@analitikisler/ai-tag-debugger 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +43 -11
- package/README.tr.md +42 -10
- package/package.json +4 -1
- package/src/agent.js +6 -1
- package/src/browser.js +262 -29
- package/src/campaign.js +7 -4
- package/src/checks.js +91 -20
- package/src/cli.js +2 -1
- package/src/cmp.js +166 -0
- package/src/consent.js +200 -26
- package/src/containers.js +163 -0
- package/src/help.js +1 -0
- package/src/hit-params.js +70 -0
- package/src/html/consent.js +34 -9
- package/src/html/journeys.js +32 -70
- package/src/html/scope.js +15 -1
- package/src/html/summary.js +1 -1
- package/src/i18n.js +230 -4
- package/src/mcp.js +341 -22
- package/src/metrics.js +13 -7
- package/src/parsers.js +62 -8
- package/src/report.js +26 -3
- package/src/verify.js +2 -0
package/src/browser.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
2
3
|
import { createRequire } from "node:module";
|
|
3
4
|
import { readFile, rm, writeFile } from "node:fs/promises";
|
|
4
5
|
import path from "node:path";
|
|
@@ -7,6 +8,9 @@ import { parseTrackingRequest, isCollectionEndpoint, providerOf, vendorOf, resol
|
|
|
7
8
|
import { redactBody, redactParams, redactPush, redactUrl } from "./redact.js";
|
|
8
9
|
import { hostScope, isPrivateAddress, navigationError, privateHostChecker } from "./guard.js";
|
|
9
10
|
import { startEgressProxy } from "./egress.js";
|
|
11
|
+
import { campaignParamsIn } from "./campaign.js";
|
|
12
|
+
import { detectCmpInPage } from "./cmp.js";
|
|
13
|
+
import { callersOf, containerOf } from "./containers.js";
|
|
10
14
|
|
|
11
15
|
// Consent banners of the common consent platforms (CMPs), by their container. A cover that matches
|
|
12
16
|
// one of these always blocks the journey, and a reject click is checked against them afterwards.
|
|
@@ -26,10 +30,12 @@ export function installBrowser() {
|
|
|
26
30
|
export const CMP_BANNER_SELECTORS = `#onetrust-banner-sdk, #onetrust-consent-sdk, #CybotCookiebotDialog, #usercentrics-root, #didomi-host, #cookiescript_injected, .cky-consent-container, .cc-window, #cmpbox, .qc-cmp2-container, \
|
|
27
31
|
#truste-consent-track, #consent_blackbar, .osano-cm-window, #iubenda-cs-banner, .iubenda-cs-container, #cmplz-cookiebanner-container, .cmplz-cookiebanner, #cookiefirst-root, .cookiefirst-root, #axeptio_overlay, \
|
|
28
32
|
#sp_message_container, [id^="sp_message_container_"], #cookie-law-info-bar, #BorlabsCookieBox, .klaro, #termly-code-snippet-support, #cookie-information-template-wrapper, #coiOverlay, \
|
|
29
|
-
#cmpwrapper, #cmpbox2, .fc-consent-root, #ppms_cm_popup_overlay, [aria-label*="cookie" i][role=dialog]
|
|
33
|
+
#cmpwrapper, #cmpbox2, .fc-consent-root, #ppms_cm_popup_overlay, [aria-label*="cookie" i][role=dialog], \
|
|
34
|
+
[data-name^="Banner"][efl-consent-not-saved]`;
|
|
35
|
+
// Efilli draws its banner inside the open shadow root of <efilli-layout-dynamic>, so the checks below also look inside open shadow roots.
|
|
30
36
|
|
|
31
37
|
// Caps on what one audit records, so a looping page can't grow memory without limit.
|
|
32
|
-
const LOG_LIMITS = { hits: 5000, dataLayerLog: 5000, network: 10000 };
|
|
38
|
+
const LOG_LIMITS = { hits: 5000, dataLayerLog: 5000, network: 10000, containerLoads: 500 };
|
|
33
39
|
|
|
34
40
|
// Secrets the browser process has no use for. It gets the rest of the environment
|
|
35
41
|
// (PATH, HOME, DISPLAY...) so Chromium still starts normally.
|
|
@@ -55,8 +61,57 @@ const hostOf = (url) => {
|
|
|
55
61
|
}
|
|
56
62
|
};
|
|
57
63
|
|
|
64
|
+
// Runs inside INIT_SCRIPT: records each Google container script (gtm.js, gtag/js) the page adds, through
|
|
65
|
+
// window.__atdLoad, with how it got there. One set by a script carries that moment's call stack, whose script
|
|
66
|
+
// addresses tell who asked for it (see src/containers.js); one the HTML parser inserted never passes through
|
|
67
|
+
// the setters, so a MutationObserver reports it. A container reading another dataLayer gets it watched.
|
|
68
|
+
const CONTAINER_HOOK = `
|
|
69
|
+
const docUrl = location.href;
|
|
70
|
+
const doc = Math.random().toString(36).slice(2, 10);
|
|
71
|
+
const isContainer = (src) => /\\/(gtm\\.js|gtag\\/js)\\?/.test(String(src)) && /[?&]id=/.test(String(src));
|
|
72
|
+
const seen = new WeakSet();
|
|
73
|
+
const report = (el, src, how) => {
|
|
74
|
+
try {
|
|
75
|
+
const abs = new URL(String(src), document.baseURI).href;
|
|
76
|
+
if (!isContainer(abs) || !window.__atdLoad) return;
|
|
77
|
+
const layer = new URL(abs).searchParams.get("l") || "dataLayer";
|
|
78
|
+
watch(layer);
|
|
79
|
+
let frames = [];
|
|
80
|
+
if (how === "script") {
|
|
81
|
+
const limit = Error.stackTraceLimit;
|
|
82
|
+
Error.stackTraceLimit = 50;
|
|
83
|
+
const stack = String(new Error().stack || "");
|
|
84
|
+
Error.stackTraceLimit = limit;
|
|
85
|
+
// The first two frames are report() and the setter, both from this script; injected inline code
|
|
86
|
+
// (a Custom HTML tag) has no address and shows as <anonymous>.
|
|
87
|
+
frames = stack.split("\\n").filter((line) => /^\\s+at /.test(line)).slice(2)
|
|
88
|
+
.map((line) => (line.match(/(https?:\\/\\/[^\\s()]+?):\\d+:\\d+/) || [])[1] || (/<anonymous>/.test(line) ? "inline" : null))
|
|
89
|
+
.filter(Boolean);
|
|
90
|
+
}
|
|
91
|
+
window.__atdLoad({ src: abs, how, frames, documentUrl: docUrl, page: location.href, doc });
|
|
92
|
+
} catch {}
|
|
93
|
+
};
|
|
94
|
+
try {
|
|
95
|
+
const desc = Object.getOwnPropertyDescriptor(HTMLScriptElement.prototype, "src");
|
|
96
|
+
Object.defineProperty(HTMLScriptElement.prototype, "src", {
|
|
97
|
+
...desc,
|
|
98
|
+
set(v) { if (!seen.has(this)) { seen.add(this); report(this, v, "script"); } return desc.set.call(this, v); },
|
|
99
|
+
});
|
|
100
|
+
const setAttribute = Element.prototype.setAttribute;
|
|
101
|
+
Element.prototype.setAttribute = function (name, v) {
|
|
102
|
+
if (this instanceof HTMLScriptElement && String(name).toLowerCase() === "src" && !seen.has(this)) { seen.add(this); report(this, v, "script"); }
|
|
103
|
+
return setAttribute.call(this, name, v);
|
|
104
|
+
};
|
|
105
|
+
new MutationObserver((list) => {
|
|
106
|
+
for (const m of list) for (const n of m.addedNodes) {
|
|
107
|
+
if (n instanceof HTMLScriptElement && n.src && !seen.has(n)) { seen.add(n); report(n, n.src, "parser"); }
|
|
108
|
+
}
|
|
109
|
+
}).observe(document, { childList: true, subtree: true });
|
|
110
|
+
} catch {}
|
|
111
|
+
`;
|
|
112
|
+
|
|
58
113
|
const INIT_SCRIPT = `(() => {
|
|
59
|
-
const record = (entry) => {
|
|
114
|
+
const record = (entry, layer = "dataLayer") => {
|
|
60
115
|
let value = entry;
|
|
61
116
|
try {
|
|
62
117
|
// gtag() pushes an arguments object; store it as a plain array.
|
|
@@ -90,22 +145,32 @@ const INIT_SCRIPT = `(() => {
|
|
|
90
145
|
};
|
|
91
146
|
value = JSON.parse(JSON.stringify(copy(value, [])));
|
|
92
147
|
} catch { value = String(entry); }
|
|
93
|
-
if (window.__atdRecord) window.__atdRecord({ url: location.href, value });
|
|
148
|
+
if (window.__atdRecord) window.__atdRecord({ url: location.href, value, ...(layer === "dataLayer" ? {} : { layer }) });
|
|
94
149
|
};
|
|
95
|
-
const wrap = (dl) => {
|
|
96
|
-
if (!dl || dl.__atdWrapped) return dl;
|
|
97
|
-
dl.forEach(record);
|
|
150
|
+
const wrap = (dl, layer) => {
|
|
151
|
+
if (!dl || dl.__atdWrapped || typeof dl.push !== "function") return dl;
|
|
152
|
+
dl.forEach((e) => record(e, layer));
|
|
98
153
|
const push = dl.push.bind(dl);
|
|
99
|
-
dl.push = (...items) => { items.forEach(record); return push(...items); };
|
|
154
|
+
dl.push = (...items) => { items.forEach((e) => record(e, layer)); return push(...items); };
|
|
100
155
|
dl.__atdWrapped = true;
|
|
101
156
|
return dl;
|
|
102
157
|
};
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
158
|
+
const watched = new Set();
|
|
159
|
+
// A container can read a dataLayer of another name (gtm.js?l=myDataLayer); it is watched once one is seen.
|
|
160
|
+
const watch = (layer) => {
|
|
161
|
+
if (watched.has(layer) || !/^[A-Za-z_$][\\w$]*$/.test(layer)) return;
|
|
162
|
+
watched.add(layer);
|
|
163
|
+
try {
|
|
164
|
+
let current = layer === "dataLayer" ? wrap(window[layer] || [], layer) : wrap(window[layer], layer);
|
|
165
|
+
Object.defineProperty(window, layer, {
|
|
166
|
+
configurable: true,
|
|
167
|
+
get: () => current,
|
|
168
|
+
set: (v) => { current = wrap(v, layer); },
|
|
169
|
+
});
|
|
170
|
+
} catch { /* a property the page made unconfigurable */ }
|
|
171
|
+
};
|
|
172
|
+
watch("dataLayer");
|
|
173
|
+
${CONTAINER_HOOK}
|
|
109
174
|
})();`;
|
|
110
175
|
|
|
111
176
|
// Labels the interactive elements of one frame with numeric refs, starting at
|
|
@@ -113,7 +178,7 @@ const INIT_SCRIPT = `(() => {
|
|
|
113
178
|
// and form fields, it finds custom controls that sites build from divs and spans
|
|
114
179
|
// (role attributes, tabindex, or a pointer cursor). Elements on screen come first,
|
|
115
180
|
// so a long menu or footer can't push the button the agent needs out of the list.
|
|
116
|
-
function snapshotInPage({ first, limit, textLimit, frameBox }) {
|
|
181
|
+
function snapshotInPage({ first, limit, textLimit, frameBox, offscreenLimit = null }) {
|
|
117
182
|
const visible = (el) => {
|
|
118
183
|
const r = el.getBoundingClientRect();
|
|
119
184
|
if (r.width <= 0 || r.height <= 0) return false;
|
|
@@ -187,8 +252,11 @@ function snapshotInPage({ first, limit, textLimit, frameBox }) {
|
|
|
187
252
|
.map((el, order) => ({ el, order, tier: tierOf(el) }))
|
|
188
253
|
.sort((a, b) => a.tier - b.tier || a.order - b.order);
|
|
189
254
|
const firstCovered = ranked.findIndex((x) => x.tier === 2);
|
|
255
|
+
let offscreen = 0;
|
|
190
256
|
const items = ranked
|
|
191
257
|
.filter((item, i) => item.tier < 2 || i - firstCovered < 30) // only a few covered or clipped ones
|
|
258
|
+
// A compact view keeps a few elements above or below the screen; scrolling brings the rest on screen.
|
|
259
|
+
.filter((item) => item.tier !== 1 || offscreenLimit === null || offscreen++ < offscreenLimit)
|
|
192
260
|
.slice(0, limit)
|
|
193
261
|
.map((item) => ({ ...item, onScreen: item.tier === 0 }));
|
|
194
262
|
const lines = [];
|
|
@@ -377,6 +445,72 @@ export function isTransientNavigationError(err, failedLoad) {
|
|
|
377
445
|
return code === "timeout" || TRANSIENT_NET_ERRORS.test(code);
|
|
378
446
|
}
|
|
379
447
|
|
|
448
|
+
/**
|
|
449
|
+
* What a step's analytics and ad cookies say, without keeping their values: the GA4 client id from _ga
|
|
450
|
+
* (the cid GA4 hits carry anyway), whether _gcl_aw and _fbc hold the click ids the landing page was opened
|
|
451
|
+
* with ("landing") or other ones ("other"), and whether _gcl_au and _fbp exist. Null when none of them do.
|
|
452
|
+
*/
|
|
453
|
+
export function identityOf(cookies, landingUrl) {
|
|
454
|
+
const value = (name) => cookies.find((c) => c.name === name)?.value ?? "";
|
|
455
|
+
const sent = landingUrl ? campaignParamsIn(landingUrl) : {};
|
|
456
|
+
const out = {};
|
|
457
|
+
const ga = /^GA\d\.\d+\.(\d+\.\d+)$/.exec(value("_ga"));
|
|
458
|
+
if (ga) out.ga_client_id = ga[1];
|
|
459
|
+
// _gcl_aw is GCL.<time>.<gclid>; _fbc is fb.<n>.<time>.<fbclid>.
|
|
460
|
+
const aw = /^GCL\.\d+\.(.+)$/.exec(value("_gcl_aw"));
|
|
461
|
+
if (aw) out.gcl_aw = aw[1] === sent.gclid ? "landing" : "other";
|
|
462
|
+
const fbc = /^fb\.\d+\.\d+\.(.+)$/.exec(value("_fbc"));
|
|
463
|
+
if (fbc) out.fbc = fbc[1] === sent.fbclid ? "landing" : "other";
|
|
464
|
+
if (value("_gcl_au")) out.gcl_au = true;
|
|
465
|
+
if (value("_fbp")) out.fbp = true;
|
|
466
|
+
return Object.keys(out).length ? out : null;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Runs in the page: the Google tag's own consent state (google_tag_data.ics), or null before a Google tag loaded.
|
|
471
|
+
* Per consent type the default and update it holds, and whether it applied a default and an update at all, whether
|
|
472
|
+
* the default came after a tag had already read consent (wasSetLate), and whether wait_for_update ran out. It is how
|
|
473
|
+
* the tag sees consent, wherever it was set: gtag() in the page, or a consent platform's GTM template, which never
|
|
474
|
+
* shows in the dataLayer. Not documented by Google; read defensively.
|
|
475
|
+
*/
|
|
476
|
+
export function googleConsentStateInPage() {
|
|
477
|
+
const ics = window.google_tag_data?.ics;
|
|
478
|
+
if (!ics || typeof ics !== "object") return null;
|
|
479
|
+
const word = (v) => (v === true ? "granted" : v === false ? "denied" : undefined);
|
|
480
|
+
const entries = {};
|
|
481
|
+
for (const [type, e] of Object.entries(ics.entries ?? {})) {
|
|
482
|
+
if (!e || typeof e !== "object") continue;
|
|
483
|
+
const d = word(e.default);
|
|
484
|
+
const u = word(e.update);
|
|
485
|
+
entries[type] = { ...(d ? { default: d } : {}), ...(u ? { update: u } : {}) };
|
|
486
|
+
}
|
|
487
|
+
return { entries, usedDefault: Boolean(ics.usedDefault), usedUpdate: Boolean(ics.usedUpdate), wasSetLate: Boolean(ics.wasSetLate), waitPeriodTimedOut: Boolean(ics.waitPeriodTimedOut) };
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* Runs in the page: whether it is something other than the site, so what a step captured there says nothing about
|
|
492
|
+
* the site's tracking. A bot challenge in its place (Cloudflare, DataDome, HUMAN, Akamai), or a page that crashed in
|
|
493
|
+
* the browser (the error screen a Next.js app shows when its scripts fail). Null for a page that looks like itself.
|
|
494
|
+
* @returns {null | { kind: "challenge"|"crash", vendor?: string }}
|
|
495
|
+
*/
|
|
496
|
+
export function pageProblemInPage() {
|
|
497
|
+
const title = (document.title || "").trim();
|
|
498
|
+
const text = (document.body?.innerText || "").replace(/\s+/g, " ").trim().slice(0, 3000);
|
|
499
|
+
const has = (selector) => {
|
|
500
|
+
try {
|
|
501
|
+
return Boolean(document.querySelector(selector));
|
|
502
|
+
} catch {
|
|
503
|
+
return false;
|
|
504
|
+
}
|
|
505
|
+
};
|
|
506
|
+
if (/^just a moment\.\.\.$|^attention required! \| cloudflare$/i.test(title) || has("#challenge-form, #challenge-running, #cf-challenge-running")) return { kind: "challenge", vendor: "Cloudflare" };
|
|
507
|
+
if (has('iframe[src*="captcha-delivery.com"]')) return { kind: "challenge", vendor: "DataDome" };
|
|
508
|
+
if (has("#px-captcha")) return { kind: "challenge", vendor: "HUMAN" };
|
|
509
|
+
if (/^access denied$/i.test(title) && /reference\s*#\s*[\d.a-f]+/i.test(text)) return { kind: "challenge", vendor: "Akamai" };
|
|
510
|
+
if (/^application error: a client-side exception has occurred/i.test(text)) return { kind: "crash", vendor: "Next.js" };
|
|
511
|
+
return null;
|
|
512
|
+
}
|
|
513
|
+
|
|
380
514
|
export class AuditSession {
|
|
381
515
|
/**
|
|
382
516
|
* @param {{ viewport: string, site?: string, allowedHosts?: string[], providers?: string[], keepBodies?: boolean, blockPrivate?: boolean | ((ip: string) => boolean), blockHits?: boolean, headless?: boolean }} opts
|
|
@@ -406,8 +540,13 @@ export class AuditSession {
|
|
|
406
540
|
this.consentStep = null;
|
|
407
541
|
this.rejectStep = null;
|
|
408
542
|
this.cookieLog = []; // each cookie name and domain with the step it was first seen at (never values)
|
|
543
|
+
this.cookieChanges = []; // { step, name, domain, change: added|changed|removed }, never values
|
|
544
|
+
this.cookieHashes = new Map(); // domain|name -> a hash of the value at the last step, kept in memory only
|
|
545
|
+
this.cmp = null; // the consent platform and its setup, from detectCmpInPage, with the step it was first seen at
|
|
546
|
+
this.refusedScripts = new Map(); // step -> how many of the page's scripts were refused for coming back as HTML
|
|
409
547
|
this.errors = [];
|
|
410
548
|
this.dataLayerLog = [];
|
|
549
|
+
this.containerLoads = []; // each Google container script the page added, with how (see src/containers.js)
|
|
411
550
|
this.network = [];
|
|
412
551
|
this.providers = resolveProviders(this.opts.providers);
|
|
413
552
|
this.landing = null; // the first page: { step, requested, chain, final }
|
|
@@ -452,9 +591,14 @@ export class AuditSession {
|
|
|
452
591
|
this.lastTrackingAt = Date.now();
|
|
453
592
|
this.record("dataLayerLog", { ...this.redactEntry(entry), step: this.currentStep, t: Date.now() });
|
|
454
593
|
});
|
|
594
|
+
await this.context.exposeBinding("__atdLoad", (_source, entry) => this.recordLoad(entry));
|
|
455
595
|
await this.context.addInitScript(INIT_SCRIPT);
|
|
456
596
|
this.page = await this.context.newPage();
|
|
457
597
|
this.page.on("pageerror", (err) => this.errors.push({ step: this.currentStep, message: err.message }));
|
|
598
|
+
// A script that comes back as an HTML page (a bot check, or files a new deployment no longer has) is refused.
|
|
599
|
+
this.page.on("console", (msg) => {
|
|
600
|
+
if (msg.type() === "error" && /^Refused to execute script from .*MIME type \('text\/html'\)/.test(msg.text())) this.refusedScripts.set(this.currentStep, (this.refusedScripts.get(this.currentStep) ?? 0) + 1);
|
|
601
|
+
});
|
|
458
602
|
this.page.on("framenavigated", (frame) => {
|
|
459
603
|
if (frame !== this.page.mainFrame()) return;
|
|
460
604
|
if (frame.url().startsWith("chrome-error:")) this.errorPageShown = true;
|
|
@@ -521,7 +665,14 @@ export class AuditSession {
|
|
|
521
665
|
});
|
|
522
666
|
this.context.on("response", (response) => {
|
|
523
667
|
const call = calls.get(response.request());
|
|
524
|
-
if (call)
|
|
668
|
+
if (!call) return;
|
|
669
|
+
call.status = response.status();
|
|
670
|
+
// A consent platform's script or setup file date tells which build of its setup the site serves. Setup files
|
|
671
|
+
// (OneTrust's, Cookiebot's) are fetched by script rather than loaded as one.
|
|
672
|
+
if (call.provider === "consent" && ["script", "fetch", "xhr"].includes(call.type)) {
|
|
673
|
+
const lastModified = response.headers()["last-modified"];
|
|
674
|
+
if (lastModified) call.lastModified = lastModified;
|
|
675
|
+
}
|
|
525
676
|
});
|
|
526
677
|
this.context.on("requestfailed", (request) => {
|
|
527
678
|
const call = calls.get(request);
|
|
@@ -667,6 +818,25 @@ export class AuditSession {
|
|
|
667
818
|
return { postData: this.opts.keepBodies ? body : redactBody(body).text };
|
|
668
819
|
}
|
|
669
820
|
|
|
821
|
+
/**
|
|
822
|
+
* A container script the page added: its ID, dataLayer and host, whether the HTML parser inserted it or a
|
|
823
|
+
* script did and, for a script, the callers on the stack. Only addresses without their query are kept, so
|
|
824
|
+
* the environment token (gtm_auth) and user data in a page's query string never reach the capture.
|
|
825
|
+
*/
|
|
826
|
+
recordLoad(entry) {
|
|
827
|
+
const container = containerOf(entry?.src);
|
|
828
|
+
if (!container) return;
|
|
829
|
+
this.record("containerLoads", {
|
|
830
|
+
...container,
|
|
831
|
+
how: entry.how === "parser" ? "parser" : "script",
|
|
832
|
+
...(entry.how === "parser" ? {} : { callers: callersOf(entry.frames, entry.documentUrl) }),
|
|
833
|
+
page: pagePath(entry.page),
|
|
834
|
+
doc: String(entry.doc ?? "").slice(0, 16),
|
|
835
|
+
step: this.currentStep,
|
|
836
|
+
t: Date.now(),
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
|
|
670
840
|
/** Adds an entry to one of the logs, up to its limit. Returns false once the limit is reached. */
|
|
671
841
|
record(log, entry) {
|
|
672
842
|
if (this[log].length >= LOG_LIMITS[log]) {
|
|
@@ -783,20 +953,21 @@ export class AuditSession {
|
|
|
783
953
|
}
|
|
784
954
|
}
|
|
785
955
|
|
|
786
|
-
|
|
956
|
+
/** @param {{ offscreenLimit?: number|null }} [opts] offscreenLimit: how many elements above or below the screen to list; null for all. */
|
|
957
|
+
async snapshot({ offscreenLimit = null } = {}) {
|
|
787
958
|
const blocked = await this.enforce();
|
|
788
959
|
if (blocked && this.page.url() === "about:blank") return { url: "about:blank", title: "", elements: "", text: `(${blocked})` };
|
|
789
960
|
try {
|
|
790
961
|
// The main page first, then iframes (booking and payment widgets often live in one).
|
|
791
962
|
const frames = await this.visibleFrames();
|
|
792
|
-
const main = await this.page.evaluate(snapshotInPage, { first: 1, limit: frames.length ? MAX_ELEMENTS - 60 : MAX_ELEMENTS, textLimit: 3000 });
|
|
963
|
+
const main = await this.page.evaluate(snapshotInPage, { first: 1, limit: frames.length ? MAX_ELEMENTS - 60 : MAX_ELEMENTS, textLimit: 3000, offscreenLimit });
|
|
793
964
|
this.refFrames = new Map();
|
|
794
965
|
const lines = [...main.lines];
|
|
795
966
|
let next = main.next;
|
|
796
967
|
let more = main.more;
|
|
797
968
|
for (const { frame, frameBox } of frames) {
|
|
798
969
|
if (next > MAX_ELEMENTS) break;
|
|
799
|
-
const sub = await frame.evaluate(snapshotInPage, { first: next, limit: MAX_ELEMENTS - next + 1, textLimit: 0, frameBox }).catch(() => null);
|
|
970
|
+
const sub = await frame.evaluate(snapshotInPage, { first: next, limit: MAX_ELEMENTS - next + 1, textLimit: 0, frameBox, offscreenLimit }).catch(() => null);
|
|
800
971
|
if (!sub?.lines.length) continue;
|
|
801
972
|
lines.push(`(inside an embedded frame from ${new URL(sub.url).host})`, ...sub.lines);
|
|
802
973
|
for (let r = next; r < sub.next; r++) this.refFrames.set(r, frame);
|
|
@@ -940,7 +1111,7 @@ export class AuditSession {
|
|
|
940
1111
|
*/
|
|
941
1112
|
async act(a) {
|
|
942
1113
|
const error = await this.perform(a);
|
|
943
|
-
await this.
|
|
1114
|
+
await this.observe();
|
|
944
1115
|
// A scripted replay of this step may fail the same way, which must not end the journey there.
|
|
945
1116
|
if (error) this.steps[this.currentStep].error = error;
|
|
946
1117
|
this.steps[this.currentStep].pagePath = pagePath(this.page.url());
|
|
@@ -948,6 +1119,28 @@ export class AuditSession {
|
|
|
948
1119
|
return error;
|
|
949
1120
|
}
|
|
950
1121
|
|
|
1122
|
+
/**
|
|
1123
|
+
* What the step just taken left behind, beyond its requests: the cookies it changed, the consent platform the
|
|
1124
|
+
* first time one is seen, and the Google tag's consent state when it changed. act() calls it after every action;
|
|
1125
|
+
* the consent scenarios call it after the steps they take themselves.
|
|
1126
|
+
*/
|
|
1127
|
+
async observe() {
|
|
1128
|
+
await this.trackCookies();
|
|
1129
|
+
if (!this.cmp) {
|
|
1130
|
+
const cmp = await this.evaluateWithin(detectCmpInPage);
|
|
1131
|
+
if (cmp) this.cmp = { ...cmp, step: this.currentStep };
|
|
1132
|
+
}
|
|
1133
|
+
const problem = await this.evaluateWithin(pageProblemInPage);
|
|
1134
|
+
if (problem) {
|
|
1135
|
+
const refused = this.refusedScripts.get(this.currentStep) ?? 0;
|
|
1136
|
+
this.steps[this.currentStep].pageProblem = { ...problem, ...(refused ? { refusedScripts: refused } : {}) };
|
|
1137
|
+
}
|
|
1138
|
+
// Kept on the step only when it changed, so the steps that hold one are where the Google tag's state moved.
|
|
1139
|
+
const tagConsent = await this.evaluateWithin(googleConsentStateInPage);
|
|
1140
|
+
const before = this.steps.slice(0, this.currentStep).map((st) => st.tagConsent).filter(Boolean).at(-1);
|
|
1141
|
+
if (tagConsent && JSON.stringify(tagConsent) !== JSON.stringify(before)) this.steps[this.currentStep].tagConsent = tagConsent;
|
|
1142
|
+
}
|
|
1143
|
+
|
|
951
1144
|
async perform(a) {
|
|
952
1145
|
const describe = a.reason || `${a.action} ${a.selector ?? (a.ref != null ? `#${a.ref}` : a.text != null ? `"${a.text}"` : a.url ?? "")}`.trim();
|
|
953
1146
|
this.beginStep(describe, { action: a.action, acceptsConsent: !!a.accepts_consent, rejectsConsent: !!a.rejects_consent });
|
|
@@ -1021,10 +1214,14 @@ export class AuditSession {
|
|
|
1021
1214
|
if (a.rejects_consent && this.rejectStep === null) {
|
|
1022
1215
|
this.rejectStep = this.currentStep;
|
|
1023
1216
|
// A rejected banner should close; one that is still showing means the button did nothing.
|
|
1024
|
-
step.bannerOpenAfter = await this.page.evaluate((sel) =>
|
|
1025
|
-
const
|
|
1026
|
-
|
|
1027
|
-
|
|
1217
|
+
step.bannerOpenAfter = await this.page.evaluate((sel) => {
|
|
1218
|
+
const roots = [document];
|
|
1219
|
+
for (let i = 0; i < roots.length; i++) for (const n of roots[i].querySelectorAll("*")) if (n.shadowRoot) roots.push(n.shadowRoot);
|
|
1220
|
+
return roots.flatMap((r) => [...r.querySelectorAll(sel)]).some((n) => {
|
|
1221
|
+
const r = n.getBoundingClientRect();
|
|
1222
|
+
return r.width > 0 && r.height > 0 && getComputedStyle(n).visibility !== "hidden";
|
|
1223
|
+
});
|
|
1224
|
+
}, CMP_BANNER_SELECTORS).catch(() => false);
|
|
1028
1225
|
}
|
|
1029
1226
|
return null;
|
|
1030
1227
|
} catch (err) {
|
|
@@ -1032,20 +1229,56 @@ export class AuditSession {
|
|
|
1032
1229
|
}
|
|
1033
1230
|
}
|
|
1034
1231
|
|
|
1035
|
-
/**
|
|
1232
|
+
/**
|
|
1233
|
+
* Notes the cookies the step just taken added, changed or removed, and what the analytics and ad cookies
|
|
1234
|
+
* say (identityOf). Values never leave this function: a short hash of each, held until the next step,
|
|
1235
|
+
* is enough to tell that one changed.
|
|
1236
|
+
*/
|
|
1036
1237
|
async trackCookies() {
|
|
1238
|
+
const all = await this.rawCookies();
|
|
1037
1239
|
const seen = new Set(this.cookieLog.map((c) => `${c.domain}|${c.name}`));
|
|
1038
|
-
|
|
1039
|
-
|
|
1240
|
+
const now = new Map();
|
|
1241
|
+
for (const c of all) {
|
|
1242
|
+
const k = `${c.domain}|${c.name}`;
|
|
1243
|
+
const hash = createHash("sha256").update(c.value).digest("base64").slice(0, 16);
|
|
1244
|
+
now.set(k, hash);
|
|
1245
|
+
if (!seen.has(k) && this.cookieLog.length < 500) this.cookieLog.push({ name: c.name, domain: c.domain, step: this.currentStep });
|
|
1246
|
+
const before = this.cookieHashes.get(k);
|
|
1247
|
+
const change = before === undefined ? "added" : before !== hash ? "changed" : null;
|
|
1248
|
+
if (change && this.cookieChanges.length < 2000) this.cookieChanges.push({ step: this.currentStep, name: c.name, domain: c.domain, change });
|
|
1040
1249
|
}
|
|
1250
|
+
for (const k of this.cookieHashes.keys()) {
|
|
1251
|
+
if (now.has(k) || this.cookieChanges.length >= 2000) continue;
|
|
1252
|
+
const [domain, ...name] = k.split("|");
|
|
1253
|
+
this.cookieChanges.push({ step: this.currentStep, name: name.join("|"), domain, change: "removed" });
|
|
1254
|
+
}
|
|
1255
|
+
this.cookieHashes = now;
|
|
1256
|
+
const identity = identityOf(all, this.landing?.requested);
|
|
1257
|
+
if (identity) this.steps[this.currentStep].identity = identity;
|
|
1041
1258
|
}
|
|
1042
1259
|
|
|
1043
|
-
|
|
1260
|
+
/** page.evaluate(fn), or null when it fails or the page does not answer within ms: a page stuck in a script never answers, and the next page read reports that. */
|
|
1261
|
+
async evaluateWithin(fn, ms = 2000) {
|
|
1262
|
+
let timer;
|
|
1263
|
+
const result = await Promise.race([
|
|
1264
|
+
this.page.evaluate(fn).catch(() => null),
|
|
1265
|
+
new Promise((resolve) => (timer = setTimeout(() => resolve(null), ms))),
|
|
1266
|
+
]);
|
|
1267
|
+
clearTimeout(timer);
|
|
1268
|
+
return result;
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
/** The browser's cookies with their values, for trackCookies only; [] when the browser takes too long. */
|
|
1272
|
+
async rawCookies() {
|
|
1044
1273
|
let timer;
|
|
1045
1274
|
const tooLong = new Promise((resolve) => (timer = setTimeout(() => resolve([]), 10_000)));
|
|
1046
1275
|
const cookies = await Promise.race([this.context.cookies().catch(() => []), tooLong]);
|
|
1047
1276
|
clearTimeout(timer);
|
|
1048
|
-
return cookies
|
|
1277
|
+
return cookies;
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
async cookies() {
|
|
1281
|
+
return (await this.rawCookies()).map((c) => ({ name: c.name, domain: c.domain }));
|
|
1049
1282
|
}
|
|
1050
1283
|
|
|
1051
1284
|
async close() {
|
package/src/campaign.js
CHANGED
|
@@ -19,10 +19,13 @@ export function campaignParamsIn(url) {
|
|
|
19
19
|
}
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
/**
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
/**
|
|
23
|
+
* The address with test campaign parameters added, unless it already carries campaign parameters.
|
|
24
|
+
* Given parameters of the caller's own (a real campaign's UTM values, a gclid), those are set instead, always.
|
|
25
|
+
*/
|
|
26
|
+
export function withCampaignParams(url, own = null) {
|
|
27
|
+
if (!own && Object.keys(campaignParamsIn(url)).length) return url;
|
|
25
28
|
const u = new URL(url);
|
|
26
|
-
for (const [k, v] of Object.entries(TEST_PARAMS)) u.searchParams.set(k, v);
|
|
29
|
+
for (const [k, v] of Object.entries(own ?? TEST_PARAMS)) u.searchParams.set(k, v);
|
|
27
30
|
return u.href;
|
|
28
31
|
}
|