@acepaste/ingress 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/acquire.d.ts +19 -1
- package/dist/src/acquire.js +130 -5
- package/dist/src/cli.js +13 -1
- package/dist/src/engine.d.ts +27 -4
- package/dist/src/engine.js +162 -59
- package/dist/src/index.d.ts +5 -3
- package/dist/src/index.js +3 -2
- package/dist/src/meaning.js +8 -7
- package/dist/src/summary.d.ts +40 -0
- package/dist/src/summary.js +62 -0
- package/dist/src/taint.js +1 -1
- package/dist/src/types.d.ts +30 -4
- package/dist/src/types.js +10 -2
- package/package.json +1 -1
package/dist/src/acquire.d.ts
CHANGED
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
import { type Dispatcher } from "undici";
|
|
13
13
|
import { NetPolicyError, type NetPolicy } from "./netpolicy.js";
|
|
14
14
|
import type { AcquireResult, IngressConfig } from "./types.js";
|
|
15
|
+
/** the gate's own name, used when agentUserAgents is empty */
|
|
16
|
+
export declare const HONEST_USER_AGENT = "AcePasteIngress/0.8 (+https://acepaste.xyz/ingress)";
|
|
15
17
|
export declare const MAX_REDIRECTS = 5;
|
|
16
18
|
export declare class AcquireError extends Error {
|
|
17
19
|
readonly code: "unsupported_scheme" | "too_large" | "timeout" | "network" | "binary" | "too_many_redirects" | "policy";
|
|
@@ -21,4 +23,20 @@ export declare class AcquireError extends Error {
|
|
|
21
23
|
export type FetchLike = (url: string, init: RequestInit & {
|
|
22
24
|
dispatcher?: Dispatcher;
|
|
23
25
|
}) => Promise<Response>;
|
|
24
|
-
export
|
|
26
|
+
export interface AcquireOptions {
|
|
27
|
+
/** user-agent to send (agent fingerprint); defaults to the gate's own name */
|
|
28
|
+
userAgent?: string;
|
|
29
|
+
/** HTTP(S) proxy for this fetch */
|
|
30
|
+
proxyUrl?: string;
|
|
31
|
+
}
|
|
32
|
+
export declare function acquire(url: string, fingerprint: "human" | "agent", config: IngressConfig, policy: NetPolicy, fetchImpl?: FetchLike, resolver?: (host: string) => Promise<string[]>, opts?: AcquireOptions): Promise<AcquireResult>;
|
|
33
|
+
/**
|
|
34
|
+
* Human fetch in a real headless Chromium: real browser handshake, JavaScript on,
|
|
35
|
+
* a fresh profile. Every request the page makes is checked against the network
|
|
36
|
+
* policy first, so a page cannot use the browser to reach private addresses.
|
|
37
|
+
* Returns the document after scripts have run.
|
|
38
|
+
*/
|
|
39
|
+
export declare function acquireWithBrowser(url: string, config: IngressConfig, policy: NetPolicy, resolver?: (host: string) => Promise<string[]>, opts?: {
|
|
40
|
+
proxyUrl?: string;
|
|
41
|
+
executablePath?: string;
|
|
42
|
+
}): Promise<AcquireResult>;
|
package/dist/src/acquire.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* the connection is pinned to the addresses that were checked (DNS rebinding
|
|
10
10
|
* defense). Redirects are followed manually so each hop is checked.
|
|
11
11
|
*/
|
|
12
|
-
import { Agent, fetch as undiciFetch } from "undici";
|
|
12
|
+
import { Agent, ProxyAgent, fetch as undiciFetch } from "undici";
|
|
13
13
|
import { checkTarget, NetPolicyError } from "./netpolicy.js";
|
|
14
14
|
const HUMAN_HEADERS = {
|
|
15
15
|
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
|
@@ -21,8 +21,10 @@ const HUMAN_HEADERS = {
|
|
|
21
21
|
"sec-fetch-user": "?1",
|
|
22
22
|
"upgrade-insecure-requests": "1",
|
|
23
23
|
};
|
|
24
|
+
/** the gate's own name, used when agentUserAgents is empty */
|
|
25
|
+
export const HONEST_USER_AGENT = "AcePasteIngress/0.8 (+https://acepaste.xyz/ingress)";
|
|
24
26
|
const AGENT_HEADERS = {
|
|
25
|
-
"user-agent":
|
|
27
|
+
"user-agent": HONEST_USER_AGENT,
|
|
26
28
|
accept: "text/html,application/xhtml+xml,text/markdown,text/plain;q=0.9,application/json;q=0.8,*/*;q=0.5",
|
|
27
29
|
};
|
|
28
30
|
export const MAX_REDIRECTS = 5;
|
|
@@ -44,13 +46,24 @@ function pinnedFetch(addrs) {
|
|
|
44
46
|
});
|
|
45
47
|
return (url, init) => undiciFetch(url, { ...init, dispatcher: agent });
|
|
46
48
|
}
|
|
47
|
-
|
|
49
|
+
/** undici fetch through an HTTP(S) proxy. The target was already checked against the network policy; the proxy sits outside our network, so it cannot be used to reach it. */
|
|
50
|
+
const proxyAgents = new Map();
|
|
51
|
+
function proxiedFetch(proxyUrl) {
|
|
52
|
+
let agent = proxyAgents.get(proxyUrl);
|
|
53
|
+
if (!agent) {
|
|
54
|
+
agent = new ProxyAgent(proxyUrl);
|
|
55
|
+
proxyAgents.set(proxyUrl, agent);
|
|
56
|
+
}
|
|
57
|
+
const a = agent;
|
|
58
|
+
return (url, init) => undiciFetch(url, { ...init, dispatcher: a });
|
|
59
|
+
}
|
|
60
|
+
export async function acquire(url, fingerprint, config, policy, fetchImpl, resolver, opts = {}) {
|
|
48
61
|
let current = new URL(url);
|
|
49
62
|
if (current.protocol !== "http:" && current.protocol !== "https:") {
|
|
50
63
|
throw new AcquireError(`unsupported scheme ${current.protocol}`, "unsupported_scheme");
|
|
51
64
|
}
|
|
52
65
|
const chain = [];
|
|
53
|
-
const headersIn = fingerprint === "human" ? HUMAN_HEADERS : AGENT_HEADERS;
|
|
66
|
+
const headersIn = fingerprint === "human" ? HUMAN_HEADERS : { ...AGENT_HEADERS, ...(opts.userAgent ? { "user-agent": opts.userAgent } : {}) };
|
|
54
67
|
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
55
68
|
let addrs;
|
|
56
69
|
try {
|
|
@@ -61,7 +74,7 @@ export async function acquire(url, fingerprint, config, policy, fetchImpl, resol
|
|
|
61
74
|
throw new AcquireError(e.message, "policy", e.code);
|
|
62
75
|
throw e;
|
|
63
76
|
}
|
|
64
|
-
const doFetch = fetchImpl ?? pinnedFetch(addrs);
|
|
77
|
+
const doFetch = fetchImpl ?? (opts.proxyUrl ? proxiedFetch(opts.proxyUrl) : pinnedFetch(addrs));
|
|
65
78
|
const controller = new AbortController();
|
|
66
79
|
const timer = setTimeout(() => controller.abort(), config.fetchTimeoutMs);
|
|
67
80
|
let res;
|
|
@@ -139,8 +152,120 @@ export async function acquire(url, fingerprint, config, policy, fetchImpl, resol
|
|
|
139
152
|
contentType,
|
|
140
153
|
body,
|
|
141
154
|
fingerprint,
|
|
155
|
+
userAgent: headersIn["user-agent"],
|
|
156
|
+
via: "http",
|
|
142
157
|
fetchedAt: new Date().toISOString(),
|
|
143
158
|
};
|
|
144
159
|
}
|
|
145
160
|
throw new AcquireError("unreachable", "network");
|
|
146
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* Human fetch in a real headless Chromium: real browser handshake, JavaScript on,
|
|
164
|
+
* a fresh profile. Every request the page makes is checked against the network
|
|
165
|
+
* policy first, so a page cannot use the browser to reach private addresses.
|
|
166
|
+
* Returns the document after scripts have run.
|
|
167
|
+
*/
|
|
168
|
+
export async function acquireWithBrowser(url, config, policy, resolver, opts = {}) {
|
|
169
|
+
const first = new URL(url);
|
|
170
|
+
if (first.protocol !== "http:" && first.protocol !== "https:")
|
|
171
|
+
throw new AcquireError(`unsupported scheme ${first.protocol}`, "unsupported_scheme");
|
|
172
|
+
try {
|
|
173
|
+
await checkTarget(first, policy, resolver);
|
|
174
|
+
}
|
|
175
|
+
catch (e) {
|
|
176
|
+
if (e instanceof NetPolicyError)
|
|
177
|
+
throw new AcquireError(e.message, "policy", e.code);
|
|
178
|
+
throw e;
|
|
179
|
+
}
|
|
180
|
+
let pw;
|
|
181
|
+
try {
|
|
182
|
+
pw = await import("playwright");
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
throw new AcquireError("browser fetch needs playwright installed", "network");
|
|
186
|
+
}
|
|
187
|
+
let proxy;
|
|
188
|
+
if (opts.proxyUrl) {
|
|
189
|
+
const p = new URL(opts.proxyUrl);
|
|
190
|
+
proxy = { server: `${p.protocol}//${p.host}`, username: decodeURIComponent(p.username) || undefined, password: decodeURIComponent(p.password) || undefined };
|
|
191
|
+
}
|
|
192
|
+
let browser;
|
|
193
|
+
try {
|
|
194
|
+
browser = await pw.chromium.launch({
|
|
195
|
+
headless: true, proxy,
|
|
196
|
+
executablePath: opts.executablePath ?? process.env.INGRESS_CHROMIUM ?? undefined,
|
|
197
|
+
args: ["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage", "--disable-extensions", "--no-first-run"],
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
catch (e) {
|
|
201
|
+
throw new AcquireError(`browser launch failed: ${e.message.split("\n")[0]}`, "network");
|
|
202
|
+
}
|
|
203
|
+
try {
|
|
204
|
+
// headless Chromium announces itself ("HeadlessChrome", navigator.webdriver); present as ordinary desktop Chrome
|
|
205
|
+
const ctx = await browser.newContext({ userAgent: HUMAN_HEADERS["user-agent"], viewport: { width: 1280, height: 900 }, locale: "en-US", storageState: { cookies: [], origins: [] } });
|
|
206
|
+
await ctx.addInitScript(() => {
|
|
207
|
+
Object.defineProperty(navigator, "webdriver", { get: () => false });
|
|
208
|
+
Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
|
|
209
|
+
});
|
|
210
|
+
const checked = new Map();
|
|
211
|
+
await ctx.route("**/*", async (route) => {
|
|
212
|
+
let u;
|
|
213
|
+
try {
|
|
214
|
+
u = new URL(route.request().url());
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
return route.abort("blockedbyclient");
|
|
218
|
+
}
|
|
219
|
+
if (u.protocol === "data:" || u.protocol === "blob:")
|
|
220
|
+
return route.continue();
|
|
221
|
+
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
222
|
+
return route.abort("blockedbyclient");
|
|
223
|
+
const key = `${u.protocol}//${u.host}`;
|
|
224
|
+
let ok = checked.get(key);
|
|
225
|
+
if (ok === undefined) {
|
|
226
|
+
try {
|
|
227
|
+
await checkTarget(u, policy, resolver);
|
|
228
|
+
ok = true;
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
ok = false;
|
|
232
|
+
}
|
|
233
|
+
checked.set(key, ok);
|
|
234
|
+
}
|
|
235
|
+
return ok ? route.continue() : route.abort("blockedbyclient");
|
|
236
|
+
});
|
|
237
|
+
const page = await ctx.newPage();
|
|
238
|
+
let status = 0;
|
|
239
|
+
let contentType = "";
|
|
240
|
+
try {
|
|
241
|
+
const res = await page.goto(url, { waitUntil: "domcontentloaded", timeout: config.fetchTimeoutMs });
|
|
242
|
+
status = res?.status() ?? 0;
|
|
243
|
+
contentType = res?.headers()["content-type"] ?? "";
|
|
244
|
+
await page.waitForTimeout(Math.min(1500, config.fetchTimeoutMs / 4));
|
|
245
|
+
}
|
|
246
|
+
catch (e) {
|
|
247
|
+
const msg = e.message.split("\n")[0];
|
|
248
|
+
throw new AcquireError(/timeout/i.test(msg) ? "browser fetch timed out" : `network error: ${msg}`, /timeout/i.test(msg) ? "timeout" : "network");
|
|
249
|
+
}
|
|
250
|
+
const finalUrl = page.url();
|
|
251
|
+
try {
|
|
252
|
+
await checkTarget(new URL(finalUrl), policy, resolver);
|
|
253
|
+
}
|
|
254
|
+
catch (e) {
|
|
255
|
+
if (e instanceof NetPolicyError)
|
|
256
|
+
throw new AcquireError(e.message, "policy", e.code);
|
|
257
|
+
throw e;
|
|
258
|
+
}
|
|
259
|
+
const body = await page.content();
|
|
260
|
+
if (body.length > config.maxBodyBytes)
|
|
261
|
+
throw new AcquireError(`body exceeds ${config.maxBodyBytes} bytes`, "too_large");
|
|
262
|
+
const userAgent = await page.evaluate(() => navigator.userAgent);
|
|
263
|
+
return {
|
|
264
|
+
url, finalUrl, redirectChain: finalUrl === url ? [] : [url], status, headers: { "content-type": contentType },
|
|
265
|
+
contentType: contentType || "text/html", body, fingerprint: "human", userAgent, via: "browser", fetchedAt: new Date().toISOString(),
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
finally {
|
|
269
|
+
await browser.close().catch(() => { });
|
|
270
|
+
}
|
|
271
|
+
}
|
package/dist/src/cli.js
CHANGED
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { readFileSync } from "node:fs";
|
|
13
13
|
import { IngressEngine } from "./engine.js";
|
|
14
|
+
import { HONEST_USER_AGENT } from "./acquire.js";
|
|
15
|
+
import { summarize, cloakingConfigFromEnv } from "./summary.js";
|
|
14
16
|
import { loadPolicyFile } from "./policyfile.js";
|
|
15
17
|
const args = process.argv.slice(2);
|
|
16
18
|
const flag = (n) => args.includes(`--${n}`);
|
|
@@ -32,8 +34,12 @@ async function main() {
|
|
|
32
34
|
return;
|
|
33
35
|
}
|
|
34
36
|
const cfg = opt("policy") ? loadPolicyFile(opt("policy")) : {};
|
|
37
|
+
Object.assign(cfg, cloakingConfigFromEnv(process.env, HONEST_USER_AGENT));
|
|
35
38
|
if (flag("no-cloaking"))
|
|
36
39
|
cfg.cloakingCheck = false;
|
|
40
|
+
const hf = opt("human-fetch");
|
|
41
|
+
if (hf === "browser" || hf === "http")
|
|
42
|
+
cfg.humanFetch = hf;
|
|
37
43
|
const sb = opt("sandbox");
|
|
38
44
|
if (sb === "always" || sb === "escalate" || sb === "off")
|
|
39
45
|
cfg.sandbox = sb;
|
|
@@ -50,13 +56,19 @@ async function main() {
|
|
|
50
56
|
else if (url)
|
|
51
57
|
result = await engine.fetch(url, { override });
|
|
52
58
|
else {
|
|
53
|
-
console.error("usage: ingress <url> | ingress --text < file [--json] [--no-cloaking] [--override] [--sandbox=always|escalate|off] [--policy=file.json] | ingress --verify < receipt.json");
|
|
59
|
+
console.error("usage: ingress <url> | ingress --text < file [--json | --summary] [--no-cloaking] [--human-fetch=http|browser] [--override] [--sandbox=always|escalate|off] [--policy=file.json] | ingress --verify < receipt.json");
|
|
54
60
|
process.exit(2);
|
|
55
61
|
}
|
|
56
62
|
if (flag("json")) {
|
|
57
63
|
console.log(JSON.stringify(result, null, 2));
|
|
58
64
|
return;
|
|
59
65
|
}
|
|
66
|
+
if (flag("summary")) {
|
|
67
|
+
console.log(JSON.stringify(summarize(result)));
|
|
68
|
+
if (result.tier === "unreachable")
|
|
69
|
+
process.exitCode = 1;
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
60
72
|
console.log(result.envelope);
|
|
61
73
|
console.error(`\n[timings] ${Object.entries(result.timings).map(([k, v]) => `${k}=${v.toFixed(1)}ms`).join(" ")}`);
|
|
62
74
|
}
|
package/dist/src/engine.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* hostile -> refuse. Not overridable at runtime. Logged.
|
|
12
12
|
*/
|
|
13
13
|
import { type KeyObject } from "node:crypto";
|
|
14
|
-
import { type FetchLike } from "./acquire.js";
|
|
14
|
+
import { acquireWithBrowser, type FetchLike } from "./acquire.js";
|
|
15
15
|
import { runSandbox } from "./sandbox.js";
|
|
16
16
|
import { type Finding, type IngressConfig, type IngressResult, type TaintReceipt, type Tier } from "./types.js";
|
|
17
17
|
export interface IngressOptions {
|
|
@@ -27,6 +27,12 @@ export interface EngineDeps {
|
|
|
27
27
|
log?: (entry: Record<string, unknown>) => void;
|
|
28
28
|
/** sandbox runner override (tests) */
|
|
29
29
|
sandboxImpl?: typeof runSandbox;
|
|
30
|
+
/** browser fetch override for humanFetch: "browser" (tests) */
|
|
31
|
+
browserAcquireImpl?: typeof acquireWithBrowser;
|
|
32
|
+
/** randomness for user-agent choice, fetch order and jitter (tests pin it) */
|
|
33
|
+
random?: () => number;
|
|
34
|
+
/** delay between the two fetches (tests skip it) */
|
|
35
|
+
sleep?: (ms: number) => Promise<void>;
|
|
30
36
|
}
|
|
31
37
|
export declare class IngressEngine {
|
|
32
38
|
readonly config: IngressConfig;
|
|
@@ -51,6 +57,7 @@ export declare class IngressEngine {
|
|
|
51
57
|
private domainRule;
|
|
52
58
|
private refuse;
|
|
53
59
|
private refuseOnAcquire;
|
|
60
|
+
private pickAgentUserAgent;
|
|
54
61
|
private receipt;
|
|
55
62
|
private signerCache?;
|
|
56
63
|
private signer;
|
|
@@ -88,12 +95,28 @@ export declare class IngressEngine {
|
|
|
88
95
|
* fact, not twelve attacks)
|
|
89
96
|
* - noisy-OR across kinds
|
|
90
97
|
* - hidden text that carries no instruction shape is informational: it is
|
|
91
|
-
*
|
|
92
|
-
*
|
|
98
|
+
* never delivered, so visibility-only evidence is capped below the
|
|
99
|
+
* suspect line. The page passes clean, the hidden text is stripped, and
|
|
100
|
+
* the findings stay on the receipt (ratified 2026-11-03: hidden menus,
|
|
101
|
+
* collapsed sections and screen-reader labels must not refuse a page)
|
|
93
102
|
* - co-occurrence of hidden-text and meaning findings, cloaking, and
|
|
94
103
|
* webdriver-keyed divergence each add a bonus
|
|
95
104
|
*/
|
|
96
|
-
|
|
105
|
+
/** below the default suspect threshold (0.25), so hidden text alone never refuses */
|
|
106
|
+
export declare const VISIBILITY_ONLY_CAP = 0.2;
|
|
107
|
+
export declare function aggregate(findings: Finding[], ctx?: {
|
|
108
|
+
cloakingCompared?: boolean;
|
|
109
|
+
}): number;
|
|
110
|
+
/** words of served text with markup, link targets and bare URLs removed, so Markdown and HTML of the same page compare equal */
|
|
111
|
+
export declare function servedWords(text: string): string[];
|
|
112
|
+
/** how much of what the agent received is absent from what the browser received (word occurrences, not distinct words) */
|
|
113
|
+
export declare function compareServed(agentText: string, humanText: string): {
|
|
114
|
+
agentWords: number;
|
|
115
|
+
agentOnlyWords: number;
|
|
116
|
+
agentOnlyShare: number;
|
|
117
|
+
longestAgentOnlyRun: number;
|
|
118
|
+
sample: string;
|
|
119
|
+
};
|
|
97
120
|
/**
|
|
98
121
|
* Spotlighting-style envelope. Random boundary so content cannot forge the
|
|
99
122
|
* closing marker. The model is told, in one line, how to treat the content.
|
package/dist/src/engine.js
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* hostile -> refuse. Not overridable at runtime. Logged.
|
|
12
12
|
*/
|
|
13
13
|
import { createHash, createHmac, randomBytes, createPrivateKey, createPublicKey, generateKeyPairSync, sign as cryptoSign, verify as cryptoVerify } from "node:crypto";
|
|
14
|
-
import { acquire, AcquireError } from "./acquire.js";
|
|
14
|
+
import { acquire, acquireWithBrowser, AcquireError, HONEST_USER_AGENT } from "./acquire.js";
|
|
15
15
|
import { analyzeHtml, rulesFromCss, colorRulesFromCss } from "./visibility.js";
|
|
16
16
|
import { scanCharacters } from "./character.js";
|
|
17
17
|
import { countHostileInvisibles } from "./acepaste-alphabet.js";
|
|
@@ -46,7 +46,13 @@ export class IngressEngine {
|
|
|
46
46
|
now: deps.now ?? (() => new Date()),
|
|
47
47
|
log: deps.log ?? ((e) => process.stderr.write(JSON.stringify(e) + "\n")),
|
|
48
48
|
sandboxImpl: deps.sandboxImpl ?? runSandbox,
|
|
49
|
+
browserAcquireImpl: deps.browserAcquireImpl ?? acquireWithBrowser,
|
|
50
|
+
random: deps.random ?? Math.random,
|
|
51
|
+
sleep: deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms))),
|
|
49
52
|
};
|
|
53
|
+
const [jmin, jmax] = this.config.cloakingJitterMs;
|
|
54
|
+
if (!(jmin >= 0 && jmax >= jmin))
|
|
55
|
+
throw new Error("IngressEngine: cloakingJitterMs must be [min, max] with 0 <= min <= max");
|
|
50
56
|
}
|
|
51
57
|
/** Gate a URL: policy check, acquire (twice when cloaking check is on), analyze, decide. */
|
|
52
58
|
async fetch(url, opts = {}) {
|
|
@@ -61,28 +67,50 @@ export class IngressEngine {
|
|
|
61
67
|
return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 1, severity: 1, message: `source ${hostOf(url)} is denied by policy`, action: "none" }], timings, t0, "text");
|
|
62
68
|
}
|
|
63
69
|
const cloaking = rule?.cloakingCheck ?? this.config.cloakingCheck;
|
|
70
|
+
// Two fetches, told apart only by what they present. The agent fetch wears the user agent of a
|
|
71
|
+
// real AI assistant's fetcher (that is what agent-targeted cloaking keys on). The human fetch looks
|
|
72
|
+
// like desktop Chrome, optionally through a different network (humanProxy) and optionally in a real
|
|
73
|
+
// browser (humanFetch: "browser"). Which goes first and the gap between them are random, so the pair
|
|
74
|
+
// does not arrive as a simultaneous, identically-timed signature.
|
|
75
|
+
const agentUA = this.pickAgentUserAgent();
|
|
76
|
+
const fetchAgent = () => acquire(url, "agent", this.config, this.config.net, this.deps.fetchImpl, this.deps.resolver, { userAgent: agentUA });
|
|
77
|
+
const fetchHuman = () => this.config.humanFetch === "browser"
|
|
78
|
+
? this.deps.browserAcquireImpl(url, this.config, this.config.net, this.deps.resolver, { proxyUrl: this.config.humanProxy })
|
|
79
|
+
: acquire(url, "human", this.config, this.config.net, this.deps.fetchImpl, this.deps.resolver, { proxyUrl: this.config.humanProxy });
|
|
80
|
+
const findings = [];
|
|
64
81
|
let agentRes, humanRes;
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
82
|
+
const ta = performance.now();
|
|
83
|
+
if (cloaking) {
|
|
84
|
+
const [jmin, jmax] = this.config.cloakingJitterMs;
|
|
85
|
+
const gap = Math.round(jmin + this.deps.random() * (jmax - jmin));
|
|
86
|
+
const humanFirst = this.deps.random() < 0.5;
|
|
87
|
+
const later = (f) => this.deps.sleep(gap).then(f);
|
|
88
|
+
const [agentS, humanS] = await Promise.allSettled(humanFirst ? [later(fetchAgent), fetchHuman()] : [fetchAgent(), later(fetchHuman)]);
|
|
89
|
+
timings.acquire = performance.now() - ta;
|
|
90
|
+
if (agentS.status === "rejected") {
|
|
91
|
+
if (agentS.reason instanceof AcquireError)
|
|
92
|
+
return this.refuseOnAcquire(url, agentS.reason, timings, t0);
|
|
93
|
+
throw agentS.reason;
|
|
72
94
|
}
|
|
95
|
+
agentRes = agentS.value;
|
|
96
|
+
if (humanS.status === "fulfilled")
|
|
97
|
+
humanRes = humanS.value;
|
|
73
98
|
else {
|
|
74
|
-
|
|
99
|
+
// the page itself was fetched; only the comparison is missing. Recorded, weightless.
|
|
100
|
+
findings.push({ pass: "acquire", kind: "acquire.cloaking_check_incomplete", confidence: 0, severity: 0, message: `cloaking check skipped: the browser-fingerprint fetch failed (${humanS.reason?.message ?? "unknown"})`, action: "none" });
|
|
75
101
|
}
|
|
76
|
-
timings.acquire = performance.now() - ta;
|
|
77
102
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
103
|
+
else {
|
|
104
|
+
try {
|
|
105
|
+
agentRes = await fetchAgent();
|
|
106
|
+
}
|
|
107
|
+
catch (e) {
|
|
108
|
+
if (e instanceof AcquireError)
|
|
109
|
+
return this.refuseOnAcquire(url, e, timings, t0);
|
|
110
|
+
throw e;
|
|
111
|
+
}
|
|
112
|
+
timings.acquire = performance.now() - ta;
|
|
82
113
|
}
|
|
83
|
-
if (!agentRes)
|
|
84
|
-
throw new Error("acquire returned nothing");
|
|
85
|
-
const findings = [];
|
|
86
114
|
if (agentRes.redirectChain.length) {
|
|
87
115
|
const hosts = new Set(agentRes.redirectChain.concat(agentRes.finalUrl).map(hostOf));
|
|
88
116
|
if (hosts.size > 2)
|
|
@@ -91,6 +119,26 @@ export class IngressEngine {
|
|
|
91
119
|
if (finalRule?.deny)
|
|
92
120
|
return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 1, severity: 1, message: `redirect target ${hostOf(agentRes.finalUrl)} is denied by policy`, action: "none" }], timings, t0, "text");
|
|
93
121
|
}
|
|
122
|
+
// Many sites refuse AI-assistant user agents outright. When the browser fetch got through, fetch
|
|
123
|
+
// once more under the gate's own name so the page can still be read and compared.
|
|
124
|
+
if (cloaking && [401, 403, 429].includes(agentRes.status) && humanRes && humanRes.status < 400 && agentUA !== HONEST_USER_AGENT) {
|
|
125
|
+
try {
|
|
126
|
+
const retry = await acquire(url, "agent", this.config, this.config.net, this.deps.fetchImpl, this.deps.resolver, { userAgent: HONEST_USER_AGENT });
|
|
127
|
+
findings.push({ pass: "acquire", kind: "acquire.ai_agent_refused", confidence: 0, severity: 0, message: `the site refused an AI-assistant user agent (HTTP ${agentRes.status}); fetched again as AcePaste Ingress (HTTP ${retry.status})`, action: "none" });
|
|
128
|
+
agentRes = retry;
|
|
129
|
+
}
|
|
130
|
+
catch { /* keep the refusal */ }
|
|
131
|
+
}
|
|
132
|
+
// An error page is not the page. If the site refused the AI fetcher (many block AI user agents or
|
|
133
|
+
// challenge non-browsers), there is nothing to deliver and nothing to compare: unreachable, not an attack.
|
|
134
|
+
if (agentRes.status >= 400) {
|
|
135
|
+
const both = humanRes ? `; the browser fetch got HTTP ${humanRes.status}` : "";
|
|
136
|
+
return this.refuse(url, [{ pass: "acquire", kind: "acquire.unreachable", confidence: 0, severity: 0, message: `the site answered the AI fetcher with HTTP ${agentRes.status}${both}`, action: "none" }], timings, t0, "text", "unreachable");
|
|
137
|
+
}
|
|
138
|
+
if (humanRes && humanRes.status >= 400) {
|
|
139
|
+
findings.push({ pass: "acquire", kind: "acquire.cloaking_check_incomplete", confidence: 0, severity: 0, message: `cloaking check skipped: the browser-fingerprint fetch got HTTP ${humanRes.status}`, action: "none" });
|
|
140
|
+
humanRes = undefined;
|
|
141
|
+
}
|
|
94
142
|
const kind = detectKind(agentRes.contentType, agentRes.body, agentRes.finalUrl);
|
|
95
143
|
// external stylesheets (same origin only, bounded)
|
|
96
144
|
let externalRules = [];
|
|
@@ -102,7 +150,7 @@ export class IngressEngine {
|
|
|
102
150
|
}
|
|
103
151
|
const result = await this.guarded(url, kind, t0, () => this.analyze({
|
|
104
152
|
url, finalUrl: agentRes.finalUrl, redirectChain: agentRes.redirectChain, raw: agentRes.body, kind,
|
|
105
|
-
humanRaw: humanRes?.body, externalRules, externalColorRules, timings, t0, override: !!opts.override, extraFindings: findings, rule,
|
|
153
|
+
humanRaw: humanRes?.body, humanContentType: humanRes?.contentType, externalRules, externalColorRules, timings, t0, override: !!opts.override, extraFindings: findings, rule,
|
|
106
154
|
}));
|
|
107
155
|
this.cacheSet(url, opts, result);
|
|
108
156
|
return result;
|
|
@@ -202,14 +250,25 @@ export class IngressEngine {
|
|
|
202
250
|
// cloaking: compare visible text under the two fingerprints
|
|
203
251
|
if (a.humanRaw !== undefined) {
|
|
204
252
|
const tc = performance.now();
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
253
|
+
// The two fetches may legitimately get different formats (a site that hands agents Markdown and
|
|
254
|
+
// browsers HTML). Compare words, not markup: the question is whether the agent was told anything
|
|
255
|
+
// the person would not have seen.
|
|
256
|
+
const humanKind = detectKind(a.humanContentType ?? "", a.humanRaw, a.finalUrl);
|
|
257
|
+
// Against everything in the browser's copy, hidden or not: cloaking means the agent got words the
|
|
258
|
+
// browser was never sent. (Text the browser got but hides is the visibility pass's business, and
|
|
259
|
+
// static CSS analysis can wrongly mark a JavaScript-revealed page as hidden.)
|
|
260
|
+
const humanText = humanKind === "html" ? analyzeHtml(a.humanRaw, a.externalRules, a.externalColorRules ?? []).segments.map((x) => x.text).join(" ") : a.humanRaw;
|
|
261
|
+
const cmp = compareServed(visibleText, humanText);
|
|
262
|
+
// two bands: most of the agent's page is unseen by the browser (a different page), or a solid
|
|
263
|
+
// block of it is (the usual cloaking move: the real page plus a passage only the AI receives)
|
|
264
|
+
const wholesale = cmp.agentWords >= 30 && cmp.agentOnlyShare > 0.4 && cmp.agentOnlyWords >= 15;
|
|
265
|
+
const block = cmp.agentOnlyWords >= 25 && cmp.agentOnlyShare >= 0.15 && cmp.longestAgentOnlyRun >= 12;
|
|
266
|
+
if (wholesale || block) {
|
|
208
267
|
findings.push({
|
|
209
268
|
pass: "cloaking", kind: "cloaking.agent_targeted_divergence",
|
|
210
|
-
confidence: Math.min(0.95, 0.5 + (
|
|
211
|
-
message:
|
|
212
|
-
excerpt:
|
|
269
|
+
confidence: wholesale ? Math.min(0.95, 0.5 + (cmp.agentOnlyShare - 0.4)) : Math.min(0.6, 0.3 + cmp.agentOnlyShare / 2), severity: 0.9,
|
|
270
|
+
message: `${(cmp.agentOnlyShare * 100).toFixed(0)}% of the words served to an AI fetcher do not appear on the page served to a browser`,
|
|
271
|
+
excerpt: `agent-only words: ${cmp.sample}`, action: "none",
|
|
213
272
|
});
|
|
214
273
|
}
|
|
215
274
|
timings.cloaking = performance.now() - tc;
|
|
@@ -299,7 +358,7 @@ export class IngressEngine {
|
|
|
299
358
|
// evidence model (Kith & Revere lineage): class, band, cited artifact, benign alternative
|
|
300
359
|
attachEvidence(findings, sha256(a.raw));
|
|
301
360
|
// score and tier (per-domain thresholds override)
|
|
302
|
-
const score = aggregate(findings);
|
|
361
|
+
const score = aggregate(findings, { cloakingCompared: passesRun.includes("cloaking") });
|
|
303
362
|
const hostileAt = a.rule?.hostileThreshold ?? this.config.hostileThreshold;
|
|
304
363
|
const suspectAt = a.rule?.suspectThreshold ?? this.config.suspectThreshold;
|
|
305
364
|
const tier = score >= hostileAt ? "hostile" : score >= suspectAt ? "suspect" : "clean";
|
|
@@ -361,19 +420,31 @@ export class IngressEngine {
|
|
|
361
420
|
return p.startsWith(".") ? host === p.slice(1) || host.endsWith(p) : host === p;
|
|
362
421
|
});
|
|
363
422
|
}
|
|
364
|
-
refuse(url, findings, timings, t0, kind) {
|
|
423
|
+
refuse(url, findings, timings, t0, kind, tier = "hostile") {
|
|
365
424
|
attachEvidence(findings, sha256(""));
|
|
366
|
-
const
|
|
367
|
-
const
|
|
425
|
+
const score = tier === "hostile" ? 1 : 0;
|
|
426
|
+
const receipt = this.receipt({ url, finalUrl: url, redirectChain: [], kind, raw: "", delivered: "", tier, score, findings, overridden: false, passesRun: tier === "hostile" ? ["policy"] : ["acquire"], acepasteInvisibleCount: 0 });
|
|
427
|
+
const envelope = buildEnvelope(url, tier, "", findings, receipt, false);
|
|
368
428
|
timings.total = performance.now() - t0;
|
|
369
|
-
|
|
370
|
-
|
|
429
|
+
// unreachable is not an attack: it goes to the operational log, never to the attack log or public feed
|
|
430
|
+
if (tier === "hostile")
|
|
431
|
+
this.deps.log(buildLogEntry(url, "hostile", 1, findings, [], receipt, false));
|
|
432
|
+
else
|
|
433
|
+
this.deps.log({ event: "ingress.unreachable", at: this.deps.now().toISOString(), url, reason: findings[0]?.message ?? "" });
|
|
434
|
+
return { tier, score, contentKind: kind, delivered: "", envelope, findings, receipt, timings, overridden: false };
|
|
371
435
|
}
|
|
372
436
|
refuseOnAcquire(url, e, timings, t0) {
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
437
|
+
// a network-policy refusal (private address, denied port) is the gate doing its job; everything else is the source failing to deliver
|
|
438
|
+
if (e.code === "policy" && e.policyCode !== "unresolvable") {
|
|
439
|
+
return this.refuse(url, [{ pass: "acquire", kind: "policy.denied_source", confidence: 1, severity: 1, message: `network policy refused the fetch (${e.policyCode}): ${e.message}`, action: "none" }], timings, t0, "text");
|
|
440
|
+
}
|
|
441
|
+
return this.refuse(url, [{ pass: "acquire", kind: "acquire.unreachable", confidence: 0, severity: 0, message: `could not fetch or read the source (${e.policyCode ?? e.code}): ${e.message}`, action: "none" }], timings, t0, "text", "unreachable");
|
|
442
|
+
}
|
|
443
|
+
pickAgentUserAgent() {
|
|
444
|
+
const list = this.config.agentUserAgents.filter((u) => typeof u === "string" && u.trim());
|
|
445
|
+
if (!list.length)
|
|
446
|
+
return HONEST_USER_AGENT;
|
|
447
|
+
return list[Math.min(list.length - 1, Math.floor(this.deps.random() * list.length))];
|
|
377
448
|
}
|
|
378
449
|
receipt(r) {
|
|
379
450
|
const ecdsa = this.signer();
|
|
@@ -496,16 +567,34 @@ function hostOf(url) {
|
|
|
496
567
|
* fact, not twelve attacks)
|
|
497
568
|
* - noisy-OR across kinds
|
|
498
569
|
* - hidden text that carries no instruction shape is informational: it is
|
|
499
|
-
*
|
|
500
|
-
*
|
|
570
|
+
* never delivered, so visibility-only evidence is capped below the
|
|
571
|
+
* suspect line. The page passes clean, the hidden text is stripped, and
|
|
572
|
+
* the findings stay on the receipt (ratified 2026-11-03: hidden menus,
|
|
573
|
+
* collapsed sections and screen-reader labels must not refuse a page)
|
|
501
574
|
* - co-occurrence of hidden-text and meaning findings, cloaking, and
|
|
502
575
|
* webdriver-keyed divergence each add a bonus
|
|
503
576
|
*/
|
|
504
|
-
|
|
505
|
-
|
|
577
|
+
/** below the default suspect threshold (0.25), so hidden text alone never refuses */
|
|
578
|
+
export const VISIBILITY_ONLY_CAP = 0.2;
|
|
579
|
+
export function aggregate(findings, ctx = {}) {
|
|
580
|
+
const weighted = findings.filter((f) => f.confidence * f.severity > 0);
|
|
581
|
+
if (!weighted.length)
|
|
582
|
+
return 0;
|
|
583
|
+
const passesAll = new Set(weighted.map((f) => f.pass));
|
|
584
|
+
// Hidden text matters when it carries something: an instruction found inside it, invisible
|
|
585
|
+
// characters, or a browser-observed change. Otherwise it is page furniture (menus, labels,
|
|
586
|
+
// collapsed sections), never delivered, and does not add to the score. It stays on the receipt.
|
|
587
|
+
const hiddenCarries = weighted.some((f) => f.pass === "meaning" && f.inHidden) || passesAll.has("character") || passesAll.has("sandbox");
|
|
588
|
+
// Page scripts are never delivered to the model, only the static text is. Their findings
|
|
589
|
+
// (automation probes, obfuscation, minified bundles) predict that the page shows bots something
|
|
590
|
+
// else. Once the two-fingerprint comparison has run and found no divergence, that prediction is
|
|
591
|
+
// answered, so they do not add to the score.
|
|
592
|
+
const scriptsAnswered = !!ctx.cloakingCompared && !passesAll.has("cloaking") && !passesAll.has("sandbox");
|
|
593
|
+
const scored = weighted.filter((f) => (f.pass !== "visibility" || hiddenCarries || passesAll.size === 1) && (f.pass !== "script" || !scriptsAnswered));
|
|
594
|
+
if (!scored.length)
|
|
506
595
|
return 0;
|
|
507
596
|
const byKind = new Map();
|
|
508
|
-
for (const f of
|
|
597
|
+
for (const f of scored) {
|
|
509
598
|
const p = Math.max(0, Math.min(1, f.confidence * f.severity));
|
|
510
599
|
const arr = byKind.get(f.kind) ?? [];
|
|
511
600
|
arr.push(p);
|
|
@@ -518,34 +607,41 @@ export function aggregate(findings) {
|
|
|
518
607
|
notP *= 1 - Math.min(1, max * bonus);
|
|
519
608
|
}
|
|
520
609
|
let score = 1 - notP;
|
|
521
|
-
const passes = new Set(
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
const hiddenAndMeaning = (passes.has("visibility") || passes.has("character") || passes.has("sandbox")) && passes.has("meaning");
|
|
610
|
+
const passes = new Set(scored.map((f) => f.pass));
|
|
611
|
+
if ([...passes].every((p) => p === "visibility"))
|
|
612
|
+
score = Math.min(score, VISIBILITY_ONLY_CAP);
|
|
613
|
+
const hiddenAndMeaning = scored.some((f) => f.pass === "meaning" && f.inHidden) || ((passes.has("character") || passes.has("sandbox")) && passes.has("meaning"));
|
|
526
614
|
if (hiddenAndMeaning)
|
|
527
615
|
score = Math.min(1, score + 0.15);
|
|
528
616
|
if (passes.has("cloaking"))
|
|
529
617
|
score = Math.min(1, score + 0.1);
|
|
530
|
-
if (
|
|
618
|
+
if (scored.some((f) => f.kind === "sandbox.webdriver_keyed_divergence"))
|
|
531
619
|
score = Math.min(1, score + 0.1);
|
|
532
620
|
return +score.toFixed(4);
|
|
533
621
|
}
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
const
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
inter++;
|
|
543
|
-
return inter / (A.size + B.size - inter);
|
|
622
|
+
/** words of served text with markup, link targets and bare URLs removed, so Markdown and HTML of the same page compare equal */
|
|
623
|
+
export function servedWords(text) {
|
|
624
|
+
const t = text
|
|
625
|
+
.replace(/!?\[([^\]]*)\]\([^)]*\)/g, " $1 ") // [label](target) and  -> label
|
|
626
|
+
.replace(/<\/?[a-z][^>]*>/gi, " ") // stray tags
|
|
627
|
+
.replace(/\b(?:https?:\/\/|www\.)\S+/gi, " ") // bare URLs
|
|
628
|
+
.replace(/[#>*_`~|\-]{1,}/g, " "); // markdown syntax
|
|
629
|
+
return t.toLowerCase().split(/[^\p{L}\p{N}']+/u).filter((w) => w.length > 2);
|
|
544
630
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
const
|
|
548
|
-
|
|
631
|
+
/** how much of what the agent received is absent from what the browser received (word occurrences, not distinct words) */
|
|
632
|
+
export function compareServed(agentText, humanText) {
|
|
633
|
+
const A = servedWords(agentText);
|
|
634
|
+
const H = new Set(servedWords(humanText));
|
|
635
|
+
const only = A.filter((w) => !H.has(w));
|
|
636
|
+
let run = 0, longestAgentOnlyRun = 0;
|
|
637
|
+
for (const w of A) {
|
|
638
|
+
run = H.has(w) ? 0 : run + 1;
|
|
639
|
+
if (run > longestAgentOnlyRun)
|
|
640
|
+
longestAgentOnlyRun = run;
|
|
641
|
+
}
|
|
642
|
+
const sampleSeen = new Set();
|
|
643
|
+
const sample = only.filter((w) => (sampleSeen.has(w) ? false : (sampleSeen.add(w), true))).slice(0, 30).join(" ");
|
|
644
|
+
return { agentWords: A.length, agentOnlyWords: only.length, agentOnlyShare: A.length ? only.length / A.length : 0, longestAgentOnlyRun, sample };
|
|
549
645
|
}
|
|
550
646
|
/**
|
|
551
647
|
* Spotlighting-style envelope. Random boundary so content cannot forge the
|
|
@@ -560,6 +656,13 @@ export function buildEnvelope(source, tier, delivered, findings, receipt, overri
|
|
|
560
656
|
`verdict: ${tier}${overridden ? " (delivered under human override)" : ""} (score ${receipt.score})`,
|
|
561
657
|
`receipt: ${receipt.digest.slice(0, 16)} issued ${receipt.issuedAt} engine ${receipt.engineVersion} ruleset ${receipt.rulesetVersion}`,
|
|
562
658
|
];
|
|
659
|
+
if (tier === "unreachable") {
|
|
660
|
+
head.push(`content: UNAVAILABLE. The source could not be fetched or read, so nothing was delivered. This is not an attack verdict; the source may be down, slow or not text.`);
|
|
661
|
+
for (const f of findings.slice(0, 2))
|
|
662
|
+
head.push(` - ${f.message}`);
|
|
663
|
+
head.push(`<</acepaste-ingress ${b}>>`);
|
|
664
|
+
return head.join("\n");
|
|
665
|
+
}
|
|
563
666
|
if (tier === "hostile" || (tier === "suspect" && !overridden)) {
|
|
564
667
|
head.push(`attack: ${kinds.join(", ")}`);
|
|
565
668
|
head.push(tier === "hostile"
|
package/dist/src/index.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
export { IngressEngine, aggregate, buildEnvelope, buildLogEntry } from "./engine.js";
|
|
1
|
+
export { IngressEngine, aggregate, buildEnvelope, buildLogEntry, compareServed, servedWords, VISIBILITY_ONLY_CAP } from "./engine.js";
|
|
2
2
|
export type { EngineDeps, IngressOptions } from "./engine.js";
|
|
3
|
-
export { acquire, AcquireError, MAX_REDIRECTS } from "./acquire.js";
|
|
4
|
-
export type { FetchLike } from "./acquire.js";
|
|
3
|
+
export { acquire, acquireWithBrowser, AcquireError, MAX_REDIRECTS, HONEST_USER_AGENT } from "./acquire.js";
|
|
4
|
+
export type { FetchLike, AcquireOptions } from "./acquire.js";
|
|
5
|
+
export { summarize, findingWeight, cloakingConfigFromEnv } from "./summary.js";
|
|
6
|
+
export type { ResultSummary, FindingGroup, TopFinding } from "./summary.js";
|
|
5
7
|
export { checkTarget, isPrivateIP, isPrivateIPv4, isPrivateIPv6, NetPolicyError, DEFAULT_NET_POLICY } from "./netpolicy.js";
|
|
6
8
|
export type { NetPolicy } from "./netpolicy.js";
|
|
7
9
|
export { analyzeHtml, findStylesheetLinks, rulesFromCss, colorRulesFromCss } from "./visibility.js";
|
package/dist/src/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
export { IngressEngine, aggregate, buildEnvelope, buildLogEntry } from "./engine.js";
|
|
2
|
-
export { acquire, AcquireError, MAX_REDIRECTS } from "./acquire.js";
|
|
1
|
+
export { IngressEngine, aggregate, buildEnvelope, buildLogEntry, compareServed, servedWords, VISIBILITY_ONLY_CAP } from "./engine.js";
|
|
2
|
+
export { acquire, acquireWithBrowser, AcquireError, MAX_REDIRECTS, HONEST_USER_AGENT } from "./acquire.js";
|
|
3
|
+
export { summarize, findingWeight, cloakingConfigFromEnv } from "./summary.js";
|
|
3
4
|
export { checkTarget, isPrivateIP, isPrivateIPv4, isPrivateIPv6, NetPolicyError, DEFAULT_NET_POLICY } from "./netpolicy.js";
|
|
4
5
|
export { analyzeHtml, findStylesheetLinks, rulesFromCss, colorRulesFromCss } from "./visibility.js";
|
|
5
6
|
export { parseColor, contrastRatio, INVISIBLE_CONTRAST } from "./color.js";
|
package/dist/src/meaning.js
CHANGED
|
@@ -136,13 +136,14 @@ function scoreText(text, hidden, where, findings, inCode = false) {
|
|
|
136
136
|
if (roleToken) {
|
|
137
137
|
findings.push({
|
|
138
138
|
pass: "meaning", kind: "meaning.role_token", confidence: Math.min(0.98, (roleToken.weight + hiddenBoost) * roleToken.discount * codeFactor), severity: 0.85,
|
|
139
|
-
message: `${roleToken.weight < 0.5 ? "line-start role label (transcript-shaped)" : "chat-template role token"} in content${inCode ? " (inside a code block)" : ""}${tag(roleToken.variant)}`, excerpt: roleToken.match.slice(0, 80), location: where, action: "neutralized",
|
|
139
|
+
message: `${roleToken.weight < 0.5 ? "line-start role label (transcript-shaped)" : "chat-template role token"} in content${inCode ? " (inside a code block)" : ""}${tag(roleToken.variant)}`, excerpt: roleToken.match.slice(0, 80), location: where, inHidden: hidden, action: "neutralized",
|
|
140
140
|
});
|
|
141
141
|
}
|
|
142
142
|
if (toolCall) {
|
|
143
143
|
findings.push({
|
|
144
|
-
|
|
145
|
-
|
|
144
|
+
// visible tool-call text is usually documentation; quoted or attributed, it is talking about one
|
|
145
|
+
pass: "meaning", kind: "meaning.tool_call_shaped", confidence: Math.min(0.95, (0.6 + hiddenBoost) * toolCall.discount * codeFactor * (!hidden && isQuotedOrAttributed(text, toolCall.match) ? 0.55 : 1)), severity: hidden ? 0.8 : 0.6,
|
|
146
|
+
message: `tool-invocation shaped text in content${inCode ? " (inside a code block)" : ""}${tag(toolCall.variant)}`, excerpt: toolCall.match.slice(0, 120), location: where, inHidden: hidden, action: "neutralized",
|
|
146
147
|
});
|
|
147
148
|
}
|
|
148
149
|
const links = [...text.matchAll(EXFIL_LINK)];
|
|
@@ -164,7 +165,7 @@ function scoreText(text, hidden, where, findings, inCode = false) {
|
|
|
164
165
|
pass: "meaning", kind: n >= 3 || /system\s+prompt/i.test(best.match) ? "meaning.system_prompt_imitation" : "meaning.instruction_shaped",
|
|
165
166
|
confidence: conf, severity,
|
|
166
167
|
message: `${best.label}${n > 1 ? ` (+${n - 1} more instruction shapes)` : ""}${hidden ? ", in text a human would not see" : ""}${inCode ? " (inside a code block)" : ""}${tag(best.variant)}`,
|
|
167
|
-
excerpt: best.match.slice(0, 120), location: where, action: hidden ? "quarantined" : "neutralized",
|
|
168
|
+
excerpt: best.match.slice(0, 120), location: where, inHidden: hidden, action: hidden ? "quarantined" : "neutralized",
|
|
168
169
|
});
|
|
169
170
|
}
|
|
170
171
|
const imgs = [...text.matchAll(MD_IMAGE_EXFIL)];
|
|
@@ -177,11 +178,11 @@ function scoreText(text, hidden, where, findings, inCode = false) {
|
|
|
177
178
|
findings.push({
|
|
178
179
|
pass: "meaning", kind: "meaning.exfil_markdown_image", confidence: (strongImg ? 0.85 : 0.25) * codeFactor, severity: 0.9,
|
|
179
180
|
message: `markdown image whose URL carries ${imgs[0][1].split("?")[1].length} bytes of query data (EchoLeak pattern)`,
|
|
180
|
-
excerpt: imgs[0][1].slice(0, 120), location: where, action: "neutralized",
|
|
181
|
+
excerpt: imgs[0][1].slice(0, 120), location: where, inHidden: hidden, action: "neutralized",
|
|
181
182
|
});
|
|
182
183
|
}
|
|
183
184
|
else if (hidden && MD_IMAGE_ANY.test(text)) {
|
|
184
|
-
findings.push({ pass: "meaning", kind: "meaning.exfil_markdown_image", confidence: 0.6, severity: 0.7, message: `markdown image reference inside hidden text`, location: where, action: "neutralized" });
|
|
185
|
+
findings.push({ pass: "meaning", kind: "meaning.exfil_markdown_image", confidence: 0.6, severity: 0.7, message: `markdown image reference inside hidden text`, location: where, inHidden: hidden, action: "neutralized" });
|
|
185
186
|
}
|
|
186
187
|
MD_IMAGE_ANY.lastIndex = 0;
|
|
187
188
|
if (links.length || hooks.length) {
|
|
@@ -189,7 +190,7 @@ function scoreText(text, hidden, where, findings, inCode = false) {
|
|
|
189
190
|
// a webhook URL in visible prose is documentation more often than exfiltration; hidden or alongside an instruction it is not
|
|
190
191
|
pass: "meaning", kind: "meaning.exfil_link", confidence: (hooks.length ? (hidden || hits.some((h) => h.weight >= 0.8) ? 0.8 : 0.3) : hidden ? 0.6 : 0.3) * codeFactor, severity: 0.8,
|
|
191
192
|
message: hooks.length ? `link to a known collection endpoint (${hooks[0][1]})` : `link with templated query parameters for context exfiltration`,
|
|
192
|
-
excerpt: (hooks[0]?.[0] ?? links[0]?.[1] ?? "").slice(0, 120), location: where, action: "neutralized",
|
|
193
|
+
excerpt: (hooks[0]?.[0] ?? links[0]?.[1] ?? "").slice(0, 120), location: where, inHidden: hidden, action: "neutralized",
|
|
193
194
|
});
|
|
194
195
|
}
|
|
195
196
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compact, machine-readable digest of a result: findings grouped by pass and
|
|
3
|
+
* kind with counts and peak weight, plus the heaviest findings first.
|
|
4
|
+
* Adapted from Briar Greenway's probe script (2026-11-03).
|
|
5
|
+
*/
|
|
6
|
+
import type { Finding, IngressResult } from "./types.js";
|
|
7
|
+
export interface FindingGroup {
|
|
8
|
+
pass: string;
|
|
9
|
+
kind: string;
|
|
10
|
+
count: number;
|
|
11
|
+
maxWeight: number | null;
|
|
12
|
+
}
|
|
13
|
+
export interface TopFinding {
|
|
14
|
+
pass: string;
|
|
15
|
+
kind: string;
|
|
16
|
+
weight: number | null;
|
|
17
|
+
message: string;
|
|
18
|
+
excerpt: string;
|
|
19
|
+
}
|
|
20
|
+
export interface ResultSummary {
|
|
21
|
+
url: string;
|
|
22
|
+
tier: IngressResult["tier"];
|
|
23
|
+
score: number;
|
|
24
|
+
contentKind: IngressResult["contentKind"];
|
|
25
|
+
findingCount: number;
|
|
26
|
+
groups: FindingGroup[];
|
|
27
|
+
top: TopFinding[];
|
|
28
|
+
}
|
|
29
|
+
/** confidence x severity, or null when either is not a finite number */
|
|
30
|
+
export declare function findingWeight(f: Pick<Finding, "confidence" | "severity">): number | null;
|
|
31
|
+
export declare function summarize(result: IngressResult, topN?: number): ResultSummary;
|
|
32
|
+
/**
|
|
33
|
+
* Cloaking-check settings from environment variables, shared by the CLI, the
|
|
34
|
+
* MCP server and the HTTP service:
|
|
35
|
+
* INGRESS_AGENT_UA "honest" to send only the gate's own name, or a "|"-separated list of user agents
|
|
36
|
+
* INGRESS_CLOAKING_JITTER "min,max" milliseconds between the two fetches
|
|
37
|
+
* INGRESS_HUMAN_PROXY HTTP(S) proxy URL for the browser-fingerprint fetch (e.g. a residential proxy)
|
|
38
|
+
* INGRESS_HUMAN_FETCH "browser" to make that fetch in a real headless Chromium
|
|
39
|
+
*/
|
|
40
|
+
export declare function cloakingConfigFromEnv(env: Record<string, string | undefined>, honestUserAgent: string): Partial<import("./types.js").IngressConfig>;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compact, machine-readable digest of a result: findings grouped by pass and
|
|
3
|
+
* kind with counts and peak weight, plus the heaviest findings first.
|
|
4
|
+
* Adapted from Briar Greenway's probe script (2026-11-03).
|
|
5
|
+
*/
|
|
6
|
+
/** confidence x severity, or null when either is not a finite number */
|
|
7
|
+
export function findingWeight(f) {
|
|
8
|
+
const n = f.confidence * f.severity;
|
|
9
|
+
return Number.isFinite(n) ? n : null;
|
|
10
|
+
}
|
|
11
|
+
const rounded = (n) => (n === null ? null : Number(n.toFixed(2)));
|
|
12
|
+
export function summarize(result, topN = 6) {
|
|
13
|
+
const groups = new Map();
|
|
14
|
+
for (const f of result.findings) {
|
|
15
|
+
// a tuple key cannot collide even if a pass or kind contains a colon
|
|
16
|
+
const key = JSON.stringify([f.pass, f.kind]);
|
|
17
|
+
const g = groups.get(key) ?? { pass: f.pass, kind: f.kind, count: 0, maxWeight: null };
|
|
18
|
+
g.count++;
|
|
19
|
+
const w = findingWeight(f);
|
|
20
|
+
if (w !== null && (g.maxWeight === null || w > g.maxWeight))
|
|
21
|
+
g.maxWeight = w;
|
|
22
|
+
groups.set(key, g);
|
|
23
|
+
}
|
|
24
|
+
const top = result.findings
|
|
25
|
+
.map((f, index) => ({ f, index, w: findingWeight(f) }))
|
|
26
|
+
.sort((a, b) => (b.w ?? -Infinity) - (a.w ?? -Infinity) || a.index - b.index)
|
|
27
|
+
.slice(0, topN)
|
|
28
|
+
.map(({ f, w }) => ({
|
|
29
|
+
pass: f.pass, kind: f.kind, weight: rounded(w),
|
|
30
|
+
message: (f.message || "").slice(0, 140),
|
|
31
|
+
excerpt: (f.excerpt || "").replace(/\s+/g, " ").slice(0, 80),
|
|
32
|
+
}));
|
|
33
|
+
return {
|
|
34
|
+
url: result.receipt.finalUrl || result.receipt.url,
|
|
35
|
+
tier: result.tier, score: result.score, contentKind: result.contentKind,
|
|
36
|
+
findingCount: result.findings.length,
|
|
37
|
+
groups: [...groups.values()].map((g) => ({ ...g, maxWeight: rounded(g.maxWeight) })),
|
|
38
|
+
top,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Cloaking-check settings from environment variables, shared by the CLI, the
|
|
43
|
+
* MCP server and the HTTP service:
|
|
44
|
+
* INGRESS_AGENT_UA "honest" to send only the gate's own name, or a "|"-separated list of user agents
|
|
45
|
+
* INGRESS_CLOAKING_JITTER "min,max" milliseconds between the two fetches
|
|
46
|
+
* INGRESS_HUMAN_PROXY HTTP(S) proxy URL for the browser-fingerprint fetch (e.g. a residential proxy)
|
|
47
|
+
* INGRESS_HUMAN_FETCH "browser" to make that fetch in a real headless Chromium
|
|
48
|
+
*/
|
|
49
|
+
export function cloakingConfigFromEnv(env, honestUserAgent) {
|
|
50
|
+
const cfg = {};
|
|
51
|
+
const ua = env.INGRESS_AGENT_UA?.trim();
|
|
52
|
+
if (ua)
|
|
53
|
+
cfg.agentUserAgents = ua === "honest" ? [honestUserAgent] : ua.split("|").map((s) => s.trim()).filter(Boolean);
|
|
54
|
+
const j = env.INGRESS_CLOAKING_JITTER?.split(",").map((x) => Number(x.trim()));
|
|
55
|
+
if (j && j.length === 2 && j.every((x) => Number.isFinite(x)))
|
|
56
|
+
cfg.cloakingJitterMs = [j[0], j[1]];
|
|
57
|
+
if (env.INGRESS_HUMAN_PROXY)
|
|
58
|
+
cfg.humanProxy = env.INGRESS_HUMAN_PROXY;
|
|
59
|
+
if (env.INGRESS_HUMAN_FETCH === "browser" || env.INGRESS_HUMAN_FETCH === "http")
|
|
60
|
+
cfg.humanFetch = env.INGRESS_HUMAN_FETCH;
|
|
61
|
+
return cfg;
|
|
62
|
+
}
|
package/dist/src/taint.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* the user-facing layer. Nothing here is mandatory; it is middleware the
|
|
8
8
|
* integrator wraps around tool dispatch.
|
|
9
9
|
*/
|
|
10
|
-
const TIER_RANK = { clean: 0, suspect: 1, hostile: 2 };
|
|
10
|
+
const TIER_RANK = { clean: 0, unreachable: 0, suspect: 1, hostile: 2 };
|
|
11
11
|
export function createTaintGuard(policy = {}) {
|
|
12
12
|
const p = {
|
|
13
13
|
taintAt: policy.taintAt ?? "suspect",
|
package/dist/src/types.d.ts
CHANGED
|
@@ -5,13 +5,18 @@
|
|
|
5
5
|
* at the same source would not see is suspect by default.
|
|
6
6
|
*/
|
|
7
7
|
import type { NetPolicy } from "./netpolicy.js";
|
|
8
|
-
export declare const ENGINE_VERSION = "0.
|
|
8
|
+
export declare const ENGINE_VERSION = "0.8.0";
|
|
9
9
|
/** bump whenever a detector's behavior changes; receipts carry it so verdicts are reproducible */
|
|
10
|
-
export declare const RULESET_VERSION = "2026.
|
|
11
|
-
|
|
10
|
+
export declare const RULESET_VERSION = "2026.11.03-1";
|
|
11
|
+
/**
|
|
12
|
+
* clean: delivered. suspect: refused, overridable. hostile: refused, not overridable.
|
|
13
|
+
* unreachable: nothing could be fetched or read (network error, timeout, redirect loop,
|
|
14
|
+
* binary or oversized body). Not an attack verdict, and never written to the attack log.
|
|
15
|
+
*/
|
|
16
|
+
export type Tier = "clean" | "suspect" | "hostile" | "unreachable";
|
|
12
17
|
export type ContentKind = "html" | "markdown" | "json" | "text" | "javascript";
|
|
13
18
|
export type PassName = "acquire" | "policy" | "visibility" | "character" | "script" | "supplychain" | "secrets" | "meaning" | "cloaking" | "sandbox" | "plugin";
|
|
14
|
-
export type FindingKind = "policy.denied_source" | "policy.redirect_chain" | "unicode.tag_characters" | "unicode.zero_width" | "unicode.bidi_control" | "unicode.variation_selector_smuggling" | "unicode.mixed_script_confusable" | "unicode.fullwidth_or_math_lookalike" | "unicode.combining_flood" | "unicode.encoded_payload" | "unicode.entity_smuggling" | "unicode.whitespace_channel" | "visibility.css_hidden" | "visibility.offscreen" | "visibility.attribute_only" | "visibility.html_comment" | "visibility.tiny_or_transparent" | "visibility.markdown_hidden" | "script.obfuscation" | "script.runtime_codegen" | "script.automation_detection" | "script.post_load_text_injection" | "script.exfil_beacon" | "script.binary_decoding" | "supplychain.lifecycle_script" | "supplychain.url_dependency" | "supplychain.offregistry_resolution" | "supplychain.install_instruction" | "secrets.credential_in_content" | "sandbox.text_injected_after_load" | "sandbox.webdriver_keyed_divergence" | "sandbox.computed_hidden_text" | "sandbox.blocked_egress_attempt" | "meaning.instruction_shaped" | "meaning.role_token" | "meaning.system_prompt_imitation" | "meaning.tool_call_shaped" | "meaning.exfil_markdown_image" | "meaning.exfil_link" | "cloaking.agent_targeted_divergence" | `plugin.${string}`;
|
|
19
|
+
export type FindingKind = "policy.denied_source" | "policy.redirect_chain" | "acquire.unreachable" | "acquire.cloaking_check_incomplete" | "acquire.ai_agent_refused" | "unicode.tag_characters" | "unicode.zero_width" | "unicode.bidi_control" | "unicode.variation_selector_smuggling" | "unicode.mixed_script_confusable" | "unicode.fullwidth_or_math_lookalike" | "unicode.combining_flood" | "unicode.encoded_payload" | "unicode.entity_smuggling" | "unicode.whitespace_channel" | "visibility.css_hidden" | "visibility.offscreen" | "visibility.attribute_only" | "visibility.html_comment" | "visibility.tiny_or_transparent" | "visibility.markdown_hidden" | "script.obfuscation" | "script.runtime_codegen" | "script.automation_detection" | "script.post_load_text_injection" | "script.exfil_beacon" | "script.binary_decoding" | "supplychain.lifecycle_script" | "supplychain.url_dependency" | "supplychain.offregistry_resolution" | "supplychain.install_instruction" | "secrets.credential_in_content" | "sandbox.text_injected_after_load" | "sandbox.webdriver_keyed_divergence" | "sandbox.computed_hidden_text" | "sandbox.blocked_egress_attempt" | "meaning.instruction_shaped" | "meaning.role_token" | "meaning.system_prompt_imitation" | "meaning.tool_call_shaped" | "meaning.exfil_markdown_image" | "meaning.exfil_link" | "cloaking.agent_targeted_divergence" | `plugin.${string}`;
|
|
15
20
|
export interface Finding {
|
|
16
21
|
pass: PassName;
|
|
17
22
|
kind: FindingKind;
|
|
@@ -25,6 +30,8 @@ export interface Finding {
|
|
|
25
30
|
excerpt?: string;
|
|
26
31
|
/** location hint: character offset, selector path, or script index */
|
|
27
32
|
location?: string;
|
|
33
|
+
/** meaning pass: the match was in text a human would not see */
|
|
34
|
+
inHidden?: boolean;
|
|
28
35
|
/** what the engine did about it */
|
|
29
36
|
action: "none" | "neutralized" | "quarantined";
|
|
30
37
|
/** Kith & Revere evidence model, attached by the engine: a finding is a conclusion, never a Fact */
|
|
@@ -47,6 +54,10 @@ export interface AcquireResult {
|
|
|
47
54
|
body: string;
|
|
48
55
|
/** which fingerprint this acquisition used */
|
|
49
56
|
fingerprint: "human" | "agent";
|
|
57
|
+
/** the user-agent string actually sent */
|
|
58
|
+
userAgent: string;
|
|
59
|
+
/** how it was fetched: plain HTTP client, or a real browser (human fingerprint only) */
|
|
60
|
+
via: "http" | "browser";
|
|
50
61
|
fetchedAt: string;
|
|
51
62
|
}
|
|
52
63
|
export interface TextSegment {
|
|
@@ -146,6 +157,21 @@ export interface IngressConfig {
|
|
|
146
157
|
suspectThreshold: number;
|
|
147
158
|
/** run the second, human-fingerprinted fetch and compare */
|
|
148
159
|
cloakingCheck: boolean;
|
|
160
|
+
/**
|
|
161
|
+
* user agents for the agent fetch, one picked at random per fetch. Defaults to the
|
|
162
|
+
* user-triggered fetchers of the major AI assistants, because those are what
|
|
163
|
+
* agent-targeted cloaking keys on. Set to one honest string to stop impersonating.
|
|
164
|
+
*/
|
|
165
|
+
agentUserAgents: string[];
|
|
166
|
+
/** random gap between the two fetches, [min, max] milliseconds; the order is random too */
|
|
167
|
+
cloakingJitterMs: [number, number];
|
|
168
|
+
/**
|
|
169
|
+
* optional HTTP(S) proxy for the human fetch only, e.g. a residential proxy, so the two
|
|
170
|
+
* fetches leave from different networks. Credentials go in the URL. Never logged.
|
|
171
|
+
*/
|
|
172
|
+
humanProxy?: string;
|
|
173
|
+
/** "http": the human fetch is a plain request with browser headers. "browser": a real headless Chromium loads the page. */
|
|
174
|
+
humanFetch: "http" | "browser";
|
|
149
175
|
/** maximum body size accepted, bytes */
|
|
150
176
|
maxBodyBytes: number;
|
|
151
177
|
/** fetch timeout, milliseconds */
|
package/dist/src/types.js
CHANGED
|
@@ -4,13 +4,21 @@
|
|
|
4
4
|
* Governing principle: anything the model would receive that a human looking
|
|
5
5
|
* at the same source would not see is suspect by default.
|
|
6
6
|
*/
|
|
7
|
-
export const ENGINE_VERSION = "0.
|
|
7
|
+
export const ENGINE_VERSION = "0.8.0";
|
|
8
8
|
/** bump whenever a detector's behavior changes; receipts carry it so verdicts are reproducible */
|
|
9
|
-
export const RULESET_VERSION = "2026.
|
|
9
|
+
export const RULESET_VERSION = "2026.11.03-1";
|
|
10
10
|
export const DEFAULT_CONFIG = {
|
|
11
11
|
hostileThreshold: 0.7,
|
|
12
12
|
suspectThreshold: 0.25,
|
|
13
13
|
cloakingCheck: true,
|
|
14
|
+
agentUserAgents: [
|
|
15
|
+
// sources: OpenAI bot docs; Cloudflare Radar bot directory; Perplexity agent docs (checked 2026-11-03)
|
|
16
|
+
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
|
|
17
|
+
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com)",
|
|
18
|
+
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://www.perplexity.ai)",
|
|
19
|
+
],
|
|
20
|
+
cloakingJitterMs: [250, 1500],
|
|
21
|
+
humanFetch: "http",
|
|
14
22
|
maxBodyBytes: 5 * 1024 * 1024,
|
|
15
23
|
fetchTimeoutMs: 15000,
|
|
16
24
|
cacheTtlSeconds: 86400,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@acepaste/ingress",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "AcePaste Ingress engine: acquire, visibility, character, script, supply-chain, sandbox, meaning, verdict, envelope, receipts.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/src/index.js",
|