@acepaste/ingress 0.0.0-stage → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +105 -0
- package/README.md +15 -2
- package/dist/src/acepaste-alphabet.d.ts +26 -0
- package/dist/src/acepaste-alphabet.js +47 -0
- package/dist/src/acquire.d.ts +42 -0
- package/dist/src/acquire.js +271 -0
- package/dist/src/adapters.d.ts +30 -0
- package/dist/src/adapters.js +160 -0
- package/dist/src/character.d.ts +22 -0
- package/dist/src/character.js +291 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +75 -0
- package/dist/src/color.d.ts +26 -0
- package/dist/src/color.js +111 -0
- package/dist/src/engine.d.ts +131 -0
- package/dist/src/engine.js +715 -0
- package/dist/src/evidence.d.ts +21 -0
- package/dist/src/evidence.js +77 -0
- package/dist/src/index.d.ts +28 -0
- package/dist/src/index.js +20 -0
- package/dist/src/meaning.d.ts +28 -0
- package/dist/src/meaning.js +227 -0
- package/dist/src/netpolicy.d.ts +34 -0
- package/dist/src/netpolicy.js +122 -0
- package/dist/src/normalize.d.ts +34 -0
- package/dist/src/normalize.js +204 -0
- package/dist/src/policyfile.d.ts +2 -0
- package/dist/src/policyfile.js +50 -0
- package/dist/src/sandbox.d.ts +45 -0
- package/dist/src/sandbox.js +231 -0
- package/dist/src/script.d.ts +23 -0
- package/dist/src/script.js +220 -0
- package/dist/src/secrets.d.ts +24 -0
- package/dist/src/secrets.js +192 -0
- package/dist/src/summary.d.ts +40 -0
- package/dist/src/summary.js +62 -0
- package/dist/src/supplychain.d.ts +24 -0
- package/dist/src/supplychain.js +182 -0
- package/dist/src/taint.d.ts +37 -0
- package/dist/src/taint.js +39 -0
- package/dist/src/types.d.ts +202 -0
- package/dist/src/types.js +33 -0
- package/dist/src/visibility.d.ts +41 -0
- package/dist/src/visibility.js +424 -0
- package/package.json +64 -4
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content adapters: turn any supported content kind into the common shape
|
|
3
|
+
* the passes consume (segments with visibility, scripts, decoded blobs).
|
|
4
|
+
*
|
|
5
|
+
* HTML goes through visibility.ts. Everything else comes through here.
|
|
6
|
+
*/
|
|
7
|
+
export function detectKind(contentType, body, urlHint = "") {
|
|
8
|
+
const ct = contentType.toLowerCase();
|
|
9
|
+
if (/html|xhtml/.test(ct))
|
|
10
|
+
return "html";
|
|
11
|
+
if (/markdown/.test(ct) || /\.(md|markdown)(\?|#|$)/i.test(urlHint))
|
|
12
|
+
return "markdown";
|
|
13
|
+
if (/json/.test(ct))
|
|
14
|
+
return "json";
|
|
15
|
+
if (/javascript|ecmascript/.test(ct) || /\.(m?js|cjs)(\?|#|$)/i.test(urlHint))
|
|
16
|
+
return "javascript";
|
|
17
|
+
const head = body.slice(0, 2048);
|
|
18
|
+
if (/^\s*<(!doctype|html|head|body|div|p|span|article|main|table)/i.test(head))
|
|
19
|
+
return "html";
|
|
20
|
+
if (/^\s*[\[{]/.test(head)) {
|
|
21
|
+
try {
|
|
22
|
+
JSON.parse(body);
|
|
23
|
+
return "json";
|
|
24
|
+
}
|
|
25
|
+
catch { /* not json */ }
|
|
26
|
+
}
|
|
27
|
+
if (/^(#{1,6}\s|\*\*|- |\d+\. |```|\[.+\]\(.+\))/m.test(head) && /\n/.test(head))
|
|
28
|
+
return "markdown";
|
|
29
|
+
return "text";
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Decode numeric and the common named entities so entity-smuggled
|
|
33
|
+
* characters (​ ​ ‍) reach the character pass as the
|
|
34
|
+
* characters they are. Returns the decoded text and the count of
|
|
35
|
+
* suspicious decodes (format controls, tags, selectors).
|
|
36
|
+
*/
|
|
37
|
+
export function decodeEntities(text) {
|
|
38
|
+
const named = {
|
|
39
|
+
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", zwj: "", zwnj: "", lrm: "", rlm: "", shy: "",
|
|
40
|
+
NoBreak: "", ZeroWidthSpace: "", NegativeMediumSpace: "",
|
|
41
|
+
};
|
|
42
|
+
let suspicious = 0;
|
|
43
|
+
let sample = "";
|
|
44
|
+
const isSuspicious = (cp) => (cp >= 0x200b && cp <= 0x200f) || cp === 0x2060 || cp === 0xfeff || cp === 0x00ad || (cp >= 0x202a && cp <= 0x202e) ||
|
|
45
|
+
(cp >= 0x2066 && cp <= 0x2069) || (cp >= 0xfe00 && cp <= 0xfe0f) || (cp >= 0xe0000 && cp <= 0xe007f) || (cp >= 0xe0100 && cp <= 0xe01ef);
|
|
46
|
+
const out = text.replace(/&(#x[0-9a-fA-F]{1,6}|#\d{1,7}|[a-zA-Z]{2,20});/g, (m, body) => {
|
|
47
|
+
let cp;
|
|
48
|
+
if (body[0] === "#")
|
|
49
|
+
cp = body[1] === "x" || body[1] === "X" ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
|
50
|
+
else if (body in named) {
|
|
51
|
+
const ch = named[body];
|
|
52
|
+
cp = ch.codePointAt(0);
|
|
53
|
+
}
|
|
54
|
+
if (cp === undefined || Number.isNaN(cp) || cp > 0x10ffff)
|
|
55
|
+
return m;
|
|
56
|
+
if (isSuspicious(cp)) {
|
|
57
|
+
suspicious++;
|
|
58
|
+
if (!sample)
|
|
59
|
+
sample = m;
|
|
60
|
+
}
|
|
61
|
+
return String.fromCodePoint(cp);
|
|
62
|
+
});
|
|
63
|
+
return { text: out, suspicious, sample };
|
|
64
|
+
}
|
|
65
|
+
/** Markdown: comments and non-rendered constructs are hidden; rendered text is visible. */
|
|
66
|
+
export function adaptMarkdown(raw) {
|
|
67
|
+
const segments = [];
|
|
68
|
+
const findings = [];
|
|
69
|
+
let text = raw;
|
|
70
|
+
// HTML comments inside markdown render nothing
|
|
71
|
+
text = text.replace(/<!--([\s\S]*?)-->/g, (_, c) => {
|
|
72
|
+
if (c.trim())
|
|
73
|
+
segments.push({ text: c, visibility: "comment", path: "md/comment" });
|
|
74
|
+
return " ";
|
|
75
|
+
});
|
|
76
|
+
// reference-style link definitions: [id]: url "title" — the title and url never render
|
|
77
|
+
text = text.replace(/^\s{0,3}\[[^\]]+\]:\s*(\S+)(?:\s+["'(]([^"')]*)["')])?\s*$/gm, (_, url, title) => {
|
|
78
|
+
if (title?.trim())
|
|
79
|
+
segments.push({ text: title, visibility: "attribute", path: "md/linkdef@title" });
|
|
80
|
+
if (/[?#].{40,}/.test(url))
|
|
81
|
+
segments.push({ text: url, visibility: "attribute", path: "md/linkdef@url" });
|
|
82
|
+
return " ";
|
|
83
|
+
});
|
|
84
|
+
// inline link/image titles and urls
|
|
85
|
+
text = text.replace(/!?\[([^\]]*)\]\(([^)\s]+)(?:\s+["']([^"']*)["'])?\)/g, (m, label, url, title) => {
|
|
86
|
+
if (title?.trim())
|
|
87
|
+
segments.push({ text: title, visibility: "attribute", path: "md/link@title" });
|
|
88
|
+
// keep the construct for the meaning pass (markdown image exfil) by leaving it in visible text
|
|
89
|
+
return m;
|
|
90
|
+
});
|
|
91
|
+
// raw HTML blocks inside markdown: hidden attributes and styles
|
|
92
|
+
text = text.replace(/<([a-z][\w-]*)([^>]*)>/gi, (m, _tag, attrs) => {
|
|
93
|
+
if (/display\s*:\s*none|visibility\s*:\s*hidden|hidden(\s|=|>|$)|aria-hidden\s*=\s*["']?true|font-size\s*:\s*0|opacity\s*:\s*0/i.test(attrs)) {
|
|
94
|
+
segments.push({ text: m, visibility: "css_hidden", path: "md/html-open-tag" });
|
|
95
|
+
findings.push({ pass: "visibility", kind: "visibility.markdown_hidden", confidence: 0.6, severity: 0.6, message: "raw HTML with hiding styles inside markdown", excerpt: m.slice(0, 120), location: "md/html", action: "none" });
|
|
96
|
+
}
|
|
97
|
+
const alt = attrs.match(/\b(alt|title|aria-label)\s*=\s*["']([^"']{20,})["']/i);
|
|
98
|
+
if (alt)
|
|
99
|
+
segments.push({ text: alt[2], visibility: "attribute", path: `md/html@${alt[1]}` });
|
|
100
|
+
return " ";
|
|
101
|
+
});
|
|
102
|
+
// fenced code is visible but is code; keep it visible (the model reads it)
|
|
103
|
+
segments.unshift({ text, visibility: "visible", path: "md" });
|
|
104
|
+
for (const s of segments) {
|
|
105
|
+
if (s.visibility === "visible")
|
|
106
|
+
continue;
|
|
107
|
+
const words = s.text.trim().split(/\s+/).length;
|
|
108
|
+
if (words < 6)
|
|
109
|
+
continue;
|
|
110
|
+
findings.push({
|
|
111
|
+
pass: "visibility", kind: "visibility.markdown_hidden", confidence: Math.min(0.9, 0.5 + words / 100), severity: 0.6,
|
|
112
|
+
message: `${words} words in a markdown construct that does not render (${s.path})`, excerpt: s.text.trim().slice(0, 160), location: s.path, action: "none",
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
return { segments, findings };
|
|
116
|
+
}
|
|
117
|
+
/** JSON: every string value is a segment; keys hint at intent. Nothing is "hidden" in JSON, but long strings in unusual keys are attribute-like. */
|
|
118
|
+
export function adaptJson(raw) {
|
|
119
|
+
const segments = [];
|
|
120
|
+
const findings = [];
|
|
121
|
+
let doc;
|
|
122
|
+
try {
|
|
123
|
+
doc = JSON.parse(raw);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return adaptText(raw);
|
|
127
|
+
}
|
|
128
|
+
const walk = (v, path, depth) => {
|
|
129
|
+
if (depth > 64)
|
|
130
|
+
return;
|
|
131
|
+
if (typeof v === "string") {
|
|
132
|
+
const keyPath = path.replace(/^\$/, "");
|
|
133
|
+
if (v.trim())
|
|
134
|
+
segments.push({ text: v, visibility: /(\.|\[)(_|__|\$|meta|hidden|internal|debug|comment|instruction|instructions|prompt|system)(\b|\.|\[|$)/i.test(keyPath) ? "attribute" : "visible", path });
|
|
135
|
+
}
|
|
136
|
+
else if (Array.isArray(v))
|
|
137
|
+
v.forEach((x, i) => walk(x, `${path}[${i}]`, depth + 1));
|
|
138
|
+
else if (v && typeof v === "object")
|
|
139
|
+
for (const [k, x] of Object.entries(v))
|
|
140
|
+
walk(x, `${path}.${k}`, depth + 1);
|
|
141
|
+
};
|
|
142
|
+
walk(doc, "$", 0);
|
|
143
|
+
for (const s of segments) {
|
|
144
|
+
if (s.visibility !== "attribute")
|
|
145
|
+
continue;
|
|
146
|
+
const words = s.text.trim().split(/\s+/).length;
|
|
147
|
+
if (words >= 6)
|
|
148
|
+
findings.push({ pass: "visibility", kind: "visibility.attribute_only", confidence: 0.4, severity: 0.5, message: `${words} words under a metadata-like JSON key`, excerpt: s.text.slice(0, 160), location: s.path, action: "none" });
|
|
149
|
+
}
|
|
150
|
+
return { segments, findings };
|
|
151
|
+
}
|
|
152
|
+
export function adaptText(raw) {
|
|
153
|
+
return { segments: [{ text: raw, visibility: "visible", path: "text" }], findings: [] };
|
|
154
|
+
}
|
|
155
|
+
/** Plain JavaScript fetched directly (a module, a bundle): the whole thing is a script; any string literals are visible text. */
|
|
156
|
+
export function adaptJavascript(raw) {
|
|
157
|
+
const strings = raw.match(/(["'`])(?:\\.|(?!\1)[^\\\n])*\1/g) ?? [];
|
|
158
|
+
const text = strings.map((s) => s.slice(1, -1)).join("\n");
|
|
159
|
+
return { segments: [{ text, visibility: "visible", path: "js/strings" }], findings: [] };
|
|
160
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Character pass (AcePaste lineage).
|
|
3
|
+
*
|
|
4
|
+
* The hostile-invisible alphabet is ported verbatim from the production
|
|
5
|
+
* AcePaste cleaner (see acepaste-alphabet.ts), so the gate strips exactly
|
|
6
|
+
* what the web tool and the Chrome extension strip. On top of that shared
|
|
7
|
+
* set, the gate adds script-aware exceptions: zero-width joiners inside
|
|
8
|
+
* emoji sequences and Indic/Arabic text, zero-width non-joiners in complex
|
|
9
|
+
* scripts, and bidirectional marks (not overrides or isolates) inside
|
|
10
|
+
* right-to-left runs are legitimate and left in place; and it adds the
|
|
11
|
+
* decode-and-rescan behavior the gate needs that the cleaner does not.
|
|
12
|
+
*/
|
|
13
|
+
import type { Finding } from "./types.js";
|
|
14
|
+
export { countHostileInvisibles } from "./acepaste-alphabet.js";
|
|
15
|
+
export interface CharacterResult {
|
|
16
|
+
/** text with hostile characters removed and encoded payloads untouched */
|
|
17
|
+
cleaned: string;
|
|
18
|
+
findings: Finding[];
|
|
19
|
+
/** decoded payloads that should be rescanned by the meaning pass */
|
|
20
|
+
decodedPayloads: string[];
|
|
21
|
+
}
|
|
22
|
+
export declare function scanCharacters(text: string): CharacterResult;
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Character pass (AcePaste lineage).
|
|
3
|
+
*
|
|
4
|
+
* The hostile-invisible alphabet is ported verbatim from the production
|
|
5
|
+
* AcePaste cleaner (see acepaste-alphabet.ts), so the gate strips exactly
|
|
6
|
+
* what the web tool and the Chrome extension strip. On top of that shared
|
|
7
|
+
* set, the gate adds script-aware exceptions: zero-width joiners inside
|
|
8
|
+
* emoji sequences and Indic/Arabic text, zero-width non-joiners in complex
|
|
9
|
+
* scripts, and bidirectional marks (not overrides or isolates) inside
|
|
10
|
+
* right-to-left runs are legitimate and left in place; and it adds the
|
|
11
|
+
* decode-and-rescan behavior the gate needs that the cleaner does not.
|
|
12
|
+
*/
|
|
13
|
+
import { decodePayloads } from "./normalize.js";
|
|
14
|
+
export { countHostileInvisibles } from "./acepaste-alphabet.js";
|
|
15
|
+
const TAG_RANGE = /[\u{E0000}-\u{E007F}]/gu;
|
|
16
|
+
// AcePaste basic-plane invisibles minus the code points with dedicated handling below
|
|
17
|
+
// (ZWJ/ZWNJ script-aware, LRM/RLM bidi-mark-aware, bidi overrides, variation selectors).
|
|
18
|
+
const ZERO_WIDTH = /[\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180E\u200B\u2060-\u206F\u2800\u3164\uFEFF\uFFA0]/g;
|
|
19
|
+
const ZWJ = "";
|
|
20
|
+
const ZWNJ = "";
|
|
21
|
+
const BIDI_OVERRIDE_ISOLATE = /[--]/g; // embeddings, overrides, isolates: never legitimate in plain text intake
|
|
22
|
+
const BIDI_MARKS = /[]/g; // LRM, RLM, ALM: legitimate in RTL text
|
|
23
|
+
const VARIATION_SELECTORS = /[︀-️]|[\u{E0100}-\u{E01EF}]/gu;
|
|
24
|
+
const EMOJI_ADJ = /\p{Extended_Pictographic}|[\u{1F3FB}-\u{1F3FF}]|️/u;
|
|
25
|
+
const INDIC_OR_ARABIC = /[-ۿݐ-ݿࢠ-ࣿऀ-က-႟ក-]/u;
|
|
26
|
+
const RTL_SCRIPT = /[--ۿݐ-ݿࢠ-ࣿיִ-﷿ﹰ-]/u;
|
|
27
|
+
// Minimal confusable table: Cyrillic/Greek letters that render like Latin.
|
|
28
|
+
const CONFUSABLE = {
|
|
29
|
+
"а": "a", "е": "e", "о": "o", "р": "p", "с": "c", "у": "y", "х": "x",
|
|
30
|
+
"і": "i", "ј": "j", "һ": "h", "ԁ": "d", "ԛ": "q", "ѕ": "s",
|
|
31
|
+
"Α": "A", "Β": "B", "Ε": "E", "Η": "H", "Ι": "I", "Κ": "K", "Μ": "M",
|
|
32
|
+
"Ν": "N", "Ο": "O", "Ρ": "P", "Τ": "T", "Υ": "Y", "Χ": "X", "ο": "o",
|
|
33
|
+
"А": "A", "В": "B", "Е": "E", "К": "K", "М": "M", "Н": "H", "О": "O",
|
|
34
|
+
"Р": "P", "С": "C", "Т": "T", "Х": "X",
|
|
35
|
+
};
|
|
36
|
+
function excerptAround(text, idx, span = 40) {
|
|
37
|
+
return text.slice(Math.max(0, idx - span), idx + span).replace(/[\p{Cf}\p{Mn}]/gu, "�");
|
|
38
|
+
}
|
|
39
|
+
export function scanCharacters(text) {
|
|
40
|
+
const findings = [];
|
|
41
|
+
const decodedPayloads = [];
|
|
42
|
+
let cleaned = text;
|
|
43
|
+
// 0. Lone surrogates: never legitimate; replace so downstream re-encoding cannot mangle them
|
|
44
|
+
const lone = cleaned.match(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g);
|
|
45
|
+
if (lone)
|
|
46
|
+
cleaned = cleaned.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, "\uFFFD");
|
|
47
|
+
// 1. Unicode tag characters: the flagship vector. Decode what they spell.
|
|
48
|
+
const tags = [...text.matchAll(TAG_RANGE)];
|
|
49
|
+
if (tags.length) {
|
|
50
|
+
const spelled = tags
|
|
51
|
+
.map((m) => {
|
|
52
|
+
const cp = m[0].codePointAt(0);
|
|
53
|
+
return cp >= 0xe0020 && cp <= 0xe007e ? String.fromCodePoint(cp - 0xe0000) : "";
|
|
54
|
+
})
|
|
55
|
+
.join("");
|
|
56
|
+
findings.push({
|
|
57
|
+
pass: "character",
|
|
58
|
+
kind: "unicode.tag_characters",
|
|
59
|
+
confidence: 0.98,
|
|
60
|
+
severity: 0.95,
|
|
61
|
+
message: `${tags.length} Unicode Plane 14 tag characters spelling hidden ASCII`,
|
|
62
|
+
excerpt: spelled.slice(0, 200),
|
|
63
|
+
location: `offset ${tags[0].index}`,
|
|
64
|
+
action: "neutralized",
|
|
65
|
+
});
|
|
66
|
+
if (spelled.trim())
|
|
67
|
+
decodedPayloads.push(spelled);
|
|
68
|
+
cleaned = cleaned.replace(TAG_RANGE, "");
|
|
69
|
+
}
|
|
70
|
+
// 2. Bidirectional overrides and isolates: never legitimate here.
|
|
71
|
+
const bidi = [...cleaned.matchAll(BIDI_OVERRIDE_ISOLATE)];
|
|
72
|
+
if (bidi.length) {
|
|
73
|
+
findings.push({
|
|
74
|
+
pass: "character",
|
|
75
|
+
kind: "unicode.bidi_control",
|
|
76
|
+
confidence: 0.9,
|
|
77
|
+
severity: 0.7,
|
|
78
|
+
message: `${bidi.length} bidirectional override/isolate controls (Trojan Source class)`,
|
|
79
|
+
excerpt: excerptAround(cleaned, bidi[0].index),
|
|
80
|
+
location: `offset ${bidi[0].index}`,
|
|
81
|
+
action: "neutralized",
|
|
82
|
+
});
|
|
83
|
+
cleaned = cleaned.replace(BIDI_OVERRIDE_ISOLATE, "");
|
|
84
|
+
}
|
|
85
|
+
// bidi marks outside any right-to-left run are suspicious; inside one they are normal
|
|
86
|
+
const marks = [...cleaned.matchAll(BIDI_MARKS)];
|
|
87
|
+
const strayMarks = marks.filter((m) => !RTL_SCRIPT.test(cleaned.slice(Math.max(0, m.index - 24), m.index + 24)));
|
|
88
|
+
if (strayMarks.length >= 3) {
|
|
89
|
+
findings.push({
|
|
90
|
+
pass: "character",
|
|
91
|
+
kind: "unicode.bidi_control",
|
|
92
|
+
confidence: 0.6,
|
|
93
|
+
severity: 0.4,
|
|
94
|
+
message: `${strayMarks.length} bidirectional marks outside any right-to-left text`,
|
|
95
|
+
location: `offset ${strayMarks[0].index}`,
|
|
96
|
+
action: "neutralized",
|
|
97
|
+
});
|
|
98
|
+
for (const m of strayMarks.reverse())
|
|
99
|
+
cleaned = cleaned.slice(0, m.index) + cleaned.slice(m.index + 1);
|
|
100
|
+
}
|
|
101
|
+
// 3. Zero-width characters, script-aware.
|
|
102
|
+
const zw = [...cleaned.matchAll(ZERO_WIDTH)];
|
|
103
|
+
const joinerStray = [];
|
|
104
|
+
for (let i = 0; i < cleaned.length; i++) {
|
|
105
|
+
const ch = cleaned[i];
|
|
106
|
+
if (ch !== ZWJ && ch !== ZWNJ)
|
|
107
|
+
continue;
|
|
108
|
+
const prev = cleaned.slice(Math.max(0, i - 2), i);
|
|
109
|
+
const next = cleaned.slice(i + 1, i + 3);
|
|
110
|
+
const emojiCtx = ch === ZWJ && (EMOJI_ADJ.test(prev) || EMOJI_ADJ.test(next));
|
|
111
|
+
const complexScriptCtx = INDIC_OR_ARABIC.test(prev) || INDIC_OR_ARABIC.test(next);
|
|
112
|
+
if (!emojiCtx && !complexScriptCtx)
|
|
113
|
+
joinerStray.push(i);
|
|
114
|
+
}
|
|
115
|
+
const total = zw.length + joinerStray.length;
|
|
116
|
+
if (total > 0) {
|
|
117
|
+
// density matters: a couple is a copy-paste artifact; dozens is a channel
|
|
118
|
+
const density = total / Math.max(1, cleaned.length);
|
|
119
|
+
const confidence = Math.min(0.97, 0.35 + Math.min(total, 40) / 50 + density * 50);
|
|
120
|
+
findings.push({
|
|
121
|
+
pass: "character",
|
|
122
|
+
kind: "unicode.zero_width",
|
|
123
|
+
confidence,
|
|
124
|
+
severity: total >= 8 ? 0.8 : 0.3,
|
|
125
|
+
message: `${total} zero-width characters outside emoji or complex-script context`,
|
|
126
|
+
excerpt: excerptAround(cleaned, zw[0]?.index ?? joinerStray[0]),
|
|
127
|
+
location: `offset ${zw[0]?.index ?? joinerStray[0]}`,
|
|
128
|
+
action: "neutralized",
|
|
129
|
+
});
|
|
130
|
+
// decode binary-channel smuggling (ZWSP/ZWNJ as bits) when dense
|
|
131
|
+
if (total >= 16) {
|
|
132
|
+
const bits = [...cleaned].filter((c) => c === "" || c === "").map((c) => (c === "" ? "0" : "1")).join("");
|
|
133
|
+
if (bits.length >= 16) {
|
|
134
|
+
let out = "";
|
|
135
|
+
for (let i = 0; i + 8 <= bits.length; i += 8)
|
|
136
|
+
out += String.fromCharCode(parseInt(bits.slice(i, i + 8), 2));
|
|
137
|
+
if (/^[\x20-\x7E\s]+$/.test(out))
|
|
138
|
+
decodedPayloads.push(out);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// single pass removal: indices in joinerStray refer to the string before any removal
|
|
142
|
+
const drop = new Set(joinerStray);
|
|
143
|
+
let out = "";
|
|
144
|
+
for (let i = 0; i < cleaned.length; i++) {
|
|
145
|
+
const ch = cleaned[i];
|
|
146
|
+
if (drop.has(i))
|
|
147
|
+
continue;
|
|
148
|
+
if (/[\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180E\u200B\u2060-\u206F\u2800\u3164\uFEFF\uFFA0]/.test(ch))
|
|
149
|
+
continue;
|
|
150
|
+
out += ch;
|
|
151
|
+
}
|
|
152
|
+
cleaned = out;
|
|
153
|
+
}
|
|
154
|
+
// 3b. Whitespace channels (AcePaste FORMATTING_RE lineage). Unicode has a dozen spaces that render identically
|
|
155
|
+
// (en, em, thin, hair, figure, punctuation, narrow no-break, medium mathematical, ideographic) plus line and
|
|
156
|
+
// paragraph separators. One or two are typography; many, interleaved with ordinary spaces in Latin text, are a
|
|
157
|
+
// channel: the choice of space at each word gap carries a bit, which no human reader can see.
|
|
158
|
+
{
|
|
159
|
+
const ODD = /[\u2000-\u200A\u2028\u2029\u202F\u205F]/g;
|
|
160
|
+
// only look-alike spaces in a word gap count; typography is excluded: French spacing before ; : ! ? and inside « »,
|
|
161
|
+
// thousands separators between digits, and indentation (a space next to a line break or another space)
|
|
162
|
+
const odd = [...cleaned.matchAll(ODD)].filter((m) => {
|
|
163
|
+
const i = m.index, prev = cleaned[i - 1] ?? "", next = cleaned[i + 1] ?? "";
|
|
164
|
+
if (!prev || !next || /\s/.test(prev) || /\s/.test(next))
|
|
165
|
+
return false;
|
|
166
|
+
// a word-gap channel sits between letters; typeset maths, ranges and units put fine spaces next to symbols and digits
|
|
167
|
+
if (!/\p{L}/u.test(prev) || !/\p{L}/u.test(next))
|
|
168
|
+
return false;
|
|
169
|
+
if (/[;:!?»%€]/.test(next) || prev === "«")
|
|
170
|
+
return false;
|
|
171
|
+
if (/\d/.test(prev) && /\d/.test(next))
|
|
172
|
+
return false;
|
|
173
|
+
return true;
|
|
174
|
+
});
|
|
175
|
+
const cjk = /[\u3040-\u30FF\u3400-\u9FFF\uAC00-\uD7AF]/.test(cleaned);
|
|
176
|
+
const ideographic = cjk ? 0 : (cleaned.match(/\u3000/g) ?? []).length;
|
|
177
|
+
const n = odd.length + ideographic;
|
|
178
|
+
const kinds = new Set([...odd.map((m) => m[0]), ...(ideographic ? ["\u3000"] : [])]);
|
|
179
|
+
const asciiGaps = (cleaned.match(/(?<=\S) (?=\S)/g) ?? []).length;
|
|
180
|
+
if (n >= 8 && (kinds.size >= 2 || asciiGaps >= 4)) {
|
|
181
|
+
findings.push({
|
|
182
|
+
pass: "character", kind: "unicode.whitespace_channel",
|
|
183
|
+
confidence: Math.min(0.95, 0.5 + n / 60), severity: n >= 16 ? 0.75 : 0.45,
|
|
184
|
+
message: `${n} look-alike Unicode spaces (${kinds.size} kinds) mixed into ordinary word gaps: the choice of space can carry hidden bits`,
|
|
185
|
+
excerpt: excerptAround(cleaned, odd[0]?.index ?? 0), location: `offset ${odd[0]?.index ?? 0}`, action: "neutralized",
|
|
186
|
+
});
|
|
187
|
+
// decode: every word gap is a bit (ordinary space = 0, any look-alike = 1), read as 8-bit ASCII
|
|
188
|
+
const gaps = [...cleaned.matchAll(/(?<=\S)([ \u2000-\u200A\u202F\u205F\u3000])(?=\S)/g)].map((m) => (m[1] === " " ? "0" : "1")).join("");
|
|
189
|
+
for (const bits of [gaps, gaps.replace(/[01]/g, (b) => (b === "0" ? "1" : "0"))]) {
|
|
190
|
+
let out = "";
|
|
191
|
+
for (let i = 0; i + 8 <= bits.length; i += 8)
|
|
192
|
+
out += String.fromCharCode(parseInt(bits.slice(i, i + 8), 2));
|
|
193
|
+
if (out.length >= 4 && /^[\x20-\x7E]+$/.test(out) && /[a-z]{3}/i.test(out)) {
|
|
194
|
+
decodedPayloads.push(out);
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
cleaned = cleaned.replace(ODD, " ").replace(cjk ? /$^/ : /\u3000/g, " ");
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
// 4. Variation-selector smuggling: runs of selectors not attached to a base that takes them.
|
|
202
|
+
const vsRuns = [...cleaned.matchAll(/(?:[︀-️]|[\u{E0100}-\u{E01EF}]){4,}/gu)];
|
|
203
|
+
if (vsRuns.length) {
|
|
204
|
+
const n = vsRuns.reduce((a, m) => a + [...m[0]].length, 0);
|
|
205
|
+
const bytes = vsRuns.flatMap((m) => [...m[0]].map((c) => {
|
|
206
|
+
const cp = c.codePointAt(0);
|
|
207
|
+
return cp <= 0xfe0f ? cp - 0xfe00 : cp - 0xe0100 + 16;
|
|
208
|
+
}));
|
|
209
|
+
const decoded = bytes.filter((b) => b >= 0x20 && b < 0x7f).slice(0, 100000).map((b) => String.fromCharCode(b)).join("");
|
|
210
|
+
findings.push({
|
|
211
|
+
pass: "character",
|
|
212
|
+
kind: "unicode.variation_selector_smuggling",
|
|
213
|
+
confidence: 0.92,
|
|
214
|
+
severity: 0.9,
|
|
215
|
+
message: `${n} chained variation selectors encoding ${bytes.length} bytes`,
|
|
216
|
+
excerpt: decoded.slice(0, 120),
|
|
217
|
+
location: `offset ${vsRuns[0].index}`,
|
|
218
|
+
action: "neutralized",
|
|
219
|
+
});
|
|
220
|
+
if (decoded.trim())
|
|
221
|
+
decodedPayloads.push(decoded);
|
|
222
|
+
cleaned = cleaned.replace(/(?:[︀-️]|[\u{E0100}-\u{E01EF}]){4,}/gu, "");
|
|
223
|
+
}
|
|
224
|
+
// 5. Mixed-script words: Latin letters sharing a word with letters of another script. Legitimate text does this rarely
|
|
225
|
+
// (brand names, transliteration glosses); injection spelled with look-alikes does it in every keyword.
|
|
226
|
+
const words = cleaned.match(/[\p{L}\p{M}]{3,}/gu) ?? [];
|
|
227
|
+
let confusableWords = 0;
|
|
228
|
+
let sample = "";
|
|
229
|
+
for (const w of words) {
|
|
230
|
+
const hasLatin = /\p{Script=Latin}/u.test(w);
|
|
231
|
+
if (!hasLatin)
|
|
232
|
+
continue;
|
|
233
|
+
const hasOther = /[^\p{Script=Latin}\p{Script=Common}\p{Script=Inherited}]/u.test(w);
|
|
234
|
+
if (hasOther) {
|
|
235
|
+
confusableWords++;
|
|
236
|
+
if (!sample)
|
|
237
|
+
sample = w;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (confusableWords > 0) {
|
|
241
|
+
findings.push({
|
|
242
|
+
pass: "character",
|
|
243
|
+
kind: "unicode.mixed_script_confusable",
|
|
244
|
+
confidence: Math.min(0.9, 0.4 + confusableWords * 0.1),
|
|
245
|
+
severity: 0.5,
|
|
246
|
+
message: `${confusableWords} words mixing Latin with look-alike Cyrillic/Greek letters`,
|
|
247
|
+
excerpt: sample,
|
|
248
|
+
action: "neutralized",
|
|
249
|
+
});
|
|
250
|
+
cleaned = cleaned.replace(/[\p{L}\p{M}]{3,}/gu, (w) => /\p{Script=Latin}/u.test(w) ? [...w].map((c) => CONFUSABLE[c] ?? c).join("") : w);
|
|
251
|
+
}
|
|
252
|
+
// 5b. Full-width and mathematical alphanumeric look-alikes: ignore, 𝗶𝗴𝗻𝗼𝗿𝗲. Humans read them fine; tokenizers and keyword filters do not.
|
|
253
|
+
const fwRuns = cleaned.match(/[\uFF21-\uFF3A\uFF41-\uFF5A]{3,}|[\u{1D400}-\u{1D7FF}]{3,}|[\u24B6-\u24E9\u{1F130}-\u{1F189}]{3,}/gu) ?? [];
|
|
254
|
+
if (fwRuns.length) {
|
|
255
|
+
const folded = (t) => t.normalize("NFKC");
|
|
256
|
+
findings.push({
|
|
257
|
+
pass: "character", kind: "unicode.fullwidth_or_math_lookalike",
|
|
258
|
+
confidence: Math.min(0.85, 0.35 + fwRuns.length * 0.1), severity: 0.5,
|
|
259
|
+
message: `${fwRuns.length} runs of full-width, mathematical or enclosed letters standing in for Latin`,
|
|
260
|
+
excerpt: fwRuns.slice(0, 3).map(folded).join(" "), action: "neutralized",
|
|
261
|
+
});
|
|
262
|
+
cleaned = cleaned.replace(/[\uFF21-\uFF3A\uFF41-\uFF5A]+|[\u{1D400}-\u{1D7FF}]+|[\u24B6-\u24E9\u{1F130}-\u{1F189}]+/gu, (r) => r.normalize("NFKC"));
|
|
263
|
+
}
|
|
264
|
+
// 5c. Combining-mark floods (zalgo): more than 4 combining marks on one base is never legitimate text
|
|
265
|
+
const floods = cleaned.match(/[\p{M}]{5,}/gu) ?? [];
|
|
266
|
+
if (floods.length) {
|
|
267
|
+
findings.push({
|
|
268
|
+
pass: "character", kind: "unicode.combining_flood",
|
|
269
|
+
confidence: Math.min(0.9, 0.5 + floods.length * 0.1), severity: 0.4,
|
|
270
|
+
message: `${floods.length} combining-mark floods (${Math.max(...floods.map((f) => [...f].length))} marks on one base)`,
|
|
271
|
+
action: "neutralized",
|
|
272
|
+
});
|
|
273
|
+
cleaned = cleaned.replace(/([^\p{M}])[\p{M}]{5,}/gu, "$1");
|
|
274
|
+
}
|
|
275
|
+
// 6. Encoded payloads: base64 (incl. short and whitespace-split), base64url, base32, hex, char codes, binary, \u escapes.
|
|
276
|
+
const decoded = decodePayloads(cleaned);
|
|
277
|
+
for (const d of decoded)
|
|
278
|
+
decodedPayloads.push(d.text);
|
|
279
|
+
if (decoded.length) {
|
|
280
|
+
findings.push({
|
|
281
|
+
pass: "character",
|
|
282
|
+
kind: "unicode.encoded_payload",
|
|
283
|
+
confidence: 0.5,
|
|
284
|
+
severity: 0.4,
|
|
285
|
+
message: `${decoded.length} encoded blobs (${[...new Set(decoded.map((d) => d.scheme))].join(", ")}) decoded to readable text; rescanned by meaning pass`,
|
|
286
|
+
excerpt: decoded[0].text.slice(0, 120),
|
|
287
|
+
action: "none",
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
return { cleaned, findings, decodedPayloads };
|
|
291
|
+
}
|
package/dist/src/cli.js
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* ingress <url> gate a URL
|
|
4
|
+
* ingress --text < file gate text/HTML/markdown/JSON on stdin
|
|
5
|
+
* ingress --json ... emit the full result as JSON
|
|
6
|
+
* ingress --override ... deliver a suspect page the human asked for
|
|
7
|
+
* ingress --sandbox=always|escalate|off
|
|
8
|
+
* ingress --policy=file.json
|
|
9
|
+
* ingress --verify < receipt.json verify a receipt's digest and signature (INGRESS_VERIFY_PUBLIC_KEY=file.pem for ECDSA, INGRESS_SIGNING_KEY for HMAC)
|
|
10
|
+
* ingress --keygen print a new ECDSA P-256 key pair for receipt signing
|
|
11
|
+
*/
|
|
12
|
+
import { readFileSync } from "node:fs";
|
|
13
|
+
import { IngressEngine } from "./engine.js";
|
|
14
|
+
import { HONEST_USER_AGENT } from "./acquire.js";
|
|
15
|
+
import { summarize, cloakingConfigFromEnv } from "./summary.js";
|
|
16
|
+
import { loadPolicyFile } from "./policyfile.js";
|
|
17
|
+
const args = process.argv.slice(2);
|
|
18
|
+
const flag = (n) => args.includes(`--${n}`);
|
|
19
|
+
const opt = (n) => args.find((a) => a.startsWith(`--${n}=`))?.split("=")[1];
|
|
20
|
+
const url = args.find((a) => /^https?:\/\//.test(a));
|
|
21
|
+
async function main() {
|
|
22
|
+
const chunksIn = async () => { const c = []; for await (const x of process.stdin)
|
|
23
|
+
c.push(x); return Buffer.concat(c).toString("utf8"); };
|
|
24
|
+
if (flag("keygen")) {
|
|
25
|
+
const k = IngressEngine.generateSigningKeys();
|
|
26
|
+
console.log(`# keyId ${k.keyId}\n# private key: keep secret; set INGRESS_SIGNING_PRIVATE_KEY_FILE to its path\n${k.privateKeyPem}\n# public key: publish; anyone can verify receipts with it\n${k.publicKeyPem}`);
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (flag("verify")) {
|
|
30
|
+
const r = JSON.parse(await chunksIn());
|
|
31
|
+
const pub = process.env.INGRESS_VERIFY_PUBLIC_KEY;
|
|
32
|
+
const key = r.signatureAlg === "ecdsa-p256-sha256" ? (pub && !pub.includes("BEGIN") ? readFileSync(pub, "utf8") : pub) : process.env.INGRESS_SIGNING_KEY;
|
|
33
|
+
console.log(JSON.stringify(IngressEngine.verifyReceipt(r, key), null, 2));
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const cfg = opt("policy") ? loadPolicyFile(opt("policy")) : {};
|
|
37
|
+
Object.assign(cfg, cloakingConfigFromEnv(process.env, HONEST_USER_AGENT));
|
|
38
|
+
if (flag("no-cloaking"))
|
|
39
|
+
cfg.cloakingCheck = false;
|
|
40
|
+
const hf = opt("human-fetch");
|
|
41
|
+
if (hf === "browser" || hf === "http")
|
|
42
|
+
cfg.humanFetch = hf;
|
|
43
|
+
const sb = opt("sandbox");
|
|
44
|
+
if (sb === "always" || sb === "escalate" || sb === "off")
|
|
45
|
+
cfg.sandbox = sb;
|
|
46
|
+
if (process.env.INGRESS_SIGNING_KEY)
|
|
47
|
+
cfg.signingKey = process.env.INGRESS_SIGNING_KEY;
|
|
48
|
+
const pk = process.env.INGRESS_SIGNING_PRIVATE_KEY_FILE ? readFileSync(process.env.INGRESS_SIGNING_PRIVATE_KEY_FILE, "utf8") : process.env.INGRESS_SIGNING_PRIVATE_KEY;
|
|
49
|
+
if (pk)
|
|
50
|
+
cfg.signingPrivateKey = pk;
|
|
51
|
+
const engine = new IngressEngine(cfg);
|
|
52
|
+
const override = flag("override");
|
|
53
|
+
let result;
|
|
54
|
+
if (flag("text"))
|
|
55
|
+
result = await engine.scan(await chunksIn(), { override });
|
|
56
|
+
else if (url)
|
|
57
|
+
result = await engine.fetch(url, { override });
|
|
58
|
+
else {
|
|
59
|
+
console.error("usage: ingress <url> | ingress --text < file [--json | --summary] [--no-cloaking] [--human-fetch=http|browser] [--override] [--sandbox=always|escalate|off] [--policy=file.json] | ingress --verify < receipt.json");
|
|
60
|
+
process.exit(2);
|
|
61
|
+
}
|
|
62
|
+
if (flag("json")) {
|
|
63
|
+
console.log(JSON.stringify(result, null, 2));
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
if (flag("summary")) {
|
|
67
|
+
console.log(JSON.stringify(summarize(result)));
|
|
68
|
+
if (result.tier === "unreachable")
|
|
69
|
+
process.exitCode = 1;
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
console.log(result.envelope);
|
|
73
|
+
console.error(`\n[timings] ${Object.entries(result.timings).map(([k, v]) => `${k}=${v.toFixed(1)}ms`).join(" ")}`);
|
|
74
|
+
}
|
|
75
|
+
main().catch((e) => { console.error(e); process.exit(1); });
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Colour parsing and contrast, for same-colour hiding (white on white, black
|
|
3
|
+
* on black, #fefefe on #fff, "snow" on "white"). Contrast is the WCAG 2
|
|
4
|
+
* ratio: 1 means identical, 21 is black on white. Text at or below the
|
|
5
|
+
* threshold is unreadable to a sighted human and is labeled hidden.
|
|
6
|
+
*/
|
|
7
|
+
export interface RGBA {
|
|
8
|
+
r: number;
|
|
9
|
+
g: number;
|
|
10
|
+
b: number;
|
|
11
|
+
a: number;
|
|
12
|
+
}
|
|
13
|
+
/** at or below this ratio text is treated as invisible against its background (WCAG body-text minimum is 4.5) */
|
|
14
|
+
export declare const INVISIBLE_CONTRAST = 1.25;
|
|
15
|
+
/** Parse a CSS colour value. Returns null for values that cannot be resolved statically (var(), currentcolor, gradients). */
|
|
16
|
+
export declare function parseColor(input: string): RGBA | null;
|
|
17
|
+
/** Composite `top` over an opaque `bottom`. */
|
|
18
|
+
export declare function blend(top: RGBA, bottom: RGBA): RGBA;
|
|
19
|
+
/** WCAG contrast ratio of text colour `fg` (alpha composited) over opaque `bg`. */
|
|
20
|
+
export declare function contrastRatio(fg: RGBA, bg: RGBA): number;
|
|
21
|
+
/** Pull colour declarations out of a style string. `bg` is null when the background is an image or gradient (unknown). */
|
|
22
|
+
export declare function colorsFromStyle(style: string): {
|
|
23
|
+
fg?: RGBA | null;
|
|
24
|
+
bg?: RGBA | null;
|
|
25
|
+
fill?: RGBA | null;
|
|
26
|
+
};
|