@acepaste/ingress 0.0.0-stage → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +105 -0
- package/README.md +15 -2
- package/dist/src/acepaste-alphabet.d.ts +26 -0
- package/dist/src/acepaste-alphabet.js +47 -0
- package/dist/src/acquire.d.ts +24 -0
- package/dist/src/acquire.js +146 -0
- package/dist/src/adapters.d.ts +30 -0
- package/dist/src/adapters.js +160 -0
- package/dist/src/character.d.ts +22 -0
- package/dist/src/character.js +291 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +63 -0
- package/dist/src/color.d.ts +26 -0
- package/dist/src/color.js +111 -0
- package/dist/src/engine.d.ts +108 -0
- package/dist/src/engine.js +612 -0
- package/dist/src/evidence.d.ts +21 -0
- package/dist/src/evidence.js +77 -0
- package/dist/src/index.d.ts +26 -0
- package/dist/src/index.js +19 -0
- package/dist/src/meaning.d.ts +28 -0
- package/dist/src/meaning.js +226 -0
- package/dist/src/netpolicy.d.ts +34 -0
- package/dist/src/netpolicy.js +122 -0
- package/dist/src/normalize.d.ts +34 -0
- package/dist/src/normalize.js +204 -0
- package/dist/src/policyfile.d.ts +2 -0
- package/dist/src/policyfile.js +50 -0
- package/dist/src/sandbox.d.ts +45 -0
- package/dist/src/sandbox.js +231 -0
- package/dist/src/script.d.ts +23 -0
- package/dist/src/script.js +220 -0
- package/dist/src/secrets.d.ts +24 -0
- package/dist/src/secrets.js +192 -0
- package/dist/src/supplychain.d.ts +24 -0
- package/dist/src/supplychain.js +182 -0
- package/dist/src/taint.d.ts +37 -0
- package/dist/src/taint.js +39 -0
- package/dist/src/types.d.ts +176 -0
- package/dist/src/types.js +25 -0
- package/dist/src/visibility.d.ts +41 -0
- package/dist/src/visibility.js +424 -0
- package/package.json +64 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# Functional Source License, Version 1.1, ALv2 Future License
|
|
2
|
+
|
|
3
|
+
## Abbreviation
|
|
4
|
+
|
|
5
|
+
FSL-1.1-ALv2
|
|
6
|
+
|
|
7
|
+
## Notice
|
|
8
|
+
|
|
9
|
+
Copyright 2026 ThinkWell Labs LLC
|
|
10
|
+
|
|
11
|
+
## Terms and Conditions
|
|
12
|
+
|
|
13
|
+
### Licensor ("We")
|
|
14
|
+
|
|
15
|
+
The party offering the Software under these Terms and Conditions.
|
|
16
|
+
|
|
17
|
+
### The Software
|
|
18
|
+
|
|
19
|
+
The "Software" is each version of the software that we make available under
|
|
20
|
+
these Terms and Conditions, as indicated by our inclusion of these Terms and
|
|
21
|
+
Conditions with the Software.
|
|
22
|
+
|
|
23
|
+
### License Grant
|
|
24
|
+
|
|
25
|
+
Subject to your compliance with this License Grant and the Patents,
|
|
26
|
+
Redistribution and Trademark clauses below, we hereby grant you the right to
|
|
27
|
+
use, copy, modify, create derivative works, publicly perform, publicly display
|
|
28
|
+
and redistribute the Software for any Permitted Purpose identified below.
|
|
29
|
+
|
|
30
|
+
### Permitted Purpose
|
|
31
|
+
|
|
32
|
+
A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
|
|
33
|
+
means making the Software available to others in a commercial product or
|
|
34
|
+
service that:
|
|
35
|
+
|
|
36
|
+
1. substitutes for the Software;
|
|
37
|
+
|
|
38
|
+
2. substitutes for any other product or service we offer using the Software
|
|
39
|
+
that exists as of the date we make the Software available; or
|
|
40
|
+
|
|
41
|
+
3. offers the same or substantially similar functionality as the Software.
|
|
42
|
+
|
|
43
|
+
Permitted Purposes specifically include using the Software:
|
|
44
|
+
|
|
45
|
+
1. for your internal use and access;
|
|
46
|
+
|
|
47
|
+
2. for non-commercial education;
|
|
48
|
+
|
|
49
|
+
3. for non-commercial research; and
|
|
50
|
+
|
|
51
|
+
4. in connection with professional services that you provide to a licensee
|
|
52
|
+
using the Software in accordance with these Terms and Conditions.
|
|
53
|
+
|
|
54
|
+
### Patents
|
|
55
|
+
|
|
56
|
+
To the extent your use for a Permitted Purpose would necessarily infringe our
|
|
57
|
+
patents, the license grant above includes a license under our patents. If you
|
|
58
|
+
make a claim against any party that the Software infringes or contributes to
|
|
59
|
+
the infringement of any patent, then your patent license to the Software ends
|
|
60
|
+
immediately.
|
|
61
|
+
|
|
62
|
+
### Redistribution
|
|
63
|
+
|
|
64
|
+
The Terms and Conditions apply to all copies, modifications and derivatives of
|
|
65
|
+
the Software.
|
|
66
|
+
|
|
67
|
+
If you redistribute any copies, modifications or derivatives of the Software,
|
|
68
|
+
you must include a copy of or a link to these Terms and Conditions and not
|
|
69
|
+
remove any copyright notices provided in or with the Software.
|
|
70
|
+
|
|
71
|
+
### Disclaimer
|
|
72
|
+
|
|
73
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
|
|
74
|
+
IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
|
|
75
|
+
PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
|
|
76
|
+
|
|
77
|
+
IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
|
|
78
|
+
SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
|
|
79
|
+
EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
|
|
80
|
+
|
|
81
|
+
### Trademarks
|
|
82
|
+
|
|
83
|
+
Except for displaying the License Details and identifying us as the origin of
|
|
84
|
+
the Software, you have no right under these Terms and Conditions to use our
|
|
85
|
+
trademarks, trade names, service marks or product names.
|
|
86
|
+
|
|
87
|
+
## Grant of Future License
|
|
88
|
+
|
|
89
|
+
We hereby irrevocably grant you an additional license to use the Software under
|
|
90
|
+
the Apache License, Version 2.0 that is effective on the second anniversary of
|
|
91
|
+
the date we make the Software available. On or after that date, you may use the
|
|
92
|
+
Software under the Apache License, Version 2.0, in which case the following
|
|
93
|
+
will apply:
|
|
94
|
+
|
|
95
|
+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
96
|
+
this file except in compliance with the License.
|
|
97
|
+
|
|
98
|
+
You may obtain a copy of the License at
|
|
99
|
+
|
|
100
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
101
|
+
|
|
102
|
+
Unless required by applicable law or agreed to in writing, software distributed
|
|
103
|
+
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
104
|
+
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
105
|
+
specific language governing permissions and limitations under the License.
|
package/README.md
CHANGED
|
@@ -1,3 +1,16 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @acepaste/ingress
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The AcePaste Ingress engine: a security gate between internet content and AI ingestion. Anything a model would receive that a human looking at the same source would not see is suspect by default.
|
|
4
|
+
|
|
5
|
+
```ts
|
|
6
|
+
import { IngressEngine } from "@acepaste/ingress";
|
|
7
|
+
const ingress = new IngressEngine({ signingKey: process.env.INGRESS_SIGNING_KEY });
|
|
8
|
+
const r = await ingress.fetch("https://example.com/doc"); // or ingress.scan(text)
|
|
9
|
+
model.push(r.envelope); // clean text in a provenance envelope, or a refusal with the reason
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Verdicts: `clean` (delivered with a signed receipt), `suspect` (refused, human-overridable), `hostile` (refused, not overridable). Passes: acquire, policy, visibility, character (the production AcePaste invisible alphabet), script and supply-chain (underlight lineage), sandbox (optional headless Chromium), meaning, plugins.
|
|
13
|
+
|
|
14
|
+
CLI: `npx @acepaste/ingress https://example.com` or `ingress --text < page.html`.
|
|
15
|
+
|
|
16
|
+
Full documentation: https://github.com/sugarcypher/acepasteINGRESS
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The AcePaste invisible-character alphabet — ported verbatim from the
|
|
3
|
+
* production tool (sugarcypher/ACE-ASTE-PRO, AcePasteBrowserExtension/
|
|
4
|
+
* cleaner.js, INVISIBLE_RE and INVISIBLE_SUPPL_RE). This is the authoritative
|
|
5
|
+
* definition of "hostile invisible character" shared by the web tool, the
|
|
6
|
+
* Chrome extension, and this gate, so a verdict here matches what AcePaste
|
|
7
|
+
* itself would strip. Keep it in sync with cleaner.js; do not fork it.
|
|
8
|
+
*
|
|
9
|
+
* Provenance: these ranges are AcePaste's, not Claude's reconstruction.
|
|
10
|
+
* The gate's script-aware exceptions (emoji/Indic/Arabic context for joiners,
|
|
11
|
+
* bidi marks inside right-to-left runs) live in character.ts and are applied
|
|
12
|
+
* on top of this set.
|
|
13
|
+
*/
|
|
14
|
+
/** Basic-plane hostile invisibles (cleaner.js INVISIBLE_RE). */
|
|
15
|
+
export declare const ACEPASTE_INVISIBLE_SOURCE: string;
|
|
16
|
+
/** Supplementary-plane hostile invisibles (cleaner.js INVISIBLE_SUPPL_RE): tag chars + variation selectors supplement. */
|
|
17
|
+
export declare const ACEPASTE_INVISIBLE_SUPPL_SOURCE = "[\\u{E0000}-\\u{E007F}\\u{E0100}-\\u{E01EF}]";
|
|
18
|
+
export declare const acepasteInvisible: () => RegExp;
|
|
19
|
+
export declare const acepasteInvisibleSuppl: () => RegExp;
|
|
20
|
+
/**
|
|
21
|
+
* Count hostile invisibles the way the AcePaste extension's page scanner does
|
|
22
|
+
* (scanner.js countHostile): the sum of matches of both sets. The gate uses
|
|
23
|
+
* this only for the receipt's parity note; its own findings come from the
|
|
24
|
+
* script-aware passes in character.ts.
|
|
25
|
+
*/
|
|
26
|
+
export declare function countHostileInvisibles(str: string): number;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The AcePaste invisible-character alphabet — ported verbatim from the
|
|
3
|
+
* production tool (sugarcypher/ACE-ASTE-PRO, AcePasteBrowserExtension/
|
|
4
|
+
* cleaner.js, INVISIBLE_RE and INVISIBLE_SUPPL_RE). This is the authoritative
|
|
5
|
+
* definition of "hostile invisible character" shared by the web tool, the
|
|
6
|
+
* Chrome extension, and this gate, so a verdict here matches what AcePaste
|
|
7
|
+
* itself would strip. Keep it in sync with cleaner.js; do not fork it.
|
|
8
|
+
*
|
|
9
|
+
* Provenance: these ranges are AcePaste's, not Claude's reconstruction.
|
|
10
|
+
* The gate's script-aware exceptions (emoji/Indic/Arabic context for joiners,
|
|
11
|
+
* bidi marks inside right-to-left runs) live in character.ts and are applied
|
|
12
|
+
* on top of this set.
|
|
13
|
+
*/
|
|
14
|
+
/** Basic-plane hostile invisibles (cleaner.js INVISIBLE_RE). */
|
|
15
|
+
export const ACEPASTE_INVISIBLE_SOURCE = "[" +
|
|
16
|
+
"\\u00AD" + // SOFT HYPHEN
|
|
17
|
+
"\\u034F" + // COMBINING GRAPHEME JOINER
|
|
18
|
+
"\\u061C" + // ARABIC LETTER MARK
|
|
19
|
+
"\\u115F\\u1160" + // HANGUL FILLERS
|
|
20
|
+
"\\u17B4\\u17B5" + // KHMER INHERENT VOWELS
|
|
21
|
+
"\\u180E" + // MONGOLIAN VOWEL SEPARATOR
|
|
22
|
+
"\\u200B-\\u200F" + // ZERO WIDTH SPACE/NJ/J + LRM/RLM
|
|
23
|
+
"\\u202A-\\u202E" + // BIDI EMBEDDINGS / OVERRIDES (Trojan Source)
|
|
24
|
+
"\\u2060-\\u206F" + // WORD JOINER + general format chars
|
|
25
|
+
"\\u2800" + // BRAILLE PATTERN BLANK (looks like a space)
|
|
26
|
+
"\\u3164" + // HANGUL FILLER
|
|
27
|
+
"\\uFE00-\\uFE0F" + // VARIATION SELECTORS 1-16
|
|
28
|
+
"\\uFEFF" + // ZERO WIDTH NO-BREAK SPACE / BOM
|
|
29
|
+
"\\uFFA0" + // HALFWIDTH HANGUL FILLER
|
|
30
|
+
"]";
|
|
31
|
+
/** Supplementary-plane hostile invisibles (cleaner.js INVISIBLE_SUPPL_RE): tag chars + variation selectors supplement. */
|
|
32
|
+
export const ACEPASTE_INVISIBLE_SUPPL_SOURCE = "[\\u{E0000}-\\u{E007F}\\u{E0100}-\\u{E01EF}]";
|
|
33
|
+
export const acepasteInvisible = () => new RegExp(ACEPASTE_INVISIBLE_SOURCE, "g");
|
|
34
|
+
export const acepasteInvisibleSuppl = () => new RegExp(ACEPASTE_INVISIBLE_SUPPL_SOURCE, "gu");
|
|
35
|
+
/**
|
|
36
|
+
* Count hostile invisibles the way the AcePaste extension's page scanner does
|
|
37
|
+
* (scanner.js countHostile): the sum of matches of both sets. The gate uses
|
|
38
|
+
* this only for the receipt's parity note; its own findings come from the
|
|
39
|
+
* script-aware passes in character.ts.
|
|
40
|
+
*/
|
|
41
|
+
export function countHostileInvisibles(str) {
|
|
42
|
+
if (!str)
|
|
43
|
+
return 0;
|
|
44
|
+
const a = str.match(acepasteInvisible());
|
|
45
|
+
const b = str.match(acepasteInvisibleSuppl());
|
|
46
|
+
return (a ? a.length : 0) + (b ? b.length : 0);
|
|
47
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Acquire pass.
|
|
3
|
+
*
|
|
4
|
+
* Fetches a URL under one of two fingerprints. The "agent" fingerprint looks
|
|
5
|
+
* like what an automated fetch tool sends; the "human" fingerprint looks like
|
|
6
|
+
* a desktop browser. The cloaking pass compares the two.
|
|
7
|
+
*
|
|
8
|
+
* Every hop is checked against the network policy before connecting, and
|
|
9
|
+
* the connection is pinned to the addresses that were checked (DNS rebinding
|
|
10
|
+
* defense). Redirects are followed manually so each hop is checked.
|
|
11
|
+
*/
|
|
12
|
+
import { type Dispatcher } from "undici";
|
|
13
|
+
import { NetPolicyError, type NetPolicy } from "./netpolicy.js";
|
|
14
|
+
import type { AcquireResult, IngressConfig } from "./types.js";
|
|
15
|
+
export declare const MAX_REDIRECTS = 5;
|
|
16
|
+
export declare class AcquireError extends Error {
|
|
17
|
+
readonly code: "unsupported_scheme" | "too_large" | "timeout" | "network" | "binary" | "too_many_redirects" | "policy";
|
|
18
|
+
readonly policyCode?: NetPolicyError["code"] | undefined;
|
|
19
|
+
constructor(message: string, code: "unsupported_scheme" | "too_large" | "timeout" | "network" | "binary" | "too_many_redirects" | "policy", policyCode?: NetPolicyError["code"] | undefined);
|
|
20
|
+
}
|
|
21
|
+
export type FetchLike = (url: string, init: RequestInit & {
|
|
22
|
+
dispatcher?: Dispatcher;
|
|
23
|
+
}) => Promise<Response>;
|
|
24
|
+
export declare function acquire(url: string, fingerprint: "human" | "agent", config: IngressConfig, policy: NetPolicy, fetchImpl?: FetchLike, resolver?: (host: string) => Promise<string[]>): Promise<AcquireResult>;
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Acquire pass.
|
|
3
|
+
*
|
|
4
|
+
* Fetches a URL under one of two fingerprints. The "agent" fingerprint looks
|
|
5
|
+
* like what an automated fetch tool sends; the "human" fingerprint looks like
|
|
6
|
+
* a desktop browser. The cloaking pass compares the two.
|
|
7
|
+
*
|
|
8
|
+
* Every hop is checked against the network policy before connecting, and
|
|
9
|
+
* the connection is pinned to the addresses that were checked (DNS rebinding
|
|
10
|
+
* defense). Redirects are followed manually so each hop is checked.
|
|
11
|
+
*/
|
|
12
|
+
import { Agent, fetch as undiciFetch } from "undici";
|
|
13
|
+
import { checkTarget, NetPolicyError } from "./netpolicy.js";
|
|
14
|
+
const HUMAN_HEADERS = {
|
|
15
|
+
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
|
16
|
+
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
17
|
+
"accept-language": "en-US,en;q=0.9",
|
|
18
|
+
"sec-fetch-dest": "document",
|
|
19
|
+
"sec-fetch-mode": "navigate",
|
|
20
|
+
"sec-fetch-site": "none",
|
|
21
|
+
"sec-fetch-user": "?1",
|
|
22
|
+
"upgrade-insecure-requests": "1",
|
|
23
|
+
};
|
|
24
|
+
const AGENT_HEADERS = {
|
|
25
|
+
"user-agent": "AcePasteIngress/0.7 (+https://acepaste.xyz/ingress)",
|
|
26
|
+
accept: "text/html,application/xhtml+xml,text/markdown,text/plain;q=0.9,application/json;q=0.8,*/*;q=0.5",
|
|
27
|
+
};
|
|
28
|
+
export const MAX_REDIRECTS = 5;
|
|
29
|
+
export class AcquireError extends Error {
|
|
30
|
+
code;
|
|
31
|
+
policyCode;
|
|
32
|
+
constructor(message, code, policyCode) {
|
|
33
|
+
super(message);
|
|
34
|
+
this.code = code;
|
|
35
|
+
this.policyCode = policyCode;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/** undici fetch pinned to pre-checked addresses */
|
|
39
|
+
function pinnedFetch(addrs) {
|
|
40
|
+
const agent = new Agent({
|
|
41
|
+
connect: {
|
|
42
|
+
lookup: (_host, _opts, cb) => cb(null, addrs.map((address) => ({ address, family: address.includes(":") ? 6 : 4 }))),
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
return (url, init) => undiciFetch(url, { ...init, dispatcher: agent });
|
|
46
|
+
}
|
|
47
|
+
export async function acquire(url, fingerprint, config, policy, fetchImpl, resolver) {
|
|
48
|
+
let current = new URL(url);
|
|
49
|
+
if (current.protocol !== "http:" && current.protocol !== "https:") {
|
|
50
|
+
throw new AcquireError(`unsupported scheme ${current.protocol}`, "unsupported_scheme");
|
|
51
|
+
}
|
|
52
|
+
const chain = [];
|
|
53
|
+
const headersIn = fingerprint === "human" ? HUMAN_HEADERS : AGENT_HEADERS;
|
|
54
|
+
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
55
|
+
let addrs;
|
|
56
|
+
try {
|
|
57
|
+
addrs = await checkTarget(current, policy, resolver);
|
|
58
|
+
}
|
|
59
|
+
catch (e) {
|
|
60
|
+
if (e instanceof NetPolicyError)
|
|
61
|
+
throw new AcquireError(e.message, "policy", e.code);
|
|
62
|
+
throw e;
|
|
63
|
+
}
|
|
64
|
+
const doFetch = fetchImpl ?? pinnedFetch(addrs);
|
|
65
|
+
const controller = new AbortController();
|
|
66
|
+
const timer = setTimeout(() => controller.abort(), config.fetchTimeoutMs);
|
|
67
|
+
let res;
|
|
68
|
+
try {
|
|
69
|
+
res = await doFetch(current.toString(), { headers: headersIn, redirect: "manual", signal: controller.signal });
|
|
70
|
+
}
|
|
71
|
+
catch (e) {
|
|
72
|
+
clearTimeout(timer);
|
|
73
|
+
const aborted = e.name === "AbortError";
|
|
74
|
+
throw new AcquireError(aborted ? "fetch timed out" : `network error: ${e.message}`, aborted ? "timeout" : "network");
|
|
75
|
+
}
|
|
76
|
+
clearTimeout(timer);
|
|
77
|
+
if (res.status >= 300 && res.status < 400 && res.headers.get("location")) {
|
|
78
|
+
chain.push(current.toString());
|
|
79
|
+
let next;
|
|
80
|
+
try {
|
|
81
|
+
next = new URL(res.headers.get("location"), current);
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
throw new AcquireError("bad redirect location", "network");
|
|
85
|
+
}
|
|
86
|
+
if (next.protocol !== "http:" && next.protocol !== "https:")
|
|
87
|
+
throw new AcquireError(`redirect to unsupported scheme ${next.protocol}`, "unsupported_scheme");
|
|
88
|
+
try {
|
|
89
|
+
await res.body?.cancel();
|
|
90
|
+
}
|
|
91
|
+
catch { /* ignore */ }
|
|
92
|
+
current = next;
|
|
93
|
+
if (hop === MAX_REDIRECTS)
|
|
94
|
+
throw new AcquireError(`more than ${MAX_REDIRECTS} redirects`, "too_many_redirects");
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
const contentType = res.headers.get("content-type") ?? "";
|
|
98
|
+
const lengthHeader = Number(res.headers.get("content-length") ?? "0");
|
|
99
|
+
if (lengthHeader > config.maxBodyBytes)
|
|
100
|
+
throw new AcquireError(`body ${lengthHeader} bytes exceeds limit`, "too_large");
|
|
101
|
+
if (contentType && !/^(text\/|application\/(xhtml\+xml|xml|json|ld\+json|javascript|x-javascript|markdown))/i.test(contentType)) {
|
|
102
|
+
throw new AcquireError(`binary or unsupported content-type ${contentType}`, "binary");
|
|
103
|
+
}
|
|
104
|
+
// stream with a hard cap so a lying content-length cannot exhaust memory
|
|
105
|
+
const reader = res.body?.getReader();
|
|
106
|
+
const chunks = [];
|
|
107
|
+
let total = 0;
|
|
108
|
+
if (reader) {
|
|
109
|
+
for (;;) {
|
|
110
|
+
const { done, value } = await reader.read();
|
|
111
|
+
if (done)
|
|
112
|
+
break;
|
|
113
|
+
total += value.byteLength;
|
|
114
|
+
if (total > config.maxBodyBytes) {
|
|
115
|
+
try {
|
|
116
|
+
await reader.cancel();
|
|
117
|
+
}
|
|
118
|
+
catch { /* ignore */ }
|
|
119
|
+
throw new AcquireError(`body exceeds ${config.maxBodyBytes} bytes`, "too_large");
|
|
120
|
+
}
|
|
121
|
+
chunks.push(value);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
const buf = new Uint8Array(total);
|
|
125
|
+
let off = 0;
|
|
126
|
+
for (const c of chunks) {
|
|
127
|
+
buf.set(c, off);
|
|
128
|
+
off += c.byteLength;
|
|
129
|
+
}
|
|
130
|
+
const body = new TextDecoder("utf-8", { fatal: false }).decode(buf);
|
|
131
|
+
const headers = {};
|
|
132
|
+
res.headers.forEach((v, k) => (headers[k] = v));
|
|
133
|
+
return {
|
|
134
|
+
url,
|
|
135
|
+
finalUrl: current.toString(),
|
|
136
|
+
redirectChain: chain,
|
|
137
|
+
status: res.status,
|
|
138
|
+
headers,
|
|
139
|
+
contentType,
|
|
140
|
+
body,
|
|
141
|
+
fingerprint,
|
|
142
|
+
fetchedAt: new Date().toISOString(),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
throw new AcquireError("unreachable", "network");
|
|
146
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content adapters: turn any supported content kind into the common shape
|
|
3
|
+
* the passes consume (segments with visibility, scripts, decoded blobs).
|
|
4
|
+
*
|
|
5
|
+
* HTML goes through visibility.ts. Everything else comes through here.
|
|
6
|
+
*/
|
|
7
|
+
import type { ContentKind, Finding, TextSegment } from "./types.js";
|
|
8
|
+
export declare function detectKind(contentType: string, body: string, urlHint?: string): ContentKind;
|
|
9
|
+
/**
|
|
10
|
+
* Decode numeric and the common named entities so entity-smuggled
|
|
11
|
+
* characters (​ ​ ‍) reach the character pass as the
|
|
12
|
+
* characters they are. Returns the decoded text and the count of
|
|
13
|
+
* suspicious decodes (format controls, tags, selectors).
|
|
14
|
+
*/
|
|
15
|
+
export declare function decodeEntities(text: string): {
|
|
16
|
+
text: string;
|
|
17
|
+
suspicious: number;
|
|
18
|
+
sample: string;
|
|
19
|
+
};
|
|
20
|
+
export interface AdapterResult {
|
|
21
|
+
segments: TextSegment[];
|
|
22
|
+
findings: Finding[];
|
|
23
|
+
}
|
|
24
|
+
/** Markdown: comments and non-rendered constructs are hidden; rendered text is visible. */
|
|
25
|
+
export declare function adaptMarkdown(raw: string): AdapterResult;
|
|
26
|
+
/** JSON: every string value is a segment; keys hint at intent. Nothing is "hidden" in JSON, but long strings in unusual keys are attribute-like. */
|
|
27
|
+
export declare function adaptJson(raw: string): AdapterResult;
|
|
28
|
+
export declare function adaptText(raw: string): AdapterResult;
|
|
29
|
+
/** Plain JavaScript fetched directly (a module, a bundle): the whole thing is a script; any string literals are visible text. */
|
|
30
|
+
export declare function adaptJavascript(raw: string): AdapterResult;
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content adapters: turn any supported content kind into the common shape
|
|
3
|
+
* the passes consume (segments with visibility, scripts, decoded blobs).
|
|
4
|
+
*
|
|
5
|
+
* HTML goes through visibility.ts. Everything else comes through here.
|
|
6
|
+
*/
|
|
7
|
+
export function detectKind(contentType, body, urlHint = "") {
|
|
8
|
+
const ct = contentType.toLowerCase();
|
|
9
|
+
if (/html|xhtml/.test(ct))
|
|
10
|
+
return "html";
|
|
11
|
+
if (/markdown/.test(ct) || /\.(md|markdown)(\?|#|$)/i.test(urlHint))
|
|
12
|
+
return "markdown";
|
|
13
|
+
if (/json/.test(ct))
|
|
14
|
+
return "json";
|
|
15
|
+
if (/javascript|ecmascript/.test(ct) || /\.(m?js|cjs)(\?|#|$)/i.test(urlHint))
|
|
16
|
+
return "javascript";
|
|
17
|
+
const head = body.slice(0, 2048);
|
|
18
|
+
if (/^\s*<(!doctype|html|head|body|div|p|span|article|main|table)/i.test(head))
|
|
19
|
+
return "html";
|
|
20
|
+
if (/^\s*[\[{]/.test(head)) {
|
|
21
|
+
try {
|
|
22
|
+
JSON.parse(body);
|
|
23
|
+
return "json";
|
|
24
|
+
}
|
|
25
|
+
catch { /* not json */ }
|
|
26
|
+
}
|
|
27
|
+
if (/^(#{1,6}\s|\*\*|- |\d+\. |```|\[.+\]\(.+\))/m.test(head) && /\n/.test(head))
|
|
28
|
+
return "markdown";
|
|
29
|
+
return "text";
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Decode numeric and the common named entities so entity-smuggled
|
|
33
|
+
* characters (​ ​ ‍) reach the character pass as the
|
|
34
|
+
* characters they are. Returns the decoded text and the count of
|
|
35
|
+
* suspicious decodes (format controls, tags, selectors).
|
|
36
|
+
*/
|
|
37
|
+
export function decodeEntities(text) {
|
|
38
|
+
const named = {
|
|
39
|
+
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", zwj: "", zwnj: "", lrm: "", rlm: "", shy: "",
|
|
40
|
+
NoBreak: "", ZeroWidthSpace: "", NegativeMediumSpace: "",
|
|
41
|
+
};
|
|
42
|
+
let suspicious = 0;
|
|
43
|
+
let sample = "";
|
|
44
|
+
const isSuspicious = (cp) => (cp >= 0x200b && cp <= 0x200f) || cp === 0x2060 || cp === 0xfeff || cp === 0x00ad || (cp >= 0x202a && cp <= 0x202e) ||
|
|
45
|
+
(cp >= 0x2066 && cp <= 0x2069) || (cp >= 0xfe00 && cp <= 0xfe0f) || (cp >= 0xe0000 && cp <= 0xe007f) || (cp >= 0xe0100 && cp <= 0xe01ef);
|
|
46
|
+
const out = text.replace(/&(#x[0-9a-fA-F]{1,6}|#\d{1,7}|[a-zA-Z]{2,20});/g, (m, body) => {
|
|
47
|
+
let cp;
|
|
48
|
+
if (body[0] === "#")
|
|
49
|
+
cp = body[1] === "x" || body[1] === "X" ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
|
50
|
+
else if (body in named) {
|
|
51
|
+
const ch = named[body];
|
|
52
|
+
cp = ch.codePointAt(0);
|
|
53
|
+
}
|
|
54
|
+
if (cp === undefined || Number.isNaN(cp) || cp > 0x10ffff)
|
|
55
|
+
return m;
|
|
56
|
+
if (isSuspicious(cp)) {
|
|
57
|
+
suspicious++;
|
|
58
|
+
if (!sample)
|
|
59
|
+
sample = m;
|
|
60
|
+
}
|
|
61
|
+
return String.fromCodePoint(cp);
|
|
62
|
+
});
|
|
63
|
+
return { text: out, suspicious, sample };
|
|
64
|
+
}
|
|
65
|
+
/** Markdown: comments and non-rendered constructs are hidden; rendered text is visible. */
|
|
66
|
+
export function adaptMarkdown(raw) {
|
|
67
|
+
const segments = [];
|
|
68
|
+
const findings = [];
|
|
69
|
+
let text = raw;
|
|
70
|
+
// HTML comments inside markdown render nothing
|
|
71
|
+
text = text.replace(/<!--([\s\S]*?)-->/g, (_, c) => {
|
|
72
|
+
if (c.trim())
|
|
73
|
+
segments.push({ text: c, visibility: "comment", path: "md/comment" });
|
|
74
|
+
return " ";
|
|
75
|
+
});
|
|
76
|
+
// reference-style link definitions: [id]: url "title" — the title and url never render
|
|
77
|
+
text = text.replace(/^\s{0,3}\[[^\]]+\]:\s*(\S+)(?:\s+["'(]([^"')]*)["')])?\s*$/gm, (_, url, title) => {
|
|
78
|
+
if (title?.trim())
|
|
79
|
+
segments.push({ text: title, visibility: "attribute", path: "md/linkdef@title" });
|
|
80
|
+
if (/[?#].{40,}/.test(url))
|
|
81
|
+
segments.push({ text: url, visibility: "attribute", path: "md/linkdef@url" });
|
|
82
|
+
return " ";
|
|
83
|
+
});
|
|
84
|
+
// inline link/image titles and urls
|
|
85
|
+
text = text.replace(/!?\[([^\]]*)\]\(([^)\s]+)(?:\s+["']([^"']*)["'])?\)/g, (m, label, url, title) => {
|
|
86
|
+
if (title?.trim())
|
|
87
|
+
segments.push({ text: title, visibility: "attribute", path: "md/link@title" });
|
|
88
|
+
// keep the construct for the meaning pass (markdown image exfil) by leaving it in visible text
|
|
89
|
+
return m;
|
|
90
|
+
});
|
|
91
|
+
// raw HTML blocks inside markdown: hidden attributes and styles
|
|
92
|
+
text = text.replace(/<([a-z][\w-]*)([^>]*)>/gi, (m, _tag, attrs) => {
|
|
93
|
+
if (/display\s*:\s*none|visibility\s*:\s*hidden|hidden(\s|=|>|$)|aria-hidden\s*=\s*["']?true|font-size\s*:\s*0|opacity\s*:\s*0/i.test(attrs)) {
|
|
94
|
+
segments.push({ text: m, visibility: "css_hidden", path: "md/html-open-tag" });
|
|
95
|
+
findings.push({ pass: "visibility", kind: "visibility.markdown_hidden", confidence: 0.6, severity: 0.6, message: "raw HTML with hiding styles inside markdown", excerpt: m.slice(0, 120), location: "md/html", action: "none" });
|
|
96
|
+
}
|
|
97
|
+
const alt = attrs.match(/\b(alt|title|aria-label)\s*=\s*["']([^"']{20,})["']/i);
|
|
98
|
+
if (alt)
|
|
99
|
+
segments.push({ text: alt[2], visibility: "attribute", path: `md/html@${alt[1]}` });
|
|
100
|
+
return " ";
|
|
101
|
+
});
|
|
102
|
+
// fenced code is visible but is code; keep it visible (the model reads it)
|
|
103
|
+
segments.unshift({ text, visibility: "visible", path: "md" });
|
|
104
|
+
for (const s of segments) {
|
|
105
|
+
if (s.visibility === "visible")
|
|
106
|
+
continue;
|
|
107
|
+
const words = s.text.trim().split(/\s+/).length;
|
|
108
|
+
if (words < 6)
|
|
109
|
+
continue;
|
|
110
|
+
findings.push({
|
|
111
|
+
pass: "visibility", kind: "visibility.markdown_hidden", confidence: Math.min(0.9, 0.5 + words / 100), severity: 0.6,
|
|
112
|
+
message: `${words} words in a markdown construct that does not render (${s.path})`, excerpt: s.text.trim().slice(0, 160), location: s.path, action: "none",
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
return { segments, findings };
|
|
116
|
+
}
|
|
117
|
+
/** JSON: every string value is a segment; keys hint at intent. Nothing is "hidden" in JSON, but long strings in unusual keys are attribute-like. */
|
|
118
|
+
export function adaptJson(raw) {
|
|
119
|
+
const segments = [];
|
|
120
|
+
const findings = [];
|
|
121
|
+
let doc;
|
|
122
|
+
try {
|
|
123
|
+
doc = JSON.parse(raw);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return adaptText(raw);
|
|
127
|
+
}
|
|
128
|
+
const walk = (v, path, depth) => {
|
|
129
|
+
if (depth > 64)
|
|
130
|
+
return;
|
|
131
|
+
if (typeof v === "string") {
|
|
132
|
+
const keyPath = path.replace(/^\$/, "");
|
|
133
|
+
if (v.trim())
|
|
134
|
+
segments.push({ text: v, visibility: /(\.|\[)(_|__|\$|meta|hidden|internal|debug|comment|instruction|instructions|prompt|system)(\b|\.|\[|$)/i.test(keyPath) ? "attribute" : "visible", path });
|
|
135
|
+
}
|
|
136
|
+
else if (Array.isArray(v))
|
|
137
|
+
v.forEach((x, i) => walk(x, `${path}[${i}]`, depth + 1));
|
|
138
|
+
else if (v && typeof v === "object")
|
|
139
|
+
for (const [k, x] of Object.entries(v))
|
|
140
|
+
walk(x, `${path}.${k}`, depth + 1);
|
|
141
|
+
};
|
|
142
|
+
walk(doc, "$", 0);
|
|
143
|
+
for (const s of segments) {
|
|
144
|
+
if (s.visibility !== "attribute")
|
|
145
|
+
continue;
|
|
146
|
+
const words = s.text.trim().split(/\s+/).length;
|
|
147
|
+
if (words >= 6)
|
|
148
|
+
findings.push({ pass: "visibility", kind: "visibility.attribute_only", confidence: 0.4, severity: 0.5, message: `${words} words under a metadata-like JSON key`, excerpt: s.text.slice(0, 160), location: s.path, action: "none" });
|
|
149
|
+
}
|
|
150
|
+
return { segments, findings };
|
|
151
|
+
}
|
|
152
|
+
export function adaptText(raw) {
|
|
153
|
+
return { segments: [{ text: raw, visibility: "visible", path: "text" }], findings: [] };
|
|
154
|
+
}
|
|
155
|
+
/** Plain JavaScript fetched directly (a module, a bundle): the whole thing is a script; any string literals are visible text. */
|
|
156
|
+
export function adaptJavascript(raw) {
|
|
157
|
+
const strings = raw.match(/(["'`])(?:\\.|(?!\1)[^\\\n])*\1/g) ?? [];
|
|
158
|
+
const text = strings.map((s) => s.slice(1, -1)).join("\n");
|
|
159
|
+
return { segments: [{ text, visibility: "visible", path: "js/strings" }], findings: [] };
|
|
160
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Character pass (AcePaste lineage).
|
|
3
|
+
*
|
|
4
|
+
* The hostile-invisible alphabet is ported verbatim from the production
|
|
5
|
+
* AcePaste cleaner (see acepaste-alphabet.ts), so the gate strips exactly
|
|
6
|
+
* what the web tool and the Chrome extension strip. On top of that shared
|
|
7
|
+
* set, the gate adds script-aware exceptions: zero-width joiners inside
|
|
8
|
+
* emoji sequences and Indic/Arabic text, zero-width non-joiners in complex
|
|
9
|
+
* scripts, and bidirectional marks (not overrides or isolates) inside
|
|
10
|
+
* right-to-left runs are legitimate and left in place; and it adds the
|
|
11
|
+
* decode-and-rescan behavior the gate needs that the cleaner does not.
|
|
12
|
+
*/
|
|
13
|
+
import type { Finding } from "./types.js";
|
|
14
|
+
export { countHostileInvisibles } from "./acepaste-alphabet.js";
|
|
15
|
+
export interface CharacterResult {
|
|
16
|
+
/** text with hostile characters removed and encoded payloads untouched */
|
|
17
|
+
cleaned: string;
|
|
18
|
+
findings: Finding[];
|
|
19
|
+
/** decoded payloads that should be rescanned by the meaning pass */
|
|
20
|
+
decodedPayloads: string[];
|
|
21
|
+
}
|
|
22
|
+
export declare function scanCharacters(text: string): CharacterResult;
|