@acepaste/ingress 0.9.6 → 0.9.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/adapters.d.ts +9 -0
- package/dist/src/adapters.js +76 -6
- package/dist/src/agent-hooks.d.ts +112 -0
- package/dist/src/agent-hooks.js +533 -0
- package/dist/src/approvers.d.ts +16 -0
- package/dist/src/approvers.js +50 -0
- package/dist/src/audio.d.ts +70 -0
- package/dist/src/audio.js +224 -0
- package/dist/src/audit-changes.d.ts +46 -0
- package/dist/src/audit-changes.js +110 -0
- package/dist/src/audit-local.d.ts +57 -0
- package/dist/src/audit-local.js +231 -0
- package/dist/src/audit-parsers.d.ts +1 -1
- package/dist/src/audit.js +48 -0
- package/dist/src/broker.d.ts +92 -1
- package/dist/src/broker.js +222 -3
- package/dist/src/calendar.d.ts +36 -0
- package/dist/src/calendar.js +227 -0
- package/dist/src/captions.d.ts +44 -0
- package/dist/src/captions.js +209 -0
- package/dist/src/changes.d.ts +105 -0
- package/dist/src/changes.js +174 -0
- package/dist/src/character.js +15 -1
- package/dist/src/classifier-model.d.ts +53 -0
- package/dist/src/classifier-model.js +3 -0
- package/dist/src/classifier.d.ts +41 -0
- package/dist/src/classifier.js +213 -0
- package/dist/src/cli.js +501 -1
- package/dist/src/egress.d.ts +71 -0
- package/dist/src/egress.js +242 -0
- package/dist/src/email.d.ts +42 -0
- package/dist/src/email.js +424 -0
- package/dist/src/engine.d.ts +36 -12
- package/dist/src/engine.js +88 -37
- package/dist/src/image-meta.d.ts +33 -0
- package/dist/src/image-meta.js +433 -0
- package/dist/src/image.d.ts +110 -0
- package/dist/src/image.js +353 -0
- package/dist/src/index.d.ts +50 -5
- package/dist/src/index.js +28 -3
- package/dist/src/localguard.d.ts +33 -0
- package/dist/src/localguard.js +61 -0
- package/dist/src/meaning.d.ts +2 -0
- package/dist/src/meaning.js +77 -22
- package/dist/src/messages.d.ts +79 -0
- package/dist/src/messages.js +431 -0
- package/dist/src/nearby.d.ts +52 -0
- package/dist/src/nearby.js +186 -0
- package/dist/src/netpolicy.js +50 -19
- package/dist/src/provenance.d.ts +56 -0
- package/dist/src/provenance.js +114 -0
- package/dist/src/qr.d.ts +25 -0
- package/dist/src/qr.js +113 -0
- package/dist/src/safeguards.d.ts +61 -0
- package/dist/src/safeguards.js +199 -0
- package/dist/src/served.d.ts +20 -0
- package/dist/src/served.js +53 -0
- package/dist/src/shell-commands.d.ts +39 -0
- package/dist/src/shell-commands.js +291 -0
- package/dist/src/shortcuts.d.ts +45 -0
- package/dist/src/shortcuts.js +220 -0
- package/dist/src/stream.d.ts +88 -0
- package/dist/src/stream.js +119 -0
- package/dist/src/terminal.d.ts +29 -0
- package/dist/src/terminal.js +313 -0
- package/dist/src/tooldefs.d.ts +84 -0
- package/dist/src/tooldefs.js +122 -0
- package/dist/src/types.d.ts +50 -6
- package/dist/src/types.js +15 -2
- package/dist/src/visibility.d.ts +23 -3
- package/dist/src/visibility.js +285 -78
- package/native/ingress-presence.swift +48 -0
- package/native/ingress-vision.swift +417 -0
- package/package.json +3 -1
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The learned pass: a small classifier that reads text the way the rules cannot, by resemblance.
|
|
3
|
+
*
|
|
4
|
+
* The rules are precise and miss rewording: on public sets they catch about one attack in nine. This pass is
|
|
5
|
+
* a logistic regression over two hashed bags (word 1-2-grams, and character 3-5-grams inside word
|
|
6
|
+
* boundaries), trained on public attack and benign text (tools/classifier). It has no dependencies, runs on
|
|
7
|
+
* the machine, and scores a text in well under a millisecond.
|
|
8
|
+
*
|
|
9
|
+
* What it is allowed to decide:
|
|
10
|
+
* - On its own it can make content suspect (refused, and a person may override). It cannot make content
|
|
11
|
+
* hostile: that needs a rule, hidden placement or another signal to agree. A statistical resemblance is
|
|
12
|
+
* not proof.
|
|
13
|
+
* - It judges what would be delivered (the visible text, by its opening) and every piece of hidden text.
|
|
14
|
+
* Later stretches of a long document are judged at a stricter line, because a long page gives it many
|
|
15
|
+
* chances to be wrong.
|
|
16
|
+
*
|
|
17
|
+
* The featurizer below must match tools/classifier/train.py exactly (scikit-learn's HashingVectorizer with
|
|
18
|
+
* the `word` and `char_wb` analyzers). The model carries "parity" vectors scored by the trainer; the tests
|
|
19
|
+
* check this implementation against them.
|
|
20
|
+
*/
|
|
21
|
+
import type { Finding, TextSegment } from "./types.js";
|
|
22
|
+
/** Probability that this text is an injection or jailbreak, 0..1. */
|
|
23
|
+
export declare function classifyText(text: string): number;
|
|
24
|
+
export declare function classifierInfo(): {
|
|
25
|
+
threshold: number;
|
|
26
|
+
trainedAt: string;
|
|
27
|
+
maxChars: number;
|
|
28
|
+
};
|
|
29
|
+
export interface ClassifierInput {
|
|
30
|
+
/** the text that would be delivered */
|
|
31
|
+
visible: string;
|
|
32
|
+
/** every segment, so hidden text can be judged piece by piece */
|
|
33
|
+
segments: ReadonlyArray<TextSegment>;
|
|
34
|
+
/** the content kind: sets how much the visible score may decide */
|
|
35
|
+
kind?: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The classifier pass. A finding from here scores between the suspect line and well under the hostile line,
|
|
39
|
+
* so on its own it refuses with an override and never more.
|
|
40
|
+
*/
|
|
41
|
+
export declare function scanClassifier(input: ClassifierInput): Finding[];
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The learned pass: a small classifier that reads text the way the rules cannot, by resemblance.
|
|
3
|
+
*
|
|
4
|
+
* The rules are precise and miss rewording: on public sets they catch about one attack in nine. This pass is
|
|
5
|
+
* a logistic regression over two hashed bags (word 1-2-grams, and character 3-5-grams inside word
|
|
6
|
+
* boundaries), trained on public attack and benign text (tools/classifier). It has no dependencies, runs on
|
|
7
|
+
* the machine, and scores a text in well under a millisecond.
|
|
8
|
+
*
|
|
9
|
+
* What it is allowed to decide:
|
|
10
|
+
* - On its own it can make content suspect (refused, and a person may override). It cannot make content
|
|
11
|
+
* hostile: that needs a rule, hidden placement or another signal to agree. A statistical resemblance is
|
|
12
|
+
* not proof.
|
|
13
|
+
* - It judges what would be delivered (the visible text, by its opening) and every piece of hidden text.
|
|
14
|
+
* Later stretches of a long document are judged at a stricter line, because a long page gives it many
|
|
15
|
+
* chances to be wrong.
|
|
16
|
+
*
|
|
17
|
+
* The featurizer below must match tools/classifier/train.py exactly (scikit-learn's HashingVectorizer with
|
|
18
|
+
* the `word` and `char_wb` analyzers). The model carries "parity" vectors scored by the trainer; the tests
|
|
19
|
+
* check this implementation against them.
|
|
20
|
+
*/
|
|
21
|
+
import { CLASSIFIER_MODEL } from "./classifier-model.js";
|
|
22
|
+
let cached;
|
|
23
|
+
function model() {
|
|
24
|
+
if (cached)
|
|
25
|
+
return cached;
|
|
26
|
+
const m = CLASSIFIER_MODEL;
|
|
27
|
+
const unpack = (b) => {
|
|
28
|
+
const buf = Buffer.from(b.int8, "base64");
|
|
29
|
+
return { scale: b.scale, weights: new Int8Array(buf.buffer, buf.byteOffset, buf.byteLength), ngram: b.ngram };
|
|
30
|
+
};
|
|
31
|
+
cached = { nFeatures: m.nFeatures, maxChars: m.maxChars, bias: m.bias, threshold: m.threshold, trainedAt: m.trainedAt, word: unpack(m.word), char: unpack(m.char) };
|
|
32
|
+
return cached;
|
|
33
|
+
}
|
|
34
|
+
/** MurmurHash3 (x86, 32-bit), seed 0, as a signed integer: the hash scikit-learn's FeatureHasher uses. */
|
|
35
|
+
function murmur3(bytes) {
|
|
36
|
+
let h = 0;
|
|
37
|
+
const c1 = 0xcc9e2d51, c2 = 0x1b873593;
|
|
38
|
+
const n = bytes.length, blocks = n >>> 2;
|
|
39
|
+
for (let i = 0; i < blocks; i++) {
|
|
40
|
+
let k = bytes[i * 4] | (bytes[i * 4 + 1] << 8) | (bytes[i * 4 + 2] << 16) | (bytes[i * 4 + 3] << 24);
|
|
41
|
+
k = Math.imul(k, c1);
|
|
42
|
+
k = (k << 15) | (k >>> 17);
|
|
43
|
+
k = Math.imul(k, c2);
|
|
44
|
+
h ^= k;
|
|
45
|
+
h = (h << 13) | (h >>> 19);
|
|
46
|
+
h = (Math.imul(h, 5) + 0xe6546b64) | 0;
|
|
47
|
+
}
|
|
48
|
+
let k = 0;
|
|
49
|
+
const tail = blocks * 4;
|
|
50
|
+
switch (n & 3) {
|
|
51
|
+
case 3: k ^= bytes[tail + 2] << 16; // falls through
|
|
52
|
+
case 2: k ^= bytes[tail + 1] << 8; // falls through
|
|
53
|
+
case 1:
|
|
54
|
+
k ^= bytes[tail];
|
|
55
|
+
k = Math.imul(k, c1);
|
|
56
|
+
k = (k << 15) | (k >>> 17);
|
|
57
|
+
k = Math.imul(k, c2);
|
|
58
|
+
h ^= k;
|
|
59
|
+
}
|
|
60
|
+
h ^= n;
|
|
61
|
+
h ^= h >>> 16;
|
|
62
|
+
h = Math.imul(h, 0x85ebca6b);
|
|
63
|
+
h ^= h >>> 13;
|
|
64
|
+
h = Math.imul(h, 0xc2b2ae35);
|
|
65
|
+
h ^= h >>> 16;
|
|
66
|
+
return h | 0;
|
|
67
|
+
}
|
|
68
|
+
const enc = new TextEncoder();
|
|
69
|
+
function column(token, nFeatures) {
|
|
70
|
+
const h = murmur3(enc.encode(token));
|
|
71
|
+
// scikit-learn's own special case: abs() of the most negative int32 does not exist
|
|
72
|
+
return h === -2147483648 ? (2147483647 - (nFeatures - 1)) % nFeatures : Math.abs(h) % nFeatures;
|
|
73
|
+
}
|
|
74
|
+
/** the characters Python's str.split() and \s treat as whitespace */
|
|
75
|
+
const PY_SPACE = "\\t\\n\\v\\f\\r \\x1c-\\x1f\\x85\\u00a0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000";
|
|
76
|
+
const SPACE_RUN = new RegExp(`[${PY_SPACE}]{2,}`, "g");
|
|
77
|
+
const SPLIT = new RegExp(`[${PY_SPACE}]+`);
|
|
78
|
+
const WORD = /[\p{L}\p{N}_]{2,}/gu;
|
|
79
|
+
/** counts -> 1 + ln(count), L2-normalised, dotted with the weights */
|
|
80
|
+
function dot(counts, b) {
|
|
81
|
+
let sumSq = 0, acc = 0;
|
|
82
|
+
for (const [i, c] of counts) {
|
|
83
|
+
const tf = 1 + Math.log(c);
|
|
84
|
+
sumSq += tf * tf;
|
|
85
|
+
acc += tf * b.weights[i];
|
|
86
|
+
}
|
|
87
|
+
return sumSq ? (acc * b.scale) / Math.sqrt(sumSq) : 0;
|
|
88
|
+
}
|
|
89
|
+
/** Probability that this text is an injection or jailbreak, 0..1. */
|
|
90
|
+
export function classifyText(text) {
|
|
91
|
+
const m = model();
|
|
92
|
+
const doc = Array.from(text.normalize("NFKC")).slice(0, m.maxChars).join("").toLowerCase();
|
|
93
|
+
// word 1-2-grams: maximal runs of two or more word characters
|
|
94
|
+
const words = new Map();
|
|
95
|
+
const toks = doc.match(WORD) ?? [];
|
|
96
|
+
const bump = (map, t) => { const i = column(t, m.nFeatures); map.set(i, (map.get(i) ?? 0) + 1); };
|
|
97
|
+
for (const t of toks)
|
|
98
|
+
bump(words, t);
|
|
99
|
+
for (let n = 2; n <= Math.min(m.word.ngram[1], toks.length); n++)
|
|
100
|
+
for (let i = 0; i + n <= toks.length; i++)
|
|
101
|
+
bump(words, toks.slice(i, i + n).join(" "));
|
|
102
|
+
// character n-grams inside word boundaries: each whitespace-separated word, padded with one space each side
|
|
103
|
+
const chars = new Map();
|
|
104
|
+
for (const raw of doc.replace(SPACE_RUN, " ").split(SPLIT)) {
|
|
105
|
+
if (!raw)
|
|
106
|
+
continue;
|
|
107
|
+
const w = Array.from(` ${raw} `);
|
|
108
|
+
for (let n = m.char.ngram[0]; n <= m.char.ngram[1]; n++) {
|
|
109
|
+
let offset = 0;
|
|
110
|
+
bump(chars, w.slice(0, n).join(""));
|
|
111
|
+
while (offset + n < w.length) {
|
|
112
|
+
offset++;
|
|
113
|
+
bump(chars, w.slice(offset, offset + n).join(""));
|
|
114
|
+
}
|
|
115
|
+
if (offset === 0)
|
|
116
|
+
break; // a word shorter than n is counted once
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const z = dot(words, m.word) + dot(chars, m.char) + m.bias;
|
|
120
|
+
return 1 / (1 + Math.exp(-z));
|
|
121
|
+
}
|
|
122
|
+
export function classifierInfo() {
|
|
123
|
+
const m = model();
|
|
124
|
+
return { threshold: m.threshold, trainedAt: m.trainedAt, maxChars: m.maxChars };
|
|
125
|
+
}
|
|
126
|
+
const WINDOW = 3000, STEP = 2400, MAX_WINDOWS = 80;
|
|
127
|
+
const AUTHORITY = { email: "message", message: "message", notification: "message", calendar: "message", nearby: "message", html: "document", markdown: "document", javascript: "document", json: "text", text: "text", log: "text", automation: "text", captions: "text" };
|
|
128
|
+
// chosen on the development samples (tools/pint, tools/email-eval): at 0.85 the public benign sets give 3
|
|
129
|
+
// false alarms in 7,855 texts; at 0.8 the development emails give 1 in 3,000
|
|
130
|
+
const VISIBLE_LINE = { message: 0.8, text: 0.85, document: 2 /* never */ };
|
|
131
|
+
const HIDDEN_LINE = 0.9;
|
|
132
|
+
/** channels made of short texts, where the classifier's visible line depends on length (see scanClassifier) */
|
|
133
|
+
const SHORT_TEXT_KINDS = new Set(["message", "notification", "nearby"]);
|
|
134
|
+
/**
|
|
135
|
+
* The same text with what it quotes taken out: quotation marks, block quotes, code. An article that reports
|
|
136
|
+
* an attack reads like one until the quotation is removed; an attack does not sit inside quotation marks.
|
|
137
|
+
* (The rules make the same allowance.) Returns undefined when too little is left to judge.
|
|
138
|
+
*/
|
|
139
|
+
function withoutQuotations(text) {
|
|
140
|
+
const stripped = text
|
|
141
|
+
.replace(/```[\s\S]*?```|~~~[\s\S]*?~~~/g, " ")
|
|
142
|
+
.replace(/^\s{0,3}>.*$/gm, " ")
|
|
143
|
+
.replace(/“[^”]{1,800}”|‘[^’]{12,800}’|«[^»]{1,800}»|"[^"\n]{12,800}"/g, " ");
|
|
144
|
+
return stripped.trim().split(/\s+/).length >= 20 ? stripped : undefined;
|
|
145
|
+
}
|
|
146
|
+
/** the classifier's score for visible text, after the allowance for quotation */
|
|
147
|
+
function visibleScore(text) {
|
|
148
|
+
const p = classifyText(text);
|
|
149
|
+
if (p < 0.5)
|
|
150
|
+
return p;
|
|
151
|
+
const rest = withoutQuotations(text);
|
|
152
|
+
return rest === undefined || rest.length === text.length ? p : Math.min(p, classifyText(rest));
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* The classifier pass. A finding from here scores between the suspect line and well under the hostile line,
|
|
156
|
+
* so on its own it refuses with an override and never more.
|
|
157
|
+
*/
|
|
158
|
+
export function scanClassifier(input) {
|
|
159
|
+
const { threshold, maxChars } = classifierInfo();
|
|
160
|
+
const out = [];
|
|
161
|
+
const authority = AUTHORITY[input.kind ?? "text"] ?? "text";
|
|
162
|
+
// Short-text mode (0.9.19). People text in imperatives ("call me now", "send me the address"), and the model
|
|
163
|
+
// was never shown how people text: on 2,419 ordinary text messages it rated 20 as attacks, at every length
|
|
164
|
+
// up to 33 words. So in chat, notifications and device names a short text is not judged by the classifier
|
|
165
|
+
// alone, and a middling one needs near-certainty. The rules still read every word.
|
|
166
|
+
const words = input.visible.trim().split(/\s+/).length;
|
|
167
|
+
const line = SHORT_TEXT_KINDS.has(input.kind ?? "") ? (words < 15 ? 2 : words < 40 ? 0.9 : VISIBLE_LINE[authority]) : VISIBLE_LINE[authority];
|
|
168
|
+
const finding = (p, at, where, hidden) => ({
|
|
169
|
+
pass: "classifier", kind: "classifier.injection_likely",
|
|
170
|
+
// from just over the suspect line at the threshold to 0.62 at certainty: never hostile alone
|
|
171
|
+
confidence: Math.min(0.95, 0.45 + 0.5 * ((p - at) / Math.max(1e-6, 1 - at)) + (hidden ? 0.1 : 0)), severity: 0.65,
|
|
172
|
+
message: `reads like an injection or jailbreak attempt (learned classifier, ${(p * 100).toFixed(0)}%)${hidden ? ", in text a human would not see" : ""}`,
|
|
173
|
+
location: where, inHidden: hidden || undefined, action: "none",
|
|
174
|
+
});
|
|
175
|
+
/** recorded on the receipt, deciding nothing */
|
|
176
|
+
const note = (p, where) => ({ pass: "classifier", kind: "classifier.injection_likely", confidence: 0, severity: 0, message: `resembles an injection or jailbreak attempt (learned classifier, ${(p * 100).toFixed(0)}%); not decisive for visible ${authority} text`, location: where, action: "none" });
|
|
177
|
+
const visible = input.visible.trim();
|
|
178
|
+
if (visible.length >= 20) {
|
|
179
|
+
const cps = Array.from(visible);
|
|
180
|
+
let best = visibleScore(cps.slice(0, maxChars).join("")), where = "visible text";
|
|
181
|
+
// the rest of a long text, in overlapping windows
|
|
182
|
+
for (let s = STEP, n = 0; s < cps.length - 200 && n < MAX_WINDOWS && cps.length > maxChars; s += STEP, n++) {
|
|
183
|
+
const p = visibleScore(cps.slice(s, s + WINDOW).join(""));
|
|
184
|
+
if (p > best) {
|
|
185
|
+
best = p;
|
|
186
|
+
where = "visible text (past the opening)";
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (best >= line)
|
|
190
|
+
out.push(finding(best, line, where, false));
|
|
191
|
+
else if (best >= Math.max(threshold, 0.9))
|
|
192
|
+
out.push(note(best, where));
|
|
193
|
+
}
|
|
194
|
+
// Hidden text, piece by piece: a hidden passage that reads like an attack is the strongest thing this pass
|
|
195
|
+
// can say. A page has hundreds of hidden pieces (menus, labels, preheaders, descriptions), each a chance to
|
|
196
|
+
// be wrong, so the line is strict and one-line furniture is not judged at all.
|
|
197
|
+
let bestHidden = 0, where = "";
|
|
198
|
+
for (const s of input.segments) {
|
|
199
|
+
if (s.visibility === "visible" || (s.visibility === "attribute" && s.path.endsWith("@alt")))
|
|
200
|
+
continue;
|
|
201
|
+
const t = s.text.trim();
|
|
202
|
+
if (t.length < 60 || t.split(/\s+/).length < 9)
|
|
203
|
+
continue;
|
|
204
|
+
const p = classifyText(t);
|
|
205
|
+
if (p > bestHidden) {
|
|
206
|
+
bestHidden = p;
|
|
207
|
+
where = s.path;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
if (bestHidden >= HIDDEN_LINE)
|
|
211
|
+
out.push(finding(bestHidden, HIDDEN_LINE, where, true));
|
|
212
|
+
return out;
|
|
213
|
+
}
|