@acepaste/ingress 0.9.6 → 0.9.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/adapters.d.ts +9 -0
- package/dist/src/adapters.js +76 -6
- package/dist/src/agent-hooks.d.ts +112 -0
- package/dist/src/agent-hooks.js +533 -0
- package/dist/src/approvers.d.ts +16 -0
- package/dist/src/approvers.js +50 -0
- package/dist/src/audio.d.ts +70 -0
- package/dist/src/audio.js +224 -0
- package/dist/src/audit-changes.d.ts +46 -0
- package/dist/src/audit-changes.js +110 -0
- package/dist/src/audit-local.d.ts +57 -0
- package/dist/src/audit-local.js +231 -0
- package/dist/src/audit-parsers.d.ts +1 -1
- package/dist/src/audit.js +48 -0
- package/dist/src/broker.d.ts +92 -1
- package/dist/src/broker.js +222 -3
- package/dist/src/calendar.d.ts +36 -0
- package/dist/src/calendar.js +227 -0
- package/dist/src/captions.d.ts +44 -0
- package/dist/src/captions.js +209 -0
- package/dist/src/changes.d.ts +105 -0
- package/dist/src/changes.js +174 -0
- package/dist/src/character.js +15 -1
- package/dist/src/classifier-model.d.ts +53 -0
- package/dist/src/classifier-model.js +3 -0
- package/dist/src/classifier.d.ts +41 -0
- package/dist/src/classifier.js +213 -0
- package/dist/src/cli.js +501 -1
- package/dist/src/egress.d.ts +71 -0
- package/dist/src/egress.js +242 -0
- package/dist/src/email.d.ts +42 -0
- package/dist/src/email.js +424 -0
- package/dist/src/engine.d.ts +36 -12
- package/dist/src/engine.js +88 -37
- package/dist/src/image-meta.d.ts +33 -0
- package/dist/src/image-meta.js +433 -0
- package/dist/src/image.d.ts +110 -0
- package/dist/src/image.js +353 -0
- package/dist/src/index.d.ts +50 -5
- package/dist/src/index.js +28 -3
- package/dist/src/localguard.d.ts +33 -0
- package/dist/src/localguard.js +61 -0
- package/dist/src/meaning.d.ts +2 -0
- package/dist/src/meaning.js +77 -22
- package/dist/src/messages.d.ts +79 -0
- package/dist/src/messages.js +431 -0
- package/dist/src/nearby.d.ts +52 -0
- package/dist/src/nearby.js +186 -0
- package/dist/src/netpolicy.js +50 -19
- package/dist/src/provenance.d.ts +56 -0
- package/dist/src/provenance.js +114 -0
- package/dist/src/qr.d.ts +25 -0
- package/dist/src/qr.js +113 -0
- package/dist/src/safeguards.d.ts +61 -0
- package/dist/src/safeguards.js +199 -0
- package/dist/src/served.d.ts +20 -0
- package/dist/src/served.js +53 -0
- package/dist/src/shell-commands.d.ts +39 -0
- package/dist/src/shell-commands.js +291 -0
- package/dist/src/shortcuts.d.ts +45 -0
- package/dist/src/shortcuts.js +220 -0
- package/dist/src/stream.d.ts +88 -0
- package/dist/src/stream.js +119 -0
- package/dist/src/terminal.d.ts +29 -0
- package/dist/src/terminal.js +313 -0
- package/dist/src/tooldefs.d.ts +84 -0
- package/dist/src/tooldefs.js +122 -0
- package/dist/src/types.d.ts +50 -6
- package/dist/src/types.js +15 -2
- package/dist/src/visibility.d.ts +23 -3
- package/dist/src/visibility.js +285 -78
- package/native/ingress-presence.swift +48 -0
- package/native/ingress-vision.swift +417 -0
- package/package.json +3 -1
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shortcuts: reading an automation before it is installed.
|
|
3
|
+
*
|
|
4
|
+
* A shared shortcut is a program someone else wrote that runs with your contacts, photos, files, messages and
|
|
5
|
+
* network. Its actions are listed in the file. This scanner reads them and says what the shortcut can do, in
|
|
6
|
+
* three questions that matter together:
|
|
7
|
+
*
|
|
8
|
+
* does it read private data? contacts, photos, files, the clipboard, location, messages, health
|
|
9
|
+
* does it take input from outside? the contents of a web address, a page, a feed, whatever is shared to it
|
|
10
|
+
* can it send things out? a web request with a body, a message, an email, a post, a remote script
|
|
11
|
+
*
|
|
12
|
+
* Any one is ordinary. All three in one shortcut is a program that can be told from outside to send your data
|
|
13
|
+
* somewhere, and it is reported as that. Running code (shell, AppleScript, JavaScript on a page, SSH) and
|
|
14
|
+
* running other shortcuts are reported. Every piece of text in it (comments, message bodies, scripts, URLs)
|
|
15
|
+
* is scanned like any other content.
|
|
16
|
+
*
|
|
17
|
+
* Input: the shortcut as JSON or an XML property list (`plutil -convert json -o - file.shortcut` gives JSON).
|
|
18
|
+
* A signed shortcut exported from the app is encrypted to Apple's format and cannot be read here; one opened
|
|
19
|
+
* from a shared link, or saved unsigned, can.
|
|
20
|
+
*/
|
|
21
|
+
import { linkTrick } from "./messages.js";
|
|
22
|
+
const short = (id) => id.replace(/^is\.workflow\.actions\./, "").replace(/^com\.apple\./, "");
|
|
23
|
+
const PRIVATE = [
|
|
24
|
+
[/contacts?$|selectcontacts|getcontact|filter\.contacts/i, "contacts"], [/photos?$|selectphoto|getlastphoto|getlatestlivephotos|filter\.photos|takephoto|getlastscreenshot/i, "photos"],
|
|
25
|
+
[/getclipboard/i, "the clipboard"], [/location$|getcurrentlocation|getmylocation/i, "your location"], [/health\.quantity|filter\.health/i, "health data"],
|
|
26
|
+
[/documentpicker\.open|file\.getfoldercontents|file$|getfile|filter\.files|file\.select/i, "files"], [/getmessages|searchmessages|filter\.messages|imessage\.search/i, "messages"],
|
|
27
|
+
[/filter\.calendarevents|getupcomingevents|filter\.reminders|getupcomingreminders/i, "calendar and reminders"], [/filter\.notes|shownote|getnote/i, "notes"],
|
|
28
|
+
[/getwifi|getipaddress|getdevicedetails|getbatterylevel/i, "device details"], [/dictatetext|recordaudio/i, "the microphone"], [/safari\.geturl|getcurrenturl|getarticle.*safari|readinglist/i, "what you are browsing"],
|
|
29
|
+
[/mail\.search|getemails|filter\.mail/i, "email"], [/password|keychain/i, "saved passwords"],
|
|
30
|
+
];
|
|
31
|
+
const OUTSIDE = [
|
|
32
|
+
[/downloadurl$/i, "the contents of a web address"], [/getwebpagecontents|getarticle|gethtmlfromrichtext/i, "a web page"], [/rss|getitemsfromfeed|rss\.extract/i, "a feed"],
|
|
33
|
+
[/url\.expand|getheadersofurl/i, "a web address's reply"], [/detect\.link|geturlcomponent/i, "links found in its input"], [/scanbarcode/i, "a scanned code"],
|
|
34
|
+
];
|
|
35
|
+
const SENDS = [
|
|
36
|
+
[/sendmessage|sendimessage/i, "sends a message"], [/sendemail|mail\.send/i, "sends an email"], [/airdropdocument/i, "sends over AirDrop"],
|
|
37
|
+
[/postonfacebook|tweet|postto|instagram|tumblr|wordpress|slack|discord|telegram|whatsapp\.send/i, "posts to a service"], [/dropbox\.savefile|box\.savefile|evernote|uploadto|imgur|cloudapp|pastebin/i, "uploads to a service"],
|
|
38
|
+
[/runsshscript/i, "runs a script on another machine"], [/share$|showsharesheet/i, "opens the share sheet"], [/openurl$|openxcallbackurl|openin$/i, "opens a web address or another app"],
|
|
39
|
+
];
|
|
40
|
+
const CODE = [
|
|
41
|
+
[/runshellscript/i, "a shell script"], [/runapplescript|runjavascriptforautomation/i, "AppleScript or JavaScript for Automation"], [/runjavascriptonwebpage/i, "JavaScript on a web page"], [/runsshscript/i, "a script over SSH"],
|
|
42
|
+
];
|
|
43
|
+
export function looksLikeShortcut(doc) {
|
|
44
|
+
return !!doc && typeof doc === "object" && !Array.isArray(doc) && Array.isArray(doc.WFWorkflowActions);
|
|
45
|
+
}
|
|
46
|
+
/** A property list in XML, as far as a shortcut needs: dictionaries, arrays, strings, numbers, booleans. */
|
|
47
|
+
export function parsePlistXml(xml) {
|
|
48
|
+
const tokens = [...xml.matchAll(/<(\/?)(plist|dict|array|key|string|integer|real|true|false|data|date)\b[^>]*?(\/?)>|([^<]+)/g)];
|
|
49
|
+
let i = 0;
|
|
50
|
+
const dec = (s) => s.replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))).replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16))).replace(/&/g, "&");
|
|
51
|
+
const textUntilClose = (tag) => { let s = ""; while (i < tokens.length && !(tokens[i][1] === "/" && tokens[i][2] === tag)) {
|
|
52
|
+
s += tokens[i][4] ?? "";
|
|
53
|
+
i++;
|
|
54
|
+
} i++; return dec(s); };
|
|
55
|
+
const value = (depth) => {
|
|
56
|
+
while (i < tokens.length && tokens[i][4] !== undefined)
|
|
57
|
+
i++; // whitespace between elements
|
|
58
|
+
const t = tokens[i++];
|
|
59
|
+
if (!t || depth > 64)
|
|
60
|
+
return undefined;
|
|
61
|
+
const tag = t[2], selfClosing = t[3] === "/";
|
|
62
|
+
if (tag === "true")
|
|
63
|
+
return true;
|
|
64
|
+
if (tag === "false")
|
|
65
|
+
return false;
|
|
66
|
+
if (selfClosing)
|
|
67
|
+
return tag === "dict" ? {} : tag === "array" ? [] : "";
|
|
68
|
+
if (tag === "string" || tag === "data" || tag === "date")
|
|
69
|
+
return textUntilClose(tag);
|
|
70
|
+
if (tag === "integer" || tag === "real")
|
|
71
|
+
return Number(textUntilClose(tag));
|
|
72
|
+
if (tag === "array") {
|
|
73
|
+
const a = [];
|
|
74
|
+
for (;;) {
|
|
75
|
+
while (i < tokens.length && tokens[i][4] !== undefined)
|
|
76
|
+
i++;
|
|
77
|
+
if (i >= tokens.length || (tokens[i][1] === "/" && tokens[i][2] === "array")) {
|
|
78
|
+
i++;
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
a.push(value(depth + 1));
|
|
82
|
+
if (a.length > 20_000)
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
return a;
|
|
86
|
+
}
|
|
87
|
+
if (tag === "dict") {
|
|
88
|
+
const d = {};
|
|
89
|
+
for (;;) {
|
|
90
|
+
while (i < tokens.length && tokens[i][4] !== undefined)
|
|
91
|
+
i++;
|
|
92
|
+
if (i >= tokens.length || (tokens[i][1] === "/" && tokens[i][2] === "dict")) {
|
|
93
|
+
i++;
|
|
94
|
+
break;
|
|
95
|
+
}
|
|
96
|
+
if (tokens[i][2] !== "key") {
|
|
97
|
+
i++;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
i++;
|
|
101
|
+
const k = textUntilClose("key");
|
|
102
|
+
d[k] = value(depth + 1);
|
|
103
|
+
}
|
|
104
|
+
return d;
|
|
105
|
+
}
|
|
106
|
+
if (tag === "plist")
|
|
107
|
+
return value(depth + 1);
|
|
108
|
+
return undefined;
|
|
109
|
+
};
|
|
110
|
+
return value(0);
|
|
111
|
+
}
|
|
112
|
+
/** every string inside an action's parameters, with the path to it */
|
|
113
|
+
function strings(v, path, out, depth = 0) {
|
|
114
|
+
if (depth > 12 || out.length > 4000)
|
|
115
|
+
return;
|
|
116
|
+
if (typeof v === "string") {
|
|
117
|
+
if (v.trim().length >= 2 && !/^[0-9A-F]{8}-[0-9A-F]{4}-/i.test(v))
|
|
118
|
+
out.push({ path, text: v });
|
|
119
|
+
}
|
|
120
|
+
else if (Array.isArray(v))
|
|
121
|
+
v.slice(0, 500).forEach((x, i) => strings(x, `${path}[${i}]`, out, depth + 1));
|
|
122
|
+
else if (v && typeof v === "object")
|
|
123
|
+
for (const [k, x] of Object.entries(v)) {
|
|
124
|
+
if (!/^(UUID|GroupingIdentifier|WFSerializationType|OutputUUID|CustomOutputName|Type|WFControlFlowMode)$/.test(k))
|
|
125
|
+
strings(x, path ? `${path}.${k}` : k, out, depth + 1);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
export function summarizeShortcut(doc) {
|
|
129
|
+
const d = (doc ?? {});
|
|
130
|
+
const actions = (Array.isArray(d.WFWorkflowActions) ? d.WFWorkflowActions : []).slice(0, 5000).map((a) => ({ id: String(a?.WFWorkflowActionIdentifier ?? ""), params: (a?.WFWorkflowActionParameters ?? {}) }));
|
|
131
|
+
const s = { actions: actions.length, readsPrivate: [], outsideInput: [], sendsOut: [], runsCode: [], runsOtherShortcuts: [], destinations: [], acceptsSharedInput: Array.isArray(d.WFWorkflowInputContentItemClasses) && d.WFWorkflowInputContentItemClasses.length > 0 && Array.isArray(d.WFWorkflowTypes) && d.WFWorkflowTypes.some((t) => /ActionExtension|Share|QuickActions|ReceivesOnScreenContent/i.test(String(t))) };
|
|
132
|
+
const push = (list, v) => { if (!list.includes(v))
|
|
133
|
+
list.push(v); };
|
|
134
|
+
for (const a of actions) {
|
|
135
|
+
const id = short(a.id);
|
|
136
|
+
for (const [re, what] of PRIVATE)
|
|
137
|
+
if (re.test(id))
|
|
138
|
+
push(s.readsPrivate, what);
|
|
139
|
+
for (const [re, what] of OUTSIDE)
|
|
140
|
+
if (re.test(id))
|
|
141
|
+
push(s.outsideInput, what);
|
|
142
|
+
for (const [re, what] of SENDS)
|
|
143
|
+
if (re.test(id))
|
|
144
|
+
push(s.sendsOut, what);
|
|
145
|
+
for (const [re, what] of CODE)
|
|
146
|
+
if (re.test(id))
|
|
147
|
+
push(s.runsCode, what);
|
|
148
|
+
if (/runworkflow$/i.test(id))
|
|
149
|
+
push(s.runsOtherShortcuts, String(a.params.WFWorkflowName ?? a.params.WFWorkflow?.workflowName ?? "another shortcut").slice(0, 60));
|
|
150
|
+
// a web request that carries a body is a way out, whatever the action is called
|
|
151
|
+
if (/downloadurl$/i.test(id)) {
|
|
152
|
+
const method = String(a.params.WFHTTPMethod ?? "GET").toUpperCase();
|
|
153
|
+
if (method !== "GET" || a.params.WFHTTPBodyType || a.params.WFFormValues || a.params.WFJSONValues || a.params.WFRequestVariable)
|
|
154
|
+
push(s.sendsOut, `sends a web request with a body (${method})`);
|
|
155
|
+
}
|
|
156
|
+
const found = [];
|
|
157
|
+
strings(a.params, "", found);
|
|
158
|
+
for (const f of found)
|
|
159
|
+
for (const m of f.text.matchAll(/\bhttps?:\/\/[^\s"'<>]+|\b[\w.+-]+@[\w-]+(?:\.[\w-]+)+\b|(?<!\d)\+\d[\d\s().-]{7,}\d/g))
|
|
160
|
+
push(s.destinations, m[0].slice(0, 120));
|
|
161
|
+
}
|
|
162
|
+
if (s.acceptsSharedInput)
|
|
163
|
+
push(s.outsideInput, "whatever is shared to it from other apps");
|
|
164
|
+
return { summary: s, actions };
|
|
165
|
+
}
|
|
166
|
+
export function adaptShortcut(raw) {
|
|
167
|
+
const segments = [];
|
|
168
|
+
const findings = [];
|
|
169
|
+
const add = (f) => findings.push({ pass: "channel", action: "none", ...f });
|
|
170
|
+
let doc;
|
|
171
|
+
const head = raw.trimStart();
|
|
172
|
+
if (head.startsWith("{")) {
|
|
173
|
+
try {
|
|
174
|
+
doc = JSON.parse(raw);
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
doc = undefined;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
else if (/^(?:<\?xml|<!DOCTYPE plist|<plist)/i.test(head)) {
|
|
181
|
+
try {
|
|
182
|
+
doc = parsePlistXml(raw);
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
doc = undefined;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (!looksLikeShortcut(doc)) {
|
|
189
|
+
// not a shortcut this reader understands: scanned as text, and said
|
|
190
|
+
segments.push({ text: raw, visibility: "visible", path: "automation/text" });
|
|
191
|
+
if (/^AEA1|^bplist00/.test(head))
|
|
192
|
+
add({ kind: "automation.unreadable", confidence: 0.1, severity: 0.2, message: head.startsWith("AEA1") ? "a signed shortcut is encrypted to Apple's format and its actions cannot be read here: nothing in it was checked" : "a binary property list: convert it with `plutil -convert json -o - <file>` and scan that" });
|
|
193
|
+
return { segments, findings };
|
|
194
|
+
}
|
|
195
|
+
const { summary: s, actions } = summarizeShortcut(doc);
|
|
196
|
+
const name = String(doc.WFWorkflowName ?? "");
|
|
197
|
+
// what it says about itself, and every piece of text inside it
|
|
198
|
+
segments.push({ text: [`Shortcut${name ? ` "${name}"` : ""}: ${s.actions} actions.`, s.readsPrivate.length ? `Reads: ${s.readsPrivate.join(", ")}.` : "", s.outsideInput.length ? `Takes in: ${s.outsideInput.join(", ")}.` : "", s.sendsOut.length ? `Can send out: ${s.sendsOut.join("; ")}.` : "", s.runsCode.length ? `Runs: ${s.runsCode.join(", ")}.` : "", s.destinations.length ? `Addresses written into it: ${s.destinations.slice(0, 12).join(", ")}.` : ""].filter(Boolean).join("\n"), visibility: "visible", path: "automation/summary" });
|
|
199
|
+
actions.forEach((a, i) => {
|
|
200
|
+
const found = [];
|
|
201
|
+
strings(a.params, "", found);
|
|
202
|
+
// a comment is written for the person; everything else is a parameter nobody reads in a long shortcut
|
|
203
|
+
for (const f of found)
|
|
204
|
+
segments.push({ text: f.text.replace(//g, " "), visibility: /comment$/i.test(a.id) ? "visible" : "attribute", path: `automation/action[${i}]:${short(a.id)}/${f.path}` });
|
|
205
|
+
});
|
|
206
|
+
if (s.readsPrivate.length && s.outsideInput.length && s.sendsOut.length) {
|
|
207
|
+
add({ kind: "automation.private_outside_send", confidence: 0.5, severity: 0.7, message: `this shortcut reads ${s.readsPrivate.slice(0, 3).join(", ")}, takes in ${s.outsideInput.slice(0, 2).join(" and ")}, and can send things out (${s.sendsOut.slice(0, 2).join("; ")}): content from outside can steer where your data goes` });
|
|
208
|
+
}
|
|
209
|
+
else if (s.readsPrivate.length && s.sendsOut.some((x) => /web request|script on another|posts to|uploads to/.test(x))) {
|
|
210
|
+
add({ kind: "automation.private_outside_send", confidence: 0.3, severity: 0.6, message: `this shortcut reads ${s.readsPrivate.slice(0, 3).join(", ")} and can send to a server (${s.sendsOut.filter((x) => /web request|script on another|posts to|uploads to/.test(x)).slice(0, 2).join("; ")})` });
|
|
211
|
+
}
|
|
212
|
+
if (s.runsCode.length)
|
|
213
|
+
add({ kind: "automation.runs_code", confidence: 0.35, severity: 0.6, message: `this shortcut runs ${s.runsCode.join(", ")}: read the script before installing it` });
|
|
214
|
+
if (s.runsOtherShortcuts.length)
|
|
215
|
+
add({ kind: "automation.runs_code", confidence: 0.1, severity: 0.3, message: `this shortcut runs other shortcuts (${s.runsOtherShortcuts.slice(0, 4).join(", ")}), which were not read` });
|
|
216
|
+
const tricks = s.destinations.filter((x) => /^https?:/i.test(x)).map((u) => [u, linkTrick(u)]).filter(([, t]) => t);
|
|
217
|
+
if (tricks.length)
|
|
218
|
+
add({ kind: "automation.destination", confidence: 0.4, severity: 0.6, message: `an address written into the shortcut is ${tricks[0][1]}`, excerpt: tricks[0][0].slice(0, 120) });
|
|
219
|
+
return { segments, findings };
|
|
220
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streams: content that arrives in pieces and is read as it arrives (server-sent events, agent-to-agent
|
|
3
|
+
* messages, live logs, chat feeds, live transcripts).
|
|
4
|
+
*
|
|
5
|
+
* There is no whole document to score before delivery, so the scanner keeps three promises instead:
|
|
6
|
+
*
|
|
7
|
+
* 1. Nothing is released before it has been scanned together with what came just before it. Each scan
|
|
8
|
+
* covers the unreleased text plus an overlap of text already released, so an instruction split across
|
|
9
|
+
* two chunks is seen whole.
|
|
10
|
+
* 2. The newest part is held back. A sentence that has only half arrived cannot be judged, so the last
|
|
11
|
+
* `holdBackChars` stay in the buffer until more arrives or the stream ends. An instruction shorter than
|
|
12
|
+
* the hold-back is never released at all once it is recognised.
|
|
13
|
+
* 3. A verdict that turns bad stops the stream. Nothing further is released, and the caller is told which
|
|
14
|
+
* already-released stretch belonged to the window that turned bad, so it can tell the agent that text
|
|
15
|
+
* is tainted too ("cut and retract").
|
|
16
|
+
*
|
|
17
|
+
* The cost is latency: the hold-back delays the newest text. Callers choose it.
|
|
18
|
+
*/
|
|
19
|
+
import type { IngressEngine } from "./engine.js";
|
|
20
|
+
import type { ContentKind, Finding, SourceInfo, Tier } from "./types.js";
|
|
21
|
+
export interface StreamOptions {
|
|
22
|
+
/** receipt source, e.g. "stream:sse:https://feed.example/events" */
|
|
23
|
+
url?: string;
|
|
24
|
+
/** how the text should be read; streams are text-like (text, markdown, log, captions, message) */
|
|
25
|
+
kind?: ContentKind;
|
|
26
|
+
source?: SourceInfo;
|
|
27
|
+
/** newest characters kept back until more arrives (default 600) */
|
|
28
|
+
holdBackChars?: number;
|
|
29
|
+
/** already-released characters rescanned with each new piece (default 1200) */
|
|
30
|
+
overlapChars?: number;
|
|
31
|
+
/** do not scan until this much unreleased text has arrived, unless the stream ends (default 400) */
|
|
32
|
+
minScanChars?: number;
|
|
33
|
+
/** the stream is refused once it has carried this much (default 20 MB) */
|
|
34
|
+
maxTotalChars?: number;
|
|
35
|
+
}
|
|
36
|
+
export interface StreamStep {
|
|
37
|
+
/** text that may now be handed to the model; "" when nothing is ready or the stream was cut */
|
|
38
|
+
release: string;
|
|
39
|
+
tier: Tier;
|
|
40
|
+
/** true once the stream has been stopped; every later step is cut too */
|
|
41
|
+
cut: boolean;
|
|
42
|
+
/** set on the step that cut the stream */
|
|
43
|
+
retract?: {
|
|
44
|
+
/** offset (in released characters) from which earlier text belonged to the window that turned bad */
|
|
45
|
+
fromOffset: number;
|
|
46
|
+
/** how many already-released characters that is */
|
|
47
|
+
chars: number;
|
|
48
|
+
findings: Finding[];
|
|
49
|
+
/** what to tell the agent */
|
|
50
|
+
notice: string;
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
export interface StreamSummary {
|
|
54
|
+
tier: Tier;
|
|
55
|
+
cut: boolean;
|
|
56
|
+
receivedChars: number;
|
|
57
|
+
releasedChars: number;
|
|
58
|
+
windows: number;
|
|
59
|
+
/** sha256 of everything released, so the caller can prove what the model was given */
|
|
60
|
+
releasedDigest: string;
|
|
61
|
+
}
|
|
62
|
+
export declare class StreamScanner {
|
|
63
|
+
private readonly engine;
|
|
64
|
+
private pending;
|
|
65
|
+
private releasedTail;
|
|
66
|
+
private releasedChars;
|
|
67
|
+
private received;
|
|
68
|
+
private windows;
|
|
69
|
+
private isCut;
|
|
70
|
+
private tier;
|
|
71
|
+
private hash;
|
|
72
|
+
private chain;
|
|
73
|
+
private readonly o;
|
|
74
|
+
constructor(engine: IngressEngine, opts?: StreamOptions);
|
|
75
|
+
/** Add the next piece. Steps are processed in order even if push is called again before the last one resolved. */
|
|
76
|
+
push(chunk: string): Promise<StreamStep>;
|
|
77
|
+
/** The stream is over: scan and release what was held back. */
|
|
78
|
+
end(): Promise<StreamStep>;
|
|
79
|
+
summary(): StreamSummary;
|
|
80
|
+
private step;
|
|
81
|
+
private cut;
|
|
82
|
+
}
|
|
83
|
+
/** Scan a complete text as if it had arrived in the given pieces. Convenience for tests and replays. */
|
|
84
|
+
export declare function scanStream(engine: IngressEngine, chunks: Iterable<string> | AsyncIterable<string>, opts?: StreamOptions): Promise<{
|
|
85
|
+
released: string;
|
|
86
|
+
steps: StreamStep[];
|
|
87
|
+
summary: StreamSummary;
|
|
88
|
+
}>;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streams: content that arrives in pieces and is read as it arrives (server-sent events, agent-to-agent
|
|
3
|
+
* messages, live logs, chat feeds, live transcripts).
|
|
4
|
+
*
|
|
5
|
+
* There is no whole document to score before delivery, so the scanner keeps three promises instead:
|
|
6
|
+
*
|
|
7
|
+
* 1. Nothing is released before it has been scanned together with what came just before it. Each scan
|
|
8
|
+
* covers the unreleased text plus an overlap of text already released, so an instruction split across
|
|
9
|
+
* two chunks is seen whole.
|
|
10
|
+
* 2. The newest part is held back. A sentence that has only half arrived cannot be judged, so the last
|
|
11
|
+
* `holdBackChars` stay in the buffer until more arrives or the stream ends. An instruction shorter than
|
|
12
|
+
* the hold-back is never released at all once it is recognised.
|
|
13
|
+
* 3. A verdict that turns bad stops the stream. Nothing further is released, and the caller is told which
|
|
14
|
+
* already-released stretch belonged to the window that turned bad, so it can tell the agent that text
|
|
15
|
+
* is tainted too ("cut and retract").
|
|
16
|
+
*
|
|
17
|
+
* The cost is latency: the hold-back delays the newest text. Callers choose it.
|
|
18
|
+
*/
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
20
|
+
export class StreamScanner {
|
|
21
|
+
engine;
|
|
22
|
+
pending = "";
|
|
23
|
+
releasedTail = "";
|
|
24
|
+
releasedChars = 0;
|
|
25
|
+
received = 0;
|
|
26
|
+
windows = 0;
|
|
27
|
+
isCut = false;
|
|
28
|
+
tier = "clean";
|
|
29
|
+
hash = createHash("sha256");
|
|
30
|
+
chain = Promise.resolve();
|
|
31
|
+
o;
|
|
32
|
+
constructor(engine, opts = {}) {
|
|
33
|
+
this.engine = engine;
|
|
34
|
+
this.o = { holdBackChars: 600, overlapChars: 1200, minScanChars: 400, maxTotalChars: 20_000_000, ...opts };
|
|
35
|
+
if (this.o.holdBackChars < 0 || this.o.overlapChars < 0)
|
|
36
|
+
throw new Error("StreamScanner: holdBackChars and overlapChars must not be negative");
|
|
37
|
+
}
|
|
38
|
+
/** Add the next piece. Steps are processed in order even if push is called again before the last one resolved. */
|
|
39
|
+
push(chunk) {
|
|
40
|
+
const next = this.chain.then(() => this.step(chunk, false));
|
|
41
|
+
this.chain = next.catch(() => undefined);
|
|
42
|
+
return next;
|
|
43
|
+
}
|
|
44
|
+
/** The stream is over: scan and release what was held back. */
|
|
45
|
+
end() {
|
|
46
|
+
const next = this.chain.then(() => this.step("", true));
|
|
47
|
+
this.chain = next.catch(() => undefined);
|
|
48
|
+
return next;
|
|
49
|
+
}
|
|
50
|
+
summary() {
|
|
51
|
+
return { tier: this.tier, cut: this.isCut, receivedChars: this.received, releasedChars: this.releasedChars, windows: this.windows, releasedDigest: this.hash.copy().digest("hex") };
|
|
52
|
+
}
|
|
53
|
+
async step(chunk, final) {
|
|
54
|
+
if (this.isCut)
|
|
55
|
+
return { release: "", tier: this.tier, cut: true };
|
|
56
|
+
this.pending += chunk;
|
|
57
|
+
this.received += chunk.length;
|
|
58
|
+
if (this.received > this.o.maxTotalChars)
|
|
59
|
+
return this.cut([{ pass: "policy", kind: "policy.denied_source", confidence: 0.5, severity: 0.5, message: `stream carried more than ${this.o.maxTotalChars} characters`, action: "none" }], "suspect");
|
|
60
|
+
if (!final && this.pending.length < this.o.minScanChars + this.o.holdBackChars)
|
|
61
|
+
return { release: "", tier: this.tier, cut: false };
|
|
62
|
+
if (!this.pending)
|
|
63
|
+
return { release: "", tier: this.tier, cut: false };
|
|
64
|
+
// scan the unreleased text together with the tail of what was already released
|
|
65
|
+
const window = this.releasedTail + this.pending;
|
|
66
|
+
this.windows++;
|
|
67
|
+
const r = await this.engine.scan(window, { url: this.o.url ?? "stream:", kind: this.o.kind ?? "text", source: this.o.source });
|
|
68
|
+
if (r.tier !== "clean")
|
|
69
|
+
return this.cut(r.findings, r.tier);
|
|
70
|
+
// release everything except the newest part, cut at a line or word boundary where there is one
|
|
71
|
+
let n = final ? this.pending.length : this.pending.length - this.o.holdBackChars;
|
|
72
|
+
if (!final && n > 0) {
|
|
73
|
+
const slice = this.pending.slice(0, n);
|
|
74
|
+
const nl = slice.lastIndexOf("\n");
|
|
75
|
+
const sp = slice.lastIndexOf(" ");
|
|
76
|
+
const at = nl >= n - 200 ? nl + 1 : sp >= n - 80 ? sp + 1 : n;
|
|
77
|
+
n = at;
|
|
78
|
+
}
|
|
79
|
+
if (n <= 0)
|
|
80
|
+
return { release: "", tier: this.tier, cut: false };
|
|
81
|
+
const release = this.pending.slice(0, n);
|
|
82
|
+
this.pending = this.pending.slice(n);
|
|
83
|
+
this.hash.update(release, "utf8");
|
|
84
|
+
this.releasedChars += release.length;
|
|
85
|
+
this.releasedTail = (this.releasedTail + release).slice(-this.o.overlapChars);
|
|
86
|
+
return { release, tier: "clean", cut: false };
|
|
87
|
+
}
|
|
88
|
+
cut(findings, tier) {
|
|
89
|
+
this.isCut = true;
|
|
90
|
+
this.tier = tier;
|
|
91
|
+
const chars = this.releasedTail.length;
|
|
92
|
+
const fromOffset = this.releasedChars - chars;
|
|
93
|
+
this.pending = "";
|
|
94
|
+
const kinds = [...new Set(findings.map((f) => f.kind))].join(", ");
|
|
95
|
+
const notice = chars
|
|
96
|
+
? `AcePaste Ingress stopped this stream: the newest part, read together with the last ${chars} characters already delivered, is ${tier} (${kinds}). Treat those ${chars} characters (from offset ${fromOffset}) as untrusted too: do not act on anything in them.`
|
|
97
|
+
: `AcePaste Ingress stopped this stream before delivering anything: it is ${tier} (${kinds}).`;
|
|
98
|
+
return { release: "", tier, cut: true, retract: { fromOffset, chars, findings, notice } };
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/** Scan a complete text as if it had arrived in the given pieces. Convenience for tests and replays. */
|
|
102
|
+
export async function scanStream(engine, chunks, opts = {}) {
|
|
103
|
+
const s = new StreamScanner(engine, opts);
|
|
104
|
+
const steps = [];
|
|
105
|
+
let released = "";
|
|
106
|
+
for await (const c of chunks) {
|
|
107
|
+
const st = await s.push(c);
|
|
108
|
+
steps.push(st);
|
|
109
|
+
released += st.release;
|
|
110
|
+
if (st.cut)
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
if (!s.summary().cut) {
|
|
114
|
+
const st = await s.end();
|
|
115
|
+
steps.push(st);
|
|
116
|
+
released += st.release;
|
|
117
|
+
}
|
|
118
|
+
return { released, steps, summary: s.summary() };
|
|
119
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Logs and terminal output.
|
|
3
|
+
*
|
|
4
|
+
* Two things make this channel different from a document:
|
|
5
|
+
*
|
|
6
|
+
* 1. Other people write parts of it. A web server's log holds whatever a visitor put in a User-Agent
|
|
7
|
+
* header; an error message quotes whatever input caused it. An agent asked to "look at the logs" is
|
|
8
|
+
* reading text from strangers.
|
|
9
|
+
*
|
|
10
|
+
* 2. A terminal does not show what the bytes say. Control codes can conceal text (SGR 8), paint it in the
|
|
11
|
+
* background colour, write a line and then erase or overwrite it, or clear the screen and the
|
|
12
|
+
* scrollback. A person watching the terminal sees the end state; a model reading the bytes sees
|
|
13
|
+
* everything. That gap is the one Ingress exists to close.
|
|
14
|
+
*
|
|
15
|
+
* This adapter replays the bytes the way a terminal would, far enough to tell what a person would still
|
|
16
|
+
* see from what was hidden from them. Hidden text becomes a concealed segment (so the meaning pass reads it
|
|
17
|
+
* as hidden), and control codes that act on the person's machine rather than the display (clipboard writes,
|
|
18
|
+
* link text that names a different address) are reported.
|
|
19
|
+
*/
|
|
20
|
+
import type { Finding, TextSegment } from "./types.js";
|
|
21
|
+
export interface TerminalAdapterResult {
|
|
22
|
+
segments: TextSegment[];
|
|
23
|
+
findings: Finding[];
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Replay terminal output. Returns what a person would still see, and what was hidden from them.
|
|
27
|
+
* Text without control codes comes back unchanged as one visible segment.
|
|
28
|
+
*/
|
|
29
|
+
export declare function adaptTerminal(raw: string): TerminalAdapterResult;
|