@stratusagent/tool-web 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +40 -15
- package/dist/index.js.map +1 -1
- package/dist/readability.d.ts.map +1 -1
- package/dist/readability.js +64 -1
- package/dist/readability.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -44,8 +44,8 @@ has approved that URL — not the redirect it answers with.
|
|
|
44
44
|
| --- | --- | --- |
|
|
45
45
|
| `allowedHosts` | none | Hosts exempt from the address check, by name or literal address. The narrow override: one internal service an agent is meant to reach. |
|
|
46
46
|
| `allowPrivateAddresses` | `false` | Reach any non-global address. The trusted-workstation posture — it turns the SSRF protection off rather than adjusting it. |
|
|
47
|
-
| `maxBytes` | `400000` | Stop reading here; the result says `truncated`. |
|
|
48
|
-
| `timeoutMs` | `20000` | Give up on the whole exchange. |
|
|
47
|
+
| `maxBytes` | `400000` | Stop reading here; the result says `truncated`. A call's own `maxBytes` may ask for less, never more. |
|
|
48
|
+
| `timeoutMs` | `20000` | Give up on the whole exchange — every redirect hop draws on the one budget. |
|
|
49
49
|
| `maxRedirects` | `5` | Hops to follow before refusing. |
|
|
50
50
|
| `userAgent` | `StratusAgent/0.5 …` | What to send. |
|
|
51
51
|
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,EAAiB,MAAM,oBAAoB,CAAC;AAC5E,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,EAAiB,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,sBAAsB,CAAC;AAK9B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAO5E,MAAM,WAAW,eAAgB,SAAQ,UAAU;IACjD,sEAAsE;IACtE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,uEAAuE;IACvE,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAiCD;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,GAC7B,QAAQ,MAAM,EACd,UAAS;IACP,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;CACjB,KACL,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,CA0DhH,CAAC;AAkDF;;;;;;;;GAQG;AACH,eAAO,MAAM,eAAe,GAAI,SAAQ,UAAe,KAAG,MAKxD,CAAC;AAEH,4DAA4D;AAC5D,eAAO,MAAM,YAAY,YARe,UAAU,KAAQ,MAQf,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { assertRequestAllowed, requestThroughPolicy, } from '@stratusagent/egress';
|
|
1
|
+
import { assertRequestAllowed, egressPolicyFrom, requestThroughPolicy, } from '@stratusagent/egress';
|
|
2
2
|
import { resolvePluginAgentConfig } from '@stratusagent/plugins';
|
|
3
3
|
import { extractTitle, htmlToText } from "./readability.js";
|
|
4
4
|
export { decodeEntities, extractTitle, htmlToText } from "./readability.js";
|
|
@@ -7,17 +7,19 @@ const DEFAULT_TIMEOUT_MS = 20_000;
|
|
|
7
7
|
const DEFAULT_MAX_REDIRECTS = 5;
|
|
8
8
|
const DEFAULT_USER_AGENT = 'StratusAgent/0.5 (+https://github.com/stratuslabs/agent)';
|
|
9
9
|
const asNumber = (value, fallback) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : fallback;
|
|
10
|
+
/**
|
|
11
|
+
* A per-call `maxBytes` may narrow the operator's cap, never raise it. A
|
|
12
|
+
* cap the model can lift is not one: a fetch of a 300 MB body asked for
|
|
13
|
+
* `maxBytes: 50000000`, got 50 MB of it, and took the daemon from 278 MB to
|
|
14
|
+
* 1.1 GB resident on the way.
|
|
15
|
+
*/
|
|
16
|
+
const narrowed = (requested, cap) => Math.min(asNumber(requested, cap), cap);
|
|
10
17
|
const settingsFor = (config, session) => {
|
|
11
18
|
// Per call, from the session: an agent allowed to reach an internal host
|
|
12
19
|
// is a decision about that agent, and closing over it at setup would
|
|
13
20
|
// give the exemption to everyone.
|
|
14
21
|
const resolved = resolvePluginAgentConfig(config, session.agent.id);
|
|
15
|
-
const policy =
|
|
16
|
-
...(resolved.allowPrivateAddresses === true ? { allowPrivateAddresses: true } : {}),
|
|
17
|
-
...(Array.isArray(resolved.allowedHosts)
|
|
18
|
-
? { allowedHosts: resolved.allowedHosts.filter((entry) => typeof entry === 'string') }
|
|
19
|
-
: {}),
|
|
20
|
-
};
|
|
22
|
+
const policy = egressPolicyFrom(resolved);
|
|
21
23
|
return {
|
|
22
24
|
policy,
|
|
23
25
|
maxBytes: asNumber(resolved.maxBytes, DEFAULT_MAX_BYTES),
|
|
@@ -42,15 +44,38 @@ export const fetchThroughPolicy = async (rawUrl, options = {}) => {
|
|
|
42
44
|
const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
43
45
|
const hops = [];
|
|
44
46
|
let current = assertRequestAllowed(rawUrl, options.policy ?? {}).href;
|
|
47
|
+
// One budget for the whole exchange, not one per hop. Each hop is its own
|
|
48
|
+
// request with its own timer, and a fresh timer per hop let a chain of
|
|
49
|
+
// slow redirects multiply the configured timeout by the redirect limit:
|
|
50
|
+
// four hops at three seconds each answered after twelve seconds under a
|
|
51
|
+
// five-second timeout, which is not what "give up on the whole exchange"
|
|
52
|
+
// promises.
|
|
53
|
+
const deadline = options.timeoutMs === undefined ? undefined : Date.now() + options.timeoutMs;
|
|
54
|
+
const timedOut = () => new Error(`Timed out after ${options.timeoutMs}ms across ${hops.length - 1} redirect(s) starting at ${rawUrl}`);
|
|
45
55
|
for (let hop = 0; hop <= maxRedirects; hop += 1) {
|
|
46
56
|
hops.push(current);
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
const remainingMs = deadline === undefined ? undefined : deadline - Date.now();
|
|
58
|
+
if (remainingMs !== undefined && remainingMs <= 0) {
|
|
59
|
+
throw timedOut();
|
|
60
|
+
}
|
|
61
|
+
let response;
|
|
62
|
+
try {
|
|
63
|
+
response = await requestThroughPolicy(current, {
|
|
64
|
+
...(options.policy ? { policy: options.policy } : {}),
|
|
65
|
+
headers: { 'user-agent': options.userAgent ?? DEFAULT_USER_AGENT, accept: 'text/html,text/plain;q=0.9,*/*;q=0.5' },
|
|
66
|
+
...(options.maxBytes === undefined ? {} : { maxBytes: options.maxBytes }),
|
|
67
|
+
...(remainingMs === undefined ? {} : { timeoutMs: remainingMs }),
|
|
68
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
// A hop that ran out the shared budget reports the budget, not the
|
|
73
|
+
// sliver of it that this hop was left.
|
|
74
|
+
if (deadline !== undefined && hops.length > 1 && Date.now() >= deadline) {
|
|
75
|
+
throw timedOut();
|
|
76
|
+
}
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
54
79
|
const location = response.headers.location;
|
|
55
80
|
if (isRedirect(response.status) && typeof location === 'string' && location.length > 0) {
|
|
56
81
|
// Resolved against the current URL, then checked from scratch — a
|
|
@@ -93,7 +118,7 @@ const createFetchTool = (config) => ({
|
|
|
93
118
|
const settings = settingsFor(config, session);
|
|
94
119
|
const response = await fetchThroughPolicy(url, {
|
|
95
120
|
policy: settings.policy,
|
|
96
|
-
maxBytes:
|
|
121
|
+
maxBytes: narrowed(input.maxBytes, settings.maxBytes),
|
|
97
122
|
timeoutMs: settings.timeoutMs,
|
|
98
123
|
maxRedirects: settings.maxRedirects,
|
|
99
124
|
userAgent: settings.userAgent,
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,oBAAoB,EACpB,oBAAoB,GAErB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE5D,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE5E,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAClC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAChC,MAAM,kBAAkB,GAAG,0DAA0D,CAAC;AAatF,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAE,QAAgB,EAAU,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAEtF,MAAM,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,GAErB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE5D,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE5E,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAClC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAChC,MAAM,kBAAkB,GAAG,0DAA0D,CAAC;AAatF,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAE,QAAgB,EAAU,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAEtF;;;;;GAKG;AACH,MAAM,QAAQ,GAAG,CAAC,SAAkB,EAAE,GAAW,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;AAEtG,MAAM,WAAW,GAAG,CAAC,MAAkB,EAAE,OAAgB,EAAE,EAAE;IAC3D,yEAAyE;IACzE,qEAAqE;IACrE,kCAAkC;IAClC,MAAM,QAAQ,GAAG,wBAAwB,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACpE,MAAM,MAAM,GAAiB,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACxD,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAC;QACxD,SAAS,EAAE,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;QAC3D,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,EAAE,qBAAqB,CAAC;QACpE,SAAS,EAAE,OAAO,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB;KAC5F,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,MAAc,EAAW,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAC;AAE9E,MAAM,aAAa,GAAG,CAAC,OAAgC,EAAU,EAAE,CACjE,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAEtD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,MAAc,EACd,UAOI,EAAE,EAC2G,EAAE;IACnH,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,qBAAqB,CAAC;IACnE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,OAAO,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;IACtE,0EAA0E;IAC1E,uEAAuE;IACvE,wEAAwE;IACxE,wEAAwE;IACxE,yEAAyE;IACzE,YAAY;IACZ,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IAC9F,MAAM,QAAQ,GAAG,GAAU,EAAE,CAC3B,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,SAAS,aAAa,IAAI,CAAC,MAAM,GAAG,CAAC,4BAA4B,MAAM,EAAE,CAAC,CAAC;IAElH,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACnB,MAAM,WAAW,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/E,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;YAClD,MAAM,QAAQ,EAAE,CAAC;QACnB,CAAC;QACD,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE;gBAC7C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB,EAAE,MAAM,EAAE,sCAAsC,EAAE;gBAClH,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACzE,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;gBAChE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtD,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,mEAAmE;YACnE,uCAAuC;YACvC,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;gBACxE,MAAM,QAAQ,EAAE,CAAC;YACnB,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC3C,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvF,kEAAkE;YAClE,gDAAgD;YAChD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAC7C,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;YAChE,SAAS;QACX,CAAC;QAED,OAAO;YACL,GAAG,EAAE,OAAO;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,aAAa,CAAC,QAAQ,CAAC,OAAkC,CAAC;YACvE,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,IAAI;SACL,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,iBAAiB,MAAM,EAAE,CAAC,CAAC;AAChF,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,MAAkB,EAAQ,EAAE,CAAC,CAAC;IACrD,IAAI,EAAE,WAAW;IACjB,WAAW,EAAE,yEAAyE;IACtF,sEAAsE;IACtE,yEAAyE;IACzE,IAAI,EAAE,OAAO;IACb,UAAU,EAAE;QACV,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACvB,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5B,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,uDAAuD,EAAE;SAC/F;QACD,QAAQ,EAAE,CAAC,KAAK,CAAC;KAClB;IACD,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO;QACnC,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE;YAC7C,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC;YACrD,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnD,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrF,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAE7D,OAAO;YACL,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,sEAAsE;YACtE,iEAAiE;YACjE,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,IAAI;YACJ,SAAS,EAAE,QAAQ,CAAC,SAAS;SAC9B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,SAAqB,EAAE,EAAU,EAAE,CAAC,CAAC;IACnE,IAAI,EAAE,wBAAwB;IAC9B,KAAK,CAAC,OAAO;QACX,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;IAClD,CAAC;CACF,CAAC,CAAC;AAEH,4DAA4D;AAC5D,MAAM,CAAC,MAAM,YAAY,GAAG,eAAe,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readability.d.ts","sourceRoot":"","sources":["../src/readability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;
|
|
1
|
+
{"version":3,"file":"readability.d.ts","sourceRoot":"","sources":["../src/readability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAwEH,eAAO,MAAM,cAAc,GAAI,OAAO,MAAM,KAAG,MAW3C,CAAC;AAEL,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,KAAG,MAAM,GAAG,SAGpD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,MAAM,MAAM,KAAG,MA2DzC,CAAC"}
|
package/dist/readability.js
CHANGED
|
@@ -27,6 +27,37 @@ const DROPPED_ELEMENTS = [
|
|
|
27
27
|
'aside',
|
|
28
28
|
'form',
|
|
29
29
|
];
|
|
30
|
+
/**
|
|
31
|
+
* Formatting that wraps text which was already adjacent, so removing it
|
|
32
|
+
* joins nothing that was apart. Deliberately a closed set: everything not
|
|
33
|
+
* named here becomes a space, which is the safe direction — see the
|
|
34
|
+
* substitution below.
|
|
35
|
+
*/
|
|
36
|
+
const INLINE_ELEMENTS = new Set([
|
|
37
|
+
'a', 'abbr', 'b', 'bdi', 'bdo', 'big', 'cite', 'code', 'data', 'del', 'dfn',
|
|
38
|
+
'em', 'font', 'i', 'ins', 'kbd', 'mark', 'nobr', 'ruby',
|
|
39
|
+
's', 'samp', 'small', 'span', 'strike', 'strong', 'sub', 'sup', 'time',
|
|
40
|
+
'tt', 'u', 'var', 'wbr',
|
|
41
|
+
]);
|
|
42
|
+
// `rt` and `rp` are deliberately absent, though they are inline elements.
|
|
43
|
+
// They hold a ruby annotation — a pronunciation printed *above* the base
|
|
44
|
+
// text, not beside it — so joining them produces a token the page never
|
|
45
|
+
// shows: `<ruby>東京<rt>とうきょう</rt></ruby>` reads back as 東京とうきょう,
|
|
46
|
+
// fusing a word with its own furigana. Separating loses nothing and fuses
|
|
47
|
+
// nothing, which is the safer of the two ways to be wrong here; dropping
|
|
48
|
+
// the annotation outright would read better still and would be a guess
|
|
49
|
+
// about whether the pronunciation is content.
|
|
50
|
+
//
|
|
51
|
+
// `sub` and `sup` stay, and are the deliberate contrast: `H<sub>2</sub>O`
|
|
52
|
+
// is one token and joining it is the whole point.
|
|
53
|
+
/**
|
|
54
|
+
* `q` renders a character of its own — a browser draws quotation marks
|
|
55
|
+
* around it from the stylesheet — so neither answer above is right for it.
|
|
56
|
+
* Removing it fuses `<q>yes</q><q>no</q>` into `yesno`; spacing it detaches
|
|
57
|
+
* the comma in `<q>yes</q>, then left`. Emitting the mark a reader sees
|
|
58
|
+
* avoids both and is the more faithful extraction.
|
|
59
|
+
*/
|
|
60
|
+
const QUOTE_ELEMENT = 'q';
|
|
30
61
|
const BLOCK_ELEMENTS = [
|
|
31
62
|
'p', 'div', 'section', 'article', 'main', 'br', 'hr',
|
|
32
63
|
'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
|
|
@@ -63,7 +94,13 @@ export const extractTitle = (html) => {
|
|
|
63
94
|
return match?.[1] ? decodeEntities(match[1]).trim() : undefined;
|
|
64
95
|
};
|
|
65
96
|
export const htmlToText = (html) => {
|
|
97
|
+
// Comments first, then the doctype and any processing instruction. These
|
|
98
|
+
// are removed by name rather than by a blanket `<[^>]*>` sweep, because
|
|
99
|
+
// that sweep reads `5 < 10 and 20 > 15` as a tag and deletes the middle
|
|
100
|
+
// of the sentence — prose about arbitrary subjects is exactly what a
|
|
101
|
+
// fetched page is.
|
|
66
102
|
let text = html.replace(/<!--[\s\S]*?-->/g, '');
|
|
103
|
+
text = text.replace(/<[!?][^>]*>/g, ' ');
|
|
67
104
|
for (const element of DROPPED_ELEMENTS) {
|
|
68
105
|
text = text.replace(new RegExp(`<${element}\\b[^>]*>[\\s\\S]*?</${element}>`, 'gi'), ' ');
|
|
69
106
|
// Unclosed or self-closing forms of the same elements.
|
|
@@ -74,7 +111,33 @@ export const htmlToText = (html) => {
|
|
|
74
111
|
for (const element of BLOCK_ELEMENTS) {
|
|
75
112
|
text = text.replace(new RegExp(`</?${element}\\b[^>]*>`, 'gi'), '\n');
|
|
76
113
|
}
|
|
77
|
-
|
|
114
|
+
// Inline formatting is removed; every other tag becomes a space.
|
|
115
|
+
//
|
|
116
|
+
// A space around formatting rewrites the page — `un<em>expected</em>`
|
|
117
|
+
// becomes `un expected`, a word the page does not contain, and
|
|
118
|
+
// `<strong>kettle</strong>.` becomes `kettle .` — and a model reading the
|
|
119
|
+
// extraction cannot tell either from the real thing.
|
|
120
|
+
//
|
|
121
|
+
// The default runs the other way on purpose. Naming the *separators*
|
|
122
|
+
// instead and deleting the rest looks equivalent and is not: the list is
|
|
123
|
+
// never complete. `td`, `dd`, and `option` are all missing from the block
|
|
124
|
+
// list above, and a custom element is missing from every list anybody
|
|
125
|
+
// will write — so that spelling silently glued `AlphaBeta` out of two
|
|
126
|
+
// table cells. Inline formatting is a closed set that has not grown in
|
|
127
|
+
// twenty years, so listing it is the half that can be finished, and an
|
|
128
|
+
// unrecognised tag falls to a space: a seam that was not needed is a
|
|
129
|
+
// blemish, while one that was is a word nobody wrote.
|
|
130
|
+
//
|
|
131
|
+
// The name is captured whole, punctuation included. Stopping at the first
|
|
132
|
+
// character outside `[a-zA-Z0-9-]` reads `<a:widget>` as the inline `a`
|
|
133
|
+
// and deletes it — and a namespaced element is precisely the unknown tag
|
|
134
|
+
// this default exists for.
|
|
135
|
+
text = text.replace(/<\/?([a-zA-Z][^\s/>]*)[^>]*>/g, (_tag, rawName) => {
|
|
136
|
+
const name = rawName.toLowerCase();
|
|
137
|
+
if (name === QUOTE_ELEMENT)
|
|
138
|
+
return '"';
|
|
139
|
+
return INLINE_ELEMENTS.has(name) ? '' : ' ';
|
|
140
|
+
});
|
|
78
141
|
text = decodeEntities(text);
|
|
79
142
|
return text
|
|
80
143
|
.split('\n')
|
package/dist/readability.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readability.js","sourceRoot":"","sources":["../src/readability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,gBAAgB,GAAG;IACvB,QAAQ;IACR,OAAO;IACP,UAAU;IACV,UAAU;IACV,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,MAAM;CACP,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;IACpD,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IAClC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK;CAC/C,CAAC;AAEF,MAAM,cAAc,GAA2B;IAC7C,GAAG,EAAE,GAAG;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,GAAG;IACX,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;CACX,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,KAAa,EAAU,EAAE,CACtD,KAAK,CAAC,OAAO,CAAC,gCAAgC,EAAE,CAAC,KAAK,EAAE,MAAc,EAAE,EAAE;IACxE,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACpE,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACpE,CAAC;IACD,OAAO,cAAc,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,KAAK,CAAC;AACvD,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAY,EAAsB,EAAE;IAC/D,MAAM,KAAK,GAAG,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5D,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAClE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,IAAY,EAAU,EAAE;IACjD,IAAI,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"readability.js","sourceRoot":"","sources":["../src/readability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,gBAAgB,GAAG;IACvB,QAAQ;IACR,OAAO;IACP,UAAU;IACV,UAAU;IACV,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,MAAM;CACP,CAAC;AAEF;;;;;GAKG;AACH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;IAC3E,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACvD,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM;IACtE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK;CACxB,CAAC,CAAC;AACH,0EAA0E;AAC1E,yEAAyE;AACzE,wEAAwE;AACxE,gEAAgE;AAChE,0EAA0E;AAC1E,yEAAyE;AACzE,uEAAuE;AACvE,8CAA8C;AAC9C,EAAE;AACF,0EAA0E;AAC1E,kDAAkD;AAElD;;;;;;GAMG;AACH,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,MAAM,cAAc,GAAG;IACrB,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;IACpD,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IAClC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK;CAC/C,CAAC;AAEF,MAAM,cAAc,GAA2B;IAC7C,GAAG,EAAE,GAAG;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,GAAG;IACX,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;CACX,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,KAAa,EAAU,EAAE,CACtD,KAAK,CAAC,OAAO,CAAC,gCAAgC,EAAE,CAAC,KAAK,EAAE,MAAc,EAAE,EAAE;IACxE,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACpE,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACpE,CAAC;IACD,OAAO,cAAc,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,KAAK,CAAC;AACvD,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAY,EAAsB,EAAE;IAC/D,MAAM,KAAK,GAAG,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5D,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAClE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,IAAY,EAAU,EAAE;IACjD,yEAAyE;IACzE,wEAAwE;IACxE,wEAAwE;IACxE,qEAAqE;IACrE,mBAAmB;IACnB,IAAI,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAChD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;QACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,OAAO,wBAAwB,OAAO,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1F,uDAAuD;QACvD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,OAAO,aAAa,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;IAE5D,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC7C,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,OAAO,WAAW,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IACxE,CAAC;IACD,iEAAiE;IACjE,EAAE;IACF,sEAAsE;IACtE,+DAA+D;IAC/D,0EAA0E;IAC1E,qDAAqD;IACrD,EAAE;IACF,qEAAqE;IACrE,yEAAyE;IACzE,0EAA0E;IAC1E,sEAAsE;IACtE,sEAAsE;IACtE,uEAAuE;IACvE,uEAAuE;IACvE,qEAAqE;IACrE,sDAAsD;IACtD,EAAE;IACF,0EAA0E;IAC1E,wEAAwE;IACxE,yEAAyE;IACzE,2BAA2B;IAC3B,IAAI,GAAG,IAAI,CAAC,OAAO,CACjB,+BAA+B,EAC/B,CAAC,IAAI,EAAE,OAAe,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACnC,IAAI,IAAI,KAAK,aAAa;YAAE,OAAO,GAAG,CAAC;QACvC,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,CAAC,CACF,CAAC;IAEF,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAE5B,OAAO,IAAI;SACR,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;SACnD,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SACtF,IAAI,CAAC,IAAI,CAAC;SACV,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@stratusagent/tool-web",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -18,9 +18,9 @@
|
|
|
18
18
|
"node": ">=22.13 <23 || >=23.4"
|
|
19
19
|
},
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@stratusagent/core": "0.
|
|
22
|
-
"@stratusagent/
|
|
23
|
-
"@stratusagent/
|
|
21
|
+
"@stratusagent/core": "0.11.1",
|
|
22
|
+
"@stratusagent/egress": "0.11.1",
|
|
23
|
+
"@stratusagent/plugins": "0.11.1"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@types/node": "^24.6.0",
|