@intentic/webq 1.331.0 → 1.333.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/http.d.ts.map +1 -1
- package/dist/lib/http.js +2 -35
- package/dist/lib/http.js.map +1 -1
- package/package.json +4 -4
- package/src/lib/http.ts +3 -37
package/dist/lib/http.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC/B;AAID,eAAO,MAAM,UAAU,+GAA+G,CAAC;AAKvI,eAAO,MAAM,SAAS,QAAe,MAAM,YAAW,gBAAgB,KAAQ,OAAO,CAAC,QAAQ,CAsB7F,CAAC"}
|
package/dist/lib/http.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readCappedBytes } from "@intentic/base/capped-read";
|
|
1
2
|
export const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 webq";
|
|
2
3
|
const DEFAULT_TIMEOUT_MS = 20_000;
|
|
3
4
|
const DEFAULT_MAX_BYTES = 10 * 1024 * 1024;
|
|
@@ -14,7 +15,7 @@ export const httpFetch = async (url, options = {}) => {
|
|
|
14
15
|
},
|
|
15
16
|
});
|
|
16
17
|
const contentType = response.headers.get("content-type") ?? "";
|
|
17
|
-
const { bytes, truncated } = await
|
|
18
|
+
const { bytes, truncated } = await readCappedBytes(response, maxBytes, { timeoutMs });
|
|
18
19
|
return {
|
|
19
20
|
url,
|
|
20
21
|
finalUrl: response.url === "" ? url : response.url,
|
|
@@ -24,40 +25,6 @@ export const httpFetch = async (url, options = {}) => {
|
|
|
24
25
|
truncated,
|
|
25
26
|
};
|
|
26
27
|
};
|
|
27
|
-
const readCapped = async (response, maxBytes, timeoutMs) => {
|
|
28
|
-
if (response.body === null) {
|
|
29
|
-
return { bytes: new Uint8Array(0), truncated: false };
|
|
30
|
-
}
|
|
31
|
-
const deadline = Date.now() + timeoutMs;
|
|
32
|
-
const chunks = [];
|
|
33
|
-
let total = 0;
|
|
34
|
-
const reader = response.body.getReader();
|
|
35
|
-
for (;;) {
|
|
36
|
-
if (Date.now() > deadline) {
|
|
37
|
-
await reader.cancel();
|
|
38
|
-
return { bytes: concat(chunks, total), truncated: true };
|
|
39
|
-
}
|
|
40
|
-
const { done, value } = await reader.read();
|
|
41
|
-
if (done) {
|
|
42
|
-
return { bytes: concat(chunks, total), truncated: false };
|
|
43
|
-
}
|
|
44
|
-
chunks.push(value);
|
|
45
|
-
total += value.byteLength;
|
|
46
|
-
if (total >= maxBytes) {
|
|
47
|
-
await reader.cancel();
|
|
48
|
-
return { bytes: concat(chunks, total).slice(0, maxBytes), truncated: true };
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
};
|
|
52
|
-
const concat = (chunks, total) => {
|
|
53
|
-
const merged = new Uint8Array(total);
|
|
54
|
-
let offset = 0;
|
|
55
|
-
for (const chunk of chunks) {
|
|
56
|
-
merged.set(chunk, offset);
|
|
57
|
-
offset += chunk.byteLength;
|
|
58
|
-
}
|
|
59
|
-
return merged;
|
|
60
|
-
};
|
|
61
28
|
const decode = (bytes, contentType) => {
|
|
62
29
|
const headerCharset = /charset=["']?([\w-]+)/i.exec(contentType)?.[1];
|
|
63
30
|
const sniffed = headerCharset ?? sniffMetaCharset(bytes);
|
package/dist/lib/http.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAqB7D,MAAM,CAAC,MAAM,UAAU,GAAG,4GAA4G,CAAC;AAEvI,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,GAAW,EAAE,OAAO,GAAqB,EAAE,EAAqB,EAAE;IAC9F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACvD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAC9B,QAAQ,EAAE,QAAQ;QAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;QACtC,OAAO,EAAE;YACL,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,UAAU;YAC7C,MAAM,EAAE,kFAAkF;YAC1F,iBAAiB,EAAE,gBAAgB;SACtC;KACJ,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACtF,OAAO;QACH,GAAG;QACH,QAAQ,EAAE,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG;QAClD,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,WAAW;QACX,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC;QAChC,SAAS;KACZ,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAiB,EAAE,WAAmB,EAAU,EAAE;IAC9D,MAAM,aAAa,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,aAAa,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACzD,IAAI,CAAC;QACD,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpE,CAAC;AACL,CAAC,CAAC;AAGF,MAAM,gBAAgB,GAAG,CAAC,KAAiB,EAAsB,EAAE;IAC/D,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACpE,OAAO,CAAC,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAClH,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@intentic/webq",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.333.0",
|
|
4
4
|
"description": "webq, agent-native web fetch CLI: any URL as clean budgeted markdown, and bounded same-site crawls, over one shared cache",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -45,8 +45,8 @@
|
|
|
45
45
|
}
|
|
46
46
|
},
|
|
47
47
|
"dependencies": {
|
|
48
|
-
"@intentic/agent-cli": "1.
|
|
49
|
-
"@intentic/base": "1.
|
|
48
|
+
"@intentic/agent-cli": "1.333.0",
|
|
49
|
+
"@intentic/base": "1.333.0",
|
|
50
50
|
"@stricli/core": "1.3.0",
|
|
51
51
|
"parse5": "8.0.1",
|
|
52
52
|
"playwright": "1.63.0",
|
|
@@ -61,7 +61,7 @@
|
|
|
61
61
|
},
|
|
62
62
|
"scripts": {
|
|
63
63
|
"build": "tsgo",
|
|
64
|
-
"typecheck": "
|
|
64
|
+
"typecheck": "bun check -p tsconfig.test.json",
|
|
65
65
|
"watch": "tsgo --build --watch",
|
|
66
66
|
"test": "suites",
|
|
67
67
|
"test:watch": "suites --watch"
|
package/src/lib/http.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/* The static fetch path — where every page starts, and where most docs pages end. */
|
|
2
2
|
|
|
3
|
+
import { readCappedBytes } from "@intentic/base/capped-read";
|
|
4
|
+
|
|
3
5
|
export interface HttpFetchOptions {
|
|
4
6
|
readonly timeoutMs?: number;
|
|
5
7
|
readonly maxBytes?: number;
|
|
@@ -37,7 +39,7 @@ export const httpFetch = async (url: string, options: HttpFetchOptions = {}): Pr
|
|
|
37
39
|
},
|
|
38
40
|
});
|
|
39
41
|
const contentType = response.headers.get("content-type") ?? "";
|
|
40
|
-
const { bytes, truncated } = await
|
|
42
|
+
const { bytes, truncated } = await readCappedBytes(response, maxBytes, { timeoutMs });
|
|
41
43
|
return {
|
|
42
44
|
url,
|
|
43
45
|
finalUrl: response.url === "" ? url : response.url,
|
|
@@ -48,42 +50,6 @@ export const httpFetch = async (url: string, options: HttpFetchOptions = {}): Pr
|
|
|
48
50
|
};
|
|
49
51
|
};
|
|
50
52
|
|
|
51
|
-
const readCapped = async (response: Response, maxBytes: number, timeoutMs: number): Promise<{ bytes: Uint8Array; truncated: boolean }> => {
|
|
52
|
-
if (response.body === null) {
|
|
53
|
-
return { bytes: new Uint8Array(0), truncated: false };
|
|
54
|
-
}
|
|
55
|
-
const deadline = Date.now() + timeoutMs;
|
|
56
|
-
const chunks: Uint8Array[] = [];
|
|
57
|
-
let total = 0;
|
|
58
|
-
const reader = response.body.getReader();
|
|
59
|
-
for (;;) {
|
|
60
|
-
if (Date.now() > deadline) {
|
|
61
|
-
await reader.cancel();
|
|
62
|
-
return { bytes: concat(chunks, total), truncated: true };
|
|
63
|
-
}
|
|
64
|
-
const { done, value } = await reader.read();
|
|
65
|
-
if (done) {
|
|
66
|
-
return { bytes: concat(chunks, total), truncated: false };
|
|
67
|
-
}
|
|
68
|
-
chunks.push(value);
|
|
69
|
-
total += value.byteLength;
|
|
70
|
-
if (total >= maxBytes) {
|
|
71
|
-
await reader.cancel();
|
|
72
|
-
return { bytes: concat(chunks, total).slice(0, maxBytes), truncated: true };
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
};
|
|
76
|
-
|
|
77
|
-
const concat = (chunks: Uint8Array[], total: number): Uint8Array => {
|
|
78
|
-
const merged = new Uint8Array(total);
|
|
79
|
-
let offset = 0;
|
|
80
|
-
for (const chunk of chunks) {
|
|
81
|
-
merged.set(chunk, offset);
|
|
82
|
-
offset += chunk.byteLength;
|
|
83
|
-
}
|
|
84
|
-
return merged;
|
|
85
|
-
};
|
|
86
|
-
|
|
87
53
|
const decode = (bytes: Uint8Array, contentType: string): string => {
|
|
88
54
|
const headerCharset = /charset=["']?([\w-]+)/i.exec(contentType)?.[1];
|
|
89
55
|
const sniffed = headerCharset ?? sniffMetaCharset(bytes);
|