@sasasin/fetch-url 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +8 -3
- package/package.json +1 -2
- package/tests/selftest.mjs +0 -57
- package/tests/test-load.mjs +0 -56
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.1.1
|
|
4
|
+
|
|
5
|
+
- `tests/` is no longer shipped in the npm tarball. Node refuses to strip TypeScript types for
|
|
6
|
+
files under `node_modules`, so `node tests/selftest.mjs` could never work from an `npm:`-installed
|
|
7
|
+
copy (`ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING`), and `tests/test-load.mjs` needs jiti, which
|
|
8
|
+
lives in `devDependencies`. The scripts stay in the repository and run from a git clone
|
|
9
|
+
(`~/.pi/agent/git/github.com/sasasin/fetch_url`) or the source tree.
|
|
10
|
+
- README: the self-test section now says where the scripts can run, and notes that the extension
|
|
11
|
+
itself is unaffected because pi loads it through jiti.
|
|
12
|
+
|
|
3
13
|
## 0.1.0
|
|
4
14
|
|
|
5
15
|
Initial public release.
|
package/README.md
CHANGED
|
@@ -126,8 +126,13 @@ in-extension self-update command:
|
|
|
126
126
|
|
|
127
127
|
## Self-test without pi
|
|
128
128
|
|
|
129
|
-
|
|
130
|
-
|
|
129
|
+
Run these from a git clone (the git source lives at `~/.pi/agent/git/github.com/sasasin/fetch_url`)
|
|
130
|
+
or from the source tree — **not** from the `npm:` installed copy: Node refuses to strip types for
|
|
131
|
+
files under `node_modules`, so `selftest.mjs` cannot import `fetch-core.ts` there. The extension
|
|
132
|
+
itself is unaffected, because pi loads it through jiti. Hence `tests/` is kept out of the npm
|
|
133
|
+
tarball.
|
|
134
|
+
|
|
135
|
+
Needs Node 22.19+ (the same floor as pi itself — `selftest.mjs` relies on native type stripping):
|
|
131
136
|
|
|
132
137
|
```bash
|
|
133
138
|
npm install --omit=dev
|
|
@@ -135,7 +140,7 @@ node tests/selftest.mjs --concurrency 4 --mode text https://news.ycombinator.com
|
|
|
135
140
|
node tests/selftest.mjs --abort-after 1500 --concurrency 2 https://httpbin.org/delay/8 https://example.com/
|
|
136
141
|
```
|
|
137
142
|
|
|
138
|
-
To check
|
|
143
|
+
To check tool registration without starting a browser (uses jiti from `devDependencies`):
|
|
139
144
|
|
|
140
145
|
```bash
|
|
141
146
|
npm install
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sasasin/fetch-url",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Pi extension that fetches many web pages at once by driving Playwright directly: one Chromium, one BrowserContext per URL, concurrent navigation, no MCP and no API billing.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -36,7 +36,6 @@
|
|
|
36
36
|
"files": [
|
|
37
37
|
"index.ts",
|
|
38
38
|
"fetch-core.ts",
|
|
39
|
-
"tests",
|
|
40
39
|
"README.md",
|
|
41
40
|
"CHANGELOG.md",
|
|
42
41
|
"LICENSE"
|
package/tests/selftest.mjs
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* 自己テスト: pi の拡張 runtime を使わず fetch-core.ts を直接叩く。
|
|
4
|
-
* node tests/selftest.mjs [--concurrency N] [--mode aria|text|html] [--max-chars N] <url...>
|
|
5
|
-
* 並列度 1(read_url 相当の直列)と 4 を同じ URL 群で比べるため、
|
|
6
|
-
* 実行のたびに --concurrency を変えて 2 回流すと差が見える。
|
|
7
|
-
*/
|
|
8
|
-
import { fetchPages, closeBrowser } from "../fetch-core.ts";
|
|
9
|
-
|
|
10
|
-
const args = process.argv.slice(2);
|
|
11
|
-
let concurrency = 4;
|
|
12
|
-
let mode = "aria";
|
|
13
|
-
let maxChars = 8000;
|
|
14
|
-
let timeoutMs;
|
|
15
|
-
let abortAfter;
|
|
16
|
-
const urls = [];
|
|
17
|
-
for (let i = 0; i < args.length; i++) {
|
|
18
|
-
if (args[i] === "--concurrency") concurrency = Number(args[++i]);
|
|
19
|
-
else if (args[i] === "--mode") mode = args[++i];
|
|
20
|
-
else if (args[i] === "--max-chars") maxChars = Number(args[++i]);
|
|
21
|
-
else if (args[i] === "--timeout") timeoutMs = Number(args[++i]);
|
|
22
|
-
else if (args[i] === "--abort-after") abortAfter = Number(args[++i]);
|
|
23
|
-
else urls.push(args[i]);
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
const controller = new AbortController();
|
|
27
|
-
if (abortAfter) setTimeout(() => controller.abort(), abortAfter).unref?.();
|
|
28
|
-
|
|
29
|
-
const started = Date.now();
|
|
30
|
-
const results = await fetchPages(urls, {
|
|
31
|
-
concurrency,
|
|
32
|
-
mode,
|
|
33
|
-
maxCharsPerPage: maxChars,
|
|
34
|
-
...(timeoutMs ? { navTimeoutMs: timeoutMs } : {}),
|
|
35
|
-
},{ signal: controller.signal });
|
|
36
|
-
const total = Date.now() - started;
|
|
37
|
-
|
|
38
|
-
for (const result of results) {
|
|
39
|
-
console.log(
|
|
40
|
-
[
|
|
41
|
-
result.ok ? "ok " : "fail ",
|
|
42
|
-
result.url,
|
|
43
|
-
`attempt=${result.attempts}`,
|
|
44
|
-
`ms=${result.ms}`,
|
|
45
|
-
result.status ? `status=${result.status}` : "",
|
|
46
|
-
result.finalUrl && result.finalUrl !== result.url ? `final=${result.finalUrl}` : "",
|
|
47
|
-
`chars=${result.characters}`,
|
|
48
|
-
result.title ? `title=${JSON.stringify(result.title)}` : "",
|
|
49
|
-
result.note ? `note=${result.note}` : "",
|
|
50
|
-
]
|
|
51
|
-
.filter(Boolean)
|
|
52
|
-
.join(" ")
|
|
53
|
-
);
|
|
54
|
-
console.log(` head: ${result.body.slice(0, 120).replace(/\s+/g, " ")}`);
|
|
55
|
-
}
|
|
56
|
-
console.log(`\nconcurrency=${concurrency} mode=${mode} urls=${urls.length} wall=${total}ms sum=${results.reduce((a, r) => a + r.ms, 0)}ms`);
|
|
57
|
-
await closeBrowser();
|
package/tests/test-load.mjs
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* pi と同じ jiti 経由で拡張をロードし、登録内容が生きていることを確認する。
|
|
4
|
-
* node tests/test-load.mjs
|
|
5
|
-
* pi と同じ jiti 経路で読むので、ブラウザを起動せずに登録内容だけ確認できる
|
|
6
|
-
* (selftest.mjs は .ts を直接 import するため型ストリッピング可能な Node 22.19+ が要る)。
|
|
7
|
-
*/
|
|
8
|
-
import { dirname, resolve } from "node:path";
|
|
9
|
-
import { fileURLToPath } from "node:url";
|
|
10
|
-
import { createJiti } from "jiti";
|
|
11
|
-
|
|
12
|
-
// このスクリプトは tests/ にあるので、パッケージルートは親ディレクトリ
|
|
13
|
-
const here = dirname(fileURLToPath(import.meta.url));
|
|
14
|
-
const root = resolve(here, "..");
|
|
15
|
-
const jiti = createJiti(import.meta.url, { moduleCache: false });
|
|
16
|
-
|
|
17
|
-
const mod = await jiti.import(resolve(root, "index.ts"));
|
|
18
|
-
if (typeof mod.default !== "function") {
|
|
19
|
-
console.error("index.ts must default-export an extension factory");
|
|
20
|
-
process.exit(1);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
const tools = [];
|
|
24
|
-
const commands = [];
|
|
25
|
-
const events = [];
|
|
26
|
-
mod.default({
|
|
27
|
-
registerTool: (tool) => tools.push(tool?.name),
|
|
28
|
-
registerCommand: (name) => commands.push(name),
|
|
29
|
-
on: (event) => events.push(event),
|
|
30
|
-
exec: async () => ({ code: 0, stdout: "", stderr: "" }),
|
|
31
|
-
});
|
|
32
|
-
|
|
33
|
-
const core = await jiti.import(resolve(root, "fetch-core.ts"));
|
|
34
|
-
|
|
35
|
-
// 拡張が実際に依存を解決できるか(インストール状態の妥当性チェック)
|
|
36
|
-
let playwrightResolvable = false;
|
|
37
|
-
try {
|
|
38
|
-
const pw = await jiti.import("playwright-core");
|
|
39
|
-
playwrightResolvable = Boolean(pw?.chromium);
|
|
40
|
-
} catch {}
|
|
41
|
-
|
|
42
|
-
const checks = [
|
|
43
|
-
["fetch_url tool registered", tools.includes("fetch_url")],
|
|
44
|
-
["registers no slash commands (updates go through pi update --extensions)", commands.length === 0],
|
|
45
|
-
["session_shutdown handler registered", events.includes("session_shutdown")],
|
|
46
|
-
["packageDir() resolves to this folder", resolve(core.packageDir()) === root],
|
|
47
|
-
["playwright-core is resolvable from the extension", playwrightResolvable],
|
|
48
|
-
];
|
|
49
|
-
|
|
50
|
-
let failed = false;
|
|
51
|
-
for (const [label, ok] of checks) {
|
|
52
|
-
if (!ok) failed = true;
|
|
53
|
-
console.log(`${ok ? "ok " : "FAIL"} ${label}`);
|
|
54
|
-
}
|
|
55
|
-
console.log(`node=${process.version} tools=${JSON.stringify(tools)} commands=${JSON.stringify(commands)}`);
|
|
56
|
-
process.exit(failed ? 1 : 0);
|