amenbo 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +188 -0
- package/README.md +13 -1
- package/dist/cache.d.ts +31 -1
- package/dist/cache.js +67 -7
- package/dist/cache.js.map +1 -1
- package/dist/cli.d.ts +3 -1
- package/dist/cli.js +59 -4
- package/dist/cli.js.map +1 -1
- package/dist/core.d.ts +6 -0
- package/dist/core.js +125 -38
- package/dist/core.js.map +1 -1
- package/dist/diff.d.ts +1 -1
- package/dist/diff.js +29 -9
- package/dist/diff.js.map +1 -1
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +13 -0
- package/dist/errors.js.map +1 -1
- package/dist/extract/dataSources.js +17 -4
- package/dist/extract/dataSources.js.map +1 -1
- package/dist/extract/outline.js +15 -1
- package/dist/extract/outline.js.map +1 -1
- package/dist/extract/pdf.js +2 -2
- package/dist/extract/pruning.d.ts +4 -2
- package/dist/extract/pruning.js +36 -7
- package/dist/extract/pruning.js.map +1 -1
- package/dist/extract/qualityScore.d.ts +3 -2
- package/dist/extract/qualityScore.js +8 -3
- package/dist/extract/qualityScore.js.map +1 -1
- package/dist/fetcher/browser.js +15 -1
- package/dist/fetcher/browser.js.map +1 -1
- package/dist/fetcher/http.d.ts +12 -1
- package/dist/fetcher/http.js +15 -4
- package/dist/fetcher/http.js.map +1 -1
- package/dist/fetcher/index.d.ts +6 -0
- package/dist/fetcher/index.js +18 -13
- package/dist/fetcher/index.js.map +1 -1
- package/dist/installBrowser.d.ts +11 -0
- package/dist/installBrowser.js +30 -0
- package/dist/installBrowser.js.map +1 -0
- package/dist/links.d.ts +12 -0
- package/dist/links.js +32 -17
- package/dist/links.js.map +1 -1
- package/dist/politeness.d.ts +23 -3
- package/dist/politeness.js +27 -6
- package/dist/politeness.js.map +1 -1
- package/dist/screenshot.d.ts +2 -0
- package/dist/screenshot.js +14 -4
- package/dist/screenshot.js.map +1 -1
- package/dist/server.d.ts +2 -0
- package/dist/server.js +67 -26
- package/dist/server.js.map +1 -1
- package/package.json +3 -6
package/dist/fetcher/index.d.ts
CHANGED
|
@@ -33,9 +33,15 @@ export interface HandoffResult {
|
|
|
33
33
|
/** プレビュー上限で本文を打ち切った場合true(ファイル全体は取得していない)。 */
|
|
34
34
|
truncated: boolean;
|
|
35
35
|
}
|
|
36
|
+
export declare function detectSpaSignals(html: string): {
|
|
37
|
+
escalate: boolean;
|
|
38
|
+
reason: string | null;
|
|
39
|
+
};
|
|
36
40
|
export interface FetchPageOptions extends HttpGetOptions {
|
|
37
41
|
/** trueの場合、SPA判定を行わず常にブラウザで取得する(将来のmode指定等で利用)。 */
|
|
38
42
|
forceBrowser?: boolean;
|
|
43
|
+
/** MCP progress notifications用。headlessブラウザへ昇格する直前にのみ呼ばれる。 */
|
|
44
|
+
onProgress?: ((message: string) => void) | undefined;
|
|
39
45
|
}
|
|
40
46
|
/** 二段フェッチ本体。条件付きGETでstatus 304が返った場合はNotModifiedResultを返す。 */
|
|
41
47
|
export declare function fetchPage(url: string, options?: FetchPageOptions): Promise<FetchResult | NotModifiedResult | HandoffResult>;
|
package/dist/fetcher/index.js
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
* 大半の静的/SSRページはHTTP GETのみで完結し、先方負荷・自機資源を最小化する。
|
|
7
7
|
*/
|
|
8
8
|
import { parseHTML } from "linkedom";
|
|
9
|
-
import { UnsupportedContentError } from "../errors.js";
|
|
10
9
|
import { fetchWithBrowser } from "./browser.js";
|
|
11
10
|
import { httpGetRouted } from "./http.js";
|
|
12
11
|
// ---- SPA判定ヒューリスティック ----
|
|
@@ -14,22 +13,27 @@ const SPA_ROOT_IDS = ["root", "app", "__next", "__nuxt"];
|
|
|
14
13
|
const MIN_HTML_LENGTH_FOR_RATIO_CHECK = 2000;
|
|
15
14
|
const TEXT_TO_HTML_RATIO_THRESHOLD = 0.02;
|
|
16
15
|
const MIN_VISIBLE_TEXT_LENGTH = 200;
|
|
17
|
-
function detectSpaSignals(html) {
|
|
16
|
+
export function detectSpaSignals(html) {
|
|
18
17
|
const { document } = parseHTML(html);
|
|
19
|
-
for (const tag of Array.from(document.querySelectorAll("script, style"))) {
|
|
20
|
-
tag.remove();
|
|
21
|
-
}
|
|
22
|
-
const visibleText = (document.body?.textContent ?? "").replace(/\s+/g, "");
|
|
23
18
|
for (const id of SPA_ROOT_IDS) {
|
|
24
19
|
const el = document.getElementById(id);
|
|
25
20
|
if (el && el.textContent.replace(/\s+/g, "").length < 10) {
|
|
26
21
|
return { escalate: true, reason: `SPAルートコンテナ(#${id})が空です` };
|
|
27
22
|
}
|
|
28
23
|
}
|
|
24
|
+
// noscriptWarningの判定はnoscriptのテキストを読むため、除去前に先に採取する。
|
|
29
25
|
const noscriptWarning = Array.from(document.querySelectorAll("noscript")).some((el) => /javascript|有効に|enable/i.test(el.textContent ?? ""));
|
|
30
26
|
if (noscriptWarning) {
|
|
31
27
|
return { escalate: true, reason: "noscriptにJavaScript要求の警告があります" };
|
|
32
28
|
}
|
|
29
|
+
// script/style/noscriptを除去してから可視テキスト量を測る(extract/markdown.tsの
|
|
30
|
+
// collectQualityInputと同じ除去対象に揃える)。noscriptを除去しないと、GTM等の
|
|
31
|
+
// <noscript><iframe>フォールバックや長文注記を持つSPAで、noscriptのテキスト長が
|
|
32
|
+
// 閾値を満たしてしまいブラウザ昇格が誤ってスキップされる。
|
|
33
|
+
for (const tag of Array.from(document.querySelectorAll("script, style, noscript"))) {
|
|
34
|
+
tag.remove();
|
|
35
|
+
}
|
|
36
|
+
const visibleText = (document.body?.textContent ?? "").replace(/\s+/g, "");
|
|
33
37
|
if (html.length > MIN_HTML_LENGTH_FOR_RATIO_CHECK && visibleText.length / html.length < TEXT_TO_HTML_RATIO_THRESHOLD) {
|
|
34
38
|
return { escalate: true, reason: "抽出テキスト量/HTMLサイズ比が低いです" };
|
|
35
39
|
}
|
|
@@ -45,13 +49,13 @@ export async function fetchPage(url, options = {}) {
|
|
|
45
49
|
return { notModified: true, finalUrl: routed.finalUrl };
|
|
46
50
|
}
|
|
47
51
|
if (routed.kind === "handoff") {
|
|
48
|
-
//
|
|
49
|
-
// URL
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
// レビュー指摘対応: 以前はcontent-type=application/pdfのみUnsupportedContentErrorを
|
|
53
|
+
// 投げて行き止まりにしていたが、URL拡張子で検出できないPDF(官公庁の
|
|
54
|
+
// `/download?id=123`型ダウンロードエンドポイント等で頻出)がここに落ちると、curl誘導も
|
|
55
|
+
// 出せないまま利用者に「サポート対象外」とだけ返す不親切な経路になっていた。
|
|
56
|
+
// PDFかどうかの判定・専用処理へのルーティングはcore.ts(handleFetchTool)側で
|
|
57
|
+
// 行う(URL拡張子で早期判明したPDFと同じhandlePdfFetch経路へ合流させるため)。
|
|
58
|
+
// ここでは他の非HTMLコンテンツと同様、通常のhandoffとして返すだけにする。
|
|
55
59
|
return {
|
|
56
60
|
handoff: true,
|
|
57
61
|
finalUrl: routed.finalUrl,
|
|
@@ -76,6 +80,7 @@ export async function fetchPage(url, options = {}) {
|
|
|
76
80
|
geometry: null,
|
|
77
81
|
};
|
|
78
82
|
}
|
|
83
|
+
options.onProgress?.("ブラウザで再取得しています…");
|
|
79
84
|
const browserResult = await fetchWithBrowser(routed.finalUrl, options.timeoutMs);
|
|
80
85
|
return {
|
|
81
86
|
finalUrl: browserResult.finalUrl,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/fetcher/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/fetcher/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,aAAa,EAAuB,MAAM,WAAW,CAAC;AAuC/D,2BAA2B;AAE3B,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACzD,MAAM,+BAA+B,GAAG,IAAI,CAAC;AAC7C,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAC1C,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAEpC,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAErC,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QACvC,IAAI,EAAE,IAAI,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACzD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CACpF,wBAAwB,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CACpD,CAAC;IACF,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,+BAA+B,EAAE,CAAC;IACrE,CAAC;IAED,8DAA8D;IAC9D,wDAAwD;IACxD,wDAAwD;IACxD,+BAA+B;IAC/B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC,EAAE,CAAC;QACnF,GAAG,CAAC,MAAM,EAAE,CAAC;IACf,CAAC;IACD,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAE3E,IAAI,IAAI,CAAC,MAAM,GAAG,+BAA+B,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,4BAA4B,EAAE,CAAC;QACrH,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC;IAC7D,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,GAAG,+BAA+B,IAAI,WAAW,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QAClG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IACtD,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC3C,CAAC;AASD,8DAA8D;AAC9D,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAAO,GAAqB,EAAE;IACzE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAEjD,IAAI,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QAClC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC1D,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,sEAAsE;QACtE,uCAAuC;QACvC,wDAAwD;QACxD,wCAAwC;QACxC,qDAAqD;QACrD,mDAAmD;QACnD,4CAA4C;QAC5C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEvH,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;QACzB,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM;YACZ,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;YAChC,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;YACjD,gBAAgB,EAAE,IAAI;YACtB,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,UAAU,EAAE,CAAC,gBAAgB,CAAC,CAAC;IACvC,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACjF,OAAO;QACL,QAAQ,EAAE,aAAa,CAAC,QAAQ;QAChC,IAAI,EAAE,aAAa,CAAC,IAAI;QACxB,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,aAAa,CAAC,MAAM;QAC5B,QAAQ,EAAE,OAAO;QACjB,IAAI,EAAE,IAAI;QACV,YAAY,EAAE,IAAI;QAClB,gBAAgB,EAAE,UAAU,CAAC,MAAM;QACnC,QAAQ,EAAE,aAAa,CAAC,QAAQ;KACjC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* amenboが依存する同梱playwrightパッケージのCLIエントリ(playwright/cli.js)を解決して
|
|
3
|
+
* `install chromium` を実行する。playwrightのグローバル/別バージョンではなく、
|
|
4
|
+
* amenboに同梱されたバージョンのCLIを使うことで、Chromiumのリビジョンとplaywright本体の
|
|
5
|
+
* バージョンが食い違う事態を原理的に防ぐ。
|
|
6
|
+
*
|
|
7
|
+
* "playwright/cli.js" はplaywrightのpackage.json exportsマップにサブパスとして
|
|
8
|
+
* 定義されていないため直接resolveできない。exportsに含まれる"playwright/package.json"を
|
|
9
|
+
* resolveし、そのディレクトリからcli.js(package.json"bin"が指す実体)を組み立てる。
|
|
10
|
+
*/
|
|
11
|
+
export declare function installBrowser(): Promise<number>;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* installBrowser.ts — `amenbo install-browser` サブコマンドの本体。
|
|
3
|
+
*
|
|
4
|
+
* Chromium遅延化(postinstall廃止)対応: package.jsonのpostinstallを撤去したため、
|
|
5
|
+
* ブラウザが必要な操作(SPA昇格・screenshot)を使う利用者は、このコマンドで明示的に
|
|
6
|
+
* 一度だけChromiumを取得する。
|
|
7
|
+
*/
|
|
8
|
+
import { createRequire } from "node:module";
|
|
9
|
+
import { spawn } from "node:child_process";
|
|
10
|
+
import { dirname, join } from "node:path";
|
|
11
|
+
/**
|
|
12
|
+
* amenboが依存する同梱playwrightパッケージのCLIエントリ(playwright/cli.js)を解決して
|
|
13
|
+
* `install chromium` を実行する。playwrightのグローバル/別バージョンではなく、
|
|
14
|
+
* amenboに同梱されたバージョンのCLIを使うことで、Chromiumのリビジョンとplaywright本体の
|
|
15
|
+
* バージョンが食い違う事態を原理的に防ぐ。
|
|
16
|
+
*
|
|
17
|
+
* "playwright/cli.js" はplaywrightのpackage.json exportsマップにサブパスとして
|
|
18
|
+
* 定義されていないため直接resolveできない。exportsに含まれる"playwright/package.json"を
|
|
19
|
+
* resolveし、そのディレクトリからcli.js(package.json"bin"が指す実体)を組み立てる。
|
|
20
|
+
*/
|
|
21
|
+
export async function installBrowser() {
|
|
22
|
+
const packageJsonPath = createRequire(import.meta.url).resolve("playwright/package.json");
|
|
23
|
+
const cliPath = join(dirname(packageJsonPath), "cli.js");
|
|
24
|
+
return new Promise((resolve, reject) => {
|
|
25
|
+
const child = spawn(process.execPath, [cliPath, "install", "chromium"], { stdio: "inherit" });
|
|
26
|
+
child.once("error", reject);
|
|
27
|
+
child.once("close", (code) => resolve(code ?? 1));
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=installBrowser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"installBrowser.js","sourceRoot":"","sources":["../src/installBrowser.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,eAAe,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAC1F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAC;IACzD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9F,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/links.d.ts
CHANGED
|
@@ -9,12 +9,24 @@ export interface LinksResult {
|
|
|
9
9
|
links: LinkEntry[];
|
|
10
10
|
/** MAX_LINKSを超えて切り捨てた場合true。 */
|
|
11
11
|
truncated: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* フィルタ適用前(重複排除後)の件数。
|
|
14
|
+
* filter指定時にlinks.length===0となった場合、これが0なら「sitemap/RSS/ページ自体が空」、
|
|
15
|
+
* 0より大きければ「filterに一致しなかった」と原因を切り分けられる(呼び出し側でのみ使う)。
|
|
16
|
+
*/
|
|
17
|
+
preFilterCount: number;
|
|
12
18
|
}
|
|
13
19
|
/** ページ内の<a href>リンクを抽出する(機能C: extract/dataSources.tsからも再利用される)。 */
|
|
14
20
|
export declare function extractPageLinks(html: string, baseUrl: string): LinkEntry[];
|
|
15
21
|
export interface DiscoverLinksOptions {
|
|
16
22
|
filter?: string;
|
|
17
23
|
timeoutMs?: number;
|
|
24
|
+
/**
|
|
25
|
+
* MCP progress notifications用。sitemapindexの子sitemap再帰・フィード取得を含め
|
|
26
|
+
* politeness待機が複数回発生しうる経路のため、累積待機が最も長引きうる。
|
|
27
|
+
* server.tsがprogressToken有無に応じて組み立てて渡す(CLIは渡さない=未指定で後方互換)。
|
|
28
|
+
*/
|
|
29
|
+
onProgress?: ((message: string) => void) | undefined;
|
|
18
30
|
}
|
|
19
31
|
/** URLからリンク一覧を発見する。sitemap → RSS/Atom → ページ内リンクの優先順で試す。 */
|
|
20
32
|
export declare function discoverLinks(url: string, politeness: PolitenessManager, options?: DiscoverLinksOptions): Promise<LinksResult>;
|
package/dist/links.js
CHANGED
|
@@ -32,18 +32,27 @@ const DEFAULT_TIMEOUT_MS = 15_000;
|
|
|
32
32
|
function escapeRegExp(value) {
|
|
33
33
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
34
34
|
}
|
|
35
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* filter未指定なら常に真。*を含む場合はglob、それ以外は部分一致。
|
|
37
|
+
*
|
|
38
|
+
* レビュー指摘対応: globは`^…$`で両端アンカーせず非アンカー(部分一致)にする。
|
|
39
|
+
* README/記事が例示する`--filter "blog/*"`は絶対URL(`https://example.com/blog/123`)全体とは
|
|
40
|
+
* 一致しないため、アンカー版だと必ず0件になっていた。URL/タイトルの一部にglobパターンが
|
|
41
|
+
* 現れれば真とする(通常の部分一致フィルタと同じ「含む」感覚に揃える)。
|
|
42
|
+
*/
|
|
36
43
|
function matchesFilter(value, filter) {
|
|
37
44
|
if (!filter)
|
|
38
45
|
return true;
|
|
39
46
|
if (filter.includes("*")) {
|
|
40
|
-
const pattern = new RegExp(
|
|
47
|
+
const pattern = new RegExp(filter.split("*").map(escapeRegExp).join(".*"), "i");
|
|
41
48
|
return pattern.test(value);
|
|
42
49
|
}
|
|
43
50
|
return value.toLowerCase().includes(filter.toLowerCase());
|
|
44
51
|
}
|
|
45
|
-
async function fetchXml(url, timeoutMs, rewrite) {
|
|
46
|
-
|
|
52
|
+
async function fetchXml(url, timeoutMs, politeness, rewrite) {
|
|
53
|
+
// レビュー指摘対応: リダイレクトで別オリジンへ着地した場合、着地先のrobots.txtも確認する
|
|
54
|
+
// (fetcher/http.tsのHttpGetOptions.checkRobots参照)。
|
|
55
|
+
const result = await httpGet(url, { timeoutMs, checkRobots: (targetUrl) => politeness.checkRobotsAllowed(targetUrl) });
|
|
47
56
|
const { document } = parseHTML(rewrite ? rewrite(result.html) : result.html);
|
|
48
57
|
return document;
|
|
49
58
|
}
|
|
@@ -56,11 +65,11 @@ async function fetchXml(url, timeoutMs, rewrite) {
|
|
|
56
65
|
function escapeRssLinkTag(xml) {
|
|
57
66
|
return xml.replace(/<link>([\s\S]*?)<\/link>/gi, "<rss-link>$1</rss-link>");
|
|
58
67
|
}
|
|
59
|
-
async function tryParseSitemap(sitemapUrl, politeness, timeoutMs, depth = 0) {
|
|
68
|
+
async function tryParseSitemap(sitemapUrl, politeness, timeoutMs, depth = 0, onProgress) {
|
|
60
69
|
let document;
|
|
61
70
|
try {
|
|
62
|
-
await politeness.guard(sitemapUrl);
|
|
63
|
-
document = await fetchXml(sitemapUrl, timeoutMs);
|
|
71
|
+
await politeness.guard(sitemapUrl, onProgress);
|
|
72
|
+
document = await fetchXml(sitemapUrl, timeoutMs, politeness);
|
|
64
73
|
}
|
|
65
74
|
catch (error) {
|
|
66
75
|
if (isFallbackBlockingError(error))
|
|
@@ -80,7 +89,7 @@ async function tryParseSitemap(sitemapUrl, politeness, timeoutMs, depth = 0) {
|
|
|
80
89
|
const childUrl = loc.textContent.trim();
|
|
81
90
|
if (!childUrl)
|
|
82
91
|
continue;
|
|
83
|
-
const childLinks = await tryParseSitemap(childUrl, politeness, timeoutMs, depth + 1);
|
|
92
|
+
const childLinks = await tryParseSitemap(childUrl, politeness, timeoutMs, depth + 1, onProgress);
|
|
84
93
|
if (childLinks)
|
|
85
94
|
results.push(...childLinks);
|
|
86
95
|
if (results.length >= MAX_LINKS)
|
|
@@ -93,12 +102,12 @@ async function tryParseSitemap(sitemapUrl, politeness, timeoutMs, depth = 0) {
|
|
|
93
102
|
return null;
|
|
94
103
|
return urlLocs.map((loc) => ({ url: loc.textContent.trim(), title: null })).filter((entry) => entry.url.length > 0);
|
|
95
104
|
}
|
|
96
|
-
async function tryParseFeed(feedUrl, politeness, timeoutMs) {
|
|
105
|
+
async function tryParseFeed(feedUrl, politeness, timeoutMs, onProgress) {
|
|
97
106
|
let document;
|
|
98
107
|
try {
|
|
99
|
-
await politeness.guard(feedUrl);
|
|
108
|
+
await politeness.guard(feedUrl, onProgress);
|
|
100
109
|
// RSS 2.0の<link>はHTMLパーサにvoid要素として扱われるためリネームして回避する(下記コメント参照)
|
|
101
|
-
document = await fetchXml(feedUrl, timeoutMs, escapeRssLinkTag);
|
|
110
|
+
document = await fetchXml(feedUrl, timeoutMs, politeness, escapeRssLinkTag);
|
|
102
111
|
}
|
|
103
112
|
catch (error) {
|
|
104
113
|
if (isFallbackBlockingError(error))
|
|
@@ -175,8 +184,14 @@ function dedupeByUrl(links) {
|
|
|
175
184
|
return result;
|
|
176
185
|
}
|
|
177
186
|
function finalize(source, links, filter) {
|
|
178
|
-
const
|
|
179
|
-
|
|
187
|
+
const deduped = dedupeByUrl(links);
|
|
188
|
+
const filtered = deduped.filter((link) => matchesFilter(link.url, filter) || matchesFilter(link.title ?? "", filter));
|
|
189
|
+
return {
|
|
190
|
+
source,
|
|
191
|
+
links: filtered.slice(0, MAX_LINKS),
|
|
192
|
+
truncated: filtered.length > MAX_LINKS,
|
|
193
|
+
preFilterCount: deduped.length,
|
|
194
|
+
};
|
|
180
195
|
}
|
|
181
196
|
/** URLからリンク一覧を発見する。sitemap → RSS/Atom → ページ内リンクの優先順で試す。 */
|
|
182
197
|
export async function discoverLinks(url, politeness, options = {}) {
|
|
@@ -190,14 +205,14 @@ export async function discoverLinks(url, politeness, options = {}) {
|
|
|
190
205
|
const declaredSitemaps = await politeness.getSitemaps(url);
|
|
191
206
|
const candidateSitemaps = declaredSitemaps.length > 0 ? declaredSitemaps : [`${origin}/sitemap.xml`];
|
|
192
207
|
for (const sitemapUrl of candidateSitemaps) {
|
|
193
|
-
const links = await tryParseSitemap(sitemapUrl, politeness, timeoutMs);
|
|
208
|
+
const links = await tryParseSitemap(sitemapUrl, politeness, timeoutMs, 0, options.onProgress);
|
|
194
209
|
if (links && links.length > 0) {
|
|
195
210
|
return finalize("sitemap", links, options.filter);
|
|
196
211
|
}
|
|
197
212
|
}
|
|
198
213
|
// 2. ページを取得し、<link rel=alternate>が指すRSS/Atomフィードを試す
|
|
199
|
-
await politeness.guard(url);
|
|
200
|
-
const pageResult = await fetchPage(url, { timeoutMs });
|
|
214
|
+
await politeness.guard(url, options.onProgress);
|
|
215
|
+
const pageResult = await fetchPage(url, { timeoutMs, checkRobots: (targetUrl) => politeness.checkRobotsAllowed(targetUrl) });
|
|
201
216
|
if ("notModified" in pageResult) {
|
|
202
217
|
return finalize("page", [], options.filter);
|
|
203
218
|
}
|
|
@@ -212,7 +227,7 @@ export async function discoverLinks(url, politeness, options = {}) {
|
|
|
212
227
|
const feedHref = feedLinkEl?.getAttribute("href");
|
|
213
228
|
if (feedHref) {
|
|
214
229
|
const feedUrl = new URL(feedHref, pageResult.finalUrl).toString();
|
|
215
|
-
const feedLinks = await tryParseFeed(feedUrl, politeness, timeoutMs);
|
|
230
|
+
const feedLinks = await tryParseFeed(feedUrl, politeness, timeoutMs, options.onProgress);
|
|
216
231
|
if (feedLinks && feedLinks.length > 0) {
|
|
217
232
|
return finalize("rss", feedLinks, options.filter);
|
|
218
233
|
}
|
package/dist/links.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"links.js","sourceRoot":"","sources":["../src/links.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAC/G,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG9D;;;;;;;;;GASG;AACH,SAAS,uBAAuB,CAAC,KAAc;IAC7C,OAAO,KAAK,YAAY,mBAAmB,IAAI,KAAK,YAAY,iBAAiB,CAAC;AACpF,CAAC;
|
|
1
|
+
{"version":3,"file":"links.js","sourceRoot":"","sources":["../src/links.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAC/G,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG9D;;;;;;;;;GASG;AACH,SAAS,uBAAuB,CAAC,KAAc;IAC7C,OAAO,KAAK,YAAY,mBAAmB,IAAI,KAAK,YAAY,iBAAiB,CAAC;AACpF,CAAC;AAsBD,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,iDAAiD;AACjD,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,aAAa,CAAC,KAAa,EAAE,MAA0B;IAC9D,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAChF,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;AAC5D,CAAC;AAaD,KAAK,UAAU,QAAQ,CACrB,GAAW,EACX,SAAiB,EACjB,UAA6B,EAC7B,OAAiC;IAEjC,oDAAoD;IACpD,kDAAkD;IAClD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACvH,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7E,OAAO,QAAsC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,OAAO,GAAG,CAAC,OAAO,CAAC,4BAA4B,EAAE,yBAAyB,CAAC,CAAC;AAC9E,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,UAAkB,EAClB,UAA6B,EAC7B,SAAiB,EACjB,KAAK,GAAG,CAAC,EACT,UAAoD;IAEpD,IAAI,QAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/C,QAAQ,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,uBAAuB,CAAC,KAAK,CAAC;YAAE,MAAM,KAAK,CAAC;QAChD,qDAAqD;QACrD,2CAA2C;QAC3C,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,CAAC,sBAAsB,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gDAAgD;IAChD,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;IACxF,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACxC,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YACxB,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,UAAU,CAAC,CAAC;YACjG,IAAI,UAAU;gBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;YAC5C,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS;gBAAE,MAAM;QACzC,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACtH,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,OAAe,EACf,UAA6B,EAC7B,SAAiB,EACjB,UAAoD;IAEpD,IAAI,QAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC5C,6DAA6D;QAC7D,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC;IAC9E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,uBAAuB,CAAC,KAAK,CAAC;YAAE,MAAM,KAAK,CAAC;QAChD,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,CAAC,mBAAmB,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,KAAK;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACnE,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;YACzE,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,KAAK,EAAsB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC;IACzC,CAAC;IAED,OAAO;IACP,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;IACnE,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,WAAW;aACxB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACb,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;YAC3E,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;YAC1E,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,KAAK,EAAsB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC;IACzC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,OAAe;IAC5D,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAE1E,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,SAAS;QAC1C,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,SAAS;QACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;QACxE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAkB;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,MAAkB,EAAE,KAAkB,EAAE,MAA0B;IAClF,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IACtH,OAAO;QACL,MAAM;QACN,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;QACnC,SAAS,EAAE,QAAQ,CAAC,MAAM,GAAG,SAAS;QACtC,cAAc,EAAE,OAAO,CAAC,MAAM;KAC/B,CAAC;AACJ,CAAC;AAaD,2DAA2D;AAC3D,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAW,EAAE,UAA6B,EAAE,OAAO,GAAyB,EAAE;IAChH,6DAA6D;IAC7D,4DAA4D;IAC5D,uCAAuC;IACvC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACtB,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IAEnC,oDAAoD;IACpD,MAAM,gBAAgB,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;IACrG,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QAC9F,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,QAAQ,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,MAAM,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAC7H,IAAI,aAAa,IAAI,UAAU,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,uDAAuD;IACvD,gEAAgE;IAChE,iCAAiC;IACjC,IAAI,SAAS,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,IAAI,uBAAuB,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,WAAW,IAAI,MAAM,CAAC,CAAC;IAC3F,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CACvC,uGAAuG,CACxG,CAAC;IACF,MAAM,QAAQ,GAAG,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAClD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAC;QAClE,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACzF,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,OAAO,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,OAAO,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAClG,CAAC"}
|
package/dist/politeness.d.ts
CHANGED
|
@@ -16,6 +16,19 @@ export interface PolitenessStore {
|
|
|
16
16
|
getLastRequestAt(host: string): number | null;
|
|
17
17
|
setLastRequestAt(host: string, at: number): void;
|
|
18
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* レビュー指摘対応: robots.txt取得結果をプロセス間で共有するための永続化ストア。
|
|
21
|
+
* PolitenessStoreと同じ考え方で、cache.ts(robots_cacheテーブル)をこの形で注入する想定。
|
|
22
|
+
* CLIは1コマンド=1プロセスのため、これが無いとREADME推奨の「CLIで複数ページ一括収集」時に
|
|
23
|
+
* コマンド毎にrobots.txtを再取得してしまい低負荷原則に反する。
|
|
24
|
+
*/
|
|
25
|
+
export interface RobotsStore {
|
|
26
|
+
get(origin: string): {
|
|
27
|
+
body: string;
|
|
28
|
+
fetchedAt: number;
|
|
29
|
+
} | null;
|
|
30
|
+
set(origin: string, body: string, fetchedAt: number): void;
|
|
31
|
+
}
|
|
19
32
|
export interface PolitenessOptions {
|
|
20
33
|
/** ドメイン毎の最小リクエスト間隔(ミリ秒、既定1000ms)。robots.txtのCrawl-Delayがこれより長ければそちらを優先。 */
|
|
21
34
|
minIntervalMs?: number;
|
|
@@ -27,6 +40,8 @@ export interface PolitenessOptions {
|
|
|
27
40
|
sleep?: (ms: number) => Promise<void>;
|
|
28
41
|
/** 未指定時は従来通りインメモリのみでレート制御する(完全後方互換)。 */
|
|
29
42
|
store?: PolitenessStore;
|
|
43
|
+
/** 未指定時は従来通りインメモリのみでrobots.txtをキャッシュする(完全後方互換)。 */
|
|
44
|
+
robotsStore?: RobotsStore;
|
|
30
45
|
}
|
|
31
46
|
export declare class PolitenessManager {
|
|
32
47
|
private readonly minIntervalMs;
|
|
@@ -34,12 +49,16 @@ export declare class PolitenessManager {
|
|
|
34
49
|
private readonly now;
|
|
35
50
|
private readonly sleep;
|
|
36
51
|
private readonly store;
|
|
52
|
+
private readonly robotsStore;
|
|
37
53
|
private readonly robotsCache;
|
|
38
54
|
private readonly locks;
|
|
39
55
|
private readonly lastRequestAt;
|
|
40
56
|
constructor(options?: PolitenessOptions);
|
|
41
|
-
/**
|
|
42
|
-
|
|
57
|
+
/**
|
|
58
|
+
* robots.txtとレート制御の両方を確認し、アクセス可能になるまで待機する。拒否時はRobotsDeniedErrorを投げる。
|
|
59
|
+
* onProgress: MCP progress notifications用。実際に待機が発生する場合のみ呼ばれる(待機0の場合は呼ばれない)。
|
|
60
|
+
*/
|
|
61
|
+
guard(url: string, onProgress?: ((message: string) => void) | undefined): Promise<void>;
|
|
43
62
|
/** robots.txtのみ確認する(waitTurnは行わない)。 */
|
|
44
63
|
checkRobotsAllowed(url: string): Promise<void>;
|
|
45
64
|
/** robots.txtが宣言するSitemap URLの一覧を返す(linksツールのsitemap優先探索用)。 */
|
|
@@ -49,8 +68,9 @@ export declare class PolitenessManager {
|
|
|
49
68
|
* 直列キュー(locks)による直列化はプロセス内のみ有効。store(PolitenessStore)経由の
|
|
50
69
|
* プロセス間共有はread-modify-writeがアトミックではないベストエフォート(PolitenessStoreの
|
|
51
70
|
* JSDoc参照)。
|
|
71
|
+
* onProgress: 実際にsleepが発生する場合のみ呼ばれる(待機0のときは呼ばれない)。
|
|
52
72
|
*/
|
|
53
|
-
waitTurn(url: string): Promise<void>;
|
|
73
|
+
waitTurn(url: string, onProgress?: ((message: string) => void) | undefined): Promise<void>;
|
|
54
74
|
/**
|
|
55
75
|
* N8: robotsCache/locks/lastRequestAtは訪問したホスト毎に1エントリずつ増え、長時間稼働する
|
|
56
76
|
* MCPサーバープロセスでは無制限に増加しうる。robotsTtlMsを「エントリの生存期間」の目安として
|
package/dist/politeness.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* politeness.ts — 収集先への低負荷を担保するpolitenessマネージャ。
|
|
3
3
|
*
|
|
4
|
-
* - robots.txt尊重(robots-parser)。取得結果はプロセス内でキャッシュ(TTL付き)
|
|
4
|
+
* - robots.txt尊重(robots-parser)。取得結果はプロセス内でキャッシュ(TTL付き)し、
|
|
5
|
+
* robotsStore注入時はプロセス間でも共有する(CLI併設対応)
|
|
5
6
|
* - Crawl-Delay尊重
|
|
6
7
|
* - ドメイン毎の直列キュー + 既定最小間隔1秒
|
|
7
8
|
*/
|
|
@@ -21,6 +22,7 @@ export class PolitenessManager {
|
|
|
21
22
|
now;
|
|
22
23
|
sleep;
|
|
23
24
|
store;
|
|
25
|
+
robotsStore;
|
|
24
26
|
robotsCache = new Map();
|
|
25
27
|
// N8: settledを持たせるのは、pruneStaleHostStateがまだ完了していない(=キューに並んでいる
|
|
26
28
|
// 呼び出しがある)ロックを誤って消してしまうと直列化が壊れる(同一ホストへの同時呼び出しが
|
|
@@ -34,11 +36,15 @@ export class PolitenessManager {
|
|
|
34
36
|
this.now = options.now ?? Date.now;
|
|
35
37
|
this.sleep = options.sleep ?? defaultSleep;
|
|
36
38
|
this.store = options.store;
|
|
39
|
+
this.robotsStore = options.robotsStore;
|
|
37
40
|
}
|
|
38
|
-
/**
|
|
39
|
-
|
|
41
|
+
/**
|
|
42
|
+
* robots.txtとレート制御の両方を確認し、アクセス可能になるまで待機する。拒否時はRobotsDeniedErrorを投げる。
|
|
43
|
+
* onProgress: MCP progress notifications用。実際に待機が発生する場合のみ呼ばれる(待機0の場合は呼ばれない)。
|
|
44
|
+
*/
|
|
45
|
+
async guard(url, onProgress) {
|
|
40
46
|
await this.checkRobotsAllowed(url);
|
|
41
|
-
await this.waitTurn(url);
|
|
47
|
+
await this.waitTurn(url, onProgress);
|
|
42
48
|
}
|
|
43
49
|
/** robots.txtのみ確認する(waitTurnは行わない)。 */
|
|
44
50
|
async checkRobotsAllowed(url) {
|
|
@@ -57,8 +63,9 @@ export class PolitenessManager {
|
|
|
57
63
|
* 直列キュー(locks)による直列化はプロセス内のみ有効。store(PolitenessStore)経由の
|
|
58
64
|
* プロセス間共有はread-modify-writeがアトミックではないベストエフォート(PolitenessStoreの
|
|
59
65
|
* JSDoc参照)。
|
|
66
|
+
* onProgress: 実際にsleepが発生する場合のみ呼ばれる(待機0のときは呼ばれない)。
|
|
60
67
|
*/
|
|
61
|
-
async waitTurn(url) {
|
|
68
|
+
async waitTurn(url, onProgress) {
|
|
62
69
|
this.pruneStaleHostState();
|
|
63
70
|
const host = new URL(url).host;
|
|
64
71
|
const crawlDelayMs = await this.getCrawlDelayMs(url);
|
|
@@ -80,6 +87,7 @@ export class PolitenessManager {
|
|
|
80
87
|
const lastAt = Math.max(memoryLastAt, storeLastAt);
|
|
81
88
|
const elapsed = this.now() - lastAt;
|
|
82
89
|
if (elapsed < intervalMs) {
|
|
90
|
+
onProgress?.("アクセス間隔を確保するため待機しています…");
|
|
83
91
|
await this.sleep(intervalMs - elapsed);
|
|
84
92
|
}
|
|
85
93
|
const requestAt = this.now();
|
|
@@ -139,6 +147,17 @@ export class PolitenessManager {
|
|
|
139
147
|
if (cached && this.now() - cached.fetchedAt < this.robotsTtlMs) {
|
|
140
148
|
return cached.robots;
|
|
141
149
|
}
|
|
150
|
+
// レビュー指摘対応: インメモリキャッシュに無い/TTL超過の場合、ネットワークへ行く前に
|
|
151
|
+
// robotsStore(cache.tsのrobots_cacheテーブル。プロセス間共有)を確認する。ここで
|
|
152
|
+
// 復元できれば別プロセス(CLIの前回実行等)が取得済みのrobots.txtを再利用でき、
|
|
153
|
+
// CLIバルク収集時の再取得(=先方への追加負荷)を避けられる。空bodyの記録(robots.txt
|
|
154
|
+
// 不在=制限なし)もfetchedAtの有無で「未記録」と区別して正しく復元する。
|
|
155
|
+
const stored = this.robotsStore?.get(origin);
|
|
156
|
+
if (stored && this.now() - stored.fetchedAt < this.robotsTtlMs) {
|
|
157
|
+
const robots = robotsParser(`${origin}/robots.txt`, stored.body);
|
|
158
|
+
this.robotsCache.set(origin, { robots, fetchedAt: stored.fetchedAt });
|
|
159
|
+
return robots;
|
|
160
|
+
}
|
|
142
161
|
const robotsUrl = `${origin}/robots.txt`;
|
|
143
162
|
let body = "";
|
|
144
163
|
try {
|
|
@@ -163,7 +182,9 @@ export class PolitenessManager {
|
|
|
163
182
|
body = "";
|
|
164
183
|
}
|
|
165
184
|
const robots = robotsParser(robotsUrl, body);
|
|
166
|
-
|
|
185
|
+
const fetchedAt = this.now();
|
|
186
|
+
this.robotsCache.set(origin, { robots, fetchedAt });
|
|
187
|
+
this.robotsStore?.set(origin, body, fetchedAt);
|
|
167
188
|
return robots;
|
|
168
189
|
}
|
|
169
190
|
}
|
package/dist/politeness.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"politeness.js","sourceRoot":"","sources":["../src/politeness.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"politeness.js","sourceRoot":"","sources":["../src/politeness.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,kBAAkB,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAiBxD,MAAM,YAAY,GAAG,kBAA0E,CAAC;AAEhG,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,MAAM;AACpD,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAoDrC,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,OAAO,iBAAiB;IACX,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,GAAG,CAAe;IAClB,KAAK,CAAgC;IACrC,KAAK,CAA8B;IACnC,WAAW,CAA0B;IAErC,WAAW,GAAG,IAAI,GAAG,EAA4B,CAAC;IACnE,8DAA8D;IAC9D,+CAA+C;IAC/C,gDAAgD;IAChD,YAAY;IACK,KAAK,GAAG,IAAI,GAAG,EAAsD,CAAC;IACtE,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3D,YAAY,OAAO,GAAsB,EAAE;QACzC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC;QACtE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,qBAAqB,CAAC;QAChE,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAAoD;QAC3E,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACvC,CAAC;IAED,uCAAuC;IACvC,KAAK,CAAC,kBAAkB,CAAC,GAAW;QAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,KAAK,KAAK,EAAE,CAAC;YAC1D,MAAM,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,WAAW,CAAC,GAAW;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,OAAO,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACrC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,UAAoD;QAC9E,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;QAC/B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAE9D,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,aAAa,EAAE,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3D,IAAI,OAAO,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5C,OAAO,GAAG,OAAO,CAAC;QACpB,CAAC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAE5B,MAAM,QAAQ,CAAC;QACf,IAAI,CAAC;YACH,yDAAyD;YACzD,uDAAuD;YACvD,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;YACpE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;YACpC,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;gBACzB,UAAU,EAAE,CAAC,uBAAuB,CAAC,CAAC;gBACtC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC;YACzC,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YACxC,IAAI,CAAC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAChD,CAAC;gBAAS,CAAC;YACT,OAAO,EAAE,CAAC;YACV,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,mBAAmB;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;QAE7C,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAChD,IAAI,MAAM,IAAI,MAAM;gBAAE,SAAS;YAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,KAAK,CAAC,SAAS,GAAG,MAAM,EAAE,CAAC;gBAC7B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;QACnD,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,qEAAqE;QACrE,wDAAwD;QACxD,mEAAmE;QACnE,qDAAqD;QACrD,gEAAgE;QAChE,gEAAgE;QAChE,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/D,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;QAED,+CAA+C;QAC/C,0DAA0D;QAC1D,gDAAgD;QAChD,qDAAqD;QACrD,2CAA2C;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/D,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,MAAM,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACjE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YACtE,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,MAAM,aAAa,CAAC;QACzC,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,uBAAuB,EAAE,CAAC,CAAC;YAChF,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,qDAAqD;YACrD,EAAE;YACF,uDAAuD;YACvD,8CAA8C;YAC9C,iDAAiD;YACjD,2DAA2D;YAC3D,2CAA2C;YAC3C,kCAAkC;YAClC,IAAI,KAAK,YAAY,mBAAmB;gBAAE,MAAM,KAAK,CAAC;YACtD,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,EAAE,CAAC;gBACxC,OAAO,CAAC,KAAK,CAAC,sCAAsC,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
package/dist/screenshot.d.ts
CHANGED
|
@@ -34,5 +34,7 @@ export interface CaptureResult {
|
|
|
34
34
|
/** N7: MAX_TILES枚を超えるページで切り捨てが発生した場合true。 */
|
|
35
35
|
truncated: boolean;
|
|
36
36
|
}
|
|
37
|
+
export declare const MIN_SCALE = 0.5;
|
|
38
|
+
export declare const MAX_SCALE = 1;
|
|
37
39
|
/** URLをブラウザでレンダリングし、タイル分割したPNGスクリーンショット群を撮影する。 */
|
|
38
40
|
export declare function captureTiledScreenshot(url: string, options?: ScreenshotOptions): Promise<CaptureResult>;
|
package/dist/screenshot.js
CHANGED
|
@@ -34,8 +34,10 @@ export function isTileCaptureTruncated(pageHeight, tileCount, tileHeight = DEFAU
|
|
|
34
34
|
return tilesNeeded > tileCount;
|
|
35
35
|
}
|
|
36
36
|
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
// レビュー指摘対応(Low): CLI側(cli.ts parseScreenshotArgs)がMCPのzod .min(0.5).max(1.0)と
|
|
38
|
+
// 同じ範囲で検証できるよう、この定数をexportしてCLIと共有する(値の重複定義によるずれを防ぐ)。
|
|
39
|
+
export const MIN_SCALE = 0.5;
|
|
40
|
+
export const MAX_SCALE = 1.0;
|
|
39
41
|
function clampScale(scale) {
|
|
40
42
|
if (scale === undefined)
|
|
41
43
|
return MAX_SCALE;
|
|
@@ -69,8 +71,16 @@ export async function captureTiledScreenshot(url, options = {}) {
|
|
|
69
71
|
const truncated = isTileCaptureTruncated(captureHeight, geometries.length);
|
|
70
72
|
// N2: タイル毎にfullPageスクリーンショットを撮り直すのではなく、1回だけ撮影して
|
|
71
73
|
// @napi-rs/canvasでタイル領域をクロップする(deviceScaleFactor分、座標をscale倍する)。
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
//
|
|
75
|
+
// レビュー指摘対応(High): 以前はここでfullPage:trueをハードコードしており、
|
|
76
|
+
// 呼び出し元がfullPage:false(CLI --viewport-only / MCP fullPage:false)を指定しても
|
|
77
|
+
// 実際には全ページ分レンダリングしてしまい、遅延読み込み画像・無限スクロールXHR等の
|
|
78
|
+
// 追加リクエストを先方へ誘発していた(明示された低負荷指定を内部で裏切っていた)。
|
|
79
|
+
// fullPage:false時はビューポート(viewport height=DEFAULT_TILE_HEIGHTで撮影済み)のみを
|
|
80
|
+
// 実際に撮影する。captureHeight/geometries/crop座標は既にfullPageの値で分岐済みのため、
|
|
81
|
+
// 1タイル(ビューポート分)のみでも整合する。
|
|
82
|
+
const capturedPng = await page.screenshot({ type: "png", fullPage });
|
|
83
|
+
const image = await loadImage(capturedPng);
|
|
74
84
|
const tiles = geometries.map((geometry) => {
|
|
75
85
|
const sx = Math.min(Math.round(geometry.x * scale), Math.max(image.width - 1, 0));
|
|
76
86
|
const sy = Math.min(Math.round(geometry.y * scale), Math.max(image.height - 1, 0));
|
package/dist/screenshot.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../src/screenshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/E,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAS/C,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AACvC,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACxC,+CAA+C;AAC/C,MAAM,SAAS,GAAG,EAAE,CAAC;AAErB;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAiB,EACjB,UAAkB,EAClB,SAAS,GAAW,kBAAkB,EACtC,UAAU,GAAW,mBAAmB;IAExC,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAC7E,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;AACvG,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,sBAAsB,CAAC,UAAkB,EAAE,SAAiB,EAAE,UAAU,GAAW,mBAAmB;IACpH,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;IACpE,OAAO,WAAW,GAAG,SAAS,CAAC;AACjC,CAAC;AA0BD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,SAAS,GAAG,GAAG,CAAC;
|
|
1
|
+
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../src/screenshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/E,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAS/C,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AACvC,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACxC,+CAA+C;AAC/C,MAAM,SAAS,GAAG,EAAE,CAAC;AAErB;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAiB,EACjB,UAAkB,EAClB,SAAS,GAAW,kBAAkB,EACtC,UAAU,GAAW,mBAAmB;IAExC,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAC7E,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;AACvG,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,sBAAsB,CAAC,UAAkB,EAAE,SAAiB,EAAE,UAAU,GAAW,mBAAmB;IACpH,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;IACpE,OAAO,WAAW,GAAG,SAAS,CAAC;AACjC,CAAC;AA0BD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,8EAA8E;AAC9E,sDAAsD;AACtD,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAC7B,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAE7B,SAAS,UAAU,CAAC,KAAyB;IAC3C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,GAAW,EAAE,OAAO,GAAsB,EAAE;IACvF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,kBAAkB,CAAC;IAClD,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAE1D,2DAA2D;IAC3D,6DAA6D;IAC7D,sDAAsD;IACtD,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,SAAS,EAAE;QAClE,SAAS,EAAE,UAAU;QACrB,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE;QAChD,iBAAiB,EAAE,KAAK;KACzB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,gDAAgD;QAChD,MAAM,kBAAkB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,0BAA0B;QAC5B,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;YAC5C,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC;YACtD,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,YAAY,CAAC;SACzD,CAAC,CAAC,CAAC;QAEJ,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;QACtG,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,mBAAmB,CAAC,CAAC;QAC7F,MAAM,SAAS,GAAG,sBAAsB,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAE3E,gDAAgD;QAChD,gEAAgE;QAChE,EAAE;QACF,kDAAkD;QAClD,uEAAuE;QACvE,6CAA6C;QAC7C,2CAA2C;QAC3C,sEAAsE;QACtE,gEAAgE;QAChE,yBAAyB;QACzB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,WAAW,CAAC,CAAC;QAE3C,MAAM,KAAK,GAAqB,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClF,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACnF,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;YACvF,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;YAEzF,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACpC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACpC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAEnD,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAChH,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC"}
|
package/dist/server.d.ts
CHANGED