amenbo 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +11 -1
- package/README.md +11 -1
- package/dist/cache.d.ts +31 -1
- package/dist/cache.js +67 -7
- package/dist/cache.js.map +1 -1
- package/dist/cli.d.ts +3 -1
- package/dist/cli.js +59 -4
- package/dist/cli.js.map +1 -1
- package/dist/core.d.ts +6 -0
- package/dist/core.js +125 -38
- package/dist/core.js.map +1 -1
- package/dist/diff.d.ts +1 -1
- package/dist/diff.js +29 -9
- package/dist/diff.js.map +1 -1
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +13 -0
- package/dist/errors.js.map +1 -1
- package/dist/extract/dataSources.js +17 -4
- package/dist/extract/dataSources.js.map +1 -1
- package/dist/extract/outline.js +15 -1
- package/dist/extract/outline.js.map +1 -1
- package/dist/extract/pdf.js +2 -2
- package/dist/extract/pruning.d.ts +4 -2
- package/dist/extract/pruning.js +36 -7
- package/dist/extract/pruning.js.map +1 -1
- package/dist/extract/qualityScore.d.ts +3 -2
- package/dist/extract/qualityScore.js +8 -3
- package/dist/extract/qualityScore.js.map +1 -1
- package/dist/fetcher/browser.js +15 -1
- package/dist/fetcher/browser.js.map +1 -1
- package/dist/fetcher/http.d.ts +11 -1
- package/dist/fetcher/http.js +6 -1
- package/dist/fetcher/http.js.map +1 -1
- package/dist/fetcher/index.d.ts +6 -0
- package/dist/fetcher/index.js +18 -13
- package/dist/fetcher/index.js.map +1 -1
- package/dist/installBrowser.d.ts +11 -0
- package/dist/installBrowser.js +30 -0
- package/dist/installBrowser.js.map +1 -0
- package/dist/links.d.ts +12 -0
- package/dist/links.js +32 -17
- package/dist/links.js.map +1 -1
- package/dist/politeness.d.ts +23 -3
- package/dist/politeness.js +27 -6
- package/dist/politeness.js.map +1 -1
- package/dist/screenshot.d.ts +2 -0
- package/dist/screenshot.js +14 -4
- package/dist/screenshot.js.map +1 -1
- package/dist/server.d.ts +2 -0
- package/dist/server.js +55 -26
- package/dist/server.js.map +1 -1
- package/package.json +3 -6
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* amenboが依存する同梱playwrightパッケージのCLIエントリ(playwright/cli.js)を解決して
|
|
3
|
+
* `install chromium` を実行する。playwrightのグローバル/別バージョンではなく、
|
|
4
|
+
* amenboに同梱されたバージョンのCLIを使うことで、Chromiumのリビジョンとplaywright本体の
|
|
5
|
+
* バージョンが食い違う事態を原理的に防ぐ。
|
|
6
|
+
*
|
|
7
|
+
* "playwright/cli.js" はplaywrightのpackage.json exportsマップにサブパスとして
|
|
8
|
+
* 定義されていないため直接resolveできない。exportsに含まれる"playwright/package.json"を
|
|
9
|
+
* resolveし、そのディレクトリからcli.js(package.json"bin"が指す実体)を組み立てる。
|
|
10
|
+
*/
|
|
11
|
+
export declare function installBrowser(): Promise<number>;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* installBrowser.ts — `amenbo install-browser` サブコマンドの本体。
|
|
3
|
+
*
|
|
4
|
+
* Chromium遅延化(postinstall廃止)対応: package.jsonのpostinstallを撤去したため、
|
|
5
|
+
* ブラウザが必要な操作(SPA昇格・screenshot)を使う利用者は、このコマンドで明示的に
|
|
6
|
+
* 一度だけChromiumを取得する。
|
|
7
|
+
*/
|
|
8
|
+
import { createRequire } from "node:module";
|
|
9
|
+
import { spawn } from "node:child_process";
|
|
10
|
+
import { dirname, join } from "node:path";
|
|
11
|
+
/**
|
|
12
|
+
* amenboが依存する同梱playwrightパッケージのCLIエントリ(playwright/cli.js)を解決して
|
|
13
|
+
* `install chromium` を実行する。playwrightのグローバル/別バージョンではなく、
|
|
14
|
+
* amenboに同梱されたバージョンのCLIを使うことで、Chromiumのリビジョンとplaywright本体の
|
|
15
|
+
* バージョンが食い違う事態を原理的に防ぐ。
|
|
16
|
+
*
|
|
17
|
+
* "playwright/cli.js" はplaywrightのpackage.json exportsマップにサブパスとして
|
|
18
|
+
* 定義されていないため直接resolveできない。exportsに含まれる"playwright/package.json"を
|
|
19
|
+
* resolveし、そのディレクトリからcli.js(package.json"bin"が指す実体)を組み立てる。
|
|
20
|
+
*/
|
|
21
|
+
export async function installBrowser() {
|
|
22
|
+
const packageJsonPath = createRequire(import.meta.url).resolve("playwright/package.json");
|
|
23
|
+
const cliPath = join(dirname(packageJsonPath), "cli.js");
|
|
24
|
+
return new Promise((resolve, reject) => {
|
|
25
|
+
const child = spawn(process.execPath, [cliPath, "install", "chromium"], { stdio: "inherit" });
|
|
26
|
+
child.once("error", reject);
|
|
27
|
+
child.once("close", (code) => resolve(code ?? 1));
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=installBrowser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"installBrowser.js","sourceRoot":"","sources":["../src/installBrowser.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,eAAe,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAC1F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAC;IACzD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9F,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/links.d.ts
CHANGED
|
@@ -9,12 +9,24 @@ export interface LinksResult {
|
|
|
9
9
|
links: LinkEntry[];
|
|
10
10
|
/** MAX_LINKSを超えて切り捨てた場合true。 */
|
|
11
11
|
truncated: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* フィルタ適用前(重複排除後)の件数。
|
|
14
|
+
* filter指定時にlinks.length===0となった場合、これが0なら「sitemap/RSS/ページ自体が空」、
|
|
15
|
+
* 0より大きければ「filterに一致しなかった」と原因を切り分けられる(呼び出し側でのみ使う)。
|
|
16
|
+
*/
|
|
17
|
+
preFilterCount: number;
|
|
12
18
|
}
|
|
13
19
|
/** ページ内の<a href>リンクを抽出する(機能C: extract/dataSources.tsからも再利用される)。 */
|
|
14
20
|
export declare function extractPageLinks(html: string, baseUrl: string): LinkEntry[];
|
|
15
21
|
export interface DiscoverLinksOptions {
|
|
16
22
|
filter?: string;
|
|
17
23
|
timeoutMs?: number;
|
|
24
|
+
/**
|
|
25
|
+
* MCP progress notifications用。sitemapindexの子sitemap再帰・フィード取得を含め
|
|
26
|
+
* politeness待機が複数回発生しうる経路のため、累積待機が最も長引きうる。
|
|
27
|
+
* server.tsがprogressToken有無に応じて組み立てて渡す(CLIは渡さない=未指定で後方互換)。
|
|
28
|
+
*/
|
|
29
|
+
onProgress?: ((message: string) => void) | undefined;
|
|
18
30
|
}
|
|
19
31
|
/** URLからリンク一覧を発見する。sitemap → RSS/Atom → ページ内リンクの優先順で試す。 */
|
|
20
32
|
export declare function discoverLinks(url: string, politeness: PolitenessManager, options?: DiscoverLinksOptions): Promise<LinksResult>;
|
package/dist/links.js
CHANGED
|
@@ -32,18 +32,27 @@ const DEFAULT_TIMEOUT_MS = 15_000;
|
|
|
32
32
|
function escapeRegExp(value) {
|
|
33
33
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
34
34
|
}
|
|
35
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* filter未指定なら常に真。*を含む場合はglob、それ以外は部分一致。
|
|
37
|
+
*
|
|
38
|
+
* レビュー指摘対応: globは`^…$`で両端アンカーせず非アンカー(部分一致)にする。
|
|
39
|
+
* README/記事が例示する`--filter "blog/*"`は絶対URL(`https://example.com/blog/123`)全体とは
|
|
40
|
+
* 一致しないため、アンカー版だと必ず0件になっていた。URL/タイトルの一部にglobパターンが
|
|
41
|
+
* 現れれば真とする(通常の部分一致フィルタと同じ「含む」感覚に揃える)。
|
|
42
|
+
*/
|
|
36
43
|
function matchesFilter(value, filter) {
|
|
37
44
|
if (!filter)
|
|
38
45
|
return true;
|
|
39
46
|
if (filter.includes("*")) {
|
|
40
|
-
const pattern = new RegExp(
|
|
47
|
+
const pattern = new RegExp(filter.split("*").map(escapeRegExp).join(".*"), "i");
|
|
41
48
|
return pattern.test(value);
|
|
42
49
|
}
|
|
43
50
|
return value.toLowerCase().includes(filter.toLowerCase());
|
|
44
51
|
}
|
|
45
|
-
async function fetchXml(url, timeoutMs, rewrite) {
|
|
46
|
-
|
|
52
|
+
async function fetchXml(url, timeoutMs, politeness, rewrite) {
|
|
53
|
+
// レビュー指摘対応: リダイレクトで別オリジンへ着地した場合、着地先のrobots.txtも確認する
|
|
54
|
+
// (fetcher/http.tsのHttpGetOptions.checkRobots参照)。
|
|
55
|
+
const result = await httpGet(url, { timeoutMs, checkRobots: (targetUrl) => politeness.checkRobotsAllowed(targetUrl) });
|
|
47
56
|
const { document } = parseHTML(rewrite ? rewrite(result.html) : result.html);
|
|
48
57
|
return document;
|
|
49
58
|
}
|
|
@@ -56,11 +65,11 @@ async function fetchXml(url, timeoutMs, rewrite) {
|
|
|
56
65
|
function escapeRssLinkTag(xml) {
|
|
57
66
|
return xml.replace(/<link>([\s\S]*?)<\/link>/gi, "<rss-link>$1</rss-link>");
|
|
58
67
|
}
|
|
59
|
-
async function tryParseSitemap(sitemapUrl, politeness, timeoutMs, depth = 0) {
|
|
68
|
+
async function tryParseSitemap(sitemapUrl, politeness, timeoutMs, depth = 0, onProgress) {
|
|
60
69
|
let document;
|
|
61
70
|
try {
|
|
62
|
-
await politeness.guard(sitemapUrl);
|
|
63
|
-
document = await fetchXml(sitemapUrl, timeoutMs);
|
|
71
|
+
await politeness.guard(sitemapUrl, onProgress);
|
|
72
|
+
document = await fetchXml(sitemapUrl, timeoutMs, politeness);
|
|
64
73
|
}
|
|
65
74
|
catch (error) {
|
|
66
75
|
if (isFallbackBlockingError(error))
|
|
@@ -80,7 +89,7 @@ async function tryParseSitemap(sitemapUrl, politeness, timeoutMs, depth = 0) {
|
|
|
80
89
|
const childUrl = loc.textContent.trim();
|
|
81
90
|
if (!childUrl)
|
|
82
91
|
continue;
|
|
83
|
-
const childLinks = await tryParseSitemap(childUrl, politeness, timeoutMs, depth + 1);
|
|
92
|
+
const childLinks = await tryParseSitemap(childUrl, politeness, timeoutMs, depth + 1, onProgress);
|
|
84
93
|
if (childLinks)
|
|
85
94
|
results.push(...childLinks);
|
|
86
95
|
if (results.length >= MAX_LINKS)
|
|
@@ -93,12 +102,12 @@ async function tryParseSitemap(sitemapUrl, politeness, timeoutMs, depth = 0) {
|
|
|
93
102
|
return null;
|
|
94
103
|
return urlLocs.map((loc) => ({ url: loc.textContent.trim(), title: null })).filter((entry) => entry.url.length > 0);
|
|
95
104
|
}
|
|
96
|
-
async function tryParseFeed(feedUrl, politeness, timeoutMs) {
|
|
105
|
+
async function tryParseFeed(feedUrl, politeness, timeoutMs, onProgress) {
|
|
97
106
|
let document;
|
|
98
107
|
try {
|
|
99
|
-
await politeness.guard(feedUrl);
|
|
108
|
+
await politeness.guard(feedUrl, onProgress);
|
|
100
109
|
// RSS 2.0の<link>はHTMLパーサにvoid要素として扱われるためリネームして回避する(下記コメント参照)
|
|
101
|
-
document = await fetchXml(feedUrl, timeoutMs, escapeRssLinkTag);
|
|
110
|
+
document = await fetchXml(feedUrl, timeoutMs, politeness, escapeRssLinkTag);
|
|
102
111
|
}
|
|
103
112
|
catch (error) {
|
|
104
113
|
if (isFallbackBlockingError(error))
|
|
@@ -175,8 +184,14 @@ function dedupeByUrl(links) {
|
|
|
175
184
|
return result;
|
|
176
185
|
}
|
|
177
186
|
function finalize(source, links, filter) {
|
|
178
|
-
const
|
|
179
|
-
|
|
187
|
+
const deduped = dedupeByUrl(links);
|
|
188
|
+
const filtered = deduped.filter((link) => matchesFilter(link.url, filter) || matchesFilter(link.title ?? "", filter));
|
|
189
|
+
return {
|
|
190
|
+
source,
|
|
191
|
+
links: filtered.slice(0, MAX_LINKS),
|
|
192
|
+
truncated: filtered.length > MAX_LINKS,
|
|
193
|
+
preFilterCount: deduped.length,
|
|
194
|
+
};
|
|
180
195
|
}
|
|
181
196
|
/** URLからリンク一覧を発見する。sitemap → RSS/Atom → ページ内リンクの優先順で試す。 */
|
|
182
197
|
export async function discoverLinks(url, politeness, options = {}) {
|
|
@@ -190,14 +205,14 @@ export async function discoverLinks(url, politeness, options = {}) {
|
|
|
190
205
|
const declaredSitemaps = await politeness.getSitemaps(url);
|
|
191
206
|
const candidateSitemaps = declaredSitemaps.length > 0 ? declaredSitemaps : [`${origin}/sitemap.xml`];
|
|
192
207
|
for (const sitemapUrl of candidateSitemaps) {
|
|
193
|
-
const links = await tryParseSitemap(sitemapUrl, politeness, timeoutMs);
|
|
208
|
+
const links = await tryParseSitemap(sitemapUrl, politeness, timeoutMs, 0, options.onProgress);
|
|
194
209
|
if (links && links.length > 0) {
|
|
195
210
|
return finalize("sitemap", links, options.filter);
|
|
196
211
|
}
|
|
197
212
|
}
|
|
198
213
|
// 2. ページを取得し、<link rel=alternate>が指すRSS/Atomフィードを試す
|
|
199
|
-
await politeness.guard(url);
|
|
200
|
-
const pageResult = await fetchPage(url, { timeoutMs });
|
|
214
|
+
await politeness.guard(url, options.onProgress);
|
|
215
|
+
const pageResult = await fetchPage(url, { timeoutMs, checkRobots: (targetUrl) => politeness.checkRobotsAllowed(targetUrl) });
|
|
201
216
|
if ("notModified" in pageResult) {
|
|
202
217
|
return finalize("page", [], options.filter);
|
|
203
218
|
}
|
|
@@ -212,7 +227,7 @@ export async function discoverLinks(url, politeness, options = {}) {
|
|
|
212
227
|
const feedHref = feedLinkEl?.getAttribute("href");
|
|
213
228
|
if (feedHref) {
|
|
214
229
|
const feedUrl = new URL(feedHref, pageResult.finalUrl).toString();
|
|
215
|
-
const feedLinks = await tryParseFeed(feedUrl, politeness, timeoutMs);
|
|
230
|
+
const feedLinks = await tryParseFeed(feedUrl, politeness, timeoutMs, options.onProgress);
|
|
216
231
|
if (feedLinks && feedLinks.length > 0) {
|
|
217
232
|
return finalize("rss", feedLinks, options.filter);
|
|
218
233
|
}
|
package/dist/links.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"links.js","sourceRoot":"","sources":["../src/links.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAC/G,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG9D;;;;;;;;;GASG;AACH,SAAS,uBAAuB,CAAC,KAAc;IAC7C,OAAO,KAAK,YAAY,mBAAmB,IAAI,KAAK,YAAY,iBAAiB,CAAC;AACpF,CAAC;
|
|
1
|
+
{"version":3,"file":"links.js","sourceRoot":"","sources":["../src/links.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAC/G,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG9D;;;;;;;;;GASG;AACH,SAAS,uBAAuB,CAAC,KAAc;IAC7C,OAAO,KAAK,YAAY,mBAAmB,IAAI,KAAK,YAAY,iBAAiB,CAAC;AACpF,CAAC;AAsBD,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,iDAAiD;AACjD,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,aAAa,CAAC,KAAa,EAAE,MAA0B;IAC9D,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAChF,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;AAC5D,CAAC;AAaD,KAAK,UAAU,QAAQ,CACrB,GAAW,EACX,SAAiB,EACjB,UAA6B,EAC7B,OAAiC;IAEjC,oDAAoD;IACpD,kDAAkD;IAClD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACvH,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7E,OAAO,QAAsC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,OAAO,GAAG,CAAC,OAAO,CAAC,4BAA4B,EAAE,yBAAyB,CAAC,CAAC;AAC9E,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,UAAkB,EAClB,UAA6B,EAC7B,SAAiB,EACjB,KAAK,GAAG,CAAC,EACT,UAAoD;IAEpD,IAAI,QAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/C,QAAQ,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,uBAAuB,CAAC,KAAK,CAAC;YAAE,MAAM,KAAK,CAAC;QAChD,qDAAqD;QACrD,2CAA2C;QAC3C,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,CAAC,sBAAsB,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gDAAgD;IAChD,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;IACxF,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACxC,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YACxB,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,UAAU,CAAC,CAAC;YACjG,IAAI,UAAU;gBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;YAC5C,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS;gBAAE,MAAM;QACzC,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACtH,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,OAAe,EACf,UAA6B,EAC7B,SAAiB,EACjB,UAAoD;IAEpD,IAAI,QAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC5C,6DAA6D;QAC7D,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC;IAC9E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,uBAAuB,CAAC,KAAK,CAAC;YAAE,MAAM,KAAK,CAAC;QAChD,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,CAAC,mBAAmB,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,KAAK;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACnE,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;YACzE,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,KAAK,EAAsB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC;IACzC,CAAC;IAED,OAAO;IACP,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;IACnE,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,WAAW;aACxB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACb,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;YAC3E,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;YAC1E,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,KAAK,EAAsB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC;IACzC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,OAAe;IAC5D,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAE1E,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,SAAS;QAC1C,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,SAAS;QACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;QACxE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAkB;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,MAAkB,EAAE,KAAkB,EAAE,MAA0B;IAClF,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IACtH,OAAO;QACL,MAAM;QACN,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;QACnC,SAAS,EAAE,QAAQ,CAAC,MAAM,GAAG,SAAS;QACtC,cAAc,EAAE,OAAO,CAAC,MAAM;KAC/B,CAAC;AACJ,CAAC;AAaD,2DAA2D;AAC3D,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAW,EAAE,UAA6B,EAAE,OAAO,GAAyB,EAAE;IAChH,6DAA6D;IAC7D,4DAA4D;IAC5D,uCAAuC;IACvC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACtB,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IAEnC,oDAAoD;IACpD,MAAM,gBAAgB,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;IACrG,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QAC9F,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,QAAQ,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,MAAM,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAC7H,IAAI,aAAa,IAAI,UAAU,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,uDAAuD;IACvD,gEAAgE;IAChE,iCAAiC;IACjC,IAAI,SAAS,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,IAAI,uBAAuB,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,WAAW,IAAI,MAAM,CAAC,CAAC;IAC3F,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CACvC,uGAAuG,CACxG,CAAC;IACF,MAAM,QAAQ,GAAG,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAClD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAC;QAClE,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACzF,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,OAAO,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,OAAO,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAClG,CAAC"}
|
package/dist/politeness.d.ts
CHANGED
|
@@ -16,6 +16,19 @@ export interface PolitenessStore {
|
|
|
16
16
|
getLastRequestAt(host: string): number | null;
|
|
17
17
|
setLastRequestAt(host: string, at: number): void;
|
|
18
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* レビュー指摘対応: robots.txt取得結果をプロセス間で共有するための永続化ストア。
|
|
21
|
+
* PolitenessStoreと同じ考え方で、cache.ts(robots_cacheテーブル)をこの形で注入する想定。
|
|
22
|
+
* CLIは1コマンド=1プロセスのため、これが無いとREADME推奨の「CLIで複数ページ一括収集」時に
|
|
23
|
+
* コマンド毎にrobots.txtを再取得してしまい低負荷原則に反する。
|
|
24
|
+
*/
|
|
25
|
+
export interface RobotsStore {
|
|
26
|
+
get(origin: string): {
|
|
27
|
+
body: string;
|
|
28
|
+
fetchedAt: number;
|
|
29
|
+
} | null;
|
|
30
|
+
set(origin: string, body: string, fetchedAt: number): void;
|
|
31
|
+
}
|
|
19
32
|
export interface PolitenessOptions {
|
|
20
33
|
/** ドメイン毎の最小リクエスト間隔(ミリ秒、既定1000ms)。robots.txtのCrawl-Delayがこれより長ければそちらを優先。 */
|
|
21
34
|
minIntervalMs?: number;
|
|
@@ -27,6 +40,8 @@ export interface PolitenessOptions {
|
|
|
27
40
|
sleep?: (ms: number) => Promise<void>;
|
|
28
41
|
/** 未指定時は従来通りインメモリのみでレート制御する(完全後方互換)。 */
|
|
29
42
|
store?: PolitenessStore;
|
|
43
|
+
/** 未指定時は従来通りインメモリのみでrobots.txtをキャッシュする(完全後方互換)。 */
|
|
44
|
+
robotsStore?: RobotsStore;
|
|
30
45
|
}
|
|
31
46
|
export declare class PolitenessManager {
|
|
32
47
|
private readonly minIntervalMs;
|
|
@@ -34,12 +49,16 @@ export declare class PolitenessManager {
|
|
|
34
49
|
private readonly now;
|
|
35
50
|
private readonly sleep;
|
|
36
51
|
private readonly store;
|
|
52
|
+
private readonly robotsStore;
|
|
37
53
|
private readonly robotsCache;
|
|
38
54
|
private readonly locks;
|
|
39
55
|
private readonly lastRequestAt;
|
|
40
56
|
constructor(options?: PolitenessOptions);
|
|
41
|
-
/**
|
|
42
|
-
|
|
57
|
+
/**
|
|
58
|
+
* robots.txtとレート制御の両方を確認し、アクセス可能になるまで待機する。拒否時はRobotsDeniedErrorを投げる。
|
|
59
|
+
* onProgress: MCP progress notifications用。実際に待機が発生する場合のみ呼ばれる(待機0の場合は呼ばれない)。
|
|
60
|
+
*/
|
|
61
|
+
guard(url: string, onProgress?: ((message: string) => void) | undefined): Promise<void>;
|
|
43
62
|
/** robots.txtのみ確認する(waitTurnは行わない)。 */
|
|
44
63
|
checkRobotsAllowed(url: string): Promise<void>;
|
|
45
64
|
/** robots.txtが宣言するSitemap URLの一覧を返す(linksツールのsitemap優先探索用)。 */
|
|
@@ -49,8 +68,9 @@ export declare class PolitenessManager {
|
|
|
49
68
|
* 直列キュー(locks)による直列化はプロセス内のみ有効。store(PolitenessStore)経由の
|
|
50
69
|
* プロセス間共有はread-modify-writeがアトミックではないベストエフォート(PolitenessStoreの
|
|
51
70
|
* JSDoc参照)。
|
|
71
|
+
* onProgress: 実際にsleepが発生する場合のみ呼ばれる(待機0のときは呼ばれない)。
|
|
52
72
|
*/
|
|
53
|
-
waitTurn(url: string): Promise<void>;
|
|
73
|
+
waitTurn(url: string, onProgress?: ((message: string) => void) | undefined): Promise<void>;
|
|
54
74
|
/**
|
|
55
75
|
* N8: robotsCache/locks/lastRequestAtは訪問したホスト毎に1エントリずつ増え、長時間稼働する
|
|
56
76
|
* MCPサーバープロセスでは無制限に増加しうる。robotsTtlMsを「エントリの生存期間」の目安として
|
package/dist/politeness.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* politeness.ts — 収集先への低負荷を担保するpolitenessマネージャ。
|
|
3
3
|
*
|
|
4
|
-
* - robots.txt尊重(robots-parser)。取得結果はプロセス内でキャッシュ(TTL付き)
|
|
4
|
+
* - robots.txt尊重(robots-parser)。取得結果はプロセス内でキャッシュ(TTL付き)し、
|
|
5
|
+
* robotsStore注入時はプロセス間でも共有する(CLI併設対応)
|
|
5
6
|
* - Crawl-Delay尊重
|
|
6
7
|
* - ドメイン毎の直列キュー + 既定最小間隔1秒
|
|
7
8
|
*/
|
|
@@ -21,6 +22,7 @@ export class PolitenessManager {
|
|
|
21
22
|
now;
|
|
22
23
|
sleep;
|
|
23
24
|
store;
|
|
25
|
+
robotsStore;
|
|
24
26
|
robotsCache = new Map();
|
|
25
27
|
// N8: settledを持たせるのは、pruneStaleHostStateがまだ完了していない(=キューに並んでいる
|
|
26
28
|
// 呼び出しがある)ロックを誤って消してしまうと直列化が壊れる(同一ホストへの同時呼び出しが
|
|
@@ -34,11 +36,15 @@ export class PolitenessManager {
|
|
|
34
36
|
this.now = options.now ?? Date.now;
|
|
35
37
|
this.sleep = options.sleep ?? defaultSleep;
|
|
36
38
|
this.store = options.store;
|
|
39
|
+
this.robotsStore = options.robotsStore;
|
|
37
40
|
}
|
|
38
|
-
/**
|
|
39
|
-
|
|
41
|
+
/**
|
|
42
|
+
* robots.txtとレート制御の両方を確認し、アクセス可能になるまで待機する。拒否時はRobotsDeniedErrorを投げる。
|
|
43
|
+
* onProgress: MCP progress notifications用。実際に待機が発生する場合のみ呼ばれる(待機0の場合は呼ばれない)。
|
|
44
|
+
*/
|
|
45
|
+
async guard(url, onProgress) {
|
|
40
46
|
await this.checkRobotsAllowed(url);
|
|
41
|
-
await this.waitTurn(url);
|
|
47
|
+
await this.waitTurn(url, onProgress);
|
|
42
48
|
}
|
|
43
49
|
/** robots.txtのみ確認する(waitTurnは行わない)。 */
|
|
44
50
|
async checkRobotsAllowed(url) {
|
|
@@ -57,8 +63,9 @@ export class PolitenessManager {
|
|
|
57
63
|
* 直列キュー(locks)による直列化はプロセス内のみ有効。store(PolitenessStore)経由の
|
|
58
64
|
* プロセス間共有はread-modify-writeがアトミックではないベストエフォート(PolitenessStoreの
|
|
59
65
|
* JSDoc参照)。
|
|
66
|
+
* onProgress: 実際にsleepが発生する場合のみ呼ばれる(待機0のときは呼ばれない)。
|
|
60
67
|
*/
|
|
61
|
-
async waitTurn(url) {
|
|
68
|
+
async waitTurn(url, onProgress) {
|
|
62
69
|
this.pruneStaleHostState();
|
|
63
70
|
const host = new URL(url).host;
|
|
64
71
|
const crawlDelayMs = await this.getCrawlDelayMs(url);
|
|
@@ -80,6 +87,7 @@ export class PolitenessManager {
|
|
|
80
87
|
const lastAt = Math.max(memoryLastAt, storeLastAt);
|
|
81
88
|
const elapsed = this.now() - lastAt;
|
|
82
89
|
if (elapsed < intervalMs) {
|
|
90
|
+
onProgress?.("アクセス間隔を確保するため待機しています…");
|
|
83
91
|
await this.sleep(intervalMs - elapsed);
|
|
84
92
|
}
|
|
85
93
|
const requestAt = this.now();
|
|
@@ -139,6 +147,17 @@ export class PolitenessManager {
|
|
|
139
147
|
if (cached && this.now() - cached.fetchedAt < this.robotsTtlMs) {
|
|
140
148
|
return cached.robots;
|
|
141
149
|
}
|
|
150
|
+
// レビュー指摘対応: インメモリキャッシュに無い/TTL超過の場合、ネットワークへ行く前に
|
|
151
|
+
// robotsStore(cache.tsのrobots_cacheテーブル。プロセス間共有)を確認する。ここで
|
|
152
|
+
// 復元できれば別プロセス(CLIの前回実行等)が取得済みのrobots.txtを再利用でき、
|
|
153
|
+
// CLIバルク収集時の再取得(=先方への追加負荷)を避けられる。空bodyの記録(robots.txt
|
|
154
|
+
// 不在=制限なし)もfetchedAtの有無で「未記録」と区別して正しく復元する。
|
|
155
|
+
const stored = this.robotsStore?.get(origin);
|
|
156
|
+
if (stored && this.now() - stored.fetchedAt < this.robotsTtlMs) {
|
|
157
|
+
const robots = robotsParser(`${origin}/robots.txt`, stored.body);
|
|
158
|
+
this.robotsCache.set(origin, { robots, fetchedAt: stored.fetchedAt });
|
|
159
|
+
return robots;
|
|
160
|
+
}
|
|
142
161
|
const robotsUrl = `${origin}/robots.txt`;
|
|
143
162
|
let body = "";
|
|
144
163
|
try {
|
|
@@ -163,7 +182,9 @@ export class PolitenessManager {
|
|
|
163
182
|
body = "";
|
|
164
183
|
}
|
|
165
184
|
const robots = robotsParser(robotsUrl, body);
|
|
166
|
-
|
|
185
|
+
const fetchedAt = this.now();
|
|
186
|
+
this.robotsCache.set(origin, { robots, fetchedAt });
|
|
187
|
+
this.robotsStore?.set(origin, body, fetchedAt);
|
|
167
188
|
return robots;
|
|
168
189
|
}
|
|
169
190
|
}
|
package/dist/politeness.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"politeness.js","sourceRoot":"","sources":["../src/politeness.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"politeness.js","sourceRoot":"","sources":["../src/politeness.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,kBAAkB,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAiBxD,MAAM,YAAY,GAAG,kBAA0E,CAAC;AAEhG,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,MAAM;AACpD,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAoDrC,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,OAAO,iBAAiB;IACX,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,GAAG,CAAe;IAClB,KAAK,CAAgC;IACrC,KAAK,CAA8B;IACnC,WAAW,CAA0B;IAErC,WAAW,GAAG,IAAI,GAAG,EAA4B,CAAC;IACnE,8DAA8D;IAC9D,+CAA+C;IAC/C,gDAAgD;IAChD,YAAY;IACK,KAAK,GAAG,IAAI,GAAG,EAAsD,CAAC;IACtE,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3D,YAAY,OAAO,GAAsB,EAAE;QACzC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC;QACtE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,qBAAqB,CAAC;QAChE,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAAoD;QAC3E,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACvC,CAAC;IAED,uCAAuC;IACvC,KAAK,CAAC,kBAAkB,CAAC,GAAW;QAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,KAAK,KAAK,EAAE,CAAC;YAC1D,MAAM,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,WAAW,CAAC,GAAW;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,OAAO,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACrC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,UAAoD;QAC9E,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;QAC/B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAE9D,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,aAAa,EAAE,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3D,IAAI,OAAO,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5C,OAAO,GAAG,OAAO,CAAC;QACpB,CAAC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAE5B,MAAM,QAAQ,CAAC;QACf,IAAI,CAAC;YACH,yDAAyD;YACzD,uDAAuD;YACvD,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;YACpE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;YACpC,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;gBACzB,UAAU,EAAE,CAAC,uBAAuB,CAAC,CAAC;gBACtC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC;YACzC,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YACxC,IAAI,CAAC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAChD,CAAC;gBAAS,CAAC;YACT,OAAO,EAAE,CAAC;YACV,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,mBAAmB;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;QAE7C,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAChD,IAAI,MAAM,IAAI,MAAM;gBAAE,SAAS;YAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,KAAK,CAAC,SAAS,GAAG,MAAM,EAAE,CAAC;gBAC7B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;QACnD,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,qEAAqE;QACrE,wDAAwD;QACxD,mEAAmE;QACnE,qDAAqD;QACrD,gEAAgE;QAChE,gEAAgE;QAChE,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/D,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;QAED,+CAA+C;QAC/C,0DAA0D;QAC1D,gDAAgD;QAChD,qDAAqD;QACrD,2CAA2C;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/D,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,MAAM,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACjE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YACtE,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,MAAM,aAAa,CAAC;QACzC,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,uBAAuB,EAAE,CAAC,CAAC;YAChF,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,qDAAqD;YACrD,EAAE;YACF,uDAAuD;YACvD,8CAA8C;YAC9C,iDAAiD;YACjD,2DAA2D;YAC3D,2CAA2C;YAC3C,kCAAkC;YAClC,IAAI,KAAK,YAAY,mBAAmB;gBAAE,MAAM,KAAK,CAAC;YACtD,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,EAAE,CAAC;gBACxC,OAAO,CAAC,KAAK,CAAC,sCAAsC,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
package/dist/screenshot.d.ts
CHANGED
|
@@ -34,5 +34,7 @@ export interface CaptureResult {
|
|
|
34
34
|
/** N7: MAX_TILES枚を超えるページで切り捨てが発生した場合true。 */
|
|
35
35
|
truncated: boolean;
|
|
36
36
|
}
|
|
37
|
+
export declare const MIN_SCALE = 0.5;
|
|
38
|
+
export declare const MAX_SCALE = 1;
|
|
37
39
|
/** URLをブラウザでレンダリングし、タイル分割したPNGスクリーンショット群を撮影する。 */
|
|
38
40
|
export declare function captureTiledScreenshot(url: string, options?: ScreenshotOptions): Promise<CaptureResult>;
|
package/dist/screenshot.js
CHANGED
|
@@ -34,8 +34,10 @@ export function isTileCaptureTruncated(pageHeight, tileCount, tileHeight = DEFAU
|
|
|
34
34
|
return tilesNeeded > tileCount;
|
|
35
35
|
}
|
|
36
36
|
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
// レビュー指摘対応(Low): CLI側(cli.ts parseScreenshotArgs)がMCPのzod .min(0.5).max(1.0)と
|
|
38
|
+
// 同じ範囲で検証できるよう、この定数をexportしてCLIと共有する(値の重複定義によるずれを防ぐ)。
|
|
39
|
+
export const MIN_SCALE = 0.5;
|
|
40
|
+
export const MAX_SCALE = 1.0;
|
|
39
41
|
function clampScale(scale) {
|
|
40
42
|
if (scale === undefined)
|
|
41
43
|
return MAX_SCALE;
|
|
@@ -69,8 +71,16 @@ export async function captureTiledScreenshot(url, options = {}) {
|
|
|
69
71
|
const truncated = isTileCaptureTruncated(captureHeight, geometries.length);
|
|
70
72
|
// N2: タイル毎にfullPageスクリーンショットを撮り直すのではなく、1回だけ撮影して
|
|
71
73
|
// @napi-rs/canvasでタイル領域をクロップする(deviceScaleFactor分、座標をscale倍する)。
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
//
|
|
75
|
+
// レビュー指摘対応(High): 以前はここでfullPage:trueをハードコードしており、
|
|
76
|
+
// 呼び出し元がfullPage:false(CLI --viewport-only / MCP fullPage:false)を指定しても
|
|
77
|
+
// 実際には全ページ分レンダリングしてしまい、遅延読み込み画像・無限スクロールXHR等の
|
|
78
|
+
// 追加リクエストを先方へ誘発していた(明示された低負荷指定を内部で裏切っていた)。
|
|
79
|
+
// fullPage:false時はビューポート(viewport height=DEFAULT_TILE_HEIGHTで撮影済み)のみを
|
|
80
|
+
// 実際に撮影する。captureHeight/geometries/crop座標は既にfullPageの値で分岐済みのため、
|
|
81
|
+
// 1タイル(ビューポート分)のみでも整合する。
|
|
82
|
+
const capturedPng = await page.screenshot({ type: "png", fullPage });
|
|
83
|
+
const image = await loadImage(capturedPng);
|
|
74
84
|
const tiles = geometries.map((geometry) => {
|
|
75
85
|
const sx = Math.min(Math.round(geometry.x * scale), Math.max(image.width - 1, 0));
|
|
76
86
|
const sy = Math.min(Math.round(geometry.y * scale), Math.max(image.height - 1, 0));
|
package/dist/screenshot.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../src/screenshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/E,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAS/C,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AACvC,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACxC,+CAA+C;AAC/C,MAAM,SAAS,GAAG,EAAE,CAAC;AAErB;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAiB,EACjB,UAAkB,EAClB,SAAS,GAAW,kBAAkB,EACtC,UAAU,GAAW,mBAAmB;IAExC,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAC7E,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;AACvG,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,sBAAsB,CAAC,UAAkB,EAAE,SAAiB,EAAE,UAAU,GAAW,mBAAmB;IACpH,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;IACpE,OAAO,WAAW,GAAG,SAAS,CAAC;AACjC,CAAC;AA0BD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,SAAS,GAAG,GAAG,CAAC;
|
|
1
|
+
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../src/screenshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/E,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAS/C,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AACvC,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACxC,+CAA+C;AAC/C,MAAM,SAAS,GAAG,EAAE,CAAC;AAErB;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAiB,EACjB,UAAkB,EAClB,SAAS,GAAW,kBAAkB,EACtC,UAAU,GAAW,mBAAmB;IAExC,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAC7E,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;AACvG,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,sBAAsB,CAAC,UAAkB,EAAE,SAAiB,EAAE,UAAU,GAAW,mBAAmB;IACpH,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;IACpE,OAAO,WAAW,GAAG,SAAS,CAAC;AACjC,CAAC;AA0BD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,8EAA8E;AAC9E,sDAAsD;AACtD,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAC7B,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAE7B,SAAS,UAAU,CAAC,KAAyB;IAC3C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,GAAW,EAAE,OAAO,GAAsB,EAAE;IACvF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,kBAAkB,CAAC;IAClD,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAE1D,2DAA2D;IAC3D,6DAA6D;IAC7D,sDAAsD;IACtD,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,SAAS,EAAE;QAClE,SAAS,EAAE,UAAU;QACrB,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE;QAChD,iBAAiB,EAAE,KAAK;KACzB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,gDAAgD;QAChD,MAAM,kBAAkB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,0BAA0B;QAC5B,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;YAC5C,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC;YACtD,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,YAAY,CAAC;SACzD,CAAC,CAAC,CAAC;QAEJ,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;QACtG,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,mBAAmB,CAAC,CAAC;QAC7F,MAAM,SAAS,GAAG,sBAAsB,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAE3E,gDAAgD;QAChD,gEAAgE;QAChE,EAAE;QACF,kDAAkD;QAClD,uEAAuE;QACvE,6CAA6C;QAC7C,2CAA2C;QAC3C,sEAAsE;QACtE,gEAAgE;QAChE,yBAAyB;QACzB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,WAAW,CAAC,CAAC;QAE3C,MAAM,KAAK,GAAqB,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClF,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACnF,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;YACvF,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;YAEzF,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACpC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACpC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAEnD,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAChH,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC"}
|
package/dist/server.d.ts
CHANGED
package/dist/server.js
CHANGED
|
@@ -17,31 +17,55 @@ import { fileURLToPath } from "node:url";
|
|
|
17
17
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
18
18
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
19
19
|
import { z } from "zod";
|
|
20
|
-
import { formatLinksResponse, handleFetchTool, handleScreenshotTool, politeness, resolvePackageVersion, } from "./core.js";
|
|
20
|
+
import { formatLinksResponse, handleFetchTool, handleScreenshotTool, politeness, registerCoreShutdownHandlers, resolvePackageVersion, } from "./core.js";
|
|
21
21
|
import { AmenboError } from "./errors.js";
|
|
22
22
|
import { discoverLinks } from "./links.js";
|
|
23
|
-
|
|
23
|
+
// テスト用export(既存スタイルに合わせた最小のテスト可能化。InMemoryTransport経由の
|
|
24
|
+
// MCP progress notifications統合テストでclient.connect()の相手として使う)。
|
|
25
|
+
export const server = new McpServer({ name: "amenbo", version: resolvePackageVersion() });
|
|
26
|
+
/**
|
|
27
|
+
* MCP progress notifications: リクエストにprogressToken(_meta.progressToken)が付いている
|
|
28
|
+
* クライアントに対してのみ、進捗メッセージをホストUIへ通知するコールバックを組み立てる。
|
|
29
|
+
* progressTokenが無い(=進捗通知に対応しない/希望しないクライアント)場合はundefinedを返し、
|
|
30
|
+
* core.ts側の各通知ポイントは呼び出し自体を行わない(条件分岐のみでオーバーヘッドは実質ゼロ)。
|
|
31
|
+
*
|
|
32
|
+
* sendNotificationはtotalを送らない(処理量が事前に不定なため)。失敗はツール本処理を
|
|
33
|
+
* 壊さないようベストエフォートで握りつぶし、原因のみstderrへ記録する。
|
|
34
|
+
*/
|
|
35
|
+
function buildProgressNotifier(extra) {
|
|
36
|
+
const progressToken = extra._meta?.progressToken;
|
|
37
|
+
if (progressToken === undefined)
|
|
38
|
+
return undefined;
|
|
39
|
+
let progress = 0;
|
|
40
|
+
return (message) => {
|
|
41
|
+
progress += 1;
|
|
42
|
+
void extra.sendNotification({ method: "notifications/progress", params: { progressToken, progress, message } }).catch((error) => {
|
|
43
|
+
console.error("進捗通知(notifications/progress)の送信に失敗しました:", error);
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
}
|
|
24
47
|
server.registerTool("fetch", {
|
|
25
48
|
title: "Fetch a web page as Markdown",
|
|
26
|
-
description: "
|
|
27
|
-
"mode: auto(
|
|
28
|
-
"
|
|
49
|
+
description: "Fetch a web page (Japanese-web-native) as low-impact, token-efficient Markdown. Built-in robots.txt compliance, rate limiting, and caching. " +
|
|
50
|
+
"mode: auto (default; quality score picks Markdown or screenshot) / markdown / outline (heading summary) / screenshot. " +
|
|
51
|
+
"Refetching a cached URL returns cache: unchanged, or diff (changed sections only), to save tokens. PDF URLs are handled automatically.",
|
|
29
52
|
inputSchema: {
|
|
30
|
-
url: z.string().url().describe("
|
|
31
|
-
mode: z.enum(["auto", "markdown", "outline", "screenshot"]).optional().describe("
|
|
32
|
-
selector: z.string().optional().describe("
|
|
33
|
-
section: z.string().optional().describe("outline
|
|
34
|
-
page: z.number().int().positive().optional().describe("
|
|
35
|
-
max_tokens: z.number().int().positive().optional().describe("
|
|
36
|
-
force_full: z.boolean().optional().describe("
|
|
53
|
+
url: z.string().url().describe("Target URL (http/https only; PDF supported)"),
|
|
54
|
+
mode: z.enum(["auto", "markdown", "outline", "screenshot"]).optional().describe("Default: auto"),
|
|
55
|
+
selector: z.string().optional().describe("CSS selector to narrow the content"),
|
|
56
|
+
section: z.string().optional().describe("Section ID obtained from outline mode; returns only that section's Markdown"),
|
|
57
|
+
page: z.number().int().positive().optional().describe("Page number (default 1)"),
|
|
58
|
+
max_tokens: z.number().int().positive().optional().describe("Approximate token budget per page (default 8000)"),
|
|
59
|
+
force_full: z.boolean().optional().describe("Default false. If true, disables diff responses (unchanged/diff) and boilerplate-block removal, always returning the full content"),
|
|
37
60
|
},
|
|
38
61
|
annotations: {
|
|
39
62
|
readOnlyHint: true,
|
|
40
63
|
openWorldHint: true,
|
|
41
64
|
},
|
|
42
|
-
}, async ({ url, mode, selector, section, page, max_tokens: maxTokens, force_full: forceFull }) => {
|
|
65
|
+
}, async ({ url, mode, selector, section, page, max_tokens: maxTokens, force_full: forceFull }, extra) => {
|
|
43
66
|
try {
|
|
44
|
-
const
|
|
67
|
+
const onProgress = buildProgressNotifier(extra);
|
|
68
|
+
const content = await handleFetchTool({ url, mode, selector, section, page, max_tokens: maxTokens, force_full: forceFull, onProgress });
|
|
45
69
|
return { content };
|
|
46
70
|
}
|
|
47
71
|
catch (error) {
|
|
@@ -54,18 +78,19 @@ server.registerTool("fetch", {
|
|
|
54
78
|
});
|
|
55
79
|
server.registerTool("links", {
|
|
56
80
|
title: "List links from a page (sitemap/RSS-first)",
|
|
57
|
-
description: "sitemap.xml/RSS
|
|
81
|
+
description: "Low-impact link discovery: prefers sitemap.xml / RSS / Atom feeds when available, otherwise extracts in-page links.",
|
|
58
82
|
inputSchema: {
|
|
59
|
-
url: z.string().url().describe("
|
|
60
|
-
filter: z.string().optional().describe("URL
|
|
83
|
+
url: z.string().url().describe("Starting URL"),
|
|
84
|
+
filter: z.string().optional().describe("Substring match against URL/link text, or a glob using *"),
|
|
61
85
|
},
|
|
62
86
|
annotations: {
|
|
63
87
|
readOnlyHint: true,
|
|
64
88
|
openWorldHint: true,
|
|
65
89
|
},
|
|
66
|
-
}, async ({ url, filter }) => {
|
|
90
|
+
}, async ({ url, filter }, extra) => {
|
|
67
91
|
try {
|
|
68
|
-
const
|
|
92
|
+
const onProgress = buildProgressNotifier(extra);
|
|
93
|
+
const result = await discoverLinks(url, politeness, { ...(filter ? { filter } : {}), onProgress });
|
|
69
94
|
return { content: [{ type: "text", text: formatLinksResponse(url, result) }] };
|
|
70
95
|
}
|
|
71
96
|
catch (error) {
|
|
@@ -78,20 +103,21 @@ server.registerTool("links", {
|
|
|
78
103
|
});
|
|
79
104
|
server.registerTool("screenshot", {
|
|
80
105
|
title: "Capture a tiled screenshot of a web page",
|
|
81
|
-
description: "
|
|
106
|
+
description: "For explicit visual inspection. Renders the page with a headless browser and returns tiled PNG screenshots. Built-in robots.txt compliance, rate limiting, and caching.",
|
|
82
107
|
inputSchema: {
|
|
83
|
-
url: z.string().url().describe("
|
|
84
|
-
fullPage: z.boolean().optional().describe("
|
|
85
|
-
width: z.number().int().positive().optional().describe("
|
|
86
|
-
scale: z.number().min(0.5).max(1.0).optional().describe("
|
|
108
|
+
url: z.string().url().describe("Target URL (http/https only)"),
|
|
109
|
+
fullPage: z.boolean().optional().describe("Default true. If false, captures only the first viewport (1 tile)"),
|
|
110
|
+
width: z.number().int().positive().optional().describe("Tile width in px (default 1280)"),
|
|
111
|
+
scale: z.number().min(0.5).max(1.0).optional().describe("Resolution scale (0.5-1.0, default 1.0); lower reduces image tokens"),
|
|
87
112
|
},
|
|
88
113
|
annotations: {
|
|
89
114
|
readOnlyHint: true,
|
|
90
115
|
openWorldHint: true,
|
|
91
116
|
},
|
|
92
|
-
}, async ({ url, fullPage, width, scale }) => {
|
|
117
|
+
}, async ({ url, fullPage, width, scale }, extra) => {
|
|
93
118
|
try {
|
|
94
|
-
const
|
|
119
|
+
const onProgress = buildProgressNotifier(extra);
|
|
120
|
+
const content = await handleScreenshotTool({ url, fullPage, width, scale, onProgress });
|
|
95
121
|
return { content };
|
|
96
122
|
}
|
|
97
123
|
catch (error) {
|
|
@@ -104,6 +130,9 @@ server.registerTool("screenshot", {
|
|
|
104
130
|
});
|
|
105
131
|
/** MCP stdioサーバーを起動する。CLI(cli.ts)が引数なし/`serve`サブコマンド時に呼ぶ後方互換経路でもある。 */
|
|
106
132
|
export async function runServer() {
|
|
133
|
+
// レビュー指摘対応: exit/SIGINT/SIGTERMハンドラ(core.ts)はサーバー常駐プロセスでのみ
|
|
134
|
+
// 必要なため、importするだけで走るモジュールトップレベルではなくここで明示的に登録する。
|
|
135
|
+
registerCoreShutdownHandlers();
|
|
107
136
|
const transport = new StdioServerTransport();
|
|
108
137
|
await server.connect(transport);
|
|
109
138
|
}
|