@nahisaho/katashiro-collector 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web-scraper.d.ts","sourceRoot":"","sources":["../../src/scraper/web-scraper.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,KAAK,MAAM,EAMZ,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhF;;;GAGG;AACH,qBAAa,UAAW,YAAW,WAAW;IAC5C;;OAEG;IACG,MAAM,CACV,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAiBzC;;OAEG;YACW,SAAS;
|
|
1
|
+
{"version":3,"file":"web-scraper.d.ts","sourceRoot":"","sources":["../../src/scraper/web-scraper.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,KAAK,MAAM,EAMZ,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhF;;;GAGG;AACH,qBAAa,UAAW,YAAW,WAAW;IAC5C;;OAEG;IACG,MAAM,CACV,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAiBzC;;OAEG;YACW,SAAS;IAsBvB;;OAEG;IACH,OAAO,CAAC,SAAS;IAqBjB;;OAEG;IACH,OAAO,CAAC,YAAY;IAKpB;;OAEG;IACH,OAAO,CAAC,cAAc;IAqBtB;;OAEG;IACH,OAAO,CAAC,aAAa;IAgBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAgBpB;;OAEG;IACH,OAAO,CAAC,UAAU;CAUnB"}
|
|
@@ -35,14 +35,14 @@ export class WebScraper {
|
|
|
35
35
|
*/
|
|
36
36
|
async fetchPage(url, options) {
|
|
37
37
|
const userAgent = options?.userAgent ?? 'Mozilla/5.0 (compatible; KATASHIRO/0.1.0)';
|
|
38
|
+
// デフォルトタイムアウト: 30秒（存在しないドメインへの無限待機を防止）
|
|
39
|
+
const timeout = options?.timeout ?? 30000;
|
|
38
40
|
const response = await fetch(url, {
|
|
39
41
|
headers: {
|
|
40
42
|
'User-Agent': userAgent,
|
|
41
43
|
Accept: 'text/html,application/xhtml+xml',
|
|
42
44
|
},
|
|
43
|
-
signal:
|
|
44
|
-
? AbortSignal.timeout(options.timeout)
|
|
45
|
-
: undefined,
|
|
45
|
+
signal: AbortSignal.timeout(timeout),
|
|
46
46
|
});
|
|
47
47
|
if (!response.ok) {
|
|
48
48
|
throw new Error(`HTTP error: ${response.status}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web-scraper.js","sourceRoot":"","sources":["../../src/scraper/web-scraper.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAEL,EAAE,EACF,GAAG,EACH,eAAe,EACf,WAAW,EACX,KAAK,GACN,MAAM,0BAA0B,CAAC;AAGlC;;;GAGG;AACH,MAAM,OAAO,UAAU;IACrB;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,GAAW,EACX,OAAyB;QAEzB,QAAQ;QACR,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,gBAAgB,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YAClD,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YACzE,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,mBAAmB,OAAO,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,OAAyB;QAC5D,MAAM,SAAS,GACb,OAAO,EAAE,SAAS,IAAI,2CAA2C,CAAC;QAEpE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO,EAAE;gBACP,YAAY,EAAE,SAAS;gBACvB,MAAM,EAAE,iCAAiC;aAC1C;YACD,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"web-scraper.js","sourceRoot":"","sources":["../../src/scraper/web-scraper.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAEL,EAAE,EACF,GAAG,EACH,eAAe,EACf,WAAW,EACX,KAAK,GACN,MAAM,0BAA0B,CAAC;AAGlC;;;GAGG;AACH,MAAM,OAAO,UAAU;IACrB;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,GAAW,EACX,OAAyB;QAEzB,QAAQ;QACR,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,gBAAgB,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YAClD,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YACzE,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,mBAAmB,OAAO,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,OAAyB;QAC5D,MAAM,SAAS,GACb,OAAO,EAAE,SAAS,IAAI,2CAA2C,CAAC;QAEpE,uCAAuC;QACvC,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAK,CAAC;QAE1C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO,EAAE;gBACP,YAAY,EAAE,SAAS;gBACvB,MAAM,EAAE,iCAAiC;aAC1C;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,SAAS,CACf,IAAY,EACZ,GAAW,EACX,OAAyB;QAEzB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,MAAM,KAAK,GAAG,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAE/E,OAAO;YACL,GAAG;YACH,KAAK;YACL,OAAO;YACP,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC5C,MAAM;YACN,KAAK;YACL,SAAS,EAAE,eAAe,EAAE;SAC7B,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAY;QAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAC/D,OAAO,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,UAAU,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,IAAY;QACjC,2CAA2C;QAC3C,IAAI,OAAO,GAAG,IAAI;aACf,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC;aAChD,OAAO,CAAC,iCAAiC,EAAE,EAAE,CAAC;aAC9C,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;aAC1C,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC;aAChD,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAC;QAEpD,YAAY;QACZ,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAE3C,SAAS;QACT,OAAO,GAAG,OAAO;aACd,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC;aACzB,IAAI,EAAE,CAAC;QAEV,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,IAAY,EAAE,OAAe;QACjD,MAAM,QAAQ,GAAG,iCAAiC,CAAC;QACnD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC;QAEV,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACrB,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAY,EAAE,OAAe;QAChD,MAAM,SAAS,GAAG,gCAAgC,CAAC;QACnD,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC;QAEV,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBACrE,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBACnD,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW,EAAE,OAAe;QAC7C,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5D,OAAO,GAAG,CAAC;QACb,CAAC;QACD,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;CACF"}
|