@j0hanz/superfetch 1.1.9 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -469
- package/dist/config/constants.d.ts +19 -0
- package/dist/config/constants.d.ts.map +1 -0
- package/dist/config/constants.js +24 -0
- package/dist/config/constants.js.map +1 -0
- package/dist/config/formatting.d.ts +0 -2
- package/dist/config/formatting.d.ts.map +1 -1
- package/dist/config/formatting.js +1 -3
- package/dist/config/formatting.js.map +1 -1
- package/dist/config/index.d.ts +9 -3
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +19 -16
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +1 -20
- package/dist/config/types/content.d.ts.map +1 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/runtime.d.ts +7 -5
- package/dist/config/types/runtime.d.ts.map +1 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/tools.d.ts +5 -50
- package/dist/config/types/tools.d.ts.map +1 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/errors/app-error.d.ts +0 -1
- package/dist/errors/app-error.js +0 -1
- package/dist/http/auth.d.ts +0 -1
- package/dist/http/auth.d.ts.map +1 -1
- package/dist/http/auth.js +17 -13
- package/dist/http/auth.js.map +1 -1
- package/dist/http/cors.d.ts +0 -1
- package/dist/http/cors.js +4 -1
- package/dist/http/cors.js.map +1 -1
- package/dist/http/download-routes.d.ts +14 -0
- package/dist/http/download-routes.d.ts.map +1 -0
- package/dist/http/download-routes.js +131 -0
- package/dist/http/download-routes.js.map +1 -0
- package/dist/http/mcp-routes.d.ts +1 -2
- package/dist/http/mcp-routes.d.ts.map +1 -1
- package/dist/http/mcp-routes.js +1 -2
- package/dist/http/mcp-routes.js.map +1 -1
- package/dist/http/mcp-session-helpers.d.ts +13 -0
- package/dist/http/mcp-session-helpers.d.ts.map +1 -0
- package/dist/http/mcp-session-helpers.js +64 -0
- package/dist/http/mcp-session-helpers.js.map +1 -0
- package/dist/http/mcp-session.d.ts +1 -3
- package/dist/http/mcp-session.d.ts.map +1 -1
- package/dist/http/mcp-session.js +7 -71
- package/dist/http/mcp-session.js.map +1 -1
- package/dist/http/mcp-validation.d.ts +1 -2
- package/dist/http/mcp-validation.d.ts.map +1 -1
- package/dist/http/mcp-validation.js +6 -27
- package/dist/http/mcp-validation.js.map +1 -1
- package/dist/http/rate-limit.d.ts +1 -2
- package/dist/http/rate-limit.d.ts.map +1 -1
- package/dist/http/rate-limit.js +0 -1
- package/dist/http/rate-limit.js.map +1 -1
- package/dist/http/server-middleware.d.ts +9 -0
- package/dist/http/server-middleware.d.ts.map +1 -0
- package/dist/http/server-middleware.js +111 -0
- package/dist/http/server-middleware.js.map +1 -0
- package/dist/http/server.d.ts +0 -1
- package/dist/http/server.d.ts.map +1 -1
- package/dist/http/server.js +20 -99
- package/dist/http/server.js.map +1 -1
- package/dist/http/session-cleanup.d.ts +2 -0
- package/dist/http/session-cleanup.d.ts.map +1 -0
- package/dist/http/session-cleanup.js +37 -0
- package/dist/http/session-cleanup.js.map +1 -0
- package/dist/http/sessions.d.ts +1 -2
- package/dist/http/sessions.d.ts.map +1 -1
- package/dist/http/sessions.js +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +13 -6
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +0 -1
- package/dist/middleware/error-handler.js +0 -1
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +76 -12
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/index.js +0 -1
- package/dist/server.d.ts +0 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +8 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +7 -4
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +86 -26
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +2 -2
- package/dist/services/context.d.ts.map +1 -1
- package/dist/services/context.js +0 -1
- package/dist/services/extractor.d.ts +1 -2
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +45 -18
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher/agents.d.ts +0 -1
- package/dist/services/fetcher/agents.d.ts.map +1 -1
- package/dist/services/fetcher/agents.js +3 -7
- package/dist/services/fetcher/agents.js.map +1 -1
- package/dist/services/fetcher/errors.d.ts +0 -1
- package/dist/services/fetcher/errors.js +0 -1
- package/dist/services/fetcher/headers.d.ts.map +1 -1
- package/dist/services/fetcher/headers.js +2 -24
- package/dist/services/fetcher/headers.js.map +1 -1
- package/dist/services/fetcher/interceptors.d.ts +2 -2
- package/dist/services/fetcher/interceptors.d.ts.map +1 -1
- package/dist/services/fetcher/interceptors.js +30 -21
- package/dist/services/fetcher/interceptors.js.map +1 -1
- package/dist/services/fetcher/redirects.d.ts +0 -2
- package/dist/services/fetcher/redirects.d.ts.map +1 -1
- package/dist/services/fetcher/redirects.js +20 -18
- package/dist/services/fetcher/redirects.js.map +1 -1
- package/dist/services/fetcher/response.d.ts +0 -1
- package/dist/services/fetcher/response.js +4 -5
- package/dist/services/fetcher/retry-policy.d.ts +1 -28
- package/dist/services/fetcher/retry-policy.d.ts.map +1 -1
- package/dist/services/fetcher/retry-policy.js +119 -126
- package/dist/services/fetcher/retry-policy.js.map +1 -1
- package/dist/services/fetcher.d.ts +1 -2
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +18 -13
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +1 -2
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +0 -1
- package/dist/services/parser.d.ts +1 -3
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +5 -39
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +15 -19
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -2
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +50 -20
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +14 -3
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.js +66 -3
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +39 -17
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls/validation.js +1 -1
- package/dist/tools/handlers/fetch-urls/validation.js.map +1 -1
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +1 -20
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/schemas.d.ts +57 -250
- package/dist/tools/schemas.d.ts.map +1 -1
- package/dist/tools/schemas.js +38 -198
- package/dist/tools/schemas.js.map +1 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -2
- package/dist/tools/utils/cache-vary.d.ts.map +1 -1
- package/dist/tools/utils/cache-vary.js +8 -40
- package/dist/tools/utils/cache-vary.js.map +1 -1
- package/dist/tools/utils/common.d.ts +2 -4
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +6 -7
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +1 -3
- package/dist/tools/utils/content-transform.d.ts.map +1 -1
- package/dist/tools/utils/content-transform.js +65 -14
- package/dist/tools/utils/content-transform.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +1 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +25 -21
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/tools/utils/inline-content.d.ts +3 -3
- package/dist/tools/utils/inline-content.d.ts.map +1 -1
- package/dist/tools/utils/inline-content.js +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +0 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -2
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +11 -7
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/code-language.d.ts +2 -0
- package/dist/utils/code-language.d.ts.map +1 -0
- package/dist/utils/code-language.js +56 -0
- package/dist/utils/code-language.js.map +1 -0
- package/dist/utils/content-cleaner.d.ts +0 -2
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +0 -4
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/crypto.d.ts +2 -0
- package/dist/utils/crypto.d.ts.map +1 -0
- package/dist/utils/crypto.js +32 -0
- package/dist/utils/crypto.js.map +1 -0
- package/dist/utils/download-url.d.ts +8 -0
- package/dist/utils/download-url.d.ts.map +1 -0
- package/dist/utils/download-url.js +27 -0
- package/dist/utils/download-url.js.map +1 -0
- package/dist/utils/error-utils.d.ts +3 -0
- package/dist/utils/error-utils.d.ts.map +1 -0
- package/dist/utils/error-utils.js +12 -0
- package/dist/utils/error-utils.js.map +1 -0
- package/dist/utils/filename-generator.d.ts +1 -0
- package/dist/utils/filename-generator.d.ts.map +1 -0
- package/dist/utils/filename-generator.js +59 -0
- package/dist/utils/filename-generator.js.map +1 -0
- package/dist/utils/header-normalizer.d.ts +7 -4
- package/dist/utils/header-normalizer.d.ts.map +1 -1
- package/dist/utils/header-normalizer.js +23 -17
- package/dist/utils/header-normalizer.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.js +0 -1
- package/dist/utils/sanitizer.d.ts +0 -1
- package/dist/utils/sanitizer.js +0 -1
- package/dist/utils/tool-error-handler.d.ts +1 -3
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +11 -6
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-sanitizer.d.ts +2 -0
- package/dist/utils/url-sanitizer.d.ts.map +1 -0
- package/dist/utils/url-sanitizer.js +12 -0
- package/dist/utils/url-sanitizer.js.map +1 -0
- package/dist/utils/url-validator.d.ts +1 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +89 -53
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +7 -9
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cached-content.js","sourceRoot":"","sources":["../../src/resources/cached-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"cached-content.js","sourceRoot":"","sources":["../../src/resources/cached-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAE3E,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,oCAAoC,CAAC;AAEzE,OAAO,KAAK,KAAK,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE1D,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;AAC/D,MAAM,YAAY,GAAG,eAAe,CAAC;AAOrC,SAAS,kBAAkB,CACzB,SAAiB,EACjB,OAAe;IAOf,OAAO;QACL,IAAI,EAAE,GAAG,SAAS,IAAI,OAAO,EAAE;QAC/B,GAAG,EAAE,sBAAsB,SAAS,IAAI,OAAO,EAAE;QACjD,WAAW,EAAE,4BAA4B,SAAS,EAAE;QACpD,QAAQ,EAAE,oBAAoB,CAAC,SAAS,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB;IAG1B,MAAM,SAAS,GAAG,KAAK;SACpB,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACvC,OAAO,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,KAAK,EAAsC,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;IAEzE,OAAO,EAAE,SAAS,EAAE,CAAC;AACvB,CAAC;AAED,SAAS,qBAAqB;IAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC/B,OAAO;QACL,YAAY,EAAE,SAAS,CAAC,MAAM;QAC9B,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,SAAS,GAAG,KAAK,EAAE,SAAS,IAAI,SAAS,CAAC;YAChD,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,SAAS,CAAC;YAC5C,OAAO;gBACL,SAAS;gBACT,OAAO;gBACP,WAAW,EAAE,sBAAsB,SAAS,IAAI,OAAO,EAAE;aAC1D,CAAC;QACJ,CAAC,CAAC;QACF,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAiB,EAAE,GAAW;IAC1D,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;QAAE,OAAO;IAClC,KAAK,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACvE,OAAO,CAAC,6CAA6C,EAAE;YACrD,GAAG;YACH,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAiB;IAC7D,4BAA4B,CAAC,MAAM,CAAC,CAAC;IACrC,yBAAyB,CAAC,MAAM,CAAC,CAAC;IAClC,+BAA+B,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,MAA+B;IAIzD,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnD,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,QAAQ,CAChB,SAAS,CAAC,aAAa,EACvB,oDAAoD,CACrD,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,QAAQ,CAChB,SAAS,CAAC,aAAa,EACvB,mCAAmC,CACpC,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,0BAA0B,CACjC,GAAQ,EACR,QAAgB,EAChB,SAAiB;IAEjB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAEnC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,QAAQ,CAChB,SAAS,CAAC,aAAa,EACvB,uCAAuC,QAAQ,0DAA0D,CAC1G,CAAC;IACJ,CAAC;IAED,IAAI,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;QACpD,OAAO;YACL,QAAQ,EAAE;gBACR;oBACE,GAAG,EAAE,GAAG,CAAC,IAAI;oBACb,QAAQ,EAAE,oBAAoB,CAAC,SAAS,CAAC;oBACzC,IAAI,EAAE,MAAM,CAAC,OAAO;iBACrB;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnD,MAAM,eAAe,GAAG,OAAO;QAC7B,CAAC,CAAC,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC;QAC3C,CAAC,CAAC,IAAI,CAAC;IAET,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,MAAM,IAAI,QAAQ,CAChB,SAAS,CAAC,aAAa,EACvB,2CAA2C,SAAS,EAAE,CACvD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ,EAAE;YACR;gBACE,GAAG,EAAE,GAAG,CAAC,IAAI;gBACb,QAAQ,EAAE,oBAAoB,CAAC,SAAS,CAAC;gBACzC,IAAI,EAAE,eAAe;aACtB;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,4BAA4B,CAAC,MAAiB;IACrD,MAAM,CAAC,gBAAgB,CACrB,gBAAgB,EAChB,IAAI,gBAAgB,CAAC,0CAA0C,EAAE;QAC/D,IAAI,EAAE,mBAAmB;KAC1B,CAAC,EACF;QACE,KAAK,EAAE,gBAAgB;QACvB,WAAW,EACT,sHAAsH;QACxH,QAAQ,EAAE,YAAY;KACvB,EACD,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;QACd,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,kBAAkB,CAC/C,MAAiC,CAClC,CAAC;QACF,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,OAAO,EAAE,CAAC;QAC3C,OAAO,0BAA0B,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC9D,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAAC,MAAiB;IAClD,MAAM,CAAC,gBAAgB,CACrB,aAAa,EACb,yBAAyB,EACzB;QACE,KAAK,EAAE,kBAAkB;QACzB,WAAW,EAAE,wDAAwD;QACrE,QAAQ,EAAE,kBAAkB;KAC7B,EACD,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACR,QAAQ,EAAE;YACR;gBACE,GAAG,EAAE,GAAG,CAAC,IAAI;gBACb,QAAQ,EAAE,kBAAkB;gBAC5B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,qBAAqB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;aACvD;SACF;KACF,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,+BAA+B,CAAC,MAAiB;IACxD,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;QACvD,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW;YAAE,OAAO;QAEzB,oBAAoB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC1C,oBAAoB,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;QACxD,IAAI,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;YACzB,MAAM,CAAC,uBAAuB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;IAC9C,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;QAC3B,eAAe,EAAE,EAAE,CAAC;QACpB,WAAW,EAAE,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,SAAiB;IAC7C,IAAI,SAAS,KAAK,UAAU;QAAE,OAAO,eAAe,CAAC;IACrD,IAAI,SAAS,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAC;IACpD,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB;IACzC,OAAO,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClD,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAC;QAC1C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACzC,OAAO,MAAuB,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAsB,EACtB,SAAiB;IAEjB,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;QAC7B,IAAI,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACzC,OAAO,OAAO,CAAC,QAAQ,CAAC;QAC1B,CAAC;QACD,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACxC,OAAO,OAAO,CAAC,OAAO,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AACtE,CAAC"}
|
package/dist/resources/index.js
CHANGED
package/dist/server.d.ts
CHANGED
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAYpE,wBAAgB,eAAe,IAAI,SAAS,CAoB3C;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAYpE,wBAAgB,eAAe,IAAI,SAAS,CAoB3C;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CA2CtD"}
|
package/dist/server.js
CHANGED
|
@@ -27,8 +27,8 @@ export async function startStdioServer() {
|
|
|
27
27
|
server.server.onerror = (error) => {
|
|
28
28
|
logError('[MCP Error]', error instanceof Error ? error : { error });
|
|
29
29
|
};
|
|
30
|
-
|
|
31
|
-
process.stderr.write(
|
|
30
|
+
const handleShutdown = (signal) => {
|
|
31
|
+
process.stderr.write(`\n${signal} received, shutting down superFetch MCP server...\n`);
|
|
32
32
|
destroyAgents();
|
|
33
33
|
server
|
|
34
34
|
.close()
|
|
@@ -38,6 +38,12 @@ export async function startStdioServer() {
|
|
|
38
38
|
.finally(() => {
|
|
39
39
|
process.exit(0);
|
|
40
40
|
});
|
|
41
|
+
};
|
|
42
|
+
process.on('SIGINT', () => {
|
|
43
|
+
handleShutdown('SIGINT');
|
|
44
|
+
});
|
|
45
|
+
process.on('SIGTERM', () => {
|
|
46
|
+
handleShutdown('SIGTERM');
|
|
41
47
|
});
|
|
42
48
|
try {
|
|
43
49
|
await server.connect(transport);
|
|
@@ -48,4 +54,3 @@ export async function startStdioServer() {
|
|
|
48
54
|
process.exit(1);
|
|
49
55
|
}
|
|
50
56
|
}
|
|
51
|
-
//# sourceMappingURL=server.js.map
|
package/dist/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B;QACE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;QACxB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;KAC/B,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE;YAC7B,SAAS,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;YACjD,OAAO,EAAE,EAAE;SACZ;QACD,YAAY,EAAE,0BAA0B,MAAM,CAAC,MAAM,CAAC,OAAO,8OAA8O;KAC5S,CACF,CAAC;IAEF,aAAa,CAAC,MAAM,CAAC,CAAC;IACtB,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE1B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE7C,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;QAChC,QAAQ,CAAC,aAAa,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC,CAAC;IAEF,
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B;QACE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;QACxB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;KAC/B,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE;YAC7B,SAAS,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;YACjD,OAAO,EAAE,EAAE;SACZ;QACD,YAAY,EAAE,0BAA0B,MAAM,CAAC,MAAM,CAAC,OAAO,8OAA8O;KAC5S,CACF,CAAC;IAEF,aAAa,CAAC,MAAM,CAAC,CAAC;IACtB,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE1B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE7C,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;QAChC,QAAQ,CAAC,aAAa,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,CAAC,MAAc,EAAQ,EAAE;QAC9C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,MAAM,qDAAqD,CACjE,CAAC;QACF,aAAa,EAAE,CAAC;QAChB,MAAM;aACH,KAAK,EAAE;aACP,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,QAAQ,CACN,uBAAuB,EACvB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CACvC,CAAC;QACJ,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;IAEF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;QACzB,cAAc,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,CAAC,wCAAwC,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,8BAA8B,EAC9B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
|
package/dist/services/cache.d.ts
CHANGED
|
@@ -1,18 +1,21 @@
|
|
|
1
|
-
import type { CacheEntry } from '../config/types.js';
|
|
2
|
-
|
|
1
|
+
import type { CacheEntry } from '../config/types/content.js';
|
|
2
|
+
interface CacheKeyParts {
|
|
3
3
|
namespace: string;
|
|
4
4
|
urlHash: string;
|
|
5
5
|
}
|
|
6
6
|
interface CacheUpdateEvent extends CacheKeyParts {
|
|
7
7
|
cacheKey: string;
|
|
8
8
|
}
|
|
9
|
+
interface CacheEntryMetadata {
|
|
10
|
+
url: string;
|
|
11
|
+
title?: string;
|
|
12
|
+
}
|
|
9
13
|
type CacheUpdateListener = (event: CacheUpdateEvent) => void;
|
|
10
14
|
export declare function createCacheKey(namespace: string, url: string, vary?: Record<string, unknown> | string): string | null;
|
|
11
15
|
export declare function parseCacheKey(cacheKey: string): CacheKeyParts | null;
|
|
12
16
|
export declare function toResourceUri(cacheKey: string): string | null;
|
|
13
17
|
export declare function onCacheUpdate(listener: CacheUpdateListener): () => void;
|
|
14
18
|
export declare function get(cacheKey: string | null): CacheEntry | undefined;
|
|
15
|
-
export declare function set(cacheKey: string | null, content: string): void;
|
|
19
|
+
export declare function set(cacheKey: string | null, content: string, metadata: CacheEntryMetadata): void;
|
|
16
20
|
export declare function keys(): string[];
|
|
17
21
|
export {};
|
|
18
|
-
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAwD7D,UAAU,aAAa;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,gBAAiB,SAAQ,aAAa;IAC9C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,kBAAkB;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,mBAAmB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;AA2D7D,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GACtC,MAAM,GAAG,IAAI,CAMf;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAMpE;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAI7D;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,mBAAmB,GAAG,MAAM,IAAI,CAKvE;AAUD,wBAAgB,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,SAAS,CAYnE;AAsBD,wBAAgB,GAAG,CACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,kBAAkB,GAC3B,IAAI,CAeN;AAED,wBAAgB,IAAI,IAAI,MAAM,EAAE,CAE/B"}
|
package/dist/services/cache.js
CHANGED
|
@@ -1,13 +1,51 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
1
|
+
import { setInterval as setIntervalPromise } from 'node:timers/promises';
|
|
2
|
+
import { CACHE_HASH } from '../config/constants.js';
|
|
3
3
|
import { config } from '../config/index.js';
|
|
4
|
+
import { sha256Hex } from '../utils/crypto.js';
|
|
5
|
+
import { getErrorMessage } from '../utils/error-utils.js';
|
|
4
6
|
import { logWarn } from './logger.js';
|
|
5
|
-
const contentCache = new
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
const contentCache = new Map();
|
|
8
|
+
let cleanupController = null;
|
|
9
|
+
function startCleanupLoop() {
|
|
10
|
+
if (cleanupController)
|
|
11
|
+
return;
|
|
12
|
+
cleanupController = new AbortController();
|
|
13
|
+
void runCleanupLoop(cleanupController.signal).catch((error) => {
|
|
14
|
+
if (error instanceof Error && error.name !== 'AbortError') {
|
|
15
|
+
logWarn('Cache cleanup loop failed', { error: getErrorMessage(error) });
|
|
16
|
+
}
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
async function runCleanupLoop(signal) {
|
|
20
|
+
const intervalMs = Math.floor(config.cache.ttl * 1000);
|
|
21
|
+
for await (const _ of setIntervalPromise(intervalMs, undefined, {
|
|
22
|
+
signal,
|
|
23
|
+
ref: false,
|
|
24
|
+
})) {
|
|
25
|
+
evictExpiredEntries();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function evictExpiredEntries() {
|
|
29
|
+
const now = Date.now();
|
|
30
|
+
for (const [key, item] of contentCache.entries()) {
|
|
31
|
+
if (now > item.expiresAt) {
|
|
32
|
+
contentCache.delete(key);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
enforceMaxKeys();
|
|
36
|
+
}
|
|
37
|
+
function enforceMaxKeys() {
|
|
38
|
+
if (contentCache.size <= config.cache.maxKeys)
|
|
39
|
+
return;
|
|
40
|
+
const keysToRemove = contentCache.size - config.cache.maxKeys;
|
|
41
|
+
const iterator = contentCache.keys();
|
|
42
|
+
for (let i = 0; i < keysToRemove; i++) {
|
|
43
|
+
const { value, done } = iterator.next();
|
|
44
|
+
if (done)
|
|
45
|
+
break;
|
|
46
|
+
contentCache.delete(value);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
11
49
|
const updateListeners = new Set();
|
|
12
50
|
function stableStringify(value) {
|
|
13
51
|
if (value === null || value === undefined) {
|
|
@@ -26,8 +64,16 @@ function stableStringify(value) {
|
|
|
26
64
|
return `{${entries.join(',')}}`;
|
|
27
65
|
}
|
|
28
66
|
function createHashFragment(input, length) {
|
|
29
|
-
return
|
|
67
|
+
return sha256Hex(input).substring(0, length);
|
|
30
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Constructs a cache key from namespace, URL hash, and optional vary hash.
|
|
71
|
+
* Format: "namespace:urlHash" or "namespace:urlHash.varyHash" if vary params exist.
|
|
72
|
+
* @param namespace - Cache namespace (e.g., "fetch-markdown")
|
|
73
|
+
* @param urlHash - SHA-256 hash of the URL (truncated to 16 chars)
|
|
74
|
+
* @param varyHash - Optional hash of vary parameters (e.g., headers, options)
|
|
75
|
+
* @returns Complete cache key string
|
|
76
|
+
*/
|
|
31
77
|
function buildCacheKey(namespace, urlHash, varyHash) {
|
|
32
78
|
return varyHash
|
|
33
79
|
? `${namespace}:${urlHash}.${varyHash}`
|
|
@@ -39,12 +85,12 @@ function getVaryHash(vary) {
|
|
|
39
85
|
const varyString = typeof vary === 'string' ? vary : stableStringify(vary);
|
|
40
86
|
if (!varyString)
|
|
41
87
|
return undefined;
|
|
42
|
-
return createHashFragment(varyString,
|
|
88
|
+
return createHashFragment(varyString, CACHE_HASH.VARY_HASH_LENGTH);
|
|
43
89
|
}
|
|
44
90
|
export function createCacheKey(namespace, url, vary) {
|
|
45
91
|
if (!namespace || !url)
|
|
46
92
|
return null;
|
|
47
|
-
const urlHash = createHashFragment(url,
|
|
93
|
+
const urlHash = createHashFragment(url, CACHE_HASH.URL_HASH_LENGTH);
|
|
48
94
|
const varyHash = getVaryHash(vary);
|
|
49
95
|
return buildCacheKey(namespace, urlHash, varyHash);
|
|
50
96
|
}
|
|
@@ -78,21 +124,36 @@ function emitCacheUpdate(cacheKey) {
|
|
|
78
124
|
}
|
|
79
125
|
}
|
|
80
126
|
export function get(cacheKey) {
|
|
81
|
-
if (!
|
|
127
|
+
if (!isCacheReadable(cacheKey))
|
|
82
128
|
return undefined;
|
|
83
|
-
}
|
|
84
129
|
try {
|
|
85
|
-
return
|
|
130
|
+
return readCacheEntry(cacheKey);
|
|
86
131
|
}
|
|
87
132
|
catch (error) {
|
|
88
133
|
logWarn('Cache get error', {
|
|
89
134
|
key: cacheKey.substring(0, 100),
|
|
90
|
-
error: error
|
|
135
|
+
error: getErrorMessage(error),
|
|
91
136
|
});
|
|
92
137
|
return undefined;
|
|
93
138
|
}
|
|
94
139
|
}
|
|
95
|
-
|
|
140
|
+
function isCacheReadable(cacheKey) {
|
|
141
|
+
return config.cache.enabled && Boolean(cacheKey);
|
|
142
|
+
}
|
|
143
|
+
function readCacheEntry(cacheKey) {
|
|
144
|
+
const item = contentCache.get(cacheKey);
|
|
145
|
+
if (!item)
|
|
146
|
+
return undefined;
|
|
147
|
+
if (isExpired(item)) {
|
|
148
|
+
contentCache.delete(cacheKey);
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
return item.entry;
|
|
152
|
+
}
|
|
153
|
+
function isExpired(item) {
|
|
154
|
+
return Date.now() > item.expiresAt;
|
|
155
|
+
}
|
|
156
|
+
export function set(cacheKey, content, metadata) {
|
|
96
157
|
if (!config.cache.enabled)
|
|
97
158
|
return;
|
|
98
159
|
if (!cacheKey)
|
|
@@ -100,32 +161,31 @@ export function set(cacheKey, content) {
|
|
|
100
161
|
if (!content)
|
|
101
162
|
return;
|
|
102
163
|
try {
|
|
103
|
-
|
|
164
|
+
startCleanupLoop();
|
|
165
|
+
const entry = buildCacheEntry(cacheKey, content, metadata);
|
|
104
166
|
persistCacheEntry(cacheKey, entry);
|
|
105
167
|
}
|
|
106
168
|
catch (error) {
|
|
107
169
|
logWarn('Cache set error', {
|
|
108
170
|
key: cacheKey.substring(0, 100),
|
|
109
|
-
error:
|
|
171
|
+
error: getErrorMessage(error),
|
|
110
172
|
});
|
|
111
173
|
}
|
|
112
174
|
}
|
|
113
175
|
export function keys() {
|
|
114
|
-
return contentCache.keys();
|
|
176
|
+
return Array.from(contentCache.keys());
|
|
115
177
|
}
|
|
116
|
-
function buildCacheEntry(cacheKey, content) {
|
|
178
|
+
function buildCacheEntry(cacheKey, content, metadata) {
|
|
117
179
|
return {
|
|
118
|
-
url:
|
|
180
|
+
url: metadata.url,
|
|
181
|
+
title: metadata.title,
|
|
119
182
|
content,
|
|
120
183
|
fetchedAt: new Date().toISOString(),
|
|
121
184
|
expiresAt: new Date(Date.now() + config.cache.ttl * 1000).toISOString(),
|
|
122
185
|
};
|
|
123
186
|
}
|
|
124
187
|
function persistCacheEntry(cacheKey, entry) {
|
|
125
|
-
|
|
188
|
+
const expiresAt = Date.now() + config.cache.ttl * 1000;
|
|
189
|
+
contentCache.set(cacheKey, { entry, expiresAt });
|
|
126
190
|
emitCacheUpdate(cacheKey);
|
|
127
191
|
}
|
|
128
|
-
function resolveErrorMessage(error) {
|
|
129
|
-
return error instanceof Error ? error.message : 'Unknown error';
|
|
130
|
-
}
|
|
131
|
-
//# sourceMappingURL=cache.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,IAAI,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAEzE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAOtC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAqB,CAAC;AAClD,IAAI,iBAAiB,GAA2B,IAAI,CAAC;AAErD,SAAS,gBAAgB;IACvB,IAAI,iBAAiB;QAAE,OAAO;IAC9B,iBAAiB,GAAG,IAAI,eAAe,EAAE,CAAC;IAC1C,KAAK,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACrE,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1D,OAAO,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,MAAmB;IAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACtD,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,kBAAkB,CAAC,UAAU,EAAE,SAAS,EAAE;QAC9D,MAAM;QACN,GAAG,EAAE,KAAK;KACX,CAAC,EAAE,CAAC;QACH,mBAAmB,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YACzB,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,cAAc,EAAE,CAAC;AACnB,CAAC;AAED,SAAS,cAAc;IACrB,IAAI,YAAY,CAAC,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO;IACtD,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;IAC9D,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,IAAI;YAAE,MAAM;QAChB,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAkBD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;AAEvD,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACrE,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC;SAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC;SACpD,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,EAAE,CACpB,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,eAAe,CAAC,UAAU,CAAC,EAAE,CAC1D,CAAC;IAEJ,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAClC,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa,EAAE,MAAc;IACvD,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,aAAa,CACpB,SAAiB,EACjB,OAAe,EACf,QAAiB;IAEjB,OAAO,QAAQ;QACb,CAAC,CAAC,GAAG,SAAS,IAAI,OAAO,IAAI,QAAQ,EAAE;QACvC,CAAC,CAAC,GAAG,SAAS,IAAI,OAAO,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,WAAW,CAClB,IAAuC;IAEvC,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,UAAU,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAC3E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,OAAO,kBAAkB,CAAC,UAAU,EAAE,UAAU,CAAC,gBAAgB,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,SAAiB,EACjB,GAAW,EACX,IAAuC;IAEvC,IAAI,CAAC,SAAS,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,EAAE,UAAU,CAAC,eAAe,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO,sBAAsB,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAA6B;IACzD,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,OAAO,GAAG,EAAE;QACV,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,QAAgB;IACvC,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,CAAC,KAAK;QAAE,OAAO;IACnB,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;QACvC,QAAQ,CAAC,EAAE,QAAQ,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,QAAuB;IACzC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;QAAE,OAAO,SAAS,CAAC;IAEjD,IAAI,CAAC;QACH,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,iBAAiB,EAAE;YACzB,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;YAC/B,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC;SAC9B,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,QAAuB;IAC9C,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB;IACtC,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAE5B,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACpB,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC;AACpB,CAAC;AAED,SAAS,SAAS,CAAC,IAAe;IAChC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,GAAG,CACjB,QAAuB,EACvB,OAAe,EACf,QAA4B;IAE5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO;IAClC,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,IAAI,CAAC,OAAO;QAAE,OAAO;IAErB,IAAI,CAAC;QACH,gBAAgB,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC3D,iBAAiB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,iBAAiB,EAAE;YACzB,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;YAC/B,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,UAAU,IAAI;IAClB,OAAO,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,eAAe,CACtB,QAAgB,EAChB,OAAe,EACf,QAA4B;IAE5B,OAAO;QACL,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE;KACxE,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB,EAAE,KAAiB;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC;IACvD,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IACjD,eAAe,CAAC,QAAQ,CAAC,CAAC;AAC5B,CAAC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
-
|
|
2
|
+
interface RequestContext {
|
|
3
3
|
requestId: string;
|
|
4
4
|
sessionId?: string;
|
|
5
5
|
}
|
|
6
6
|
export declare const requestContext: AsyncLocalStorage<RequestContext>;
|
|
7
7
|
export declare function getRequestId(): string | undefined;
|
|
8
8
|
export declare function getSessionId(): string | undefined;
|
|
9
|
-
|
|
9
|
+
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/services/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,
|
|
1
|
+
{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/services/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,UAAU,cAAc;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,cAAc,mCAA0C,CAAC;AAEtE,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAEjD;AAED,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAEjD"}
|
package/dist/services/context.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type { ExtractionResult } from '../config/types.js';
|
|
1
|
+
import type { ExtractionResult } from '../config/types/content.js';
|
|
2
2
|
export declare function extractContent(html: string, url: string, options?: {
|
|
3
3
|
extractArticle?: boolean;
|
|
4
4
|
}): ExtractionResult;
|
|
5
|
-
//# sourceMappingURL=extractor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAGV,gBAAgB,EACjB,MAAM,
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAGV,gBAAgB,EACjB,MAAM,4BAA4B,CAAC;AAyKpC,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAA6B,GAC/D,gBAAgB,CAMlB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { parseHTML } from 'linkedom';
|
|
2
2
|
import { Readability } from '@mozilla/readability';
|
|
3
|
+
import { getErrorMessage } from '../utils/error-utils.js';
|
|
3
4
|
import { truncateHtml } from '../utils/html-truncator.js';
|
|
4
|
-
import { logError, logWarn } from './logger.js';
|
|
5
|
+
import { logError, logInfo, logWarn } from './logger.js';
|
|
5
6
|
function resolveMetaField(state, field) {
|
|
6
7
|
const sources = state[field];
|
|
7
8
|
return sources.og ?? sources.twitter ?? sources.standard;
|
|
@@ -81,38 +82,62 @@ function extractMetadata(document) {
|
|
|
81
82
|
author: resolveMetaField(state, 'author'),
|
|
82
83
|
};
|
|
83
84
|
}
|
|
85
|
+
function isReadabilityCompatible(doc) {
|
|
86
|
+
if (!doc || typeof doc !== 'object')
|
|
87
|
+
return false;
|
|
88
|
+
if (!('documentElement' in doc))
|
|
89
|
+
return false;
|
|
90
|
+
if (!('querySelectorAll' in doc))
|
|
91
|
+
return false;
|
|
92
|
+
if (!('querySelector' in doc))
|
|
93
|
+
return false;
|
|
94
|
+
return true;
|
|
95
|
+
}
|
|
84
96
|
function extractArticle(document) {
|
|
97
|
+
if (!isReadabilityCompatible(document)) {
|
|
98
|
+
logWarn('Document not compatible with Readability');
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
const parsed = parseReadabilityArticle(document);
|
|
102
|
+
return parsed ? mapReadabilityResult(parsed) : null;
|
|
103
|
+
}
|
|
104
|
+
function parseReadabilityArticle(document) {
|
|
85
105
|
try {
|
|
106
|
+
// Type assertion is safe here due to isReadabilityCompatible check
|
|
86
107
|
const reader = new Readability(document);
|
|
87
|
-
|
|
88
|
-
if (!parsed)
|
|
89
|
-
return null;
|
|
90
|
-
return {
|
|
91
|
-
title: parsed.title ?? undefined,
|
|
92
|
-
byline: parsed.byline ?? undefined,
|
|
93
|
-
content: parsed.content ?? '',
|
|
94
|
-
textContent: parsed.textContent ?? '',
|
|
95
|
-
excerpt: parsed.excerpt ?? undefined,
|
|
96
|
-
siteName: parsed.siteName ?? undefined,
|
|
97
|
-
};
|
|
108
|
+
return reader.parse();
|
|
98
109
|
}
|
|
99
110
|
catch (error) {
|
|
100
111
|
logError('Failed to extract article with Readability', error instanceof Error ? error : undefined);
|
|
101
112
|
return null;
|
|
102
113
|
}
|
|
103
114
|
}
|
|
115
|
+
function mapReadabilityResult(parsed) {
|
|
116
|
+
return {
|
|
117
|
+
title: toOptional(parsed.title),
|
|
118
|
+
byline: toOptional(parsed.byline),
|
|
119
|
+
content: parsed.content ?? '',
|
|
120
|
+
textContent: parsed.textContent ?? '',
|
|
121
|
+
excerpt: toOptional(parsed.excerpt),
|
|
122
|
+
siteName: toOptional(parsed.siteName),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
function toOptional(value) {
|
|
126
|
+
return value ?? undefined;
|
|
127
|
+
}
|
|
104
128
|
export function extractContent(html, url, options = { extractArticle: true }) {
|
|
105
129
|
if (!isValidInput(html, url)) {
|
|
106
130
|
return { article: null, metadata: {} };
|
|
107
131
|
}
|
|
132
|
+
return tryExtractContent(html, url, options);
|
|
133
|
+
}
|
|
134
|
+
function tryExtractContent(html, url, options) {
|
|
108
135
|
try {
|
|
109
136
|
const processedHtml = truncateHtml(html);
|
|
110
137
|
const { document } = parseHTML(processedHtml);
|
|
111
138
|
applyBaseUri(document, url);
|
|
112
139
|
const metadata = extractMetadata(document);
|
|
113
|
-
const article = options.extractArticle
|
|
114
|
-
? extractArticle(document)
|
|
115
|
-
: null;
|
|
140
|
+
const article = options.extractArticle ? extractArticle(document) : null;
|
|
116
141
|
return { article, metadata };
|
|
117
142
|
}
|
|
118
143
|
catch (error) {
|
|
@@ -138,8 +163,10 @@ function applyBaseUri(document, url) {
|
|
|
138
163
|
writable: true,
|
|
139
164
|
});
|
|
140
165
|
}
|
|
141
|
-
catch {
|
|
142
|
-
|
|
166
|
+
catch (error) {
|
|
167
|
+
logInfo('Failed to set baseURI (non-critical)', {
|
|
168
|
+
url: url.substring(0, 100),
|
|
169
|
+
error: getErrorMessage(error),
|
|
170
|
+
});
|
|
143
171
|
}
|
|
144
172
|
}
|
|
145
|
-
//# sourceMappingURL=extractor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAQnD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAQnD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAWzD,SAAS,gBAAgB,CACvB,KAAyB,EACzB,KAAgB;IAEhB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB;IAC/B,OAAO;QACL,KAAK,EAAE,EAAE;QACT,WAAW,EAAE,EAAE;QACf,MAAM,EAAE,EAAE;KACX,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB,EAAE,GAAoB;IACrE,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC,OAAO;QAAE,OAAO;IAErB,IAAI,oBAAoB,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC;QAAE,OAAO;IACtD,IAAI,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC;QAAE,OAAO;IACpD,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,cAAc,CAAC,GAAoB;IAC1C,OAAO,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;AACrD,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAyB,EACzB,GAAoB,EACpB,OAAe;IAEf,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;IAC9C,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/C,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxC,IAAI,GAAG,KAAK,OAAO;QAAE,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC;IAC9C,IAAI,GAAG,KAAK,aAAa;QAAE,KAAK,CAAC,WAAW,CAAC,EAAE,GAAG,OAAO,CAAC;IAC1D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAyB,EACzB,GAAoB,EACpB,OAAe;IAEf,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACzC,IAAI,GAAG,KAAK,OAAO;QAAE,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;IACnD,IAAI,GAAG,KAAK,aAAa;QAAE,KAAK,CAAC,WAAW,CAAC,OAAO,GAAG,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAyB,EACzB,GAAoB,EACpB,OAAe;IAEf,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAC;IACvC,CAAC;IAED,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC;IAClC,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,KAAyB;IACjE,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,QAAkB,EAClB,KAAyB;IAEzB,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ;QAAE,OAAO;IACjC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,OAAO,EAAE,WAAW,EAAE,CAAC;QACzB,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,QAAkB;IACzC,MAAM,KAAK,GAAG,wBAAwB,EAAE,CAAC;IAEzC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC9B,mBAAmB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAErC,OAAO;QACL,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC;QACvC,WAAW,EAAE,gBAAgB,CAAC,KAAK,EAAE,aAAa,CAAC;QACnD,MAAM,EAAE,gBAAgB,CAAC,KAAK,EAAE,QAAQ,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAAC,GAAY;IAC3C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClD,IAAI,CAAC,CAAC,iBAAiB,IAAI,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,CAAC,CAAC,kBAAkB,IAAI,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,IAAI,CAAC,CAAC,eAAe,IAAI,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,QAAiB;IACvC,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,0CAA0C,CAAC,CAAC;QACpD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,uBAAuB,CAC9B,QAAkB;IAElB,IAAI,CAAC;QACH,mEAAmE;QACnE,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,4CAA4C,EAC5C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAC3B,MAAqD;IAErD,OAAO;QACL,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;QAC/B,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC;QACjC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;QACrC,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC;QACnC,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAgC;IAClD,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,GAAW,EACX,UAAwC,EAAE,cAAc,EAAE,IAAI,EAAE;IAEhE,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,OAAO,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAY,EACZ,GAAW,EACX,OAAqC;IAErC,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;QAE9C,YAAY,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAE5B,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEzE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,GAAW;IAC7C,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,CAAC,+CAA+C,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,GAAW;IACnD,IAAI,CAAC;QACH,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE;YACzC,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sCAAsC,EAAE;YAC9C,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;YAC1B,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agents.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/agents.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"agents.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/agents.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAiM/B,eAAO,MAAM,UAAU,OAA+B,CAAC;AAEvD,wBAAgB,aAAa,IAAI,IAAI,CAEpC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import dns from 'node:dns';
|
|
2
2
|
import os from 'node:os';
|
|
3
3
|
import { Agent } from 'undici';
|
|
4
|
+
import { createErrorWithCode } from '../../utils/error-utils.js';
|
|
4
5
|
import { isBlockedIp } from '../../utils/url-validator.js';
|
|
5
6
|
function resolveDns(hostname, options, callback) {
|
|
6
7
|
const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
|
|
@@ -84,16 +85,12 @@ function findBlockedIpError(list, hostname) {
|
|
|
84
85
|
if (!isBlockedIp(ip)) {
|
|
85
86
|
continue;
|
|
86
87
|
}
|
|
87
|
-
|
|
88
|
-
error.code = 'EBLOCKED';
|
|
89
|
-
return error;
|
|
88
|
+
return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
|
|
90
89
|
}
|
|
91
90
|
return null;
|
|
92
91
|
}
|
|
93
92
|
function createNoDnsResultsError(hostname) {
|
|
94
|
-
|
|
95
|
-
error.code = 'ENODATA';
|
|
96
|
-
return error;
|
|
93
|
+
return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
|
|
97
94
|
}
|
|
98
95
|
function getAgentOptions() {
|
|
99
96
|
const cpuCount = os.availableParallelism();
|
|
@@ -108,4 +105,3 @@ export const dispatcher = new Agent(getAgentOptions());
|
|
|
108
105
|
export function destroyAgents() {
|
|
109
106
|
void dispatcher.close();
|
|
110
107
|
}
|
|
111
|
-
//# sourceMappingURL=agents.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agents.js","sourceRoot":"","sources":["../../../src/services/fetcher/agents.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAE3D,SAAS,UAAU,CACjB,QAAgB,EAChB,OAA0B,EAC1B,QAIS;IAET,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,GACjD,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC9B,GAAG,CAAC,MAAM,CACR,QAAQ,EACR,EAAE,GAAG,iBAAiB,EAAE,GAAG,EAAE,IAAI,EAAE,EACnC,oBAAoB,CAAC,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CACjE,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAmC;IAEnC,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;AACrE,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAmC;IAK7D,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1D,OAAO;QACL,iBAAiB;QACjB,MAAM,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC;QACtC,cAAc,EAAE,aAAa,CAAC,iBAAiB,CAAC,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,cAAkC,EAClC,MAAe,EACf,QAIS;IAKT,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;QACxB,kBAAkB,CAChB,GAAG,EACH,SAAS,EACT,QAAQ,EACR,cAAc,EACd,MAAM,EACN,QAAQ,CACT,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,MAAmC;IAEnC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,sBAAsB,CAC7B,SAAuC,EACvC,MAA0B;IAE1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAmC,EACnC,SAAuC,EACvC,QAAgB,EAChB,cAAkC,EAClC,MAAe,EACf,QAIS;IAET,IAAI,KAAK,EAAE,CAAC;QACV,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC3B,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,sBAAsB,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACxD,IAAI,YAAY,EAAE,CAAC;QACjB,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAC7B,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7D,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACpB,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC9C,OAAO;IACT,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAyB,EACzB,MAAe,EACf,QAAgB;IAOhB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,uBAAuB,CAAC,QAAQ,CAAC;YACxC,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC3C,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,uBAAuB,CAAC,QAAQ,CAAC;YACxC,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,IAAI;KACf,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAyB,EACzB,QAAgB;IAEhB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QAED,
|
|
1
|
+
{"version":3,"file":"agents.js","sourceRoot":"","sources":["../../../src/services/fetcher/agents.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAE3D,SAAS,UAAU,CACjB,QAAgB,EAChB,OAA0B,EAC1B,QAIS;IAET,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,GACjD,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC9B,GAAG,CAAC,MAAM,CACR,QAAQ,EACR,EAAE,GAAG,iBAAiB,EAAE,GAAG,EAAE,IAAI,EAAE,EACnC,oBAAoB,CAAC,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CACjE,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAmC;IAEnC,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;AACrE,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAmC;IAK7D,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1D,OAAO;QACL,iBAAiB;QACjB,MAAM,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC;QACtC,cAAc,EAAE,aAAa,CAAC,iBAAiB,CAAC,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,cAAkC,EAClC,MAAe,EACf,QAIS;IAKT,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;QACxB,kBAAkB,CAChB,GAAG,EACH,SAAS,EACT,QAAQ,EACR,cAAc,EACd,MAAM,EACN,QAAQ,CACT,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,MAAmC;IAEnC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,sBAAsB,CAC7B,SAAuC,EACvC,MAA0B;IAE1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAmC,EACnC,SAAuC,EACvC,QAAgB,EAChB,cAAkC,EAClC,MAAe,EACf,QAIS;IAET,IAAI,KAAK,EAAE,CAAC;QACV,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC3B,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,sBAAsB,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACxD,IAAI,YAAY,EAAE,CAAC;QACjB,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAC7B,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7D,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACpB,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC9C,OAAO;IACT,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAyB,EACzB,MAAe,EACf,QAAgB;IAOhB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,uBAAuB,CAAC,QAAQ,CAAC;YACxC,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC3C,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,uBAAuB,CAAC,QAAQ,CAAC;YACxC,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,IAAI;KACf,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAyB,EACzB,QAAgB;IAEhB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QAED,OAAO,mBAAmB,CACxB,2BAA2B,QAAQ,EAAE,EACrC,UAAU,CACX,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgB;IAC/C,OAAO,mBAAmB,CACxB,+BAA+B,QAAQ,EAAE,EACzC,SAAS,CACV,CAAC;AACJ,CAAC;AAED,SAAS,eAAe;IACtB,MAAM,QAAQ,GAAG,EAAE,CAAC,oBAAoB,EAAE,CAAC;IAC3C,OAAO;QACL,gBAAgB,EAAE,KAAK;QACvB,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,EAAE,CAAC;QACvC,UAAU,EAAE,CAAC;QACb,OAAO,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;AAEvD,MAAM,UAAU,aAAa;IAC3B,KAAK,UAAU,CAAC,KAAK,EAAE,CAAC;AAC1B,CAAC"}
|
|
@@ -2,4 +2,3 @@ import { FetchError } from '../../errors/app-error.js';
|
|
|
2
2
|
export declare function createRateLimitError(url: string, headerValue: string | null): FetchError;
|
|
3
3
|
export declare function createHttpError(url: string, status: number, statusText: string): FetchError;
|
|
4
4
|
export declare function mapFetchError(error: unknown, fallbackUrl: string, timeoutMs: number): FetchError;
|
|
5
|
-
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/headers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/headers.ts"],"names":[],"mappings":"AAIA,wBAAgB,eAAe,CAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAEpC"}
|
|
@@ -1,28 +1,6 @@
|
|
|
1
1
|
import { config } from '../../config/index.js';
|
|
2
|
+
import { normalizeHeaderRecord } from '../../utils/header-normalizer.js';
|
|
2
3
|
export function sanitizeHeaders(headers) {
|
|
3
|
-
|
|
4
|
-
return undefined;
|
|
5
|
-
const normalized = normalizeHeaders(headers, config.security.blockedHeaders);
|
|
6
|
-
const iterator = normalized.keys();
|
|
7
|
-
if (iterator.next().done)
|
|
8
|
-
return undefined;
|
|
9
|
-
return Object.fromEntries(normalized.entries());
|
|
10
|
-
}
|
|
11
|
-
function normalizeHeaders(headers, blockedHeaders) {
|
|
12
|
-
const normalized = new Headers();
|
|
13
|
-
for (const [key, value] of Object.entries(headers)) {
|
|
14
|
-
if (blockedHeaders.has(key.toLowerCase()))
|
|
15
|
-
continue;
|
|
16
|
-
setHeaderSafe(normalized, key, value);
|
|
17
|
-
}
|
|
18
|
-
return normalized;
|
|
19
|
-
}
|
|
20
|
-
function setHeaderSafe(headers, key, value) {
|
|
21
|
-
try {
|
|
22
|
-
headers.set(key, value);
|
|
23
|
-
}
|
|
24
|
-
catch {
|
|
25
|
-
// Ignore invalid header values
|
|
26
|
-
}
|
|
4
|
+
return normalizeHeaderRecord(headers, config.security.blockedHeaders);
|
|
27
5
|
}
|
|
28
6
|
//# sourceMappingURL=headers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../../src/services/fetcher/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,
|
|
1
|
+
{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../../src/services/fetcher/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,UAAU,eAAe,CAC7B,OAAgC;IAEhC,OAAO,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;AACxE,CAAC"}
|