@j0hanz/superfetch 2.1.4 → 2.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts +1 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +1 -0
- package/dist/cache.js.map +1 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -0
- package/dist/crypto.d.ts +1 -0
- package/dist/crypto.d.ts.map +1 -0
- package/dist/crypto.js +1 -0
- package/dist/crypto.js.map +1 -0
- package/dist/errors.d.ts +1 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +1 -0
- package/dist/errors.js.map +1 -0
- package/dist/fetch.d.ts +1 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +1 -0
- package/dist/fetch.js.map +1 -0
- package/dist/http.d.ts +1 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +1 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts +1 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +1 -0
- package/dist/mcp.js.map +1 -0
- package/dist/observability.d.ts +1 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/observability.js +1 -0
- package/dist/observability.js.map +1 -0
- package/dist/tools.d.ts +1 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +1 -0
- package/dist/tools.js.map +1 -0
- package/dist/transform.d.ts +1 -0
- package/dist/transform.d.ts.map +1 -0
- package/dist/transform.js +1 -0
- package/dist/transform.js.map +1 -0
- package/dist/utils.d.ts +1 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +1 -0
- package/dist/utils.js.map +1 -0
- package/dist/workers/transform-worker.d.ts +1 -0
- package/dist/workers/transform-worker.d.ts.map +1 -0
- package/dist/workers/transform-worker.js +1 -0
- package/dist/workers/transform-worker.js.map +1 -0
- package/package.json +5 -2
- package/dist/config/auth-config.d.ts +0 -16
- package/dist/config/auth-config.js +0 -53
- package/dist/config/constants.d.ts +0 -17
- package/dist/config/constants.d.ts.map +0 -1
- package/dist/config/constants.js +0 -22
- package/dist/config/constants.js.map +0 -1
- package/dist/config/env-parsers.d.ts +0 -8
- package/dist/config/env-parsers.js +0 -96
- package/dist/config/formatting.d.ts +0 -7
- package/dist/config/formatting.d.ts.map +0 -1
- package/dist/config/formatting.js +0 -9
- package/dist/config/formatting.js.map +0 -1
- package/dist/config/index.d.ts +0 -61
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -107
- package/dist/config/index.js.map +0 -1
- package/dist/config/types/content.d.ts +0 -41
- package/dist/config/types/content.d.ts.map +0 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/content.js.map +0 -1
- package/dist/config/types/runtime.d.ts +0 -72
- package/dist/config/types/runtime.d.ts.map +0 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/runtime.js.map +0 -1
- package/dist/config/types/tools.d.ts +0 -28
- package/dist/config/types/tools.d.ts.map +0 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/config/types/tools.js.map +0 -1
- package/dist/config/types.d.ts +0 -4
- package/dist/config/types.d.ts.map +0 -1
- package/dist/config/types.js +0 -2
- package/dist/config/types.js.map +0 -1
- package/dist/errors/app-error.d.ts +0 -7
- package/dist/errors/app-error.d.ts.map +0 -1
- package/dist/errors/app-error.js +0 -16
- package/dist/errors/app-error.js.map +0 -1
- package/dist/errors/index.d.ts +0 -2
- package/dist/errors/index.d.ts.map +0 -1
- package/dist/errors/index.js +0 -2
- package/dist/errors/index.js.map +0 -1
- package/dist/http/accept-policy.d.ts +0 -3
- package/dist/http/accept-policy.js +0 -45
- package/dist/http/async-handler.d.ts +0 -2
- package/dist/http/async-handler.js +0 -5
- package/dist/http/auth-introspection.d.ts +0 -2
- package/dist/http/auth-introspection.js +0 -141
- package/dist/http/auth-static.d.ts +0 -2
- package/dist/http/auth-static.js +0 -23
- package/dist/http/auth.d.ts +0 -3
- package/dist/http/auth.d.ts.map +0 -1
- package/dist/http/auth.js +0 -269
- package/dist/http/auth.js.map +0 -1
- package/dist/http/base-middleware.d.ts +0 -7
- package/dist/http/base-middleware.js +0 -143
- package/dist/http/cors.d.ts +0 -2
- package/dist/http/cors.d.ts.map +0 -1
- package/dist/http/cors.js +0 -9
- package/dist/http/cors.js.map +0 -1
- package/dist/http/download-routes.d.ts +0 -2
- package/dist/http/download-routes.d.ts.map +0 -1
- package/dist/http/download-routes.js +0 -104
- package/dist/http/download-routes.js.map +0 -1
- package/dist/http/error-handler.d.ts +0 -2
- package/dist/http/error-handler.js +0 -55
- package/dist/http/host-allowlist.d.ts +0 -3
- package/dist/http/host-allowlist.js +0 -117
- package/dist/http/jsonrpc-http.d.ts +0 -2
- package/dist/http/jsonrpc-http.js +0 -10
- package/dist/http/mcp-routes.d.ts +0 -9
- package/dist/http/mcp-routes.d.ts.map +0 -1
- package/dist/http/mcp-routes.js +0 -204
- package/dist/http/mcp-routes.js.map +0 -1
- package/dist/http/mcp-session-eviction.d.ts +0 -3
- package/dist/http/mcp-session-eviction.js +0 -24
- package/dist/http/mcp-session-helpers.d.ts +0 -12
- package/dist/http/mcp-session-helpers.d.ts.map +0 -1
- package/dist/http/mcp-session-helpers.js +0 -64
- package/dist/http/mcp-session-helpers.js.map +0 -1
- package/dist/http/mcp-session-init.d.ts +0 -7
- package/dist/http/mcp-session-init.js +0 -94
- package/dist/http/mcp-session-slots.d.ts +0 -17
- package/dist/http/mcp-session-slots.js +0 -55
- package/dist/http/mcp-session-transport-init.d.ts +0 -7
- package/dist/http/mcp-session-transport-init.js +0 -41
- package/dist/http/mcp-session-transport.d.ts +0 -7
- package/dist/http/mcp-session-transport.js +0 -57
- package/dist/http/mcp-session-types.d.ts +0 -5
- package/dist/http/mcp-session-types.js +0 -1
- package/dist/http/mcp-session.d.ts +0 -10
- package/dist/http/mcp-session.d.ts.map +0 -1
- package/dist/http/mcp-session.js +0 -23
- package/dist/http/mcp-session.js.map +0 -1
- package/dist/http/mcp-sessions.d.ts +0 -41
- package/dist/http/mcp-sessions.js +0 -392
- package/dist/http/mcp-validation.d.ts +0 -3
- package/dist/http/mcp-validation.d.ts.map +0 -1
- package/dist/http/mcp-validation.js +0 -14
- package/dist/http/mcp-validation.js.map +0 -1
- package/dist/http/protocol-policy.d.ts +0 -2
- package/dist/http/protocol-policy.js +0 -31
- package/dist/http/rate-limit.d.ts +0 -12
- package/dist/http/rate-limit.d.ts.map +0 -1
- package/dist/http/rate-limit.js +0 -93
- package/dist/http/rate-limit.js.map +0 -1
- package/dist/http/server-config.d.ts +0 -1
- package/dist/http/server-config.js +0 -40
- package/dist/http/server-middleware.d.ts +0 -7
- package/dist/http/server-middleware.d.ts.map +0 -1
- package/dist/http/server-middleware.js +0 -52
- package/dist/http/server-middleware.js.map +0 -1
- package/dist/http/server-shutdown.d.ts +0 -4
- package/dist/http/server-shutdown.js +0 -43
- package/dist/http/server-tuning.d.ts +0 -9
- package/dist/http/server-tuning.js +0 -45
- package/dist/http/server.d.ts +0 -3
- package/dist/http/server.d.ts.map +0 -1
- package/dist/http/server.js +0 -291
- package/dist/http/server.js.map +0 -1
- package/dist/http/session-cleanup.d.ts +0 -2
- package/dist/http/session-cleanup.d.ts.map +0 -1
- package/dist/http/session-cleanup.js +0 -40
- package/dist/http/session-cleanup.js.map +0 -1
- package/dist/http/sessions.d.ts +0 -14
- package/dist/http/sessions.d.ts.map +0 -1
- package/dist/http/sessions.js +0 -63
- package/dist/http/sessions.js.map +0 -1
- package/dist/middleware/error-handler.d.ts +0 -2
- package/dist/middleware/error-handler.d.ts.map +0 -1
- package/dist/middleware/error-handler.js +0 -56
- package/dist/middleware/error-handler.js.map +0 -1
- package/dist/middleware/rate-limiter.d.ts +0 -16
- package/dist/middleware/rate-limiter.d.ts.map +0 -1
- package/dist/middleware/rate-limiter.js +0 -111
- package/dist/middleware/rate-limiter.js.map +0 -1
- package/dist/parsers/base-html-element-parser.d.ts +0 -43
- package/dist/parsers/base-html-element-parser.d.ts.map +0 -1
- package/dist/parsers/base-html-element-parser.js +0 -59
- package/dist/parsers/base-html-element-parser.js.map +0 -1
- package/dist/parsers/heading-element-parser.d.ts +0 -14
- package/dist/parsers/heading-element-parser.d.ts.map +0 -1
- package/dist/parsers/heading-element-parser.js +0 -26
- package/dist/parsers/heading-element-parser.js.map +0 -1
- package/dist/parsers/image-element-parser.d.ts +0 -16
- package/dist/parsers/image-element-parser.d.ts.map +0 -1
- package/dist/parsers/image-element-parser.js +0 -33
- package/dist/parsers/image-element-parser.js.map +0 -1
- package/dist/parsers/link-element-parser.d.ts +0 -15
- package/dist/parsers/link-element-parser.d.ts.map +0 -1
- package/dist/parsers/link-element-parser.js +0 -28
- package/dist/parsers/link-element-parser.js.map +0 -1
- package/dist/parsers/open-graph-parser.d.ts +0 -17
- package/dist/parsers/open-graph-parser.d.ts.map +0 -1
- package/dist/parsers/open-graph-parser.js +0 -41
- package/dist/parsers/open-graph-parser.js.map +0 -1
- package/dist/parsers/schema-org-parser.d.ts +0 -17
- package/dist/parsers/schema-org-parser.d.ts.map +0 -1
- package/dist/parsers/schema-org-parser.js +0 -32
- package/dist/parsers/schema-org-parser.js.map +0 -1
- package/dist/parsers/standard-meta-parser.d.ts +0 -18
- package/dist/parsers/standard-meta-parser.d.ts.map +0 -1
- package/dist/parsers/standard-meta-parser.js +0 -32
- package/dist/parsers/standard-meta-parser.js.map +0 -1
- package/dist/parsers/twitter-card-parser.d.ts +0 -17
- package/dist/parsers/twitter-card-parser.d.ts.map +0 -1
- package/dist/parsers/twitter-card-parser.js +0 -41
- package/dist/parsers/twitter-card-parser.js.map +0 -1
- package/dist/prompts/index.d.ts +0 -3
- package/dist/prompts/index.d.ts.map +0 -1
- package/dist/prompts/index.js +0 -73
- package/dist/prompts/index.js.map +0 -1
- package/dist/resources/cached-content-params.d.ts +0 -5
- package/dist/resources/cached-content-params.js +0 -36
- package/dist/resources/cached-content.d.ts +0 -2
- package/dist/resources/cached-content.d.ts.map +0 -1
- package/dist/resources/cached-content.js +0 -132
- package/dist/resources/cached-content.js.map +0 -1
- package/dist/resources/index.d.ts +0 -2
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js +0 -4
- package/dist/resources/index.js.map +0 -1
- package/dist/server.d.ts +0 -3
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js +0 -94
- package/dist/server.js.map +0 -1
- package/dist/services/cache-events.d.ts +0 -8
- package/dist/services/cache-events.js +0 -19
- package/dist/services/cache-keys.d.ts +0 -7
- package/dist/services/cache-keys.js +0 -57
- package/dist/services/cache.d.ts +0 -17
- package/dist/services/cache.d.ts.map +0 -1
- package/dist/services/cache.js +0 -145
- package/dist/services/cache.js.map +0 -1
- package/dist/services/cache.service.d.ts +0 -52
- package/dist/services/cache.service.d.ts.map +0 -1
- package/dist/services/cache.service.js +0 -113
- package/dist/services/cache.service.js.map +0 -1
- package/dist/services/card-extractor.d.ts +0 -6
- package/dist/services/card-extractor.d.ts.map +0 -1
- package/dist/services/card-extractor.js +0 -199
- package/dist/services/card-extractor.js.map +0 -1
- package/dist/services/context.d.ts +0 -10
- package/dist/services/context.d.ts.map +0 -1
- package/dist/services/context.js +0 -14
- package/dist/services/context.js.map +0 -1
- package/dist/services/extractor.d.ts +0 -5
- package/dist/services/extractor.d.ts.map +0 -1
- package/dist/services/extractor.js +0 -142
- package/dist/services/extractor.js.map +0 -1
- package/dist/services/extractor.service.d.ts +0 -18
- package/dist/services/extractor.service.d.ts.map +0 -1
- package/dist/services/extractor.service.js +0 -75
- package/dist/services/extractor.service.js.map +0 -1
- package/dist/services/fetcher/agents.d.ts +0 -3
- package/dist/services/fetcher/agents.d.ts.map +0 -1
- package/dist/services/fetcher/agents.js +0 -100
- package/dist/services/fetcher/agents.js.map +0 -1
- package/dist/services/fetcher/dns-selection.d.ts +0 -2
- package/dist/services/fetcher/dns-selection.js +0 -72
- package/dist/services/fetcher/errors.d.ts +0 -4
- package/dist/services/fetcher/errors.d.ts.map +0 -1
- package/dist/services/fetcher/errors.js +0 -70
- package/dist/services/fetcher/errors.js.map +0 -1
- package/dist/services/fetcher/headers.d.ts +0 -2
- package/dist/services/fetcher/headers.d.ts.map +0 -1
- package/dist/services/fetcher/headers.js +0 -6
- package/dist/services/fetcher/headers.js.map +0 -1
- package/dist/services/fetcher/interceptors.d.ts +0 -10
- package/dist/services/fetcher/interceptors.d.ts.map +0 -1
- package/dist/services/fetcher/interceptors.js +0 -108
- package/dist/services/fetcher/interceptors.js.map +0 -1
- package/dist/services/fetcher/redirects.d.ts +0 -4
- package/dist/services/fetcher/redirects.d.ts.map +0 -1
- package/dist/services/fetcher/redirects.js +0 -78
- package/dist/services/fetcher/redirects.js.map +0 -1
- package/dist/services/fetcher/response.d.ts +0 -4
- package/dist/services/fetcher/response.d.ts.map +0 -1
- package/dist/services/fetcher/response.js +0 -104
- package/dist/services/fetcher/response.js.map +0 -1
- package/dist/services/fetcher/retry-policy.d.ts +0 -1
- package/dist/services/fetcher/retry-policy.d.ts.map +0 -1
- package/dist/services/fetcher/retry-policy.js +0 -131
- package/dist/services/fetcher/retry-policy.js.map +0 -1
- package/dist/services/fetcher.d.ts +0 -25
- package/dist/services/fetcher.d.ts.map +0 -1
- package/dist/services/fetcher.js +0 -607
- package/dist/services/fetcher.js.map +0 -1
- package/dist/services/fetcher.service.d.ts +0 -18
- package/dist/services/fetcher.service.d.ts.map +0 -1
- package/dist/services/fetcher.service.js +0 -122
- package/dist/services/fetcher.service.js.map +0 -1
- package/dist/services/fifo-queue.d.ts +0 -8
- package/dist/services/fifo-queue.js +0 -25
- package/dist/services/logger.d.ts +0 -5
- package/dist/services/logger.d.ts.map +0 -1
- package/dist/services/logger.js +0 -52
- package/dist/services/logger.js.map +0 -1
- package/dist/services/logger.service.d.ts +0 -5
- package/dist/services/logger.service.d.ts.map +0 -1
- package/dist/services/logger.service.js +0 -57
- package/dist/services/logger.service.js.map +0 -1
- package/dist/services/metadata-collector.d.ts +0 -2
- package/dist/services/metadata-collector.js +0 -80
- package/dist/services/parser.d.ts +0 -6
- package/dist/services/parser.d.ts.map +0 -1
- package/dist/services/parser.js +0 -278
- package/dist/services/parser.js.map +0 -1
- package/dist/services/parser.service.d.ts +0 -42
- package/dist/services/parser.service.d.ts.map +0 -1
- package/dist/services/parser.service.js +0 -209
- package/dist/services/parser.service.js.map +0 -1
- package/dist/services/session-manager.d.ts +0 -18
- package/dist/services/session-manager.d.ts.map +0 -1
- package/dist/services/session-manager.js +0 -73
- package/dist/services/session-manager.js.map +0 -1
- package/dist/services/telemetry.d.ts +0 -19
- package/dist/services/telemetry.js +0 -43
- package/dist/services/transform-worker-pool.d.ts +0 -11
- package/dist/services/transform-worker-pool.js +0 -244
- package/dist/services/transform-worker-types.d.ts +0 -32
- package/dist/services/transform-worker-types.js +0 -14
- package/dist/strategies/exponential-backoff-strategy.d.ts +0 -13
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +0 -1
- package/dist/strategies/exponential-backoff-strategy.js +0 -32
- package/dist/strategies/exponential-backoff-strategy.js.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +0 -4
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +0 -159
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -98
- package/dist/tools/handlers/fetch-links.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +0 -11
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +0 -97
- package/dist/tools/handlers/fetch-markdown.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +0 -31
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.js +0 -223
- package/dist/tools/handlers/fetch-single.shared.js.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +0 -10
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.js +0 -119
- package/dist/tools/handlers/fetch-url.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +0 -13
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.js +0 -153
- package/dist/tools/handlers/fetch-urls/processor.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.d.ts +0 -3
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.js +0 -79
- package/dist/tools/handlers/fetch-urls/response.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.js +0 -18
- package/dist/tools/handlers/fetch-urls/validation.js.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.js +0 -124
- package/dist/tools/handlers/fetch-urls.tool.js.map +0 -1
- package/dist/tools/index.d.ts +0 -3
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js +0 -37
- package/dist/tools/index.js.map +0 -1
- package/dist/tools/schemas.d.ts +0 -12
- package/dist/tools/schemas.d.ts.map +0 -1
- package/dist/tools/schemas.js +0 -21
- package/dist/tools/schemas.js.map +0 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -1
- package/dist/tools/utils/cache-vary.d.ts.map +0 -1
- package/dist/tools/utils/cache-vary.js +0 -12
- package/dist/tools/utils/cache-vary.js.map +0 -1
- package/dist/tools/utils/cached-markdown.d.ts +0 -5
- package/dist/tools/utils/cached-markdown.js +0 -46
- package/dist/tools/utils/common.d.ts +0 -5
- package/dist/tools/utils/common.d.ts.map +0 -1
- package/dist/tools/utils/common.js +0 -42
- package/dist/tools/utils/common.js.map +0 -1
- package/dist/tools/utils/content-shaping.d.ts +0 -4
- package/dist/tools/utils/content-shaping.js +0 -67
- package/dist/tools/utils/content-transform-async.d.ts +0 -6
- package/dist/tools/utils/content-transform-async.js +0 -33
- package/dist/tools/utils/content-transform-core.d.ts +0 -5
- package/dist/tools/utils/content-transform-core.js +0 -180
- package/dist/tools/utils/content-transform-workers.d.ts +0 -1
- package/dist/tools/utils/content-transform-workers.js +0 -1
- package/dist/tools/utils/content-transform.d.ts +0 -3
- package/dist/tools/utils/content-transform.d.ts.map +0 -1
- package/dist/tools/utils/content-transform.js +0 -41
- package/dist/tools/utils/content-transform.js.map +0 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +0 -1
- package/dist/tools/utils/fetch-pipeline.js +0 -98
- package/dist/tools/utils/fetch-pipeline.js.map +0 -1
- package/dist/tools/utils/frontmatter.d.ts +0 -3
- package/dist/tools/utils/frontmatter.js +0 -73
- package/dist/tools/utils/index.d.ts +0 -4
- package/dist/tools/utils/index.d.ts.map +0 -1
- package/dist/tools/utils/index.js +0 -3
- package/dist/tools/utils/index.js.map +0 -1
- package/dist/tools/utils/inline-content.d.ts +0 -10
- package/dist/tools/utils/inline-content.d.ts.map +0 -1
- package/dist/tools/utils/inline-content.js +0 -35
- package/dist/tools/utils/inline-content.js.map +0 -1
- package/dist/tools/utils/markdown-heuristics.d.ts +0 -1
- package/dist/tools/utils/markdown-heuristics.js +0 -19
- package/dist/tools/utils/markdown-signals.d.ts +0 -1
- package/dist/tools/utils/markdown-signals.js +0 -19
- package/dist/tools/utils/markdown-toc.d.ts +0 -3
- package/dist/tools/utils/markdown-toc.d.ts.map +0 -1
- package/dist/tools/utils/markdown-toc.js +0 -35
- package/dist/tools/utils/markdown-toc.js.map +0 -1
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +0 -3
- package/dist/tools/utils/raw-markdown-frontmatter.js +0 -73
- package/dist/tools/utils/raw-markdown.d.ts +0 -6
- package/dist/tools/utils/raw-markdown.js +0 -149
- package/dist/tools/utils/response-builder.d.ts +0 -3
- package/dist/tools/utils/response-builder.d.ts.map +0 -1
- package/dist/tools/utils/response-builder.js +0 -24
- package/dist/tools/utils/response-builder.js.map +0 -1
- package/dist/tools/utils/tool-response.d.ts +0 -9
- package/dist/tools/utils/tool-response.d.ts.map +0 -1
- package/dist/tools/utils/tool-response.js +0 -19
- package/dist/tools/utils/tool-response.js.map +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +0 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +0 -1
- package/dist/transformers/jsonl.transformer.js +0 -75
- package/dist/transformers/jsonl.transformer.js.map +0 -1
- package/dist/transformers/markdown/fenced-code-rule.d.ts +0 -2
- package/dist/transformers/markdown/fenced-code-rule.js +0 -38
- package/dist/transformers/markdown/frontmatter.d.ts +0 -2
- package/dist/transformers/markdown/frontmatter.js +0 -45
- package/dist/transformers/markdown/noise-rule.d.ts +0 -2
- package/dist/transformers/markdown/noise-rule.js +0 -80
- package/dist/transformers/markdown/turndown-instance.d.ts +0 -2
- package/dist/transformers/markdown/turndown-instance.js +0 -19
- package/dist/transformers/markdown.d.ts +0 -5
- package/dist/transformers/markdown.js +0 -314
- package/dist/transformers/markdown.transformer.d.ts +0 -2
- package/dist/transformers/markdown.transformer.d.ts.map +0 -1
- package/dist/transformers/markdown.transformer.js +0 -14
- package/dist/transformers/markdown.transformer.js.map +0 -1
- package/dist/types/content.types.d.ts +0 -63
- package/dist/types/content.types.d.ts.map +0 -1
- package/dist/types/content.types.js +0 -2
- package/dist/types/content.types.js.map +0 -1
- package/dist/types/index.d.ts +0 -2
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -2
- package/dist/types/index.js.map +0 -1
- package/dist/types/schemas.d.ts +0 -49
- package/dist/types/schemas.d.ts.map +0 -1
- package/dist/types/schemas.js +0 -5
- package/dist/types/schemas.js.map +0 -1
- package/dist/utils/cached-payload.d.ts +0 -7
- package/dist/utils/cached-payload.js +0 -36
- package/dist/utils/cancellation.d.ts +0 -1
- package/dist/utils/cancellation.js +0 -18
- package/dist/utils/code-language-bash.d.ts +0 -1
- package/dist/utils/code-language-bash.js +0 -48
- package/dist/utils/code-language-core.d.ts +0 -2
- package/dist/utils/code-language-core.js +0 -13
- package/dist/utils/code-language-detectors.d.ts +0 -5
- package/dist/utils/code-language-detectors.js +0 -142
- package/dist/utils/code-language-helpers.d.ts +0 -5
- package/dist/utils/code-language-helpers.js +0 -62
- package/dist/utils/code-language-parsing.d.ts +0 -5
- package/dist/utils/code-language-parsing.js +0 -62
- package/dist/utils/code-language.d.ts +0 -2
- package/dist/utils/code-language.d.ts.map +0 -1
- package/dist/utils/code-language.js +0 -260
- package/dist/utils/code-language.js.map +0 -1
- package/dist/utils/concurrency.d.ts +0 -3
- package/dist/utils/concurrency.d.ts.map +0 -1
- package/dist/utils/concurrency.js +0 -38
- package/dist/utils/concurrency.js.map +0 -1
- package/dist/utils/content-cleaner.d.ts +0 -5
- package/dist/utils/content-cleaner.d.ts.map +0 -1
- package/dist/utils/content-cleaner.js +0 -77
- package/dist/utils/content-cleaner.js.map +0 -1
- package/dist/utils/crypto.d.ts +0 -2
- package/dist/utils/crypto.d.ts.map +0 -1
- package/dist/utils/crypto.js +0 -32
- package/dist/utils/crypto.js.map +0 -1
- package/dist/utils/download-url.d.ts +0 -16
- package/dist/utils/download-url.d.ts.map +0 -1
- package/dist/utils/download-url.js +0 -30
- package/dist/utils/download-url.js.map +0 -1
- package/dist/utils/error-details.d.ts +0 -3
- package/dist/utils/error-details.js +0 -12
- package/dist/utils/error-utils.d.ts +0 -3
- package/dist/utils/error-utils.d.ts.map +0 -1
- package/dist/utils/error-utils.js +0 -12
- package/dist/utils/error-utils.js.map +0 -1
- package/dist/utils/filename-generator.d.ts +0 -1
- package/dist/utils/filename-generator.d.ts.map +0 -1
- package/dist/utils/filename-generator.js +0 -81
- package/dist/utils/filename-generator.js.map +0 -1
- package/dist/utils/guards.d.ts +0 -1
- package/dist/utils/guards.js +0 -3
- package/dist/utils/header-normalizer.d.ts +0 -5
- package/dist/utils/header-normalizer.d.ts.map +0 -1
- package/dist/utils/header-normalizer.js +0 -31
- package/dist/utils/header-normalizer.js.map +0 -1
- package/dist/utils/host-normalizer.d.ts +0 -1
- package/dist/utils/host-normalizer.js +0 -37
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.d.ts.map +0 -1
- package/dist/utils/html-truncator.js +0 -13
- package/dist/utils/html-truncator.js.map +0 -1
- package/dist/utils/ip-address.d.ts +0 -4
- package/dist/utils/ip-address.js +0 -6
- package/dist/utils/language-detector.d.ts +0 -2
- package/dist/utils/language-detector.d.ts.map +0 -1
- package/dist/utils/language-detector.js +0 -39
- package/dist/utils/language-detector.js.map +0 -1
- package/dist/utils/sanitizer.d.ts +0 -2
- package/dist/utils/sanitizer.d.ts.map +0 -1
- package/dist/utils/sanitizer.js +0 -20
- package/dist/utils/sanitizer.js.map +0 -1
- package/dist/utils/tool-error-handler.d.ts +0 -3
- package/dist/utils/tool-error-handler.d.ts.map +0 -1
- package/dist/utils/tool-error-handler.js +0 -31
- package/dist/utils/tool-error-handler.js.map +0 -1
- package/dist/utils/url-redactor.d.ts +0 -1
- package/dist/utils/url-redactor.js +0 -13
- package/dist/utils/url-sanitizer.d.ts +0 -2
- package/dist/utils/url-sanitizer.d.ts.map +0 -1
- package/dist/utils/url-sanitizer.js +0 -12
- package/dist/utils/url-sanitizer.js.map +0 -1
- package/dist/utils/url-transformer.d.ts +0 -7
- package/dist/utils/url-transformer.js +0 -147
- package/dist/utils/url-validator.d.ts +0 -6
- package/dist/utils/url-validator.d.ts.map +0 -1
- package/dist/utils/url-validator.js +0 -156
- package/dist/utils/url-validator.js.map +0 -1
- package/dist/workers/content-transform.worker.d.ts +0 -1
- package/dist/workers/content-transform.worker.js +0 -40
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import diagnosticsChannel from 'node:diagnostics_channel';
|
|
2
|
-
import { performance } from 'node:perf_hooks';
|
|
3
|
-
import { redactUrl } from '../utils/url-redactor.js';
|
|
4
|
-
import { getOperationId, getRequestId } from './context.js';
|
|
5
|
-
const transformChannel = diagnosticsChannel.channel('superfetch.transform');
|
|
6
|
-
function publishTransformEvent(event) {
|
|
7
|
-
if (!transformChannel.hasSubscribers)
|
|
8
|
-
return;
|
|
9
|
-
try {
|
|
10
|
-
transformChannel.publish(event);
|
|
11
|
-
}
|
|
12
|
-
catch {
|
|
13
|
-
// Avoid crashing the publisher if a subscriber throws.
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
export function startTransformStage(url, stage) {
|
|
17
|
-
if (!transformChannel.hasSubscribers)
|
|
18
|
-
return null;
|
|
19
|
-
return {
|
|
20
|
-
stage,
|
|
21
|
-
startTime: performance.now(),
|
|
22
|
-
url: redactUrl(url),
|
|
23
|
-
};
|
|
24
|
-
}
|
|
25
|
-
export function endTransformStage(context, options) {
|
|
26
|
-
if (!context)
|
|
27
|
-
return;
|
|
28
|
-
const requestId = getRequestId();
|
|
29
|
-
const operationId = getOperationId();
|
|
30
|
-
const event = {
|
|
31
|
-
v: 1,
|
|
32
|
-
type: 'stage',
|
|
33
|
-
stage: context.stage,
|
|
34
|
-
durationMs: performance.now() - context.startTime,
|
|
35
|
-
url: context.url,
|
|
36
|
-
...(requestId ? { requestId } : {}),
|
|
37
|
-
...(operationId ? { operationId } : {}),
|
|
38
|
-
...(options?.truncated !== undefined
|
|
39
|
-
? { truncated: options.truncated }
|
|
40
|
-
: {}),
|
|
41
|
-
};
|
|
42
|
-
publishTransformEvent(event);
|
|
43
|
-
}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import type { MarkdownTransformResult } from '../config/types/content.js';
|
|
2
|
-
interface TransformWorkerPool {
|
|
3
|
-
transform(html: string, url: string, options: {
|
|
4
|
-
includeMetadata: boolean;
|
|
5
|
-
signal?: AbortSignal;
|
|
6
|
-
}): Promise<MarkdownTransformResult>;
|
|
7
|
-
close(): Promise<void>;
|
|
8
|
-
}
|
|
9
|
-
export declare function getOrCreateTransformWorkerPool(): TransformWorkerPool;
|
|
10
|
-
export declare function shutdownTransformWorkerPool(): Promise<void>;
|
|
11
|
-
export {};
|
|
@@ -1,244 +0,0 @@
|
|
|
1
|
-
import { randomUUID } from 'node:crypto';
|
|
2
|
-
import os from 'node:os';
|
|
3
|
-
import { Worker } from 'node:worker_threads';
|
|
4
|
-
import { FetchError } from '../errors/app-error.js';
|
|
5
|
-
import { getErrorMessage } from '../utils/error-details.js';
|
|
6
|
-
let pool = null;
|
|
7
|
-
function resolveDefaultWorkerCount() {
|
|
8
|
-
const parallelism = typeof os.availableParallelism === 'function'
|
|
9
|
-
? os.availableParallelism()
|
|
10
|
-
: os.cpus().length;
|
|
11
|
-
// Leave 1 core for the event loop; cap to avoid runaway memory.
|
|
12
|
-
return Math.min(16, Math.max(1, parallelism - 1));
|
|
13
|
-
}
|
|
14
|
-
const DEFAULT_TIMEOUT_MS = 30000;
|
|
15
|
-
export function getOrCreateTransformWorkerPool() {
|
|
16
|
-
pool ??= new WorkerPool(resolveDefaultWorkerCount(), DEFAULT_TIMEOUT_MS);
|
|
17
|
-
return pool;
|
|
18
|
-
}
|
|
19
|
-
export async function shutdownTransformWorkerPool() {
|
|
20
|
-
if (!pool)
|
|
21
|
-
return;
|
|
22
|
-
await pool.close();
|
|
23
|
-
pool = null;
|
|
24
|
-
}
|
|
25
|
-
class WorkerPool {
|
|
26
|
-
workers = [];
|
|
27
|
-
queue = [];
|
|
28
|
-
inflight = new Map();
|
|
29
|
-
timeoutMs;
|
|
30
|
-
queueMax;
|
|
31
|
-
closed = false;
|
|
32
|
-
constructor(size, timeoutMs) {
|
|
33
|
-
const safeSize = Math.max(1, size);
|
|
34
|
-
this.timeoutMs = timeoutMs;
|
|
35
|
-
this.queueMax = safeSize * 2;
|
|
36
|
-
for (let index = 0; index < safeSize; index += 1) {
|
|
37
|
-
this.workers.push(this.spawnWorker(index));
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
spawnWorker(workerIndex) {
|
|
41
|
-
const worker = new Worker(new URL('../workers/transform-worker.js', import.meta.url));
|
|
42
|
-
// Workers must not keep the process alive by themselves.
|
|
43
|
-
worker.unref();
|
|
44
|
-
const slot = {
|
|
45
|
-
worker,
|
|
46
|
-
busy: false,
|
|
47
|
-
currentTaskId: null,
|
|
48
|
-
};
|
|
49
|
-
worker.on('message', (raw) => {
|
|
50
|
-
this.onWorkerMessage(workerIndex, raw);
|
|
51
|
-
});
|
|
52
|
-
worker.on('error', (error) => {
|
|
53
|
-
this.onWorkerBroken(workerIndex, `Transform worker error: ${getErrorMessage(error)}`);
|
|
54
|
-
});
|
|
55
|
-
worker.on('exit', (code) => {
|
|
56
|
-
this.onWorkerBroken(workerIndex, `Transform worker exited (code ${code})`);
|
|
57
|
-
});
|
|
58
|
-
return slot;
|
|
59
|
-
}
|
|
60
|
-
onWorkerBroken(workerIndex, message) {
|
|
61
|
-
if (this.closed)
|
|
62
|
-
return;
|
|
63
|
-
const slot = this.workers[workerIndex];
|
|
64
|
-
if (!slot)
|
|
65
|
-
return;
|
|
66
|
-
if (slot.busy && slot.currentTaskId) {
|
|
67
|
-
this.failTask(slot.currentTaskId, new Error(message));
|
|
68
|
-
}
|
|
69
|
-
void slot.worker.terminate();
|
|
70
|
-
this.workers[workerIndex] = this.spawnWorker(workerIndex);
|
|
71
|
-
this.drainQueue();
|
|
72
|
-
}
|
|
73
|
-
onWorkerMessage(workerIndex, raw) {
|
|
74
|
-
if (!raw ||
|
|
75
|
-
typeof raw !== 'object' ||
|
|
76
|
-
!('type' in raw) ||
|
|
77
|
-
!('id' in raw) ||
|
|
78
|
-
typeof raw.id !== 'string' ||
|
|
79
|
-
typeof raw.type !== 'string') {
|
|
80
|
-
return;
|
|
81
|
-
}
|
|
82
|
-
const message = raw;
|
|
83
|
-
const inflight = this.inflight.get(message.id);
|
|
84
|
-
if (!inflight)
|
|
85
|
-
return;
|
|
86
|
-
clearTimeout(inflight.timer);
|
|
87
|
-
if (inflight.signal && inflight.abortListener) {
|
|
88
|
-
inflight.signal.removeEventListener('abort', inflight.abortListener);
|
|
89
|
-
}
|
|
90
|
-
this.inflight.delete(message.id);
|
|
91
|
-
const slot = this.workers[workerIndex];
|
|
92
|
-
if (slot) {
|
|
93
|
-
slot.busy = false;
|
|
94
|
-
slot.currentTaskId = null;
|
|
95
|
-
}
|
|
96
|
-
if (message.type === 'result') {
|
|
97
|
-
inflight.resolve(message.result);
|
|
98
|
-
}
|
|
99
|
-
else {
|
|
100
|
-
const { error } = message;
|
|
101
|
-
if (error.name === 'FetchError') {
|
|
102
|
-
inflight.reject(new FetchError(error.message, error.url, error.statusCode, error.details ?? {}));
|
|
103
|
-
}
|
|
104
|
-
else {
|
|
105
|
-
inflight.reject(new Error(error.message));
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
this.drainQueue();
|
|
109
|
-
}
|
|
110
|
-
failTask(id, error) {
|
|
111
|
-
const inflight = this.inflight.get(id);
|
|
112
|
-
if (!inflight)
|
|
113
|
-
return;
|
|
114
|
-
clearTimeout(inflight.timer);
|
|
115
|
-
if (inflight.signal && inflight.abortListener) {
|
|
116
|
-
inflight.signal.removeEventListener('abort', inflight.abortListener);
|
|
117
|
-
}
|
|
118
|
-
this.inflight.delete(id);
|
|
119
|
-
inflight.reject(error);
|
|
120
|
-
const slot = this.workers[inflight.workerIndex];
|
|
121
|
-
if (slot) {
|
|
122
|
-
slot.busy = false;
|
|
123
|
-
slot.currentTaskId = null;
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
async transform(html, url, options) {
|
|
127
|
-
if (this.closed) {
|
|
128
|
-
throw new Error('Transform worker pool closed');
|
|
129
|
-
}
|
|
130
|
-
if (this.queue.length >= this.queueMax) {
|
|
131
|
-
throw new Error('Transform worker queue is full');
|
|
132
|
-
}
|
|
133
|
-
return new Promise((resolve, reject) => {
|
|
134
|
-
this.queue.push({
|
|
135
|
-
id: randomUUID(),
|
|
136
|
-
html,
|
|
137
|
-
url,
|
|
138
|
-
includeMetadata: options.includeMetadata,
|
|
139
|
-
signal: options.signal,
|
|
140
|
-
resolve,
|
|
141
|
-
reject,
|
|
142
|
-
});
|
|
143
|
-
this.drainQueue();
|
|
144
|
-
});
|
|
145
|
-
}
|
|
146
|
-
drainQueue() {
|
|
147
|
-
if (this.queue.length === 0)
|
|
148
|
-
return;
|
|
149
|
-
for (let workerIndex = 0; workerIndex < this.workers.length; workerIndex += 1) {
|
|
150
|
-
const slot = this.workers[workerIndex];
|
|
151
|
-
if (!slot || slot.busy)
|
|
152
|
-
continue;
|
|
153
|
-
const task = this.queue.shift();
|
|
154
|
-
if (!task)
|
|
155
|
-
return;
|
|
156
|
-
this.dispatch(workerIndex, slot, task);
|
|
157
|
-
if (this.queue.length === 0)
|
|
158
|
-
return;
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
dispatch(workerIndex, slot, task) {
|
|
162
|
-
if (task.signal?.aborted) {
|
|
163
|
-
task.reject(new FetchError('Request was canceled', task.url, 499, {
|
|
164
|
-
reason: 'aborted',
|
|
165
|
-
stage: 'transform:dispatch',
|
|
166
|
-
}));
|
|
167
|
-
return;
|
|
168
|
-
}
|
|
169
|
-
slot.busy = true;
|
|
170
|
-
slot.currentTaskId = task.id;
|
|
171
|
-
const timer = setTimeout(() => {
|
|
172
|
-
try {
|
|
173
|
-
slot.worker.postMessage({ type: 'cancel', id: task.id });
|
|
174
|
-
}
|
|
175
|
-
catch {
|
|
176
|
-
// ignore
|
|
177
|
-
}
|
|
178
|
-
const inflight = this.inflight.get(task.id);
|
|
179
|
-
if (!inflight)
|
|
180
|
-
return;
|
|
181
|
-
clearTimeout(inflight.timer);
|
|
182
|
-
if (inflight.signal && inflight.abortListener) {
|
|
183
|
-
inflight.signal.removeEventListener('abort', inflight.abortListener);
|
|
184
|
-
}
|
|
185
|
-
this.inflight.delete(task.id);
|
|
186
|
-
inflight.reject(new FetchError('Request timeout', task.url, 504, {
|
|
187
|
-
reason: 'timeout',
|
|
188
|
-
stage: 'transform:worker-timeout',
|
|
189
|
-
}));
|
|
190
|
-
if (!this.closed) {
|
|
191
|
-
void slot.worker.terminate();
|
|
192
|
-
this.workers[workerIndex] = this.spawnWorker(workerIndex);
|
|
193
|
-
this.drainQueue();
|
|
194
|
-
}
|
|
195
|
-
}, this.timeoutMs).unref();
|
|
196
|
-
let abortListener;
|
|
197
|
-
if (task.signal) {
|
|
198
|
-
abortListener = () => {
|
|
199
|
-
try {
|
|
200
|
-
slot.worker.postMessage({ type: 'cancel', id: task.id });
|
|
201
|
-
}
|
|
202
|
-
catch {
|
|
203
|
-
// ignore
|
|
204
|
-
}
|
|
205
|
-
};
|
|
206
|
-
task.signal.addEventListener('abort', abortListener, { once: true });
|
|
207
|
-
}
|
|
208
|
-
this.inflight.set(task.id, {
|
|
209
|
-
resolve: task.resolve,
|
|
210
|
-
reject: task.reject,
|
|
211
|
-
timer,
|
|
212
|
-
signal: task.signal,
|
|
213
|
-
abortListener,
|
|
214
|
-
workerIndex,
|
|
215
|
-
});
|
|
216
|
-
slot.worker.postMessage({
|
|
217
|
-
type: 'transform',
|
|
218
|
-
id: task.id,
|
|
219
|
-
html: task.html,
|
|
220
|
-
url: task.url,
|
|
221
|
-
includeMetadata: task.includeMetadata,
|
|
222
|
-
});
|
|
223
|
-
}
|
|
224
|
-
async close() {
|
|
225
|
-
if (this.closed)
|
|
226
|
-
return;
|
|
227
|
-
this.closed = true;
|
|
228
|
-
const terminations = this.workers.map((slot) => slot.worker.terminate());
|
|
229
|
-
this.workers.length = 0;
|
|
230
|
-
for (const [id, inflight] of this.inflight.entries()) {
|
|
231
|
-
clearTimeout(inflight.timer);
|
|
232
|
-
if (inflight.signal && inflight.abortListener) {
|
|
233
|
-
inflight.signal.removeEventListener('abort', inflight.abortListener);
|
|
234
|
-
}
|
|
235
|
-
inflight.reject(new Error('Transform worker pool closed'));
|
|
236
|
-
this.inflight.delete(id);
|
|
237
|
-
}
|
|
238
|
-
for (const task of this.queue) {
|
|
239
|
-
task.reject(new Error('Transform worker pool closed'));
|
|
240
|
-
}
|
|
241
|
-
this.queue.length = 0;
|
|
242
|
-
await Promise.allSettled(terminations);
|
|
243
|
-
}
|
|
244
|
-
}
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import type { Worker } from 'node:worker_threads';
|
|
2
|
-
import type { MarkdownTransformResult, TransformOptions } from '../config/types/content.js';
|
|
3
|
-
export interface WorkerTransformRequest {
|
|
4
|
-
id: number;
|
|
5
|
-
html: string;
|
|
6
|
-
url: string;
|
|
7
|
-
options: TransformOptions;
|
|
8
|
-
}
|
|
9
|
-
export type WorkerTransformResponse = {
|
|
10
|
-
id: number;
|
|
11
|
-
ok: true;
|
|
12
|
-
result: MarkdownTransformResult;
|
|
13
|
-
} | {
|
|
14
|
-
id: number;
|
|
15
|
-
ok: false;
|
|
16
|
-
error: string;
|
|
17
|
-
};
|
|
18
|
-
export interface TransformTask {
|
|
19
|
-
id: number;
|
|
20
|
-
request: WorkerTransformRequest;
|
|
21
|
-
resolve: (result: MarkdownTransformResult) => void;
|
|
22
|
-
reject: (error: Error) => void;
|
|
23
|
-
signal: AbortSignal | undefined;
|
|
24
|
-
abortHandler: (() => void) | undefined;
|
|
25
|
-
status: 'queued' | 'running';
|
|
26
|
-
}
|
|
27
|
-
export interface WorkerSlot {
|
|
28
|
-
worker: Worker;
|
|
29
|
-
busy: boolean;
|
|
30
|
-
current: TransformTask | undefined;
|
|
31
|
-
}
|
|
32
|
-
export declare function isWorkerResponse(value: unknown): value is WorkerTransformResponse;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { isRecord } from '../utils/guards.js';
|
|
2
|
-
export function isWorkerResponse(value) {
|
|
3
|
-
if (!isRecord(value))
|
|
4
|
-
return false;
|
|
5
|
-
if (typeof value.id !== 'number')
|
|
6
|
-
return false;
|
|
7
|
-
if (value.ok === true) {
|
|
8
|
-
return 'result' in value;
|
|
9
|
-
}
|
|
10
|
-
if (value.ok === false) {
|
|
11
|
-
return typeof value.error === 'string';
|
|
12
|
-
}
|
|
13
|
-
return false;
|
|
14
|
-
}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
export interface RetryStrategy {
|
|
2
|
-
calculateDelay(attempt: number): number;
|
|
3
|
-
shouldRetry(attempt: number, maxRetries: number, error: Error): boolean;
|
|
4
|
-
}
|
|
5
|
-
export declare class ExponentialBackoffStrategy implements RetryStrategy {
|
|
6
|
-
private readonly baseDelayMs;
|
|
7
|
-
private readonly maxDelayMs;
|
|
8
|
-
private readonly jitterFactor;
|
|
9
|
-
constructor(baseDelayMs?: number, maxDelayMs?: number, jitterFactor?: number);
|
|
10
|
-
calculateDelay(attempt: number): number;
|
|
11
|
-
shouldRetry(attempt: number, maxRetries: number, error: Error): boolean;
|
|
12
|
-
}
|
|
13
|
-
//# sourceMappingURL=exponential-backoff-strategy.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"exponential-backoff-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/exponential-backoff-strategy.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACxC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC;CACzE;AAED,qBAAa,0BAA2B,YAAW,aAAa;IAE5D,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,YAAY;gBAFZ,WAAW,SAAO,EAClB,UAAU,SAAQ,EAClB,YAAY,SAAO;IAGtC,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAUvC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO;CAkBxE"}
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
export class ExponentialBackoffStrategy {
|
|
2
|
-
baseDelayMs;
|
|
3
|
-
maxDelayMs;
|
|
4
|
-
jitterFactor;
|
|
5
|
-
constructor(baseDelayMs = 1000, maxDelayMs = 10000, jitterFactor = 0.25) {
|
|
6
|
-
this.baseDelayMs = baseDelayMs;
|
|
7
|
-
this.maxDelayMs = maxDelayMs;
|
|
8
|
-
this.jitterFactor = jitterFactor;
|
|
9
|
-
}
|
|
10
|
-
calculateDelay(attempt) {
|
|
11
|
-
const exponentialDelay = Math.min(this.baseDelayMs * Math.pow(2, attempt - 1), this.maxDelayMs);
|
|
12
|
-
const jitter = exponentialDelay * this.jitterFactor * (Math.random() * 2 - 1);
|
|
13
|
-
return Math.round(exponentialDelay + jitter);
|
|
14
|
-
}
|
|
15
|
-
shouldRetry(attempt, maxRetries, error) {
|
|
16
|
-
if (attempt >= maxRetries)
|
|
17
|
-
return false;
|
|
18
|
-
// Don't retry on abort
|
|
19
|
-
if (error.name === 'AbortError' || error.name === 'CanceledError') {
|
|
20
|
-
return false;
|
|
21
|
-
}
|
|
22
|
-
// Don't retry on client errors (4xx except 429)
|
|
23
|
-
if ('httpStatus' in error) {
|
|
24
|
-
const status = error.httpStatus;
|
|
25
|
-
if (status && status >= 400 && status < 500 && status !== 429) {
|
|
26
|
-
return false;
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
return true;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
//# sourceMappingURL=exponential-backoff-strategy.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"exponential-backoff-strategy.js","sourceRoot":"","sources":["../../src/strategies/exponential-backoff-strategy.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,0BAA0B;IAElB;IACA;IACA;IAHnB,YACmB,cAAc,IAAI,EAClB,aAAa,KAAK,EAClB,eAAe,IAAI;QAFnB,gBAAW,GAAX,WAAW,CAAO;QAClB,eAAU,GAAV,UAAU,CAAQ;QAClB,iBAAY,GAAZ,YAAY,CAAO;IACnC,CAAC;IAEJ,cAAc,CAAC,OAAe;QAC5B,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EAC3C,IAAI,CAAC,UAAU,CAChB,CAAC;QACF,MAAM,MAAM,GACV,gBAAgB,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,WAAW,CAAC,OAAe,EAAE,UAAkB,EAAE,KAAY;QAC3D,IAAI,OAAO,IAAI,UAAU;YAAE,OAAO,KAAK,CAAC;QAExC,uBAAuB;QACvB,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;YAClE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,gDAAgD;QAChD,IAAI,YAAY,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAI,KAAiC,CAAC,UAAU,CAAC;YAC7D,IAAI,MAAM,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC9D,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
import type { ExtractLinksOptions, LinksTransformResult, ToolResponseBase } from '../../../config/types.js';
|
|
2
|
-
export declare function resolveFilterPattern(pattern: string | undefined, url: string): RegExp | undefined | ToolResponseBase;
|
|
3
|
-
export declare function extractLinks(html: string, baseUrl: string, options: ExtractLinksOptions): LinksTransformResult;
|
|
4
|
-
//# sourceMappingURL=link-extractor.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"link-extractor.d.ts","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-links/link-extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,mBAAmB,EACnB,oBAAoB,EAEpB,gBAAgB,EACjB,MAAM,0BAA0B,CAAC;AA0ClC,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,SAAS,GAAG,gBAAgB,CAavC;AAoLD,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,mBAAmB,GAC3B,oBAAoB,CAiBtB"}
|
|
@@ -1,159 +0,0 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio';
|
|
2
|
-
import safeRegex from 'safe-regex';
|
|
3
|
-
import { createToolErrorResponse } from '../../../utils/tool-error-handler.js';
|
|
4
|
-
import { isInternalUrl } from '../../../utils/url-validator.js';
|
|
5
|
-
function isLinkAllowed(type, options) {
|
|
6
|
-
const allowed = {
|
|
7
|
-
internal: options.includeInternal,
|
|
8
|
-
external: options.includeExternal,
|
|
9
|
-
image: options.includeImages,
|
|
10
|
-
};
|
|
11
|
-
return allowed[type];
|
|
12
|
-
}
|
|
13
|
-
function matchesFilter(url, filterPattern) {
|
|
14
|
-
if (!filterPattern)
|
|
15
|
-
return true;
|
|
16
|
-
return filterPattern.test(url);
|
|
17
|
-
}
|
|
18
|
-
function evaluateLink(link, options, seen) {
|
|
19
|
-
if (seen.has(link.href)) {
|
|
20
|
-
return { accepted: false, filtered: false };
|
|
21
|
-
}
|
|
22
|
-
if (!matchesFilter(link.href, options.filterPattern)) {
|
|
23
|
-
return { accepted: false, filtered: true };
|
|
24
|
-
}
|
|
25
|
-
if (!isLinkAllowed(link.type, options)) {
|
|
26
|
-
return { accepted: false, filtered: true };
|
|
27
|
-
}
|
|
28
|
-
return { accepted: true, filtered: false };
|
|
29
|
-
}
|
|
30
|
-
export function resolveFilterPattern(pattern, url) {
|
|
31
|
-
if (!pattern)
|
|
32
|
-
return undefined;
|
|
33
|
-
const lengthError = validatePatternLength(pattern, url);
|
|
34
|
-
if (lengthError)
|
|
35
|
-
return lengthError;
|
|
36
|
-
const filterPattern = buildFilterRegex(pattern, url);
|
|
37
|
-
if (isToolResponseBase(filterPattern))
|
|
38
|
-
return filterPattern;
|
|
39
|
-
const safetyError = validatePatternSafety(filterPattern, url);
|
|
40
|
-
if (safetyError)
|
|
41
|
-
return safetyError;
|
|
42
|
-
return filterPattern;
|
|
43
|
-
}
|
|
44
|
-
function validatePatternLength(pattern, url) {
|
|
45
|
-
if (pattern.length <= 200)
|
|
46
|
-
return null;
|
|
47
|
-
return createToolErrorResponse('Filter pattern too long (max 200 characters)', url, 'VALIDATION_ERROR');
|
|
48
|
-
}
|
|
49
|
-
function buildFilterRegex(pattern, url) {
|
|
50
|
-
try {
|
|
51
|
-
return new RegExp(pattern, 'i');
|
|
52
|
-
}
|
|
53
|
-
catch {
|
|
54
|
-
return createToolErrorResponse(`Invalid filter pattern: ${pattern}`, url, 'VALIDATION_ERROR');
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
function validatePatternSafety(pattern, url) {
|
|
58
|
-
if (safeRegex(pattern))
|
|
59
|
-
return null;
|
|
60
|
-
return createToolErrorResponse('Filter pattern is unsafe (potential catastrophic backtracking)', url, 'VALIDATION_ERROR');
|
|
61
|
-
}
|
|
62
|
-
function isToolResponseBase(value) {
|
|
63
|
-
return (value !== null &&
|
|
64
|
-
typeof value === 'object' &&
|
|
65
|
-
'content' in value &&
|
|
66
|
-
Array.isArray(value.content));
|
|
67
|
-
}
|
|
68
|
-
function tryResolveUrl(href, baseUrl) {
|
|
69
|
-
if (!URL.canParse(href, baseUrl)) {
|
|
70
|
-
return null;
|
|
71
|
-
}
|
|
72
|
-
return new URL(href, baseUrl).href;
|
|
73
|
-
}
|
|
74
|
-
function buildLinkType(url, baseUrl) {
|
|
75
|
-
return isInternalUrl(url, baseUrl) ? 'internal' : 'external';
|
|
76
|
-
}
|
|
77
|
-
function isSkippableHref(href) {
|
|
78
|
-
return href.startsWith('#') || href.startsWith('javascript:');
|
|
79
|
-
}
|
|
80
|
-
function getAnchorHref($, el) {
|
|
81
|
-
const href = $(el).attr('href');
|
|
82
|
-
if (!href)
|
|
83
|
-
return null;
|
|
84
|
-
if (isSkippableHref(href))
|
|
85
|
-
return null;
|
|
86
|
-
return href;
|
|
87
|
-
}
|
|
88
|
-
function resolveAnchorLink($, el, baseUrl) {
|
|
89
|
-
const href = getAnchorHref($, el);
|
|
90
|
-
if (!href)
|
|
91
|
-
return null;
|
|
92
|
-
const url = tryResolveUrl(href, baseUrl);
|
|
93
|
-
if (!url)
|
|
94
|
-
return null;
|
|
95
|
-
return {
|
|
96
|
-
href: url,
|
|
97
|
-
text: $(el).text().trim() || url,
|
|
98
|
-
type: buildLinkType(url, baseUrl),
|
|
99
|
-
};
|
|
100
|
-
}
|
|
101
|
-
function resolveImageLink($, el, baseUrl) {
|
|
102
|
-
const src = getImageSrc($, el);
|
|
103
|
-
if (!src)
|
|
104
|
-
return null;
|
|
105
|
-
const url = tryResolveUrl(src, baseUrl);
|
|
106
|
-
if (!url)
|
|
107
|
-
return null;
|
|
108
|
-
return {
|
|
109
|
-
href: url,
|
|
110
|
-
text: $(el).attr('alt')?.trim() ?? url,
|
|
111
|
-
type: 'image',
|
|
112
|
-
};
|
|
113
|
-
}
|
|
114
|
-
function getImageSrc($, el) {
|
|
115
|
-
const src = $(el).attr('src');
|
|
116
|
-
if (!src || src.startsWith('data:'))
|
|
117
|
-
return null;
|
|
118
|
-
return src;
|
|
119
|
-
}
|
|
120
|
-
function collectLinks($, selector, resolveLink, options, seen, links) {
|
|
121
|
-
let filtered = 0;
|
|
122
|
-
$(selector).each((_, el) => {
|
|
123
|
-
const link = resolveLink($, el);
|
|
124
|
-
if (!link)
|
|
125
|
-
return;
|
|
126
|
-
const result = evaluateLink(link, options, seen);
|
|
127
|
-
if (result.filtered)
|
|
128
|
-
filtered += 1;
|
|
129
|
-
if (!result.accepted)
|
|
130
|
-
return;
|
|
131
|
-
seen.add(link.href);
|
|
132
|
-
links.push(link);
|
|
133
|
-
});
|
|
134
|
-
return filtered;
|
|
135
|
-
}
|
|
136
|
-
function collectAnchorLinks($, baseUrl, options, seen, links) {
|
|
137
|
-
return collectLinks($, 'a[href]', (instance, el) => resolveAnchorLink(instance, el, baseUrl), options, seen, links);
|
|
138
|
-
}
|
|
139
|
-
function collectImageLinks($, baseUrl, options, seen, links) {
|
|
140
|
-
if (!options.includeImages)
|
|
141
|
-
return 0;
|
|
142
|
-
return collectLinks($, 'img[src]', (instance, el) => resolveImageLink(instance, el, baseUrl), options, seen, links);
|
|
143
|
-
}
|
|
144
|
-
export function extractLinks(html, baseUrl, options) {
|
|
145
|
-
const $ = cheerio.load(html);
|
|
146
|
-
const links = [];
|
|
147
|
-
const seen = new Set();
|
|
148
|
-
let filtered = collectAnchorLinks($, baseUrl, options, seen, links);
|
|
149
|
-
filtered += collectImageLinks($, baseUrl, options, seen, links);
|
|
150
|
-
const truncated = options.maxLinks ? links.length > options.maxLinks : false;
|
|
151
|
-
const resultLinks = truncated ? links.slice(0, options.maxLinks) : links;
|
|
152
|
-
return {
|
|
153
|
-
links: resultLinks,
|
|
154
|
-
linkCount: resultLinks.length,
|
|
155
|
-
filtered,
|
|
156
|
-
truncated,
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
//# sourceMappingURL=link-extractor.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"link-extractor.js","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-links/link-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,SAAS,MAAM,YAAY,CAAC;AAUnC,OAAO,EAAE,uBAAuB,EAAE,MAAM,sCAAsC,CAAC;AAC/E,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAEhE,SAAS,aAAa,CAAC,IAAc,EAAE,OAA4B;IACjE,MAAM,OAAO,GAA8B;QACzC,QAAQ,EAAE,OAAO,CAAC,eAAe;QACjC,QAAQ,EAAE,OAAO,CAAC,eAAe;QACjC,KAAK,EAAE,OAAO,CAAC,aAAa;KAC7B,CAAC;IACF,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,aAAa,CACpB,GAAW,EACX,aAAiC;IAEjC,IAAI,CAAC,aAAa;QAAE,OAAO,IAAI,CAAC;IAChC,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,YAAY,CACnB,IAAmB,EACnB,OAA4B,EAC5B,IAAiB;IAEjB,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAC9C,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACrD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAA2B,EAC3B,GAAW;IAEX,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAE/B,MAAM,WAAW,GAAG,qBAAqB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACxD,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IAEpC,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACrD,IAAI,kBAAkB,CAAC,aAAa,CAAC;QAAE,OAAO,aAAa,CAAC;IAE5D,MAAM,WAAW,GAAG,qBAAqB,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;IAC9D,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IAEpC,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAe,EACf,GAAW;IAEX,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,uBAAuB,CAC5B,8CAA8C,EAC9C,GAAG,EACH,kBAAkB,CACnB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAe,EACf,GAAW;IAEX,IAAI,CAAC;QACH,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,uBAAuB,CAC5B,2BAA2B,OAAO,EAAE,EACpC,GAAG,EACH,kBAAkB,CACnB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAe,EACf,GAAW;IAEX,IAAI,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACpC,OAAO,uBAAuB,CAC5B,gEAAgE,EAChE,GAAG,EACH,kBAAkB,CACnB,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,CACL,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,SAAS,IAAI,KAAK;QAClB,KAAK,CAAC,OAAO,CAAE,KAA0B,CAAC,OAAO,CAAC,CACnD,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,OAAe;IAClD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;AACrC,CAAC;AAED,SAAS,aAAa,CAAC,GAAW,EAAE,OAAe;IACjD,OAAO,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;AAC/D,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,aAAa,CAAC,CAAqB,EAAE,EAAW;IACvD,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,eAAe,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CACxB,CAAqB,EACrB,EAAW,EACX,OAAe;IAEf,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,GAAG;QACT,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,GAAG;QAChC,IAAI,EAAE,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC;KAClC,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACvB,CAAqB,EACrB,EAAW,EACX,OAAe;IAEf,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACxC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,GAAG;QACT,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG;QACtC,IAAI,EAAE,OAAO;KACd,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,CAAqB,EAAE,EAAW;IACrD,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACjD,OAAO,GAAG,CAAC;AACb,CAAC;AAOD,SAAS,YAAY,CACnB,CAAqB,EACrB,QAAgB,EAChB,WAAyB,EACzB,OAA4B,EAC5B,IAAiB,EACjB,KAAsB;IAEtB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACzB,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,EAAa,CAAC,CAAC;QAC3C,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QACjD,IAAI,MAAM,CAAC,QAAQ;YAAE,QAAQ,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ;YAAE,OAAO;QAE7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CACzB,CAAqB,EACrB,OAAe,EACf,OAA4B,EAC5B,IAAiB,EACjB,KAAsB;IAEtB,OAAO,YAAY,CACjB,CAAC,EACD,SAAS,EACT,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,CAAC,EAC1D,OAAO,EACP,IAAI,EACJ,KAAK,CACN,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,CAAqB,EACrB,OAAe,EACf,OAA4B,EAC5B,IAAiB,EACjB,KAAsB;IAEtB,IAAI,CAAC,OAAO,CAAC,aAAa;QAAE,OAAO,CAAC,CAAC;IAErC,OAAO,YAAY,CACjB,CAAC,EACD,UAAU,EACV,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,CAAC,EACzD,OAAO,EACP,IAAI,EACJ,KAAK,CACN,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,OAAe,EACf,OAA4B;IAE5B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,IAAI,QAAQ,GAAG,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IACpE,QAAQ,IAAI,iBAAiB,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IAEhE,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7E,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,WAAW;QAClB,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,QAAQ;QACR,SAAS;KACV,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import type { FetchLinksInput, ToolResponseBase } from '../../config/types.js';
|
|
2
|
-
export declare const FETCH_LINKS_TOOL_NAME = "fetch-links";
|
|
3
|
-
export declare const FETCH_LINKS_TOOL_DESCRIPTION = "Extracts all hyperlinks from a webpage with anchor text and type classification. Supports filtering, image links, and link limits.";
|
|
4
|
-
export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<ToolResponseBase>;
|
|
5
|
-
//# sourceMappingURL=fetch-links.tool.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,eAAe,EAGf,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAgB/B,eAAO,MAAM,qBAAqB,gBAAgB,CAAC;AACnD,eAAO,MAAM,4BAA4B,uIAC6F,CAAC;AAgEvI,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,gBAAgB,CAAC,CAU3B"}
|