@j0hanz/superfetch 2.1.4 → 2.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts +1 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +1 -0
- package/dist/cache.js.map +1 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -0
- package/dist/crypto.d.ts +1 -0
- package/dist/crypto.d.ts.map +1 -0
- package/dist/crypto.js +1 -0
- package/dist/crypto.js.map +1 -0
- package/dist/errors.d.ts +1 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +1 -0
- package/dist/errors.js.map +1 -0
- package/dist/fetch.d.ts +1 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +1 -0
- package/dist/fetch.js.map +1 -0
- package/dist/http.d.ts +1 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +1 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts +1 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +1 -0
- package/dist/mcp.js.map +1 -0
- package/dist/observability.d.ts +1 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/observability.js +1 -0
- package/dist/observability.js.map +1 -0
- package/dist/tools.d.ts +1 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +1 -0
- package/dist/tools.js.map +1 -0
- package/dist/transform.d.ts +1 -0
- package/dist/transform.d.ts.map +1 -0
- package/dist/transform.js +1 -0
- package/dist/transform.js.map +1 -0
- package/dist/utils.d.ts +1 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +1 -0
- package/dist/utils.js.map +1 -0
- package/dist/workers/transform-worker.d.ts +1 -0
- package/dist/workers/transform-worker.d.ts.map +1 -0
- package/dist/workers/transform-worker.js +1 -0
- package/dist/workers/transform-worker.js.map +1 -0
- package/package.json +5 -2
- package/dist/config/auth-config.d.ts +0 -16
- package/dist/config/auth-config.js +0 -53
- package/dist/config/constants.d.ts +0 -17
- package/dist/config/constants.d.ts.map +0 -1
- package/dist/config/constants.js +0 -22
- package/dist/config/constants.js.map +0 -1
- package/dist/config/env-parsers.d.ts +0 -8
- package/dist/config/env-parsers.js +0 -96
- package/dist/config/formatting.d.ts +0 -7
- package/dist/config/formatting.d.ts.map +0 -1
- package/dist/config/formatting.js +0 -9
- package/dist/config/formatting.js.map +0 -1
- package/dist/config/index.d.ts +0 -61
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -107
- package/dist/config/index.js.map +0 -1
- package/dist/config/types/content.d.ts +0 -41
- package/dist/config/types/content.d.ts.map +0 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/content.js.map +0 -1
- package/dist/config/types/runtime.d.ts +0 -72
- package/dist/config/types/runtime.d.ts.map +0 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/runtime.js.map +0 -1
- package/dist/config/types/tools.d.ts +0 -28
- package/dist/config/types/tools.d.ts.map +0 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/config/types/tools.js.map +0 -1
- package/dist/config/types.d.ts +0 -4
- package/dist/config/types.d.ts.map +0 -1
- package/dist/config/types.js +0 -2
- package/dist/config/types.js.map +0 -1
- package/dist/errors/app-error.d.ts +0 -7
- package/dist/errors/app-error.d.ts.map +0 -1
- package/dist/errors/app-error.js +0 -16
- package/dist/errors/app-error.js.map +0 -1
- package/dist/errors/index.d.ts +0 -2
- package/dist/errors/index.d.ts.map +0 -1
- package/dist/errors/index.js +0 -2
- package/dist/errors/index.js.map +0 -1
- package/dist/http/accept-policy.d.ts +0 -3
- package/dist/http/accept-policy.js +0 -45
- package/dist/http/async-handler.d.ts +0 -2
- package/dist/http/async-handler.js +0 -5
- package/dist/http/auth-introspection.d.ts +0 -2
- package/dist/http/auth-introspection.js +0 -141
- package/dist/http/auth-static.d.ts +0 -2
- package/dist/http/auth-static.js +0 -23
- package/dist/http/auth.d.ts +0 -3
- package/dist/http/auth.d.ts.map +0 -1
- package/dist/http/auth.js +0 -269
- package/dist/http/auth.js.map +0 -1
- package/dist/http/base-middleware.d.ts +0 -7
- package/dist/http/base-middleware.js +0 -143
- package/dist/http/cors.d.ts +0 -2
- package/dist/http/cors.d.ts.map +0 -1
- package/dist/http/cors.js +0 -9
- package/dist/http/cors.js.map +0 -1
- package/dist/http/download-routes.d.ts +0 -2
- package/dist/http/download-routes.d.ts.map +0 -1
- package/dist/http/download-routes.js +0 -104
- package/dist/http/download-routes.js.map +0 -1
- package/dist/http/error-handler.d.ts +0 -2
- package/dist/http/error-handler.js +0 -55
- package/dist/http/host-allowlist.d.ts +0 -3
- package/dist/http/host-allowlist.js +0 -117
- package/dist/http/jsonrpc-http.d.ts +0 -2
- package/dist/http/jsonrpc-http.js +0 -10
- package/dist/http/mcp-routes.d.ts +0 -9
- package/dist/http/mcp-routes.d.ts.map +0 -1
- package/dist/http/mcp-routes.js +0 -204
- package/dist/http/mcp-routes.js.map +0 -1
- package/dist/http/mcp-session-eviction.d.ts +0 -3
- package/dist/http/mcp-session-eviction.js +0 -24
- package/dist/http/mcp-session-helpers.d.ts +0 -12
- package/dist/http/mcp-session-helpers.d.ts.map +0 -1
- package/dist/http/mcp-session-helpers.js +0 -64
- package/dist/http/mcp-session-helpers.js.map +0 -1
- package/dist/http/mcp-session-init.d.ts +0 -7
- package/dist/http/mcp-session-init.js +0 -94
- package/dist/http/mcp-session-slots.d.ts +0 -17
- package/dist/http/mcp-session-slots.js +0 -55
- package/dist/http/mcp-session-transport-init.d.ts +0 -7
- package/dist/http/mcp-session-transport-init.js +0 -41
- package/dist/http/mcp-session-transport.d.ts +0 -7
- package/dist/http/mcp-session-transport.js +0 -57
- package/dist/http/mcp-session-types.d.ts +0 -5
- package/dist/http/mcp-session-types.js +0 -1
- package/dist/http/mcp-session.d.ts +0 -10
- package/dist/http/mcp-session.d.ts.map +0 -1
- package/dist/http/mcp-session.js +0 -23
- package/dist/http/mcp-session.js.map +0 -1
- package/dist/http/mcp-sessions.d.ts +0 -41
- package/dist/http/mcp-sessions.js +0 -392
- package/dist/http/mcp-validation.d.ts +0 -3
- package/dist/http/mcp-validation.d.ts.map +0 -1
- package/dist/http/mcp-validation.js +0 -14
- package/dist/http/mcp-validation.js.map +0 -1
- package/dist/http/protocol-policy.d.ts +0 -2
- package/dist/http/protocol-policy.js +0 -31
- package/dist/http/rate-limit.d.ts +0 -12
- package/dist/http/rate-limit.d.ts.map +0 -1
- package/dist/http/rate-limit.js +0 -93
- package/dist/http/rate-limit.js.map +0 -1
- package/dist/http/server-config.d.ts +0 -1
- package/dist/http/server-config.js +0 -40
- package/dist/http/server-middleware.d.ts +0 -7
- package/dist/http/server-middleware.d.ts.map +0 -1
- package/dist/http/server-middleware.js +0 -52
- package/dist/http/server-middleware.js.map +0 -1
- package/dist/http/server-shutdown.d.ts +0 -4
- package/dist/http/server-shutdown.js +0 -43
- package/dist/http/server-tuning.d.ts +0 -9
- package/dist/http/server-tuning.js +0 -45
- package/dist/http/server.d.ts +0 -3
- package/dist/http/server.d.ts.map +0 -1
- package/dist/http/server.js +0 -291
- package/dist/http/server.js.map +0 -1
- package/dist/http/session-cleanup.d.ts +0 -2
- package/dist/http/session-cleanup.d.ts.map +0 -1
- package/dist/http/session-cleanup.js +0 -40
- package/dist/http/session-cleanup.js.map +0 -1
- package/dist/http/sessions.d.ts +0 -14
- package/dist/http/sessions.d.ts.map +0 -1
- package/dist/http/sessions.js +0 -63
- package/dist/http/sessions.js.map +0 -1
- package/dist/middleware/error-handler.d.ts +0 -2
- package/dist/middleware/error-handler.d.ts.map +0 -1
- package/dist/middleware/error-handler.js +0 -56
- package/dist/middleware/error-handler.js.map +0 -1
- package/dist/middleware/rate-limiter.d.ts +0 -16
- package/dist/middleware/rate-limiter.d.ts.map +0 -1
- package/dist/middleware/rate-limiter.js +0 -111
- package/dist/middleware/rate-limiter.js.map +0 -1
- package/dist/parsers/base-html-element-parser.d.ts +0 -43
- package/dist/parsers/base-html-element-parser.d.ts.map +0 -1
- package/dist/parsers/base-html-element-parser.js +0 -59
- package/dist/parsers/base-html-element-parser.js.map +0 -1
- package/dist/parsers/heading-element-parser.d.ts +0 -14
- package/dist/parsers/heading-element-parser.d.ts.map +0 -1
- package/dist/parsers/heading-element-parser.js +0 -26
- package/dist/parsers/heading-element-parser.js.map +0 -1
- package/dist/parsers/image-element-parser.d.ts +0 -16
- package/dist/parsers/image-element-parser.d.ts.map +0 -1
- package/dist/parsers/image-element-parser.js +0 -33
- package/dist/parsers/image-element-parser.js.map +0 -1
- package/dist/parsers/link-element-parser.d.ts +0 -15
- package/dist/parsers/link-element-parser.d.ts.map +0 -1
- package/dist/parsers/link-element-parser.js +0 -28
- package/dist/parsers/link-element-parser.js.map +0 -1
- package/dist/parsers/open-graph-parser.d.ts +0 -17
- package/dist/parsers/open-graph-parser.d.ts.map +0 -1
- package/dist/parsers/open-graph-parser.js +0 -41
- package/dist/parsers/open-graph-parser.js.map +0 -1
- package/dist/parsers/schema-org-parser.d.ts +0 -17
- package/dist/parsers/schema-org-parser.d.ts.map +0 -1
- package/dist/parsers/schema-org-parser.js +0 -32
- package/dist/parsers/schema-org-parser.js.map +0 -1
- package/dist/parsers/standard-meta-parser.d.ts +0 -18
- package/dist/parsers/standard-meta-parser.d.ts.map +0 -1
- package/dist/parsers/standard-meta-parser.js +0 -32
- package/dist/parsers/standard-meta-parser.js.map +0 -1
- package/dist/parsers/twitter-card-parser.d.ts +0 -17
- package/dist/parsers/twitter-card-parser.d.ts.map +0 -1
- package/dist/parsers/twitter-card-parser.js +0 -41
- package/dist/parsers/twitter-card-parser.js.map +0 -1
- package/dist/prompts/index.d.ts +0 -3
- package/dist/prompts/index.d.ts.map +0 -1
- package/dist/prompts/index.js +0 -73
- package/dist/prompts/index.js.map +0 -1
- package/dist/resources/cached-content-params.d.ts +0 -5
- package/dist/resources/cached-content-params.js +0 -36
- package/dist/resources/cached-content.d.ts +0 -2
- package/dist/resources/cached-content.d.ts.map +0 -1
- package/dist/resources/cached-content.js +0 -132
- package/dist/resources/cached-content.js.map +0 -1
- package/dist/resources/index.d.ts +0 -2
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js +0 -4
- package/dist/resources/index.js.map +0 -1
- package/dist/server.d.ts +0 -3
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js +0 -94
- package/dist/server.js.map +0 -1
- package/dist/services/cache-events.d.ts +0 -8
- package/dist/services/cache-events.js +0 -19
- package/dist/services/cache-keys.d.ts +0 -7
- package/dist/services/cache-keys.js +0 -57
- package/dist/services/cache.d.ts +0 -17
- package/dist/services/cache.d.ts.map +0 -1
- package/dist/services/cache.js +0 -145
- package/dist/services/cache.js.map +0 -1
- package/dist/services/cache.service.d.ts +0 -52
- package/dist/services/cache.service.d.ts.map +0 -1
- package/dist/services/cache.service.js +0 -113
- package/dist/services/cache.service.js.map +0 -1
- package/dist/services/card-extractor.d.ts +0 -6
- package/dist/services/card-extractor.d.ts.map +0 -1
- package/dist/services/card-extractor.js +0 -199
- package/dist/services/card-extractor.js.map +0 -1
- package/dist/services/context.d.ts +0 -10
- package/dist/services/context.d.ts.map +0 -1
- package/dist/services/context.js +0 -14
- package/dist/services/context.js.map +0 -1
- package/dist/services/extractor.d.ts +0 -5
- package/dist/services/extractor.d.ts.map +0 -1
- package/dist/services/extractor.js +0 -142
- package/dist/services/extractor.js.map +0 -1
- package/dist/services/extractor.service.d.ts +0 -18
- package/dist/services/extractor.service.d.ts.map +0 -1
- package/dist/services/extractor.service.js +0 -75
- package/dist/services/extractor.service.js.map +0 -1
- package/dist/services/fetcher/agents.d.ts +0 -3
- package/dist/services/fetcher/agents.d.ts.map +0 -1
- package/dist/services/fetcher/agents.js +0 -100
- package/dist/services/fetcher/agents.js.map +0 -1
- package/dist/services/fetcher/dns-selection.d.ts +0 -2
- package/dist/services/fetcher/dns-selection.js +0 -72
- package/dist/services/fetcher/errors.d.ts +0 -4
- package/dist/services/fetcher/errors.d.ts.map +0 -1
- package/dist/services/fetcher/errors.js +0 -70
- package/dist/services/fetcher/errors.js.map +0 -1
- package/dist/services/fetcher/headers.d.ts +0 -2
- package/dist/services/fetcher/headers.d.ts.map +0 -1
- package/dist/services/fetcher/headers.js +0 -6
- package/dist/services/fetcher/headers.js.map +0 -1
- package/dist/services/fetcher/interceptors.d.ts +0 -10
- package/dist/services/fetcher/interceptors.d.ts.map +0 -1
- package/dist/services/fetcher/interceptors.js +0 -108
- package/dist/services/fetcher/interceptors.js.map +0 -1
- package/dist/services/fetcher/redirects.d.ts +0 -4
- package/dist/services/fetcher/redirects.d.ts.map +0 -1
- package/dist/services/fetcher/redirects.js +0 -78
- package/dist/services/fetcher/redirects.js.map +0 -1
- package/dist/services/fetcher/response.d.ts +0 -4
- package/dist/services/fetcher/response.d.ts.map +0 -1
- package/dist/services/fetcher/response.js +0 -104
- package/dist/services/fetcher/response.js.map +0 -1
- package/dist/services/fetcher/retry-policy.d.ts +0 -1
- package/dist/services/fetcher/retry-policy.d.ts.map +0 -1
- package/dist/services/fetcher/retry-policy.js +0 -131
- package/dist/services/fetcher/retry-policy.js.map +0 -1
- package/dist/services/fetcher.d.ts +0 -25
- package/dist/services/fetcher.d.ts.map +0 -1
- package/dist/services/fetcher.js +0 -607
- package/dist/services/fetcher.js.map +0 -1
- package/dist/services/fetcher.service.d.ts +0 -18
- package/dist/services/fetcher.service.d.ts.map +0 -1
- package/dist/services/fetcher.service.js +0 -122
- package/dist/services/fetcher.service.js.map +0 -1
- package/dist/services/fifo-queue.d.ts +0 -8
- package/dist/services/fifo-queue.js +0 -25
- package/dist/services/logger.d.ts +0 -5
- package/dist/services/logger.d.ts.map +0 -1
- package/dist/services/logger.js +0 -52
- package/dist/services/logger.js.map +0 -1
- package/dist/services/logger.service.d.ts +0 -5
- package/dist/services/logger.service.d.ts.map +0 -1
- package/dist/services/logger.service.js +0 -57
- package/dist/services/logger.service.js.map +0 -1
- package/dist/services/metadata-collector.d.ts +0 -2
- package/dist/services/metadata-collector.js +0 -80
- package/dist/services/parser.d.ts +0 -6
- package/dist/services/parser.d.ts.map +0 -1
- package/dist/services/parser.js +0 -278
- package/dist/services/parser.js.map +0 -1
- package/dist/services/parser.service.d.ts +0 -42
- package/dist/services/parser.service.d.ts.map +0 -1
- package/dist/services/parser.service.js +0 -209
- package/dist/services/parser.service.js.map +0 -1
- package/dist/services/session-manager.d.ts +0 -18
- package/dist/services/session-manager.d.ts.map +0 -1
- package/dist/services/session-manager.js +0 -73
- package/dist/services/session-manager.js.map +0 -1
- package/dist/services/telemetry.d.ts +0 -19
- package/dist/services/telemetry.js +0 -43
- package/dist/services/transform-worker-pool.d.ts +0 -11
- package/dist/services/transform-worker-pool.js +0 -244
- package/dist/services/transform-worker-types.d.ts +0 -32
- package/dist/services/transform-worker-types.js +0 -14
- package/dist/strategies/exponential-backoff-strategy.d.ts +0 -13
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +0 -1
- package/dist/strategies/exponential-backoff-strategy.js +0 -32
- package/dist/strategies/exponential-backoff-strategy.js.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +0 -4
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +0 -159
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -98
- package/dist/tools/handlers/fetch-links.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +0 -11
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +0 -97
- package/dist/tools/handlers/fetch-markdown.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +0 -31
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.js +0 -223
- package/dist/tools/handlers/fetch-single.shared.js.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +0 -10
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.js +0 -119
- package/dist/tools/handlers/fetch-url.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +0 -13
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.js +0 -153
- package/dist/tools/handlers/fetch-urls/processor.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.d.ts +0 -3
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.js +0 -79
- package/dist/tools/handlers/fetch-urls/response.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.js +0 -18
- package/dist/tools/handlers/fetch-urls/validation.js.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.js +0 -124
- package/dist/tools/handlers/fetch-urls.tool.js.map +0 -1
- package/dist/tools/index.d.ts +0 -3
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js +0 -37
- package/dist/tools/index.js.map +0 -1
- package/dist/tools/schemas.d.ts +0 -12
- package/dist/tools/schemas.d.ts.map +0 -1
- package/dist/tools/schemas.js +0 -21
- package/dist/tools/schemas.js.map +0 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -1
- package/dist/tools/utils/cache-vary.d.ts.map +0 -1
- package/dist/tools/utils/cache-vary.js +0 -12
- package/dist/tools/utils/cache-vary.js.map +0 -1
- package/dist/tools/utils/cached-markdown.d.ts +0 -5
- package/dist/tools/utils/cached-markdown.js +0 -46
- package/dist/tools/utils/common.d.ts +0 -5
- package/dist/tools/utils/common.d.ts.map +0 -1
- package/dist/tools/utils/common.js +0 -42
- package/dist/tools/utils/common.js.map +0 -1
- package/dist/tools/utils/content-shaping.d.ts +0 -4
- package/dist/tools/utils/content-shaping.js +0 -67
- package/dist/tools/utils/content-transform-async.d.ts +0 -6
- package/dist/tools/utils/content-transform-async.js +0 -33
- package/dist/tools/utils/content-transform-core.d.ts +0 -5
- package/dist/tools/utils/content-transform-core.js +0 -180
- package/dist/tools/utils/content-transform-workers.d.ts +0 -1
- package/dist/tools/utils/content-transform-workers.js +0 -1
- package/dist/tools/utils/content-transform.d.ts +0 -3
- package/dist/tools/utils/content-transform.d.ts.map +0 -1
- package/dist/tools/utils/content-transform.js +0 -41
- package/dist/tools/utils/content-transform.js.map +0 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +0 -1
- package/dist/tools/utils/fetch-pipeline.js +0 -98
- package/dist/tools/utils/fetch-pipeline.js.map +0 -1
- package/dist/tools/utils/frontmatter.d.ts +0 -3
- package/dist/tools/utils/frontmatter.js +0 -73
- package/dist/tools/utils/index.d.ts +0 -4
- package/dist/tools/utils/index.d.ts.map +0 -1
- package/dist/tools/utils/index.js +0 -3
- package/dist/tools/utils/index.js.map +0 -1
- package/dist/tools/utils/inline-content.d.ts +0 -10
- package/dist/tools/utils/inline-content.d.ts.map +0 -1
- package/dist/tools/utils/inline-content.js +0 -35
- package/dist/tools/utils/inline-content.js.map +0 -1
- package/dist/tools/utils/markdown-heuristics.d.ts +0 -1
- package/dist/tools/utils/markdown-heuristics.js +0 -19
- package/dist/tools/utils/markdown-signals.d.ts +0 -1
- package/dist/tools/utils/markdown-signals.js +0 -19
- package/dist/tools/utils/markdown-toc.d.ts +0 -3
- package/dist/tools/utils/markdown-toc.d.ts.map +0 -1
- package/dist/tools/utils/markdown-toc.js +0 -35
- package/dist/tools/utils/markdown-toc.js.map +0 -1
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +0 -3
- package/dist/tools/utils/raw-markdown-frontmatter.js +0 -73
- package/dist/tools/utils/raw-markdown.d.ts +0 -6
- package/dist/tools/utils/raw-markdown.js +0 -149
- package/dist/tools/utils/response-builder.d.ts +0 -3
- package/dist/tools/utils/response-builder.d.ts.map +0 -1
- package/dist/tools/utils/response-builder.js +0 -24
- package/dist/tools/utils/response-builder.js.map +0 -1
- package/dist/tools/utils/tool-response.d.ts +0 -9
- package/dist/tools/utils/tool-response.d.ts.map +0 -1
- package/dist/tools/utils/tool-response.js +0 -19
- package/dist/tools/utils/tool-response.js.map +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +0 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +0 -1
- package/dist/transformers/jsonl.transformer.js +0 -75
- package/dist/transformers/jsonl.transformer.js.map +0 -1
- package/dist/transformers/markdown/fenced-code-rule.d.ts +0 -2
- package/dist/transformers/markdown/fenced-code-rule.js +0 -38
- package/dist/transformers/markdown/frontmatter.d.ts +0 -2
- package/dist/transformers/markdown/frontmatter.js +0 -45
- package/dist/transformers/markdown/noise-rule.d.ts +0 -2
- package/dist/transformers/markdown/noise-rule.js +0 -80
- package/dist/transformers/markdown/turndown-instance.d.ts +0 -2
- package/dist/transformers/markdown/turndown-instance.js +0 -19
- package/dist/transformers/markdown.d.ts +0 -5
- package/dist/transformers/markdown.js +0 -314
- package/dist/transformers/markdown.transformer.d.ts +0 -2
- package/dist/transformers/markdown.transformer.d.ts.map +0 -1
- package/dist/transformers/markdown.transformer.js +0 -14
- package/dist/transformers/markdown.transformer.js.map +0 -1
- package/dist/types/content.types.d.ts +0 -63
- package/dist/types/content.types.d.ts.map +0 -1
- package/dist/types/content.types.js +0 -2
- package/dist/types/content.types.js.map +0 -1
- package/dist/types/index.d.ts +0 -2
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -2
- package/dist/types/index.js.map +0 -1
- package/dist/types/schemas.d.ts +0 -49
- package/dist/types/schemas.d.ts.map +0 -1
- package/dist/types/schemas.js +0 -5
- package/dist/types/schemas.js.map +0 -1
- package/dist/utils/cached-payload.d.ts +0 -7
- package/dist/utils/cached-payload.js +0 -36
- package/dist/utils/cancellation.d.ts +0 -1
- package/dist/utils/cancellation.js +0 -18
- package/dist/utils/code-language-bash.d.ts +0 -1
- package/dist/utils/code-language-bash.js +0 -48
- package/dist/utils/code-language-core.d.ts +0 -2
- package/dist/utils/code-language-core.js +0 -13
- package/dist/utils/code-language-detectors.d.ts +0 -5
- package/dist/utils/code-language-detectors.js +0 -142
- package/dist/utils/code-language-helpers.d.ts +0 -5
- package/dist/utils/code-language-helpers.js +0 -62
- package/dist/utils/code-language-parsing.d.ts +0 -5
- package/dist/utils/code-language-parsing.js +0 -62
- package/dist/utils/code-language.d.ts +0 -2
- package/dist/utils/code-language.d.ts.map +0 -1
- package/dist/utils/code-language.js +0 -260
- package/dist/utils/code-language.js.map +0 -1
- package/dist/utils/concurrency.d.ts +0 -3
- package/dist/utils/concurrency.d.ts.map +0 -1
- package/dist/utils/concurrency.js +0 -38
- package/dist/utils/concurrency.js.map +0 -1
- package/dist/utils/content-cleaner.d.ts +0 -5
- package/dist/utils/content-cleaner.d.ts.map +0 -1
- package/dist/utils/content-cleaner.js +0 -77
- package/dist/utils/content-cleaner.js.map +0 -1
- package/dist/utils/crypto.d.ts +0 -2
- package/dist/utils/crypto.d.ts.map +0 -1
- package/dist/utils/crypto.js +0 -32
- package/dist/utils/crypto.js.map +0 -1
- package/dist/utils/download-url.d.ts +0 -16
- package/dist/utils/download-url.d.ts.map +0 -1
- package/dist/utils/download-url.js +0 -30
- package/dist/utils/download-url.js.map +0 -1
- package/dist/utils/error-details.d.ts +0 -3
- package/dist/utils/error-details.js +0 -12
- package/dist/utils/error-utils.d.ts +0 -3
- package/dist/utils/error-utils.d.ts.map +0 -1
- package/dist/utils/error-utils.js +0 -12
- package/dist/utils/error-utils.js.map +0 -1
- package/dist/utils/filename-generator.d.ts +0 -1
- package/dist/utils/filename-generator.d.ts.map +0 -1
- package/dist/utils/filename-generator.js +0 -81
- package/dist/utils/filename-generator.js.map +0 -1
- package/dist/utils/guards.d.ts +0 -1
- package/dist/utils/guards.js +0 -3
- package/dist/utils/header-normalizer.d.ts +0 -5
- package/dist/utils/header-normalizer.d.ts.map +0 -1
- package/dist/utils/header-normalizer.js +0 -31
- package/dist/utils/header-normalizer.js.map +0 -1
- package/dist/utils/host-normalizer.d.ts +0 -1
- package/dist/utils/host-normalizer.js +0 -37
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.d.ts.map +0 -1
- package/dist/utils/html-truncator.js +0 -13
- package/dist/utils/html-truncator.js.map +0 -1
- package/dist/utils/ip-address.d.ts +0 -4
- package/dist/utils/ip-address.js +0 -6
- package/dist/utils/language-detector.d.ts +0 -2
- package/dist/utils/language-detector.d.ts.map +0 -1
- package/dist/utils/language-detector.js +0 -39
- package/dist/utils/language-detector.js.map +0 -1
- package/dist/utils/sanitizer.d.ts +0 -2
- package/dist/utils/sanitizer.d.ts.map +0 -1
- package/dist/utils/sanitizer.js +0 -20
- package/dist/utils/sanitizer.js.map +0 -1
- package/dist/utils/tool-error-handler.d.ts +0 -3
- package/dist/utils/tool-error-handler.d.ts.map +0 -1
- package/dist/utils/tool-error-handler.js +0 -31
- package/dist/utils/tool-error-handler.js.map +0 -1
- package/dist/utils/url-redactor.d.ts +0 -1
- package/dist/utils/url-redactor.js +0 -13
- package/dist/utils/url-sanitizer.d.ts +0 -2
- package/dist/utils/url-sanitizer.d.ts.map +0 -1
- package/dist/utils/url-sanitizer.js +0 -12
- package/dist/utils/url-sanitizer.js.map +0 -1
- package/dist/utils/url-transformer.d.ts +0 -7
- package/dist/utils/url-transformer.js +0 -147
- package/dist/utils/url-validator.d.ts +0 -6
- package/dist/utils/url-validator.d.ts.map +0 -1
- package/dist/utils/url-validator.js +0 -156
- package/dist/utils/url-validator.js.map +0 -1
- package/dist/workers/content-transform.worker.d.ts +0 -1
- package/dist/workers/content-transform.worker.js +0 -40
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
import { logDebug, logError } from '../../services/logger.js';
|
|
2
|
-
import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
|
|
3
|
-
import { appendHeaderVary } from '../utils/cache-vary.js';
|
|
4
|
-
import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
|
|
5
|
-
import { extractLinks, resolveFilterPattern, } from './fetch-links/link-extractor.js';
|
|
6
|
-
export const FETCH_LINKS_TOOL_NAME = 'fetch-links';
|
|
7
|
-
export const FETCH_LINKS_TOOL_DESCRIPTION = 'Extracts all hyperlinks from a webpage with anchor text and type classification. Supports filtering, image links, and link limits.';
|
|
8
|
-
function isRecord(value) {
|
|
9
|
-
return value !== null && typeof value === 'object';
|
|
10
|
-
}
|
|
11
|
-
function isToolResponseBase(value) {
|
|
12
|
-
if (!isRecord(value))
|
|
13
|
-
return false;
|
|
14
|
-
return Array.isArray(value.content);
|
|
15
|
-
}
|
|
16
|
-
function logFetchLinksStart(url, options, filterPattern) {
|
|
17
|
-
logDebug('Extracting links', {
|
|
18
|
-
url,
|
|
19
|
-
...options,
|
|
20
|
-
filterPattern,
|
|
21
|
-
});
|
|
22
|
-
}
|
|
23
|
-
async function fetchLinksPipeline(url, input, options) {
|
|
24
|
-
return executeFetchPipeline({
|
|
25
|
-
url,
|
|
26
|
-
cacheNamespace: 'links',
|
|
27
|
-
customHeaders: input.customHeaders,
|
|
28
|
-
retries: input.retries,
|
|
29
|
-
timeout: input.timeout,
|
|
30
|
-
cacheVary: appendHeaderVary({
|
|
31
|
-
includeInternal: options.includeInternal,
|
|
32
|
-
includeExternal: options.includeExternal,
|
|
33
|
-
includeImages: options.includeImages,
|
|
34
|
-
maxLinks: options.maxLinks,
|
|
35
|
-
filterPattern: input.filterPattern ?? null,
|
|
36
|
-
}, input.customHeaders),
|
|
37
|
-
transform: (html, normalizedUrl) => extractLinks(html, normalizedUrl, options),
|
|
38
|
-
});
|
|
39
|
-
}
|
|
40
|
-
function buildLinksResponse(result) {
|
|
41
|
-
const structuredContent = buildLinksStructuredContent(result);
|
|
42
|
-
return {
|
|
43
|
-
content: [
|
|
44
|
-
{
|
|
45
|
-
type: 'text',
|
|
46
|
-
text: JSON.stringify(structuredContent, null, 2),
|
|
47
|
-
},
|
|
48
|
-
],
|
|
49
|
-
structuredContent,
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
|
-
export async function fetchLinksToolHandler(input) {
|
|
53
|
-
try {
|
|
54
|
-
return await executeFetchLinks(input);
|
|
55
|
-
}
|
|
56
|
-
catch (error) {
|
|
57
|
-
logError('fetch-links tool error', error instanceof Error ? error : undefined);
|
|
58
|
-
return handleToolError(error, input.url, 'Failed to extract links');
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
async function executeFetchLinks(input) {
|
|
62
|
-
const { url } = input;
|
|
63
|
-
if (!url) {
|
|
64
|
-
return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
|
|
65
|
-
}
|
|
66
|
-
const filterPattern = resolveFilterPattern(input.filterPattern, url);
|
|
67
|
-
if (isToolResponseBase(filterPattern)) {
|
|
68
|
-
return filterPattern;
|
|
69
|
-
}
|
|
70
|
-
const options = buildExtractOptions(input, filterPattern);
|
|
71
|
-
logFetchLinksStart(url, options, input.filterPattern);
|
|
72
|
-
const result = await fetchLinksPipeline(url, input, options);
|
|
73
|
-
return buildLinksResponse(result);
|
|
74
|
-
}
|
|
75
|
-
function buildExtractOptions(input, filterPattern) {
|
|
76
|
-
return {
|
|
77
|
-
includeInternal: input.includeInternal ?? true,
|
|
78
|
-
includeExternal: input.includeExternal ?? true,
|
|
79
|
-
includeImages: input.includeImages ?? false,
|
|
80
|
-
maxLinks: input.maxLinks,
|
|
81
|
-
filterPattern,
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
function buildLinksStructuredContent(result) {
|
|
85
|
-
const structuredContent = {
|
|
86
|
-
url: result.url,
|
|
87
|
-
linkCount: result.data.linkCount,
|
|
88
|
-
links: result.data.links,
|
|
89
|
-
};
|
|
90
|
-
if (result.data.filtered > 0) {
|
|
91
|
-
structuredContent.filtered = result.data.filtered;
|
|
92
|
-
}
|
|
93
|
-
if (result.data.truncated) {
|
|
94
|
-
structuredContent.truncated = result.data.truncated;
|
|
95
|
-
}
|
|
96
|
-
return structuredContent;
|
|
97
|
-
}
|
|
98
|
-
//# sourceMappingURL=fetch-links.tool.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EACL,YAAY,EACZ,oBAAoB,GACrB,MAAM,iCAAiC,CAAC;AAEzC,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,oIAAoI,CAAC;AAEvI,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC;AACrD,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,kBAAkB,CACzB,GAAW,EACX,OAA4B,EAC5B,aAAiC;IAEjC,QAAQ,CAAC,kBAAkB,EAAE;QAC3B,GAAG;QACH,GAAG,OAAO;QACV,aAAa;KACd,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,GAAW,EACX,KAAsB,EACtB,OAA4B;IAE5B,OAAO,oBAAoB,CAAuB;QAChD,GAAG;QACH,cAAc,EAAE,OAAO;QACvB,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,gBAAgB,CACzB;YACE,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;SAC3C,EACD,KAAK,CAAC,aAAa,CACpB;QACD,SAAS,EAAE,CAAC,IAAI,EAAE,aAAa,EAAE,EAAE,CACjC,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,CAAC;KAC7C,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CACzB,MAA4C;IAE5C,MAAM,iBAAiB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IAC9D,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;aACjD;SACF;QACD,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,KAAsB;IAEtB,IAAI,CAAC;QACH,OAAO,MAAM,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,wBAAwB,EACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,KAAsB;IAEtB,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;IACtB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IACD,MAAM,aAAa,GAAG,oBAAoB,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;IACrE,IAAI,kBAAkB,CAAC,aAAa,CAAC,EAAE,CAAC;QACtC,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;IAE1D,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7D,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAsB,EACtB,aAAiC;IAEjC,OAAO;QACL,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC9C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC9C,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,KAAK;QAC3C,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,aAAa;KACd,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAClC,MAA4C;IAE5C,MAAM,iBAAiB,GAA4B;QACjD,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS;QAChC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;KACzB,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QAC7B,iBAAiB,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;IACpD,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC1B,iBAAiB,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;IACtD,CAAC;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC"}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import type { FetchMarkdownInput, ToolResponseBase } from '../../config/types/tools.js';
|
|
2
|
-
import { transformHtmlToMarkdown } from '../utils/content-transform.js';
|
|
3
|
-
import { performSharedFetch } from './fetch-single.shared.js';
|
|
4
|
-
export declare const FETCH_MARKDOWN_TOOL_NAME = "fetch-markdown";
|
|
5
|
-
export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter";
|
|
6
|
-
interface FetchMarkdownDeps {
|
|
7
|
-
readonly performSharedFetch?: typeof performSharedFetch;
|
|
8
|
-
readonly transformHtmlToMarkdown?: typeof transformHtmlToMarkdown;
|
|
9
|
-
}
|
|
10
|
-
export declare function createFetchMarkdownToolHandler(deps?: FetchMarkdownDeps): (input: FetchMarkdownInput) => Promise<ToolResponseBase>;
|
|
11
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,kBAAkB,EAElB,gBAAgB,EACjB,MAAM,6BAA6B,CAAC;AAuBrC,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,mHACsE,CAAC;AA0GnH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAU3B"}
|
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
import { config } from '../../config/index.js';
|
|
2
|
-
import { logDebug, logError } from '../../services/logger.js';
|
|
3
|
-
import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
|
|
4
|
-
import { transformHtmlToMarkdown } from '../utils/content-transform.js';
|
|
5
|
-
import { buildToolContentBlocks, performSharedFetch, } from './fetch-single.shared.js';
|
|
6
|
-
export const FETCH_MARKDOWN_TOOL_NAME = 'fetch-markdown';
|
|
7
|
-
export const FETCH_MARKDOWN_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format with optional frontmatter';
|
|
8
|
-
function isRecord(value) {
|
|
9
|
-
return value !== null && typeof value === 'object';
|
|
10
|
-
}
|
|
11
|
-
function deserializeMarkdownPipelineResult(cached) {
|
|
12
|
-
try {
|
|
13
|
-
const parsed = JSON.parse(cached);
|
|
14
|
-
if (!isRecord(parsed))
|
|
15
|
-
return undefined;
|
|
16
|
-
const { content, markdown, title } = parsed;
|
|
17
|
-
if (typeof content !== 'string')
|
|
18
|
-
return undefined;
|
|
19
|
-
if (typeof markdown !== 'string')
|
|
20
|
-
return undefined;
|
|
21
|
-
if (title !== undefined && typeof title !== 'string')
|
|
22
|
-
return undefined;
|
|
23
|
-
return {
|
|
24
|
-
content,
|
|
25
|
-
markdown,
|
|
26
|
-
title: typeof title === 'string' ? title : undefined,
|
|
27
|
-
truncated: false,
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
catch {
|
|
31
|
-
return undefined;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
function resolveMarkdownOptions(input) {
|
|
35
|
-
return {
|
|
36
|
-
includeMetadata: input.includeMetadata ?? config.extraction.includeMetadata,
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
function buildMarkdownStructuredContent(pipeline, inlineResult) {
|
|
40
|
-
return {
|
|
41
|
-
url: pipeline.url,
|
|
42
|
-
title: pipeline.data.title,
|
|
43
|
-
markdown: inlineResult.content,
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
function logFetchMarkdownStart(url, options) {
|
|
47
|
-
logDebug('Fetching markdown', { url, ...options });
|
|
48
|
-
}
|
|
49
|
-
function buildMarkdownTransform(options, transform) {
|
|
50
|
-
return async (html, url) => {
|
|
51
|
-
const markdownResult = await transform(html, url, options);
|
|
52
|
-
return { ...markdownResult, content: markdownResult.markdown };
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
async function fetchMarkdownPipeline(url, options, performSharedFetchImpl, transformImpl) {
|
|
56
|
-
const sharedOptions = {
|
|
57
|
-
url,
|
|
58
|
-
format: 'markdown',
|
|
59
|
-
includeMetadata: options.includeMetadata,
|
|
60
|
-
transform: buildMarkdownTransform(options, transformImpl),
|
|
61
|
-
deserialize: deserializeMarkdownPipelineResult,
|
|
62
|
-
};
|
|
63
|
-
return performSharedFetchImpl(sharedOptions);
|
|
64
|
-
}
|
|
65
|
-
function buildMarkdownResponse(pipeline, inlineResult) {
|
|
66
|
-
const structuredContent = buildMarkdownStructuredContent(pipeline, inlineResult);
|
|
67
|
-
return {
|
|
68
|
-
content: buildToolContentBlocks(structuredContent, pipeline.fromCache, inlineResult, 'Fetched markdown', pipeline.cacheKey, pipeline.data.content, 'markdown', pipeline.url, pipeline.data.title),
|
|
69
|
-
structuredContent,
|
|
70
|
-
};
|
|
71
|
-
}
|
|
72
|
-
export function createFetchMarkdownToolHandler(deps = {}) {
|
|
73
|
-
const performSharedFetchImpl = deps.performSharedFetch ?? performSharedFetch;
|
|
74
|
-
const transformImpl = deps.transformHtmlToMarkdown ?? transformHtmlToMarkdown;
|
|
75
|
-
return async (input) => {
|
|
76
|
-
try {
|
|
77
|
-
return await executeFetchMarkdown(input, performSharedFetchImpl, transformImpl);
|
|
78
|
-
}
|
|
79
|
-
catch (error) {
|
|
80
|
-
logError('fetch-markdown tool error', error instanceof Error ? error : undefined);
|
|
81
|
-
return handleToolError(error, input.url, 'Failed to fetch markdown');
|
|
82
|
-
}
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
|
-
async function executeFetchMarkdown(input, performSharedFetchImpl, transformImpl) {
|
|
86
|
-
const { url } = input;
|
|
87
|
-
if (!url) {
|
|
88
|
-
return createToolErrorResponse('URL is required', '');
|
|
89
|
-
}
|
|
90
|
-
const options = resolveMarkdownOptions(input);
|
|
91
|
-
logFetchMarkdownStart(url, options);
|
|
92
|
-
const { pipeline, inlineResult } = await fetchMarkdownPipeline(url, options, performSharedFetchImpl, transformImpl);
|
|
93
|
-
if (inlineResult.error) {
|
|
94
|
-
return createToolErrorResponse(inlineResult.error, url);
|
|
95
|
-
}
|
|
96
|
-
return buildMarkdownResponse(pipeline, inlineResult);
|
|
97
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAExE,OAAO,EACL,oCAAoC,EACpC,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EAEtB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAMlC,MAAM,CAAC,MAAM,wBAAwB,GAAG,gBAAgB,CAAC;AACzD,MAAM,CAAC,MAAM,+BAA+B,GAC1C,gHAAgH,CAAC;AAQnH,SAAS,sBAAsB,CAAC,KAAyB;IACvD,OAAO;QACL,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;QACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC9C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,8BAA8B,CACrC,QAAgD,EAChD,YAA0B,EAC1B,YAAqC;IAErC,MAAM,iBAAiB,GAA4B;QACjD,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;QAC1B,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,WAAW,EAAE,YAAY,CAAC,WAAW;QACrC,MAAM,EAAE,QAAQ,CAAC,SAAS;KAC3B,CAAC;IAEF,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;QACtD,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,oCAAoC,CAClC,iBAAiB,EACjB,YAAY,EACZ,UAAU,CACX,CAAC;IAEF,IAAI,YAAY,EAAE,CAAC;QACjB,iBAAiB,CAAC,IAAI,GAAG,YAAY,CAAC;IACxC,CAAC;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAW,EAAE,OAAyB;IACnE,QAAQ,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAyB;IACvD,OAAO,CAAC,IAAY,EAAE,GAAW,EAA0B,EAAE;QAC3D,MAAM,cAAc,GAAG,uBAAuB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,EAAE,GAAG,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,QAAQ,EAAE,CAAC;IACjE,CAAC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,GAAW,EACX,KAAyB,EACzB,OAAwB,EACxB,gBAAkC;IAKlC,OAAO,kBAAkB,CAAyB;QAChD,GAAG;QACH,MAAM,EAAE,UAAU;QAClB,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,sBAAsB,CAAC,gBAAgB,CAAC;KACpD,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAC5B,QAAgD,EAChD,YAA0B,EAC1B,YAAqC;IAErC,MAAM,iBAAiB,GAAG,8BAA8B,CACtD,QAAQ,EACR,YAAY,EACZ,YAAY,CACb,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,sBAAsB,CAC7B,iBAAiB,EACjB,QAAQ,CAAC,SAAS,EAClB,YAAY,EACZ,kBAAkB,EAClB,QAAQ,CAAC,QAAQ,EACjB,QAAQ,CAAC,IAAI,CAAC,OAAO,EACrB,UAAU,EACV,QAAQ,CAAC,GAAG,EACZ,QAAQ,CAAC,IAAI,CAAC,KAAK,CACpB;QACD,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,KAAyB;IAEzB,IAAI,CAAC;QACH,OAAO,MAAM,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,KAAyB;IAEzB,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;IACtB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,OAAO,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,gBAAgB,GAAqB,EAAE,GAAG,OAAO,EAAE,CAAC;IAE1D,qBAAqB,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IAE7C,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAC5D,GAAG,EACH,KAAK,EACL,OAAO,EACP,gBAAgB,CACjB,CAAC;IAEF,MAAM,WAAW,GAAG,sBAAsB,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IAC9D,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IAEpC,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW;QAC3C,CAAC,CAAC,mBAAmB,CAAC;YAClB,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,IAAI;YACnC,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;SAC3B,CAAC;QACJ,CAAC,CAAC,IAAI,CAAC;IAET,OAAO,qBAAqB,CAAC,QAAQ,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;AACrE,CAAC"}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import type { FetchPipelineOptions, PipelineResult, ToolContentBlock } from '../../config/types/runtime.js';
|
|
2
|
-
interface SharedFetchOptions<T extends {
|
|
3
|
-
content: string;
|
|
4
|
-
}> {
|
|
5
|
-
readonly url: string;
|
|
6
|
-
readonly transform: (html: string, normalizedUrl: string) => T | Promise<T>;
|
|
7
|
-
readonly serialize?: (result: T) => string;
|
|
8
|
-
readonly deserialize?: (cached: string) => T | undefined;
|
|
9
|
-
}
|
|
10
|
-
interface SharedFetchDeps {
|
|
11
|
-
readonly executeFetchPipeline?: typeof executeFetchPipeline;
|
|
12
|
-
}
|
|
13
|
-
export declare function performSharedFetch<T extends {
|
|
14
|
-
content: string;
|
|
15
|
-
}>(options: SharedFetchOptions<T>, deps?: SharedFetchDeps): Promise<{
|
|
16
|
-
pipeline: PipelineResult<T>;
|
|
17
|
-
inlineResult: ReturnType<typeof applyInlineContentLimit>;
|
|
18
|
-
}>;
|
|
19
|
-
export type InlineResult = ReturnType<typeof applyInlineContentLimit>;
|
|
20
|
-
export declare function buildToolContentBlocks(structuredContent: Record<string, unknown>, fromCache: boolean, inlineResult: InlineResult, resourceName: string, cacheKey?: string | null, fullContent?: string, url?: string, title?: string): ToolContentBlock[];
|
|
21
|
-
interface InlineContentResult {
|
|
22
|
-
content?: string;
|
|
23
|
-
contentSize: number;
|
|
24
|
-
resourceUri?: string;
|
|
25
|
-
resourceMimeType?: string;
|
|
26
|
-
error?: string;
|
|
27
|
-
truncated?: boolean;
|
|
28
|
-
}
|
|
29
|
-
declare function applyInlineContentLimit(content: string, cacheKey: string | null): InlineContentResult;
|
|
30
|
-
export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;
|
|
31
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-single.shared.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-single.shared.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,cAAc,EACd,gBAAgB,EACjB,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,6BAA6B,CAAC;AAOrC,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AAErE,KAAK,iBAAiB,GAAG,OAAO,GAAG,UAAU,CAAC;AAE9C,UAAU,kBAAkB,CAAC,CAAC,SAAS;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE;IACxD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC,CAAC;CAChE;AAED,wBAAsB,kBAAkB,CAAC,CAAC,SAAS;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,EACpE,OAAO,EAAE,kBAAkB,CAAC,CAAC,CAAC,GAC7B,OAAO,CAAC;IACT,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;IAC5B,YAAY,EAAE,UAAU,CAAC,OAAO,uBAAuB,CAAC,CAAC;CAC1D,CAAC,CA8BD;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAEtE,UAAU,eAAe;IACvB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,eAAe,GACvB,gBAAgB,GAAG,IAAI,CAMzB;AAED,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,MAAM,GACV,gBAAgB,GAAG,IAAI,CAGzB;AAED,wBAAgB,oCAAoC,CAClD,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC1C,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,MAAM,GACjB,IAAI,CAaN;AAyDD,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC1C,SAAS,EAAE,OAAO,EAClB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,MAAM,EACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,CAAC,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,iBAAiB,EAC1B,GAAG,CAAC,EAAE,MAAM,EACZ,KAAK,CAAC,EAAE,MAAM,GACb,gBAAgB,EAAE,CA+BpB"}
|
|
@@ -1,223 +0,0 @@
|
|
|
1
|
-
import { TRUNCATION_MARKER } from '../../config/formatting.js';
|
|
2
|
-
import { config } from '../../config/index.js';
|
|
3
|
-
import * as cache from '../../services/cache.js';
|
|
4
|
-
import { createCacheKey, toResourceUri } from '../../services/cache-keys.js';
|
|
5
|
-
import { fetchNormalizedUrl } from '../../services/fetcher.js';
|
|
6
|
-
import { logDebug } from '../../services/logger.js';
|
|
7
|
-
import { generateSafeFilename } from '../../utils/filename-generator.js';
|
|
8
|
-
import { isRecord } from '../../utils/guards.js';
|
|
9
|
-
import { transformToRawUrl } from '../../utils/url-transformer.js';
|
|
10
|
-
import { normalizeUrl } from '../../utils/url-validator.js';
|
|
11
|
-
function applyOptionalPipelineSerialization(pipelineOptions, options) {
|
|
12
|
-
if (options.serialize !== undefined) {
|
|
13
|
-
pipelineOptions.serialize = options.serialize;
|
|
14
|
-
}
|
|
15
|
-
if (options.deserialize !== undefined) {
|
|
16
|
-
pipelineOptions.deserialize = options.deserialize;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
export async function performSharedFetch(options, deps = {}) {
|
|
20
|
-
const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
|
|
21
|
-
const pipelineOptions = {
|
|
22
|
-
url: options.url,
|
|
23
|
-
cacheNamespace: 'markdown',
|
|
24
|
-
transform: options.transform,
|
|
25
|
-
};
|
|
26
|
-
applyOptionalPipelineSerialization(pipelineOptions, options);
|
|
27
|
-
const pipeline = await executePipeline(pipelineOptions);
|
|
28
|
-
const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null);
|
|
29
|
-
return { pipeline, inlineResult };
|
|
30
|
-
}
|
|
31
|
-
function serializeStructuredContent(structuredContent, fromCache) {
|
|
32
|
-
return JSON.stringify(structuredContent, fromCache ? undefined : null, fromCache ? undefined : 2);
|
|
33
|
-
}
|
|
34
|
-
function buildResourceLink(inlineResult, name) {
|
|
35
|
-
if (!inlineResult.resourceUri) {
|
|
36
|
-
return null;
|
|
37
|
-
}
|
|
38
|
-
const block = {
|
|
39
|
-
type: 'resource_link',
|
|
40
|
-
uri: inlineResult.resourceUri,
|
|
41
|
-
name,
|
|
42
|
-
description: `Content exceeds inline limit (${config.constants.maxInlineContentChars} chars)`,
|
|
43
|
-
};
|
|
44
|
-
if (inlineResult.resourceMimeType !== undefined) {
|
|
45
|
-
block.mimeType = inlineResult.resourceMimeType;
|
|
46
|
-
}
|
|
47
|
-
return block;
|
|
48
|
-
}
|
|
49
|
-
function buildEmbeddedResource(content, url, title) {
|
|
50
|
-
if (!content) {
|
|
51
|
-
return null;
|
|
52
|
-
}
|
|
53
|
-
const filename = generateSafeFilename(url, title, undefined, '.md');
|
|
54
|
-
const uri = `file:///${filename}`;
|
|
55
|
-
return {
|
|
56
|
-
type: 'resource',
|
|
57
|
-
resource: {
|
|
58
|
-
uri,
|
|
59
|
-
mimeType: 'text/markdown',
|
|
60
|
-
text: content,
|
|
61
|
-
},
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
function resolveContentToEmbed(inlineResult, fullContent, useInlineInHttpMode) {
|
|
65
|
-
if (useInlineInHttpMode) {
|
|
66
|
-
return inlineResult.content;
|
|
67
|
-
}
|
|
68
|
-
return fullContent ?? inlineResult.content;
|
|
69
|
-
}
|
|
70
|
-
function maybeAppendEmbeddedResource(blocks, contentToEmbed, url, title) {
|
|
71
|
-
if (typeof contentToEmbed !== 'string')
|
|
72
|
-
return;
|
|
73
|
-
if (!url)
|
|
74
|
-
return;
|
|
75
|
-
const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
|
|
76
|
-
if (embeddedResource) {
|
|
77
|
-
blocks.push(embeddedResource);
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
function maybeAppendResourceLink(blocks, inlineResult, resourceName) {
|
|
81
|
-
const resourceLink = buildResourceLink(inlineResult, resourceName);
|
|
82
|
-
if (resourceLink) {
|
|
83
|
-
blocks.push(resourceLink);
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
function buildTextBlock(structuredContent, fromCache) {
|
|
87
|
-
return {
|
|
88
|
-
type: 'text',
|
|
89
|
-
text: serializeStructuredContent(structuredContent, fromCache),
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
export function buildToolContentBlocks(structuredContent, fromCache, inlineResult, resourceName, cacheKey, fullContent, url, title) {
|
|
93
|
-
const blocks = [
|
|
94
|
-
buildTextBlock(structuredContent, fromCache),
|
|
95
|
-
];
|
|
96
|
-
const contentToEmbed = resolveContentToEmbed(inlineResult, fullContent, config.runtime.httpMode);
|
|
97
|
-
maybeAppendEmbeddedResource(blocks, contentToEmbed, url, title);
|
|
98
|
-
maybeAppendResourceLink(blocks, inlineResult, resourceName);
|
|
99
|
-
return blocks;
|
|
100
|
-
}
|
|
101
|
-
function applyInlineContentLimit(content, cacheKey) {
|
|
102
|
-
const contentSize = content.length;
|
|
103
|
-
const inlineLimit = config.constants.maxInlineContentChars;
|
|
104
|
-
if (contentSize <= inlineLimit) {
|
|
105
|
-
return { content, contentSize };
|
|
106
|
-
}
|
|
107
|
-
const resourceUri = resolveResourceUri(cacheKey);
|
|
108
|
-
if (!resourceUri) {
|
|
109
|
-
return buildTruncatedFallback(content, contentSize, inlineLimit);
|
|
110
|
-
}
|
|
111
|
-
return {
|
|
112
|
-
contentSize,
|
|
113
|
-
resourceUri,
|
|
114
|
-
resourceMimeType: 'text/markdown',
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
function resolveResourceUri(cacheKey) {
|
|
118
|
-
if (!config.cache.enabled || !cacheKey)
|
|
119
|
-
return null;
|
|
120
|
-
return toResourceUri(cacheKey);
|
|
121
|
-
}
|
|
122
|
-
function buildTruncatedFallback(content, contentSize, inlineLimit) {
|
|
123
|
-
const maxContentLength = Math.max(0, inlineLimit - TRUNCATION_MARKER.length);
|
|
124
|
-
const truncatedContent = content.length > inlineLimit
|
|
125
|
-
? `${content.substring(0, maxContentLength)}${TRUNCATION_MARKER}`
|
|
126
|
-
: content;
|
|
127
|
-
return {
|
|
128
|
-
content: truncatedContent,
|
|
129
|
-
contentSize,
|
|
130
|
-
truncated: true,
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normalizedUrl, }) {
|
|
134
|
-
if (!cacheKey)
|
|
135
|
-
return null;
|
|
136
|
-
const cached = cache.get(cacheKey);
|
|
137
|
-
if (!cached)
|
|
138
|
-
return null;
|
|
139
|
-
if (!deserialize) {
|
|
140
|
-
logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
|
|
141
|
-
return null;
|
|
142
|
-
}
|
|
143
|
-
const data = deserialize(cached.content);
|
|
144
|
-
if (data === undefined) {
|
|
145
|
-
logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
|
|
146
|
-
return null;
|
|
147
|
-
}
|
|
148
|
-
logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
|
|
149
|
-
return {
|
|
150
|
-
data,
|
|
151
|
-
fromCache: true,
|
|
152
|
-
url: normalizedUrl,
|
|
153
|
-
fetchedAt: cached.fetchedAt,
|
|
154
|
-
cacheKey,
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
function resolveNormalizedUrl(url) {
|
|
158
|
-
const { normalizedUrl: validatedUrl } = normalizeUrl(url);
|
|
159
|
-
const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
|
|
160
|
-
return { normalizedUrl, originalUrl: validatedUrl, transformed };
|
|
161
|
-
}
|
|
162
|
-
export async function executeFetchPipeline(options) {
|
|
163
|
-
const resolvedUrl = resolveNormalizedUrl(options.url);
|
|
164
|
-
logRawUrlTransformation(resolvedUrl);
|
|
165
|
-
const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
|
|
166
|
-
const cachedResult = attemptCacheRetrieval({
|
|
167
|
-
cacheKey,
|
|
168
|
-
deserialize: options.deserialize,
|
|
169
|
-
cacheNamespace: options.cacheNamespace,
|
|
170
|
-
normalizedUrl: resolvedUrl.normalizedUrl,
|
|
171
|
-
});
|
|
172
|
-
if (cachedResult)
|
|
173
|
-
return cachedResult;
|
|
174
|
-
logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
|
|
175
|
-
const fetchOptions = options.signal === undefined ? {} : { signal: options.signal };
|
|
176
|
-
const html = await fetchNormalizedUrl(resolvedUrl.normalizedUrl, fetchOptions);
|
|
177
|
-
const data = await options.transform(html, resolvedUrl.normalizedUrl);
|
|
178
|
-
if (cache.isEnabled()) {
|
|
179
|
-
persistCache({
|
|
180
|
-
cacheKey,
|
|
181
|
-
data,
|
|
182
|
-
serialize: options.serialize,
|
|
183
|
-
normalizedUrl: resolvedUrl.normalizedUrl,
|
|
184
|
-
});
|
|
185
|
-
}
|
|
186
|
-
return {
|
|
187
|
-
data,
|
|
188
|
-
fromCache: false,
|
|
189
|
-
url: resolvedUrl.normalizedUrl,
|
|
190
|
-
fetchedAt: new Date().toISOString(),
|
|
191
|
-
cacheKey,
|
|
192
|
-
};
|
|
193
|
-
}
|
|
194
|
-
function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
|
|
195
|
-
if (!cacheKey)
|
|
196
|
-
return;
|
|
197
|
-
const serializer = serialize ?? JSON.stringify;
|
|
198
|
-
const title = extractTitle(data);
|
|
199
|
-
const metadata = {
|
|
200
|
-
url: normalizedUrl,
|
|
201
|
-
...(title === undefined ? {} : { title }),
|
|
202
|
-
};
|
|
203
|
-
cache.set(cacheKey, serializer(data), metadata);
|
|
204
|
-
}
|
|
205
|
-
function extractTitle(value) {
|
|
206
|
-
if (!isRecord(value))
|
|
207
|
-
return undefined;
|
|
208
|
-
const { title } = value;
|
|
209
|
-
return typeof title === 'string' ? title : undefined;
|
|
210
|
-
}
|
|
211
|
-
function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
|
|
212
|
-
logDebug(`Cache miss due to ${reason}`, {
|
|
213
|
-
namespace: cacheNamespace,
|
|
214
|
-
url: normalizedUrl,
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
function logRawUrlTransformation(resolvedUrl) {
|
|
218
|
-
if (!resolvedUrl.transformed)
|
|
219
|
-
return;
|
|
220
|
-
logDebug('Using transformed raw content URL', {
|
|
221
|
-
original: resolvedUrl.originalUrl,
|
|
222
|
-
});
|
|
223
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-single.shared.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-single.shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAU/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AAgBrE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAA8B;IAK9B,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1E,MAAM,SAAS,GAAG,gBAAgB,CAChC;QACE,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;KAClE,EACD,OAAO,CAAC,aAAa,CACtB,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAI;QAC7C,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,cAAc;QACd,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS;QACT,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,uBAAuB,CAC1C,QAAQ,CAAC,IAAI,CAAC,OAAO,EACrB,QAAQ,CAAC,QAAQ,IAAI,IAAI,EACzB,OAAO,CAAC,MAAM,CACf,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;AACpC,CAAC;AAUD,MAAM,UAAU,mBAAmB,CACjC,OAAwB;IAExB,OAAO,qBAAqB,CAAC;QAC3B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,YAA0B,EAC1B,GAAW;IAEX,IAAI,CAAC,YAAY,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,uBAAuB,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,gBAAgB,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,oCAAoC,CAClD,iBAA0C,EAC1C,YAA0B,EAC1B,UAAkB;IAElB,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;QAC3B,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC7C,iBAAiB,CAAC,UAAU,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC;IACvD,CAAC;IAED,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC;QAC7B,iBAAiB,CAAC,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC;QACzD,iBAAiB,CAAC,gBAAgB,GAAG,YAAY,CAAC,gBAAgB,CAAC;IACrE,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CACjC,iBAA0C,EAC1C,SAAkB;IAElB,OAAO,IAAI,CAAC,SAAS,CACnB,iBAAiB,EACjB,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAC5B,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAC1B,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,YAA0B,EAC1B,IAAY;IAEZ,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,GAAG,EAAE,YAAY,CAAC,WAAW;QAC7B,IAAI;QACJ,QAAQ,EAAE,YAAY,CAAC,gBAAgB;QACvC,WAAW,EAAE,iCAAiC,MAAM,CAAC,SAAS,CAAC,qBAAqB,SAAS;KAC9F,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAe,EACf,QAAgB,EAChB,GAAW,EACX,KAAc;IAEd,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4CAA4C;IAC5C,MAAM,SAAS,GAAG,QAAQ,KAAK,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IAClE,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAExE,oEAAoE;IACpE,MAAM,GAAG,GAAG,WAAW,QAAQ,EAAE,CAAC;IAElC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,GAAG;YACH,QAAQ;YACR,IAAI,EAAE,OAAO;SACd;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,iBAA0C,EAC1C,SAAkB,EAClB,YAA0B,EAC1B,YAAoB,EACpB,QAAwB,EACxB,WAAoB,EACpB,MAA0B,EAC1B,GAAY,EACZ,KAAc;IAEd,MAAM,SAAS,GAAqB;QAClC,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,0BAA0B,CAAC,iBAAiB,EAAE,SAAS,CAAC;KAC/D,CAAC;IAEF,MAAM,MAAM,GAAuB,CAAC,SAAS,CAAC,CAAC;IAE/C,0EAA0E;IAC1E,MAAM,QAAQ,GACZ,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAChE,MAAM,cAAc,GAAG,WAAW,IAAI,YAAY,CAAC,OAAO,CAAC;IAC3D,IAAI,cAAc,IAAI,GAAG,EAAE,CAAC;QAC1B,MAAM,gBAAgB,GAAG,qBAAqB,CAC5C,cAAc,EACd,QAAQ,EACR,GAAG,EACH,KAAK,CACN,CAAC;QACF,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IACnE,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { MarkdownTransformResult } from '../../config/types/content.js';
|
|
2
|
-
import type { FetchUrlInput, ToolResponseBase } from '../../config/types/tools.js';
|
|
3
|
-
export declare const FETCH_URL_TOOL_NAME = "fetch-url";
|
|
4
|
-
export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format";
|
|
5
|
-
type MarkdownPipelineResult = MarkdownTransformResult & {
|
|
6
|
-
readonly content: string;
|
|
7
|
-
};
|
|
8
|
-
export declare function parseCachedMarkdownResult(cached: string): MarkdownPipelineResult | undefined;
|
|
9
|
-
export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<ToolResponseBase>;
|
|
10
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EACjB,MAAM,6BAA6B,CAAC;AAqBrC,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,6JACqH,CAAC;AAoH7J,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,GACnB,OAAO,CAAC,gBAAgB,CAAC,CAU3B"}
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
import { logDebug, logError } from '../../services/logger.js';
|
|
2
|
-
import { isRecord } from '../../utils/guards.js';
|
|
3
|
-
import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
|
|
4
|
-
import { transformHtmlToMarkdown } from '../utils/content-transform.js';
|
|
5
|
-
import { buildToolContentBlocks, performSharedFetch, } from './fetch-single.shared.js';
|
|
6
|
-
export const FETCH_URL_TOOL_NAME = 'fetch-url';
|
|
7
|
-
export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format';
|
|
8
|
-
function parseJsonRecord(input) {
|
|
9
|
-
try {
|
|
10
|
-
const parsed = JSON.parse(input);
|
|
11
|
-
return isRecord(parsed) ? parsed : undefined;
|
|
12
|
-
}
|
|
13
|
-
catch {
|
|
14
|
-
return undefined;
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
function resolveMarkdownContent(parsed) {
|
|
18
|
-
const { markdown } = parsed;
|
|
19
|
-
if (typeof markdown === 'string')
|
|
20
|
-
return markdown;
|
|
21
|
-
const { content } = parsed;
|
|
22
|
-
if (typeof content === 'string')
|
|
23
|
-
return content;
|
|
24
|
-
return undefined;
|
|
25
|
-
}
|
|
26
|
-
function resolveOptionalTitle(parsed) {
|
|
27
|
-
const { title } = parsed;
|
|
28
|
-
if (title === undefined)
|
|
29
|
-
return undefined;
|
|
30
|
-
return typeof title === 'string' ? title : undefined;
|
|
31
|
-
}
|
|
32
|
-
function resolveTruncatedFlag(parsed) {
|
|
33
|
-
const { truncated } = parsed;
|
|
34
|
-
return typeof truncated === 'boolean' ? truncated : false;
|
|
35
|
-
}
|
|
36
|
-
export function parseCachedMarkdownResult(cached) {
|
|
37
|
-
const parsed = parseJsonRecord(cached);
|
|
38
|
-
if (!parsed)
|
|
39
|
-
return undefined;
|
|
40
|
-
const resolvedContent = resolveMarkdownContent(parsed);
|
|
41
|
-
if (resolvedContent === undefined)
|
|
42
|
-
return undefined;
|
|
43
|
-
const title = resolveOptionalTitle(parsed);
|
|
44
|
-
if (parsed.title !== undefined && title === undefined)
|
|
45
|
-
return undefined;
|
|
46
|
-
return {
|
|
47
|
-
content: resolvedContent,
|
|
48
|
-
markdown: resolvedContent,
|
|
49
|
-
title,
|
|
50
|
-
truncated: resolveTruncatedFlag(parsed),
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
function deserializeMarkdownResult(cached) {
|
|
54
|
-
return parseCachedMarkdownResult(cached);
|
|
55
|
-
}
|
|
56
|
-
function buildMarkdownTransform() {
|
|
57
|
-
return async (html, url) => {
|
|
58
|
-
const result = await transformHtmlToMarkdown(html, url, {
|
|
59
|
-
includeMetadata: true,
|
|
60
|
-
});
|
|
61
|
-
return { ...result, content: result.markdown };
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
function serializeMarkdownResult(result) {
|
|
65
|
-
return JSON.stringify({
|
|
66
|
-
markdown: result.markdown,
|
|
67
|
-
title: result.title,
|
|
68
|
-
truncated: result.truncated,
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
function buildStructuredContent(pipeline, inlineResult, inputUrl) {
|
|
72
|
-
return {
|
|
73
|
-
url: pipeline.url,
|
|
74
|
-
resolvedUrl: pipeline.url,
|
|
75
|
-
inputUrl,
|
|
76
|
-
title: pipeline.data.title,
|
|
77
|
-
markdown: inlineResult.content,
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
function buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult) {
|
|
81
|
-
return buildToolContentBlocks(structuredContent, pipeline.fromCache, inlineResult, 'Fetched markdown', pipeline.cacheKey, pipeline.data.content, pipeline.url, pipeline.data.title);
|
|
82
|
-
}
|
|
83
|
-
function logFetchStart(url) {
|
|
84
|
-
logDebug('Fetching URL', { url });
|
|
85
|
-
}
|
|
86
|
-
async function fetchPipeline(url) {
|
|
87
|
-
return performSharedFetch({
|
|
88
|
-
url,
|
|
89
|
-
transform: buildMarkdownTransform(),
|
|
90
|
-
serialize: serializeMarkdownResult,
|
|
91
|
-
deserialize: deserializeMarkdownResult,
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
function buildResponse(pipeline, inlineResult, inputUrl) {
|
|
95
|
-
const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
|
|
96
|
-
const content = buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult);
|
|
97
|
-
return {
|
|
98
|
-
content,
|
|
99
|
-
structuredContent,
|
|
100
|
-
};
|
|
101
|
-
}
|
|
102
|
-
export async function fetchUrlToolHandler(input) {
|
|
103
|
-
return executeFetch(input).catch((error) => {
|
|
104
|
-
logError('fetch-url tool error', error instanceof Error ? error : undefined);
|
|
105
|
-
return handleToolError(error, input.url, 'Failed to fetch URL');
|
|
106
|
-
});
|
|
107
|
-
}
|
|
108
|
-
async function executeFetch(input) {
|
|
109
|
-
const { url } = input;
|
|
110
|
-
if (!url) {
|
|
111
|
-
return createToolErrorResponse('URL is required', '');
|
|
112
|
-
}
|
|
113
|
-
logFetchStart(url);
|
|
114
|
-
const { pipeline, inlineResult } = await fetchPipeline(url);
|
|
115
|
-
if (inlineResult.error) {
|
|
116
|
-
return createToolErrorResponse(inlineResult.error, url);
|
|
117
|
-
}
|
|
118
|
-
return buildResponse(pipeline, inlineResult, url);
|
|
119
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,oBAAoB,EACpB,iCAAiC,GAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EACL,oCAAoC,EACpC,sBAAsB,EACtB,sBAAsB,EAEtB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAElC,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GACrC,0JAA0J,CAAC;AAW7J,SAAS,sBAAsB,CAAC,KAAoB;IAClD,OAAO;QACL,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;QACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC9C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,OAAO;KAChC,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAwB;IACtD,OAAO,CAAC,IAAY,EAAE,GAAW,EAAE,EAAE,CACnC,OAAO,CAAC,MAAM,KAAK,UAAU;QAC3B,CAAC,CAAC,iCAAiC,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;QACvD,CAAC,CAAC,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,8BAA8B,CACrC,MAAc,EACd,QAA8C,EAC9C,YAA0B;IAE1B,MAAM,iBAAiB,GAA4B;QACjD,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;QAC1B,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,aAAa;QAC1C,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,MAAM;QACN,WAAW,EAAE,YAAY,CAAC,WAAW;QACrC,MAAM,EAAE,QAAQ,CAAC,SAAS;KAC3B,CAAC;IAEF,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;QAC3B,iBAAiB,CAAC,SAAS,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,oCAAoC,CAClC,iBAAiB,EACjB,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,OAAwB;IAC7D,QAAQ,CAAC,cAAc,EAAE;QACvB,GAAG;QACH,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,KAAoB,EACpB,OAAwB;IAKxB,OAAO,kBAAkB,CAAuB;QAC9C,GAAG;QACH,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,sBAAsB,CAAC,OAAO,CAAC;KAC3C,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAC5B,QAA8C,EAC9C,YAA0B,EAC1B,MAAc;IAEd,MAAM,iBAAiB,GAAG,8BAA8B,CACtD,MAAM,EACN,QAAQ,EACR,YAAY,CACb,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,sBAAsB,CAC7B,iBAAiB,EACjB,QAAQ,CAAC,SAAS,EAClB,YAAY,EACZ,iBAAiB,EACjB,QAAQ,CAAC,QAAQ,EACjB,QAAQ,CAAC,IAAI,CAAC,OAAO,EACrB,MAAM,EACN,QAAQ,CAAC,GAAG,EACZ,QAAQ,CAAC,IAAI,CAAC,KAAK,CACpB;QACD,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAoB;IAEpB,IAAI,CAAC;QACH,OAAO,MAAM,eAAe,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,sBAAsB,EACtB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,KAAoB;IAEpB,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;IACtB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,OAAO,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAC9C,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAE/B,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,gBAAgB,CACvD,GAAG,EACH,KAAK,EACL,OAAO,CACR,CAAC;IAEF,MAAM,WAAW,GAAG,sBAAsB,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IAC9D,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IACpC,OAAO,qBAAqB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { BatchUrlResult, FetchOptions, FetchUrlsInput } from '../../../config/types.js';
|
|
2
|
-
type Format = NonNullable<FetchUrlsInput['format']>;
|
|
3
|
-
interface SingleUrlProcessOptions {
|
|
4
|
-
readonly extractMainContent: boolean;
|
|
5
|
-
readonly includeMetadata: boolean;
|
|
6
|
-
readonly maxContentLength?: number;
|
|
7
|
-
readonly format: Format;
|
|
8
|
-
readonly requestOptions?: FetchOptions;
|
|
9
|
-
readonly maxRetries?: number;
|
|
10
|
-
}
|
|
11
|
-
export declare function processSingleUrl(url: string, options: SingleUrlProcessOptions): Promise<BatchUrlResult>;
|
|
12
|
-
export type { SingleUrlProcessOptions };
|
|
13
|
-
//# sourceMappingURL=processor.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../../../../src/tools/handlers/fetch-urls/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EAEf,MAAM,0BAA0B,CAAC;AAalC,KAAK,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;AAEpD,UAAU,uBAAuB;IAC/B,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC;IACvC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AA0ED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,uBAAuB,GAC/B,OAAO,CAAC,cAAc,CAAC,CAMzB;AAuGD,YAAY,EAAE,uBAAuB,EAAE,CAAC"}
|