@j0hanz/superfetch 2.1.4 → 2.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts +1 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +2 -1
- package/dist/cache.js.map +1 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -0
- package/dist/crypto.d.ts +1 -0
- package/dist/crypto.d.ts.map +1 -0
- package/dist/crypto.js +1 -0
- package/dist/crypto.js.map +1 -0
- package/dist/errors.d.ts +1 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +1 -0
- package/dist/errors.js.map +1 -0
- package/dist/fetch.d.ts +1 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +2 -1
- package/dist/fetch.js.map +1 -0
- package/dist/http.d.ts +1 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +2 -1
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts +1 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +1 -0
- package/dist/mcp.js.map +1 -0
- package/dist/observability.d.ts +1 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/observability.js +1 -0
- package/dist/observability.js.map +1 -0
- package/dist/tools.d.ts +1 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +2 -1
- package/dist/tools.js.map +1 -0
- package/dist/transform.d.ts +1 -0
- package/dist/transform.d.ts.map +1 -0
- package/dist/transform.js +2 -5
- package/dist/transform.js.map +1 -0
- package/dist/{utils.d.ts → type-guards.d.ts} +1 -0
- package/dist/type-guards.d.ts.map +1 -0
- package/dist/{utils.js → type-guards.js} +1 -0
- package/dist/type-guards.js.map +1 -0
- package/dist/workers/transform-worker.d.ts +1 -0
- package/dist/workers/transform-worker.d.ts.map +1 -0
- package/dist/workers/transform-worker.js +2 -1
- package/dist/workers/transform-worker.js.map +1 -0
- package/package.json +7 -4
- package/dist/config/auth-config.d.ts +0 -16
- package/dist/config/auth-config.js +0 -53
- package/dist/config/constants.d.ts +0 -17
- package/dist/config/constants.d.ts.map +0 -1
- package/dist/config/constants.js +0 -22
- package/dist/config/constants.js.map +0 -1
- package/dist/config/env-parsers.d.ts +0 -8
- package/dist/config/env-parsers.js +0 -96
- package/dist/config/formatting.d.ts +0 -7
- package/dist/config/formatting.d.ts.map +0 -1
- package/dist/config/formatting.js +0 -9
- package/dist/config/formatting.js.map +0 -1
- package/dist/config/index.d.ts +0 -61
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -107
- package/dist/config/index.js.map +0 -1
- package/dist/config/types/content.d.ts +0 -41
- package/dist/config/types/content.d.ts.map +0 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/content.js.map +0 -1
- package/dist/config/types/runtime.d.ts +0 -72
- package/dist/config/types/runtime.d.ts.map +0 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/runtime.js.map +0 -1
- package/dist/config/types/tools.d.ts +0 -28
- package/dist/config/types/tools.d.ts.map +0 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/config/types/tools.js.map +0 -1
- package/dist/config/types.d.ts +0 -4
- package/dist/config/types.d.ts.map +0 -1
- package/dist/config/types.js +0 -2
- package/dist/config/types.js.map +0 -1
- package/dist/errors/app-error.d.ts +0 -7
- package/dist/errors/app-error.d.ts.map +0 -1
- package/dist/errors/app-error.js +0 -16
- package/dist/errors/app-error.js.map +0 -1
- package/dist/errors/index.d.ts +0 -2
- package/dist/errors/index.d.ts.map +0 -1
- package/dist/errors/index.js +0 -2
- package/dist/errors/index.js.map +0 -1
- package/dist/http/accept-policy.d.ts +0 -3
- package/dist/http/accept-policy.js +0 -45
- package/dist/http/async-handler.d.ts +0 -2
- package/dist/http/async-handler.js +0 -5
- package/dist/http/auth-introspection.d.ts +0 -2
- package/dist/http/auth-introspection.js +0 -141
- package/dist/http/auth-static.d.ts +0 -2
- package/dist/http/auth-static.js +0 -23
- package/dist/http/auth.d.ts +0 -3
- package/dist/http/auth.d.ts.map +0 -1
- package/dist/http/auth.js +0 -269
- package/dist/http/auth.js.map +0 -1
- package/dist/http/base-middleware.d.ts +0 -7
- package/dist/http/base-middleware.js +0 -143
- package/dist/http/cors.d.ts +0 -2
- package/dist/http/cors.d.ts.map +0 -1
- package/dist/http/cors.js +0 -9
- package/dist/http/cors.js.map +0 -1
- package/dist/http/download-routes.d.ts +0 -2
- package/dist/http/download-routes.d.ts.map +0 -1
- package/dist/http/download-routes.js +0 -104
- package/dist/http/download-routes.js.map +0 -1
- package/dist/http/error-handler.d.ts +0 -2
- package/dist/http/error-handler.js +0 -55
- package/dist/http/host-allowlist.d.ts +0 -3
- package/dist/http/host-allowlist.js +0 -117
- package/dist/http/jsonrpc-http.d.ts +0 -2
- package/dist/http/jsonrpc-http.js +0 -10
- package/dist/http/mcp-routes.d.ts +0 -9
- package/dist/http/mcp-routes.d.ts.map +0 -1
- package/dist/http/mcp-routes.js +0 -204
- package/dist/http/mcp-routes.js.map +0 -1
- package/dist/http/mcp-session-eviction.d.ts +0 -3
- package/dist/http/mcp-session-eviction.js +0 -24
- package/dist/http/mcp-session-helpers.d.ts +0 -12
- package/dist/http/mcp-session-helpers.d.ts.map +0 -1
- package/dist/http/mcp-session-helpers.js +0 -64
- package/dist/http/mcp-session-helpers.js.map +0 -1
- package/dist/http/mcp-session-init.d.ts +0 -7
- package/dist/http/mcp-session-init.js +0 -94
- package/dist/http/mcp-session-slots.d.ts +0 -17
- package/dist/http/mcp-session-slots.js +0 -55
- package/dist/http/mcp-session-transport-init.d.ts +0 -7
- package/dist/http/mcp-session-transport-init.js +0 -41
- package/dist/http/mcp-session-transport.d.ts +0 -7
- package/dist/http/mcp-session-transport.js +0 -57
- package/dist/http/mcp-session-types.d.ts +0 -5
- package/dist/http/mcp-session-types.js +0 -1
- package/dist/http/mcp-session.d.ts +0 -10
- package/dist/http/mcp-session.d.ts.map +0 -1
- package/dist/http/mcp-session.js +0 -23
- package/dist/http/mcp-session.js.map +0 -1
- package/dist/http/mcp-sessions.d.ts +0 -41
- package/dist/http/mcp-sessions.js +0 -392
- package/dist/http/mcp-validation.d.ts +0 -3
- package/dist/http/mcp-validation.d.ts.map +0 -1
- package/dist/http/mcp-validation.js +0 -14
- package/dist/http/mcp-validation.js.map +0 -1
- package/dist/http/protocol-policy.d.ts +0 -2
- package/dist/http/protocol-policy.js +0 -31
- package/dist/http/rate-limit.d.ts +0 -12
- package/dist/http/rate-limit.d.ts.map +0 -1
- package/dist/http/rate-limit.js +0 -93
- package/dist/http/rate-limit.js.map +0 -1
- package/dist/http/server-config.d.ts +0 -1
- package/dist/http/server-config.js +0 -40
- package/dist/http/server-middleware.d.ts +0 -7
- package/dist/http/server-middleware.d.ts.map +0 -1
- package/dist/http/server-middleware.js +0 -52
- package/dist/http/server-middleware.js.map +0 -1
- package/dist/http/server-shutdown.d.ts +0 -4
- package/dist/http/server-shutdown.js +0 -43
- package/dist/http/server-tuning.d.ts +0 -9
- package/dist/http/server-tuning.js +0 -45
- package/dist/http/server.d.ts +0 -3
- package/dist/http/server.d.ts.map +0 -1
- package/dist/http/server.js +0 -291
- package/dist/http/server.js.map +0 -1
- package/dist/http/session-cleanup.d.ts +0 -2
- package/dist/http/session-cleanup.d.ts.map +0 -1
- package/dist/http/session-cleanup.js +0 -40
- package/dist/http/session-cleanup.js.map +0 -1
- package/dist/http/sessions.d.ts +0 -14
- package/dist/http/sessions.d.ts.map +0 -1
- package/dist/http/sessions.js +0 -63
- package/dist/http/sessions.js.map +0 -1
- package/dist/middleware/error-handler.d.ts +0 -2
- package/dist/middleware/error-handler.d.ts.map +0 -1
- package/dist/middleware/error-handler.js +0 -56
- package/dist/middleware/error-handler.js.map +0 -1
- package/dist/middleware/rate-limiter.d.ts +0 -16
- package/dist/middleware/rate-limiter.d.ts.map +0 -1
- package/dist/middleware/rate-limiter.js +0 -111
- package/dist/middleware/rate-limiter.js.map +0 -1
- package/dist/parsers/base-html-element-parser.d.ts +0 -43
- package/dist/parsers/base-html-element-parser.d.ts.map +0 -1
- package/dist/parsers/base-html-element-parser.js +0 -59
- package/dist/parsers/base-html-element-parser.js.map +0 -1
- package/dist/parsers/heading-element-parser.d.ts +0 -14
- package/dist/parsers/heading-element-parser.d.ts.map +0 -1
- package/dist/parsers/heading-element-parser.js +0 -26
- package/dist/parsers/heading-element-parser.js.map +0 -1
- package/dist/parsers/image-element-parser.d.ts +0 -16
- package/dist/parsers/image-element-parser.d.ts.map +0 -1
- package/dist/parsers/image-element-parser.js +0 -33
- package/dist/parsers/image-element-parser.js.map +0 -1
- package/dist/parsers/link-element-parser.d.ts +0 -15
- package/dist/parsers/link-element-parser.d.ts.map +0 -1
- package/dist/parsers/link-element-parser.js +0 -28
- package/dist/parsers/link-element-parser.js.map +0 -1
- package/dist/parsers/open-graph-parser.d.ts +0 -17
- package/dist/parsers/open-graph-parser.d.ts.map +0 -1
- package/dist/parsers/open-graph-parser.js +0 -41
- package/dist/parsers/open-graph-parser.js.map +0 -1
- package/dist/parsers/schema-org-parser.d.ts +0 -17
- package/dist/parsers/schema-org-parser.d.ts.map +0 -1
- package/dist/parsers/schema-org-parser.js +0 -32
- package/dist/parsers/schema-org-parser.js.map +0 -1
- package/dist/parsers/standard-meta-parser.d.ts +0 -18
- package/dist/parsers/standard-meta-parser.d.ts.map +0 -1
- package/dist/parsers/standard-meta-parser.js +0 -32
- package/dist/parsers/standard-meta-parser.js.map +0 -1
- package/dist/parsers/twitter-card-parser.d.ts +0 -17
- package/dist/parsers/twitter-card-parser.d.ts.map +0 -1
- package/dist/parsers/twitter-card-parser.js +0 -41
- package/dist/parsers/twitter-card-parser.js.map +0 -1
- package/dist/prompts/index.d.ts +0 -3
- package/dist/prompts/index.d.ts.map +0 -1
- package/dist/prompts/index.js +0 -73
- package/dist/prompts/index.js.map +0 -1
- package/dist/resources/cached-content-params.d.ts +0 -5
- package/dist/resources/cached-content-params.js +0 -36
- package/dist/resources/cached-content.d.ts +0 -2
- package/dist/resources/cached-content.d.ts.map +0 -1
- package/dist/resources/cached-content.js +0 -132
- package/dist/resources/cached-content.js.map +0 -1
- package/dist/resources/index.d.ts +0 -2
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js +0 -4
- package/dist/resources/index.js.map +0 -1
- package/dist/server.d.ts +0 -3
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js +0 -94
- package/dist/server.js.map +0 -1
- package/dist/services/cache-events.d.ts +0 -8
- package/dist/services/cache-events.js +0 -19
- package/dist/services/cache-keys.d.ts +0 -7
- package/dist/services/cache-keys.js +0 -57
- package/dist/services/cache.d.ts +0 -17
- package/dist/services/cache.d.ts.map +0 -1
- package/dist/services/cache.js +0 -145
- package/dist/services/cache.js.map +0 -1
- package/dist/services/cache.service.d.ts +0 -52
- package/dist/services/cache.service.d.ts.map +0 -1
- package/dist/services/cache.service.js +0 -113
- package/dist/services/cache.service.js.map +0 -1
- package/dist/services/card-extractor.d.ts +0 -6
- package/dist/services/card-extractor.d.ts.map +0 -1
- package/dist/services/card-extractor.js +0 -199
- package/dist/services/card-extractor.js.map +0 -1
- package/dist/services/context.d.ts +0 -10
- package/dist/services/context.d.ts.map +0 -1
- package/dist/services/context.js +0 -14
- package/dist/services/context.js.map +0 -1
- package/dist/services/extractor.d.ts +0 -5
- package/dist/services/extractor.d.ts.map +0 -1
- package/dist/services/extractor.js +0 -142
- package/dist/services/extractor.js.map +0 -1
- package/dist/services/extractor.service.d.ts +0 -18
- package/dist/services/extractor.service.d.ts.map +0 -1
- package/dist/services/extractor.service.js +0 -75
- package/dist/services/extractor.service.js.map +0 -1
- package/dist/services/fetcher/agents.d.ts +0 -3
- package/dist/services/fetcher/agents.d.ts.map +0 -1
- package/dist/services/fetcher/agents.js +0 -100
- package/dist/services/fetcher/agents.js.map +0 -1
- package/dist/services/fetcher/dns-selection.d.ts +0 -2
- package/dist/services/fetcher/dns-selection.js +0 -72
- package/dist/services/fetcher/errors.d.ts +0 -4
- package/dist/services/fetcher/errors.d.ts.map +0 -1
- package/dist/services/fetcher/errors.js +0 -70
- package/dist/services/fetcher/errors.js.map +0 -1
- package/dist/services/fetcher/headers.d.ts +0 -2
- package/dist/services/fetcher/headers.d.ts.map +0 -1
- package/dist/services/fetcher/headers.js +0 -6
- package/dist/services/fetcher/headers.js.map +0 -1
- package/dist/services/fetcher/interceptors.d.ts +0 -10
- package/dist/services/fetcher/interceptors.d.ts.map +0 -1
- package/dist/services/fetcher/interceptors.js +0 -108
- package/dist/services/fetcher/interceptors.js.map +0 -1
- package/dist/services/fetcher/redirects.d.ts +0 -4
- package/dist/services/fetcher/redirects.d.ts.map +0 -1
- package/dist/services/fetcher/redirects.js +0 -78
- package/dist/services/fetcher/redirects.js.map +0 -1
- package/dist/services/fetcher/response.d.ts +0 -4
- package/dist/services/fetcher/response.d.ts.map +0 -1
- package/dist/services/fetcher/response.js +0 -104
- package/dist/services/fetcher/response.js.map +0 -1
- package/dist/services/fetcher/retry-policy.d.ts +0 -1
- package/dist/services/fetcher/retry-policy.d.ts.map +0 -1
- package/dist/services/fetcher/retry-policy.js +0 -131
- package/dist/services/fetcher/retry-policy.js.map +0 -1
- package/dist/services/fetcher.d.ts +0 -25
- package/dist/services/fetcher.d.ts.map +0 -1
- package/dist/services/fetcher.js +0 -607
- package/dist/services/fetcher.js.map +0 -1
- package/dist/services/fetcher.service.d.ts +0 -18
- package/dist/services/fetcher.service.d.ts.map +0 -1
- package/dist/services/fetcher.service.js +0 -122
- package/dist/services/fetcher.service.js.map +0 -1
- package/dist/services/fifo-queue.d.ts +0 -8
- package/dist/services/fifo-queue.js +0 -25
- package/dist/services/logger.d.ts +0 -5
- package/dist/services/logger.d.ts.map +0 -1
- package/dist/services/logger.js +0 -52
- package/dist/services/logger.js.map +0 -1
- package/dist/services/logger.service.d.ts +0 -5
- package/dist/services/logger.service.d.ts.map +0 -1
- package/dist/services/logger.service.js +0 -57
- package/dist/services/logger.service.js.map +0 -1
- package/dist/services/metadata-collector.d.ts +0 -2
- package/dist/services/metadata-collector.js +0 -80
- package/dist/services/parser.d.ts +0 -6
- package/dist/services/parser.d.ts.map +0 -1
- package/dist/services/parser.js +0 -278
- package/dist/services/parser.js.map +0 -1
- package/dist/services/parser.service.d.ts +0 -42
- package/dist/services/parser.service.d.ts.map +0 -1
- package/dist/services/parser.service.js +0 -209
- package/dist/services/parser.service.js.map +0 -1
- package/dist/services/session-manager.d.ts +0 -18
- package/dist/services/session-manager.d.ts.map +0 -1
- package/dist/services/session-manager.js +0 -73
- package/dist/services/session-manager.js.map +0 -1
- package/dist/services/telemetry.d.ts +0 -19
- package/dist/services/telemetry.js +0 -43
- package/dist/services/transform-worker-pool.d.ts +0 -11
- package/dist/services/transform-worker-pool.js +0 -244
- package/dist/services/transform-worker-types.d.ts +0 -32
- package/dist/services/transform-worker-types.js +0 -14
- package/dist/strategies/exponential-backoff-strategy.d.ts +0 -13
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +0 -1
- package/dist/strategies/exponential-backoff-strategy.js +0 -32
- package/dist/strategies/exponential-backoff-strategy.js.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +0 -4
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +0 -159
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -98
- package/dist/tools/handlers/fetch-links.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +0 -11
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +0 -97
- package/dist/tools/handlers/fetch-markdown.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +0 -31
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.js +0 -223
- package/dist/tools/handlers/fetch-single.shared.js.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +0 -10
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.js +0 -119
- package/dist/tools/handlers/fetch-url.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +0 -13
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.js +0 -153
- package/dist/tools/handlers/fetch-urls/processor.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.d.ts +0 -3
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.js +0 -79
- package/dist/tools/handlers/fetch-urls/response.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.js +0 -18
- package/dist/tools/handlers/fetch-urls/validation.js.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.js +0 -124
- package/dist/tools/handlers/fetch-urls.tool.js.map +0 -1
- package/dist/tools/index.d.ts +0 -3
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js +0 -37
- package/dist/tools/index.js.map +0 -1
- package/dist/tools/schemas.d.ts +0 -12
- package/dist/tools/schemas.d.ts.map +0 -1
- package/dist/tools/schemas.js +0 -21
- package/dist/tools/schemas.js.map +0 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -1
- package/dist/tools/utils/cache-vary.d.ts.map +0 -1
- package/dist/tools/utils/cache-vary.js +0 -12
- package/dist/tools/utils/cache-vary.js.map +0 -1
- package/dist/tools/utils/cached-markdown.d.ts +0 -5
- package/dist/tools/utils/cached-markdown.js +0 -46
- package/dist/tools/utils/common.d.ts +0 -5
- package/dist/tools/utils/common.d.ts.map +0 -1
- package/dist/tools/utils/common.js +0 -42
- package/dist/tools/utils/common.js.map +0 -1
- package/dist/tools/utils/content-shaping.d.ts +0 -4
- package/dist/tools/utils/content-shaping.js +0 -67
- package/dist/tools/utils/content-transform-async.d.ts +0 -6
- package/dist/tools/utils/content-transform-async.js +0 -33
- package/dist/tools/utils/content-transform-core.d.ts +0 -5
- package/dist/tools/utils/content-transform-core.js +0 -180
- package/dist/tools/utils/content-transform-workers.d.ts +0 -1
- package/dist/tools/utils/content-transform-workers.js +0 -1
- package/dist/tools/utils/content-transform.d.ts +0 -3
- package/dist/tools/utils/content-transform.d.ts.map +0 -1
- package/dist/tools/utils/content-transform.js +0 -41
- package/dist/tools/utils/content-transform.js.map +0 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +0 -1
- package/dist/tools/utils/fetch-pipeline.js +0 -98
- package/dist/tools/utils/fetch-pipeline.js.map +0 -1
- package/dist/tools/utils/frontmatter.d.ts +0 -3
- package/dist/tools/utils/frontmatter.js +0 -73
- package/dist/tools/utils/index.d.ts +0 -4
- package/dist/tools/utils/index.d.ts.map +0 -1
- package/dist/tools/utils/index.js +0 -3
- package/dist/tools/utils/index.js.map +0 -1
- package/dist/tools/utils/inline-content.d.ts +0 -10
- package/dist/tools/utils/inline-content.d.ts.map +0 -1
- package/dist/tools/utils/inline-content.js +0 -35
- package/dist/tools/utils/inline-content.js.map +0 -1
- package/dist/tools/utils/markdown-heuristics.d.ts +0 -1
- package/dist/tools/utils/markdown-heuristics.js +0 -19
- package/dist/tools/utils/markdown-signals.d.ts +0 -1
- package/dist/tools/utils/markdown-signals.js +0 -19
- package/dist/tools/utils/markdown-toc.d.ts +0 -3
- package/dist/tools/utils/markdown-toc.d.ts.map +0 -1
- package/dist/tools/utils/markdown-toc.js +0 -35
- package/dist/tools/utils/markdown-toc.js.map +0 -1
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +0 -3
- package/dist/tools/utils/raw-markdown-frontmatter.js +0 -73
- package/dist/tools/utils/raw-markdown.d.ts +0 -6
- package/dist/tools/utils/raw-markdown.js +0 -149
- package/dist/tools/utils/response-builder.d.ts +0 -3
- package/dist/tools/utils/response-builder.d.ts.map +0 -1
- package/dist/tools/utils/response-builder.js +0 -24
- package/dist/tools/utils/response-builder.js.map +0 -1
- package/dist/tools/utils/tool-response.d.ts +0 -9
- package/dist/tools/utils/tool-response.d.ts.map +0 -1
- package/dist/tools/utils/tool-response.js +0 -19
- package/dist/tools/utils/tool-response.js.map +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +0 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +0 -1
- package/dist/transformers/jsonl.transformer.js +0 -75
- package/dist/transformers/jsonl.transformer.js.map +0 -1
- package/dist/transformers/markdown/fenced-code-rule.d.ts +0 -2
- package/dist/transformers/markdown/fenced-code-rule.js +0 -38
- package/dist/transformers/markdown/frontmatter.d.ts +0 -2
- package/dist/transformers/markdown/frontmatter.js +0 -45
- package/dist/transformers/markdown/noise-rule.d.ts +0 -2
- package/dist/transformers/markdown/noise-rule.js +0 -80
- package/dist/transformers/markdown/turndown-instance.d.ts +0 -2
- package/dist/transformers/markdown/turndown-instance.js +0 -19
- package/dist/transformers/markdown.d.ts +0 -5
- package/dist/transformers/markdown.js +0 -314
- package/dist/transformers/markdown.transformer.d.ts +0 -2
- package/dist/transformers/markdown.transformer.d.ts.map +0 -1
- package/dist/transformers/markdown.transformer.js +0 -14
- package/dist/transformers/markdown.transformer.js.map +0 -1
- package/dist/types/content.types.d.ts +0 -63
- package/dist/types/content.types.d.ts.map +0 -1
- package/dist/types/content.types.js +0 -2
- package/dist/types/content.types.js.map +0 -1
- package/dist/types/index.d.ts +0 -2
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -2
- package/dist/types/index.js.map +0 -1
- package/dist/types/schemas.d.ts +0 -49
- package/dist/types/schemas.d.ts.map +0 -1
- package/dist/types/schemas.js +0 -5
- package/dist/types/schemas.js.map +0 -1
- package/dist/utils/cached-payload.d.ts +0 -7
- package/dist/utils/cached-payload.js +0 -36
- package/dist/utils/cancellation.d.ts +0 -1
- package/dist/utils/cancellation.js +0 -18
- package/dist/utils/code-language-bash.d.ts +0 -1
- package/dist/utils/code-language-bash.js +0 -48
- package/dist/utils/code-language-core.d.ts +0 -2
- package/dist/utils/code-language-core.js +0 -13
- package/dist/utils/code-language-detectors.d.ts +0 -5
- package/dist/utils/code-language-detectors.js +0 -142
- package/dist/utils/code-language-helpers.d.ts +0 -5
- package/dist/utils/code-language-helpers.js +0 -62
- package/dist/utils/code-language-parsing.d.ts +0 -5
- package/dist/utils/code-language-parsing.js +0 -62
- package/dist/utils/code-language.d.ts +0 -2
- package/dist/utils/code-language.d.ts.map +0 -1
- package/dist/utils/code-language.js +0 -260
- package/dist/utils/code-language.js.map +0 -1
- package/dist/utils/concurrency.d.ts +0 -3
- package/dist/utils/concurrency.d.ts.map +0 -1
- package/dist/utils/concurrency.js +0 -38
- package/dist/utils/concurrency.js.map +0 -1
- package/dist/utils/content-cleaner.d.ts +0 -5
- package/dist/utils/content-cleaner.d.ts.map +0 -1
- package/dist/utils/content-cleaner.js +0 -77
- package/dist/utils/content-cleaner.js.map +0 -1
- package/dist/utils/crypto.d.ts +0 -2
- package/dist/utils/crypto.d.ts.map +0 -1
- package/dist/utils/crypto.js +0 -32
- package/dist/utils/crypto.js.map +0 -1
- package/dist/utils/download-url.d.ts +0 -16
- package/dist/utils/download-url.d.ts.map +0 -1
- package/dist/utils/download-url.js +0 -30
- package/dist/utils/download-url.js.map +0 -1
- package/dist/utils/error-details.d.ts +0 -3
- package/dist/utils/error-details.js +0 -12
- package/dist/utils/error-utils.d.ts +0 -3
- package/dist/utils/error-utils.d.ts.map +0 -1
- package/dist/utils/error-utils.js +0 -12
- package/dist/utils/error-utils.js.map +0 -1
- package/dist/utils/filename-generator.d.ts +0 -1
- package/dist/utils/filename-generator.d.ts.map +0 -1
- package/dist/utils/filename-generator.js +0 -81
- package/dist/utils/filename-generator.js.map +0 -1
- package/dist/utils/guards.d.ts +0 -1
- package/dist/utils/guards.js +0 -3
- package/dist/utils/header-normalizer.d.ts +0 -5
- package/dist/utils/header-normalizer.d.ts.map +0 -1
- package/dist/utils/header-normalizer.js +0 -31
- package/dist/utils/header-normalizer.js.map +0 -1
- package/dist/utils/host-normalizer.d.ts +0 -1
- package/dist/utils/host-normalizer.js +0 -37
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.d.ts.map +0 -1
- package/dist/utils/html-truncator.js +0 -13
- package/dist/utils/html-truncator.js.map +0 -1
- package/dist/utils/ip-address.d.ts +0 -4
- package/dist/utils/ip-address.js +0 -6
- package/dist/utils/language-detector.d.ts +0 -2
- package/dist/utils/language-detector.d.ts.map +0 -1
- package/dist/utils/language-detector.js +0 -39
- package/dist/utils/language-detector.js.map +0 -1
- package/dist/utils/sanitizer.d.ts +0 -2
- package/dist/utils/sanitizer.d.ts.map +0 -1
- package/dist/utils/sanitizer.js +0 -20
- package/dist/utils/sanitizer.js.map +0 -1
- package/dist/utils/tool-error-handler.d.ts +0 -3
- package/dist/utils/tool-error-handler.d.ts.map +0 -1
- package/dist/utils/tool-error-handler.js +0 -31
- package/dist/utils/tool-error-handler.js.map +0 -1
- package/dist/utils/url-redactor.d.ts +0 -1
- package/dist/utils/url-redactor.js +0 -13
- package/dist/utils/url-sanitizer.d.ts +0 -2
- package/dist/utils/url-sanitizer.d.ts.map +0 -1
- package/dist/utils/url-sanitizer.js +0 -12
- package/dist/utils/url-sanitizer.js.map +0 -1
- package/dist/utils/url-transformer.d.ts +0 -7
- package/dist/utils/url-transformer.js +0 -147
- package/dist/utils/url-validator.d.ts +0 -6
- package/dist/utils/url-validator.d.ts.map +0 -1
- package/dist/utils/url-validator.js +0 -156
- package/dist/utils/url-validator.js.map +0 -1
- package/dist/workers/content-transform.worker.d.ts +0 -1
- package/dist/workers/content-transform.worker.js +0 -40
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
const MAX_CONCURRENCY_LIMIT = 10;
|
|
2
|
-
const MIN_CONCURRENCY = 1;
|
|
3
|
-
function createConcurrencyLimiter(limit) {
|
|
4
|
-
const maxConcurrency = Math.min(Math.max(MIN_CONCURRENCY, limit), MAX_CONCURRENCY_LIMIT);
|
|
5
|
-
let activeCount = 0;
|
|
6
|
-
const waitingQueue = [];
|
|
7
|
-
return async (task) => {
|
|
8
|
-
while (activeCount >= maxConcurrency) {
|
|
9
|
-
await new Promise((resolve) => waitingQueue.push(resolve));
|
|
10
|
-
}
|
|
11
|
-
activeCount++;
|
|
12
|
-
try {
|
|
13
|
-
return await task();
|
|
14
|
-
}
|
|
15
|
-
finally {
|
|
16
|
-
activeCount--;
|
|
17
|
-
const nextWaiting = waitingQueue.shift();
|
|
18
|
-
if (nextWaiting)
|
|
19
|
-
nextWaiting();
|
|
20
|
-
}
|
|
21
|
-
};
|
|
22
|
-
}
|
|
23
|
-
export async function runWithConcurrency(limit, tasks, options) {
|
|
24
|
-
const limiter = createConcurrencyLimiter(limit);
|
|
25
|
-
const totalTasks = tasks.length;
|
|
26
|
-
let completedCount = 0;
|
|
27
|
-
const wrappedTasks = tasks.map((task) => async () => {
|
|
28
|
-
try {
|
|
29
|
-
return await limiter(task);
|
|
30
|
-
}
|
|
31
|
-
finally {
|
|
32
|
-
completedCount++;
|
|
33
|
-
options?.onProgress?.(completedCount, totalTasks);
|
|
34
|
-
}
|
|
35
|
-
});
|
|
36
|
-
return Promise.allSettled(wrappedTasks.map((task) => task()));
|
|
37
|
-
}
|
|
38
|
-
//# sourceMappingURL=concurrency.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"concurrency.js","sourceRoot":"","sources":["../../src/utils/concurrency.ts"],"names":[],"mappings":"AAKA,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,SAAS,wBAAwB,CAAC,KAAa;IAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC7B,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,EAChC,qBAAqB,CACtB,CAAC;IAEF,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,YAAY,GAAmB,EAAE,CAAC;IAExC,OAAO,KAAK,EAAK,IAAsB,EAAc,EAAE;QACrD,OAAO,WAAW,IAAI,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACnE,CAAC;QAED,WAAW,EAAE,CAAC;QACd,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,WAAW,EAAE,CAAC;YACd,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,WAAW;gBAAE,WAAW,EAAE,CAAC;QACjC,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAa,EACb,KAAoC,EACpC,OAAqC;IAErC,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAChC,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAgB,EAAE;QAC9D,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;gBAAS,CAAC;YACT,cAAc,EAAE,CAAC;YACjB,OAAO,EAAE,UAAU,EAAE,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
export declare function cleanParagraph(text: string): string | null;
|
|
2
|
-
export declare function cleanHeading(text: string): string | null;
|
|
3
|
-
export declare function cleanListItems(items: readonly string[]): readonly string[];
|
|
4
|
-
export declare function cleanCodeBlock(code: string): string | null;
|
|
5
|
-
export declare function removeInlineTimestamps(text: string): string;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"content-cleaner.d.ts","sourceRoot":"","sources":["../../src/utils/content-cleaner.ts"],"names":[],"mappings":"AA2BA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAc1D;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAgBxD;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAQxD;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAY1D;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAgB3D"}
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
const NOISE_PATTERN = new RegExp([
|
|
2
|
-
'^(share|copy|like|follow|subscribe|download|print|save)$',
|
|
3
|
-
'^(copy to clipboard|copied!?|copy code|copy link)$',
|
|
4
|
-
'^(show more|show less|load more|view more|read more|see all|view all)$',
|
|
5
|
-
'^(next|previous|prev|back|forward|home|menu|close|skip to)$',
|
|
6
|
-
'^(table of contents|toc|on this page)$',
|
|
7
|
-
'^(loading\\.{0,3}|please wait\\.{0,3}|\\.{2,})$',
|
|
8
|
-
'^(n\\/a|tbd|todo|coming soon|placeholder)$',
|
|
9
|
-
'^\\d+\\s*(comments?|replies?|likes?|shares?|views?)$',
|
|
10
|
-
'^[/\\\\>→»›]+$',
|
|
11
|
-
].join('|'), 'i');
|
|
12
|
-
const MIN_PARAGRAPH_LENGTH = 20;
|
|
13
|
-
const MIN_HEADING_LENGTH = 2;
|
|
14
|
-
const MIN_LIST_ITEM_LENGTH = 3;
|
|
15
|
-
const MAX_REGEX_INPUT_LENGTH = 500;
|
|
16
|
-
function isNoiseText(text) {
|
|
17
|
-
const trimmed = text.trim();
|
|
18
|
-
if (!trimmed)
|
|
19
|
-
return true;
|
|
20
|
-
if (trimmed.length > MAX_REGEX_INPUT_LENGTH)
|
|
21
|
-
return false;
|
|
22
|
-
return NOISE_PATTERN.test(trimmed);
|
|
23
|
-
}
|
|
24
|
-
export function cleanParagraph(text) {
|
|
25
|
-
const trimmed = text.trim();
|
|
26
|
-
if (trimmed.length < MIN_PARAGRAPH_LENGTH) {
|
|
27
|
-
if (!/[.!?]$/.test(trimmed)) {
|
|
28
|
-
return null;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
if (isNoiseText(trimmed)) {
|
|
32
|
-
return null;
|
|
33
|
-
}
|
|
34
|
-
return trimmed;
|
|
35
|
-
}
|
|
36
|
-
export function cleanHeading(text) {
|
|
37
|
-
let cleaned = text.trim();
|
|
38
|
-
if (cleaned.length < MIN_HEADING_LENGTH) {
|
|
39
|
-
return null;
|
|
40
|
-
}
|
|
41
|
-
cleaned = cleaned.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1');
|
|
42
|
-
cleaned = cleaned.replace(/\s*Link for (this heading|[\w\s]+)\s*$/i, '');
|
|
43
|
-
cleaned = cleaned.replace(/\s*#+\s*$/, '');
|
|
44
|
-
if (isNoiseText(cleaned)) {
|
|
45
|
-
return null;
|
|
46
|
-
}
|
|
47
|
-
return cleaned.trim();
|
|
48
|
-
}
|
|
49
|
-
export function cleanListItems(items) {
|
|
50
|
-
return items
|
|
51
|
-
.map((item) => item.trim())
|
|
52
|
-
.filter((item) => {
|
|
53
|
-
if (item.length < MIN_LIST_ITEM_LENGTH)
|
|
54
|
-
return false;
|
|
55
|
-
if (isNoiseText(item))
|
|
56
|
-
return false;
|
|
57
|
-
return true;
|
|
58
|
-
});
|
|
59
|
-
}
|
|
60
|
-
export function cleanCodeBlock(code) {
|
|
61
|
-
const trimmed = code.trim();
|
|
62
|
-
if (trimmed.length === 0) {
|
|
63
|
-
return null;
|
|
64
|
-
}
|
|
65
|
-
if (trimmed.length < 3 && !/^[{}[\]();<>]$/.test(trimmed)) {
|
|
66
|
-
return null;
|
|
67
|
-
}
|
|
68
|
-
return trimmed;
|
|
69
|
-
}
|
|
70
|
-
export function removeInlineTimestamps(text) {
|
|
71
|
-
return text
|
|
72
|
-
.replace(/\b\d+\s*(seconds?|minutes?|hours?|days?|weeks?|months?|years?)\s*ago\b/gi, '')
|
|
73
|
-
.replace(/\b(updated|modified|edited|created|published)\s*:?\s*\d+\s*(seconds?|minutes?|hours?|days?|weeks?|months?|years?)\s*ago\b/gi, '')
|
|
74
|
-
.replace(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2},?\s+\d{4}\b/gi, '')
|
|
75
|
-
.replace(/\s{2,}/g, ' ')
|
|
76
|
-
.trim();
|
|
77
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"content-cleaner.js","sourceRoot":"","sources":["../../src/utils/content-cleaner.ts"],"names":[],"mappings":"AAAA,MAAM,aAAa,GAAG,IAAI,MAAM,CAC9B;IACE,0DAA0D;IAC1D,oDAAoD;IACpD,wEAAwE;IACxE,6DAA6D;IAC7D,wCAAwC;IACxC,iDAAiD;IACjD,4CAA4C;IAC5C,sDAAsD;IACtD,gBAAgB;CACjB,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,OAAO,CAAC,MAAM,GAAG,sBAAsB;QAAE,OAAO,KAAK,CAAC;IAC1D,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE1B,IAAI,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC;IAC1D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,yCAAyC,EAAE,EAAE,CAAC,CAAC;IACzE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAE3C,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAe;IAC5C,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACf,IAAI,IAAI,CAAC,MAAM,GAAG,oBAAoB;YAAE,OAAO,KAAK,CAAC;QACrD,IAAI,WAAW,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,OAAO,IAAI;SACR,OAAO,CACN,0EAA0E,EAC1E,EAAE,CACH;SACA,OAAO,CACN,6HAA6H,EAC7H,EAAE,CACH;SACA,OAAO,CACN,6EAA6E,EAC7E,EAAE,CACH;SACA,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,IAAI,EAAE,CAAC;AACZ,CAAC"}
|
package/dist/utils/crypto.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.d.ts","sourceRoot":"","sources":["../../src/utils/crypto.ts"],"names":[],"mappings":"AAiBA,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAKjE;AAmBD,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAE5D"}
|
package/dist/utils/crypto.js
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import { createHash, hash as oneShotHash, timingSafeEqual } from 'node:crypto';
|
|
2
|
-
const MAX_HASH_INPUT_BYTES = 5 * 1024 * 1024;
|
|
3
|
-
const ALLOWED_HASH_ALGORITHMS = new Set([
|
|
4
|
-
'sha256',
|
|
5
|
-
'sha512',
|
|
6
|
-
]);
|
|
7
|
-
function byteLength(input) {
|
|
8
|
-
return typeof input === 'string'
|
|
9
|
-
? Buffer.byteLength(input, 'utf8')
|
|
10
|
-
: input.byteLength;
|
|
11
|
-
}
|
|
12
|
-
export function timingSafeEqualUtf8(a, b) {
|
|
13
|
-
const aBuffer = Buffer.from(a, 'utf8');
|
|
14
|
-
const bBuffer = Buffer.from(b, 'utf8');
|
|
15
|
-
if (aBuffer.length !== bBuffer.length)
|
|
16
|
-
return false;
|
|
17
|
-
return timingSafeEqual(aBuffer, bBuffer);
|
|
18
|
-
}
|
|
19
|
-
function hashHex(algorithm, input) {
|
|
20
|
-
if (!ALLOWED_HASH_ALGORITHMS.has(algorithm)) {
|
|
21
|
-
throw new Error(`Hash algorithm not allowed: ${algorithm}`);
|
|
22
|
-
}
|
|
23
|
-
if (byteLength(input) <= MAX_HASH_INPUT_BYTES) {
|
|
24
|
-
return oneShotHash(algorithm, input, 'hex');
|
|
25
|
-
}
|
|
26
|
-
const hasher = createHash(algorithm);
|
|
27
|
-
hasher.update(input);
|
|
28
|
-
return hasher.digest('hex');
|
|
29
|
-
}
|
|
30
|
-
export function sha256Hex(input) {
|
|
31
|
-
return hashHex('sha256', input);
|
|
32
|
-
}
|
package/dist/utils/crypto.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.js","sourceRoot":"","sources":["../../src/utils/crypto.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,IAAI,IAAI,WAAW,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE/E,MAAM,oBAAoB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAI7C,MAAM,uBAAuB,GAAsC,IAAI,GAAG,CAAC;IACzE,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,KAA0B;IAC5C,OAAO,OAAO,KAAK,KAAK,QAAQ;QAC9B,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC;QAClC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,CAAS,EAAE,CAAS;IACtD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACpD,OAAO,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,OAAO,CACd,SAA+B,EAC/B,KAA0B;IAE1B,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,oBAAoB,EAAE,CAAC;QAC9C,OAAO,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA0B;IAClD,OAAO,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAClC,CAAC"}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { config } from '../config/index.js';
|
|
2
|
-
import type { FileDownloadInfo } from '../config/types/tools.js';
|
|
3
|
-
import * as cache from '../services/cache.js';
|
|
4
|
-
import { generateSafeFilename } from './filename-generator.js';
|
|
5
|
-
interface DownloadInfoOptions {
|
|
6
|
-
cacheKey: string | null;
|
|
7
|
-
url: string;
|
|
8
|
-
title?: string;
|
|
9
|
-
}
|
|
10
|
-
interface DownloadInfoDeps {
|
|
11
|
-
readonly config?: typeof config;
|
|
12
|
-
readonly cache?: Pick<typeof cache, 'get' | 'parseCacheKey'>;
|
|
13
|
-
readonly generateSafeFilename?: typeof generateSafeFilename;
|
|
14
|
-
}
|
|
15
|
-
export declare function buildFileDownloadInfo(options: DownloadInfoOptions, deps?: DownloadInfoDeps): FileDownloadInfo | null;
|
|
16
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"download-url.d.ts","sourceRoot":"","sources":["../../src/utils/download-url.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAMjE,UAAU,mBAAmB;IAC3B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,mBAAmB,GAC3B,gBAAgB,GAAG,IAAI,CA0BzB"}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import { config } from '../config/index.js';
|
|
2
|
-
import * as cache from '../services/cache.js';
|
|
3
|
-
import { generateSafeFilename } from './filename-generator.js';
|
|
4
|
-
export function buildFileDownloadInfo(options, deps = {}) {
|
|
5
|
-
const resolvedConfig = deps.config ?? config;
|
|
6
|
-
const resolvedCache = deps.cache ?? cache;
|
|
7
|
-
const resolveFilename = deps.generateSafeFilename ?? generateSafeFilename;
|
|
8
|
-
if (!resolvedConfig.runtime.httpMode) {
|
|
9
|
-
return null;
|
|
10
|
-
}
|
|
11
|
-
if (!resolvedConfig.cache.enabled || !options.cacheKey) {
|
|
12
|
-
return null;
|
|
13
|
-
}
|
|
14
|
-
const parts = resolvedCache.parseCacheKey(options.cacheKey);
|
|
15
|
-
if (!parts)
|
|
16
|
-
return null;
|
|
17
|
-
const cacheEntry = resolvedCache.get(options.cacheKey);
|
|
18
|
-
if (!cacheEntry)
|
|
19
|
-
return null;
|
|
20
|
-
const { expiresAt, title, url } = cacheEntry;
|
|
21
|
-
const downloadUrl = buildDownloadUrl(parts.namespace, parts.urlHash);
|
|
22
|
-
const fileName = resolveFilename(url, title ?? options.title, parts.urlHash, resolveExtension(parts.namespace));
|
|
23
|
-
return { downloadUrl, fileName, expiresAt };
|
|
24
|
-
}
|
|
25
|
-
function buildDownloadUrl(namespace, hash) {
|
|
26
|
-
return `/mcp/downloads/${namespace}/${hash}`;
|
|
27
|
-
}
|
|
28
|
-
function resolveExtension(namespace) {
|
|
29
|
-
return namespace === 'markdown' ? '.md' : '.jsonl';
|
|
30
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"download-url.js","sourceRoot":"","sources":["../../src/utils/download-url.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,KAAK,KAAK,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAQ/D,MAAM,UAAU,qBAAqB,CACnC,OAA4B;IAE5B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,IAAI,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAE7B,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,UAAU,CAAC;IAE7C,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,oBAAoB,CACnC,GAAG,EACH,KAAK,IAAI,OAAO,CAAC,KAAK,EACtB,KAAK,CAAC,OAAO,EACb,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC,CAClC,CAAC;IAEF,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AAC9C,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB,EAAE,IAAY;IACvD,OAAO,kBAAkB,SAAS,IAAI,IAAI,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB;IACzC,OAAO,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AACrD,CAAC"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
export function getErrorMessage(error) {
|
|
2
|
-
return error instanceof Error ? error.message : 'Unknown error';
|
|
3
|
-
}
|
|
4
|
-
export function createErrorWithCode(message, code) {
|
|
5
|
-
const error = new Error(message);
|
|
6
|
-
return Object.assign(error, { code });
|
|
7
|
-
}
|
|
8
|
-
export function isSystemError(error) {
|
|
9
|
-
return (error instanceof Error &&
|
|
10
|
-
'code' in error &&
|
|
11
|
-
typeof Reflect.get(error, 'code') === 'string');
|
|
12
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"error-utils.d.ts","sourceRoot":"","sources":["../../src/utils/error-utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEtD;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,GACX,MAAM,CAAC,cAAc,CAIvB;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,cAAc,CAM5E"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
export function getErrorMessage(error) {
|
|
2
|
-
return error instanceof Error ? error.message : 'Unknown error';
|
|
3
|
-
}
|
|
4
|
-
export function createErrorWithCode(message, code) {
|
|
5
|
-
const error = new Error(message);
|
|
6
|
-
return Object.assign(error, { code });
|
|
7
|
-
}
|
|
8
|
-
export function isSystemError(error) {
|
|
9
|
-
return (error instanceof Error &&
|
|
10
|
-
'code' in error &&
|
|
11
|
-
typeof Reflect.get(error, 'code') === 'string');
|
|
12
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"error-utils.js","sourceRoot":"","sources":["../../src/utils/error-utils.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAAe,EACf,IAAY;IAEZ,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAA0B,CAAC;IAC1D,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;IAClB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,OAAO,CACL,KAAK,YAAY,KAAK;QACtB,MAAM,IAAI,KAAK;QACf,OAAQ,KAA+B,CAAC,IAAI,KAAK,QAAQ,CAC1D,CAAC;AACJ,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"filename-generator.d.ts","sourceRoot":"","sources":["../../src/utils/filename-generator.ts"],"names":[],"mappings":"AAMA,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,GAAE,MAA0B,GACpC,MAAM,CAcR"}
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
const MAX_FILENAME_LENGTH = 200;
|
|
2
|
-
const UNSAFE_CHARS_REGEX = /[<>:"/\\|?*]|\p{C}/gu;
|
|
3
|
-
const WHITESPACE_REGEX = /\s+/g;
|
|
4
|
-
const DEFAULT_EXTENSION = '.md';
|
|
5
|
-
function trimHyphens(value) {
|
|
6
|
-
let start = 0;
|
|
7
|
-
let end = value.length;
|
|
8
|
-
while (start < end && value[start] === '-') {
|
|
9
|
-
start += 1;
|
|
10
|
-
}
|
|
11
|
-
while (end > start && value[end - 1] === '-') {
|
|
12
|
-
end -= 1;
|
|
13
|
-
}
|
|
14
|
-
return value.slice(start, end);
|
|
15
|
-
}
|
|
16
|
-
export function generateSafeFilename(url, title, hashFallback, extension = DEFAULT_EXTENSION) {
|
|
17
|
-
const fromUrl = extractFilenameFromUrl(url);
|
|
18
|
-
if (fromUrl)
|
|
19
|
-
return sanitizeFilename(fromUrl, extension);
|
|
20
|
-
if (title) {
|
|
21
|
-
const fromTitle = slugifyTitle(title);
|
|
22
|
-
if (fromTitle)
|
|
23
|
-
return sanitizeFilename(fromTitle, extension);
|
|
24
|
-
}
|
|
25
|
-
if (hashFallback) {
|
|
26
|
-
return `${hashFallback.substring(0, 16)}${extension}`;
|
|
27
|
-
}
|
|
28
|
-
return `download-${Date.now()}${extension}`;
|
|
29
|
-
}
|
|
30
|
-
function getLastPathSegment(url) {
|
|
31
|
-
const segments = url.pathname.split('/').filter(Boolean);
|
|
32
|
-
if (segments.length === 0)
|
|
33
|
-
return null;
|
|
34
|
-
const lastSegment = segments[segments.length - 1];
|
|
35
|
-
return lastSegment ?? null;
|
|
36
|
-
}
|
|
37
|
-
function stripCommonPageExtension(segment) {
|
|
38
|
-
return segment.replace(/\.(html?|php|aspx?|jsp)$/i, '');
|
|
39
|
-
}
|
|
40
|
-
function normalizeUrlFilenameSegment(segment) {
|
|
41
|
-
const cleaned = stripCommonPageExtension(segment);
|
|
42
|
-
if (!cleaned)
|
|
43
|
-
return null;
|
|
44
|
-
if (cleaned === 'index')
|
|
45
|
-
return null;
|
|
46
|
-
return cleaned;
|
|
47
|
-
}
|
|
48
|
-
function extractFilenameFromUrl(url) {
|
|
49
|
-
try {
|
|
50
|
-
const urlObj = new URL(url);
|
|
51
|
-
const lastSegment = getLastPathSegment(urlObj);
|
|
52
|
-
if (!lastSegment)
|
|
53
|
-
return null;
|
|
54
|
-
return normalizeUrlFilenameSegment(lastSegment);
|
|
55
|
-
}
|
|
56
|
-
catch {
|
|
57
|
-
return null;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
function slugifyTitle(title) {
|
|
61
|
-
const slug = title
|
|
62
|
-
.toLowerCase()
|
|
63
|
-
.trim()
|
|
64
|
-
.replace(UNSAFE_CHARS_REGEX, '')
|
|
65
|
-
.replace(WHITESPACE_REGEX, '-')
|
|
66
|
-
.replace(/-+/g, '-');
|
|
67
|
-
const trimmed = trimHyphens(slug);
|
|
68
|
-
return trimmed || null;
|
|
69
|
-
}
|
|
70
|
-
function sanitizeFilename(name, extension) {
|
|
71
|
-
let sanitized = name
|
|
72
|
-
.replace(UNSAFE_CHARS_REGEX, '')
|
|
73
|
-
.replace(WHITESPACE_REGEX, '-')
|
|
74
|
-
.trim();
|
|
75
|
-
// Truncate if too long
|
|
76
|
-
const maxBase = MAX_FILENAME_LENGTH - extension.length;
|
|
77
|
-
if (sanitized.length > maxBase) {
|
|
78
|
-
sanitized = sanitized.substring(0, maxBase);
|
|
79
|
-
}
|
|
80
|
-
return `${sanitized}${extension}`;
|
|
81
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"filename-generator.js","sourceRoot":"","sources":["../../src/utils/filename-generator.ts"],"names":[],"mappings":"AAAA,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAClD,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAEhC,MAAM,UAAU,oBAAoB,CAClC,GAAW,EACX,KAAc,EACd,YAAqB,EACrB,YAAoB,iBAAiB;IAErC,MAAM,OAAO,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,OAAO;QAAE,OAAO,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAEzD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,SAAS;YAAE,OAAO,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC;IACxD,CAAC;IAED,OAAO,YAAY,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;AAC9C,CAAC;AAED,SAAS,sBAAsB,CAAC,GAAW;IACzC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAE5B,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEvC,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAE9B,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC;QAErE,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QAEjD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,MAAM,IAAI,GAAG,KAAK;SACf,WAAW,EAAE;SACb,IAAI,EAAE;SACN,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC;SAC9B,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEzB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAiB;IACvD,IAAI,SAAS,GAAG,IAAI;SACjB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC;SAC9B,IAAI,EAAE,CAAC;IAEV,uBAAuB;IACvB,MAAM,OAAO,GAAG,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC;IACvD,IAAI,SAAS,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;QAC/B,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,GAAG,SAAS,GAAG,SAAS,EAAE,CAAC;AACpC,CAAC"}
|
package/dist/utils/guards.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function isRecord(value: unknown): value is Record<string, unknown>;
|
package/dist/utils/guards.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"header-normalizer.d.ts","sourceRoot":"","sources":["../../src/utils/header-normalizer.ts"],"names":[],"mappings":"AAAA,UAAU,gBAAgB;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EAC3C,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,OAAO,GAAE,gBAAqB,GAC7B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAOpC;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAOT;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAE1D;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAExE"}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
export function normalizeHeaderRecord(headers, blockedHeaders, options = {}) {
|
|
2
|
-
if (!headers || Object.keys(headers).length === 0)
|
|
3
|
-
return undefined;
|
|
4
|
-
const normalized = normalizeHeaderEntries(headers, blockedHeaders, options);
|
|
5
|
-
if (!hasHeaderEntries(normalized))
|
|
6
|
-
return undefined;
|
|
7
|
-
return headersToRecord(normalized);
|
|
8
|
-
}
|
|
9
|
-
function normalizeHeaderEntries(headers, blockedHeaders, options = {}) {
|
|
10
|
-
const normalized = new Headers();
|
|
11
|
-
for (const [key, value] of Object.entries(headers)) {
|
|
12
|
-
if (blockedHeaders.has(key.toLowerCase()))
|
|
13
|
-
continue;
|
|
14
|
-
setHeaderValue(normalized, key, value, options.trimValues === true);
|
|
15
|
-
}
|
|
16
|
-
return normalized;
|
|
17
|
-
}
|
|
18
|
-
function hasHeaderEntries(headers) {
|
|
19
|
-
return !headers.keys().next().done;
|
|
20
|
-
}
|
|
21
|
-
function headersToRecord(headers) {
|
|
22
|
-
return Object.fromEntries(headers.entries());
|
|
23
|
-
}
|
|
24
|
-
function setHeaderValue(headers, key, value, trimValue) {
|
|
25
|
-
try {
|
|
26
|
-
headers.set(key, trimValue ? value.trim() : value);
|
|
27
|
-
}
|
|
28
|
-
catch {
|
|
29
|
-
// Ignore invalid header values
|
|
30
|
-
}
|
|
31
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"header-normalizer.js","sourceRoot":"","sources":["../../src/utils/header-normalizer.ts"],"names":[],"mappings":"AAIA,MAAM,UAAU,qBAAqB,CACnC,OAA2C,EAC3C,cAA2B,EAC3B,UAA4B,EAAE;IAE9B,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAEpE,MAAM,UAAU,GAAG,sBAAsB,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;IAC5E,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC;IAEpD,OAAO,eAAe,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAA+B,EAC/B,cAA2B,EAC3B,UAA4B,EAAE;IAE9B,MAAM,UAAU,GAAG,IAAI,OAAO,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACpD,cAAc,CAAC,UAAU,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAgB;IAC/C,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAgB;IAC9C,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,cAAc,CACrB,OAAgB,EAChB,GAAW,EACX,KAAa,EACb,SAAkB;IAElB,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;AACH,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function normalizeHost(value: string): string | null;
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import { isIP } from 'node:net';
|
|
2
|
-
function takeFirstHostValue(value) {
|
|
3
|
-
const first = value.split(',')[0];
|
|
4
|
-
if (!first)
|
|
5
|
-
return null;
|
|
6
|
-
const trimmed = first.trim();
|
|
7
|
-
return trimmed ? trimmed : null;
|
|
8
|
-
}
|
|
9
|
-
function stripIpv6Brackets(value) {
|
|
10
|
-
if (!value.startsWith('['))
|
|
11
|
-
return null;
|
|
12
|
-
const end = value.indexOf(']');
|
|
13
|
-
if (end === -1)
|
|
14
|
-
return null;
|
|
15
|
-
return value.slice(1, end);
|
|
16
|
-
}
|
|
17
|
-
function stripPortIfPresent(value) {
|
|
18
|
-
const colonIndex = value.indexOf(':');
|
|
19
|
-
if (colonIndex === -1)
|
|
20
|
-
return value;
|
|
21
|
-
return value.slice(0, colonIndex);
|
|
22
|
-
}
|
|
23
|
-
export function normalizeHost(value) {
|
|
24
|
-
const trimmed = value.trim().toLowerCase();
|
|
25
|
-
if (!trimmed)
|
|
26
|
-
return null;
|
|
27
|
-
const first = takeFirstHostValue(trimmed);
|
|
28
|
-
if (!first)
|
|
29
|
-
return null;
|
|
30
|
-
const ipv6 = stripIpv6Brackets(first);
|
|
31
|
-
if (ipv6)
|
|
32
|
-
return ipv6;
|
|
33
|
-
if (isIP(first) === 6) {
|
|
34
|
-
return first;
|
|
35
|
-
}
|
|
36
|
-
return stripPortIfPresent(first);
|
|
37
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function truncateHtml(html: string): string;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"html-truncator.d.ts","sourceRoot":"","sources":["../../src/utils/html-truncator.ts"],"names":[],"mappings":"AAIA,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAajD"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { config } from '../config/index.js';
|
|
2
|
-
import { logWarn } from '../services/logger.js';
|
|
3
|
-
export function truncateHtml(html) {
|
|
4
|
-
const maxSize = config.constants.maxHtmlSize;
|
|
5
|
-
if (html.length <= maxSize) {
|
|
6
|
-
return html;
|
|
7
|
-
}
|
|
8
|
-
logWarn('HTML content exceeds maximum size, truncating', {
|
|
9
|
-
size: html.length,
|
|
10
|
-
maxSize,
|
|
11
|
-
});
|
|
12
|
-
return html.substring(0, maxSize);
|
|
13
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"html-truncator.js","sourceRoot":"","sources":["../../src/utils/html-truncator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC;IAE7C,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,+CAA+C,EAAE;QACvD,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,OAAO;KACR,CAAC,CAAC;IAEH,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACpC,CAAC"}
|
package/dist/utils/ip-address.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"language-detector.d.ts","sourceRoot":"","sources":["../../src/utils/language-detector.ts"],"names":[],"mappings":"AAoCA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE/D"}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
const LANGUAGE_PATTERNS = [
|
|
2
|
-
[
|
|
3
|
-
/^\s*import\s+.*\s+from\s+['"]react['"]|<[A-Z][a-zA-Z]*[\s/>]|jsx\s*:|className=/m,
|
|
4
|
-
'jsx',
|
|
5
|
-
],
|
|
6
|
-
[
|
|
7
|
-
/:\s*(string|number|boolean|void|any|unknown|never)\b|interface\s+\w+|type\s+\w+\s*=/m,
|
|
8
|
-
'typescript',
|
|
9
|
-
],
|
|
10
|
-
[/^\s*(fn|let\s+mut|impl|struct|enum|use\s+\w+::)/m, 'rust'],
|
|
11
|
-
[
|
|
12
|
-
/^\s*(export|const|let|var|function|class|async|await)\b|^\s*import\s+.*['"]/m,
|
|
13
|
-
'javascript',
|
|
14
|
-
],
|
|
15
|
-
// Python patterns
|
|
16
|
-
[/^\s*(def|class|import|from|if __name__|print\()/m, 'python'],
|
|
17
|
-
// Bash/Shell patterns
|
|
18
|
-
[
|
|
19
|
-
/^\s*(npm|yarn|pnpm|npx|brew|apt|pip|cargo|go )\s+(install|add|run|build|start)/m,
|
|
20
|
-
'bash',
|
|
21
|
-
],
|
|
22
|
-
[/^\s*[$#]\s+\w+|^\s*#!|^\s*(sudo|chmod|mkdir|cd|ls|cat|echo)\s+/m, 'bash'],
|
|
23
|
-
// CSS patterns
|
|
24
|
-
[/^\s*[.#@]?[\w-]+\s*\{[^}]*\}|@media|@import|@keyframes/m, 'css'],
|
|
25
|
-
// HTML patterns
|
|
26
|
-
[/^\s*<(!DOCTYPE|html|head|body|div|span|p|a|script|style)\b/im, 'html'],
|
|
27
|
-
// JSON patterns
|
|
28
|
-
[/^\s*\{\s*"|^\s*\[\s*("|\d|true|false|null)/m, 'json'],
|
|
29
|
-
// YAML patterns
|
|
30
|
-
[/^\s*[\w-]+:\s*.+$/m, 'yaml'],
|
|
31
|
-
// SQL patterns
|
|
32
|
-
[/^\s*(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)\s+/im, 'sql'],
|
|
33
|
-
// Go patterns
|
|
34
|
-
[/^\s*(func|package|import\s+")/m, 'go'],
|
|
35
|
-
];
|
|
36
|
-
export function detectLanguage(code) {
|
|
37
|
-
return LANGUAGE_PATTERNS.find(([pattern]) => pattern.test(code))?.[1];
|
|
38
|
-
}
|
|
39
|
-
//# sourceMappingURL=language-detector.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"language-detector.js","sourceRoot":"","sources":["../../src/utils/language-detector.ts"],"names":[],"mappings":"AAAA,MAAM,iBAAiB,GAAG;IACxB;QACE,kFAAkF;QAClF,KAAK;KACN;IACD;QACE,sFAAsF;QACtF,YAAY;KACb;IACD,CAAC,kDAAkD,EAAE,MAAM,CAAC;IAC5D;QACE,8EAA8E;QAC9E,YAAY;KACb;IACD,kBAAkB;IAClB,CAAC,kDAAkD,EAAE,QAAQ,CAAC;IAC9D,sBAAsB;IACtB;QACE,iFAAiF;QACjF,MAAM;KACP;IACD,CAAC,iEAAiE,EAAE,MAAM,CAAC;IAC3E,eAAe;IACf,CAAC,yDAAyD,EAAE,KAAK,CAAC;IAClE,gBAAgB;IAChB,CAAC,8DAA8D,EAAE,MAAM,CAAC;IACxE,gBAAgB;IAChB,CAAC,6CAA6C,EAAE,MAAM,CAAC;IACvD,gBAAgB;IAChB,CAAC,oBAAoB,EAAE,MAAM,CAAC;IAC9B,eAAe;IACf,CAAC,0DAA0D,EAAE,KAAK,CAAC;IACnE,cAAc;IACd,CAAC,gCAAgC,EAAE,IAAI,CAAC;CAChC,CAAC;AAEX,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sanitizer.d.ts","sourceRoot":"","sources":["../../src/utils/sanitizer.ts"],"names":[],"mappings":"AAIA,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAKpE;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAWpE"}
|
package/dist/utils/sanitizer.js
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
const CONSECUTIVE_WHITESPACE = /\s+/g;
|
|
2
|
-
const MIN_TRUNCATION_LENGTH = 4;
|
|
3
|
-
const TRUNCATION_SUFFIX = '...';
|
|
4
|
-
export function sanitizeText(text) {
|
|
5
|
-
if (text == null)
|
|
6
|
-
return '';
|
|
7
|
-
if (typeof text !== 'string')
|
|
8
|
-
return String(text);
|
|
9
|
-
return text.replace(CONSECUTIVE_WHITESPACE, ' ').trim();
|
|
10
|
-
}
|
|
11
|
-
export function truncateText(text, maxLength) {
|
|
12
|
-
if (maxLength < MIN_TRUNCATION_LENGTH) {
|
|
13
|
-
return text.length > 0 ? text.charAt(0) : '';
|
|
14
|
-
}
|
|
15
|
-
if (text.length <= maxLength) {
|
|
16
|
-
return text;
|
|
17
|
-
}
|
|
18
|
-
const truncationPoint = maxLength - TRUNCATION_SUFFIX.length;
|
|
19
|
-
return `${text.substring(0, truncationPoint)}${TRUNCATION_SUFFIX}`;
|
|
20
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sanitizer.js","sourceRoot":"","sources":["../../src/utils/sanitizer.ts"],"names":[],"mappings":"AAAA,MAAM,sBAAsB,GAAG,MAAM,CAAC;AACtC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAChC,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAEhC,MAAM,UAAU,YAAY,CAAC,IAA+B;IAC1D,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC5B,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IAElD,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,SAAiB;IAC1D,IAAI,SAAS,GAAG,qBAAqB,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,eAAe,GAAG,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC;IAC7D,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,eAAe,CAAC,GAAG,iBAAiB,EAAE,CAAC;AACrE,CAAC"}
|
|
@@ -1,3 +0,0 @@
|
|
|
1
|
-
import type { ToolErrorResponse } from '../config/types/tools.js';
|
|
2
|
-
export declare function createToolErrorResponse(message: string, url: string): ToolErrorResponse;
|
|
3
|
-
export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AA6BlE,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,GACX,iBAAiB,CAanB;AAgBD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CAoBnB"}
|