@j0hanz/superfetch 2.1.4 → 2.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts +1 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +1 -0
- package/dist/cache.js.map +1 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -0
- package/dist/crypto.d.ts +1 -0
- package/dist/crypto.d.ts.map +1 -0
- package/dist/crypto.js +1 -0
- package/dist/crypto.js.map +1 -0
- package/dist/errors.d.ts +1 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +1 -0
- package/dist/errors.js.map +1 -0
- package/dist/fetch.d.ts +1 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +1 -0
- package/dist/fetch.js.map +1 -0
- package/dist/http.d.ts +1 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +1 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts +1 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +1 -0
- package/dist/mcp.js.map +1 -0
- package/dist/observability.d.ts +1 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/observability.js +1 -0
- package/dist/observability.js.map +1 -0
- package/dist/tools.d.ts +1 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +1 -0
- package/dist/tools.js.map +1 -0
- package/dist/transform.d.ts +1 -0
- package/dist/transform.d.ts.map +1 -0
- package/dist/transform.js +1 -0
- package/dist/transform.js.map +1 -0
- package/dist/utils.d.ts +1 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +1 -0
- package/dist/utils.js.map +1 -0
- package/dist/workers/transform-worker.d.ts +1 -0
- package/dist/workers/transform-worker.d.ts.map +1 -0
- package/dist/workers/transform-worker.js +1 -0
- package/dist/workers/transform-worker.js.map +1 -0
- package/package.json +5 -2
- package/dist/config/auth-config.d.ts +0 -16
- package/dist/config/auth-config.js +0 -53
- package/dist/config/constants.d.ts +0 -17
- package/dist/config/constants.d.ts.map +0 -1
- package/dist/config/constants.js +0 -22
- package/dist/config/constants.js.map +0 -1
- package/dist/config/env-parsers.d.ts +0 -8
- package/dist/config/env-parsers.js +0 -96
- package/dist/config/formatting.d.ts +0 -7
- package/dist/config/formatting.d.ts.map +0 -1
- package/dist/config/formatting.js +0 -9
- package/dist/config/formatting.js.map +0 -1
- package/dist/config/index.d.ts +0 -61
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -107
- package/dist/config/index.js.map +0 -1
- package/dist/config/types/content.d.ts +0 -41
- package/dist/config/types/content.d.ts.map +0 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/content.js.map +0 -1
- package/dist/config/types/runtime.d.ts +0 -72
- package/dist/config/types/runtime.d.ts.map +0 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/runtime.js.map +0 -1
- package/dist/config/types/tools.d.ts +0 -28
- package/dist/config/types/tools.d.ts.map +0 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/config/types/tools.js.map +0 -1
- package/dist/config/types.d.ts +0 -4
- package/dist/config/types.d.ts.map +0 -1
- package/dist/config/types.js +0 -2
- package/dist/config/types.js.map +0 -1
- package/dist/errors/app-error.d.ts +0 -7
- package/dist/errors/app-error.d.ts.map +0 -1
- package/dist/errors/app-error.js +0 -16
- package/dist/errors/app-error.js.map +0 -1
- package/dist/errors/index.d.ts +0 -2
- package/dist/errors/index.d.ts.map +0 -1
- package/dist/errors/index.js +0 -2
- package/dist/errors/index.js.map +0 -1
- package/dist/http/accept-policy.d.ts +0 -3
- package/dist/http/accept-policy.js +0 -45
- package/dist/http/async-handler.d.ts +0 -2
- package/dist/http/async-handler.js +0 -5
- package/dist/http/auth-introspection.d.ts +0 -2
- package/dist/http/auth-introspection.js +0 -141
- package/dist/http/auth-static.d.ts +0 -2
- package/dist/http/auth-static.js +0 -23
- package/dist/http/auth.d.ts +0 -3
- package/dist/http/auth.d.ts.map +0 -1
- package/dist/http/auth.js +0 -269
- package/dist/http/auth.js.map +0 -1
- package/dist/http/base-middleware.d.ts +0 -7
- package/dist/http/base-middleware.js +0 -143
- package/dist/http/cors.d.ts +0 -2
- package/dist/http/cors.d.ts.map +0 -1
- package/dist/http/cors.js +0 -9
- package/dist/http/cors.js.map +0 -1
- package/dist/http/download-routes.d.ts +0 -2
- package/dist/http/download-routes.d.ts.map +0 -1
- package/dist/http/download-routes.js +0 -104
- package/dist/http/download-routes.js.map +0 -1
- package/dist/http/error-handler.d.ts +0 -2
- package/dist/http/error-handler.js +0 -55
- package/dist/http/host-allowlist.d.ts +0 -3
- package/dist/http/host-allowlist.js +0 -117
- package/dist/http/jsonrpc-http.d.ts +0 -2
- package/dist/http/jsonrpc-http.js +0 -10
- package/dist/http/mcp-routes.d.ts +0 -9
- package/dist/http/mcp-routes.d.ts.map +0 -1
- package/dist/http/mcp-routes.js +0 -204
- package/dist/http/mcp-routes.js.map +0 -1
- package/dist/http/mcp-session-eviction.d.ts +0 -3
- package/dist/http/mcp-session-eviction.js +0 -24
- package/dist/http/mcp-session-helpers.d.ts +0 -12
- package/dist/http/mcp-session-helpers.d.ts.map +0 -1
- package/dist/http/mcp-session-helpers.js +0 -64
- package/dist/http/mcp-session-helpers.js.map +0 -1
- package/dist/http/mcp-session-init.d.ts +0 -7
- package/dist/http/mcp-session-init.js +0 -94
- package/dist/http/mcp-session-slots.d.ts +0 -17
- package/dist/http/mcp-session-slots.js +0 -55
- package/dist/http/mcp-session-transport-init.d.ts +0 -7
- package/dist/http/mcp-session-transport-init.js +0 -41
- package/dist/http/mcp-session-transport.d.ts +0 -7
- package/dist/http/mcp-session-transport.js +0 -57
- package/dist/http/mcp-session-types.d.ts +0 -5
- package/dist/http/mcp-session-types.js +0 -1
- package/dist/http/mcp-session.d.ts +0 -10
- package/dist/http/mcp-session.d.ts.map +0 -1
- package/dist/http/mcp-session.js +0 -23
- package/dist/http/mcp-session.js.map +0 -1
- package/dist/http/mcp-sessions.d.ts +0 -41
- package/dist/http/mcp-sessions.js +0 -392
- package/dist/http/mcp-validation.d.ts +0 -3
- package/dist/http/mcp-validation.d.ts.map +0 -1
- package/dist/http/mcp-validation.js +0 -14
- package/dist/http/mcp-validation.js.map +0 -1
- package/dist/http/protocol-policy.d.ts +0 -2
- package/dist/http/protocol-policy.js +0 -31
- package/dist/http/rate-limit.d.ts +0 -12
- package/dist/http/rate-limit.d.ts.map +0 -1
- package/dist/http/rate-limit.js +0 -93
- package/dist/http/rate-limit.js.map +0 -1
- package/dist/http/server-config.d.ts +0 -1
- package/dist/http/server-config.js +0 -40
- package/dist/http/server-middleware.d.ts +0 -7
- package/dist/http/server-middleware.d.ts.map +0 -1
- package/dist/http/server-middleware.js +0 -52
- package/dist/http/server-middleware.js.map +0 -1
- package/dist/http/server-shutdown.d.ts +0 -4
- package/dist/http/server-shutdown.js +0 -43
- package/dist/http/server-tuning.d.ts +0 -9
- package/dist/http/server-tuning.js +0 -45
- package/dist/http/server.d.ts +0 -3
- package/dist/http/server.d.ts.map +0 -1
- package/dist/http/server.js +0 -291
- package/dist/http/server.js.map +0 -1
- package/dist/http/session-cleanup.d.ts +0 -2
- package/dist/http/session-cleanup.d.ts.map +0 -1
- package/dist/http/session-cleanup.js +0 -40
- package/dist/http/session-cleanup.js.map +0 -1
- package/dist/http/sessions.d.ts +0 -14
- package/dist/http/sessions.d.ts.map +0 -1
- package/dist/http/sessions.js +0 -63
- package/dist/http/sessions.js.map +0 -1
- package/dist/middleware/error-handler.d.ts +0 -2
- package/dist/middleware/error-handler.d.ts.map +0 -1
- package/dist/middleware/error-handler.js +0 -56
- package/dist/middleware/error-handler.js.map +0 -1
- package/dist/middleware/rate-limiter.d.ts +0 -16
- package/dist/middleware/rate-limiter.d.ts.map +0 -1
- package/dist/middleware/rate-limiter.js +0 -111
- package/dist/middleware/rate-limiter.js.map +0 -1
- package/dist/parsers/base-html-element-parser.d.ts +0 -43
- package/dist/parsers/base-html-element-parser.d.ts.map +0 -1
- package/dist/parsers/base-html-element-parser.js +0 -59
- package/dist/parsers/base-html-element-parser.js.map +0 -1
- package/dist/parsers/heading-element-parser.d.ts +0 -14
- package/dist/parsers/heading-element-parser.d.ts.map +0 -1
- package/dist/parsers/heading-element-parser.js +0 -26
- package/dist/parsers/heading-element-parser.js.map +0 -1
- package/dist/parsers/image-element-parser.d.ts +0 -16
- package/dist/parsers/image-element-parser.d.ts.map +0 -1
- package/dist/parsers/image-element-parser.js +0 -33
- package/dist/parsers/image-element-parser.js.map +0 -1
- package/dist/parsers/link-element-parser.d.ts +0 -15
- package/dist/parsers/link-element-parser.d.ts.map +0 -1
- package/dist/parsers/link-element-parser.js +0 -28
- package/dist/parsers/link-element-parser.js.map +0 -1
- package/dist/parsers/open-graph-parser.d.ts +0 -17
- package/dist/parsers/open-graph-parser.d.ts.map +0 -1
- package/dist/parsers/open-graph-parser.js +0 -41
- package/dist/parsers/open-graph-parser.js.map +0 -1
- package/dist/parsers/schema-org-parser.d.ts +0 -17
- package/dist/parsers/schema-org-parser.d.ts.map +0 -1
- package/dist/parsers/schema-org-parser.js +0 -32
- package/dist/parsers/schema-org-parser.js.map +0 -1
- package/dist/parsers/standard-meta-parser.d.ts +0 -18
- package/dist/parsers/standard-meta-parser.d.ts.map +0 -1
- package/dist/parsers/standard-meta-parser.js +0 -32
- package/dist/parsers/standard-meta-parser.js.map +0 -1
- package/dist/parsers/twitter-card-parser.d.ts +0 -17
- package/dist/parsers/twitter-card-parser.d.ts.map +0 -1
- package/dist/parsers/twitter-card-parser.js +0 -41
- package/dist/parsers/twitter-card-parser.js.map +0 -1
- package/dist/prompts/index.d.ts +0 -3
- package/dist/prompts/index.d.ts.map +0 -1
- package/dist/prompts/index.js +0 -73
- package/dist/prompts/index.js.map +0 -1
- package/dist/resources/cached-content-params.d.ts +0 -5
- package/dist/resources/cached-content-params.js +0 -36
- package/dist/resources/cached-content.d.ts +0 -2
- package/dist/resources/cached-content.d.ts.map +0 -1
- package/dist/resources/cached-content.js +0 -132
- package/dist/resources/cached-content.js.map +0 -1
- package/dist/resources/index.d.ts +0 -2
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js +0 -4
- package/dist/resources/index.js.map +0 -1
- package/dist/server.d.ts +0 -3
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js +0 -94
- package/dist/server.js.map +0 -1
- package/dist/services/cache-events.d.ts +0 -8
- package/dist/services/cache-events.js +0 -19
- package/dist/services/cache-keys.d.ts +0 -7
- package/dist/services/cache-keys.js +0 -57
- package/dist/services/cache.d.ts +0 -17
- package/dist/services/cache.d.ts.map +0 -1
- package/dist/services/cache.js +0 -145
- package/dist/services/cache.js.map +0 -1
- package/dist/services/cache.service.d.ts +0 -52
- package/dist/services/cache.service.d.ts.map +0 -1
- package/dist/services/cache.service.js +0 -113
- package/dist/services/cache.service.js.map +0 -1
- package/dist/services/card-extractor.d.ts +0 -6
- package/dist/services/card-extractor.d.ts.map +0 -1
- package/dist/services/card-extractor.js +0 -199
- package/dist/services/card-extractor.js.map +0 -1
- package/dist/services/context.d.ts +0 -10
- package/dist/services/context.d.ts.map +0 -1
- package/dist/services/context.js +0 -14
- package/dist/services/context.js.map +0 -1
- package/dist/services/extractor.d.ts +0 -5
- package/dist/services/extractor.d.ts.map +0 -1
- package/dist/services/extractor.js +0 -142
- package/dist/services/extractor.js.map +0 -1
- package/dist/services/extractor.service.d.ts +0 -18
- package/dist/services/extractor.service.d.ts.map +0 -1
- package/dist/services/extractor.service.js +0 -75
- package/dist/services/extractor.service.js.map +0 -1
- package/dist/services/fetcher/agents.d.ts +0 -3
- package/dist/services/fetcher/agents.d.ts.map +0 -1
- package/dist/services/fetcher/agents.js +0 -100
- package/dist/services/fetcher/agents.js.map +0 -1
- package/dist/services/fetcher/dns-selection.d.ts +0 -2
- package/dist/services/fetcher/dns-selection.js +0 -72
- package/dist/services/fetcher/errors.d.ts +0 -4
- package/dist/services/fetcher/errors.d.ts.map +0 -1
- package/dist/services/fetcher/errors.js +0 -70
- package/dist/services/fetcher/errors.js.map +0 -1
- package/dist/services/fetcher/headers.d.ts +0 -2
- package/dist/services/fetcher/headers.d.ts.map +0 -1
- package/dist/services/fetcher/headers.js +0 -6
- package/dist/services/fetcher/headers.js.map +0 -1
- package/dist/services/fetcher/interceptors.d.ts +0 -10
- package/dist/services/fetcher/interceptors.d.ts.map +0 -1
- package/dist/services/fetcher/interceptors.js +0 -108
- package/dist/services/fetcher/interceptors.js.map +0 -1
- package/dist/services/fetcher/redirects.d.ts +0 -4
- package/dist/services/fetcher/redirects.d.ts.map +0 -1
- package/dist/services/fetcher/redirects.js +0 -78
- package/dist/services/fetcher/redirects.js.map +0 -1
- package/dist/services/fetcher/response.d.ts +0 -4
- package/dist/services/fetcher/response.d.ts.map +0 -1
- package/dist/services/fetcher/response.js +0 -104
- package/dist/services/fetcher/response.js.map +0 -1
- package/dist/services/fetcher/retry-policy.d.ts +0 -1
- package/dist/services/fetcher/retry-policy.d.ts.map +0 -1
- package/dist/services/fetcher/retry-policy.js +0 -131
- package/dist/services/fetcher/retry-policy.js.map +0 -1
- package/dist/services/fetcher.d.ts +0 -25
- package/dist/services/fetcher.d.ts.map +0 -1
- package/dist/services/fetcher.js +0 -607
- package/dist/services/fetcher.js.map +0 -1
- package/dist/services/fetcher.service.d.ts +0 -18
- package/dist/services/fetcher.service.d.ts.map +0 -1
- package/dist/services/fetcher.service.js +0 -122
- package/dist/services/fetcher.service.js.map +0 -1
- package/dist/services/fifo-queue.d.ts +0 -8
- package/dist/services/fifo-queue.js +0 -25
- package/dist/services/logger.d.ts +0 -5
- package/dist/services/logger.d.ts.map +0 -1
- package/dist/services/logger.js +0 -52
- package/dist/services/logger.js.map +0 -1
- package/dist/services/logger.service.d.ts +0 -5
- package/dist/services/logger.service.d.ts.map +0 -1
- package/dist/services/logger.service.js +0 -57
- package/dist/services/logger.service.js.map +0 -1
- package/dist/services/metadata-collector.d.ts +0 -2
- package/dist/services/metadata-collector.js +0 -80
- package/dist/services/parser.d.ts +0 -6
- package/dist/services/parser.d.ts.map +0 -1
- package/dist/services/parser.js +0 -278
- package/dist/services/parser.js.map +0 -1
- package/dist/services/parser.service.d.ts +0 -42
- package/dist/services/parser.service.d.ts.map +0 -1
- package/dist/services/parser.service.js +0 -209
- package/dist/services/parser.service.js.map +0 -1
- package/dist/services/session-manager.d.ts +0 -18
- package/dist/services/session-manager.d.ts.map +0 -1
- package/dist/services/session-manager.js +0 -73
- package/dist/services/session-manager.js.map +0 -1
- package/dist/services/telemetry.d.ts +0 -19
- package/dist/services/telemetry.js +0 -43
- package/dist/services/transform-worker-pool.d.ts +0 -11
- package/dist/services/transform-worker-pool.js +0 -244
- package/dist/services/transform-worker-types.d.ts +0 -32
- package/dist/services/transform-worker-types.js +0 -14
- package/dist/strategies/exponential-backoff-strategy.d.ts +0 -13
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +0 -1
- package/dist/strategies/exponential-backoff-strategy.js +0 -32
- package/dist/strategies/exponential-backoff-strategy.js.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +0 -4
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +0 -159
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -98
- package/dist/tools/handlers/fetch-links.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +0 -11
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +0 -97
- package/dist/tools/handlers/fetch-markdown.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +0 -31
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.js +0 -223
- package/dist/tools/handlers/fetch-single.shared.js.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +0 -10
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.js +0 -119
- package/dist/tools/handlers/fetch-url.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +0 -13
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.js +0 -153
- package/dist/tools/handlers/fetch-urls/processor.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.d.ts +0 -3
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.js +0 -79
- package/dist/tools/handlers/fetch-urls/response.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.js +0 -18
- package/dist/tools/handlers/fetch-urls/validation.js.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.js +0 -124
- package/dist/tools/handlers/fetch-urls.tool.js.map +0 -1
- package/dist/tools/index.d.ts +0 -3
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js +0 -37
- package/dist/tools/index.js.map +0 -1
- package/dist/tools/schemas.d.ts +0 -12
- package/dist/tools/schemas.d.ts.map +0 -1
- package/dist/tools/schemas.js +0 -21
- package/dist/tools/schemas.js.map +0 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -1
- package/dist/tools/utils/cache-vary.d.ts.map +0 -1
- package/dist/tools/utils/cache-vary.js +0 -12
- package/dist/tools/utils/cache-vary.js.map +0 -1
- package/dist/tools/utils/cached-markdown.d.ts +0 -5
- package/dist/tools/utils/cached-markdown.js +0 -46
- package/dist/tools/utils/common.d.ts +0 -5
- package/dist/tools/utils/common.d.ts.map +0 -1
- package/dist/tools/utils/common.js +0 -42
- package/dist/tools/utils/common.js.map +0 -1
- package/dist/tools/utils/content-shaping.d.ts +0 -4
- package/dist/tools/utils/content-shaping.js +0 -67
- package/dist/tools/utils/content-transform-async.d.ts +0 -6
- package/dist/tools/utils/content-transform-async.js +0 -33
- package/dist/tools/utils/content-transform-core.d.ts +0 -5
- package/dist/tools/utils/content-transform-core.js +0 -180
- package/dist/tools/utils/content-transform-workers.d.ts +0 -1
- package/dist/tools/utils/content-transform-workers.js +0 -1
- package/dist/tools/utils/content-transform.d.ts +0 -3
- package/dist/tools/utils/content-transform.d.ts.map +0 -1
- package/dist/tools/utils/content-transform.js +0 -41
- package/dist/tools/utils/content-transform.js.map +0 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +0 -1
- package/dist/tools/utils/fetch-pipeline.js +0 -98
- package/dist/tools/utils/fetch-pipeline.js.map +0 -1
- package/dist/tools/utils/frontmatter.d.ts +0 -3
- package/dist/tools/utils/frontmatter.js +0 -73
- package/dist/tools/utils/index.d.ts +0 -4
- package/dist/tools/utils/index.d.ts.map +0 -1
- package/dist/tools/utils/index.js +0 -3
- package/dist/tools/utils/index.js.map +0 -1
- package/dist/tools/utils/inline-content.d.ts +0 -10
- package/dist/tools/utils/inline-content.d.ts.map +0 -1
- package/dist/tools/utils/inline-content.js +0 -35
- package/dist/tools/utils/inline-content.js.map +0 -1
- package/dist/tools/utils/markdown-heuristics.d.ts +0 -1
- package/dist/tools/utils/markdown-heuristics.js +0 -19
- package/dist/tools/utils/markdown-signals.d.ts +0 -1
- package/dist/tools/utils/markdown-signals.js +0 -19
- package/dist/tools/utils/markdown-toc.d.ts +0 -3
- package/dist/tools/utils/markdown-toc.d.ts.map +0 -1
- package/dist/tools/utils/markdown-toc.js +0 -35
- package/dist/tools/utils/markdown-toc.js.map +0 -1
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +0 -3
- package/dist/tools/utils/raw-markdown-frontmatter.js +0 -73
- package/dist/tools/utils/raw-markdown.d.ts +0 -6
- package/dist/tools/utils/raw-markdown.js +0 -149
- package/dist/tools/utils/response-builder.d.ts +0 -3
- package/dist/tools/utils/response-builder.d.ts.map +0 -1
- package/dist/tools/utils/response-builder.js +0 -24
- package/dist/tools/utils/response-builder.js.map +0 -1
- package/dist/tools/utils/tool-response.d.ts +0 -9
- package/dist/tools/utils/tool-response.d.ts.map +0 -1
- package/dist/tools/utils/tool-response.js +0 -19
- package/dist/tools/utils/tool-response.js.map +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +0 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +0 -1
- package/dist/transformers/jsonl.transformer.js +0 -75
- package/dist/transformers/jsonl.transformer.js.map +0 -1
- package/dist/transformers/markdown/fenced-code-rule.d.ts +0 -2
- package/dist/transformers/markdown/fenced-code-rule.js +0 -38
- package/dist/transformers/markdown/frontmatter.d.ts +0 -2
- package/dist/transformers/markdown/frontmatter.js +0 -45
- package/dist/transformers/markdown/noise-rule.d.ts +0 -2
- package/dist/transformers/markdown/noise-rule.js +0 -80
- package/dist/transformers/markdown/turndown-instance.d.ts +0 -2
- package/dist/transformers/markdown/turndown-instance.js +0 -19
- package/dist/transformers/markdown.d.ts +0 -5
- package/dist/transformers/markdown.js +0 -314
- package/dist/transformers/markdown.transformer.d.ts +0 -2
- package/dist/transformers/markdown.transformer.d.ts.map +0 -1
- package/dist/transformers/markdown.transformer.js +0 -14
- package/dist/transformers/markdown.transformer.js.map +0 -1
- package/dist/types/content.types.d.ts +0 -63
- package/dist/types/content.types.d.ts.map +0 -1
- package/dist/types/content.types.js +0 -2
- package/dist/types/content.types.js.map +0 -1
- package/dist/types/index.d.ts +0 -2
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -2
- package/dist/types/index.js.map +0 -1
- package/dist/types/schemas.d.ts +0 -49
- package/dist/types/schemas.d.ts.map +0 -1
- package/dist/types/schemas.js +0 -5
- package/dist/types/schemas.js.map +0 -1
- package/dist/utils/cached-payload.d.ts +0 -7
- package/dist/utils/cached-payload.js +0 -36
- package/dist/utils/cancellation.d.ts +0 -1
- package/dist/utils/cancellation.js +0 -18
- package/dist/utils/code-language-bash.d.ts +0 -1
- package/dist/utils/code-language-bash.js +0 -48
- package/dist/utils/code-language-core.d.ts +0 -2
- package/dist/utils/code-language-core.js +0 -13
- package/dist/utils/code-language-detectors.d.ts +0 -5
- package/dist/utils/code-language-detectors.js +0 -142
- package/dist/utils/code-language-helpers.d.ts +0 -5
- package/dist/utils/code-language-helpers.js +0 -62
- package/dist/utils/code-language-parsing.d.ts +0 -5
- package/dist/utils/code-language-parsing.js +0 -62
- package/dist/utils/code-language.d.ts +0 -2
- package/dist/utils/code-language.d.ts.map +0 -1
- package/dist/utils/code-language.js +0 -260
- package/dist/utils/code-language.js.map +0 -1
- package/dist/utils/concurrency.d.ts +0 -3
- package/dist/utils/concurrency.d.ts.map +0 -1
- package/dist/utils/concurrency.js +0 -38
- package/dist/utils/concurrency.js.map +0 -1
- package/dist/utils/content-cleaner.d.ts +0 -5
- package/dist/utils/content-cleaner.d.ts.map +0 -1
- package/dist/utils/content-cleaner.js +0 -77
- package/dist/utils/content-cleaner.js.map +0 -1
- package/dist/utils/crypto.d.ts +0 -2
- package/dist/utils/crypto.d.ts.map +0 -1
- package/dist/utils/crypto.js +0 -32
- package/dist/utils/crypto.js.map +0 -1
- package/dist/utils/download-url.d.ts +0 -16
- package/dist/utils/download-url.d.ts.map +0 -1
- package/dist/utils/download-url.js +0 -30
- package/dist/utils/download-url.js.map +0 -1
- package/dist/utils/error-details.d.ts +0 -3
- package/dist/utils/error-details.js +0 -12
- package/dist/utils/error-utils.d.ts +0 -3
- package/dist/utils/error-utils.d.ts.map +0 -1
- package/dist/utils/error-utils.js +0 -12
- package/dist/utils/error-utils.js.map +0 -1
- package/dist/utils/filename-generator.d.ts +0 -1
- package/dist/utils/filename-generator.d.ts.map +0 -1
- package/dist/utils/filename-generator.js +0 -81
- package/dist/utils/filename-generator.js.map +0 -1
- package/dist/utils/guards.d.ts +0 -1
- package/dist/utils/guards.js +0 -3
- package/dist/utils/header-normalizer.d.ts +0 -5
- package/dist/utils/header-normalizer.d.ts.map +0 -1
- package/dist/utils/header-normalizer.js +0 -31
- package/dist/utils/header-normalizer.js.map +0 -1
- package/dist/utils/host-normalizer.d.ts +0 -1
- package/dist/utils/host-normalizer.js +0 -37
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.d.ts.map +0 -1
- package/dist/utils/html-truncator.js +0 -13
- package/dist/utils/html-truncator.js.map +0 -1
- package/dist/utils/ip-address.d.ts +0 -4
- package/dist/utils/ip-address.js +0 -6
- package/dist/utils/language-detector.d.ts +0 -2
- package/dist/utils/language-detector.d.ts.map +0 -1
- package/dist/utils/language-detector.js +0 -39
- package/dist/utils/language-detector.js.map +0 -1
- package/dist/utils/sanitizer.d.ts +0 -2
- package/dist/utils/sanitizer.d.ts.map +0 -1
- package/dist/utils/sanitizer.js +0 -20
- package/dist/utils/sanitizer.js.map +0 -1
- package/dist/utils/tool-error-handler.d.ts +0 -3
- package/dist/utils/tool-error-handler.d.ts.map +0 -1
- package/dist/utils/tool-error-handler.js +0 -31
- package/dist/utils/tool-error-handler.js.map +0 -1
- package/dist/utils/url-redactor.d.ts +0 -1
- package/dist/utils/url-redactor.js +0 -13
- package/dist/utils/url-sanitizer.d.ts +0 -2
- package/dist/utils/url-sanitizer.d.ts.map +0 -1
- package/dist/utils/url-sanitizer.js +0 -12
- package/dist/utils/url-sanitizer.js.map +0 -1
- package/dist/utils/url-transformer.d.ts +0 -7
- package/dist/utils/url-transformer.js +0 -147
- package/dist/utils/url-validator.d.ts +0 -6
- package/dist/utils/url-validator.d.ts.map +0 -1
- package/dist/utils/url-validator.js +0 -156
- package/dist/utils/url-validator.js.map +0 -1
- package/dist/workers/content-transform.worker.d.ts +0 -1
- package/dist/workers/content-transform.worker.js +0 -40
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
import axios from 'axios';
|
|
2
|
-
import { config } from '../config/index.js';
|
|
3
|
-
import { FetchError, TimeoutError } from '../errors/app-error.js';
|
|
4
|
-
// Headers that should not be overridden by custom headers (security)
|
|
5
|
-
const BLOCKED_HEADERS = new Set([
|
|
6
|
-
'host',
|
|
7
|
-
'authorization',
|
|
8
|
-
'cookie',
|
|
9
|
-
'x-forwarded-for',
|
|
10
|
-
'x-real-ip',
|
|
11
|
-
'proxy-authorization',
|
|
12
|
-
]);
|
|
13
|
-
/**
|
|
14
|
-
* Sanitize custom headers to prevent injection of sensitive headers
|
|
15
|
-
*/
|
|
16
|
-
function sanitizeHeaders(headers) {
|
|
17
|
-
if (!headers || Object.keys(headers).length === 0) {
|
|
18
|
-
return undefined;
|
|
19
|
-
}
|
|
20
|
-
const sanitized = {};
|
|
21
|
-
for (const [key, value] of Object.entries(headers)) {
|
|
22
|
-
if (!BLOCKED_HEADERS.has(key.toLowerCase())) {
|
|
23
|
-
sanitized[key] = value;
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
return Object.keys(sanitized).length > 0 ? sanitized : undefined;
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Calculates exponential backoff delay with jitter
|
|
30
|
-
*/
|
|
31
|
-
function calculateBackoff(attempt, maxDelay = 10000) {
|
|
32
|
-
const baseDelay = Math.min(1000 * Math.pow(2, attempt - 1), maxDelay);
|
|
33
|
-
// Add jitter (±25%)
|
|
34
|
-
const jitter = baseDelay * 0.25 * (Math.random() * 2 - 1);
|
|
35
|
-
return Math.round(baseDelay + jitter);
|
|
36
|
-
}
|
|
37
|
-
class FetcherService {
|
|
38
|
-
client;
|
|
39
|
-
constructor() {
|
|
40
|
-
this.client = axios.create({
|
|
41
|
-
timeout: config.fetcher.timeout,
|
|
42
|
-
maxRedirects: config.fetcher.maxRedirects,
|
|
43
|
-
maxContentLength: config.fetcher.maxContentLength,
|
|
44
|
-
headers: {
|
|
45
|
-
'User-Agent': config.fetcher.userAgent,
|
|
46
|
-
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
47
|
-
'Accept-Language': 'en-US,en;q=0.5',
|
|
48
|
-
'Accept-Encoding': 'gzip, deflate, br',
|
|
49
|
-
Connection: 'keep-alive',
|
|
50
|
-
},
|
|
51
|
-
// Validate status - only 2xx responses are valid
|
|
52
|
-
validateStatus: (status) => status >= 200 && status < 300,
|
|
53
|
-
});
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Fetches HTML content from a URL
|
|
57
|
-
* @throws {FetchError} on network or HTTP errors
|
|
58
|
-
* @throws {TimeoutError} on request timeout
|
|
59
|
-
*/
|
|
60
|
-
async fetchUrl(url, customHeaders) {
|
|
61
|
-
const requestConfig = {
|
|
62
|
-
method: 'GET',
|
|
63
|
-
url,
|
|
64
|
-
responseType: 'text',
|
|
65
|
-
};
|
|
66
|
-
// Add sanitized custom headers if provided
|
|
67
|
-
const sanitized = sanitizeHeaders(customHeaders);
|
|
68
|
-
if (sanitized) {
|
|
69
|
-
requestConfig.headers = { ...requestConfig.headers, ...sanitized };
|
|
70
|
-
}
|
|
71
|
-
try {
|
|
72
|
-
const response = await this.client.request(requestConfig);
|
|
73
|
-
return response.data;
|
|
74
|
-
}
|
|
75
|
-
catch (error) {
|
|
76
|
-
if (!axios.isAxiosError(error)) {
|
|
77
|
-
throw new FetchError(`Unexpected error: ${error instanceof Error ? error.message : 'Unknown'}`, url);
|
|
78
|
-
}
|
|
79
|
-
const axiosError = error;
|
|
80
|
-
if (axiosError.code === 'ECONNABORTED' || axiosError.code === 'ETIMEDOUT') {
|
|
81
|
-
throw new TimeoutError(config.fetcher.timeout, true);
|
|
82
|
-
}
|
|
83
|
-
if (axiosError.response) {
|
|
84
|
-
throw new FetchError(`HTTP ${axiosError.response.status}: ${axiosError.response.statusText}`, url, axiosError.response.status);
|
|
85
|
-
}
|
|
86
|
-
if (axiosError.request) {
|
|
87
|
-
throw new FetchError(`Network error: Could not reach ${url}`, url);
|
|
88
|
-
}
|
|
89
|
-
throw new FetchError(axiosError.message, url);
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
/**
|
|
93
|
-
* Fetches URL with exponential backoff retry logic
|
|
94
|
-
* @throws {FetchError} after all retries exhausted
|
|
95
|
-
*/
|
|
96
|
-
async fetchUrlWithRetry(url, customHeaders, maxRetries = 3) {
|
|
97
|
-
let lastError;
|
|
98
|
-
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
99
|
-
try {
|
|
100
|
-
return await this.fetchUrl(url, customHeaders);
|
|
101
|
-
}
|
|
102
|
-
catch (error) {
|
|
103
|
-
lastError = error instanceof Error ? error : new Error('Unknown error');
|
|
104
|
-
// Don't retry on client errors (4xx) except 429 (rate limited)
|
|
105
|
-
if (error instanceof FetchError && error.httpStatus) {
|
|
106
|
-
const status = error.httpStatus;
|
|
107
|
-
if (status >= 400 && status < 500 && status !== 429) {
|
|
108
|
-
throw error;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
if (attempt < maxRetries) {
|
|
112
|
-
const delay = calculateBackoff(attempt);
|
|
113
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
throw new FetchError(`Failed after ${maxRetries} attempts: ${lastError?.message ?? 'Unknown error'}`, url);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
// Singleton instance
|
|
121
|
-
export const fetcherService = new FetcherService();
|
|
122
|
-
//# sourceMappingURL=fetcher.service.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.service.js","sourceRoot":"","sources":["../../src/services/fetcher.service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAwD,MAAM,OAAO,CAAC;AAC7E,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAElE,qEAAqE;AACrE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,MAAM;IACN,eAAe;IACf,QAAQ;IACR,iBAAiB;IACjB,WAAW;IACX,qBAAqB;CACtB,CAAC,CAAC;AAEH;;GAEG;AACH,SAAS,eAAe,CACtB,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,SAAS,GAA2B,EAAE,CAAC;IAE7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC5C,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe,EAAE,QAAQ,GAAG,KAAK;IACzD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACtE,oBAAoB;IACpB,MAAM,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,cAAc;IACD,MAAM,CAAgB;IAEvC;QACE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACzB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;YAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;YACzC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB;YACjD,OAAO,EAAE;gBACP,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;gBACtC,MAAM,EACJ,4EAA4E;gBAC9E,iBAAiB,EAAE,gBAAgB;gBACnC,iBAAiB,EAAE,mBAAmB;gBACtC,UAAU,EAAE,YAAY;aACzB;YACD,iDAAiD;YACjD,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;SAC1D,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CACZ,GAAW,EACX,aAAsC;QAEtC,MAAM,aAAa,GAAuB;YACxC,MAAM,EAAE,KAAK;YACb,GAAG;YACH,YAAY,EAAE,MAAM;SACrB,CAAC;QAEF,2CAA2C;QAC3C,MAAM,SAAS,GAAG,eAAe,CAAC,aAAa,CAAC,CAAC;QACjD,IAAI,SAAS,EAAE,CAAC;YACd,aAAa,CAAC,OAAO,GAAG,EAAE,GAAG,aAAa,CAAC,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC;QACrE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAS,aAAa,CAAC,CAAC;YAClE,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,UAAU,CAClB,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EACzE,GAAG,CACJ,CAAC;YACJ,CAAC;YAED,MAAM,UAAU,GAAG,KAAmB,CAAC;YAEvC,IAAI,UAAU,CAAC,IAAI,KAAK,cAAc,IAAI,UAAU,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC1E,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACvD,CAAC;YAED,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACxB,MAAM,IAAI,UAAU,CAClB,QAAQ,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,UAAU,CAAC,QAAQ,CAAC,UAAU,EAAE,EACvE,GAAG,EACH,UAAU,CAAC,QAAQ,CAAC,MAAM,CAC3B,CAAC;YACJ,CAAC;YAED,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;gBACvB,MAAM,IAAI,UAAU,CAAC,kCAAkC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YACrE,CAAC;YAED,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CACrB,GAAW,EACX,aAAsC,EACtC,UAAU,GAAG,CAAC;QAEd,IAAI,SAA4B,CAAC;QAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;gBAExE,+DAA+D;gBAC/D,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;oBACpD,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC;oBAChC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;wBACpD,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;oBACzB,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,UAAU,CAClB,gBAAgB,UAAU,cAAc,SAAS,EAAE,OAAO,IAAI,eAAe,EAAE,EAC/E,GAAG,CACJ,CAAC;IACJ,CAAC;CACF;AAED,qBAAqB;AACrB,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
export class FifoQueue {
|
|
2
|
-
items = [];
|
|
3
|
-
head = 0;
|
|
4
|
-
get length() {
|
|
5
|
-
return this.items.length - this.head;
|
|
6
|
-
}
|
|
7
|
-
push(item) {
|
|
8
|
-
this.items.push(item);
|
|
9
|
-
}
|
|
10
|
-
shift() {
|
|
11
|
-
if (this.head >= this.items.length)
|
|
12
|
-
return undefined;
|
|
13
|
-
const item = this.items[this.head];
|
|
14
|
-
this.head += 1;
|
|
15
|
-
if (this.head > 64 && this.head * 2 >= this.items.length) {
|
|
16
|
-
this.items = this.items.slice(this.head);
|
|
17
|
-
this.head = 0;
|
|
18
|
-
}
|
|
19
|
-
return item;
|
|
20
|
-
}
|
|
21
|
-
clear() {
|
|
22
|
-
this.items.length = 0;
|
|
23
|
-
this.head = 0;
|
|
24
|
-
}
|
|
25
|
-
}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import type { LogMetadata } from '../config/types/runtime.js';
|
|
2
|
-
export declare function logInfo(message: string, meta?: LogMetadata): void;
|
|
3
|
-
export declare function logDebug(message: string, meta?: LogMetadata): void;
|
|
4
|
-
export declare function logWarn(message: string, meta?: LogMetadata): void;
|
|
5
|
-
export declare function logError(message: string, error?: Error | LogMetadata): void;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAkCxE,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIlE;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAS3E"}
|
package/dist/services/logger.js
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import { config } from '../config/index.js';
|
|
2
|
-
import { getOperationId, getRequestId, getSessionId } from './context.js';
|
|
3
|
-
function formatMetadata(meta) {
|
|
4
|
-
const requestId = getRequestId();
|
|
5
|
-
const sessionId = getSessionId();
|
|
6
|
-
const operationId = getOperationId();
|
|
7
|
-
const contextMeta = {};
|
|
8
|
-
if (requestId)
|
|
9
|
-
contextMeta.requestId = requestId;
|
|
10
|
-
if (sessionId)
|
|
11
|
-
contextMeta.sessionId = sessionId;
|
|
12
|
-
if (operationId)
|
|
13
|
-
contextMeta.operationId = operationId;
|
|
14
|
-
const merged = { ...contextMeta, ...meta };
|
|
15
|
-
return Object.keys(merged).length > 0 ? ` ${JSON.stringify(merged)}` : '';
|
|
16
|
-
}
|
|
17
|
-
function createTimestamp() {
|
|
18
|
-
return new Date().toISOString();
|
|
19
|
-
}
|
|
20
|
-
function formatLogEntry(level, message, meta) {
|
|
21
|
-
return `[${createTimestamp()}] ${level.toUpperCase()}: ${message}${formatMetadata(meta)}`;
|
|
22
|
-
}
|
|
23
|
-
function shouldLog(level) {
|
|
24
|
-
// Debug logs only when LOG_LEVEL=debug
|
|
25
|
-
if (level === 'debug')
|
|
26
|
-
return config.logging.level === 'debug';
|
|
27
|
-
// All other levels always log
|
|
28
|
-
return true;
|
|
29
|
-
}
|
|
30
|
-
export function logInfo(message, meta) {
|
|
31
|
-
if (shouldLog('info')) {
|
|
32
|
-
process.stderr.write(`${formatLogEntry('info', message, meta)}\n`);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
export function logDebug(message, meta) {
|
|
36
|
-
if (shouldLog('debug')) {
|
|
37
|
-
process.stderr.write(`${formatLogEntry('debug', message, meta)}\n`);
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
export function logWarn(message, meta) {
|
|
41
|
-
if (shouldLog('warn')) {
|
|
42
|
-
process.stderr.write(`${formatLogEntry('warn', message, meta)}\n`);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
export function logError(message, error) {
|
|
46
|
-
if (!shouldLog('error'))
|
|
47
|
-
return;
|
|
48
|
-
const errorMeta = error instanceof Error
|
|
49
|
-
? { error: error.message, stack: error.stack }
|
|
50
|
-
: (error ?? {});
|
|
51
|
-
process.stderr.write(`${formatLogEntry('error', message, errorMeta)}\n`);
|
|
52
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,SAAS,cAAc,CAAC,IAAkB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IACjD,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IAEjD,MAAM,MAAM,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,OAAe,EACf,IAAkB;IAElB,OAAO,IAAI,eAAe,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,IAAkB;IAC1D,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,KAA2B;IACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC"}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
export declare const logInfo: (message: string, meta?: Record<string, unknown>) => void;
|
|
2
|
-
export declare const logWarn: (message: string, meta?: Record<string, unknown>) => void;
|
|
3
|
-
export declare const logDebug: (message: string, meta?: Record<string, unknown>) => void;
|
|
4
|
-
export declare const logError: (message: string, error?: Error | Record<string, unknown>) => void;
|
|
5
|
-
//# sourceMappingURL=logger.service.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"logger.service.d.ts","sourceRoot":"","sources":["../../src/services/logger.service.ts"],"names":[],"mappings":"AA+CA,eAAO,MAAM,OAAO,GAAI,SAAS,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAItE,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,SAAS,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAItE,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,SAAS,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAIvE,CAAC;AAEF,eAAO,MAAM,QAAQ,GACnB,SAAS,MAAM,EACf,QAAQ,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SASxC,CAAC"}
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
import winston from 'winston';
|
|
2
|
-
import { config } from '../config/index.js';
|
|
3
|
-
import path from 'path';
|
|
4
|
-
// Create logs directory path
|
|
5
|
-
const logsDir = path.join(process.cwd(), 'logs');
|
|
6
|
-
// Create logger instance
|
|
7
|
-
const logger = winston.createLogger({
|
|
8
|
-
level: config.logging.level,
|
|
9
|
-
format: winston.format.combine(winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), winston.format.errors({ stack: true }), winston.format.splat(), winston.format.json()),
|
|
10
|
-
defaultMeta: { service: 'superfetch' },
|
|
11
|
-
transports: [
|
|
12
|
-
// Write all logs to combined.log
|
|
13
|
-
new winston.transports.File({
|
|
14
|
-
filename: path.join(logsDir, 'combined.log'),
|
|
15
|
-
maxsize: 5242880, // 5MB
|
|
16
|
-
maxFiles: 5,
|
|
17
|
-
}),
|
|
18
|
-
// Write error logs to error.log
|
|
19
|
-
new winston.transports.File({
|
|
20
|
-
filename: path.join(logsDir, 'error.log'),
|
|
21
|
-
level: 'error',
|
|
22
|
-
maxsize: 5242880, // 5MB
|
|
23
|
-
maxFiles: 5,
|
|
24
|
-
}),
|
|
25
|
-
],
|
|
26
|
-
});
|
|
27
|
-
// Add console transport in development
|
|
28
|
-
if (process.env.NODE_ENV !== 'production') {
|
|
29
|
-
logger.add(new winston.transports.Console({
|
|
30
|
-
format: winston.format.combine(winston.format.colorize(), winston.format.simple()),
|
|
31
|
-
}));
|
|
32
|
-
}
|
|
33
|
-
// Export convenience methods
|
|
34
|
-
export const logInfo = (message, meta) => {
|
|
35
|
-
if (config.logging.enabled) {
|
|
36
|
-
logger.info(message, meta);
|
|
37
|
-
}
|
|
38
|
-
};
|
|
39
|
-
export const logWarn = (message, meta) => {
|
|
40
|
-
if (config.logging.enabled) {
|
|
41
|
-
logger.warn(message, meta);
|
|
42
|
-
}
|
|
43
|
-
};
|
|
44
|
-
export const logDebug = (message, meta) => {
|
|
45
|
-
if (config.logging.enabled) {
|
|
46
|
-
logger.debug(message, meta);
|
|
47
|
-
}
|
|
48
|
-
};
|
|
49
|
-
export const logError = (message, error) => {
|
|
50
|
-
if (config.logging.enabled) {
|
|
51
|
-
const errorMeta = error instanceof Error
|
|
52
|
-
? { error: error.message, stack: error.stack }
|
|
53
|
-
: error;
|
|
54
|
-
logger.error(message, errorMeta);
|
|
55
|
-
}
|
|
56
|
-
};
|
|
57
|
-
//# sourceMappingURL=logger.service.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"logger.service.js","sourceRoot":"","sources":["../../src/services/logger.service.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,6BAA6B;AAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;AAEjD,yBAAyB;AACzB,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK;IAC3B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAC5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,EAC3D,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EACtB,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CACtB;IACD,WAAW,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE;IACtC,UAAU,EAAE;QACV,iCAAiC;QACjC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;YAC1B,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC;YAC5C,OAAO,EAAE,OAAO,EAAE,MAAM;YACxB,QAAQ,EAAE,CAAC;SACZ,CAAC;QACF,gCAAgC;QAChC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;YAC1B,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC;YACzC,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,OAAO,EAAE,MAAM;YACxB,QAAQ,EAAE,CAAC;SACZ,CAAC;KACH;CACF,CAAC,CAAC;AAEH,uCAAuC;AACvC,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;IAC1C,MAAM,CAAC,GAAG,CACR,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;QAC7B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAC5B,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,EACzB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CACxB;KACF,CAAC,CACH,CAAC;AACJ,CAAC;AAED,6BAA6B;AAC7B,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,IAA8B,EAAE,EAAE;IACzE,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,IAA8B,EAAE,EAAE;IACzE,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,IAA8B,EAAE,EAAE;IAC1E,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,OAAe,EACf,KAAuC,EACvC,EAAE;IACF,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;YACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;YAC9C,CAAC,CAAC,KAAK,CAAC;QACZ,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
function createMetaCollectorState() {
|
|
2
|
-
return {
|
|
3
|
-
title: {},
|
|
4
|
-
description: {},
|
|
5
|
-
author: {},
|
|
6
|
-
};
|
|
7
|
-
}
|
|
8
|
-
function resolveMetaField(state, field) {
|
|
9
|
-
const sources = state[field];
|
|
10
|
-
return sources.og ?? sources.twitter ?? sources.standard;
|
|
11
|
-
}
|
|
12
|
-
function parseOpenGraphKey(property) {
|
|
13
|
-
if (!property?.startsWith('og:'))
|
|
14
|
-
return null;
|
|
15
|
-
const key = property.replace('og:', '');
|
|
16
|
-
return key === 'title' || key === 'description' ? key : null;
|
|
17
|
-
}
|
|
18
|
-
function parseTwitterKey(name) {
|
|
19
|
-
if (!name?.startsWith('twitter:'))
|
|
20
|
-
return null;
|
|
21
|
-
const key = name.replace('twitter:', '');
|
|
22
|
-
return key === 'title' || key === 'description' ? key : null;
|
|
23
|
-
}
|
|
24
|
-
function parseStandardKey(name) {
|
|
25
|
-
if (name === 'description')
|
|
26
|
-
return 'description';
|
|
27
|
-
if (name === 'author')
|
|
28
|
-
return 'author';
|
|
29
|
-
return null;
|
|
30
|
-
}
|
|
31
|
-
function collectMetaTag(state, tag) {
|
|
32
|
-
const content = tag.getAttribute('content')?.trim();
|
|
33
|
-
if (!content)
|
|
34
|
-
return;
|
|
35
|
-
const ogKey = parseOpenGraphKey(tag.getAttribute('property'));
|
|
36
|
-
if (ogKey) {
|
|
37
|
-
state[ogKey].og = content;
|
|
38
|
-
return;
|
|
39
|
-
}
|
|
40
|
-
const name = tag.getAttribute('name');
|
|
41
|
-
const twitterKey = parseTwitterKey(name);
|
|
42
|
-
if (twitterKey) {
|
|
43
|
-
state[twitterKey].twitter = content;
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
const standardKey = parseStandardKey(name);
|
|
47
|
-
if (standardKey) {
|
|
48
|
-
state[standardKey].standard = content;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
function scanMetaTags(document, state) {
|
|
52
|
-
const metaTags = document.querySelectorAll('meta');
|
|
53
|
-
for (const tag of metaTags) {
|
|
54
|
-
collectMetaTag(state, tag);
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
function ensureTitleFallback(document, state) {
|
|
58
|
-
if (state.title.standard)
|
|
59
|
-
return;
|
|
60
|
-
const titleEl = document.querySelector('title');
|
|
61
|
-
if (titleEl?.textContent) {
|
|
62
|
-
state.title.standard = titleEl.textContent.trim();
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
export function extractMetadata(document) {
|
|
66
|
-
const state = createMetaCollectorState();
|
|
67
|
-
scanMetaTags(document, state);
|
|
68
|
-
ensureTitleFallback(document, state);
|
|
69
|
-
const metadata = {};
|
|
70
|
-
const title = resolveMetaField(state, 'title');
|
|
71
|
-
const description = resolveMetaField(state, 'description');
|
|
72
|
-
const author = resolveMetaField(state, 'author');
|
|
73
|
-
if (title !== undefined)
|
|
74
|
-
metadata.title = title;
|
|
75
|
-
if (description !== undefined)
|
|
76
|
-
metadata.description = description;
|
|
77
|
-
if (author !== undefined)
|
|
78
|
-
metadata.author = author;
|
|
79
|
-
return metadata;
|
|
80
|
-
}
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { ContentBlockUnion, ExtractedMetadata } from '../config/types/content.js';
|
|
2
|
-
export declare function parseHtml(html: string): ContentBlockUnion[];
|
|
3
|
-
export declare function parseHtmlWithMetadata(html: string): {
|
|
4
|
-
blocks: ContentBlockUnion[];
|
|
5
|
-
metadata: ExtractedMetadata;
|
|
6
|
-
};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,4BAA4B,CAAC;AAoPpC,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAS3D"}
|