@j0hanz/superfetch 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -204
- package/dist/config/constants.d.ts +0 -1
- package/dist/config/constants.js +0 -1
- package/dist/config/formatting.d.ts +0 -1
- package/dist/config/formatting.d.ts.map +1 -1
- package/dist/config/formatting.js +1 -3
- package/dist/config/formatting.js.map +1 -1
- package/dist/config/index.d.ts +1 -2
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +5 -11
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +0 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/runtime.d.ts +0 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/tools.d.ts +0 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/errors/app-error.d.ts +0 -1
- package/dist/errors/app-error.js +0 -1
- package/dist/http/auth.d.ts +0 -1
- package/dist/http/auth.js +0 -1
- package/dist/http/cors.d.ts +0 -1
- package/dist/http/cors.js +0 -1
- package/dist/http/download-routes.d.ts +1 -2
- package/dist/http/download-routes.d.ts.map +1 -1
- package/dist/http/download-routes.js +2 -2
- package/dist/http/mcp-routes.d.ts +0 -1
- package/dist/http/mcp-routes.js +0 -1
- package/dist/http/mcp-session-helpers.d.ts +0 -1
- package/dist/http/mcp-session-helpers.js +0 -1
- package/dist/http/mcp-session.d.ts +1 -2
- package/dist/http/mcp-session.d.ts.map +1 -1
- package/dist/http/mcp-session.js +0 -1
- package/dist/http/mcp-validation.d.ts +1 -2
- package/dist/http/mcp-validation.d.ts.map +1 -1
- package/dist/http/mcp-validation.js +6 -27
- package/dist/http/mcp-validation.js.map +1 -1
- package/dist/http/rate-limit.d.ts +1 -2
- package/dist/http/rate-limit.d.ts.map +1 -1
- package/dist/http/rate-limit.js +0 -1
- package/dist/http/rate-limit.js.map +1 -1
- package/dist/http/server-middleware.d.ts +0 -1
- package/dist/http/server-middleware.d.ts.map +1 -1
- package/dist/http/server-middleware.js +60 -4
- package/dist/http/server-middleware.js.map +1 -1
- package/dist/http/server.d.ts +0 -1
- package/dist/http/server.js +0 -1
- package/dist/http/session-cleanup.d.ts +0 -1
- package/dist/http/session-cleanup.js +0 -1
- package/dist/http/sessions.d.ts +1 -2
- package/dist/http/sessions.d.ts.map +1 -1
- package/dist/http/sessions.js +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/middleware/error-handler.d.ts +0 -1
- package/dist/middleware/error-handler.js +0 -1
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.js +0 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/index.js +0 -1
- package/dist/server.d.ts +0 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +8 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +2 -2
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +4 -2
- package/dist/services/context.d.ts +2 -1
- package/dist/services/context.js +10 -1
- package/dist/services/extractor.d.ts +1 -2
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +0 -1
- package/dist/services/fetcher/agents.d.ts +0 -1
- package/dist/services/fetcher/agents.js +55 -2
- package/dist/services/fetcher/errors.d.ts +0 -1
- package/dist/services/fetcher/errors.js +0 -1
- package/dist/services/fetcher/interceptors.d.ts +0 -1
- package/dist/services/fetcher/interceptors.js +44 -24
- package/dist/services/fetcher/redirects.d.ts +0 -1
- package/dist/services/fetcher/redirects.js +2 -3
- package/dist/services/fetcher/response.d.ts +1 -2
- package/dist/services/fetcher/response.js +28 -16
- package/dist/services/fetcher/retry-policy.d.ts +0 -1
- package/dist/services/fetcher/retry-policy.js +0 -1
- package/dist/services/fetcher.d.ts +2 -2
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +9 -7
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +1 -2
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +0 -1
- package/dist/services/parser.d.ts +5 -2
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +72 -3
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +34 -3
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +5 -3
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.js +8 -4
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +36 -3
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.js +0 -1
- package/dist/tools/schemas.d.ts +32 -33
- package/dist/tools/schemas.js +21 -5
- package/dist/tools/utils/cache-vary.d.ts +0 -1
- package/dist/tools/utils/cache-vary.d.ts.map +1 -1
- package/dist/tools/utils/cache-vary.js +7 -25
- package/dist/tools/utils/cache-vary.js.map +1 -1
- package/dist/tools/utils/common.d.ts +2 -3
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +6 -7
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +1 -3
- package/dist/tools/utils/content-transform.d.ts.map +1 -1
- package/dist/tools/utils/content-transform.js +73 -8
- package/dist/tools/utils/content-transform.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +1 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +18 -22
- package/dist/tools/utils/inline-content.d.ts +0 -1
- package/dist/tools/utils/inline-content.js +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +0 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -2
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +8 -1
- package/dist/utils/code-language.d.ts +0 -1
- package/dist/utils/code-language.js +0 -1
- package/dist/utils/content-cleaner.d.ts +0 -1
- package/dist/utils/content-cleaner.js +0 -1
- package/dist/utils/crypto.d.ts +0 -1
- package/dist/utils/crypto.js +0 -1
- package/dist/utils/download-url.d.ts +1 -2
- package/dist/utils/download-url.d.ts.map +1 -1
- package/dist/utils/download-url.js +0 -1
- package/dist/utils/error-utils.d.ts +0 -1
- package/dist/utils/error-utils.js +1 -3
- package/dist/utils/filename-generator.d.ts +0 -1
- package/dist/utils/filename-generator.js +0 -1
- package/dist/utils/header-normalizer.d.ts +0 -1
- package/dist/utils/header-normalizer.js +0 -1
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.js +0 -1
- package/dist/utils/sanitizer.d.ts +0 -1
- package/dist/utils/sanitizer.js +0 -1
- package/dist/utils/tool-error-handler.d.ts +1 -2
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +0 -1
- package/dist/utils/url-validator.d.ts +6 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +49 -11
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +4 -4
package/dist/http/rate-limit.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rate-limit.js","sourceRoot":"","sources":["../../src/http/rate-limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,IAAI,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"rate-limit.js","sourceRoot":"","sources":["../../src/http/rate-limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,IAAI,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAmBzE,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa,IAAI,SAAS,CAAC;AACzD,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAkC,EAClC,OAAwB;IAExB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IAEzC,KAAK,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,CAC5D,kBAAkB,CACnB,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,OAAwB;IAExB,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,GAAS,EAAE;QACtB,iBAAiB,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,CACjB,GAAY,EACZ,GAAa,EACb,IAAkB,EACZ,EAAE;QACR,IAAI,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACtC,IAAI,EAAE,CAAC;YACP,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,qBAAqB,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAEnE,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,EAAE,CAAC;YACP,OAAO;QACT,CAAC;QAED,IAAI,uBAAuB,CAAC,GAAG,EAAE,UAAU,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACjE,OAAO;QACT,CAAC;QAED,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;IAEF,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACrC,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,KAAkC,EAClC,OAAwB,EACxB,MAAmB;IAEnB,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,kBAAkB,CACtC,OAAO,CAAC,iBAAiB,EACzB,SAAS,EACT,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CACvB,EAAE,CAAC;QACF,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAkC,EAClC,OAAwB,EACxB,GAAW;IAEX,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,IAAI,GAAG,GAAG,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpD,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;AAC/D,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;IACT,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAY,EAAE,OAAwB;IACjE,OAAO,CAAC,OAAO,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAkC,EAClC,GAAW,EACX,GAAW,EACX,OAAwB;IAExB,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3C,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC3B,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,OAAwB;IAC3D,OAAO;QACL,KAAK,EAAE,CAAC;QACR,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,QAAQ;QACjC,YAAY,EAAE,GAAG;KAClB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,KAAqB,EAAE,GAAW;IACrD,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IACjB,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;AAC3B,CAAC;AAED,SAAS,uBAAuB,CAC9B,GAAa,EACb,KAAqB,EACrB,GAAW,EACX,OAAwB;IAExB,IAAI,KAAK,CAAC,KAAK,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACvC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3C,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;QACnB,KAAK,EAAE,qBAAqB;QAC5B,UAAU;KACX,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -7,4 +7,3 @@ export declare function createJsonParseErrorHandler(): (err: Error, _req: Reques
|
|
|
7
7
|
export declare function createContextMiddleware(): (req: Request, _res: Response, next: NextFunction) => void;
|
|
8
8
|
export declare function registerHealthRoute(app: Express): void;
|
|
9
9
|
export declare function attachBaseMiddleware(app: Express, jsonParser: RequestHandler, rateLimitMiddleware: RequestHandler, authMiddleware: RequestHandler, corsMiddleware: RequestHandler): void;
|
|
10
|
-
//# sourceMappingURL=server-middleware.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server-middleware.d.ts","sourceRoot":"","sources":["../../src/http/server-middleware.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,OAAO,EACP,YAAY,EACZ,OAAO,EACP,cAAc,EACd,QAAQ,EACT,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"server-middleware.d.ts","sourceRoot":"","sources":["../../src/http/server-middleware.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,OAAO,EACP,YAAY,EACZ,OAAO,EACP,cAAc,EACd,QAAQ,EACT,MAAM,SAAS,CAAC;AA0DjB,wBAAgB,8BAA8B,IAAI,cAAc,CAmB/D;AAED,wBAAgB,gBAAgB,IAAI;IAClC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;CAC1B,CAMA;AAED,wBAAgB,2BAA2B,IAAI,CAC7C,GAAG,EAAE,KAAK,EACV,IAAI,EAAE,OAAO,EACb,GAAG,EAAE,QAAQ,EACb,IAAI,EAAE,YAAY,KACf,IAAI,CAoBR;AAED,wBAAgB,uBAAuB,IAAI,CACzC,GAAG,EAAE,OAAO,EACZ,IAAI,EAAE,QAAQ,EACd,IAAI,EAAE,YAAY,KACf,IAAI,CASR;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAStD;AAED,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,OAAO,EACZ,UAAU,EAAE,cAAc,EAC1B,mBAAmB,EAAE,cAAc,EACnC,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,cAAc,GAC7B,IAAI,CASN"}
|
|
@@ -1,7 +1,62 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { config } from '../config/index.js';
|
|
3
|
-
import {
|
|
3
|
+
import { bindToRequestContext, runWithRequestContext, } from '../services/context.js';
|
|
4
4
|
import { getSessionId } from './sessions.js';
|
|
5
|
+
const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1']);
|
|
6
|
+
function normalizeHost(value) {
|
|
7
|
+
const trimmed = value.trim().toLowerCase();
|
|
8
|
+
if (!trimmed)
|
|
9
|
+
return null;
|
|
10
|
+
const first = trimmed.split(',')[0]?.trim();
|
|
11
|
+
if (!first)
|
|
12
|
+
return null;
|
|
13
|
+
if (first.startsWith('[')) {
|
|
14
|
+
const end = first.indexOf(']');
|
|
15
|
+
if (end === -1)
|
|
16
|
+
return null;
|
|
17
|
+
return first.slice(1, end);
|
|
18
|
+
}
|
|
19
|
+
const colonIndex = first.indexOf(':');
|
|
20
|
+
if (colonIndex !== -1) {
|
|
21
|
+
return first.slice(0, colonIndex);
|
|
22
|
+
}
|
|
23
|
+
return first;
|
|
24
|
+
}
|
|
25
|
+
function buildAllowedHosts() {
|
|
26
|
+
const allowedHosts = new Set();
|
|
27
|
+
const raw = process.env.ALLOWED_HOSTS ?? '';
|
|
28
|
+
for (const entry of raw.split(',')) {
|
|
29
|
+
const normalized = normalizeHost(entry);
|
|
30
|
+
if (normalized) {
|
|
31
|
+
allowedHosts.add(normalized);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
for (const host of LOOPBACK_HOSTS) {
|
|
35
|
+
allowedHosts.add(host);
|
|
36
|
+
}
|
|
37
|
+
const configuredHost = normalizeHost(config.server.host);
|
|
38
|
+
if (configuredHost &&
|
|
39
|
+
configuredHost !== '0.0.0.0' &&
|
|
40
|
+
configuredHost !== '::') {
|
|
41
|
+
allowedHosts.add(configuredHost);
|
|
42
|
+
}
|
|
43
|
+
return allowedHosts;
|
|
44
|
+
}
|
|
45
|
+
function createHostValidationMiddleware() {
|
|
46
|
+
const allowedHosts = buildAllowedHosts();
|
|
47
|
+
return (req, res, next) => {
|
|
48
|
+
const hostHeader = typeof req.headers.host === 'string' ? req.headers.host : '';
|
|
49
|
+
const normalized = normalizeHost(hostHeader);
|
|
50
|
+
if (!normalized || !allowedHosts.has(normalized)) {
|
|
51
|
+
res.status(403).json({
|
|
52
|
+
error: 'Host not allowed',
|
|
53
|
+
code: 'HOST_NOT_ALLOWED',
|
|
54
|
+
});
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
next();
|
|
58
|
+
};
|
|
59
|
+
}
|
|
5
60
|
export function buildCorsOptions() {
|
|
6
61
|
const allowedOrigins = process.env.ALLOWED_ORIGINS
|
|
7
62
|
? process.env.ALLOWED_ORIGINS.split(',').map((o) => o.trim())
|
|
@@ -29,8 +84,9 @@ export function createContextMiddleware() {
|
|
|
29
84
|
return (req, _res, next) => {
|
|
30
85
|
const requestId = randomUUID();
|
|
31
86
|
const sessionId = getSessionId(req);
|
|
32
|
-
|
|
33
|
-
next
|
|
87
|
+
runWithRequestContext({ requestId, sessionId }, () => {
|
|
88
|
+
const boundNext = bindToRequestContext(next);
|
|
89
|
+
boundNext();
|
|
34
90
|
});
|
|
35
91
|
};
|
|
36
92
|
}
|
|
@@ -45,6 +101,7 @@ export function registerHealthRoute(app) {
|
|
|
45
101
|
});
|
|
46
102
|
}
|
|
47
103
|
export function attachBaseMiddleware(app, jsonParser, rateLimitMiddleware, authMiddleware, corsMiddleware) {
|
|
104
|
+
app.use(createHostValidationMiddleware());
|
|
48
105
|
app.use(jsonParser);
|
|
49
106
|
app.use(createContextMiddleware());
|
|
50
107
|
app.use(createJsonParseErrorHandler());
|
|
@@ -53,4 +110,3 @@ export function attachBaseMiddleware(app, jsonParser, rateLimitMiddleware, authM
|
|
|
53
110
|
app.use(authMiddleware);
|
|
54
111
|
registerHealthRoute(app);
|
|
55
112
|
}
|
|
56
|
-
//# sourceMappingURL=server-middleware.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server-middleware.js","sourceRoot":"","sources":["../../src/http/server-middleware.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAUzC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,UAAU,gBAAgB;IAI9B,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe;QAChD,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,MAAM,CAAC;IAC9D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,2BAA2B;IAMzC,OAAO,CACL,GAAU,EACV,IAAa,EACb,GAAa,EACb,IAAkB,EACZ,EAAE;QACR,IAAI,GAAG,YAAY,WAAW,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YAChD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE;oBACL,IAAI,EAAE,CAAC,KAAK;oBACZ,OAAO,EAAE,2BAA2B;iBACrC;gBACD,EAAE,EAAE,IAAI;aACT,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB;IAKrC,OAAO,CAAC,GAAY,EAAE,IAAc,EAAE,IAAkB,EAAQ,EAAE;QAChE,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAEpC,cAAc,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE;YAChD,IAAI,EAAE,CAAC;QACT,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC9C,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,IAAI,CAAC;YACP,MAAM,EAAE,SAAS;YACjB,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;YACxB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;SACzB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,GAAY,EACZ,UAA0B,EAC1B,mBAAmC,EACnC,cAA8B,EAC9B,cAA8B;IAE9B,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACpB,GAAG,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACnC,GAAG,CAAC,GAAG,CAAC,2BAA2B,EAAE,CAAC,CAAC;IACvC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACxB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;IACrC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACxB,mBAAmB,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC"}
|
|
1
|
+
{"version":3,"file":"server-middleware.js","sourceRoot":"","sources":["../../src/http/server-middleware.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAUzC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;AAElE,SAAS,aAAa,CAAC,KAAa;IAClC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;IAC5C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB;IACxB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;IAE5C,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzD,IACE,cAAc;QACd,cAAc,KAAK,SAAS;QAC5B,cAAc,KAAK,IAAI,EACvB,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,8BAA8B;IAC5C,MAAM,YAAY,GAAG,iBAAiB,EAAE,CAAC;IAEzC,OAAO,CAAC,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAQ,EAAE;QAC/D,MAAM,UAAU,GACd,OAAO,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAE/D,MAAM,UAAU,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;QAE7C,IAAI,CAAC,UAAU,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YACjD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,kBAAkB;gBACzB,IAAI,EAAE,kBAAkB;aACzB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB;IAI9B,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe;QAChD,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,MAAM,CAAC;IAC9D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,2BAA2B;IAMzC,OAAO,CACL,GAAU,EACV,IAAa,EACb,GAAa,EACb,IAAkB,EACZ,EAAE;QACR,IAAI,GAAG,YAAY,WAAW,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YAChD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE;oBACL,IAAI,EAAE,CAAC,KAAK;oBACZ,OAAO,EAAE,2BAA2B;iBACrC;gBACD,EAAE,EAAE,IAAI;aACT,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB;IAKrC,OAAO,CAAC,GAAY,EAAE,IAAc,EAAE,IAAkB,EAAQ,EAAE;QAChE,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAEpC,cAAc,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE;YAChD,IAAI,EAAE,CAAC;QACT,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC9C,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,IAAI,CAAC;YACP,MAAM,EAAE,SAAS;YACjB,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;YACxB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;SACzB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,GAAY,EACZ,UAA0B,EAC1B,mBAAmC,EACnC,cAA8B,EAC9B,cAA8B;IAE9B,GAAG,CAAC,GAAG,CAAC,8BAA8B,EAAE,CAAC,CAAC;IAC1C,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACpB,GAAG,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACnC,GAAG,CAAC,GAAG,CAAC,2BAA2B,EAAE,CAAC,CAAC;IACvC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACxB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;IACrC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACxB,mBAAmB,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC"}
|
package/dist/http/server.d.ts
CHANGED
package/dist/http/server.js
CHANGED
package/dist/http/sessions.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Request } from 'express';
|
|
2
|
-
import type { SessionEntry } from '../config/types.js';
|
|
2
|
+
import type { SessionEntry } from '../config/types/runtime.js';
|
|
3
3
|
export interface SessionStore {
|
|
4
4
|
get: (sessionId: string) => SessionEntry | undefined;
|
|
5
5
|
touch: (sessionId: string) => void;
|
|
@@ -12,4 +12,3 @@ export interface SessionStore {
|
|
|
12
12
|
}
|
|
13
13
|
export declare function getSessionId(req: Request): string | undefined;
|
|
14
14
|
export declare function createSessionStore(sessionTtlMs: number): SessionStore;
|
|
15
|
-
//# sourceMappingURL=sessions.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sessions.d.ts","sourceRoot":"","sources":["../../src/http/sessions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAEvC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"sessions.d.ts","sourceRoot":"","sources":["../../src/http/sessions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAEvC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,YAAY,GAAG,SAAS,CAAC;IACrD,KAAK,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,GAAG,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACtD,MAAM,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,YAAY,GAAG,SAAS,CAAC;IACxD,IAAI,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,YAAY,EAAE,CAAC;IAC5B,YAAY,EAAE,MAAM,YAAY,EAAE,CAAC;IACnC,WAAW,EAAE,MAAM,YAAY,GAAG,SAAS,CAAC;CAC7C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAG7D;AAED,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,YAAY,CAiBrE"}
|
package/dist/http/sessions.js
CHANGED
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
package/dist/resources/index.js
CHANGED
package/dist/server.d.ts
CHANGED
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAYpE,wBAAgB,eAAe,IAAI,SAAS,CAoB3C;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAYpE,wBAAgB,eAAe,IAAI,SAAS,CAoB3C;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CA2CtD"}
|
package/dist/server.js
CHANGED
|
@@ -27,8 +27,8 @@ export async function startStdioServer() {
|
|
|
27
27
|
server.server.onerror = (error) => {
|
|
28
28
|
logError('[MCP Error]', error instanceof Error ? error : { error });
|
|
29
29
|
};
|
|
30
|
-
|
|
31
|
-
process.stderr.write(
|
|
30
|
+
const handleShutdown = (signal) => {
|
|
31
|
+
process.stderr.write(`\n${signal} received, shutting down superFetch MCP server...\n`);
|
|
32
32
|
destroyAgents();
|
|
33
33
|
server
|
|
34
34
|
.close()
|
|
@@ -38,6 +38,12 @@ export async function startStdioServer() {
|
|
|
38
38
|
.finally(() => {
|
|
39
39
|
process.exit(0);
|
|
40
40
|
});
|
|
41
|
+
};
|
|
42
|
+
process.on('SIGINT', () => {
|
|
43
|
+
handleShutdown('SIGINT');
|
|
44
|
+
});
|
|
45
|
+
process.on('SIGTERM', () => {
|
|
46
|
+
handleShutdown('SIGTERM');
|
|
41
47
|
});
|
|
42
48
|
try {
|
|
43
49
|
await server.connect(transport);
|
|
@@ -48,4 +54,3 @@ export async function startStdioServer() {
|
|
|
48
54
|
process.exit(1);
|
|
49
55
|
}
|
|
50
56
|
}
|
|
51
|
-
//# sourceMappingURL=server.js.map
|
package/dist/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B;QACE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;QACxB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;KAC/B,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE;YAC7B,SAAS,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;YACjD,OAAO,EAAE,EAAE;SACZ;QACD,YAAY,EAAE,0BAA0B,MAAM,CAAC,MAAM,CAAC,OAAO,8OAA8O;KAC5S,CACF,CAAC;IAEF,aAAa,CAAC,MAAM,CAAC,CAAC;IACtB,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE1B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE7C,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;QAChC,QAAQ,CAAC,aAAa,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC,CAAC;IAEF,
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B;QACE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;QACxB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;KAC/B,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE;YAC7B,SAAS,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;YACjD,OAAO,EAAE,EAAE;SACZ;QACD,YAAY,EAAE,0BAA0B,MAAM,CAAC,MAAM,CAAC,OAAO,8OAA8O;KAC5S,CACF,CAAC;IAEF,aAAa,CAAC,MAAM,CAAC,CAAC;IACtB,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE1B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE7C,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;QAChC,QAAQ,CAAC,aAAa,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,CAAC,MAAc,EAAQ,EAAE;QAC9C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,MAAM,qDAAqD,CACjE,CAAC;QACF,aAAa,EAAE,CAAC;QAChB,MAAM;aACH,KAAK,EAAE;aACP,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,QAAQ,CACN,uBAAuB,EACvB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CACvC,CAAC;QACJ,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;IAEF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;QACzB,cAAc,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,CAAC,wCAAwC,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,8BAA8B,EAC9B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
|
package/dist/services/cache.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CacheEntry } from '../config/types.js';
|
|
1
|
+
import type { CacheEntry } from '../config/types/content.js';
|
|
2
2
|
interface CacheKeyParts {
|
|
3
3
|
namespace: string;
|
|
4
4
|
urlHash: string;
|
|
@@ -18,5 +18,5 @@ export declare function onCacheUpdate(listener: CacheUpdateListener): () => void
|
|
|
18
18
|
export declare function get(cacheKey: string | null): CacheEntry | undefined;
|
|
19
19
|
export declare function set(cacheKey: string | null, content: string, metadata: CacheEntryMetadata): void;
|
|
20
20
|
export declare function keys(): string[];
|
|
21
|
+
export declare function isEnabled(): boolean;
|
|
21
22
|
export {};
|
|
22
|
-
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAwD7D,UAAU,aAAa;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,gBAAiB,SAAQ,aAAa;IAC9C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,kBAAkB;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,mBAAmB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;AA2D7D,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GACtC,MAAM,GAAG,IAAI,CAMf;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAMpE;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAI7D;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,mBAAmB,GAAG,MAAM,IAAI,CAKvE;AAUD,wBAAgB,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,SAAS,CAYnE;AAsBD,wBAAgB,GAAG,CACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,kBAAkB,GAC3B,IAAI,CAeN;AAED,wBAAgB,IAAI,IAAI,MAAM,EAAE,CAE/B"}
|
package/dist/services/cache.js
CHANGED
|
@@ -17,7 +17,7 @@ function startCleanupLoop() {
|
|
|
17
17
|
});
|
|
18
18
|
}
|
|
19
19
|
async function runCleanupLoop(signal) {
|
|
20
|
-
const intervalMs = Math.floor(config.cache.ttl *
|
|
20
|
+
const intervalMs = Math.floor(config.cache.ttl * 1000);
|
|
21
21
|
for await (const _ of setIntervalPromise(intervalMs, undefined, {
|
|
22
22
|
signal,
|
|
23
23
|
ref: false,
|
|
@@ -175,6 +175,9 @@ export function set(cacheKey, content, metadata) {
|
|
|
175
175
|
export function keys() {
|
|
176
176
|
return Array.from(contentCache.keys());
|
|
177
177
|
}
|
|
178
|
+
export function isEnabled() {
|
|
179
|
+
return config.cache.enabled;
|
|
180
|
+
}
|
|
178
181
|
function buildCacheEntry(cacheKey, content, metadata) {
|
|
179
182
|
return {
|
|
180
183
|
url: metadata.url,
|
|
@@ -189,4 +192,3 @@ function persistCacheEntry(cacheKey, entry) {
|
|
|
189
192
|
contentCache.set(cacheKey, { entry, expiresAt });
|
|
190
193
|
emitCacheUpdate(cacheKey);
|
|
191
194
|
}
|
|
192
|
-
//# sourceMappingURL=cache.js.map
|
|
@@ -4,7 +4,8 @@ interface RequestContext {
|
|
|
4
4
|
sessionId?: string;
|
|
5
5
|
}
|
|
6
6
|
export declare const requestContext: AsyncLocalStorage<RequestContext>;
|
|
7
|
+
export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
|
|
8
|
+
export declare function bindToRequestContext<T extends (...args: unknown[]) => unknown>(fn: T): T;
|
|
7
9
|
export declare function getRequestId(): string | undefined;
|
|
8
10
|
export declare function getSessionId(): string | undefined;
|
|
9
11
|
export {};
|
|
10
|
-
//# sourceMappingURL=context.d.ts.map
|
package/dist/services/context.js
CHANGED
|
@@ -1,9 +1,18 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
2
|
export const requestContext = new AsyncLocalStorage();
|
|
3
|
+
export function runWithRequestContext(context, fn) {
|
|
4
|
+
return requestContext.run(context, fn);
|
|
5
|
+
}
|
|
6
|
+
export function bindToRequestContext(fn) {
|
|
7
|
+
const store = requestContext.getStore();
|
|
8
|
+
if (!store) {
|
|
9
|
+
return fn;
|
|
10
|
+
}
|
|
11
|
+
return ((...args) => requestContext.run(store, () => fn(...args)));
|
|
12
|
+
}
|
|
3
13
|
export function getRequestId() {
|
|
4
14
|
return requestContext.getStore()?.requestId;
|
|
5
15
|
}
|
|
6
16
|
export function getSessionId() {
|
|
7
17
|
return requestContext.getStore()?.sessionId;
|
|
8
18
|
}
|
|
9
|
-
//# sourceMappingURL=context.js.map
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type { ExtractionResult } from '../config/types.js';
|
|
1
|
+
import type { ExtractionResult } from '../config/types/content.js';
|
|
2
2
|
export declare function extractContent(html: string, url: string, options?: {
|
|
3
3
|
extractArticle?: boolean;
|
|
4
4
|
}): ExtractionResult;
|
|
5
|
-
//# sourceMappingURL=extractor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAGV,gBAAgB,EACjB,MAAM,
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAGV,gBAAgB,EACjB,MAAM,4BAA4B,CAAC;AAyKpC,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAA6B,GAC/D,gBAAgB,CAMlB"}
|
|
@@ -3,9 +3,26 @@ import os from 'node:os';
|
|
|
3
3
|
import { Agent } from 'undici';
|
|
4
4
|
import { createErrorWithCode } from '../../utils/error-utils.js';
|
|
5
5
|
import { isBlockedIp } from '../../utils/url-validator.js';
|
|
6
|
+
const DNS_LOOKUP_TIMEOUT_MS = 5000;
|
|
6
7
|
function resolveDns(hostname, options, callback) {
|
|
7
8
|
const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
|
|
8
|
-
|
|
9
|
+
const lookupOptions = buildLookupOptions(normalizedOptions);
|
|
10
|
+
let done = false;
|
|
11
|
+
const timer = setTimeout(() => {
|
|
12
|
+
if (done)
|
|
13
|
+
return;
|
|
14
|
+
done = true;
|
|
15
|
+
callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
|
|
16
|
+
}, DNS_LOOKUP_TIMEOUT_MS);
|
|
17
|
+
timer.unref();
|
|
18
|
+
const safeCallback = (err, address, family) => {
|
|
19
|
+
if (done)
|
|
20
|
+
return;
|
|
21
|
+
done = true;
|
|
22
|
+
clearTimeout(timer);
|
|
23
|
+
callback(err, address, family);
|
|
24
|
+
};
|
|
25
|
+
dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
|
|
9
26
|
}
|
|
10
27
|
function normalizeLookupOptions(options) {
|
|
11
28
|
return typeof options === 'number' ? { family: options } : options;
|
|
@@ -18,6 +35,29 @@ function buildLookupContext(options) {
|
|
|
18
35
|
resolvedFamily: resolveFamily(normalizedOptions.family),
|
|
19
36
|
};
|
|
20
37
|
}
|
|
38
|
+
const DEFAULT_DNS_ORDER = 'verbatim';
|
|
39
|
+
function resolveResultOrder(options) {
|
|
40
|
+
if (options.order)
|
|
41
|
+
return options.order;
|
|
42
|
+
const legacyVerbatim = getLegacyVerbatim(options);
|
|
43
|
+
if (legacyVerbatim !== undefined) {
|
|
44
|
+
return legacyVerbatim ? 'verbatim' : 'ipv4first';
|
|
45
|
+
}
|
|
46
|
+
return DEFAULT_DNS_ORDER;
|
|
47
|
+
}
|
|
48
|
+
function getLegacyVerbatim(options) {
|
|
49
|
+
const legacy = options.verbatim;
|
|
50
|
+
return typeof legacy === 'boolean' ? legacy : undefined;
|
|
51
|
+
}
|
|
52
|
+
function buildLookupOptions(normalizedOptions) {
|
|
53
|
+
const options = {
|
|
54
|
+
...normalizedOptions,
|
|
55
|
+
order: resolveResultOrder(normalizedOptions),
|
|
56
|
+
all: true,
|
|
57
|
+
};
|
|
58
|
+
delete options.verbatim;
|
|
59
|
+
return options;
|
|
60
|
+
}
|
|
21
61
|
function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
|
|
22
62
|
return (err, addresses) => {
|
|
23
63
|
handleLookupResult(err, addresses, hostname, resolvedFamily, useAll, callback);
|
|
@@ -42,6 +82,11 @@ function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll,
|
|
|
42
82
|
return;
|
|
43
83
|
}
|
|
44
84
|
const list = normalizeLookupResults(addresses, resolvedFamily);
|
|
85
|
+
const invalidFamilyError = findInvalidFamilyError(list, hostname);
|
|
86
|
+
if (invalidFamilyError) {
|
|
87
|
+
callback(invalidFamilyError, list);
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
45
90
|
const blockedError = findBlockedIpError(list, hostname);
|
|
46
91
|
if (blockedError) {
|
|
47
92
|
callback(blockedError, list);
|
|
@@ -89,6 +134,15 @@ function findBlockedIpError(list, hostname) {
|
|
|
89
134
|
}
|
|
90
135
|
return null;
|
|
91
136
|
}
|
|
137
|
+
function findInvalidFamilyError(list, hostname) {
|
|
138
|
+
for (const addr of list) {
|
|
139
|
+
const family = typeof addr === 'string' ? 0 : addr.family;
|
|
140
|
+
if (family === 4 || family === 6)
|
|
141
|
+
continue;
|
|
142
|
+
return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
|
|
143
|
+
}
|
|
144
|
+
return null;
|
|
145
|
+
}
|
|
92
146
|
function createNoDnsResultsError(hostname) {
|
|
93
147
|
return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
|
|
94
148
|
}
|
|
@@ -105,4 +159,3 @@ export const dispatcher = new Agent(getAgentOptions());
|
|
|
105
159
|
export function destroyAgents() {
|
|
106
160
|
void dispatcher.close();
|
|
107
161
|
}
|
|
108
|
-
//# sourceMappingURL=agents.js.map
|
|
@@ -2,4 +2,3 @@ import { FetchError } from '../../errors/app-error.js';
|
|
|
2
2
|
export declare function createRateLimitError(url: string, headerValue: string | null): FetchError;
|
|
3
3
|
export declare function createHttpError(url: string, status: number, statusText: string): FetchError;
|
|
4
4
|
export declare function mapFetchError(error: unknown, fallbackUrl: string, timeoutMs: number): FetchError;
|
|
5
|
-
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -8,4 +8,3 @@ export declare function startFetchTelemetry(url: string, method: string): FetchT
|
|
|
8
8
|
export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
|
|
9
9
|
export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
|
|
10
10
|
export {};
|
|
11
|
-
//# sourceMappingURL=interceptors.d.ts.map
|
|
@@ -4,21 +4,44 @@ import { performance } from 'node:perf_hooks';
|
|
|
4
4
|
import { isSystemError } from '../../utils/error-utils.js';
|
|
5
5
|
import { logDebug, logError, logWarn } from '../logger.js';
|
|
6
6
|
const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
|
|
7
|
+
function redactUrl(rawUrl) {
|
|
8
|
+
try {
|
|
9
|
+
const url = new URL(rawUrl);
|
|
10
|
+
url.username = '';
|
|
11
|
+
url.password = '';
|
|
12
|
+
url.hash = '';
|
|
13
|
+
url.search = '';
|
|
14
|
+
return url.toString();
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return rawUrl;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function publishFetchEvent(event) {
|
|
21
|
+
if (!fetchChannel.hasSubscribers)
|
|
22
|
+
return;
|
|
23
|
+
try {
|
|
24
|
+
fetchChannel.publish(event);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
// Avoid crashing the publisher if a subscriber throws.
|
|
28
|
+
}
|
|
29
|
+
}
|
|
7
30
|
export function startFetchTelemetry(url, method) {
|
|
31
|
+
const safeUrl = redactUrl(url);
|
|
8
32
|
const context = {
|
|
9
33
|
requestId: randomUUID(),
|
|
10
34
|
startTime: performance.now(),
|
|
11
|
-
url,
|
|
35
|
+
url: safeUrl,
|
|
12
36
|
method: method.toUpperCase(),
|
|
13
37
|
};
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
}
|
|
38
|
+
publishFetchEvent({
|
|
39
|
+
v: 1,
|
|
40
|
+
type: 'start',
|
|
41
|
+
requestId: context.requestId,
|
|
42
|
+
method: context.method,
|
|
43
|
+
url: context.url,
|
|
44
|
+
});
|
|
22
45
|
logDebug('HTTP Request', {
|
|
23
46
|
requestId: context.requestId,
|
|
24
47
|
method: context.method,
|
|
@@ -38,9 +61,8 @@ export function recordFetchResponse(context, response, contentSize) {
|
|
|
38
61
|
logSlowRequestIfNeeded(context, duration);
|
|
39
62
|
}
|
|
40
63
|
function publishFetchEnd(context, status, duration) {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
fetchChannel.publish({
|
|
64
|
+
publishFetchEvent({
|
|
65
|
+
v: 1,
|
|
44
66
|
type: 'end',
|
|
45
67
|
requestId: context.requestId,
|
|
46
68
|
status,
|
|
@@ -69,17 +91,16 @@ export function recordFetchError(context, error, status) {
|
|
|
69
91
|
const duration = performance.now() - context.startTime;
|
|
70
92
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
71
93
|
const code = isSystemError(err) ? err.code : undefined;
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
}
|
|
94
|
+
publishFetchEvent({
|
|
95
|
+
v: 1,
|
|
96
|
+
type: 'error',
|
|
97
|
+
requestId: context.requestId,
|
|
98
|
+
url: context.url,
|
|
99
|
+
error: err.message,
|
|
100
|
+
code,
|
|
101
|
+
status,
|
|
102
|
+
duration,
|
|
103
|
+
});
|
|
83
104
|
const log = status === 429 ? logWarn : logError;
|
|
84
105
|
log('HTTP Request Error', {
|
|
85
106
|
requestId: context.requestId,
|
|
@@ -89,4 +110,3 @@ export function recordFetchError(context, error, status) {
|
|
|
89
110
|
error: err.message,
|
|
90
111
|
});
|
|
91
112
|
}
|
|
92
|
-
//# sourceMappingURL=interceptors.js.map
|
|
@@ -15,7 +15,7 @@ async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount)
|
|
|
15
15
|
void response.body?.cancel();
|
|
16
16
|
return {
|
|
17
17
|
response,
|
|
18
|
-
nextUrl: resolveRedirectTarget(currentUrl, location),
|
|
18
|
+
nextUrl: await resolveRedirectTarget(currentUrl, location),
|
|
19
19
|
};
|
|
20
20
|
}
|
|
21
21
|
function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
|
|
@@ -36,7 +36,7 @@ function annotateRedirectError(error, url) {
|
|
|
36
36
|
return;
|
|
37
37
|
error.requestUrl = url;
|
|
38
38
|
}
|
|
39
|
-
function resolveRedirectTarget(baseUrl, location) {
|
|
39
|
+
async function resolveRedirectTarget(baseUrl, location) {
|
|
40
40
|
if (!URL.canParse(location, baseUrl)) {
|
|
41
41
|
throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
|
|
42
42
|
}
|
|
@@ -67,4 +67,3 @@ async function performFetchCycleSafely(currentUrl, init, redirectLimit, redirect
|
|
|
67
67
|
throw error;
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
|
-
//# sourceMappingURL=redirects.js.map
|