@j0hanz/superfetch 1.0.6 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +228 -36
- package/dist/config/index.d.ts +10 -5
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +73 -19
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +98 -57
- package/dist/config/types.d.ts.map +1 -1
- package/dist/errors/app-error.d.ts +4 -28
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +10 -51
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +10 -55
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +12 -14
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +0 -8
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/parsers/base-html-element-parser.d.ts +43 -0
- package/dist/parsers/base-html-element-parser.d.ts.map +1 -0
- package/dist/parsers/base-html-element-parser.js +59 -0
- package/dist/parsers/base-html-element-parser.js.map +1 -0
- package/dist/parsers/heading-element-parser.d.ts +14 -0
- package/dist/parsers/heading-element-parser.d.ts.map +1 -0
- package/dist/parsers/heading-element-parser.js +26 -0
- package/dist/parsers/heading-element-parser.js.map +1 -0
- package/dist/parsers/image-element-parser.d.ts +16 -0
- package/dist/parsers/image-element-parser.d.ts.map +1 -0
- package/dist/parsers/image-element-parser.js +33 -0
- package/dist/parsers/image-element-parser.js.map +1 -0
- package/dist/parsers/link-element-parser.d.ts +15 -0
- package/dist/parsers/link-element-parser.d.ts.map +1 -0
- package/dist/parsers/link-element-parser.js +28 -0
- package/dist/parsers/link-element-parser.js.map +1 -0
- package/dist/parsers/open-graph-parser.d.ts +17 -0
- package/dist/parsers/open-graph-parser.d.ts.map +1 -0
- package/dist/parsers/open-graph-parser.js +41 -0
- package/dist/parsers/open-graph-parser.js.map +1 -0
- package/dist/parsers/schema-org-parser.d.ts +17 -0
- package/dist/parsers/schema-org-parser.d.ts.map +1 -0
- package/dist/parsers/schema-org-parser.js +32 -0
- package/dist/parsers/schema-org-parser.js.map +1 -0
- package/dist/parsers/standard-meta-parser.d.ts +18 -0
- package/dist/parsers/standard-meta-parser.d.ts.map +1 -0
- package/dist/parsers/standard-meta-parser.js +32 -0
- package/dist/parsers/standard-meta-parser.js.map +1 -0
- package/dist/parsers/twitter-card-parser.d.ts +17 -0
- package/dist/parsers/twitter-card-parser.d.ts.map +1 -0
- package/dist/parsers/twitter-card-parser.js +41 -0
- package/dist/parsers/twitter-card-parser.js.map +1 -0
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +3 -9
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +8 -8
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +10 -10
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +0 -28
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +10 -173
- package/dist/services/cache.js.map +1 -1
- package/dist/services/extractor.d.ts +1 -11
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +86 -84
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +2 -13
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +195 -211
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +5 -4
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +27 -42
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +35 -26
- package/dist/services/parser.js.map +1 -1
- package/dist/services/session-manager.d.ts +18 -0
- package/dist/services/session-manager.d.ts.map +1 -0
- package/dist/services/session-manager.js +73 -0
- package/dist/services/session-manager.js.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts +13 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.js +32 -0
- package/dist/strategies/exponential-backoff-strategy.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -1
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +23 -33
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +15 -20
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +124 -105
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +0 -4
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/utils/common.d.ts +6 -7
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +8 -8
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +8 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +47 -79
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +15 -10
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +58 -62
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/concurrency.d.ts +2 -5
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +19 -19
- package/dist/utils/concurrency.js.map +1 -1
- package/dist/utils/content-cleaner.d.ts +0 -25
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +12 -187
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +2 -0
- package/dist/utils/html-truncator.d.ts.map +1 -0
- package/dist/utils/html-truncator.js +14 -0
- package/dist/utils/html-truncator.js.map +1 -0
- package/dist/utils/language-detector.d.ts +0 -3
- package/dist/utils/language-detector.d.ts.map +1 -1
- package/dist/utils/language-detector.js +0 -11
- package/dist/utils/language-detector.js.map +1 -1
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +7 -5
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +15 -42
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -6
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +12 -81
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +5 -6
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { BaseHTMLElementParser } from './base-html-element-parser.js';
|
|
2
|
+
/**
|
|
3
|
+
* Parser for Twitter Card meta tags (twitter:*).
|
|
4
|
+
*/
|
|
5
|
+
export class TwitterCardParser extends BaseHTMLElementParser {
|
|
6
|
+
parse(document) {
|
|
7
|
+
const data = {};
|
|
8
|
+
// Extract all Twitter Card meta tags
|
|
9
|
+
const twitterTags = this.querySelectorAll(document, 'meta[name^="twitter:"]');
|
|
10
|
+
for (const tag of twitterTags) {
|
|
11
|
+
const name = this.getAttribute(tag, 'name');
|
|
12
|
+
const content = this.getAttribute(tag, 'content');
|
|
13
|
+
if (!name || !content)
|
|
14
|
+
continue;
|
|
15
|
+
// Extract property name (e.g., "twitter:card" → "card")
|
|
16
|
+
const key = name.replace('twitter:', '');
|
|
17
|
+
switch (key) {
|
|
18
|
+
case 'card':
|
|
19
|
+
data.card = content;
|
|
20
|
+
break;
|
|
21
|
+
case 'site':
|
|
22
|
+
data.site = content;
|
|
23
|
+
break;
|
|
24
|
+
case 'title':
|
|
25
|
+
data.title = content;
|
|
26
|
+
break;
|
|
27
|
+
case 'description':
|
|
28
|
+
data.description = content;
|
|
29
|
+
break;
|
|
30
|
+
case 'image':
|
|
31
|
+
data.image = content;
|
|
32
|
+
break;
|
|
33
|
+
case 'creator':
|
|
34
|
+
data.creator = content;
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return Object.keys(data).length > 0 ? data : null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=twitter-card-parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"twitter-card-parser.js","sourceRoot":"","sources":["../../src/parsers/twitter-card-parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAWtE;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,qBAAsC;IAC3E,KAAK,CAAC,QAAkB;QACtB,MAAM,IAAI,GAAoB,EAAE,CAAC;QAEjC,qCAAqC;QACrC,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CACvC,QAAQ,EACR,wBAAwB,CACzB,CAAC;QAEF,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAElD,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEhC,wDAAwD;YACxD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAEzC,QAAQ,GAAG,EAAE,CAAC;gBACZ,KAAK,MAAM;oBACT,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;oBACpB,MAAM;gBACR,KAAK,MAAM;oBACT,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;oBACpB,MAAM;gBACR,KAAK,OAAO;oBACV,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;oBACrB,MAAM;gBACR,KAAK,aAAa;oBAChB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;oBAC3B,MAAM;gBACR,KAAK,OAAO;oBACV,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;oBACrB,MAAM;gBACR,KAAK,SAAS;oBACZ,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;oBACvB,MAAM;YACV,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACpD,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cached-content.d.ts","sourceRoot":"","sources":["../../src/resources/cached-content.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAIzE,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"cached-content.d.ts","sourceRoot":"","sources":["../../src/resources/cached-content.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAIzE,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CA6ErE"}
|
|
@@ -28,16 +28,15 @@ export function registerCachedContentResource(server) {
|
|
|
28
28
|
],
|
|
29
29
|
};
|
|
30
30
|
});
|
|
31
|
-
// Helper resource to list cached URLs
|
|
32
31
|
server.registerResource('cached-urls', 'superfetch://cache/list', {
|
|
33
32
|
title: 'Cached URLs List',
|
|
34
33
|
description: 'List all URLs currently in cache with their namespaces',
|
|
35
34
|
mimeType: 'application/json',
|
|
36
35
|
}, (uri) => {
|
|
37
|
-
const
|
|
36
|
+
const cacheKeys = cache.keys();
|
|
38
37
|
const cacheList = {
|
|
39
|
-
totalEntries:
|
|
40
|
-
entries:
|
|
38
|
+
totalEntries: cacheKeys.length,
|
|
39
|
+
entries: cacheKeys.map((key) => {
|
|
41
40
|
const parts = key.split(':');
|
|
42
41
|
const namespace = parts[0] ?? 'unknown';
|
|
43
42
|
const urlHash = parts.slice(1).join(':') || 'unknown';
|
|
@@ -60,9 +59,4 @@ export function registerCachedContentResource(server) {
|
|
|
60
59
|
};
|
|
61
60
|
});
|
|
62
61
|
}
|
|
63
|
-
// Subscription notifications - placeholder until MCP SDK fully supports sendResourceUpdated
|
|
64
|
-
export function setupCacheSubscriptions() {
|
|
65
|
-
// No-op: SDK doesn't support resource update notifications yet
|
|
66
|
-
// When it does, listen to cache.onUpdate() and call server.sendResourceUpdated()
|
|
67
|
-
}
|
|
68
62
|
//# sourceMappingURL=cached-content.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cached-content.js","sourceRoot":"","sources":["../../src/resources/cached-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAG3E,OAAO,KAAK,KAAK,MAAM,sBAAsB,CAAC;AAE9C,MAAM,UAAU,6BAA6B,CAAC,MAAiB;IAC7D,MAAM,CAAC,gBAAgB,CACrB,gBAAgB,EAChB,IAAI,gBAAgB,CAAC,0CAA0C,EAAE;QAC/D,IAAI,EAAE,SAAS;KAChB,CAAC,EACF;QACE,KAAK,EAAE,gBAAgB;QACvB,WAAW,EACT,sHAAsH;QACxH,QAAQ,EAAE,kBAAkB;KAC7B,EACD,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;QACd,MAAM,SAAS,GAAG,MAAM,CAAC,SAAmB,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;QAEzC,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,OAAO,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,uCAAuC,QAAQ,0DAA0D,CAC1G,CAAC;QACJ,CAAC;QAED,OAAO;YACL,QAAQ,EAAE;gBACR;oBACE,GAAG,EAAE,GAAG,CAAC,IAAI;oBACb,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE,MAAM,CAAC,OAAO;iBACrB;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,
|
|
1
|
+
{"version":3,"file":"cached-content.js","sourceRoot":"","sources":["../../src/resources/cached-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAG3E,OAAO,KAAK,KAAK,MAAM,sBAAsB,CAAC;AAE9C,MAAM,UAAU,6BAA6B,CAAC,MAAiB;IAC7D,MAAM,CAAC,gBAAgB,CACrB,gBAAgB,EAChB,IAAI,gBAAgB,CAAC,0CAA0C,EAAE;QAC/D,IAAI,EAAE,SAAS;KAChB,CAAC,EACF;QACE,KAAK,EAAE,gBAAgB;QACvB,WAAW,EACT,sHAAsH;QACxH,QAAQ,EAAE,kBAAkB;KAC7B,EACD,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;QACd,MAAM,SAAS,GAAG,MAAM,CAAC,SAAmB,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;QAEzC,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,OAAO,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,uCAAuC,QAAQ,0DAA0D,CAC1G,CAAC;QACJ,CAAC;QAED,OAAO;YACL,QAAQ,EAAE;gBACR;oBACE,GAAG,EAAE,GAAG,CAAC,IAAI;oBACb,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE,MAAM,CAAC,OAAO;iBACrB;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,MAAM,CAAC,gBAAgB,CACrB,aAAa,EACb,yBAAyB,EACzB;QACE,KAAK,EAAE,kBAAkB;QACzB,WAAW,EAAE,wDAAwD;QACrE,QAAQ,EAAE,kBAAkB;KAC7B,EACD,CAAC,GAAG,EAAE,EAAE;QACN,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG;YAChB,YAAY,EAAE,SAAS,CAAC,MAAM;YAC9B,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,GAAW,EAAE,EAAE;gBACrC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;gBACxC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC;gBACtD,OAAO;oBACL,SAAS;oBACT,OAAO;oBACP,WAAW,EAAE,sBAAsB,SAAS,IAAI,OAAO,EAAE;iBAC1D,CAAC;YACJ,CAAC,CAAC;YACF,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;QAEF,OAAO;YACL,QAAQ,EAAE;gBACR;oBACE,GAAG,EAAE,GAAG,CAAC,IAAI;oBACb,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;iBACzC;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/resources/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/resources/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAQzE,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAyFzD"}
|
package/dist/resources/index.js
CHANGED
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
import { config } from '../config/index.js';
|
|
2
2
|
import * as cache from '../services/cache.js';
|
|
3
|
-
import { registerCachedContentResource
|
|
3
|
+
import { registerCachedContentResource } from './cached-content.js';
|
|
4
4
|
export function registerResources(server) {
|
|
5
|
-
// Register dynamic cached content resources
|
|
6
5
|
registerCachedContentResource(server);
|
|
7
|
-
// Setup cache update subscriptions
|
|
8
|
-
setupCacheSubscriptions();
|
|
9
|
-
// Register health check resource
|
|
10
6
|
server.registerResource('health', 'superfetch://health', {
|
|
11
7
|
title: 'Server Health',
|
|
12
8
|
description: 'Real-time server health and dependency status',
|
|
@@ -24,7 +20,7 @@ export function registerResources(server) {
|
|
|
24
20
|
heapUsed: heapUsedMB,
|
|
25
21
|
heapTotal: heapTotalMB,
|
|
26
22
|
percentage: Math.round((heapUsedMB / heapTotalMB) * 100),
|
|
27
|
-
healthy: heapUsedMB < 400,
|
|
23
|
+
healthy: heapUsedMB < 400,
|
|
28
24
|
},
|
|
29
25
|
},
|
|
30
26
|
timestamp: new Date().toISOString(),
|
|
@@ -39,7 +35,6 @@ export function registerResources(server) {
|
|
|
39
35
|
],
|
|
40
36
|
};
|
|
41
37
|
});
|
|
42
|
-
// Register server statistics resource
|
|
43
38
|
server.registerResource('stats', 'superfetch://stats', {
|
|
44
39
|
title: 'Server Statistics',
|
|
45
40
|
description: 'Fetch statistics and cache performance metrics',
|
|
@@ -53,7 +48,12 @@ export function registerResources(server) {
|
|
|
53
48
|
nodeVersion: process.version,
|
|
54
49
|
memoryUsage: process.memoryUsage(),
|
|
55
50
|
},
|
|
56
|
-
cache:
|
|
51
|
+
cache: {
|
|
52
|
+
enabled: config.cache.enabled,
|
|
53
|
+
ttl: config.cache.ttl,
|
|
54
|
+
maxKeys: config.cache.maxKeys,
|
|
55
|
+
totalKeys: cache.keys().length,
|
|
56
|
+
},
|
|
57
57
|
config: {
|
|
58
58
|
fetcher: {
|
|
59
59
|
timeout: config.fetcher.timeout,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/resources/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,KAAK,KAAK,MAAM,sBAAsB,CAAC;AAE9C,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/resources/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,KAAK,KAAK,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAC;AAEpE,MAAM,UAAU,iBAAiB,CAAC,MAAiB;IACjD,6BAA6B,CAAC,MAAM,CAAC,CAAC;IAEtC,MAAM,CAAC,gBAAgB,CACrB,QAAQ,EACR,qBAAqB,EACrB;QACE,KAAK,EAAE,eAAe;QACtB,WAAW,EAAE,+CAA+C;QAC5D,QAAQ,EAAE,kBAAkB;KAC7B,EACD,CAAC,GAAG,EAAE,EAAE;QACN,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;QAEjE,MAAM,MAAM,GAAG;YACb,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;YACxB,MAAM,EAAE;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO;gBAC3B,MAAM,EAAE;oBACN,QAAQ,EAAE,UAAU;oBACpB,SAAS,EAAE,WAAW;oBACtB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC;oBACxD,OAAO,EAAE,UAAU,GAAG,GAAG;iBAC1B;aACF;YACD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;QAEF,OAAO;YACL,QAAQ,EAAE;gBACR;oBACE,GAAG,EAAE,GAAG,CAAC,IAAI;oBACb,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;iBACtC;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,oBAAoB,EACpB;QACE,KAAK,EAAE,mBAAmB;QAC1B,WAAW,EAAE,gDAAgD;QAC7D,QAAQ,EAAE,kBAAkB;KAC7B,EACD,CAAC,GAAG,EAAE,EAAE;QACN,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;gBACxB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;gBAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;gBACxB,WAAW,EAAE,OAAO,CAAC,OAAO;gBAC5B,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE;aACnC;YACD,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO;gBAC7B,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG;gBACrB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO;gBAC7B,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM;aAC/B;YACD,MAAM,EAAE;gBACN,OAAO,EAAE;oBACP,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;oBAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;iBAC1C;gBACD,UAAU,EAAE;oBACV,kBAAkB,EAAE,MAAM,CAAC,UAAU,CAAC,kBAAkB;oBACxD,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,eAAe;iBACnD;aACF;SACF,CAAC;QAEF,OAAO;YACL,QAAQ,EAAE;gBACR;oBACE,GAAG,EAAE,GAAG,CAAC,IAAI;oBACb,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;iBACrC;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAYpE,wBAAgB,eAAe,IAAI,SAAS,CAoB3C;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CA0BtD"}
|
package/dist/server.js
CHANGED
|
@@ -4,7 +4,6 @@ import { config } from './config/index.js';
|
|
|
4
4
|
import { destroyAgents } from './services/fetcher.js';
|
|
5
5
|
import { logError, logInfo } from './services/logger.js';
|
|
6
6
|
import { registerTools } from './tools/index.js';
|
|
7
|
-
import { registerPrompts } from './prompts/index.js';
|
|
8
7
|
import { registerResources } from './resources/index.js';
|
|
9
8
|
export function createMcpServer() {
|
|
10
9
|
const server = new McpServer({
|
|
@@ -14,30 +13,31 @@ export function createMcpServer() {
|
|
|
14
13
|
capabilities: {
|
|
15
14
|
tools: { listChanged: false },
|
|
16
15
|
resources: { listChanged: true, subscribe: true },
|
|
17
|
-
prompts: { listChanged: false },
|
|
18
16
|
logging: {},
|
|
19
17
|
},
|
|
20
18
|
instructions: `superFetch MCP server v${config.server.version} - AI-optimized web content fetching with JSONL/Markdown output. Provides tools for fetching, parsing, and transforming web content into structured formats suitable for LLM consumption. Supports resource subscriptions for cache updates.`,
|
|
21
19
|
});
|
|
22
|
-
// Register all features using the modern API
|
|
23
20
|
registerTools(server);
|
|
24
21
|
registerResources(server);
|
|
25
|
-
registerPrompts(server);
|
|
26
22
|
return server;
|
|
27
23
|
}
|
|
28
|
-
// Export function to start server with stdio transport
|
|
29
24
|
export async function startStdioServer() {
|
|
30
25
|
const server = createMcpServer();
|
|
31
26
|
const transport = new StdioServerTransport();
|
|
32
|
-
// Error handlers
|
|
33
27
|
server.server.onerror = (error) => {
|
|
34
28
|
logError('[MCP Error]', error instanceof Error ? error : { error });
|
|
35
29
|
};
|
|
36
|
-
process.on('SIGINT',
|
|
30
|
+
process.on('SIGINT', () => {
|
|
37
31
|
process.stdout.write('\nShutting down superFetch MCP server...\n');
|
|
38
|
-
destroyAgents();
|
|
39
|
-
|
|
40
|
-
|
|
32
|
+
destroyAgents();
|
|
33
|
+
server
|
|
34
|
+
.close()
|
|
35
|
+
.catch((err) => {
|
|
36
|
+
logError('Error during shutdown', err instanceof Error ? err : undefined);
|
|
37
|
+
})
|
|
38
|
+
.finally(() => {
|
|
39
|
+
process.exit(0);
|
|
40
|
+
});
|
|
41
41
|
});
|
|
42
42
|
await server.connect(transport);
|
|
43
43
|
logInfo('superFetch MCP server running on stdio');
|
package/dist/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B;QACE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;QACxB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;KAC/B,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE;YAC7B,SAAS,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;YACjD,OAAO,EAAE,EAAE;SACZ;QACD,YAAY,EAAE,0BAA0B,MAAM,CAAC,MAAM,CAAC,OAAO,8OAA8O;KAC5S,CACF,CAAC;IAEF,aAAa,CAAC,MAAM,CAAC,CAAC;IACtB,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE1B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE7C,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;QAChC,QAAQ,CAAC,aAAa,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC,CAAC;IAEF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACnE,aAAa,EAAE,CAAC;QAChB,MAAM;aACH,KAAK,EAAE;aACP,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,QAAQ,CACN,uBAAuB,EACvB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CACvC,CAAC;QACJ,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,OAAO,CAAC,wCAAwC,CAAC,CAAC;AACpD,CAAC"}
|
package/dist/services/cache.d.ts
CHANGED
|
@@ -2,33 +2,5 @@ import type { CacheEntry } from '../config/types.js';
|
|
|
2
2
|
export declare function createCacheKey(namespace: string, url: string): string | null;
|
|
3
3
|
export declare function get(cacheKey: string | null): CacheEntry | undefined;
|
|
4
4
|
export declare function set(cacheKey: string | null, content: string): void;
|
|
5
|
-
export declare function getHtml(url: string): string | undefined;
|
|
6
|
-
export declare function setHtml(url: string, html: string): void;
|
|
7
5
|
export declare function keys(): string[];
|
|
8
|
-
type CacheUpdateCallback = (key: string, namespace: string) => void;
|
|
9
|
-
export declare function onUpdate(callback: CacheUpdateCallback): () => void;
|
|
10
|
-
export declare function getStats(): {
|
|
11
|
-
size: number;
|
|
12
|
-
maxKeys: number;
|
|
13
|
-
ttl: number;
|
|
14
|
-
hits: number;
|
|
15
|
-
misses: number;
|
|
16
|
-
sets: number;
|
|
17
|
-
errors: number;
|
|
18
|
-
evictions: number;
|
|
19
|
-
hitRate: string;
|
|
20
|
-
htmlCacheSize: number;
|
|
21
|
-
htmlCacheMaxKeys: number;
|
|
22
|
-
htmlCacheTtl: number;
|
|
23
|
-
htmlHits: number;
|
|
24
|
-
htmlMisses: number;
|
|
25
|
-
htmlEvictions: number;
|
|
26
|
-
htmlHitRate: string;
|
|
27
|
-
efficiency: {
|
|
28
|
-
hitRate: string;
|
|
29
|
-
missRate: string;
|
|
30
|
-
errorRate: string;
|
|
31
|
-
};
|
|
32
|
-
};
|
|
33
|
-
export {};
|
|
34
6
|
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAarD,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK5E;AAED,wBAAgB,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,SAAS,CAcnE;AAED,wBAAgB,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAiBlE;AAED,wBAAgB,IAAI,IAAI,MAAM,EAAE,CAE/B"}
|
package/dist/services/cache.js
CHANGED
|
@@ -1,72 +1,27 @@
|
|
|
1
|
-
import crypto from 'crypto';
|
|
2
1
|
import NodeCache from 'node-cache';
|
|
3
2
|
import { config } from '../config/index.js';
|
|
4
|
-
import {
|
|
3
|
+
import { logWarn } from './logger.js';
|
|
4
|
+
const MAX_KEY_LENGTH = 500;
|
|
5
5
|
const contentCache = new NodeCache({
|
|
6
6
|
stdTTL: config.cache.ttl,
|
|
7
7
|
checkperiod: Math.floor(config.cache.ttl / 10),
|
|
8
8
|
useClones: false,
|
|
9
9
|
maxKeys: config.cache.maxKeys,
|
|
10
10
|
});
|
|
11
|
-
const HTML_CACHE_TTL = 60;
|
|
12
|
-
const HTML_CACHE_MAX_KEYS = 50;
|
|
13
|
-
const htmlCache = new NodeCache({
|
|
14
|
-
stdTTL: HTML_CACHE_TTL,
|
|
15
|
-
checkperiod: 30,
|
|
16
|
-
useClones: false,
|
|
17
|
-
maxKeys: HTML_CACHE_MAX_KEYS,
|
|
18
|
-
});
|
|
19
|
-
// Track cache evictions
|
|
20
|
-
contentCache.on('del', (key) => {
|
|
21
|
-
stats.evictions++;
|
|
22
|
-
logDebug('Cache eviction', { key: String(key).substring(0, 100) });
|
|
23
|
-
});
|
|
24
|
-
htmlCache.on('del', () => {
|
|
25
|
-
stats.htmlEvictions++;
|
|
26
|
-
});
|
|
27
|
-
const stats = {
|
|
28
|
-
hits: 0,
|
|
29
|
-
misses: 0,
|
|
30
|
-
sets: 0,
|
|
31
|
-
errors: 0,
|
|
32
|
-
evictions: 0,
|
|
33
|
-
htmlHits: 0,
|
|
34
|
-
htmlMisses: 0,
|
|
35
|
-
htmlEvictions: 0,
|
|
36
|
-
};
|
|
37
|
-
const MAX_CONTENT_SIZE = 5242880;
|
|
38
|
-
const MAX_HTML_SIZE = 10485760;
|
|
39
|
-
const MAX_KEY_LENGTH = 500;
|
|
40
11
|
export function createCacheKey(namespace, url) {
|
|
41
12
|
if (!namespace || !url)
|
|
42
13
|
return null;
|
|
43
14
|
const key = `${namespace}:${url}`;
|
|
44
|
-
|
|
45
|
-
return key;
|
|
46
|
-
// SHA-256 hash for long URLs (consistent with cached-content.ts)
|
|
47
|
-
const hash = crypto
|
|
48
|
-
.createHash('sha256')
|
|
49
|
-
.update(url)
|
|
50
|
-
.digest('hex')
|
|
51
|
-
.substring(0, 64);
|
|
52
|
-
return `${namespace}:hash:${hash}`;
|
|
15
|
+
return key.length <= MAX_KEY_LENGTH ? key : key.substring(0, MAX_KEY_LENGTH);
|
|
53
16
|
}
|
|
54
17
|
export function get(cacheKey) {
|
|
55
|
-
if (!config.cache.enabled)
|
|
56
|
-
return undefined;
|
|
57
|
-
if (!cacheKey)
|
|
18
|
+
if (!config.cache.enabled || !cacheKey) {
|
|
58
19
|
return undefined;
|
|
20
|
+
}
|
|
59
21
|
try {
|
|
60
|
-
|
|
61
|
-
if (entry) {
|
|
62
|
-
stats.hits++;
|
|
63
|
-
return entry;
|
|
64
|
-
}
|
|
65
|
-
stats.misses++;
|
|
66
|
-
return undefined;
|
|
22
|
+
return contentCache.get(cacheKey);
|
|
67
23
|
}
|
|
68
24
|
catch (error) {
|
|
69
|
-
stats.errors++;
|
|
70
25
|
logWarn('Cache get error', {
|
|
71
26
|
key: cacheKey.substring(0, 100),
|
|
72
27
|
error: error instanceof Error ? error.message : 'Unknown error',
|
|
@@ -75,143 +30,25 @@ export function get(cacheKey) {
|
|
|
75
30
|
}
|
|
76
31
|
}
|
|
77
32
|
export function set(cacheKey, content) {
|
|
78
|
-
if (!config.cache.enabled)
|
|
33
|
+
if (!config.cache.enabled || !cacheKey || !content)
|
|
79
34
|
return;
|
|
80
|
-
if (!cacheKey)
|
|
81
|
-
return;
|
|
82
|
-
if (!content || typeof content !== 'string')
|
|
83
|
-
return;
|
|
84
|
-
if (content.length > MAX_CONTENT_SIZE) {
|
|
85
|
-
logWarn('Cache set skipped: content too large', {
|
|
86
|
-
key: cacheKey.substring(0, 100),
|
|
87
|
-
size: content.length,
|
|
88
|
-
maxSize: MAX_CONTENT_SIZE,
|
|
89
|
-
});
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
35
|
try {
|
|
93
|
-
const nowMs = Date.now();
|
|
94
36
|
const entry = {
|
|
95
37
|
url: cacheKey,
|
|
96
38
|
content,
|
|
97
|
-
fetchedAt: new Date(
|
|
98
|
-
expiresAt: new Date(
|
|
39
|
+
fetchedAt: new Date().toISOString(),
|
|
40
|
+
expiresAt: new Date(Date.now() + config.cache.ttl * 1000).toISOString(),
|
|
99
41
|
};
|
|
100
42
|
contentCache.set(cacheKey, entry);
|
|
101
|
-
stats.sets++;
|
|
102
|
-
// Notify subscribers of cache update
|
|
103
|
-
notifyUpdate(cacheKey);
|
|
104
43
|
}
|
|
105
44
|
catch (error) {
|
|
106
|
-
stats.errors++;
|
|
107
45
|
logWarn('Cache set error', {
|
|
108
46
|
key: cacheKey.substring(0, 100),
|
|
109
47
|
error: error instanceof Error ? error.message : 'Unknown error',
|
|
110
48
|
});
|
|
111
49
|
}
|
|
112
50
|
}
|
|
113
|
-
export function getHtml(url) {
|
|
114
|
-
if (!config.cache.enabled)
|
|
115
|
-
return undefined;
|
|
116
|
-
try {
|
|
117
|
-
const html = htmlCache.get(url);
|
|
118
|
-
if (html) {
|
|
119
|
-
// Validate cached HTML is within limits
|
|
120
|
-
if (typeof html === 'string' && html.length <= MAX_HTML_SIZE) {
|
|
121
|
-
stats.htmlHits++;
|
|
122
|
-
logDebug('HTML cache hit', { url: url.substring(0, 100) });
|
|
123
|
-
return html;
|
|
124
|
-
}
|
|
125
|
-
// Invalid cache entry, remove it
|
|
126
|
-
htmlCache.del(url);
|
|
127
|
-
logWarn('Removed oversized HTML from cache', {
|
|
128
|
-
url: url.substring(0, 100),
|
|
129
|
-
size: html.length,
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
stats.htmlMisses++;
|
|
133
|
-
return undefined;
|
|
134
|
-
}
|
|
135
|
-
catch (error) {
|
|
136
|
-
logDebug('HTML cache get error (non-critical)', {
|
|
137
|
-
url: url.substring(0, 100),
|
|
138
|
-
error: error instanceof Error ? error.message : 'Unknown',
|
|
139
|
-
});
|
|
140
|
-
return undefined;
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
export function setHtml(url, html) {
|
|
144
|
-
if (!config.cache.enabled)
|
|
145
|
-
return;
|
|
146
|
-
if (!html || html.length > MAX_HTML_SIZE)
|
|
147
|
-
return;
|
|
148
|
-
try {
|
|
149
|
-
htmlCache.set(url, html);
|
|
150
|
-
logDebug('HTML cached', { url: url.substring(0, 100), size: html.length });
|
|
151
|
-
}
|
|
152
|
-
catch (error) {
|
|
153
|
-
logDebug('HTML cache set error (non-critical)', {
|
|
154
|
-
url: url.substring(0, 100),
|
|
155
|
-
error: error instanceof Error ? error.message : 'Unknown',
|
|
156
|
-
});
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
51
|
export function keys() {
|
|
160
|
-
return
|
|
161
|
-
}
|
|
162
|
-
const updateCallbacks = [];
|
|
163
|
-
export function onUpdate(callback) {
|
|
164
|
-
updateCallbacks.push(callback);
|
|
165
|
-
// Return unsubscribe function
|
|
166
|
-
return () => {
|
|
167
|
-
const index = updateCallbacks.indexOf(callback);
|
|
168
|
-
if (index > -1) {
|
|
169
|
-
updateCallbacks.splice(index, 1);
|
|
170
|
-
}
|
|
171
|
-
};
|
|
172
|
-
}
|
|
173
|
-
// Notify callbacks when cache is updated
|
|
174
|
-
function notifyUpdate(key) {
|
|
175
|
-
const parts = key.split(':');
|
|
176
|
-
const namespace = parts[0] ?? 'unknown';
|
|
177
|
-
updateCallbacks.forEach((callback) => {
|
|
178
|
-
try {
|
|
179
|
-
callback(key, namespace);
|
|
180
|
-
}
|
|
181
|
-
catch {
|
|
182
|
-
// Silently ignore callback errors
|
|
183
|
-
}
|
|
184
|
-
});
|
|
185
|
-
}
|
|
186
|
-
export function getStats() {
|
|
187
|
-
const total = stats.hits + stats.misses;
|
|
188
|
-
const hitRate = total > 0 ? ((stats.hits / total) * 100).toFixed(2) : '0.00';
|
|
189
|
-
const htmlTotal = stats.htmlHits + stats.htmlMisses;
|
|
190
|
-
const htmlHitRate = htmlTotal > 0 ? ((stats.htmlHits / htmlTotal) * 100).toFixed(2) : '0.00';
|
|
191
|
-
const missRate = total > 0 ? ((stats.misses / total) * 100).toFixed(2) : '0.00';
|
|
192
|
-
const errorRate = stats.sets > 0 ? ((stats.errors / stats.sets) * 100).toFixed(2) : '0.00';
|
|
193
|
-
return {
|
|
194
|
-
size: contentCache.keys().length,
|
|
195
|
-
maxKeys: config.cache.maxKeys,
|
|
196
|
-
ttl: config.cache.ttl,
|
|
197
|
-
hits: stats.hits,
|
|
198
|
-
misses: stats.misses,
|
|
199
|
-
sets: stats.sets,
|
|
200
|
-
errors: stats.errors,
|
|
201
|
-
evictions: stats.evictions,
|
|
202
|
-
hitRate: `${hitRate}%`,
|
|
203
|
-
htmlCacheSize: htmlCache.keys().length,
|
|
204
|
-
htmlCacheMaxKeys: HTML_CACHE_MAX_KEYS,
|
|
205
|
-
htmlCacheTtl: HTML_CACHE_TTL,
|
|
206
|
-
htmlHits: stats.htmlHits,
|
|
207
|
-
htmlMisses: stats.htmlMisses,
|
|
208
|
-
htmlEvictions: stats.htmlEvictions,
|
|
209
|
-
htmlHitRate: `${htmlHitRate}%`,
|
|
210
|
-
efficiency: {
|
|
211
|
-
hitRate: `${hitRate}%`,
|
|
212
|
-
missRate: `${missRate}%`,
|
|
213
|
-
errorRate: `${errorRate}%`,
|
|
214
|
-
},
|
|
215
|
-
};
|
|
52
|
+
return contentCache.keys();
|
|
216
53
|
}
|
|
217
54
|
//# sourceMappingURL=cache.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/services/cache.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AAEnC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG;IACxB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC;IAC9C,SAAS,EAAE,KAAK;IAChB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO;CAC9B,CAAC,CAAC;AAEH,MAAM,UAAU,cAAc,CAAC,SAAiB,EAAE,GAAW;IAC3D,IAAI,CAAC,SAAS,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,GAAG,GAAG,GAAG,SAAS,IAAI,GAAG,EAAE,CAAC;IAClC,OAAO,GAAG,CAAC,MAAM,IAAI,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,QAAuB;IACzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,GAAG,CAAa,QAAQ,CAAC,CAAC;IAChD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,iBAAiB,EAAE;YACzB,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;YAC/B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;SAChE,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,QAAuB,EAAE,OAAe;IAC1D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO;IAE3D,IAAI,CAAC;QACH,MAAM,KAAK,GAAe;YACxB,GAAG,EAAE,QAAQ;YACb,OAAO;YACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE;SACxE,CAAC;QACF,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,iBAAiB,EAAE;YACzB,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;YAC/B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;SAChE,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,UAAU,IAAI;IAClB,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC"}
|
|
@@ -1,14 +1,4 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { ExtractedMetadata, ExtractionResult } from '../config/types.js';
|
|
3
|
-
/**
|
|
4
|
-
* Extract metadata using Cheerio (fast, no full DOM)
|
|
5
|
-
* This avoids JSDOM overhead for simple meta tag extraction
|
|
6
|
-
*/
|
|
7
|
-
export declare function extractMetadataWithCheerio($: CheerioAPI): ExtractedMetadata;
|
|
8
|
-
/**
|
|
9
|
-
* Main extraction function - uses Cheerio for metadata (fast)
|
|
10
|
-
* and lazy-loads JSDOM only when article extraction is needed
|
|
11
|
-
*/
|
|
1
|
+
import type { ExtractionResult } from '../config/types.js';
|
|
12
2
|
export declare function extractContent(html: string, url: string, options?: {
|
|
13
3
|
extractArticle?: boolean;
|
|
14
4
|
}): ExtractionResult;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAGV,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAyE5B,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAA6B,GAC/D,gBAAgB,CA6DlB"}
|