@j0hanz/superfetch 1.0.5 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -36
- package/dist/config/index.d.ts +10 -5
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +41 -17
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +98 -57
- package/dist/config/types.d.ts.map +1 -1
- package/dist/errors/app-error.d.ts +4 -28
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +10 -51
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +31 -46
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +12 -14
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +31 -14
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/parsers/base-html-element-parser.d.ts +43 -0
- package/dist/parsers/base-html-element-parser.d.ts.map +1 -0
- package/dist/parsers/base-html-element-parser.js +59 -0
- package/dist/parsers/base-html-element-parser.js.map +1 -0
- package/dist/parsers/heading-element-parser.d.ts +14 -0
- package/dist/parsers/heading-element-parser.d.ts.map +1 -0
- package/dist/parsers/heading-element-parser.js +26 -0
- package/dist/parsers/heading-element-parser.js.map +1 -0
- package/dist/parsers/image-element-parser.d.ts +16 -0
- package/dist/parsers/image-element-parser.d.ts.map +1 -0
- package/dist/parsers/image-element-parser.js +33 -0
- package/dist/parsers/image-element-parser.js.map +1 -0
- package/dist/parsers/link-element-parser.d.ts +15 -0
- package/dist/parsers/link-element-parser.d.ts.map +1 -0
- package/dist/parsers/link-element-parser.js +28 -0
- package/dist/parsers/link-element-parser.js.map +1 -0
- package/dist/parsers/open-graph-parser.d.ts +17 -0
- package/dist/parsers/open-graph-parser.d.ts.map +1 -0
- package/dist/parsers/open-graph-parser.js +41 -0
- package/dist/parsers/open-graph-parser.js.map +1 -0
- package/dist/parsers/schema-org-parser.d.ts +17 -0
- package/dist/parsers/schema-org-parser.d.ts.map +1 -0
- package/dist/parsers/schema-org-parser.js +32 -0
- package/dist/parsers/schema-org-parser.js.map +1 -0
- package/dist/parsers/standard-meta-parser.d.ts +18 -0
- package/dist/parsers/standard-meta-parser.d.ts.map +1 -0
- package/dist/parsers/standard-meta-parser.js +32 -0
- package/dist/parsers/standard-meta-parser.js.map +1 -0
- package/dist/parsers/twitter-card-parser.d.ts +17 -0
- package/dist/parsers/twitter-card-parser.d.ts.map +1 -0
- package/dist/parsers/twitter-card-parser.js +41 -0
- package/dist/parsers/twitter-card-parser.js.map +1 -0
- package/dist/resources/cached-content.d.ts +0 -2
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +3 -34
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +8 -8
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +12 -11
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +0 -28
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +10 -166
- package/dist/services/cache.js.map +1 -1
- package/dist/services/card-extractor.d.ts +0 -4
- package/dist/services/card-extractor.d.ts.map +1 -1
- package/dist/services/card-extractor.js +6 -1
- package/dist/services/card-extractor.js.map +1 -1
- package/dist/services/extractor.d.ts +1 -11
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +86 -84
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +2 -13
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +79 -79
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +5 -4
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +27 -42
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +35 -27
- package/dist/services/parser.js.map +1 -1
- package/dist/services/session-manager.d.ts +18 -0
- package/dist/services/session-manager.d.ts.map +1 -0
- package/dist/services/session-manager.js +73 -0
- package/dist/services/session-manager.js.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts +13 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.js +32 -0
- package/dist/strategies/exponential-backoff-strategy.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +3 -0
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +23 -33
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +15 -20
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +141 -108
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +0 -4
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/utils/common.d.ts +6 -7
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +8 -8
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +8 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +60 -63
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +15 -10
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +58 -62
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/concurrency.d.ts +2 -5
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +19 -19
- package/dist/utils/concurrency.js.map +1 -1
- package/dist/utils/content-cleaner.d.ts +0 -25
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +14 -171
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +2 -0
- package/dist/utils/html-truncator.d.ts.map +1 -0
- package/dist/utils/html-truncator.js +14 -0
- package/dist/utils/html-truncator.js.map +1 -0
- package/dist/utils/language-detector.d.ts +0 -3
- package/dist/utils/language-detector.d.ts.map +1 -1
- package/dist/utils/language-detector.js +0 -11
- package/dist/utils/language-detector.js.map +1 -1
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +7 -5
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +16 -41
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +1 -0
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +42 -23
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +9 -8
|
@@ -1,14 +1,4 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { ExtractedMetadata, ExtractionResult } from '../config/types.js';
|
|
3
|
-
/**
|
|
4
|
-
* Extract metadata using Cheerio (fast, no full DOM)
|
|
5
|
-
* This avoids JSDOM overhead for simple meta tag extraction
|
|
6
|
-
*/
|
|
7
|
-
export declare function extractMetadataWithCheerio($: CheerioAPI): ExtractedMetadata;
|
|
8
|
-
/**
|
|
9
|
-
* Main extraction function - uses Cheerio for metadata (fast)
|
|
10
|
-
* and lazy-loads JSDOM only when article extraction is needed
|
|
11
|
-
*/
|
|
1
|
+
import type { ExtractionResult } from '../config/types.js';
|
|
12
2
|
export declare function extractContent(html: string, url: string, options?: {
|
|
13
3
|
extractArticle?: boolean;
|
|
14
4
|
}): ExtractionResult;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAGV,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAyE5B,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAA6B,GAC/D,gBAAgB,CA6DlB"}
|
|
@@ -1,78 +1,60 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { JSDOM, VirtualConsole } from 'jsdom';
|
|
1
|
+
import { JSDOM } from 'jsdom';
|
|
3
2
|
import { Readability } from '@mozilla/readability';
|
|
4
|
-
import {
|
|
3
|
+
import { truncateHtml } from '../utils/html-truncator.js';
|
|
5
4
|
import { logError, logWarn } from './logger.js';
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
for (const selector of selectors) {
|
|
22
|
-
const content = $(selector).attr('content');
|
|
23
|
-
if (content)
|
|
24
|
-
return content;
|
|
25
|
-
}
|
|
26
|
-
return undefined;
|
|
27
|
-
};
|
|
28
|
-
const title = getMetaContent([
|
|
29
|
-
'meta[property="og:title"]',
|
|
30
|
-
'meta[name="twitter:title"]',
|
|
31
|
-
]) ??
|
|
32
|
-
($('title').text() || undefined);
|
|
33
|
-
const description = getMetaContent([
|
|
34
|
-
'meta[property="og:description"]',
|
|
35
|
-
'meta[name="twitter:description"]',
|
|
36
|
-
'meta[name="description"]',
|
|
37
|
-
]);
|
|
38
|
-
const author = getMetaContent([
|
|
39
|
-
'meta[name="author"]',
|
|
40
|
-
'meta[property="article:author"]',
|
|
41
|
-
]);
|
|
42
|
-
return { title, description, author };
|
|
5
|
+
function extractOpenGraph(document) {
|
|
6
|
+
const data = {};
|
|
7
|
+
const ogTags = document.querySelectorAll('meta[property^="og:"]');
|
|
8
|
+
for (const tag of ogTags) {
|
|
9
|
+
const property = tag.getAttribute('property');
|
|
10
|
+
const content = tag.getAttribute('content')?.trim();
|
|
11
|
+
if (!property || !content)
|
|
12
|
+
continue;
|
|
13
|
+
const key = property.replace('og:', '');
|
|
14
|
+
if (key === 'title')
|
|
15
|
+
data.title = content;
|
|
16
|
+
else if (key === 'description')
|
|
17
|
+
data.description = content;
|
|
18
|
+
}
|
|
19
|
+
return data;
|
|
43
20
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
return null;
|
|
58
|
-
return {
|
|
59
|
-
title: article.title ?? undefined,
|
|
60
|
-
byline: article.byline ?? undefined,
|
|
61
|
-
content: article.content ?? '',
|
|
62
|
-
textContent: article.textContent ?? '',
|
|
63
|
-
excerpt: article.excerpt ?? undefined,
|
|
64
|
-
siteName: article.siteName ?? undefined,
|
|
65
|
-
};
|
|
21
|
+
function extractTwitterCard(document) {
|
|
22
|
+
const data = {};
|
|
23
|
+
const twitterTags = document.querySelectorAll('meta[name^="twitter:"]');
|
|
24
|
+
for (const tag of twitterTags) {
|
|
25
|
+
const name = tag.getAttribute('name');
|
|
26
|
+
const content = tag.getAttribute('content')?.trim();
|
|
27
|
+
if (!name || !content)
|
|
28
|
+
continue;
|
|
29
|
+
const key = name.replace('twitter:', '');
|
|
30
|
+
if (key === 'title')
|
|
31
|
+
data.title = content;
|
|
32
|
+
else if (key === 'description')
|
|
33
|
+
data.description = content;
|
|
66
34
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
35
|
+
return data;
|
|
36
|
+
}
|
|
37
|
+
function extractStandardMeta(document) {
|
|
38
|
+
const data = {};
|
|
39
|
+
const metaTags = document.querySelectorAll('meta[name][content]');
|
|
40
|
+
for (const tag of metaTags) {
|
|
41
|
+
const name = tag.getAttribute('name');
|
|
42
|
+
const content = tag.getAttribute('content')?.trim();
|
|
43
|
+
if (!name || !content)
|
|
44
|
+
continue;
|
|
45
|
+
if (name === 'description')
|
|
46
|
+
data.description = content;
|
|
47
|
+
else if (name === 'author')
|
|
48
|
+
data.author = content;
|
|
49
|
+
}
|
|
50
|
+
if (!data.title) {
|
|
51
|
+
const titleEl = document.querySelector('title');
|
|
52
|
+
if (titleEl?.textContent)
|
|
53
|
+
data.title = titleEl.textContent.trim();
|
|
70
54
|
}
|
|
55
|
+
return data;
|
|
71
56
|
}
|
|
72
|
-
|
|
73
|
-
* Main extraction function - uses Cheerio for metadata (fast)
|
|
74
|
-
* and lazy-loads JSDOM only when article extraction is needed
|
|
75
|
-
*/
|
|
57
|
+
// Main extraction function
|
|
76
58
|
export function extractContent(html, url, options = { extractArticle: true }) {
|
|
77
59
|
if (!html || typeof html !== 'string') {
|
|
78
60
|
logWarn('extractContent called with invalid HTML input');
|
|
@@ -82,22 +64,42 @@ export function extractContent(html, url, options = { extractArticle: true }) {
|
|
|
82
64
|
logWarn('extractContent called with invalid URL');
|
|
83
65
|
return { article: null, metadata: {} };
|
|
84
66
|
}
|
|
85
|
-
let processedHtml = html;
|
|
86
|
-
if (html.length > MAX_HTML_SIZE) {
|
|
87
|
-
logWarn('HTML content exceeds maximum size for extraction, truncating', {
|
|
88
|
-
size: html.length,
|
|
89
|
-
maxSize: MAX_HTML_SIZE,
|
|
90
|
-
});
|
|
91
|
-
processedHtml = html.substring(0, MAX_HTML_SIZE);
|
|
92
|
-
}
|
|
93
67
|
try {
|
|
94
|
-
//
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
|
|
68
|
+
// Truncate HTML to improve performance
|
|
69
|
+
const processedHtml = truncateHtml(html);
|
|
70
|
+
// Parse HTML with JSDOM
|
|
71
|
+
const dom = new JSDOM(processedHtml, { url });
|
|
72
|
+
const { document } = dom.window;
|
|
73
|
+
const ogData = extractOpenGraph(document);
|
|
74
|
+
const twitterData = extractTwitterCard(document);
|
|
75
|
+
const standardData = extractStandardMeta(document);
|
|
76
|
+
const metadata = {
|
|
77
|
+
title: ogData.title ?? twitterData.title ?? standardData.title,
|
|
78
|
+
description: ogData.description ??
|
|
79
|
+
twitterData.description ??
|
|
80
|
+
standardData.description,
|
|
81
|
+
author: standardData.author,
|
|
82
|
+
};
|
|
83
|
+
let article = null;
|
|
84
|
+
if (options.extractArticle) {
|
|
85
|
+
try {
|
|
86
|
+
const reader = new Readability(document);
|
|
87
|
+
const parsed = reader.parse();
|
|
88
|
+
if (parsed) {
|
|
89
|
+
article = {
|
|
90
|
+
title: parsed.title ?? undefined,
|
|
91
|
+
byline: parsed.byline ?? undefined,
|
|
92
|
+
content: parsed.content ?? '',
|
|
93
|
+
textContent: parsed.textContent ?? '',
|
|
94
|
+
excerpt: parsed.excerpt ?? undefined,
|
|
95
|
+
siteName: parsed.siteName ?? undefined,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
logError('Failed to extract article with Readability', error instanceof Error ? error : undefined);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
101
103
|
return { article, metadata };
|
|
102
104
|
}
|
|
103
105
|
catch (error) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAQnD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEhD,SAAS,gBAAgB,CAAC,QAAkB;IAK1C,MAAM,IAAI,GAA6C,EAAE,CAAC;IAC1D,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IAElE,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;YAAE,SAAS;QAEpC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,GAAG,KAAK,OAAO;YAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;aACrC,IAAI,GAAG,KAAK,aAAa;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAkB;IAI5C,MAAM,IAAI,GAA6C,EAAE,CAAC;IAC1D,MAAM,WAAW,GAAG,QAAQ,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC;IAExE,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC;QACpD,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;YAAE,SAAS;QAEhC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,OAAO;YAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;aACrC,IAAI,GAAG,KAAK,aAAa;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAkB;IAK7C,MAAM,IAAI,GAA8D,EAAE,CAAC;IAE3E,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC;IAClE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC;QACpD,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;YAAE,SAAS;QAEhC,IAAI,IAAI,KAAK,aAAa;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;aAClD,IAAI,IAAI,KAAK,QAAQ;YAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;IACpD,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAChB,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,OAAO,EAAE,WAAW;YAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACpE,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,GAAW,EACX,UAAwC,EAAE,cAAc,EAAE,IAAI,EAAE;IAEhE,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,CAAC,+CAA+C,CAAC,CAAC;QACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC;QACH,uCAAuC;QACvC,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACzC,wBAAwB;QACxB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9C,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAsB;YAClC,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,WAAW,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK;YAC9D,WAAW,EACT,MAAM,CAAC,WAAW;gBAClB,WAAW,CAAC,WAAW;gBACvB,YAAY,CAAC,WAAW;YAC1B,MAAM,EAAE,YAAY,CAAC,MAAM;SAC5B,CAAC;QACF,IAAI,OAAO,GAA4B,IAAI,CAAC;QAC5C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACzC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;gBAE9B,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,GAAG;wBACR,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS;wBAChC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;wBAClC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;wBAC7B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,SAAS;wBACpC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,SAAS;qBACvC,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CACN,4CAA4C,EAC5C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;AACH,CAAC"}
|
|
@@ -1,15 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
export interface FetchOptions {
|
|
3
|
-
/** Custom HTTP headers to include in the request */
|
|
4
|
-
customHeaders?: Record<string, string>;
|
|
5
|
-
/** AbortSignal for request cancellation */
|
|
6
|
-
signal?: AbortSignal;
|
|
7
|
-
/** Per-request timeout override in milliseconds */
|
|
8
|
-
timeout?: number;
|
|
9
|
-
}
|
|
1
|
+
import type { FetchOptions } from '../config/types.js';
|
|
10
2
|
export declare function destroyAgents(): void;
|
|
11
|
-
export declare function fetchUrlWithRetry(url: string, options?: FetchOptions, maxRetries?: number
|
|
12
|
-
html: string;
|
|
13
|
-
fromHtmlCache: boolean;
|
|
14
|
-
}>;
|
|
3
|
+
export declare function fetchUrlWithRetry(url: string, options?: FetchOptions, maxRetries?: number): Promise<string>;
|
|
15
4
|
//# sourceMappingURL=fetcher.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAgEvD,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AAwND,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,YAAY,EACtB,UAAU,SAAI,GACb,OAAO,CAAC,MAAM,CAAC,CA4CjB"}
|
package/dist/services/fetcher.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import axios, { isCancel, } from 'axios';
|
|
2
|
+
import crypto from 'crypto';
|
|
2
3
|
import http from 'http';
|
|
3
4
|
import https from 'https';
|
|
4
5
|
import os from 'os';
|
|
5
6
|
import { config } from '../config/index.js';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
7
|
+
import { FetchError } from '../errors/app-error.js';
|
|
8
|
+
import { validateResolvedIps } from '../utils/url-validator.js';
|
|
8
9
|
import { logDebug, logError, logWarn } from './logger.js';
|
|
9
|
-
// Use Symbol for request timings (20-30% faster than WeakMap)
|
|
10
10
|
const REQUEST_START_TIME = Symbol('requestStartTime');
|
|
11
|
+
const REQUEST_ID = Symbol('requestId');
|
|
11
12
|
const BLOCKED_HEADERS = new Set([
|
|
12
13
|
'host',
|
|
13
14
|
'authorization',
|
|
@@ -22,19 +23,17 @@ function sanitizeHeaders(headers) {
|
|
|
22
23
|
const sanitized = {};
|
|
23
24
|
for (const [key, value] of Object.entries(headers)) {
|
|
24
25
|
if (!BLOCKED_HEADERS.has(key.toLowerCase())) {
|
|
26
|
+
// Prevent HTTP header injection via CRLF
|
|
27
|
+
if (/[\r\n]/.test(key) || /[\r\n]/.test(value)) {
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
25
30
|
sanitized[key] = value;
|
|
26
31
|
}
|
|
27
32
|
}
|
|
28
33
|
return Object.keys(sanitized).length > 0 ? sanitized : undefined;
|
|
29
34
|
}
|
|
30
|
-
function calculateBackoff(attempt, maxDelay = 10000) {
|
|
31
|
-
const baseDelay = Math.min(1000 * Math.pow(2, attempt - 1), maxDelay);
|
|
32
|
-
const jitter = baseDelay * 0.25 * (Math.random() * 2 - 1);
|
|
33
|
-
return Math.round(baseDelay + jitter);
|
|
34
|
-
}
|
|
35
|
-
// Dynamic connection pool sizing based on CPU cores (2-4x throughput on multi-core)
|
|
36
35
|
const CPU_COUNT = os.cpus().length;
|
|
37
|
-
const MAX_SOCKETS = Math.max(CPU_COUNT * 2, 25);
|
|
36
|
+
const MAX_SOCKETS = Math.max(CPU_COUNT * 2, 25);
|
|
38
37
|
const MAX_FREE_SOCKETS = Math.max(Math.floor(CPU_COUNT * 0.5), 10);
|
|
39
38
|
const httpAgent = new http.Agent({
|
|
40
39
|
keepAlive: true,
|
|
@@ -70,10 +69,11 @@ const client = axios.create({
|
|
|
70
69
|
validateStatus: (status) => status >= 200 && status < 300,
|
|
71
70
|
});
|
|
72
71
|
client.interceptors.request.use((requestConfig) => {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
72
|
+
const timedConfig = requestConfig;
|
|
73
|
+
timedConfig[REQUEST_START_TIME] = Date.now();
|
|
74
|
+
timedConfig[REQUEST_ID] = crypto.randomUUID().substring(0, 8);
|
|
76
75
|
logDebug('HTTP Request', {
|
|
76
|
+
requestId: timedConfig[REQUEST_ID],
|
|
77
77
|
method: requestConfig.method?.toUpperCase(),
|
|
78
78
|
url: requestConfig.url,
|
|
79
79
|
});
|
|
@@ -83,56 +83,53 @@ client.interceptors.request.use((requestConfig) => {
|
|
|
83
83
|
throw error;
|
|
84
84
|
});
|
|
85
85
|
client.interceptors.response.use((response) => {
|
|
86
|
-
const
|
|
87
|
-
const startTime =
|
|
86
|
+
const timedConfig = response.config;
|
|
87
|
+
const startTime = timedConfig[REQUEST_START_TIME];
|
|
88
|
+
const requestId = timedConfig[REQUEST_ID];
|
|
88
89
|
const duration = startTime ? Date.now() - startTime : 0;
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
if (timedConfig[REQUEST_START_TIME] !== undefined) {
|
|
91
|
+
timedConfig[REQUEST_START_TIME] = undefined;
|
|
92
|
+
}
|
|
93
|
+
if (timedConfig[REQUEST_ID] !== undefined) {
|
|
94
|
+
timedConfig[REQUEST_ID] = undefined;
|
|
92
95
|
}
|
|
93
96
|
const contentType = response.headers['content-type'];
|
|
94
97
|
const contentTypeStr = typeof contentType === 'string' ? contentType : undefined;
|
|
95
98
|
logDebug('HTTP Response', {
|
|
99
|
+
requestId,
|
|
96
100
|
status: response.status,
|
|
97
101
|
url: response.config.url ?? 'unknown',
|
|
98
102
|
contentType: contentTypeStr,
|
|
99
103
|
duration: `${duration}ms`,
|
|
100
104
|
size: response.headers['content-length'],
|
|
101
105
|
});
|
|
102
|
-
// Log slow requests
|
|
103
106
|
if (duration > 5000) {
|
|
104
107
|
logWarn('Slow HTTP request detected', {
|
|
108
|
+
requestId,
|
|
105
109
|
url: response.config.url ?? 'unknown',
|
|
106
110
|
duration: `${duration}ms`,
|
|
107
111
|
});
|
|
108
112
|
}
|
|
109
|
-
// Early content-type validation before processing
|
|
110
|
-
if (contentTypeStr && !isHtmlContentType(contentTypeStr)) {
|
|
111
|
-
throw new FetchError(`Unexpected content type: ${contentTypeStr}. Expected HTML content.`, response.config.url ?? 'unknown');
|
|
112
|
-
}
|
|
113
113
|
return response;
|
|
114
114
|
}, (error) => {
|
|
115
115
|
const url = error.config?.url ?? 'unknown';
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
logDebug('HTTP Request Canceled (timeout signal)', { url });
|
|
124
|
-
throw new TimeoutError(config.fetcher.timeout, true);
|
|
116
|
+
if (isCancel(error) ||
|
|
117
|
+
error.name === 'AbortError' ||
|
|
118
|
+
error.name === 'CanceledError') {
|
|
119
|
+
logDebug('HTTP Request Aborted/Canceled', { url });
|
|
120
|
+
throw new FetchError('Request was canceled', url, 499, {
|
|
121
|
+
reason: 'aborted',
|
|
122
|
+
});
|
|
125
123
|
}
|
|
126
124
|
if (error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT') {
|
|
127
125
|
logError('HTTP Timeout', { url, timeout: config.fetcher.timeout });
|
|
128
|
-
throw new
|
|
126
|
+
throw new FetchError(`Request timeout after ${config.fetcher.timeout}ms`, url, 504, { timeout: config.fetcher.timeout });
|
|
129
127
|
}
|
|
130
128
|
if (error.response) {
|
|
131
129
|
const { status, statusText, headers } = error.response;
|
|
132
|
-
// Handle 429 Too Many Requests with Retry-After header
|
|
133
130
|
if (status === 429) {
|
|
134
131
|
const retryAfterHeader = headers['retry-after'];
|
|
135
|
-
let retryAfterSeconds = 60;
|
|
132
|
+
let retryAfterSeconds = 60;
|
|
136
133
|
if (retryAfterHeader) {
|
|
137
134
|
const parsed = parseInt(retryAfterHeader, 10);
|
|
138
135
|
if (!isNaN(parsed)) {
|
|
@@ -143,7 +140,9 @@ client.interceptors.response.use((response) => {
|
|
|
143
140
|
url,
|
|
144
141
|
retryAfter: `${retryAfterSeconds}s`,
|
|
145
142
|
});
|
|
146
|
-
throw new
|
|
143
|
+
throw new FetchError('Too many requests', url, 429, {
|
|
144
|
+
retryAfter: retryAfterSeconds,
|
|
145
|
+
});
|
|
147
146
|
}
|
|
148
147
|
logError('HTTP Error Response', { url, status, statusText });
|
|
149
148
|
throw new FetchError(`HTTP ${status}: ${statusText}`, url, status);
|
|
@@ -156,18 +155,25 @@ client.interceptors.response.use((response) => {
|
|
|
156
155
|
throw new FetchError(error.message, url);
|
|
157
156
|
});
|
|
158
157
|
async function fetchUrl(url, options) {
|
|
158
|
+
try {
|
|
159
|
+
const urlObj = new URL(url);
|
|
160
|
+
await validateResolvedIps(urlObj.hostname);
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
if (error instanceof Error) {
|
|
164
|
+
throw new FetchError(error.message, url);
|
|
165
|
+
}
|
|
166
|
+
throw error;
|
|
167
|
+
}
|
|
159
168
|
const requestConfig = {
|
|
160
169
|
method: 'GET',
|
|
161
170
|
url,
|
|
162
171
|
responseType: 'text',
|
|
163
172
|
};
|
|
164
|
-
// Apply per-request timeout via AbortSignal.timeout() if provided
|
|
165
|
-
// This is cleaner than axios timeout as it properly cancels the request
|
|
166
173
|
if (options?.signal) {
|
|
167
174
|
requestConfig.signal = options.signal;
|
|
168
175
|
}
|
|
169
176
|
else if (options?.timeout) {
|
|
170
|
-
// Use AbortSignal.timeout() for per-request timeout (Node 17.3+)
|
|
171
177
|
requestConfig.signal = AbortSignal.timeout(options.timeout);
|
|
172
178
|
}
|
|
173
179
|
const sanitized = sanitizeHeaders(options?.customHeaders);
|
|
@@ -182,53 +188,51 @@ async function fetchUrl(url, options) {
|
|
|
182
188
|
return response.data;
|
|
183
189
|
}
|
|
184
190
|
catch (error) {
|
|
185
|
-
if (error instanceof FetchError
|
|
186
|
-
error instanceof TimeoutError ||
|
|
187
|
-
error instanceof AbortError ||
|
|
188
|
-
error instanceof RateLimitError) {
|
|
191
|
+
if (error instanceof FetchError) {
|
|
189
192
|
throw error;
|
|
190
193
|
}
|
|
191
194
|
throw new FetchError(`Unexpected error: ${error instanceof Error ? error.message : 'Unknown'}`, url);
|
|
192
195
|
}
|
|
193
196
|
}
|
|
194
|
-
function
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
197
|
+
function calculateRetryDelay(attempt) {
|
|
198
|
+
const baseDelayMs = 1000;
|
|
199
|
+
const maxDelayMs = 10000;
|
|
200
|
+
const jitterFactor = 0.25;
|
|
201
|
+
const exponentialDelay = Math.min(baseDelayMs * Math.pow(2, attempt - 1), maxDelayMs);
|
|
202
|
+
const jitter = exponentialDelay * jitterFactor * (Math.random() * 2 - 1);
|
|
203
|
+
return Math.round(exponentialDelay + jitter);
|
|
199
204
|
}
|
|
200
|
-
|
|
201
|
-
if (
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
205
|
+
function shouldRetryError(attempt, maxRetries, error) {
|
|
206
|
+
if (attempt >= maxRetries)
|
|
207
|
+
return false;
|
|
208
|
+
// Don't retry aborted requests
|
|
209
|
+
if (error instanceof FetchError && error.details.reason === 'aborted')
|
|
210
|
+
return false;
|
|
211
|
+
// Don't retry on client errors (4xx except 429)
|
|
212
|
+
if (error instanceof FetchError) {
|
|
213
|
+
const status = error.details.httpStatus;
|
|
214
|
+
if (status && status >= 400 && status < 500 && status !== 429)
|
|
215
|
+
return false;
|
|
207
216
|
}
|
|
217
|
+
return true;
|
|
218
|
+
}
|
|
219
|
+
export async function fetchUrlWithRetry(url, options, maxRetries = 3) {
|
|
208
220
|
const retries = Math.min(Math.max(1, maxRetries), 10);
|
|
209
221
|
let lastError = new Error(`Failed to fetch ${url}`);
|
|
210
222
|
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
211
|
-
// Check if aborted before attempting (early exit for batch operations)
|
|
212
223
|
if (options?.signal?.aborted) {
|
|
213
|
-
|
|
214
|
-
throw abortError;
|
|
224
|
+
throw new FetchError('Request was aborted before execution', url);
|
|
215
225
|
}
|
|
216
226
|
try {
|
|
217
|
-
|
|
218
|
-
setHtml(url, html);
|
|
219
|
-
logDebug('HTML Cache Set', { url });
|
|
220
|
-
return { html, fromHtmlCache: false };
|
|
227
|
+
return await fetchUrl(url, options);
|
|
221
228
|
}
|
|
222
229
|
catch (error) {
|
|
223
230
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
224
|
-
//
|
|
225
|
-
if (error instanceof
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
if (error instanceof RateLimitError) {
|
|
230
|
-
if (attempt < retries) {
|
|
231
|
-
const waitTime = Math.min(error.retryAfter * 1000, 30000); // Cap at 30s
|
|
231
|
+
// Handle rate limiting with smart retry (429 with Retry-After)
|
|
232
|
+
if (error instanceof FetchError && error.details.httpStatus === 429) {
|
|
233
|
+
const retryAfter = error.details.retryAfter;
|
|
234
|
+
if (attempt < retries && retryAfter) {
|
|
235
|
+
const waitTime = Math.min(retryAfter * 1000, 30000);
|
|
232
236
|
logWarn('Rate limited, waiting before retry', {
|
|
233
237
|
url,
|
|
234
238
|
attempt,
|
|
@@ -239,18 +243,14 @@ export async function fetchUrlWithRetry(url, options, maxRetries = 3, skipCache
|
|
|
239
243
|
}
|
|
240
244
|
throw error;
|
|
241
245
|
}
|
|
242
|
-
if (
|
|
243
|
-
const
|
|
244
|
-
// Don't retry client errors (except 429 which is handled above)
|
|
245
|
-
if (status >= 400 && status < 500 && status !== 429) {
|
|
246
|
-
throw error;
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
if (attempt < retries) {
|
|
250
|
-
const delay = calculateBackoff(attempt);
|
|
246
|
+
if (shouldRetryError(attempt, retries, lastError)) {
|
|
247
|
+
const delay = calculateRetryDelay(attempt);
|
|
251
248
|
logDebug('Retrying request', { url, attempt, delay: `${delay}ms` });
|
|
252
249
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
253
250
|
}
|
|
251
|
+
else {
|
|
252
|
+
throw error;
|
|
253
|
+
}
|
|
254
254
|
}
|
|
255
255
|
}
|
|
256
256
|
throw new FetchError(`Failed after ${retries} attempts: ${lastError.message}`, url);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAGZ,QAAQ,GACT,MAAM,OAAO,CAAC;AACf,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAGZ,QAAQ,GACT,MAAM,OAAO,CAAC;AACf,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEhE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE1D,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;AACtD,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;AAOvC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,MAAM;IACN,eAAe;IACf,QAAQ;IACR,iBAAiB;IACjB,WAAW;IACX,qBAAqB;CACtB,CAAC,CAAC;AAEH,SAAS,eAAe,CACtB,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAEpE,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC5C,yCAAyC;YACzC,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED,MAAM,SAAS,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;AACnC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;AAChD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;AAEnE,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC;IAC/B,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,WAAW;IACvB,cAAc,EAAE,gBAAgB;IAChC,OAAO,EAAE,KAAK;IACd,UAAU,EAAE,MAAM;CACnB,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC;IACjC,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,WAAW;IACvB,cAAc,EAAE,gBAAgB;IAChC,OAAO,EAAE,KAAK;IACd,UAAU,EAAE,MAAM;CACnB,CAAC,CAAC;AAEH,MAAM,UAAU,aAAa;IAC3B,SAAS,CAAC,OAAO,EAAE,CAAC;IACpB,UAAU,CAAC,OAAO,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;IAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;IACzC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB;IACjD,SAAS;IACT,UAAU;IACV,OAAO,EAAE;QACP,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;QACtC,MAAM,EACJ,4EAA4E;QAC9E,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,UAAU,EAAE,YAAY;KACzB;IACD,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;CAC1D,CAAC,CAAC;AAEH,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAC7B,CAAC,aAAa,EAAE,EAAE;IAChB,MAAM,WAAW,GAAG,aAAwC,CAAC;IAC7D,WAAW,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7C,WAAW,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9D,QAAQ,CAAC,cAAc,EAAE;QACvB,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC;QAClC,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE;QAC3C,GAAG,EAAE,aAAa,CAAC,GAAG;KACvB,CAAC,CAAC;IACH,OAAO,aAAa,CAAC;AACvB,CAAC,EACD,CAAC,KAAiB,EAAE,EAAE;IACpB,QAAQ,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC;AACd,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAC9B,CAAC,QAAQ,EAAE,EAAE;IACX,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAiC,CAAC;IAC/D,MAAM,SAAS,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAExD,IAAI,WAAW,CAAC,kBAAkB,CAAC,KAAK,SAAS,EAAE,CAAC;QAClD,WAAW,CAAC,kBAAkB,CAAC,GAAG,SAAS,CAAC;IAC9C,CAAC;IACD,IAAI,WAAW,CAAC,UAAU,CAAC,KAAK,SAAS,EAAE,CAAC;QAC1C,WAAW,CAAC,UAAU,CAAC,GAAG,SAAS,CAAC;IACtC,CAAC;IAED,MAAM,WAAW,GAAY,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC9D,MAAM,cAAc,GAClB,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5D,QAAQ,CAAC,eAAe,EAAE;QACxB,SAAS;QACT,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;QACrC,WAAW,EAAE,cAAc;QAC3B,QAAQ,EAAE,GAAG,QAAQ,IAAI;QACzB,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,IAAI,EAAE,CAAC;QACpB,OAAO,CAAC,4BAA4B,EAAE;YACpC,SAAS;YACT,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;YACrC,QAAQ,EAAE,GAAG,QAAQ,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,EACD,CAAC,KAAiB,EAAE,EAAE;IACpB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC;IAE3C,IACE,QAAQ,CAAC,KAAK,CAAC;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,KAAK,CAAC,IAAI,KAAK,eAAe,EAC9B,CAAC;QACD,QAAQ,CAAC,+BAA+B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,UAAU,CAAC,sBAAsB,EAAE,GAAG,EAAE,GAAG,EAAE;YACrD,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChE,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,UAAU,CAClB,yBAAyB,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,EACnD,GAAG,EACH,GAAG,EACH,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CACpC,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QAEvD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAuB,CAAC;YACtE,IAAI,iBAAiB,GAAG,EAAE,CAAC;YAE3B,IAAI,gBAAgB,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnB,iBAAiB,GAAG,MAAM,CAAC;gBAC7B,CAAC;YACH,CAAC;YAED,OAAO,CAAC,wBAAwB,EAAE;gBAChC,GAAG;gBACH,UAAU,EAAE,GAAG,iBAAiB,GAAG;aACpC,CAAC,CAAC;YACH,MAAM,IAAI,UAAU,CAAC,mBAAmB,EAAE,GAAG,EAAE,GAAG,EAAE;gBAClD,UAAU,EAAE,iBAAiB;aAC9B,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,UAAU,CAAC,QAAQ,MAAM,KAAK,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,kCAAkC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,QAAQ,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC3C,CAAC,CACF,CAAC;AAEF,KAAK,UAAU,QAAQ,CAAC,GAAW,EAAE,OAAsB;IACzD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,aAAa,GAAuB;QACxC,MAAM,EAAE,KAAK;QACb,GAAG;QACH,YAAY,EAAE,MAAM;KACrB,CAAC;IAEF,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACxC,CAAC;SAAM,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;QAC5B,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC1D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,eAAe,GACnB,aAAa,CAAC,OAAO,IAAI,OAAO,aAAa,CAAC,OAAO,KAAK,QAAQ;YAChE,CAAC,CAAE,aAAa,CAAC,OAAkC;YACnD,CAAC,CAAC,EAAE,CAAC;QACT,aAAa,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,SAAS,EAAE,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAS,aAAa,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,IAAI,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YAChC,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,IAAI,UAAU,CAClB,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EACzE,GAAG,CACJ,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAe;IAC1C,MAAM,WAAW,GAAG,IAAI,CAAC;IACzB,MAAM,UAAU,GAAG,KAAK,CAAC;IACzB,MAAM,YAAY,GAAG,IAAI,CAAC;IAE1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EACtC,UAAU,CACX,CAAC;IACF,MAAM,MAAM,GAAG,gBAAgB,GAAG,YAAY,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACzE,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAe,EACf,UAAkB,EAClB,KAAY;IAEZ,IAAI,OAAO,IAAI,UAAU;QAAE,OAAO,KAAK,CAAC;IAExC,+BAA+B;IAC/B,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS;QACnE,OAAO,KAAK,CAAC;IAEf,gDAAgD;IAChD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,UAAgC,CAAC;QAC9D,IAAI,MAAM,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,GAAG;YAAE,OAAO,KAAK,CAAC;IAC9E,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC;IAEd,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;IACtD,IAAI,SAAS,GAAU,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IAE3D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,UAAU,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;QACpE,CAAC;QAED,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,+DAA+D;YAC/D,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBACpE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,UAAoB,CAAC;gBACtD,IAAI,OAAO,GAAG,OAAO,IAAI,UAAU,EAAE,CAAC;oBACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;oBACpD,OAAO,CAAC,oCAAoC,EAAE;wBAC5C,GAAG;wBACH,OAAO;wBACP,QAAQ,EAAE,GAAG,QAAQ,IAAI;qBAC1B,CAAC,CAAC;oBACH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAC9D,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;gBAClD,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;gBAC3C,QAAQ,CAAC,kBAAkB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,KAAK,IAAI,EAAE,CAAC,CAAC;gBACpE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,UAAU,CAClB,gBAAgB,OAAO,cAAc,SAAS,CAAC,OAAO,EAAE,EACxD,GAAG,CACJ,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
export declare function
|
|
3
|
-
export declare function
|
|
4
|
-
export declare function
|
|
1
|
+
import type { LogMetadata } from '../config/types.js';
|
|
2
|
+
export declare function logInfo(message: string, meta?: LogMetadata): void;
|
|
3
|
+
export declare function logDebug(message: string, meta?: LogMetadata): void;
|
|
4
|
+
export declare function logWarn(message: string, meta?: LogMetadata): void;
|
|
5
|
+
export declare function logError(message: string, error?: Error | LogMetadata): void;
|
|
5
6
|
//# sourceMappingURL=logger.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAwBhE,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIlE;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAS3E"}
|