@j0hanz/superfetch 2.5.3 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +350 -226
- package/dist/assets/logo.svg +24837 -24835
- package/dist/cache.d.ts +28 -20
- package/dist/cache.js +292 -514
- package/dist/config.d.ts +41 -7
- package/dist/config.js +298 -148
- package/dist/crypto.js +25 -12
- package/dist/dom-noise-removal.js +379 -421
- package/dist/errors.d.ts +2 -2
- package/dist/errors.js +25 -8
- package/dist/fetch.d.ts +18 -16
- package/dist/fetch.js +1132 -526
- package/dist/host-normalization.js +40 -10
- package/dist/http-native.js +628 -287
- package/dist/index.js +67 -7
- package/dist/instructions.md +44 -31
- package/dist/ip-blocklist.d.ts +8 -0
- package/dist/ip-blocklist.js +65 -0
- package/dist/json.js +14 -9
- package/dist/language-detection.d.ts +2 -11
- package/dist/language-detection.js +289 -280
- package/dist/markdown-cleanup.d.ts +0 -1
- package/dist/markdown-cleanup.js +391 -429
- package/dist/mcp-validator.js +4 -2
- package/dist/mcp.js +184 -135
- package/dist/observability.js +89 -21
- package/dist/resources.js +16 -6
- package/dist/server-tuning.d.ts +2 -0
- package/dist/server-tuning.js +25 -23
- package/dist/session.d.ts +1 -0
- package/dist/session.js +41 -33
- package/dist/tasks.d.ts +2 -0
- package/dist/tasks.js +91 -9
- package/dist/timer-utils.d.ts +5 -0
- package/dist/timer-utils.js +20 -0
- package/dist/tools.d.ts +28 -5
- package/dist/tools.js +317 -183
- package/dist/transform-types.d.ts +5 -1
- package/dist/transform.d.ts +3 -2
- package/dist/transform.js +1138 -421
- package/dist/type-guards.d.ts +1 -0
- package/dist/type-guards.js +7 -0
- package/dist/workers/transform-child.d.ts +1 -0
- package/dist/workers/transform-child.js +118 -0
- package/dist/workers/transform-worker.js +87 -78
- package/package.json +14 -6
package/dist/errors.d.ts
CHANGED
|
@@ -3,8 +3,8 @@ export declare class FetchError extends Error {
|
|
|
3
3
|
readonly statusCode: number;
|
|
4
4
|
readonly code: string;
|
|
5
5
|
readonly details: Readonly<Record<string, unknown>>;
|
|
6
|
-
constructor(message: string, url: string, httpStatus?: number, details?: Record<string, unknown
|
|
6
|
+
constructor(message: string, url: string, httpStatus?: number, details?: Record<string, unknown>, options?: ErrorOptions);
|
|
7
7
|
}
|
|
8
8
|
export declare function getErrorMessage(error: unknown): string;
|
|
9
|
-
export declare function createErrorWithCode(message: string, code: string): NodeJS.ErrnoException;
|
|
9
|
+
export declare function createErrorWithCode(message: string, code: string, options?: ErrorOptions): NodeJS.ErrnoException;
|
|
10
10
|
export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;
|
package/dist/errors.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
import { isError, isObject } from './type-guards.js';
|
|
2
3
|
const DEFAULT_HTTP_STATUS = 502;
|
|
3
4
|
export class FetchError extends Error {
|
|
4
5
|
url;
|
|
5
6
|
statusCode;
|
|
6
7
|
code;
|
|
7
8
|
details;
|
|
8
|
-
constructor(message, url, httpStatus, details = {}) {
|
|
9
|
-
super(message);
|
|
9
|
+
constructor(message, url, httpStatus, details = {}, options) {
|
|
10
|
+
super(message, options);
|
|
10
11
|
this.url = url;
|
|
11
12
|
this.name = 'FetchError';
|
|
12
13
|
this.statusCode = httpStatus ?? DEFAULT_HTTP_STATUS;
|
|
@@ -16,13 +17,13 @@ export class FetchError extends Error {
|
|
|
16
17
|
}
|
|
17
18
|
}
|
|
18
19
|
export function getErrorMessage(error) {
|
|
19
|
-
if (error
|
|
20
|
+
if (isError(error))
|
|
20
21
|
return error.message;
|
|
21
22
|
if (typeof error === 'string' && error.length > 0)
|
|
22
23
|
return error;
|
|
23
24
|
if (isErrorWithMessage(error))
|
|
24
25
|
return error.message;
|
|
25
|
-
return
|
|
26
|
+
return formatUnknownError(error);
|
|
26
27
|
}
|
|
27
28
|
function isErrorWithMessage(error) {
|
|
28
29
|
if (!isObject(error))
|
|
@@ -30,12 +31,28 @@ function isErrorWithMessage(error) {
|
|
|
30
31
|
const { message } = error;
|
|
31
32
|
return typeof message === 'string' && message.length > 0;
|
|
32
33
|
}
|
|
33
|
-
|
|
34
|
-
|
|
34
|
+
function formatUnknownError(error) {
|
|
35
|
+
if (error === null || error === undefined)
|
|
36
|
+
return 'Unknown error';
|
|
37
|
+
try {
|
|
38
|
+
return inspect(error, {
|
|
39
|
+
depth: 2,
|
|
40
|
+
maxStringLength: 200,
|
|
41
|
+
breakLength: Infinity,
|
|
42
|
+
compact: true,
|
|
43
|
+
colors: false,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return 'Unknown error';
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export function createErrorWithCode(message, code, options) {
|
|
51
|
+
const error = new Error(message, options);
|
|
35
52
|
return Object.assign(error, { code });
|
|
36
53
|
}
|
|
37
54
|
export function isSystemError(error) {
|
|
38
|
-
if (!(error
|
|
55
|
+
if (!isError(error))
|
|
39
56
|
return false;
|
|
40
57
|
if (!('code' in error))
|
|
41
58
|
return false;
|
package/dist/fetch.d.ts
CHANGED
|
@@ -6,17 +6,6 @@ export interface TransformResult {
|
|
|
6
6
|
readonly transformed: boolean;
|
|
7
7
|
readonly platform?: string;
|
|
8
8
|
}
|
|
9
|
-
/** Backwards-compatible export */
|
|
10
|
-
export declare function isBlockedIp(ip: string): boolean;
|
|
11
|
-
/** Backwards-compatible exports */
|
|
12
|
-
export declare function normalizeUrl(urlString: string): {
|
|
13
|
-
normalizedUrl: string;
|
|
14
|
-
hostname: string;
|
|
15
|
-
};
|
|
16
|
-
export declare function validateAndNormalizeUrl(urlString: string): string;
|
|
17
|
-
/** Backwards-compatible exports */
|
|
18
|
-
export declare function transformToRawUrl(url: string): TransformResult;
|
|
19
|
-
export declare function isRawTextContentUrl(url: string): boolean;
|
|
20
9
|
export interface FetchTelemetryContext {
|
|
21
10
|
requestId: string;
|
|
22
11
|
startTime: number;
|
|
@@ -25,19 +14,32 @@ export interface FetchTelemetryContext {
|
|
|
25
14
|
contextRequestId?: string;
|
|
26
15
|
operationId?: string;
|
|
27
16
|
}
|
|
28
|
-
|
|
17
|
+
export declare function isBlockedIp(ip: string): boolean;
|
|
18
|
+
export declare function normalizeUrl(urlString: string): {
|
|
19
|
+
normalizedUrl: string;
|
|
20
|
+
hostname: string;
|
|
21
|
+
};
|
|
22
|
+
export declare function validateAndNormalizeUrl(urlString: string): string;
|
|
23
|
+
export declare function transformToRawUrl(url: string): TransformResult;
|
|
24
|
+
export declare function isRawTextContentUrl(url: string): boolean;
|
|
29
25
|
export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
|
|
30
26
|
export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
|
|
31
27
|
export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
|
|
32
|
-
/** Backwards-compatible export */
|
|
33
28
|
export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
|
|
34
29
|
response: Response;
|
|
35
30
|
url: string;
|
|
36
31
|
}>;
|
|
37
|
-
|
|
38
|
-
export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal): Promise<{
|
|
32
|
+
export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal, encoding?: string): Promise<{
|
|
39
33
|
text: string;
|
|
40
34
|
size: number;
|
|
41
35
|
}>;
|
|
42
|
-
|
|
36
|
+
export declare function readResponseBuffer(response: Response, url: string, maxBytes: number, signal?: AbortSignal, encoding?: string): Promise<{
|
|
37
|
+
buffer: Uint8Array;
|
|
38
|
+
encoding: string;
|
|
39
|
+
size: number;
|
|
40
|
+
}>;
|
|
43
41
|
export declare function fetchNormalizedUrl(normalizedUrl: string, options?: FetchOptions): Promise<string>;
|
|
42
|
+
export declare function fetchNormalizedUrlBuffer(normalizedUrl: string, options?: FetchOptions): Promise<{
|
|
43
|
+
buffer: Uint8Array;
|
|
44
|
+
encoding: string;
|
|
45
|
+
}>;
|