@j0hanz/superfetch 1.2.0 → 1.2.2

package/dist/tools/schemas.d.ts

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 export declare const fetchUrlInputSchema: z.ZodObject<{
-    customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    customHeaders: z.ZodOptional<z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodString>, Record<string, string>, Record<string, string>>>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
 } & {
@@ -13,25 +13,25 @@ export declare const fetchUrlInputSchema: z.ZodObject<{
     format: z.ZodDefault<z.ZodEnum<["jsonl", "markdown"]>>;
 }, "strict", z.ZodTypeAny, {
     url: string;
+    timeout: number;
+    retries: number;
     extractMainContent: boolean;
     includeMetadata: boolean;
     format: "jsonl" | "markdown";
-    timeout: number;
-    retries: number;
-    maxContentLength?: number | undefined;
     customHeaders?: Record<string, string> | undefined;
+    maxContentLength?: number | undefined;
 }, {
     url: string;
+    customHeaders?: Record<string, string> | undefined;
+    timeout?: number | undefined;
+    retries?: number | undefined;
     extractMainContent?: boolean | undefined;
     includeMetadata?: boolean | undefined;
     maxContentLength?: number | undefined;
     format?: "jsonl" | "markdown" | undefined;
-    customHeaders?: Record<string, string> | undefined;
-    timeout?: number | undefined;
-    retries?: number | undefined;
 }>;
 export declare const fetchMarkdownInputSchema: z.ZodObject<{
-    customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    customHeaders: z.ZodOptional<z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodString>, Record<string, string>, Record<string, string>>>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
 } & {
@@ -42,20 +42,20 @@ export declare const fetchMarkdownInputSchema: z.ZodObject<{
     maxContentLength: z.ZodOptional<z.ZodNumber>;
 }, "strict", z.ZodTypeAny, {
     url: string;
-    extractMainContent: boolean;
-    includeMetadata: boolean;
     timeout: number;
     retries: number;
-    maxContentLength?: number | undefined;
+    extractMainContent: boolean;
+    includeMetadata: boolean;
     customHeaders?: Record<string, string> | undefined;
+    maxContentLength?: number | undefined;
 }, {
     url: string;
-    extractMainContent?: boolean | undefined;
-    includeMetadata?: boolean | undefined;
-    maxContentLength?: number | undefined;
     customHeaders?: Record<string, string> | undefined;
     timeout?: number | undefined;
     retries?: number | undefined;
+    extractMainContent?: boolean | undefined;
+    includeMetadata?: boolean | undefined;
+    maxContentLength?: number | undefined;
 }>;
 export declare const fetchUrlOutputSchema: z.ZodObject<{
     url: z.ZodString;
@@ -75,31 +75,31 @@ export declare const fetchUrlOutputSchema: z.ZodObject<{
 }, "strict", z.ZodTypeAny, {
     url: string;
     format: "jsonl" | "markdown";
+    contentBlocks: number;
     fetchedAt: string;
     cached: boolean;
-    contentBlocks: number;
     error?: string | undefined;
-    content?: string | undefined;
-    errorCode?: string | undefined;
     title?: string | undefined;
+    content?: string | undefined;
     contentSize?: number | undefined;
-    truncated?: boolean | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
+    truncated?: boolean | undefined;
+    errorCode?: string | undefined;
 }, {
     url: string;
     format: "jsonl" | "markdown";
+    contentBlocks: number;
     fetchedAt: string;
     cached: boolean;
-    contentBlocks: number;
     error?: string | undefined;
-    content?: string | undefined;
-    errorCode?: string | undefined;
     title?: string | undefined;
+    content?: string | undefined;
     contentSize?: number | undefined;
-    truncated?: boolean | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
+    truncated?: boolean | undefined;
+    errorCode?: string | undefined;
 }>;
 export declare const fetchMarkdownOutputSchema: z.ZodObject<{
     url: z.ZodString;
@@ -111,12 +111,12 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
         fileName: z.ZodString;
         expiresAt: z.ZodString;
     }, "strip", z.ZodTypeAny, {
-        fileName: string;
         downloadUrl: string;
+        fileName: string;
         expiresAt: string;
     }, {
-        fileName: string;
         downloadUrl: string;
+        fileName: string;
         expiresAt: string;
     }>>;
 } & {
@@ -133,33 +133,32 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
     cached: boolean;
     error?: string | undefined;
     markdown?: string | undefined;
-    errorCode?: string | undefined;
     title?: string | undefined;
     contentSize?: number | undefined;
+    resourceUri?: string | undefined;
+    resourceMimeType?: string | undefined;
     truncated?: boolean | undefined;
+    errorCode?: string | undefined;
     file?: {
-        fileName: string;
         downloadUrl: string;
+        fileName: string;
         expiresAt: string;
     } | undefined;
-    resourceUri?: string | undefined;
-    resourceMimeType?: string | undefined;
 }, {
     url: string;
     fetchedAt: string;
     cached: boolean;
     error?: string | undefined;
     markdown?: string | undefined;
-    errorCode?: string | undefined;
     title?: string | undefined;
     contentSize?: number | undefined;
+    resourceUri?: string | undefined;
+    resourceMimeType?: string | undefined;
     truncated?: boolean | undefined;
+    errorCode?: string | undefined;
     file?: {
-        fileName: string;
         downloadUrl: string;
+        fileName: string;
         expiresAt: string;
     } | undefined;
-    resourceUri?: string | undefined;
-    resourceMimeType?: string | undefined;
 }>;
-//# sourceMappingURL=schemas.d.ts.map

package/dist/tools/schemas.js

@@ -1,8 +1,16 @@
 import { z } from 'zod';
 import { config } from '../config/index.js';
+const MAX_HEADER_NAME_LENGTH = 128;
+const MAX_HEADER_VALUE_LENGTH = 2048;
+const MAX_HEADER_COUNT = 50;
+const MAX_CONTENT_LENGTH = config.constants.maxContentSize;
+const customHeadersSchema = z
+    .record(z.string().max(MAX_HEADER_NAME_LENGTH), z.string().max(MAX_HEADER_VALUE_LENGTH))
+    .refine((headers) => Object.keys(headers).length <= MAX_HEADER_COUNT, {
+    message: `customHeaders must have at most ${MAX_HEADER_COUNT} entries`,
+});
 const requestOptionsSchema = z.object({
-    customHeaders: z
-        .record(z.string())
+    customHeaders: customHeadersSchema
         .optional()
         .describe('Custom HTTP headers for the request'),
     timeout: z
@@ -30,6 +38,7 @@ const extractionOptionsSchema = z.object({
     maxContentLength: z
         .number()
         .positive()
+        .max(MAX_CONTENT_LENGTH)
         .optional()
         .describe('Maximum content length in characters'),
 });
@@ -64,14 +73,22 @@ const fileDownloadSchema = z.object({
 });
 export const fetchUrlInputSchema = requestOptionsSchema
     .extend({
-    url: z.string().min(1).describe('The URL to fetch'),
+    url: z
+        .string()
+        .min(1)
+        .max(config.constants.maxUrlLength)
+        .describe('The URL to fetch'),
 })
     .merge(extractionOptionsSchema)
     .merge(formatOptionsSchema)
     .strict();
 export const fetchMarkdownInputSchema = requestOptionsSchema
     .extend({
-    url: z.string().min(1).describe('The URL to fetch'),
+    url: z
+        .string()
+        .min(1)
+        .max(config.constants.maxUrlLength)
+        .describe('The URL to fetch'),
 })
     .merge(extractionOptionsSchema)
     .strict();
@@ -110,4 +127,3 @@ export const fetchMarkdownOutputSchema = z
 })
     .merge(resourceFieldsSchema)
     .strict();
-//# sourceMappingURL=schemas.js.map

package/dist/tools/utils/cache-vary.d.ts

@@ -1,2 +1 @@
 export declare function appendHeaderVary(cacheVary: Record<string, unknown> | string | undefined, customHeaders?: Record<string, string>): Record<string, unknown> | string | undefined;
-//# sourceMappingURL=cache-vary.d.ts.map

package/dist/tools/utils/cache-vary.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cache-vary.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/cache-vary.ts"],"names":[],"mappings":"AAYA,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,SAAS,EACvD,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACrC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,SAAS,CAG9C"}
+{"version":3,"file":"cache-vary.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/cache-vary.ts"],"names":[],"mappings":"AAIA,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,SAAS,EACvD,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACrC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,SAAS,CAY9C"}

package/dist/tools/utils/cache-vary.js

@@ -1,30 +1,12 @@
 import { config } from '../../config/index.js';
 import { normalizeHeaderRecord } from '../../utils/header-normalizer.js';
-function normalizeHeadersForCache(headers) {
-    return normalizeHeaderRecord(headers, config.security.blockedHeaders, {
-        trimValues: true,
-    });
-}
 export function appendHeaderVary(cacheVary, customHeaders) {
-    const headerVary = normalizeHeadersForCache(customHeaders);
-    return mergeCacheVary(cacheVary, headerVary);
-}
-function mergeCacheVary(cacheVary, headerVary) {
-    if (!cacheVary && !headerVary)
-        return undefined;
-    if (typeof cacheVary === 'string') {
-        return buildStringVary(cacheVary, headerVary);
-    }
-    return mergeObjectVary(cacheVary, headerVary);
-}
-function mergeObjectVary(cacheVary, headerVary) {
-    if (!headerVary)
+    const headers = normalizeHeaderRecord(customHeaders, config.security.blockedHeaders, { trimValues: true });
+    if (!headers)
         return cacheVary;
-    return { ...(cacheVary ?? {}), headers: headerVary };
-}
-function buildStringVary(key, headerVary) {
-    if (!headerVary)
-        return { key };
-    return { key, headers: headerVary };
+    if (!cacheVary)
+        return { headers };
+    return typeof cacheVary === 'string'
+        ? { key: cacheVary, headers }
+        : { ...cacheVary, headers };
 }
-//# sourceMappingURL=cache-vary.js.map

package/dist/tools/utils/cache-vary.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache-vary.js","sourceRoot":"","sources":["../../../src/tools/utils/cache-vary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,SAAS,wBAAwB,CAC/B,OAAgC;IAEhC,OAAO,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE;QACpE,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,SAAuD,EACvD,aAAsC;IAEtC,MAAM,UAAU,GAAG,wBAAwB,CAAC,aAAa,CAAC,CAAC;IAC3D,OAAO,cAAc,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,cAAc,CACrB,SAAuD,EACvD,UAA8C;IAE9C,IAAI,CAAC,SAAS,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,eAAe,CACtB,SAA8C,EAC9C,UAA8C;IAE9C,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,OAAO,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;AACvD,CAAC;AAED,SAAS,eAAe,CACtB,GAAW,EACX,UAA8C;IAE9C,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAChC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;AACtC,CAAC"}
+{"version":3,"file":"cache-vary.js","sourceRoot":"","sources":["../../../src/tools/utils/cache-vary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,UAAU,gBAAgB,CAC9B,SAAuD,EACvD,aAAsC;IAEtC,MAAM,OAAO,GAAG,qBAAqB,CACnC,aAAa,EACb,MAAM,CAAC,QAAQ,CAAC,cAAc,EAC9B,EAAE,UAAU,EAAE,IAAI,EAAE,CACrB,CAAC;IAEF,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,IAAI,CAAC,SAAS;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IACnC,OAAO,OAAO,SAAS,KAAK,QAAQ;QAClC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE;QAC7B,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,CAAC;AAChC,CAAC"}

package/dist/tools/utils/common.d.ts

@@ -1,6 +1,5 @@
-import type { ExtractedArticle, ExtractedMetadata, MetadataBlock, TruncationResult } from '../../config/types.js';
+import type { ExtractedArticle, ExtractedMetadata, MetadataBlock } from '../../config/types/content.js';
+import type { TruncationResult } from '../../config/types/runtime.js';
 export declare function determineContentExtractionSource(extractMainContent: boolean, article: ExtractedArticle | null): article is ExtractedArticle;
 export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
-export declare function enforceContentLengthLimit(content: string, maxLength?: number): TruncationResult;
 export declare function truncateContent(content: string, maxLength?: number, suffix?: string): TruncationResult;
-//# sourceMappingURL=common.d.ts.map

package/dist/tools/utils/common.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/common.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wBAAgB,gCAAgC,CAC9C,kBAAkB,EAAE,OAAO,EAC3B,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAmB3B;AAED,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,MAAM,EACf,SAAS,CAAC,EAAE,MAAM,GACjB,gBAAgB,CAElB;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,SAAS,CAAC,EAAE,MAAM,EAClB,MAAM,SAAoB,GACzB,gBAAgB,CAYlB"}
+{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/common.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACd,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAEtE,wBAAgB,gCAAgC,CAC9C,kBAAkB,EAAE,OAAO,EAC3B,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAmB3B;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,SAAS,CAAC,EAAE,MAAM,EAClB,MAAM,SAAoB,GACzB,gBAAgB,CAYlB"}

package/dist/tools/utils/common.js

@@ -23,17 +23,16 @@ export function createContentMetadataBlock(url, article, extractedMeta, shouldEx
             fetchedAt: now,
         };
 }
-export function enforceContentLengthLimit(content, maxLength) {
-    return truncateContent(content, maxLength);
-}
 export function truncateContent(content, maxLength, suffix = TRUNCATION_MARKER) {
-    const shouldTruncate = maxLength !== undefined && maxLength > 0 && content.length > maxLength;
-    if (!shouldTruncate) {
+    if (maxLength === undefined ||
+        maxLength <= 0 ||
+        content.length <= maxLength) {
         return { content, truncated: false };
     }
+    const safeMax = Math.max(0, maxLength - suffix.length);
+    const marker = suffix.length > maxLength ? suffix.substring(0, maxLength) : suffix;
     return {
-        content: `${content.substring(0, maxLength)}${suffix}`,
+        content: `${content.substring(0, safeMax)}${marker}`,
         truncated: true,
     };
 }
-//# sourceMappingURL=common.js.map

package/dist/tools/utils/common.js.map

@@ -1 +1 @@
-{"version":3,"file":"common.js","sourceRoot":"","sources":["../../../src/tools/utils/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAQ/D,MAAM,UAAU,gCAAgC,CAC9C,kBAA2B,EAC3B,OAAgC;IAEhC,OAAO,kBAAkB,IAAI,CAAC,CAAC,OAAO,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,GAAW,EACX,OAAgC,EAChC,aAAgC,EAChC,wBAAiC,EACjC,eAAwB;IAExB,IAAI,CAAC,eAAe;QAAE,OAAO,SAAS,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,OAAO,wBAAwB,IAAI,OAAO;QACxC,CAAC,CAAC;YACE,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG;YACH,SAAS,EAAE,GAAG;SACf;QACH,CAAC,CAAC;YACE,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;YACtC,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,GAAG;YACH,SAAS,EAAE,GAAG;SACf,CAAC;AACR,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,OAAe,EACf,SAAkB;IAElB,OAAO,eAAe,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,SAAkB,EAClB,MAAM,GAAG,iBAAiB;IAE1B,MAAM,cAAc,GAClB,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAEzE,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvC,CAAC;IAED,OAAO;QACL,OAAO,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,MAAM,EAAE;QACtD,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC"}
+{"version":3,"file":"common.js","sourceRoot":"","sources":["../../../src/tools/utils/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAQ/D,MAAM,UAAU,gCAAgC,CAC9C,kBAA2B,EAC3B,OAAgC;IAEhC,OAAO,kBAAkB,IAAI,CAAC,CAAC,OAAO,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,GAAW,EACX,OAAgC,EAChC,aAAgC,EAChC,wBAAiC,EACjC,eAAwB;IAExB,IAAI,CAAC,eAAe;QAAE,OAAO,SAAS,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,OAAO,wBAAwB,IAAI,OAAO;QACxC,CAAC,CAAC;YACE,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG;YACH,SAAS,EAAE,GAAG;SACf;QACH,CAAC,CAAC;YACE,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;YACtC,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,GAAG;YACH,SAAS,EAAE,GAAG;SACf,CAAC;AACR,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,SAAkB,EAClB,MAAM,GAAG,iBAAiB;IAE1B,MAAM,cAAc,GAClB,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAEzE,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvC,CAAC;IAED,OAAO;QACL,OAAO,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,MAAM,EAAE;QACtD,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC"}

package/dist/tools/utils/content-transform.d.ts

@@ -1,4 +1,4 @@
-import type { JsonlTransformResult, MarkdownTransformResult } from '../../config/types.js';
+import type { JsonlTransformResult, MarkdownTransformResult } from '../../config/types/content.js';
 interface ExtractionOptions {
     readonly extractMainContent: boolean;
     readonly includeMetadata: boolean;
@@ -7,10 +7,8 @@ interface ContentLengthOptions {
     readonly maxContentLength?: number;
 }
 interface MarkdownOptions extends ExtractionOptions, ContentLengthOptions {
-    readonly generateToc?: boolean;
 }
 export declare function transformHtmlToJsonl(html: string, url: string, options: ExtractionOptions & ContentLengthOptions): JsonlTransformResult;
 export declare function transformHtmlToMarkdown(html: string, url: string, options: MarkdownOptions): MarkdownTransformResult;
 export declare function transformHtmlToMarkdownWithBlocks(html: string, url: string, options: ExtractionOptions & ContentLengthOptions): JsonlTransformResult;
 export {};
-//# sourceMappingURL=content-transform.d.ts.map

package/dist/tools/utils/content-transform.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"content-transform.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EACpB,uBAAuB,EACxB,MAAM,uBAAuB,CAAC;AAc/B,UAAU,iBAAiB;IACzB,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAQD,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,UAAU,eAAgB,SAAQ,iBAAiB,EAAE,oBAAoB;IACvE,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AA0ED,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,GAChD,oBAAoB,CAatB;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,eAAe,GACvB,uBAAuB,CAYzB;AAED,wBAAgB,iCAAiC,CAC/C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,GAChD,oBAAoB,CActB"}
+{"version":3,"file":"content-transform.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EACpB,uBAAuB,EACxB,MAAM,+BAA+B,CAAC;AAcvC,UAAU,iBAAiB;IACzB,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAQD,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,UAAU,eAAgB,SAAQ,iBAAiB,EAAE,oBAAoB;CAAG;AA4D5E,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,GAChD,oBAAoB,CAatB;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,eAAe,GACvB,uBAAuB,CAYzB;AAED,wBAAgB,iCAAiC,CAC/C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,GAChD,oBAAoB,CActB"}

package/dist/tools/utils/content-transform.js

@@ -1,10 +1,19 @@
 import { TRUNCATION_MARKER } from '../../config/formatting.js';
 import { extractContent } from '../../services/extractor.js';
-import { parseHtml } from '../../services/parser.js';
+import { parseHtml, parseHtmlWithMetadata } from '../../services/parser.js';
+import { sanitizeText } from '../../utils/sanitizer.js';
 import { toJsonl } from '../../transformers/jsonl.transformer.js';
 import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
 import { createContentMetadataBlock, determineContentExtractionSource, truncateContent, } from './common.js';
+const TITLE_PATTERN = /<title[^>]*>([\s\S]*?)<\/title>/i;
 function resolveContentSource(html, url, options) {
+    if (!options.extractMainContent && !options.includeMetadata) {
+        return {
+            sourceHtml: html,
+            title: extractTitleFromHtml(html),
+            metadata: undefined,
+        };
+    }
     const { article, metadata: extractedMeta } = extractContent(html, url, {
         extractArticle: options.extractMainContent,
     });
@@ -14,12 +23,43 @@ function resolveContentSource(html, url, options) {
     const title = shouldExtractFromArticle ? article.title : extractedMeta.title;
     return { sourceHtml, title, metadata };
 }
-function buildTransformContext(html, url, options) {
-    return resolveContentSource(html, url, options);
+function extractTitleFromHtml(html) {
+    const match = TITLE_PATTERN.exec(html);
+    if (!match?.[1])
+        return undefined;
+    const decoded = decodeHtmlEntities(match[1]);
+    const text = sanitizeText(decoded);
+    return text || undefined;
+}
+function decodeHtmlEntities(value) {
+    if (!value.includes('&'))
+        return value;
+    const basicDecoded = value
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'");
+    return basicDecoded
+        .replace(/&#(\d+);/g, (match, code) => {
+        const parsed = Number.parseInt(code, 10);
+        return Number.isFinite(parsed) && parsed >= 0 && parsed <= 0x10ffff
+            ? String.fromCodePoint(parsed)
+            : match;
+    })
+        .replace(/&#x([0-9a-fA-F]+);/g, (match, code) => {
+        const parsed = Number.parseInt(code, 16);
+        return Number.isFinite(parsed) && parsed >= 0 && parsed <= 0x10ffff
+            ? String.fromCodePoint(parsed)
+            : match;
+    });
 }
 function buildJsonlPayload(context, maxContentLength) {
     const contentBlocks = parseHtml(context.sourceHtml);
-    const { content, truncated } = truncateContent(toJsonl(contentBlocks, context.metadata), maxContentLength);
+    return buildJsonlPayloadFromBlocks(contentBlocks, context.metadata, maxContentLength);
+}
+function buildJsonlPayloadFromBlocks(contentBlocks, metadata, maxContentLength) {
+    const { content, truncated } = truncateContent(toJsonl(contentBlocks, metadata), maxContentLength);
     return {
         content,
         contentBlocks: contentBlocks.length,
@@ -32,7 +72,18 @@ function buildMarkdownPayload(context, maxContentLength) {
     return { content, truncated };
 }
 export function transformHtmlToJsonl(html, url, options) {
-    const context = buildTransformContext(html, url, options);
+    if (!options.extractMainContent && options.includeMetadata) {
+        const parsed = parseHtmlWithMetadata(html);
+        const metadataBlock = createContentMetadataBlock(url, null, parsed.metadata, false, true);
+        const { content, contentBlocks, truncated } = buildJsonlPayloadFromBlocks(parsed.blocks, metadataBlock, options.maxContentLength);
+        return {
+            content,
+            contentBlocks,
+            title: parsed.metadata.title,
+            ...(truncated && { truncated }),
+        };
+    }
+    const context = resolveContentSource(html, url, options);
     const { content, contentBlocks, truncated } = buildJsonlPayload(context, options.maxContentLength);
     return {
         content,
@@ -42,7 +93,7 @@ export function transformHtmlToJsonl(html, url, options) {
     };
 }
 export function transformHtmlToMarkdown(html, url, options) {
-    const context = buildTransformContext(html, url, options);
+    const context = resolveContentSource(html, url, options);
     const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
     return {
         markdown: content,
@@ -51,7 +102,22 @@ export function transformHtmlToMarkdown(html, url, options) {
     };
 }
 export function transformHtmlToMarkdownWithBlocks(html, url, options) {
-    const context = buildTransformContext(html, url, options);
+    if (!options.extractMainContent && options.includeMetadata) {
+        const parsed = parseHtmlWithMetadata(html);
+        const context = {
+            sourceHtml: html,
+            title: parsed.metadata.title,
+            metadata: createContentMetadataBlock(url, null, parsed.metadata, false, true),
+        };
+        const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
+        return {
+            content,
+            contentBlocks: parsed.blocks.length,
+            title: context.title,
+            ...(truncated && { truncated }),
+        };
+    }
+    const context = resolveContentSource(html, url, options);
     const contentBlocks = parseHtml(context.sourceHtml);
     const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
     return {
@@ -61,4 +127,3 @@ export function transformHtmlToMarkdownWithBlocks(html, url, options) {
         ...(truncated && { truncated }),
     };
 }
-//# sourceMappingURL=content-transform.js.map

package/dist/tools/utils/content-transform.js.map

@@ -1 +1 @@
-{"version":3,"file":"content-transform.js","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAM/D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,eAAe,GAChB,MAAM,aAAa,CAAC;AA2BrB,SAAS,oBAAoB,CAC3B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IAEH,MAAM,wBAAwB,GAAG,gCAAgC,CAC/D,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CACR,CAAC;IAEF,MAAM,UAAU,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM,QAAQ,GAAG,0BAA0B,CACzC,GAAG,EACH,OAAO,EACP,aAAa,EACb,wBAAwB,EACxB,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE7E,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,qBAAqB,CAC5B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,OAAO,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAyB,EACzB,gBAAyB;IAEzB,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,EACxC,gBAAgB,CACjB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAAyB,EACzB,gBAAyB;IAEzB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,QAAQ,EACR,gBAAgB,EAChB,iBAAiB,CAClB,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,iBAAiB,CAC7D,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa;QACb,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,IAAY,EACZ,GAAW,EACX,OAAwB;IAExB,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC/C,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"content-transform.js","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAM/D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,eAAe,GAChB,MAAM,aAAa,CAAC;AAmBrB,SAAS,oBAAoB,CAC3B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IAEH,MAAM,wBAAwB,GAAG,gCAAgC,CAC/D,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CACR,CAAC;IAEF,MAAM,UAAU,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM,QAAQ,GAAG,0BAA0B,CACzC,GAAG,EACH,OAAO,EACP,aAAa,EACb,wBAAwB,EACxB,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE7E,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAsB,EACtB,gBAAyB;IAEzB,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,EACxC,gBAAgB,CACjB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAAsB,EACtB,gBAAyB;IAEzB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,QAAQ,EACR,gBAAgB,EAChB,iBAAiB,CAClB,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,iBAAiB,CAC7D,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa;QACb,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,IAAY,EACZ,GAAW,EACX,OAAwB;IAExB,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC/C,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC"}

package/dist/tools/utils/fetch-pipeline.d.ts

@@ -1,4 +1,4 @@
-import type { FetchPipelineOptions, PipelineResult } from '../../config/types.js';
+import type { FetchPipelineOptions, PipelineResult } from '../../config/types/runtime.js';
 /**
  * Unified fetch pipeline that handles caching, fetching, and transformation.
  * Implements cache-first strategy with automatic serialization.
@@ -8,4 +8,3 @@ import type { FetchPipelineOptions, PipelineResult } from '../../config/types.js
  * @returns Promise resolving to the pipeline result
  */
 export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;
-//# sourceMappingURL=fetch-pipeline.d.ts.map

package/dist/tools/utils/fetch-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/fetch-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,oBAAoB,EACpB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAuD/B;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAwB5B"}
+{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/fetch-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,oBAAoB,EACpB,cAAc,EACf,MAAM,+BAA+B,CAAC;AAuDvC;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAwB5B"}

package/dist/tools/utils/fetch-pipeline.js

@@ -1,29 +1,22 @@
 import * as cache from '../../services/cache.js';
-import { fetchUrlWithRetry } from '../../services/fetcher.js';
-import { logDebug, logWarn } from '../../services/logger.js';
-import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
+import { fetchNormalizedUrlWithRetry } from '../../services/fetcher.js';
+import { logDebug } from '../../services/logger.js';
+import { assertResolvedAddressesAllowed, normalizeUrl, } from '../../utils/url-validator.js';
 import { appendHeaderVary } from './cache-vary.js';
-function safeJsonParse(cached, cacheKey) {
-    try {
-        return JSON.parse(cached);
-    }
-    catch {
-        logWarn('Cache deserialize failed, treating as miss', {
-            key: cacheKey.substring(0, 100),
-        });
-        return undefined;
-    }
-}
 function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalizedUrl) {
     if (!cacheKey)
         return null;
     const cached = cache.get(cacheKey);
     if (!cached)
         return null;
-    logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
-    const data = deserialize
-        ? deserialize(cached.content)
-        : safeJsonParse(cached.content, cacheKey);
+    if (!deserialize) {
+        logDebug('Cache miss due to missing deserializer', {
+            namespace: cacheNamespace,
+            url: normalizedUrl,
+        });
+        return null;
+    }
+    const data = deserialize(cached.content);
     if (data === undefined) {
         logDebug('Cache miss due to deserialize failure', {
             namespace: cacheNamespace,
@@ -31,6 +24,7 @@ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalized
         });
         return null;
     }
+    logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
     return {
         data,
         fromCache: true,
@@ -48,16 +42,19 @@ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalized
  * @returns Promise resolving to the pipeline result
  */
 export async function executeFetchPipeline(options) {
-    const normalizedUrl = validateAndNormalizeUrl(options.url);
+    const { normalizedUrl, hostname } = normalizeUrl(options.url);
     const cacheKey = resolveCacheKey(options, normalizedUrl);
     const cachedResult = attemptCacheRetrieval(cacheKey, options.deserialize, options.cacheNamespace, normalizedUrl);
     if (cachedResult)
         return cachedResult;
+    await assertResolvedAddressesAllowed(hostname);
     const fetchOptions = buildFetchOptions(options);
     logDebug('Fetching URL', { url: normalizedUrl, retries: options.retries });
-    const html = await fetchUrlWithRetry(normalizedUrl, fetchOptions, options.retries);
+    const html = await fetchNormalizedUrlWithRetry(normalizedUrl, fetchOptions, options.retries);
     const data = options.transform(html, normalizedUrl);
-    persistCache(cacheKey, data, options.serialize, normalizedUrl);
+    if (cache.isEnabled()) {
+        persistCache(cacheKey, data, options.serialize, normalizedUrl);
+    }
     return buildPipelineResult(normalizedUrl, data, cacheKey);
 }
 function resolveCacheKey(options, normalizedUrl) {
@@ -97,4 +94,3 @@ function buildPipelineResult(url, data, cacheKey) {
         cacheKey,
     };
 }
-//# sourceMappingURL=fetch-pipeline.js.map

package/dist/tools/utils/inline-content.d.ts

@@ -9,4 +9,3 @@ interface InlineContentResult {
 }
 export declare function applyInlineContentLimit(content: string, cacheKey: string | null, format: InlineContentFormat): InlineContentResult;
 export {};
-//# sourceMappingURL=inline-content.d.ts.map

package/dist/tools/utils/inline-content.js

@@ -36,4 +36,3 @@ function buildTruncatedFallback(content, contentSize, inlineLimit) {
         truncated: true,
     };
 }
-//# sourceMappingURL=inline-content.js.map

package/dist/transformers/jsonl.transformer.d.ts

@@ -1,3 +1,2 @@
-import type { ContentBlockUnion, MetadataBlock } from '../config/types.js';
+import type { ContentBlockUnion, MetadataBlock } from '../config/types/content.js';
 export declare function toJsonl(blocks: readonly ContentBlockUnion[], metadata?: MetadataBlock): string;
-//# sourceMappingURL=jsonl.transformer.d.ts.map

package/dist/transformers/jsonl.transformer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAkE3E,wBAAgB,OAAO,CACrB,MAAM,EAAE,SAAS,iBAAiB,EAAE,EACpC,QAAQ,CAAC,EAAE,aAAa,GACvB,MAAM,CAGR"}
+{"version":3,"file":"jsonl.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,iBAAiB,EACjB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAkEpC,wBAAgB,OAAO,CACrB,MAAM,EAAE,SAAS,iBAAiB,EAAE,EACpC,QAAQ,CAAC,EAAE,aAAa,GACvB,MAAM,CAGR"}

package/dist/transformers/jsonl.transformer.js

@@ -73,4 +73,3 @@ function serializeMetadata(metadata) {
         return null;
     }
 }
-//# sourceMappingURL=jsonl.transformer.js.map

package/dist/transformers/jsonl.transformer.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl.transformer.js","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,MAAM,IAAI,KAAK,CAAC;AACzB,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;AAC/B,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAmD,EACnD,SAAiB;IAEjB,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACtD,OAAO,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAsD,EACtD,SAAiB;IAEjB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC9C,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAC9B,CAAC;IACF,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CACpC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAC7C,CAAC;IACF,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AAClE,CAAC;AAED,SAAS,aAAa,CAAC,KAAwB;IAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC;IAEnD,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAwB;IAC9C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CACrB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClD,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,iBAAiB,CACxB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAwB;IACjD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG;YACtB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;SAClB,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
+{"version":3,"file":"jsonl.transformer.js","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAM5C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,MAAM,IAAI,KAAK,CAAC;AACzB,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;AAC/B,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAmD,EACnD,SAAiB;IAEjB,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACtD,OAAO,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAsD,EACtD,SAAiB;IAEjB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC9C,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAC9B,CAAC;IACF,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CACpC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAC7C,CAAC;IACF,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AAClE,CAAC;AAED,SAAS,aAAa,CAAC,KAAwB;IAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC;IAEnD,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAwB;IAC9C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CACrB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClD,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,iBAAiB,CACxB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAwB;IACjD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG;YACtB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;SAClB,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/transformers/markdown.transformer.d.ts

@@ -1,3 +1,2 @@
-import type { MetadataBlock } from '../config/types.js';
+import type { MetadataBlock } from '../config/types/content.js';
 export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock): string;
-//# sourceMappingURL=markdown.transformer.d.ts.map

package/dist/transformers/markdown.transformer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/markdown.transformer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AA+HxD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,aAAa,GAAG,MAAM,CAa7E"}
+{"version":3,"file":"markdown.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/markdown.transformer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AA+HhE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,aAAa,GAAG,MAAM,CAa7E"}