@j0hanz/superfetch 2.2.1 → 2.3.0

package/dist/tools.js

@@ -6,7 +6,7 @@ import { FetchError, getErrorMessage, isSystemError } from './errors.js';
 import { fetchNormalizedUrl, normalizeUrl, transformToRawUrl, } from './fetch.js';
 import { getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from './observability.js';
 import { transformHtmlToMarkdown, } from './transform.js';
-import { isRecord } from './type-guards.js';
+import { isObject } from './type-guards.js';
 const TRUNCATION_MARKER = '...[truncated]';
 const FETCH_PROGRESS_TOTAL = 4;
 const fetchUrlInputSchema = z.strictObject({
@@ -106,23 +106,16 @@ function buildEmbeddedResource(content, url, title) {
         },
     };
 }
-function resolveContentToEmbed(inlineResult, fullContent, useInlineInHttpMode) {
-    if (useInlineInHttpMode) {
-        return inlineResult.content;
-    }
-    return fullContent ?? inlineResult.content;
-}
-function maybeAppendEmbeddedResource(blocks, contentToEmbed, url, title) {
-    if (!contentToEmbed)
-        return;
-    if (!url)
-        return;
-    const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
-    if (embeddedResource) {
-        blocks.push(embeddedResource);
+function appendResourceBlocks({ blocks, inlineResult, resourceName, url, title, fullContent, }) {
+    const contentToEmbed = config.runtime.httpMode
+        ? inlineResult.content
+        : (fullContent ?? inlineResult.content);
+    if (contentToEmbed && url) {
+        const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
+        if (embeddedResource) {
+            blocks.push(embeddedResource);
+        }
     }
-}
-function maybeAppendResourceLink(blocks, inlineResult, resourceName) {
     const resourceLink = buildResourceLink(inlineResult, resourceName);
     if (resourceLink) {
         blocks.push(resourceLink);
@@ -136,9 +129,14 @@ function buildTextBlock(structuredContent) {
 }
 function buildToolContentBlocks(structuredContent, fromCache, inlineResult, resourceName, cacheKey, fullContent, url, title) {
     const blocks = [buildTextBlock(structuredContent)];
-    const contentToEmbed = resolveContentToEmbed(inlineResult, fullContent, config.runtime.httpMode);
-    maybeAppendEmbeddedResource(blocks, contentToEmbed, url, title);
-    maybeAppendResourceLink(blocks, inlineResult, resourceName);
+    appendResourceBlocks({
+        blocks,
+        inlineResult,
+        resourceName,
+        url,
+        title,
+        fullContent,
+    });
     return blocks;
 }
 function applyInlineContentLimit(content, cacheKey) {
@@ -246,7 +244,7 @@ function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
     cache.set(cacheKey, serializer(data), metadata);
 }
 function extractTitle(value) {
-    if (!isRecord(value))
+    if (!isObject(value))
         return undefined;
     const { title } = value;
     return typeof title === 'string' ? title : undefined;
@@ -266,14 +264,6 @@ function logRawUrlTransformation(resolvedUrl) {
         original: resolvedUrl.originalUrl,
     });
 }
-function applyOptionalPipelineSerialization(pipelineOptions, options) {
-    if (options.serialize !== undefined) {
-        pipelineOptions.serialize = options.serialize;
-    }
-    if (options.deserialize !== undefined) {
-        pipelineOptions.deserialize = options.deserialize;
-    }
-}
 export async function performSharedFetch(options, deps = {}) {
     const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
     const pipelineOptions = {
@@ -281,8 +271,9 @@ export async function performSharedFetch(options, deps = {}) {
         cacheNamespace: 'markdown',
         ...(options.signal === undefined ? {} : { signal: options.signal }),
         transform: options.transform,
+        ...(options.serialize ? { serialize: options.serialize } : {}),
+        ...(options.deserialize ? { deserialize: options.deserialize } : {}),
     };
-    applyOptionalPipelineSerialization(pipelineOptions, options);
     const pipeline = await executePipeline(pipelineOptions);
     const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null);
     return { pipeline, inlineResult };
@@ -319,7 +310,7 @@ function resolveToolErrorMessage(error, fallbackMessage) {
 function parseJsonRecord(input) {
     try {
         const parsed = JSON.parse(input);
-        return isRecord(parsed) ? parsed : undefined;
+        return isObject(parsed) ? parsed : undefined;
     }
     catch {
         return undefined;
@@ -467,7 +458,7 @@ export function withRequestContextIfMissing(handler) {
     };
 }
 function resolveRequestIdFromExtra(extra) {
-    if (!isRecord(extra))
+    if (!isObject(extra))
         return undefined;
     const { requestId } = extra;
     if (typeof requestId === 'string')
@@ -485,4 +476,3 @@ export function registerTools(server) {
         annotations: TOOL_DEFINITION.annotations,
     }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
 }
-//# sourceMappingURL=tools.js.map

package/dist/transform-types.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Shared types for the transform pipeline.
+ * Extracted to avoid circular dependencies between transform modules.
+ */
+/**
+ * Metadata block for attaching source information to markdown output.
+ */
+export interface MetadataBlock {
+    type: 'metadata';
+    title?: string;
+    description?: string;
+    author?: string;
+    url: string;
+    fetchedAt: string;
+}
+/**
+ * Article extracted by Readability.
+ */
+export interface ExtractedArticle {
+    title?: string;
+    byline?: string;
+    content: string;
+    textContent: string;
+    excerpt?: string;
+    siteName?: string;
+}
+/**
+ * Metadata extracted from HTML meta tags.
+ */
+export interface ExtractedMetadata {
+    title?: string;
+    description?: string;
+    author?: string;
+    image?: string;
+    publishedAt?: string;
+    modifiedAt?: string;
+}
+/**
+ * Result of content extraction (article + metadata).
+ */
+export interface ExtractionResult {
+    article: ExtractedArticle | null;
+    metadata: ExtractedMetadata;
+}
+/**
+ * Result of HTML to markdown transformation.
+ */
+export interface MarkdownTransformResult {
+    markdown: string;
+    title: string | undefined;
+    truncated: boolean;
+}
+/**
+ * Options for transform operations.
+ */
+export interface TransformOptions {
+    includeMetadata: boolean;
+    signal?: AbortSignal;
+}
+/**
+ * Telemetry event emitted during transform stages.
+ */
+export interface TransformStageEvent {
+    v: 1;
+    type: 'stage';
+    stage: string;
+    durationMs: number;
+    url: string;
+    requestId?: string;
+    operationId?: string;
+    truncated?: boolean;
+}
+/**
+ * Context for tracking transform stage timing.
+ */
+export interface TransformStageContext {
+    readonly stage: string;
+    readonly startTime: number;
+    readonly url: string;
+}

package/dist/transform-types.js

@@ -0,0 +1,5 @@
+/**
+ * Shared types for the transform pipeline.
+ * Extracted to avoid circular dependencies between transform modules.
+ */
+export {};

package/dist/transform.d.ts

@@ -1,52 +1,8 @@
-export interface MetadataBlock {
-    type: 'metadata';
-    title?: string;
-    description?: string;
-    author?: string;
-    url: string;
-    fetchedAt: string;
-}
-export interface ExtractedArticle {
-    title?: string;
-    byline?: string;
-    content: string;
-    textContent: string;
-    excerpt?: string;
-    siteName?: string;
-}
-export interface ExtractedMetadata {
-    title?: string;
-    description?: string;
-    author?: string;
-}
-export interface ExtractionResult {
-    article: ExtractedArticle | null;
-    metadata: ExtractedMetadata;
-}
-export interface MarkdownTransformResult {
-    markdown: string;
-    title: string | undefined;
-    truncated: boolean;
-}
-export interface TransformOptions {
-    includeMetadata: boolean;
-    signal?: AbortSignal;
-}
-export interface TransformStageEvent {
-    v: 1;
-    type: 'stage';
-    stage: string;
-    durationMs: number;
-    url: string;
-    requestId?: string;
-    operationId?: string;
-    truncated?: boolean;
-}
-export interface TransformStageContext {
-    readonly stage: string;
-    readonly startTime: number;
-    readonly url: string;
-}
+import type { ExtractedArticle, ExtractedMetadata, ExtractionResult, MarkdownTransformResult, MetadataBlock, TransformOptions, TransformStageContext } from './transform-types.js';
+export { detectLanguageFromCode, resolveLanguageFromAttributes, } from './language-detection.js';
+export { cleanupMarkdownArtifacts, promoteOrphanHeadings, } from './markdown-cleanup.js';
+export { removeNoiseFromHtml } from './dom-noise-removal.js';
+export type { MetadataBlock, ExtractedArticle, ExtractedMetadata, ExtractionResult, MarkdownTransformResult, TransformOptions, TransformStageEvent, TransformStageContext, } from './transform-types.js';
 export declare function startTransformStage(url: string, stage: string): TransformStageContext | null;
 export declare function endTransformStage(context: TransformStageContext | null, options?: {
     truncated?: boolean;
@@ -55,17 +11,15 @@ export declare function extractContent(html: string, url: string, options?: {
     extractArticle?: boolean;
     signal?: AbortSignal;
 }): ExtractionResult;
-export declare function detectLanguageFromCode(code: string): string | undefined;
-export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;
 export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock, options?: {
     url?: string;
     signal?: AbortSignal;
     document?: Document;
+    skipNoiseRemoval?: boolean;
 }): string;
-export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
+export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtmlOrDocument: string | Document): boolean;
 export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
 export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
 export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
 export declare function shutdownTransformWorkerPool(): Promise<void>;
 export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
-//# sourceMappingURL=transform.d.ts.map