@j0hanz/superfetch 1.2.2 → 1.2.4

package/dist/tools/handlers/fetch-single.shared.d.ts

@@ -1,5 +1,6 @@
 import type { PipelineResult, ToolContentBlock } from '../../config/types/runtime.js';
 import type { FileDownloadInfo, ToolResponseBase } from '../../config/types/tools.js';
+import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
 import { applyInlineContentLimit } from '../utils/inline-content.js';
 type SharedFetchFormat = 'jsonl' | 'markdown';
 interface SharedFetchOptions<T extends {
@@ -10,17 +11,21 @@ interface SharedFetchOptions<T extends {
     readonly extractMainContent: boolean;
     readonly includeMetadata: boolean;
     readonly maxContentLength?: number;
+    readonly includeContentBlocks?: boolean;
     readonly cacheVariant?: string;
     readonly customHeaders?: Record<string, string>;
     readonly retries?: number;
     readonly timeout?: number;
-    readonly transform: (html: string, normalizedUrl: string) => T;
+    readonly transform: (html: string, normalizedUrl: string) => T | Promise<T>;
     readonly serialize?: (result: T) => string;
     readonly deserialize?: (cached: string) => T | undefined;
 }
+interface SharedFetchDeps {
+    readonly executeFetchPipeline?: typeof executeFetchPipeline;
+}
 export declare function performSharedFetch<T extends {
     content: string;
-}>(options: SharedFetchOptions<T>): Promise<{
+}>(options: SharedFetchOptions<T>, deps?: SharedFetchDeps): Promise<{
     pipeline: PipelineResult<T>;
     inlineResult: ReturnType<typeof applyInlineContentLimit>;
 }>;
@@ -31,7 +36,7 @@ interface DownloadContext {
     title?: string;
 }
 export declare function getFileDownloadInfo(context: DownloadContext): FileDownloadInfo | null;
-export declare function getInlineErrorResponse(inlineResult: InlineResult, url: string): ToolResponseBase | null;
+export declare function getInlineErrorResponse(inlineResult: InlineResult, url: string, details?: Record<string, unknown>): ToolResponseBase | null;
 export declare function applyInlineResultToStructuredContent(structuredContent: Record<string, unknown>, inlineResult: InlineResult, contentKey: string): void;
 export declare function buildToolContentBlocks(structuredContent: Record<string, unknown>, fromCache: boolean, inlineResult: InlineResult, resourceName: string, cacheKey?: string | null, fullContent?: string, format?: SharedFetchFormat, url?: string, title?: string): ToolContentBlock[];
 export {};

package/dist/tools/handlers/fetch-single.shared.js

@@ -5,7 +5,8 @@ import { createToolErrorResponse } from '../../utils/tool-error-handler.js';
 import { appendHeaderVary } from '../utils/cache-vary.js';
 import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
 import { applyInlineContentLimit } from '../utils/inline-content.js';
-export async function performSharedFetch(options) {
+export async function performSharedFetch(options, deps = {}) {
+    const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
     const cacheNamespace = options.format === 'markdown' ? 'markdown' : 'url';
     const cacheVary = appendHeaderVary({
         format: options.format,
@@ -13,33 +14,54 @@ export async function performSharedFetch(options) {
         includeMetadata: options.includeMetadata,
         maxContentLength: options.maxContentLength,
         ...(options.cacheVariant ? { variant: options.cacheVariant } : {}),
-        ...(options.format === 'markdown' ? {} : { contentBlocks: true }),
+        ...(options.format === 'markdown'
+            ? { includeContentBlocks: options.includeContentBlocks }
+            : { contentBlocks: true }),
     }, options.customHeaders);
-    const pipeline = await executeFetchPipeline({
+    const pipelineOptions = {
         url: options.url,
         cacheNamespace,
-        customHeaders: options.customHeaders,
-        retries: options.retries,
-        timeout: options.timeout,
-        cacheVary,
         transform: options.transform,
-        serialize: options.serialize,
-        deserialize: options.deserialize,
-    });
+    };
+    if (options.customHeaders !== undefined) {
+        pipelineOptions.customHeaders = options.customHeaders;
+    }
+    if (options.retries !== undefined) {
+        pipelineOptions.retries = options.retries;
+    }
+    if (options.timeout !== undefined) {
+        pipelineOptions.timeout = options.timeout;
+    }
+    if (cacheVary !== undefined) {
+        pipelineOptions.cacheVary = cacheVary;
+    }
+    if (options.serialize !== undefined) {
+        pipelineOptions.serialize = options.serialize;
+    }
+    if (options.deserialize !== undefined) {
+        pipelineOptions.deserialize = options.deserialize;
+    }
+    const pipeline = await executePipeline(pipelineOptions);
     const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null, options.format);
     return { pipeline, inlineResult };
 }
 export function getFileDownloadInfo(context) {
-    return buildFileDownloadInfo({
+    const infoOptions = {
         cacheKey: context.cacheKey,
         url: context.url,
-        title: context.title,
-    });
+    };
+    if (context.title !== undefined) {
+        return buildFileDownloadInfo({
+            ...infoOptions,
+            title: context.title,
+        });
+    }
+    return buildFileDownloadInfo(infoOptions);
 }
-export function getInlineErrorResponse(inlineResult, url) {
+export function getInlineErrorResponse(inlineResult, url, details) {
     if (!inlineResult.error)
         return null;
-    return createToolErrorResponse(inlineResult.error, url, 'INTERNAL_ERROR');
+    return createToolErrorResponse(inlineResult.error, url, 'INTERNAL_ERROR', details);
 }
 export function applyInlineResultToStructuredContent(structuredContent, inlineResult, contentKey) {
     if (inlineResult.truncated) {
@@ -60,13 +82,16 @@ function buildResourceLink(inlineResult, name) {
     if (!inlineResult.resourceUri) {
         return null;
     }
-    return {
+    const block = {
         type: 'resource_link',
         uri: inlineResult.resourceUri,
         name,
-        mimeType: inlineResult.resourceMimeType,
         description: `Content exceeds inline limit (${config.constants.maxInlineContentChars} chars)`,
     };
+    if (inlineResult.resourceMimeType !== undefined) {
+        block.mimeType = inlineResult.resourceMimeType;
+    }
+    return block;
 }
 function buildEmbeddedResource(content, mimeType, url, title) {
     if (!content) {

package/dist/tools/handlers/fetch-url.tool.js

@@ -1,7 +1,7 @@
 import { config } from '../../config/index.js';
 import { logDebug, logError } from '../../services/logger.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
-import { transformHtmlToJsonl, transformHtmlToMarkdownWithBlocks, } from '../utils/content-transform.js';
+import { transformHtmlToJsonlAsync, transformHtmlToMarkdownWithBlocksAsync, } from '../utils/content-transform-async.js';
 import { applyInlineResultToStructuredContent, buildToolContentBlocks, getInlineErrorResponse, performSharedFetch, } from './fetch-single.shared.js';
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
 export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks. Supports custom headers, retries, and content length limits.';
@@ -37,17 +37,36 @@ function deserializeJsonlTransformResult(cached) {
     }
 }
 function resolveFetchUrlOptions(input) {
+    const format = input.format ?? 'jsonl';
     return {
         extractMainContent: input.extractMainContent ?? config.extraction.extractMainContent,
         includeMetadata: input.includeMetadata ?? config.extraction.includeMetadata,
-        maxContentLength: input.maxContentLength,
-        format: input.format ?? 'jsonl',
+        format,
+        includeContentBlocks: input.includeContentBlocks ?? (format === 'markdown' ? false : true),
+        ...(input.maxContentLength !== undefined && {
+            maxContentLength: input.maxContentLength,
+        }),
+    };
+}
+function buildFetchUrlErrorDetails(format) {
+    return {
+        contentBlocks: 0,
+        fetchedAt: new Date().toISOString(),
+        format,
+        cached: false,
     };
 }
 function buildFetchUrlTransform(options) {
-    return (html, url) => options.format === 'markdown'
-        ? transformHtmlToMarkdownWithBlocks(html, url, options)
-        : transformHtmlToJsonl(html, url, options);
+    return async (html, url) => options.format === 'markdown'
+        ? transformHtmlToMarkdownWithBlocksAsync(html, url, {
+            extractMainContent: options.extractMainContent,
+            includeMetadata: options.includeMetadata,
+            ...(options.maxContentLength !== undefined && {
+                maxContentLength: options.maxContentLength,
+            }),
+            includeContentBlocks: options.includeContentBlocks,
+        })
+        : transformHtmlToJsonlAsync(html, url, options);
 }
 function buildFetchUrlStructuredContent(format, pipeline, inlineResult) {
     const structuredContent = {
@@ -74,22 +93,31 @@ function logFetchUrlStart(url, options) {
         extractMainContent: options.extractMainContent,
         includeMetadata: options.includeMetadata,
         format: options.format,
+        includeContentBlocks: options.includeContentBlocks,
     });
 }
 async function fetchUrlPipeline(url, input, options) {
-    return performSharedFetch({
+    const sharedOptions = {
         url,
         format: options.format,
         extractMainContent: options.extractMainContent,
         includeMetadata: options.includeMetadata,
-        maxContentLength: options.maxContentLength,
-        customHeaders: input.customHeaders,
-        retries: input.retries,
-        timeout: input.timeout,
-        cacheVariant: options.format === 'markdown' ? 'markdown-with-blocks' : undefined,
+        includeContentBlocks: options.includeContentBlocks,
+        ...(options.maxContentLength !== undefined && {
+            maxContentLength: options.maxContentLength,
+        }),
+        ...(input.customHeaders !== undefined && {
+            customHeaders: input.customHeaders,
+        }),
+        ...(input.retries !== undefined && { retries: input.retries }),
+        ...(input.timeout !== undefined && { timeout: input.timeout }),
+        ...(options.format === 'markdown' && {
+            cacheVariant: 'markdown-with-blocks',
+        }),
         transform: buildFetchUrlTransform(options),
         deserialize: deserializeJsonlTransformResult,
-    });
+    };
+    return performSharedFetch(sharedOptions);
 }
 function buildFetchUrlResponse(pipeline, inlineResult, format) {
     const structuredContent = buildFetchUrlStructuredContent(format, pipeline, inlineResult);
@@ -104,18 +132,20 @@ export async function fetchUrlToolHandler(input) {
     }
     catch (error) {
         logError('fetch-url tool error', error instanceof Error ? error : undefined);
-        return handleToolError(error, input.url, 'Failed to fetch URL');
+        const errorDetails = buildFetchUrlErrorDetails(input.format ?? 'jsonl');
+        return handleToolError(error, input.url, 'Failed to fetch URL', errorDetails);
     }
 }
 async function executeFetchUrl(input) {
     const { url } = input;
+    const format = input.format ?? 'jsonl';
     if (!url) {
-        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
+        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR', buildFetchUrlErrorDetails(format));
     }
     const options = resolveFetchUrlOptions(input);
     logFetchUrlStart(url, options);
     const { pipeline, inlineResult } = await fetchUrlPipeline(url, input, options);
-    const inlineError = getInlineErrorResponse(inlineResult, url);
+    const inlineError = getInlineErrorResponse(inlineResult, url, buildFetchUrlErrorDetails(options.format));
     if (inlineError)
         return inlineError;
     return buildFetchUrlResponse(pipeline, inlineResult, options.format);

package/dist/tools/index.js

@@ -9,6 +9,12 @@ const TOOL_DEFINITIONS = [
         inputSchema: fetchUrlInputSchema,
         outputSchema: fetchUrlOutputSchema,
         handler: fetchUrlToolHandler,
+        annotations: {
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
     },
     {
         name: FETCH_MARKDOWN_TOOL_NAME,
@@ -17,6 +23,12 @@ const TOOL_DEFINITIONS = [
         inputSchema: fetchMarkdownInputSchema,
         outputSchema: fetchMarkdownOutputSchema,
         handler: fetchMarkdownToolHandler,
+        annotations: {
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
     },
 ];
 export function registerTools(server) {
@@ -26,6 +38,7 @@ export function registerTools(server) {
             description: tool.description,
             inputSchema: tool.inputSchema,
             outputSchema: tool.outputSchema,
+            annotations: tool.annotations,
         }, tool.handler);
     }
 }

package/dist/tools/schemas.d.ts

@@ -1,70 +1,46 @@
 import { z } from 'zod';
 export declare const fetchUrlInputSchema: z.ZodObject<{
-    customHeaders: z.ZodOptional<z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodString>, Record<string, string>, Record<string, string>>>;
-    timeout: z.ZodDefault<z.ZodNumber>;
-    retries: z.ZodDefault<z.ZodNumber>;
-} & {
-    url: z.ZodString;
-} & {
+    format: z.ZodDefault<z.ZodEnum<{
+        jsonl: "jsonl";
+        markdown: "markdown";
+    }>>;
+    includeContentBlocks: z.ZodOptional<z.ZodBoolean>;
     extractMainContent: z.ZodDefault<z.ZodBoolean>;
     includeMetadata: z.ZodDefault<z.ZodBoolean>;
     maxContentLength: z.ZodOptional<z.ZodNumber>;
-} & {
-    format: z.ZodDefault<z.ZodEnum<["jsonl", "markdown"]>>;
-}, "strict", z.ZodTypeAny, {
-    url: string;
-    timeout: number;
-    retries: number;
-    extractMainContent: boolean;
-    includeMetadata: boolean;
-    format: "jsonl" | "markdown";
-    customHeaders?: Record<string, string> | undefined;
-    maxContentLength?: number | undefined;
-}, {
-    url: string;
-    customHeaders?: Record<string, string> | undefined;
-    timeout?: number | undefined;
-    retries?: number | undefined;
-    extractMainContent?: boolean | undefined;
-    includeMetadata?: boolean | undefined;
-    maxContentLength?: number | undefined;
-    format?: "jsonl" | "markdown" | undefined;
-}>;
-export declare const fetchMarkdownInputSchema: z.ZodObject<{
-    customHeaders: z.ZodOptional<z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodString>, Record<string, string>, Record<string, string>>>;
+    url: z.ZodURL;
+    customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
-} & {
-    url: z.ZodString;
-} & {
+}, z.core.$strict>;
+export declare const fetchMarkdownInputSchema: z.ZodObject<{
     extractMainContent: z.ZodDefault<z.ZodBoolean>;
     includeMetadata: z.ZodDefault<z.ZodBoolean>;
     maxContentLength: z.ZodOptional<z.ZodNumber>;
-}, "strict", z.ZodTypeAny, {
-    url: string;
-    timeout: number;
-    retries: number;
-    extractMainContent: boolean;
-    includeMetadata: boolean;
-    customHeaders?: Record<string, string> | undefined;
-    maxContentLength?: number | undefined;
-}, {
-    url: string;
-    customHeaders?: Record<string, string> | undefined;
-    timeout?: number | undefined;
-    retries?: number | undefined;
-    extractMainContent?: boolean | undefined;
-    includeMetadata?: boolean | undefined;
-    maxContentLength?: number | undefined;
-}>;
+    url: z.ZodURL;
+    customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    timeout: z.ZodDefault<z.ZodNumber>;
+    retries: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strict>;
 export declare const fetchUrlOutputSchema: z.ZodObject<{
+    contentSize: z.ZodOptional<z.ZodNumber>;
+    resourceUri: z.ZodOptional<z.ZodString>;
+    resourceMimeType: z.ZodOptional<z.ZodString>;
+    cached: z.ZodBoolean;
+    truncated: z.ZodOptional<z.ZodBoolean>;
+    error: z.ZodOptional<z.ZodString>;
+    errorCode: z.ZodOptional<z.ZodString>;
     url: z.ZodString;
     title: z.ZodOptional<z.ZodString>;
     contentBlocks: z.ZodNumber;
     fetchedAt: z.ZodString;
-    format: z.ZodEnum<["jsonl", "markdown"]>;
+    format: z.ZodEnum<{
+        jsonl: "jsonl";
+        markdown: "markdown";
+    }>;
     content: z.ZodOptional<z.ZodString>;
-} & {
+}, z.core.$strict>;
+export declare const fetchMarkdownOutputSchema: z.ZodObject<{
     contentSize: z.ZodOptional<z.ZodNumber>;
     resourceUri: z.ZodOptional<z.ZodString>;
     resourceMimeType: z.ZodOptional<z.ZodString>;
@@ -72,36 +48,6 @@ export declare const fetchUrlOutputSchema: z.ZodObject<{
     truncated: z.ZodOptional<z.ZodBoolean>;
     error: z.ZodOptional<z.ZodString>;
     errorCode: z.ZodOptional<z.ZodString>;
-}, "strict", z.ZodTypeAny, {
-    url: string;
-    format: "jsonl" | "markdown";
-    contentBlocks: number;
-    fetchedAt: string;
-    cached: boolean;
-    error?: string | undefined;
-    title?: string | undefined;
-    content?: string | undefined;
-    contentSize?: number | undefined;
-    resourceUri?: string | undefined;
-    resourceMimeType?: string | undefined;
-    truncated?: boolean | undefined;
-    errorCode?: string | undefined;
-}, {
-    url: string;
-    format: "jsonl" | "markdown";
-    contentBlocks: number;
-    fetchedAt: string;
-    cached: boolean;
-    error?: string | undefined;
-    title?: string | undefined;
-    content?: string | undefined;
-    contentSize?: number | undefined;
-    resourceUri?: string | undefined;
-    resourceMimeType?: string | undefined;
-    truncated?: boolean | undefined;
-    errorCode?: string | undefined;
-}>;
-export declare const fetchMarkdownOutputSchema: z.ZodObject<{
     url: z.ZodString;
     title: z.ZodOptional<z.ZodString>;
     fetchedAt: z.ZodString;
@@ -110,55 +56,5 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
         downloadUrl: z.ZodString;
         fileName: z.ZodString;
         expiresAt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        downloadUrl: string;
-        fileName: string;
-        expiresAt: string;
-    }, {
-        downloadUrl: string;
-        fileName: string;
-        expiresAt: string;
-    }>>;
-} & {
-    contentSize: z.ZodOptional<z.ZodNumber>;
-    resourceUri: z.ZodOptional<z.ZodString>;
-    resourceMimeType: z.ZodOptional<z.ZodString>;
-    cached: z.ZodBoolean;
-    truncated: z.ZodOptional<z.ZodBoolean>;
-    error: z.ZodOptional<z.ZodString>;
-    errorCode: z.ZodOptional<z.ZodString>;
-}, "strict", z.ZodTypeAny, {
-    url: string;
-    fetchedAt: string;
-    cached: boolean;
-    error?: string | undefined;
-    markdown?: string | undefined;
-    title?: string | undefined;
-    contentSize?: number | undefined;
-    resourceUri?: string | undefined;
-    resourceMimeType?: string | undefined;
-    truncated?: boolean | undefined;
-    errorCode?: string | undefined;
-    file?: {
-        downloadUrl: string;
-        fileName: string;
-        expiresAt: string;
-    } | undefined;
-}, {
-    url: string;
-    fetchedAt: string;
-    cached: boolean;
-    error?: string | undefined;
-    markdown?: string | undefined;
-    title?: string | undefined;
-    contentSize?: number | undefined;
-    resourceUri?: string | undefined;
-    resourceMimeType?: string | undefined;
-    truncated?: boolean | undefined;
-    errorCode?: string | undefined;
-    file?: {
-        downloadUrl: string;
-        fileName: string;
-        expiresAt: string;
-    } | undefined;
-}>;
+    }, z.core.$strip>>;
+}, z.core.$strict>;

package/dist/tools/schemas.js

@@ -7,7 +7,7 @@ const MAX_CONTENT_LENGTH = config.constants.maxContentSize;
 const customHeadersSchema = z
     .record(z.string().max(MAX_HEADER_NAME_LENGTH), z.string().max(MAX_HEADER_VALUE_LENGTH))
     .refine((headers) => Object.keys(headers).length <= MAX_HEADER_COUNT, {
-    message: `customHeaders must have at most ${MAX_HEADER_COUNT} entries`,
+    error: `customHeaders must have at most ${MAX_HEADER_COUNT} entries`,
 });
 const requestOptionsSchema = z.object({
     customHeaders: customHeadersSchema
@@ -47,6 +47,10 @@ const formatOptionsSchema = z.object({
         .enum(['jsonl', 'markdown'])
         .default('jsonl')
         .describe('Output format'),
+    includeContentBlocks: z
+        .boolean()
+        .optional()
+        .describe('Include content block counts when format=markdown'),
 });
 const resourceFieldsSchema = z.object({
     contentSize: z.number().optional().describe('Content length in characters'),
@@ -71,29 +75,18 @@ const fileDownloadSchema = z.object({
     fileName: z.string().describe('Suggested filename for download'),
     expiresAt: z.string().describe('ISO timestamp when download expires'),
 });
-export const fetchUrlInputSchema = requestOptionsSchema
-    .extend({
-    url: z
-        .string()
-        .min(1)
-        .max(config.constants.maxUrlLength)
-        .describe('The URL to fetch'),
-})
-    .merge(extractionOptionsSchema)
-    .merge(formatOptionsSchema)
-    .strict();
-export const fetchMarkdownInputSchema = requestOptionsSchema
-    .extend({
-    url: z
-        .string()
-        .min(1)
-        .max(config.constants.maxUrlLength)
-        .describe('The URL to fetch'),
-})
-    .merge(extractionOptionsSchema)
-    .strict();
-export const fetchUrlOutputSchema = z
-    .object({
+export const fetchUrlInputSchema = z.strictObject({
+    ...requestOptionsSchema.shape,
+    url: z.url({ protocol: /^https?:$/i }).describe('The URL to fetch'),
+    ...extractionOptionsSchema.shape,
+    ...formatOptionsSchema.shape,
+});
+export const fetchMarkdownInputSchema = z.strictObject({
+    ...requestOptionsSchema.shape,
+    url: z.url({ protocol: /^https?:$/i }).describe('The URL to fetch'),
+    ...extractionOptionsSchema.shape,
+});
+export const fetchUrlOutputSchema = z.strictObject({
     url: z.string().describe('The fetched URL'),
     title: z.string().optional().describe('Page title'),
     contentBlocks: z
@@ -107,11 +100,9 @@ export const fetchUrlOutputSchema = z
         .string()
         .optional()
         .describe('The extracted content in JSONL or Markdown format'),
-})
-    .merge(resourceFieldsSchema)
-    .strict();
-export const fetchMarkdownOutputSchema = z
-    .object({
+    ...resourceFieldsSchema.shape,
+});
+export const fetchMarkdownOutputSchema = z.strictObject({
     url: z.string().describe('The fetched URL'),
     title: z.string().optional().describe('Page title'),
     fetchedAt: z
@@ -124,6 +115,5 @@ export const fetchMarkdownOutputSchema = z
     file: fileDownloadSchema
         .optional()
         .describe('Download information when content is cached'),
-})
-    .merge(resourceFieldsSchema)
-    .strict();
+    ...resourceFieldsSchema.shape,
+});

package/dist/tools/utils/common.js

@@ -6,22 +6,26 @@ export function createContentMetadataBlock(url, article, extractedMeta, shouldEx
     if (!includeMetadata)
         return undefined;
     const now = new Date().toISOString();
-    return shouldExtractFromArticle && article
-        ? {
-            type: 'metadata',
-            title: article.title,
-            author: article.byline,
-            url,
-            fetchedAt: now,
-        }
-        : {
-            type: 'metadata',
-            title: extractedMeta.title,
-            description: extractedMeta.description,
-            author: extractedMeta.author,
-            url,
-            fetchedAt: now,
-        };
+    const metadata = {
+        type: 'metadata',
+        url,
+        fetchedAt: now,
+    };
+    if (shouldExtractFromArticle && article) {
+        if (article.title !== undefined)
+            metadata.title = article.title;
+        if (article.byline !== undefined)
+            metadata.author = article.byline;
+        return metadata;
+    }
+    if (extractedMeta.title !== undefined)
+        metadata.title = extractedMeta.title;
+    if (extractedMeta.description !== undefined) {
+        metadata.description = extractedMeta.description;
+    }
+    if (extractedMeta.author !== undefined)
+        metadata.author = extractedMeta.author;
+    return metadata;
 }
 export function truncateContent(content, maxLength, suffix = TRUNCATION_MARKER) {
     if (maxLength === undefined ||

package/dist/tools/utils/content-transform-async.d.ts

@@ -0,0 +1,6 @@
+import type { JsonlTransformResult, MarkdownTransformResult, TransformOptions } from '../../config/types/content.js';
+export declare function transformHtmlToJsonlAsync(html: string, url: string, options: TransformOptions): Promise<JsonlTransformResult>;
+export declare function transformHtmlToMarkdownAsync(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
+export declare function transformHtmlToMarkdownWithBlocksAsync(html: string, url: string, options: TransformOptions & {
+    includeContentBlocks?: boolean;
+}): Promise<JsonlTransformResult>;

package/dist/tools/utils/content-transform-async.js

@@ -0,0 +1,33 @@
+import { logWarn } from '../../services/logger.js';
+import { runTransformInWorker, } from '../../services/transform-worker-pool.js';
+import { transformHtmlToJsonl, transformHtmlToMarkdown, transformHtmlToMarkdownWithBlocks, } from './content-transform.js';
+async function runOrFallback(job, fallback) {
+    try {
+        const result = await runTransformInWorker(job);
+        if (result)
+            return result;
+    }
+    catch (error) {
+        logWarn('Transform worker unavailable; using main thread', {
+            error: error instanceof Error ? error.message : String(error),
+        });
+    }
+    return fallback();
+}
+export async function transformHtmlToJsonlAsync(html, url, options) {
+    const result = await runOrFallback({ mode: 'jsonl', html, url, options }, () => transformHtmlToJsonl(html, url, options));
+    return result;
+}
+export async function transformHtmlToMarkdownAsync(html, url, options) {
+    const result = await runOrFallback({ mode: 'markdown', html, url, options }, () => transformHtmlToMarkdown(html, url, options));
+    return result;
+}
+export async function transformHtmlToMarkdownWithBlocksAsync(html, url, options) {
+    const result = await runOrFallback({
+        mode: 'markdown-blocks',
+        html,
+        url,
+        options,
+    }, () => transformHtmlToMarkdownWithBlocks(html, url, options));
+    return result;
+}

package/dist/tools/utils/content-transform.d.ts

@@ -8,7 +8,10 @@ interface ContentLengthOptions {
 }
 interface MarkdownOptions extends ExtractionOptions, ContentLengthOptions {
 }
+interface MarkdownWithBlocksOptions extends ExtractionOptions, ContentLengthOptions {
+    readonly includeContentBlocks?: boolean;
+}
 export declare function transformHtmlToJsonl(html: string, url: string, options: ExtractionOptions & ContentLengthOptions): JsonlTransformResult;
 export declare function transformHtmlToMarkdown(html: string, url: string, options: MarkdownOptions): MarkdownTransformResult;
-export declare function transformHtmlToMarkdownWithBlocks(html: string, url: string, options: ExtractionOptions & ContentLengthOptions): JsonlTransformResult;
+export declare function transformHtmlToMarkdownWithBlocks(html: string, url: string, options: MarkdownWithBlocksOptions): JsonlTransformResult;
 export {};