@j0hanz/superfetch 2.2.1 → 2.2.2

package/dist/tools.js

@@ -6,7 +6,7 @@ import { FetchError, getErrorMessage, isSystemError } from './errors.js';
 import { fetchNormalizedUrl, normalizeUrl, transformToRawUrl, } from './fetch.js';
 import { getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from './observability.js';
 import { transformHtmlToMarkdown, } from './transform.js';
-import { isRecord } from './type-guards.js';
+import { isObject } from './type-guards.js';
 const TRUNCATION_MARKER = '...[truncated]';
 const FETCH_PROGRESS_TOTAL = 4;
 const fetchUrlInputSchema = z.strictObject({
@@ -246,7 +246,7 @@ function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
     cache.set(cacheKey, serializer(data), metadata);
 }
 function extractTitle(value) {
-    if (!isRecord(value))
+    if (!isObject(value))
         return undefined;
     const { title } = value;
     return typeof title === 'string' ? title : undefined;
@@ -319,7 +319,7 @@ function resolveToolErrorMessage(error, fallbackMessage) {
 function parseJsonRecord(input) {
     try {
         const parsed = JSON.parse(input);
-        return isRecord(parsed) ? parsed : undefined;
+        return isObject(parsed) ? parsed : undefined;
     }
     catch {
         return undefined;
@@ -467,7 +467,7 @@ export function withRequestContextIfMissing(handler) {
     };
 }
 function resolveRequestIdFromExtra(extra) {
-    if (!isRecord(extra))
+    if (!isObject(extra))
         return undefined;
     const { requestId } = extra;
     if (typeof requestId === 'string')

package/dist/transform-types.d.ts

@@ -0,0 +1,81 @@
+/**
+ * Shared types for the transform pipeline.
+ * Extracted to avoid circular dependencies between transform modules.
+ */
+/**
+ * Metadata block for attaching source information to markdown output.
+ */
+export interface MetadataBlock {
+    type: 'metadata';
+    title?: string;
+    description?: string;
+    author?: string;
+    url: string;
+    fetchedAt: string;
+}
+/**
+ * Article extracted by Readability.
+ */
+export interface ExtractedArticle {
+    title?: string;
+    byline?: string;
+    content: string;
+    textContent: string;
+    excerpt?: string;
+    siteName?: string;
+}
+/**
+ * Metadata extracted from HTML meta tags.
+ */
+export interface ExtractedMetadata {
+    title?: string;
+    description?: string;
+    author?: string;
+    image?: string;
+    publishedAt?: string;
+    modifiedAt?: string;
+}
+/**
+ * Result of content extraction (article + metadata).
+ */
+export interface ExtractionResult {
+    article: ExtractedArticle | null;
+    metadata: ExtractedMetadata;
+}
+/**
+ * Result of HTML to markdown transformation.
+ */
+export interface MarkdownTransformResult {
+    markdown: string;
+    title: string | undefined;
+    truncated: boolean;
+}
+/**
+ * Options for transform operations.
+ */
+export interface TransformOptions {
+    includeMetadata: boolean;
+    signal?: AbortSignal;
+}
+/**
+ * Telemetry event emitted during transform stages.
+ */
+export interface TransformStageEvent {
+    v: 1;
+    type: 'stage';
+    stage: string;
+    durationMs: number;
+    url: string;
+    requestId?: string;
+    operationId?: string;
+    truncated?: boolean;
+}
+/**
+ * Context for tracking transform stage timing.
+ */
+export interface TransformStageContext {
+    readonly stage: string;
+    readonly startTime: number;
+    readonly url: string;
+}
+//# sourceMappingURL=transform-types.d.ts.map

package/dist/transform-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transform-types.d.ts","sourceRoot":"","sources":["../src/transform-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB"}

package/dist/transform-types.js

@@ -0,0 +1,6 @@
+/**
+ * Shared types for the transform pipeline.
+ * Extracted to avoid circular dependencies between transform modules.
+ */
+export {};
+//# sourceMappingURL=transform-types.js.map

package/dist/transform-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transform-types.js","sourceRoot":"","sources":["../src/transform-types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}

package/dist/transform.d.ts

@@ -1,52 +1,8 @@
-export interface MetadataBlock {
-    type: 'metadata';
-    title?: string;
-    description?: string;
-    author?: string;
-    url: string;
-    fetchedAt: string;
-}
-export interface ExtractedArticle {
-    title?: string;
-    byline?: string;
-    content: string;
-    textContent: string;
-    excerpt?: string;
-    siteName?: string;
-}
-export interface ExtractedMetadata {
-    title?: string;
-    description?: string;
-    author?: string;
-}
-export interface ExtractionResult {
-    article: ExtractedArticle | null;
-    metadata: ExtractedMetadata;
-}
-export interface MarkdownTransformResult {
-    markdown: string;
-    title: string | undefined;
-    truncated: boolean;
-}
-export interface TransformOptions {
-    includeMetadata: boolean;
-    signal?: AbortSignal;
-}
-export interface TransformStageEvent {
-    v: 1;
-    type: 'stage';
-    stage: string;
-    durationMs: number;
-    url: string;
-    requestId?: string;
-    operationId?: string;
-    truncated?: boolean;
-}
-export interface TransformStageContext {
-    readonly stage: string;
-    readonly startTime: number;
-    readonly url: string;
-}
+import type { ExtractedArticle, ExtractedMetadata, ExtractionResult, MarkdownTransformResult, MetadataBlock, TransformOptions, TransformStageContext } from './transform-types.js';
+export { detectLanguageFromCode, resolveLanguageFromAttributes, } from './language-detection.js';
+export { cleanupMarkdownArtifacts, promoteOrphanHeadings, } from './markdown-cleanup.js';
+export { removeNoiseFromHtml } from './dom-noise-removal.js';
+export type { MetadataBlock, ExtractedArticle, ExtractedMetadata, ExtractionResult, MarkdownTransformResult, TransformOptions, TransformStageEvent, TransformStageContext, } from './transform-types.js';
 export declare function startTransformStage(url: string, stage: string): TransformStageContext | null;
 export declare function endTransformStage(context: TransformStageContext | null, options?: {
     truncated?: boolean;
@@ -55,14 +11,13 @@ export declare function extractContent(html: string, url: string, options?: {
     extractArticle?: boolean;
     signal?: AbortSignal;
 }): ExtractionResult;
-export declare function detectLanguageFromCode(code: string): string | undefined;
-export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;
 export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock, options?: {
     url?: string;
     signal?: AbortSignal;
     document?: Document;
+    skipNoiseRemoval?: boolean;
 }): string;
-export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
+export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtmlOrDocument: string | Document): boolean;
 export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
 export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
 export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;

package/dist/transform.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../src/transform.ts"],"names":[],"mappings":"AA+BA,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAMD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAwCD,MAAM,WAAW,mBAAmB;IAClC,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAaD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,qBAAqB,GAAG,IAAI,CAQ9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,IAAI,CAoBN;AAgJD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA2ZD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMvE;AAED,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,MAAM,GAAG,SAAS,CAGpB;AA6gBD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,EAAE,QAAQ,CAAA;CAAE,GACpE,MAAM,CAkBR;AAkgBD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,YAAY,EAAE,MAAM,GACnB,OAAO,CAWT;AAED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AAgND,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAkBzB;AA6ED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAIjE;AA0eD,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAelC"}
+{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../src/transform.ts"],"names":[],"mappings":"AAqCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EACL,sBAAsB,EACtB,6BAA6B,GAC9B,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACL,wBAAwB,EACxB,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAG7D,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,mBAAmB,EACnB,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAgC9B,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,qBAAqB,GAAG,IAAI,CAQ9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,IAAI,CAoBN;AAuOD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA4XD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAyBR;AAgaD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAUT;AA0BD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AA0UD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAkBzB;AA6ED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAIjE;AA8dD,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAelC"}