html-get 2.21.20 → 2.21.22

package/index.d.ts

@@ -0,0 +1,100 @@
+import type { CheerioAPI } from 'cheerio'
+
+/**
+ * Result returned by html-get
+ */
+export interface HtmlGetResult {
+  /** The HTML content */
+  html: string
+  /** Response headers */
+  headers: Record<string, string | string[] | undefined>
+  /** Final URL after redirects */
+  url: string
+  /** HTTP status code */
+  statusCode: number
+  /** Redirect history */
+  redirects: Array<{ statusCode: number; url: string }>
+  /** Mode used: 'fetch' or 'prerender' */
+  mode: 'fetch' | 'prerender'
+  /** Parsed HTML (Cheerio) */
+  $?: CheerioAPI
+  /** Statistics about the request */
+  stats: {
+    mode: 'fetch' | 'prerender'
+    timing: number
+  }
+}
+
+/**
+ * Options for html-get
+ */
+export interface HtmlGetOptions {
+  /** Character encoding for HTML (default: 'utf-8') */
+  encoding?: string
+  /** Function that returns a browserless instance (required unless prerender is false) */
+  getBrowserless?: () => Promise<any>
+  /** Function to determine the mode ('fetch' or 'prerender') */
+  getMode?: (url: string, options: { prerender: boolean | 'auto' }) => 'fetch' | 'prerender'
+  /** Function to create temporary files */
+  getTemporalFile?: (input: string, ext?: string) => { path: string }
+  /** Options passed to got (the HTTP client) */
+  gotOpts?: Record<string, any>
+  /** Request headers */
+  headers?: Record<string, string>
+  /** Mutool function for PDF processing, or false to disable */
+  mutool?: ((...args: string[]) => any) | false
+  /** Prerender mode: true, false, or 'auto' (default) */
+  prerender?: boolean | 'auto'
+  /** Options passed to Puppeteer */
+  puppeteerOpts?: Record<string, any>
+  /** Rewrite relative URLs to absolute */
+  rewriteUrls?: boolean
+  /** Rewrite common HTML meta tag mistakes */
+  rewriteHtml?: boolean
+  /** Function to serialize HTML (default: $ => ({ html: $.html() })) */
+  serializeHtml?: ($: CheerioAPI) => { html: string }
+}
+
+/**
+ * Main function to get HTML from a URL
+ */
+export function htmlGet(
+  targetUrl: string,
+  options?: HtmlGetOptions
+): Promise<HtmlGetResult>
+
+/**
+ * Check if a URL should use 'fetch' mode (no prerender needed)
+ */
+export function isFetchMode(url: string): boolean
+
+/**
+ * Get content directly with a specific mode
+ */
+export function getContent(
+  url: string,
+  mode: 'fetch' | 'prerender',
+  options?: HtmlGetOptions
+): Promise<HtmlGetResult>
+
+/**
+ * Default mutool function (returns undefined if mutool is not installed)
+ */
+export function defaultMutool(): ((...args: string[]) => any) | undefined
+
+/**
+ * Default request timeout in milliseconds
+ */
+export const REQ_TIMEOUT: number
+
+/**
+ * Default abort types for prerendering
+ */
+export const ABORT_TYPES: string[]
+
+/**
+ * PDF size threshold in bytes (150KB)
+ */
+export const PDF_SIZE_TRESHOLD: number
+
+export default htmlGet

package/package.json

@@ -2,7 +2,8 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.21.20",
+  "types": "index.d.ts",
+  "version": "2.21.22",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -16,6 +17,10 @@
     {
       "name": "Divyansh Singh",
       "email": "40380293+brc-dd@users.noreply.github.com"
+    },
+    {
+      "name": "Michael Lip",
+      "email": "51033404+theluckystrike@users.noreply.github.com"
     }
   ],
   "repository": {
@@ -86,6 +91,7 @@
   },
   "files": [
     "bin",
+    "index.d.ts",
     "scripts",
     "src"
   ],

package/scripts/postinstall

@@ -17,7 +17,6 @@ const domains = [
   [['domainWithoutSuffix', 'csdn']],
   [['domainWithoutSuffix', 'deviantart']],
   [['domainWithoutSuffix', 'digg']],
-  [['domainWithoutSuffix', 'dribbble']],
   [['domainWithoutSuffix', 'engadget']],
   [['domainWithoutSuffix', 'etsy']],
   [['domainWithoutSuffix', 'eventbrite']],
@@ -78,6 +77,7 @@ const { top, rest } = reduce(
   { top: new Array(topsites.length), rest: [] }
 )
 
-writeFile('./src/auto-domains.json', JSON.stringify(compact(top).concat(rest), null, 2)).catch(
-  error => console.log(error)
-)
+writeFile(
+  './src/auto-domains.json',
+  JSON.stringify(compact(top).concat(rest), null, 2)
+).catch(error => console.log(error))

package/src/auto-domains.json

@@ -44,67 +44,67 @@
   [
     [
       "domainWithoutSuffix",
-      "github"
+      "vimeo"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "vimeo"
+      "github"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "nytimes"
+      "bbc"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "bbc"
+      "nytimes"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "imdb"
+      "theguardian"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "slideshare"
+      "imdb"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "theguardian"
+      "pinterest"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "spotify"
+      "telegraph"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "huffingtonpost"
+      "slideshare"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "pinterest"
+      "huffingtonpost"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "telegraph"
+      "spotify"
     ]
   ],
   [
@@ -122,19 +122,19 @@
   [
     [
       "domainWithoutSuffix",
-      "techcrunch"
+      "engadget"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "zoom"
+      "techcrunch"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "engadget"
+      "zoom"
     ]
   ],
   [
@@ -146,121 +146,121 @@
   [
     [
       "domainWithoutSuffix",
-      "eventbrite"
+      "arxiv"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "yelp"
+      "eventbrite"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "theverge"
+      "yelp"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "arxiv"
+      "theverge"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "imgur"
+      "dribbble"
     ]
   ],
   [
     [
-      "domainWithoutSuffix",
-      "reddit"
+      "domain",
+      "bsky.app"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "stackoverflow"
+      "csdn"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "flickr"
+      "deviantart"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "sourceforge"
+      "digg"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "dribbble"
+      "etsy"
     ]
   ],
   [
     [
-      "domain",
-      "bsky.app"
+      "domainWithoutSuffix",
+      "flickr"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "csdn"
+      "ghost"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "deviantart"
+      "giphy"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "digg"
+      "gitlab"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "etsy"
+      "imgur"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "ghost"
+      "meetup"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "giphy"
+      "producthunt"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "gitlab"
+      "reddit"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "meetup"
+      "sourceforge"
     ]
   ],
   [
     [
       "domainWithoutSuffix",
-      "producthunt"
+      "stackoverflow"
     ]
   ],
   [