microlink.io 0.1.1 → 0.2.0

package/bin/help.js

@@ -0,0 +1,334 @@
+'use strict'
+
+const { gray, white } = require('./style')
+
+const col = (name, desc) => `  ${white(name.padEnd(21))}   ${gray(desc)}`
+
+const cmd = (rest, comment) =>
+  `${comment ? `  ${gray(`# ${comment}`)}\n` : ''}  ${white(
+    'microlink'
+  )} ${gray(rest)}`
+
+const rows = items => items.map(([name, desc]) => col(name, desc)).join('\n')
+
+const CLI = [
+  ['--api-key', 'Microlink API key (defaults to MICROLINK_API_KEY env)'],
+  ['--endpoint', 'Microlink API endpoint'],
+  ['--header, -H', "Extra request header as 'Name: value' (repeatable)"],
+  [
+    '--http.header.<name>',
+    'HTTP request header (e.g. --http.header.authorization)'
+  ],
+  ['--trace', 'Print request & response payload (API key masked)'],
+  ['--trace-full', 'Same as --trace, including the full API key'],
+  ['--help', 'Show this help']
+]
+
+const CLI_NO_TRACE = CLI.filter(
+  ([name]) => name !== '--trace' && name !== '--trace-full'
+)
+
+const BROWSER = [
+  ['--adblock', 'Block ads and trackers'],
+  ['--animations', 'Enable CSS animations'],
+  ['--cacheKey', 'Custom cache key'],
+  ['--click', 'CSS selector(s) to click before capture'],
+  ['--colorScheme', 'Color scheme: no-preference, light, dark'],
+  ['--device', "Emulate a device (e.g. 'iPhone 11')"],
+  ['--filename', 'Suggested download filename'],
+  ['--filter', 'Pick response fields'],
+  ['--force', 'Bypass the cache'],
+  ['--javascript', 'Enable or disable JavaScript'],
+  ['--mediaType', 'Emulate media: screen, print'],
+  ['--modules', 'Inject ES module URLs'],
+  ['--prerender', 'Prerender: auto, true, false'],
+  ['--proxy', 'Proxy URL or country'],
+  ['--retry', 'Retry count'],
+  ['--scripts', 'Inject script URLs'],
+  ['--scroll', 'CSS selector to scroll into view'],
+  ['--staleTtl', 'Stale-while-revalidate TTL'],
+  ['--styles', 'Inject stylesheet URLs'],
+  ['--timeout', 'Request timeout'],
+  ['--ttl', 'Cache TTL'],
+  ['--viewport', 'Viewport as JSON (width, height, ...)'],
+  ['--waitForSelector', 'Wait until a CSS selector matches'],
+  ['--waitForTimeout', 'Wait for a duration before continuing'],
+  ['--waitUntil', 'auto, load, domcontentloaded, networkidle0, networkidle2']
+]
+
+const CONTENT = [
+  ['--selector', 'CSS selector to scope the extraction'],
+  ['--selectorAll', 'CSS selector(s) matching many nodes'],
+  ['--type', 'Cast the extracted value (url, image, ...)']
+]
+
+const COLLECTION = [
+  ['--selector', 'CSS selector (single node)'],
+  ['--selectorAll', 'CSS selector(s) matching many nodes'],
+  ['--attr', 'Attribute to read (href, src, ...)'],
+  ['--type', 'Cast each value (url, image, email, ...)']
+]
+
+const ALIAS = { run: 'function' }
+
+const content = (name, desc) => ({
+  usage: `${name} <url> [options]`,
+  desc,
+  flags: CONTENT,
+  browser: true,
+  examples: [[`${name} https://example.com`, desc]]
+})
+
+const collection = (name, desc) => ({
+  usage: `${name} <url> [options]`,
+  desc,
+  flags: COLLECTION,
+  browser: true,
+  examples: [[`${name} https://example.com`, desc]]
+})
+
+const PRODUCTS = {
+  metadata: {
+    usage: ['metadata <url> [options]', '<url> [options]'],
+    desc: 'Unified metadata (title, description, image, ...); default',
+    flags: [
+      ['--palette', 'Also extract dominant colors from images'],
+      ['--meta', 'Include metadata fields, or false to skip']
+    ],
+    browser: true,
+    examples: [
+      ['https://example.com', 'unified metadata (default)'],
+      [
+        'https://example.com --trace',
+        'print request & response, API key masked'
+      ]
+    ]
+  },
+  logo: {
+    usage: 'logo <url> [options]',
+    desc: 'Brand logo of the site (--square prefers the square variant)',
+    flags: [
+      ['--square', 'Prefer the square logo variant'],
+      ['--palette', 'Also extract dominant colors']
+    ],
+    browser: true,
+    examples: [['logo https://github.com --square', 'square brand logo']]
+  },
+  markdown: content('markdown', 'Page content as Markdown'),
+  html: content('html', 'Page content as HTML'),
+  text: content('text', 'Page content as plain text'),
+  video: {
+    usage: 'video <url> [options]',
+    desc: 'Primary video of the page (returns the asset object)',
+    flags: [['--meta', 'Include metadata fields, or false to skip']],
+    browser: true,
+    examples: [['video https://example.com', 'primary video asset']]
+  },
+  audio: {
+    usage: 'audio <url> [options]',
+    desc: 'Primary audio of the page (returns the asset object)',
+    flags: [['--meta', 'Include metadata fields, or false to skip']],
+    browser: true,
+    examples: [['audio https://example.com', 'primary audio asset']]
+  },
+  emails: collection('emails', 'Every email address present on the page'),
+  links: collection('links', 'Every absolute link URL on the page'),
+  images: collection('images', 'Every absolute image URL on the page'),
+  videos: collection('videos', 'Every absolute video URL on the page'),
+  audios: collection('audios', 'Every absolute audio URL on the page'),
+  extract: {
+    usage: 'extract <url> --data <json> [options]',
+    desc: 'Custom MQL data rules',
+    flags: [['--data', 'JSON data rules (e.g. --data \'{"field":{...}}\')']],
+    browser: true,
+    examples: [
+      [
+        'extract https://microlink.io --data \'{"image":{"selector":"meta[property=og:image]","attr":"content","type":"image"}}\'',
+        'extract og:image via MQL'
+      ]
+    ]
+  },
+  screenshot: {
+    usage: 'screenshot <url> [options]',
+    desc: 'Take a screenshot (returns the asset object)',
+    flags: [
+      ['--fullPage', 'Capture the full scrollable page'],
+      ['--type', 'Image format: png, jpeg'],
+      ['--element', 'CSS selector of the element to capture'],
+      ['--omitBackground', 'Transparent background (png)'],
+      ['--optimizeForSpeed', 'Faster encode, larger file'],
+      ['--overlay', 'Browser chrome overlay as JSON'],
+      ['--codeScheme', 'Syntax theme for code pages'],
+      ['--animated', 'Animated screenshot (GIF/MP4)'],
+      ['--palette', 'Also extract dominant colors'],
+      ['--quality', 'JPEG quality (0–100)']
+    ],
+    browser: true,
+    examples: [
+      ['screenshot https://example.com --fullPage', 'full-page screenshot']
+    ]
+  },
+  pdf: {
+    usage: 'pdf <url> [options]',
+    desc: 'Generate a PDF (returns the asset object)',
+    flags: [
+      ['--format', 'Page format: Letter, Legal, A4, ...'],
+      ['--margin', "Margin (e.g. '0.5cm' or JSON)"],
+      ['--scale', 'Scale (0.1–2)'],
+      ['--landscape', 'Landscape orientation'],
+      ['--pageRanges', "Pages to print (e.g. '1-3')"],
+      ['--width', 'Page width'],
+      ['--height', 'Page height'],
+      ['--printBackground', 'Print background graphics']
+    ],
+    browser: true,
+    examples: [['pdf https://example.com --format A4', 'A4 PDF']]
+  },
+  embed: {
+    usage: 'embed <url> [options]',
+    desc: 'oEmbed-style embeddable iframe ({ html, scripts })',
+    flags: [
+      ['--maxWidth', 'Maximum iframe width'],
+      ['--maxHeight', 'Maximum iframe height']
+    ],
+    browser: true,
+    examples: [['embed https://example.com', 'embeddable iframe']]
+  },
+  technologies: {
+    usage: 'technologies <url> [options]',
+    desc: 'Detect the tech stack behind the site',
+    flags: [],
+    browser: true,
+    examples: [['technologies https://example.com', 'detect the tech stack']]
+  },
+  lighthouse: {
+    usage: 'lighthouse <url> [options]',
+    desc: 'Run a Lighthouse report',
+    flags: [
+      ['--onlyCategories', 'Limit to these categories'],
+      ['--onlyAudits', 'Limit to these audits'],
+      ['--skipAudits', 'Skip these audits'],
+      ['--output', 'Report format: json, html, csv']
+    ],
+    browser: true,
+    examples: [['lighthouse https://example.com', 'run a Lighthouse report']]
+  },
+  search: {
+    usage: 'search <query> [options]',
+    desc: 'Google as structured data (query instead of url)',
+    flags: [
+      [
+        '--type',
+        'news, images, videos, places, maps, shopping, scholar, patents, autocomplete'
+      ],
+      ['--limit', 'Maximum number of results'],
+      ['--location', 'Country or locale (e.g. es)'],
+      ['--period', 'Recency: hour, day, week, month, year'],
+      ['--timeout', 'Request timeout']
+    ],
+    cli: CLI_NO_TRACE,
+    examples: [
+      [
+        'search "best coffee" --limit 10 --location es',
+        'Google results in Spain, limit 10'
+      ],
+      [
+        'search "open source llm" --type news --period week',
+        'news results from the past week'
+      ]
+    ]
+  },
+  function: {
+    usage: 'function <url> --file <path> [options]',
+    desc: 'Run code remotely with browser access',
+    flags: [['--file', 'Path to the code file']],
+    browser: true,
+    cli: CLI_NO_TRACE,
+    note: 'Extra flags are injected as variables in the function scope.',
+    examples: [
+      [
+        'function https://example.com --file ./fn.js',
+        'run ./fn.js with browser access'
+      ]
+    ]
+  }
+}
+
+const productList = Object.entries(PRODUCTS)
+  .map(([name, product]) => col(name, product.desc))
+  .join('\n')
+
+const global = `Usage
+${cmd('<url> [options]')}
+${cmd('<product> <url|query> [options]')}
+
+Products
+${productList}
+
+Options
+${rows(CLI)}
+
+Examples
+${cmd('https://example.com', 'unified metadata (default)')}
+${cmd(
+  'https://example.com --trace',
+  'print request & response, API key masked'
+)}
+${cmd(
+  'https://example.com --trace-full',
+  'same as --trace, including the full API key'
+)}
+${cmd('markdown https://example.com', 'page content as Markdown')}
+${cmd('screenshot https://example.com --fullPage', 'full-page screenshot')}
+${cmd('logo https://github.com --square', 'square brand logo')}
+${cmd('links https://example.com', 'every absolute link on the page')}
+${cmd(
+  'search "best coffee" --limit 10 --location es',
+  'Google results in Spain, limit 10'
+)}
+${cmd(
+  'search "open source llm" --type news --period week',
+  'news results from the past week'
+)}
+${cmd(
+  'extract https://microlink.io --data \'{"image":{"selector":"meta[property=og:image]","attr":"content","type":"image"}}\'',
+  'extract og:image via MQL'
+)}
+${cmd(
+  'function https://example.com --file ./fn.js',
+  'run ./fn.js with browser access'
+)}
+`
+
+const render = (name, product) => {
+  const usage = []
+    .concat(product.usage)
+    .map(line => cmd(line))
+    .join('\n')
+  const cli = product.cli ?? CLI
+  const options = [...product.flags, ...cli]
+  const parts = [
+    'Usage',
+    usage,
+    '',
+    gray(product.desc),
+    '',
+    'Options',
+    rows(options)
+  ]
+  if (product.browser) parts.push('', 'Browser', rows(BROWSER))
+  if (product.note) parts.push('', gray(product.note))
+  if (product.examples) {
+    parts.push(
+      '',
+      'Examples',
+      ...product.examples.map(([rest, comment]) => cmd(rest, comment))
+    )
+  }
+  return parts.join('\n') + '\n'
+}
+
+module.exports = command => {
+  const name = ALIAS[command] ?? command
+  return PRODUCTS[name] ? render(name, PRODUCTS[name]) : global
+}

package/bin/index.js

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 'use strict'
 
-const { styleText } = require('node:util')
 const { readFileSync } = require('fs')
 const path = require('path')
 const mri = require('mri')
+const helpText = require('./help')
+const { gray, white, green, red, styleText } = require('./style')
 
 const create = require('../src')
 
@@ -13,10 +14,6 @@ const HIDE_CURSOR = '\u001b[?25l'
 const CLEAR_LINE = '\r\u001b[K'
 const FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
 
-const gray = str => styleText('gray', str)
-const white = str => styleText('white', str)
-const green = str => styleText('green', str)
-const red = str => styleText('red', str)
 const label = (text, color) =>
   styleText(['inverse', 'bold', color], ` ${text.toUpperCase()} `)
 const keyValue = (key, value) => key + ' ' + gray(value)
@@ -98,8 +95,10 @@ const tracePayload = ({
   const rest = { ...requestOptions }
   delete rest.responseType
   const headers = { ...rest.headers }
-  if (!full && headers['x-api-key']) {
-    headers['x-api-key'] = humanizeApiKey(headers['x-api-key'])
+  if (!full) {
+    for (const key of ['x-api-key', 'authorization', 'cookie']) {
+      if (headers[key]) headers[key] = humanizeApiKey(headers[key])
+    }
   }
   return {
     request: { url: requestUrl, ...rest, headers },
@@ -214,11 +213,13 @@ const printFail = error => {
   if (error.more) console.error('  ', keyValue(red('more'), error.more))
 }
 
-const showHelp = () => {
-  console.log(readFileSync(path.join(__dirname, 'help.txt'), 'utf8'))
+const showHelp = command => {
+  console.log(helpText(command).trimEnd())
   process.exit(0)
 }
 
+const HTTP_HEADER = 'http.header.'
+
 const parseHeaders = input => {
   const headers = {}
   for (const item of [].concat(input ?? [])) {
@@ -231,10 +232,24 @@ const parseHeaders = input => {
   return headers
 }
 
+const takeHttpHeaders = flags => {
+  const headers = {}
+  for (const key of Object.keys(flags)) {
+    if (!key.startsWith(HTTP_HEADER)) continue
+    let value = flags[key]
+    delete flags[key]
+    if (Array.isArray(value)) value = value.at(-1)
+    if (typeof value !== 'string' && typeof value !== 'number') continue
+    const name = key.slice(HTTP_HEADER.length).toLowerCase()
+    if (name) headers[name] = String(value)
+  }
+  return headers
+}
+
 const argv = mri(process.argv.slice(2), {
   alias: { H: 'header' },
-  boolean: ['trace', 'trace-full'],
-  string: ['header', 'api-key', 'data', 'file']
+  boolean: ['trace', 'trace-full', 'help'],
+  string: ['header', 'api-key', 'data', 'file', 'endpoint']
 })
 
 let {
@@ -245,6 +260,7 @@ let {
   file,
   'api-key': apiKeyFlag,
   apiKey: apiKeyCamel,
+  endpoint: endpointFlag,
   trace,
   'trace-full': traceFull,
   ...flags
@@ -252,15 +268,21 @@ let {
 
 const isTrace = trace || traceFull
 
-if (help || !command) showHelp()
+if (!command) showHelp()
 
 const apiKey = apiKeyFlag || apiKeyCamel || process.env.MICROLINK_API_KEY
-const client = create(apiKey ? { apiKey } : {})
+const endpoint = endpointFlag
+const client = create({
+  ...(apiKey && { apiKey }),
+  ...(endpoint && { endpoint })
+})
 
 if (typeof client[command] !== 'function') {
   if (!target && URL.canParse(command)) {
     target = command
     command = 'metadata'
+  } else if (help) {
+    showHelp()
   } else {
     console.error(
       `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.`
@@ -269,6 +291,8 @@ if (typeof client[command] !== 'function') {
   }
 }
 
+if (help || !target) showHelp(command)
+
 if (
   isTrace &&
   (command === 'search' || command === 'function' || command === 'run')
@@ -278,7 +302,7 @@ if (
 }
 
 const options = { ...flags }
-const headers = parseHeaders(header)
+const headers = { ...takeHttpHeaders(options), ...parseHeaders(header) }
 if (Object.keys(headers).length > 0) options.headers = headers
 
 const invoke = () => {

package/bin/style.js

@@ -0,0 +1,10 @@
+'use strict'
+
+const { styleText } = require('node:util')
+
+const gray = str => styleText('gray', str)
+const white = str => styleText('white', str)
+const green = str => styleText('green', str)
+const red = str => styleText('red', str)
+
+module.exports = { gray, white, green, red, styleText }

package/package.json

@@ -2,7 +2,7 @@
   "name": "microlink.io",
   "description": "The Microlink API organized into products, each returning a direct result",
   "homepage": "https://github.com/microlinkhq/microlink",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "types": "./src/index.d.ts",
   "main": "./src/index.js",
   "exports": {
@@ -76,5 +76,5 @@
     },
     "directory": "test"
   },
-  "gitHead": "a12fcd2a7bc918e741a71b50fd2cd93876f4ca8f"
+  "gitHead": "3766602b11599a8284e4aa31815d409e0ffa0788"
 }

package/bin/help.txt

@@ -1,53 +0,0 @@
-Usage
-  $ microlink <url> [options]
-  $ microlink <product> <url|query> [options]
-
-Products
-  metadata      Unified metadata (title, description, image, ...); default
-  logo          Brand logo of the site (--square prefers the square variant)
-  markdown      Page content as Markdown
-  html          Page content as HTML
-  text          Page content as plain text
-  video         Primary video of the page (returns the asset object)
-  audio         Primary audio of the page (returns the asset object)
-  emails        Every email address present on the page
-  links         Every absolute link URL on the page
-  images        Every absolute image URL on the page
-  videos        Every absolute video URL on the page
-  audios        Every absolute audio URL on the page
-  extract       Custom MQL data rules (--data '{"field":{...}}')
-  screenshot    Take a screenshot (returns the asset object)
-  pdf           Generate a PDF (returns the asset object)
-  embed         oEmbed-style embeddable iframe ({ html, scripts })
-  technologies  Detect the tech stack behind the site
-  lighthouse    Run a Lighthouse report
-  search        Google as structured data (query instead of url); --type
-                routes to news, images, videos, places, maps, shopping,
-                scholar, patents or autocomplete
-  function      Run code remotely with browser access (--file ./fn.js);
-                extra flags are injected as variables in the function scope
-
-Options
-  --api-key     Microlink API key (defaults to MICROLINK_API_KEY env)
-  --header, -H  Extra request header as 'Name: value' (repeatable)
-  --data        JSON data rules for the extract command
-  --file        Path to the code file for the function command
-  --trace       Print request & response payload (API key masked)
-  --trace-full  Same as --trace, including the full API key
-  --help        Show this help
-
-Any other flag is passed as an option to the product, e.g. --fullPage,
---device 'iPhone 11', --waitUntil networkidle0, --selector article.
-
-Examples
-  $ microlink https://example.com
-  $ microlink https://example.com --trace
-  $ microlink https://example.com --trace-full
-  $ microlink markdown https://example.com
-  $ microlink screenshot https://example.com --fullPage
-  $ microlink logo https://github.com --square
-  $ microlink links https://example.com
-  $ microlink search "best coffee" --limit 10 --location es
-  $ microlink search "open source llm" --type news --period week
-  $ microlink extract https://microlink.io --data '{"image":{"selector":"meta[property=og:image]","attr":"content","type":"image"}}'
-  $ microlink function https://example.com --file ./fn.js