microlink.io 0.0.5 → 0.2.0

package/bin/help.js

@@ -0,0 +1,334 @@
+'use strict'
+
+const { gray, white } = require('./style')
+
+const col = (name, desc) => `  ${white(name.padEnd(21))}   ${gray(desc)}`
+
+const cmd = (rest, comment) =>
+  `${comment ? `  ${gray(`# ${comment}`)}\n` : ''}  ${white(
+    'microlink'
+  )} ${gray(rest)}`
+
+const rows = items => items.map(([name, desc]) => col(name, desc)).join('\n')
+
+const CLI = [
+  ['--api-key', 'Microlink API key (defaults to MICROLINK_API_KEY env)'],
+  ['--endpoint', 'Microlink API endpoint'],
+  ['--header, -H', "Extra request header as 'Name: value' (repeatable)"],
+  [
+    '--http.header.<name>',
+    'HTTP request header (e.g. --http.header.authorization)'
+  ],
+  ['--trace', 'Print request & response payload (API key masked)'],
+  ['--trace-full', 'Same as --trace, including the full API key'],
+  ['--help', 'Show this help']
+]
+
+const CLI_NO_TRACE = CLI.filter(
+  ([name]) => name !== '--trace' && name !== '--trace-full'
+)
+
+const BROWSER = [
+  ['--adblock', 'Block ads and trackers'],
+  ['--animations', 'Enable CSS animations'],
+  ['--cacheKey', 'Custom cache key'],
+  ['--click', 'CSS selector(s) to click before capture'],
+  ['--colorScheme', 'Color scheme: no-preference, light, dark'],
+  ['--device', "Emulate a device (e.g. 'iPhone 11')"],
+  ['--filename', 'Suggested download filename'],
+  ['--filter', 'Pick response fields'],
+  ['--force', 'Bypass the cache'],
+  ['--javascript', 'Enable or disable JavaScript'],
+  ['--mediaType', 'Emulate media: screen, print'],
+  ['--modules', 'Inject ES module URLs'],
+  ['--prerender', 'Prerender: auto, true, false'],
+  ['--proxy', 'Proxy URL or country'],
+  ['--retry', 'Retry count'],
+  ['--scripts', 'Inject script URLs'],
+  ['--scroll', 'CSS selector to scroll into view'],
+  ['--staleTtl', 'Stale-while-revalidate TTL'],
+  ['--styles', 'Inject stylesheet URLs'],
+  ['--timeout', 'Request timeout'],
+  ['--ttl', 'Cache TTL'],
+  ['--viewport', 'Viewport as JSON (width, height, ...)'],
+  ['--waitForSelector', 'Wait until a CSS selector matches'],
+  ['--waitForTimeout', 'Wait for a duration before continuing'],
+  ['--waitUntil', 'auto, load, domcontentloaded, networkidle0, networkidle2']
+]
+
+const CONTENT = [
+  ['--selector', 'CSS selector to scope the extraction'],
+  ['--selectorAll', 'CSS selector(s) matching many nodes'],
+  ['--type', 'Cast the extracted value (url, image, ...)']
+]
+
+const COLLECTION = [
+  ['--selector', 'CSS selector (single node)'],
+  ['--selectorAll', 'CSS selector(s) matching many nodes'],
+  ['--attr', 'Attribute to read (href, src, ...)'],
+  ['--type', 'Cast each value (url, image, email, ...)']
+]
+
+const ALIAS = { run: 'function' }
+
+const content = (name, desc) => ({
+  usage: `${name} <url> [options]`,
+  desc,
+  flags: CONTENT,
+  browser: true,
+  examples: [[`${name} https://example.com`, desc]]
+})
+
+const collection = (name, desc) => ({
+  usage: `${name} <url> [options]`,
+  desc,
+  flags: COLLECTION,
+  browser: true,
+  examples: [[`${name} https://example.com`, desc]]
+})
+
+const PRODUCTS = {
+  metadata: {
+    usage: ['metadata <url> [options]', '<url> [options]'],
+    desc: 'Unified metadata (title, description, image, ...); default',
+    flags: [
+      ['--palette', 'Also extract dominant colors from images'],
+      ['--meta', 'Include metadata fields, or false to skip']
+    ],
+    browser: true,
+    examples: [
+      ['https://example.com', 'unified metadata (default)'],
+      [
+        'https://example.com --trace',
+        'print request & response, API key masked'
+      ]
+    ]
+  },
+  logo: {
+    usage: 'logo <url> [options]',
+    desc: 'Brand logo of the site (--square prefers the square variant)',
+    flags: [
+      ['--square', 'Prefer the square logo variant'],
+      ['--palette', 'Also extract dominant colors']
+    ],
+    browser: true,
+    examples: [['logo https://github.com --square', 'square brand logo']]
+  },
+  markdown: content('markdown', 'Page content as Markdown'),
+  html: content('html', 'Page content as HTML'),
+  text: content('text', 'Page content as plain text'),
+  video: {
+    usage: 'video <url> [options]',
+    desc: 'Primary video of the page (returns the asset object)',
+    flags: [['--meta', 'Include metadata fields, or false to skip']],
+    browser: true,
+    examples: [['video https://example.com', 'primary video asset']]
+  },
+  audio: {
+    usage: 'audio <url> [options]',
+    desc: 'Primary audio of the page (returns the asset object)',
+    flags: [['--meta', 'Include metadata fields, or false to skip']],
+    browser: true,
+    examples: [['audio https://example.com', 'primary audio asset']]
+  },
+  emails: collection('emails', 'Every email address present on the page'),
+  links: collection('links', 'Every absolute link URL on the page'),
+  images: collection('images', 'Every absolute image URL on the page'),
+  videos: collection('videos', 'Every absolute video URL on the page'),
+  audios: collection('audios', 'Every absolute audio URL on the page'),
+  extract: {
+    usage: 'extract <url> --data <json> [options]',
+    desc: 'Custom MQL data rules',
+    flags: [['--data', 'JSON data rules (e.g. --data \'{"field":{...}}\')']],
+    browser: true,
+    examples: [
+      [
+        'extract https://microlink.io --data \'{"image":{"selector":"meta[property=og:image]","attr":"content","type":"image"}}\'',
+        'extract og:image via MQL'
+      ]
+    ]
+  },
+  screenshot: {
+    usage: 'screenshot <url> [options]',
+    desc: 'Take a screenshot (returns the asset object)',
+    flags: [
+      ['--fullPage', 'Capture the full scrollable page'],
+      ['--type', 'Image format: png, jpeg'],
+      ['--element', 'CSS selector of the element to capture'],
+      ['--omitBackground', 'Transparent background (png)'],
+      ['--optimizeForSpeed', 'Faster encode, larger file'],
+      ['--overlay', 'Browser chrome overlay as JSON'],
+      ['--codeScheme', 'Syntax theme for code pages'],
+      ['--animated', 'Animated screenshot (GIF/MP4)'],
+      ['--palette', 'Also extract dominant colors'],
+      ['--quality', 'JPEG quality (0–100)']
+    ],
+    browser: true,
+    examples: [
+      ['screenshot https://example.com --fullPage', 'full-page screenshot']
+    ]
+  },
+  pdf: {
+    usage: 'pdf <url> [options]',
+    desc: 'Generate a PDF (returns the asset object)',
+    flags: [
+      ['--format', 'Page format: Letter, Legal, A4, ...'],
+      ['--margin', "Margin (e.g. '0.5cm' or JSON)"],
+      ['--scale', 'Scale (0.1–2)'],
+      ['--landscape', 'Landscape orientation'],
+      ['--pageRanges', "Pages to print (e.g. '1-3')"],
+      ['--width', 'Page width'],
+      ['--height', 'Page height'],
+      ['--printBackground', 'Print background graphics']
+    ],
+    browser: true,
+    examples: [['pdf https://example.com --format A4', 'A4 PDF']]
+  },
+  embed: {
+    usage: 'embed <url> [options]',
+    desc: 'oEmbed-style embeddable iframe ({ html, scripts })',
+    flags: [
+      ['--maxWidth', 'Maximum iframe width'],
+      ['--maxHeight', 'Maximum iframe height']
+    ],
+    browser: true,
+    examples: [['embed https://example.com', 'embeddable iframe']]
+  },
+  technologies: {
+    usage: 'technologies <url> [options]',
+    desc: 'Detect the tech stack behind the site',
+    flags: [],
+    browser: true,
+    examples: [['technologies https://example.com', 'detect the tech stack']]
+  },
+  lighthouse: {
+    usage: 'lighthouse <url> [options]',
+    desc: 'Run a Lighthouse report',
+    flags: [
+      ['--onlyCategories', 'Limit to these categories'],
+      ['--onlyAudits', 'Limit to these audits'],
+      ['--skipAudits', 'Skip these audits'],
+      ['--output', 'Report format: json, html, csv']
+    ],
+    browser: true,
+    examples: [['lighthouse https://example.com', 'run a Lighthouse report']]
+  },
+  search: {
+    usage: 'search <query> [options]',
+    desc: 'Google as structured data (query instead of url)',
+    flags: [
+      [
+        '--type',
+        'news, images, videos, places, maps, shopping, scholar, patents, autocomplete'
+      ],
+      ['--limit', 'Maximum number of results'],
+      ['--location', 'Country or locale (e.g. es)'],
+      ['--period', 'Recency: hour, day, week, month, year'],
+      ['--timeout', 'Request timeout']
+    ],
+    cli: CLI_NO_TRACE,
+    examples: [
+      [
+        'search "best coffee" --limit 10 --location es',
+        'Google results in Spain, limit 10'
+      ],
+      [
+        'search "open source llm" --type news --period week',
+        'news results from the past week'
+      ]
+    ]
+  },
+  function: {
+    usage: 'function <url> --file <path> [options]',
+    desc: 'Run code remotely with browser access',
+    flags: [['--file', 'Path to the code file']],
+    browser: true,
+    cli: CLI_NO_TRACE,
+    note: 'Extra flags are injected as variables in the function scope.',
+    examples: [
+      [
+        'function https://example.com --file ./fn.js',
+        'run ./fn.js with browser access'
+      ]
+    ]
+  }
+}
+
+const productList = Object.entries(PRODUCTS)
+  .map(([name, product]) => col(name, product.desc))
+  .join('\n')
+
+const global = `Usage
+${cmd('<url> [options]')}
+${cmd('<product> <url|query> [options]')}
+
+Products
+${productList}
+
+Options
+${rows(CLI)}
+
+Examples
+${cmd('https://example.com', 'unified metadata (default)')}
+${cmd(
+  'https://example.com --trace',
+  'print request & response, API key masked'
+)}
+${cmd(
+  'https://example.com --trace-full',
+  'same as --trace, including the full API key'
+)}
+${cmd('markdown https://example.com', 'page content as Markdown')}
+${cmd('screenshot https://example.com --fullPage', 'full-page screenshot')}
+${cmd('logo https://github.com --square', 'square brand logo')}
+${cmd('links https://example.com', 'every absolute link on the page')}
+${cmd(
+  'search "best coffee" --limit 10 --location es',
+  'Google results in Spain, limit 10'
+)}
+${cmd(
+  'search "open source llm" --type news --period week',
+  'news results from the past week'
+)}
+${cmd(
+  'extract https://microlink.io --data \'{"image":{"selector":"meta[property=og:image]","attr":"content","type":"image"}}\'',
+  'extract og:image via MQL'
+)}
+${cmd(
+  'function https://example.com --file ./fn.js',
+  'run ./fn.js with browser access'
+)}
+`
+
+const render = (name, product) => {
+  const usage = []
+    .concat(product.usage)
+    .map(line => cmd(line))
+    .join('\n')
+  const cli = product.cli ?? CLI
+  const options = [...product.flags, ...cli]
+  const parts = [
+    'Usage',
+    usage,
+    '',
+    gray(product.desc),
+    '',
+    'Options',
+    rows(options)
+  ]
+  if (product.browser) parts.push('', 'Browser', rows(BROWSER))
+  if (product.note) parts.push('', gray(product.note))
+  if (product.examples) {
+    parts.push(
+      '',
+      'Examples',
+      ...product.examples.map(([rest, comment]) => cmd(rest, comment))
+    )
+  }
+  return parts.join('\n') + '\n'
+}
+
+module.exports = command => {
+  const name = ALIAS[command] ?? command
+  return PRODUCTS[name] ? render(name, PRODUCTS[name]) : global
+}

package/bin/index.js

@@ -3,16 +3,223 @@
 
 const { readFileSync } = require('fs')
 const path = require('path')
-const jsome = require('jsome')
 const mri = require('mri')
+const helpText = require('./help')
+const { gray, white, green, red, styleText } = require('./style')
 
 const create = require('../src')
 
-const showHelp = () => {
-  console.log(readFileSync(path.join(__dirname, 'help.txt'), 'utf8'))
+const SHOW_CURSOR = '\u001b[?25h'
+const HIDE_CURSOR = '\u001b[?25l'
+const CLEAR_LINE = '\r\u001b[K'
+const FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
+
+const label = (text, color) =>
+  styleText(['inverse', 'bold', color], ` ${text.toUpperCase()} `)
+const keyValue = (key, value) => key + ' ' + gray(value)
+
+const prettyMs = ms => {
+  if (!Number.isFinite(ms)) return 'unknown'
+  const sign = ms < 0 ? '-' : ''
+  let n = Math.abs(ms)
+  if (n < 1000) return `${sign}${Math.round(n)}ms`
+  n /= 1000
+  if (n < 60) return `${sign}${n.toFixed(1).replace(/\.0$/, '')}s`
+  const hours = Math.floor(n / 3600)
+  n %= 3600
+  const mins = Math.floor(n / 60)
+  const secs = (n % 60).toFixed(1).replace(/\.0$/, '')
+  if (hours) return `${sign}${hours}h ${mins}m ${secs}s`
+  return secs === '0' ? `${sign}${mins}m` : `${sign}${mins}m ${secs}s`
+}
+
+const prettyBytes = n => {
+  if (!Number.isFinite(n) || n < 1000) return `${Math.round(n || 0)} B`
+  if (n < 1e6) {
+    const val = n / 1000
+    return `${
+      val >= 100 ? Math.round(val) : val.toFixed(1).replace(/\.0$/, '')
+    } kB`
+  }
+  return `${(n / 1e6).toFixed(1).replace(/\.0$/, '')} MB`
+}
+
+const toPlainHeaders = headers => {
+  if (!headers) return {}
+  if (typeof headers.entries === 'function') {
+    return Object.fromEntries(headers.entries())
+  }
+  return headers
+}
+
+const humanizeApiKey = apiKey => `${String(apiKey).slice(0, 5)}…`
+
+const quote = str =>
+  gray('"') + white(JSON.stringify(str).slice(1, -1)) + gray('"')
+
+const printPretty = (value, indent = 0) => {
+  if (value === null) return white('null')
+  if (typeof value === 'string') return quote(value)
+  if (typeof value !== 'object') return white(String(value))
+
+  const isArray = Array.isArray(value)
+  const keys = isArray ? value : Object.keys(value)
+  if (keys.length === 0) return gray(isArray ? '[]' : '{}')
+
+  const pad = '  '.repeat(indent)
+  const inner = '  '.repeat(indent + 1)
+  const open = gray(isArray ? '[' : '{')
+  const close = gray(isArray ? ']' : '}')
+  const lines = keys.map(key => {
+    if (isArray) return inner + printPretty(key, indent + 1)
+    const name = /^[A-Za-z_$][\w$]*$/.test(key) ? white(key) : quote(key)
+    return inner + name + gray(':') + ' ' + printPretty(value[key], indent + 1)
+  })
+  return open + '\n' + lines.join(gray(',') + '\n') + '\n' + pad + close
+}
+
+const printJson = payload => {
+  console.log(
+    process.stdout.hasColors?.()
+      ? printPretty(payload)
+      : JSON.stringify(payload, null, 2)
+  )
+}
+
+const tracePayload = ({
+  requestUrl,
+  requestOptions = {},
+  response,
+  full = false
+}) => {
+  const rest = { ...requestOptions }
+  delete rest.responseType
+  const headers = { ...rest.headers }
+  if (!full) {
+    for (const key of ['x-api-key', 'authorization', 'cookie']) {
+      if (headers[key]) headers[key] = humanizeApiKey(headers[key])
+    }
+  }
+  return {
+    request: { url: requestUrl, ...rest, headers },
+    response: {
+      ...response,
+      headers: toPlainHeaders(response?.headers)
+    }
+  }
+}
+
+const shouldSpin = () =>
+  !process.env.NO_COLOR &&
+  process.env.FORCE_COLOR !== '0' &&
+  Boolean(process.stdout?.hasColors?.())
+
+const spinner = () => {
+  const now = Date.now()
+  let i = 0
+  let timer
+  const draw = () => {
+    process.stderr.write(
+      `${CLEAR_LINE}${FRAMES[i++ % FRAMES.length]} ${prettyMs(
+        Date.now() - now
+      )}`
+    )
+  }
+  return {
+    start () {
+      process.stderr.write(HIDE_CURSOR)
+      draw()
+      process.on('SIGINT', () => {
+        process.stderr.write(CLEAR_LINE + SHOW_CURSOR)
+        process.exit(130)
+      })
+      timer = setInterval(draw, 50)
+    },
+    stop () {
+      clearInterval(timer)
+      process.stderr.write(CLEAR_LINE + SHOW_CURSOR)
+    }
+  }
+}
+
+const printFooter = ({ duration, response }) => {
+  const headers = toPlainHeaders(response?.headers)
+  const time = prettyMs(duration)
+  const size = Number(headers['content-length']) || 0
+  const serverTiming = headers['server-timing']
+  const id = headers['x-request-id']
+  const edgeCacheStatus = headers['cf-cache-status']
+  const unifiedCacheStatus = headers['x-cache-status']
+  const cacheStatus =
+    unifiedCacheStatus === 'MISS' && edgeCacheStatus === 'HIT'
+      ? edgeCacheStatus
+      : unifiedCacheStatus
+  const timestamp = Number(headers['x-timestamp'])
+  const ttl = Number(headers['x-cache-ttl'])
+  const expires = timestamp + ttl - Date.now()
+  const expiredAt =
+    cacheStatus === 'HIT' && Number.isFinite(expires)
+      ? `(${prettyMs(expires)})`
+      : ''
+  const fetchMode = headers['x-fetch-mode']
+  const fetchTime = fetchMode && `(${headers['x-fetch-time']})`
+  const uri = response?.url
+
+  if (process.stdout.isTTY) console.error()
+  console.error(
+    label('success', 'green'),
+    gray(`${prettyBytes(size)} in ${time}`)
+  )
+  console.error()
+
+  if (serverTiming) {
+    console.error('  ', keyValue(green('timing'), serverTiming))
+  }
+  if (cacheStatus) {
+    console.error(
+      '   ',
+      keyValue(green('cache'), `${cacheStatus} ${gray(expiredAt)}`.trim())
+    )
+  }
+  if (fetchMode) {
+    console.error(
+      '    ',
+      keyValue(green('mode'), `${fetchMode} ${gray(fetchTime)}`.trim())
+    )
+  }
+  if (uri) console.error('     ', keyValue(green('uri'), uri))
+  if (id) console.error('      ', keyValue(green('id'), id))
+}
+
+const printFail = error => {
+  if (process.stdout.isTTY) console.error()
+  console.error(
+    label(error.status || 'fail', 'red'),
+    gray(String(error.message).replace(`${error.code}, `, ''))
+  )
+  console.error()
+  const id = error.headers?.['x-request-id']
+  if (id) console.error('    ', keyValue(red('id'), id))
+  if (error.url) console.error('   ', keyValue(red('uri'), error.url))
+  if (error.code) {
+    console.error(
+      '  ',
+      keyValue(
+        red('code'),
+        `${error.code}${error.statusCode ? ` (${error.statusCode})` : ''}`
+      )
+    )
+  }
+  if (error.more) console.error('  ', keyValue(red('more'), error.more))
+}
+
+const showHelp = command => {
+  console.log(helpText(command).trimEnd())
   process.exit(0)
 }
 
+const HTTP_HEADER = 'http.header.'
+
 const parseHeaders = input => {
   const headers = {}
   for (const item of [].concat(input ?? [])) {
@@ -25,12 +232,27 @@ const parseHeaders = input => {
   return headers
 }
 
+const takeHttpHeaders = flags => {
+  const headers = {}
+  for (const key of Object.keys(flags)) {
+    if (!key.startsWith(HTTP_HEADER)) continue
+    let value = flags[key]
+    delete flags[key]
+    if (Array.isArray(value)) value = value.at(-1)
+    if (typeof value !== 'string' && typeof value !== 'number') continue
+    const name = key.slice(HTTP_HEADER.length).toLowerCase()
+    if (name) headers[name] = String(value)
+  }
+  return headers
+}
+
 const argv = mri(process.argv.slice(2), {
   alias: { H: 'header' },
-  string: ['header', 'api-key', 'data', 'file']
+  boolean: ['trace', 'trace-full', 'help'],
+  string: ['header', 'api-key', 'data', 'file', 'endpoint']
 })
 
-const {
+let {
   _: [command, target],
   header,
   help,
@@ -38,23 +260,49 @@ const {
   file,
   'api-key': apiKeyFlag,
   apiKey: apiKeyCamel,
+  endpoint: endpointFlag,
+  trace,
+  'trace-full': traceFull,
   ...flags
 } = argv
 
-if (help || !command) showHelp()
+const isTrace = trace || traceFull
+
+if (!command) showHelp()
 
 const apiKey = apiKeyFlag || apiKeyCamel || process.env.MICROLINK_API_KEY
-const client = create(apiKey ? { apiKey } : {})
+const endpoint = endpointFlag
+const client = create({
+  ...(apiKey && { apiKey }),
+  ...(endpoint && { endpoint })
+})
 
 if (typeof client[command] !== 'function') {
-  console.error(
-    `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.`
-  )
+  if (!target && URL.canParse(command)) {
+    target = command
+    command = 'metadata'
+  } else if (help) {
+    showHelp()
+  } else {
+    console.error(
+      `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.`
+    )
+    process.exit(1)
+  }
+}
+
+if (help || !target) showHelp(command)
+
+if (
+  isTrace &&
+  (command === 'search' || command === 'function' || command === 'run')
+) {
+  console.error(`\`--trace\` is not supported for \`${command}\`.`)
   process.exit(1)
 }
 
 const options = { ...flags }
-const headers = parseHeaders(header)
+const headers = { ...takeHttpHeaders(options), ...parseHeaders(header) }
 if (Object.keys(headers).length > 0) options.headers = headers
 
 const invoke = () => {
@@ -68,14 +316,23 @@ const invoke = () => {
   return client[command](target, options)
 }
 
-Promise.resolve()
-  .then(invoke)
-  .then(result => {
-    if (typeof result === 'string') console.log(result)
-    else jsome(result)
+const spin = !isTrace && shouldSpin() ? spinner() : null
+
+;(async () => {
+  spin?.start()
+  const started = Date.now()
+  try {
+    const result = await invoke()
+    const duration = Date.now() - started
+    spin?.stop()
+    if (isTrace) printJson(tracePayload({ ...client.last, full: traceFull }))
+    else if (typeof result === 'string') console.log(result)
+    else printJson({ status: 'success', data: result })
+    if (!isTrace) printFooter({ duration, response: client.last.response })
     process.exit(0)
-  })
-  .catch(error => {
-    console.error(error.message)
+  } catch (error) {
+    spin?.stop()
+    printFail(error)
     process.exit(1)
-  })
+  }
+})()

package/bin/style.js

@@ -0,0 +1,10 @@
+'use strict'
+
+const { styleText } = require('node:util')
+
+const gray = str => styleText('gray', str)
+const white = str => styleText('white', str)
+const green = str => styleText('green', str)
+const red = str => styleText('red', str)
+
+module.exports = { gray, white, green, red, styleText }

package/package.json

@@ -2,7 +2,7 @@
   "name": "microlink.io",
   "description": "The Microlink API organized into products, each returning a direct result",
   "homepage": "https://github.com/microlinkhq/microlink",
-  "version": "0.0.5",
+  "version": "0.2.0",
   "types": "./src/index.d.ts",
   "main": "./src/index.js",
   "exports": {
@@ -46,7 +46,6 @@
     "@microlink/function": "0.3.0",
     "@microlink/google": "1.0.6",
     "@microlink/mql": "0.18.0",
-    "jsome": "~2.5.0",
     "mri": "~1.2.0"
   },
   "devDependencies": {
@@ -77,5 +76,5 @@
     },
     "directory": "test"
   },
-  "gitHead": "f51f9edb297ebe9d8ac6cdb3208547420cbea6ad"
+  "gitHead": "3766602b11599a8284e4aa31815d409e0ffa0788"
 }

package/src/index.js

@@ -41,6 +41,17 @@ const LIGHTHOUSE_KEYS = ['onlyCategories', 'onlyAudits', 'skipAudits', 'output']
 const isEmpty = obj => Object.keys(obj).length === 0
 
 const create = (ctx = {}) => {
+  const last = {}
+  const request = (...args) => {
+    const [requestUrl, requestOptions] = mql.getApiUrl(...args)
+    last.requestUrl = requestUrl
+    last.requestOptions = requestOptions
+    return mql(...args).then(result => {
+      last.response = result.response
+      return result
+    })
+  }
+
   /**
    * Split the single options bag into the three destinations mql has:
    * `got.headers` (HTTP layer, 3rd arg), `sub` (capability nested keys)
@@ -61,7 +72,7 @@ const create = (ctx = {}) => {
 
   const content = field => (url, options) => {
     const { top, sub, got } = route(options, CONTENT_KEYS)
-    return mql(
+    return request(
       url,
       { ...top, meta: false, data: { [field]: { attr: field, ...sub } } },
       got
@@ -70,7 +81,7 @@ const create = (ctx = {}) => {
 
   const collection = (field, rule) => (url, options) => {
     const { top, sub, got } = route(options, COLLECTION_KEYS)
-    return mql(
+    return request(
       url,
       { ...top, meta: false, data: { [field]: { ...rule, ...sub } } },
       got
@@ -79,7 +90,7 @@ const create = (ctx = {}) => {
 
   const capability = (field, nested) => (url, options) => {
     const { top, sub, got } = route(options, nested)
-    return mql(
+    return request(
       url,
       { ...top, meta: false, [field]: isEmpty(sub) ? true : sub },
       got
@@ -89,7 +100,7 @@ const create = (ctx = {}) => {
   /* Primary media detection (`data.video` / `data.audio`). */
   const media = field => (url, options) => {
     const { top, got } = route(options)
-    return mql(url, { ...top, meta: false, [field]: true }, got).then(
+    return request(url, { ...top, meta: false, [field]: true }, got).then(
       ({ data }) => data[field]
     )
   }
@@ -101,14 +112,14 @@ const create = (ctx = {}) => {
     return fn(code, top, got)(url)
   }
 
-  return {
+  const client = {
     metadata: (url, options) => {
       const { top, got } = route(options)
-      return mql(url, top, got).then(({ data }) => data)
+      return request(url, top, got).then(({ data }) => data)
     },
     logo: (url, options) => {
       const { top, sub, got } = route(options, LOGO_KEYS)
-      return mql(
+      return request(
         url,
         { ...top, meta: isEmpty(sub) ? true : { logo: sub } },
         got
@@ -142,7 +153,7 @@ const create = (ctx = {}) => {
     }),
     extract: (url, rules, options) => {
       const { top, got } = route(options)
-      return mql(url, { ...top, meta: false, data: rules }, got).then(
+      return request(url, { ...top, meta: false, data: rules }, got).then(
         ({ data }) => data
       )
     },
@@ -150,7 +161,7 @@ const create = (ctx = {}) => {
     pdf: capability('pdf', PDF_KEYS),
     embed: (url, options) => {
       const { top, sub, got } = route(options, EMBED_KEYS)
-      return mql(
+      return request(
         url,
         { ...top, meta: false, iframe: isEmpty(sub) ? true : sub },
         got
@@ -158,7 +169,7 @@ const create = (ctx = {}) => {
     },
     technologies: (url, options) => {
       const { top, got } = route(options)
-      return mql(
+      return request(
         url,
         {
           ...top,
@@ -170,7 +181,7 @@ const create = (ctx = {}) => {
     },
     lighthouse: (url, options) => {
       const { top, sub, got } = route(options, LIGHTHOUSE_KEYS)
-      return mql(
+      return request(
         url,
         {
           ...top,
@@ -190,6 +201,9 @@ const create = (ctx = {}) => {
     function: run,
     run
   }
+
+  Object.defineProperty(client, 'last', { value: last })
+  return client
 }
 
 module.exports = create

package/bin/help.txt

@@ -1,47 +0,0 @@
-Usage
-  $ microlink <product> <url|query> [options]
-
-Products
-  metadata      Unified metadata (title, description, image, ...)
-  logo          Brand logo of the site (--square prefers the square variant)
-  markdown      Page content as Markdown
-  html          Page content as HTML
-  text          Page content as plain text
-  video         Primary video of the page (returns the asset object)
-  audio         Primary audio of the page (returns the asset object)
-  emails        Every email address present on the page
-  links         Every absolute link URL on the page
-  images        Every absolute image URL on the page
-  videos        Every absolute video URL on the page
-  audios        Every absolute audio URL on the page
-  extract       Custom MQL data rules (--data '{"field":{...}}')
-  screenshot    Take a screenshot (returns the asset object)
-  pdf           Generate a PDF (returns the asset object)
-  embed         oEmbed-style embeddable iframe ({ html, scripts })
-  technologies  Detect the tech stack behind the site
-  lighthouse    Run a Lighthouse report
-  search        Google as structured data (query instead of url); --type
-                routes to news, images, videos, places, maps, shopping,
-                scholar, patents or autocomplete
-  function      Run code remotely with browser access (--file ./fn.js);
-                extra flags are injected as variables in the function scope
-
-Options
-  --api-key     Microlink API key (defaults to MICROLINK_API_KEY env)
-  --header, -H  Extra request header as 'Name: value' (repeatable)
-  --data        JSON data rules for the extract command
-  --file        Path to the code file for the function command
-  --help        Show this help
-
-Any other flag is passed as an option to the product, e.g. --fullPage,
---device 'iPhone 11', --waitUntil networkidle0, --selector article.
-
-Examples
-  $ microlink markdown https://example.com
-  $ microlink screenshot https://example.com --fullPage
-  $ microlink logo https://github.com --square
-  $ microlink links https://example.com
-  $ microlink search "best coffee" --limit 10 --location es
-  $ microlink search "open source llm" --type news --period week
-  $ microlink extract https://microlink.io --data '{"image":{"selector":"meta[property=og:image]","attr":"content","type":"image"}}'
-  $ microlink function https://example.com --file ./fn.js