microlink.io 0.0.5 → 0.1.1

package/bin/help.txt

@@ -1,8 +1,9 @@
 Usage
+  $ microlink <url> [options]
   $ microlink <product> <url|query> [options]
 
 Products
-  metadata      Unified metadata (title, description, image, ...)
+  metadata      Unified metadata (title, description, image, ...); default
   logo          Brand logo of the site (--square prefers the square variant)
   markdown      Page content as Markdown
   html          Page content as HTML
@@ -31,12 +32,17 @@ Options
   --header, -H  Extra request header as 'Name: value' (repeatable)
   --data        JSON data rules for the extract command
   --file        Path to the code file for the function command
+  --trace       Print request & response payload (API key masked)
+  --trace-full  Same as --trace, including the full API key
   --help        Show this help
 
 Any other flag is passed as an option to the product, e.g. --fullPage,
 --device 'iPhone 11', --waitUntil networkidle0, --selector article.
 
 Examples
+  $ microlink https://example.com
+  $ microlink https://example.com --trace
+  $ microlink https://example.com --trace-full
   $ microlink markdown https://example.com
   $ microlink screenshot https://example.com --fullPage
   $ microlink logo https://github.com --square

package/bin/index.js

@@ -1,13 +1,219 @@
 #!/usr/bin/env node
 'use strict'
 
+const { styleText } = require('node:util')
 const { readFileSync } = require('fs')
 const path = require('path')
-const jsome = require('jsome')
 const mri = require('mri')
 
 const create = require('../src')
 
+const SHOW_CURSOR = '\u001b[?25h'
+const HIDE_CURSOR = '\u001b[?25l'
+const CLEAR_LINE = '\r\u001b[K'
+const FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
+
+const gray = str => styleText('gray', str)
+const white = str => styleText('white', str)
+const green = str => styleText('green', str)
+const red = str => styleText('red', str)
+const label = (text, color) =>
+  styleText(['inverse', 'bold', color], ` ${text.toUpperCase()} `)
+const keyValue = (key, value) => key + ' ' + gray(value)
+
+const prettyMs = ms => {
+  if (!Number.isFinite(ms)) return 'unknown'
+  const sign = ms < 0 ? '-' : ''
+  let n = Math.abs(ms)
+  if (n < 1000) return `${sign}${Math.round(n)}ms`
+  n /= 1000
+  if (n < 60) return `${sign}${n.toFixed(1).replace(/\.0$/, '')}s`
+  const hours = Math.floor(n / 3600)
+  n %= 3600
+  const mins = Math.floor(n / 60)
+  const secs = (n % 60).toFixed(1).replace(/\.0$/, '')
+  if (hours) return `${sign}${hours}h ${mins}m ${secs}s`
+  return secs === '0' ? `${sign}${mins}m` : `${sign}${mins}m ${secs}s`
+}
+
+const prettyBytes = n => {
+  if (!Number.isFinite(n) || n < 1000) return `${Math.round(n || 0)} B`
+  if (n < 1e6) {
+    const val = n / 1000
+    return `${
+      val >= 100 ? Math.round(val) : val.toFixed(1).replace(/\.0$/, '')
+    } kB`
+  }
+  return `${(n / 1e6).toFixed(1).replace(/\.0$/, '')} MB`
+}
+
+const toPlainHeaders = headers => {
+  if (!headers) return {}
+  if (typeof headers.entries === 'function') {
+    return Object.fromEntries(headers.entries())
+  }
+  return headers
+}
+
+const humanizeApiKey = apiKey => `${String(apiKey).slice(0, 5)}…`
+
+const quote = str =>
+  gray('"') + white(JSON.stringify(str).slice(1, -1)) + gray('"')
+
+const printPretty = (value, indent = 0) => {
+  if (value === null) return white('null')
+  if (typeof value === 'string') return quote(value)
+  if (typeof value !== 'object') return white(String(value))
+
+  const isArray = Array.isArray(value)
+  const keys = isArray ? value : Object.keys(value)
+  if (keys.length === 0) return gray(isArray ? '[]' : '{}')
+
+  const pad = '  '.repeat(indent)
+  const inner = '  '.repeat(indent + 1)
+  const open = gray(isArray ? '[' : '{')
+  const close = gray(isArray ? ']' : '}')
+  const lines = keys.map(key => {
+    if (isArray) return inner + printPretty(key, indent + 1)
+    const name = /^[A-Za-z_$][\w$]*$/.test(key) ? white(key) : quote(key)
+    return inner + name + gray(':') + ' ' + printPretty(value[key], indent + 1)
+  })
+  return open + '\n' + lines.join(gray(',') + '\n') + '\n' + pad + close
+}
+
+const printJson = payload => {
+  console.log(
+    process.stdout.hasColors?.()
+      ? printPretty(payload)
+      : JSON.stringify(payload, null, 2)
+  )
+}
+
+const tracePayload = ({
+  requestUrl,
+  requestOptions = {},
+  response,
+  full = false
+}) => {
+  const rest = { ...requestOptions }
+  delete rest.responseType
+  const headers = { ...rest.headers }
+  if (!full && headers['x-api-key']) {
+    headers['x-api-key'] = humanizeApiKey(headers['x-api-key'])
+  }
+  return {
+    request: { url: requestUrl, ...rest, headers },
+    response: {
+      ...response,
+      headers: toPlainHeaders(response?.headers)
+    }
+  }
+}
+
+const shouldSpin = () =>
+  !process.env.NO_COLOR &&
+  process.env.FORCE_COLOR !== '0' &&
+  Boolean(process.stdout?.hasColors?.())
+
+const spinner = () => {
+  const now = Date.now()
+  let i = 0
+  let timer
+  const draw = () => {
+    process.stderr.write(
+      `${CLEAR_LINE}${FRAMES[i++ % FRAMES.length]} ${prettyMs(
+        Date.now() - now
+      )}`
+    )
+  }
+  return {
+    start () {
+      process.stderr.write(HIDE_CURSOR)
+      draw()
+      process.on('SIGINT', () => {
+        process.stderr.write(CLEAR_LINE + SHOW_CURSOR)
+        process.exit(130)
+      })
+      timer = setInterval(draw, 50)
+    },
+    stop () {
+      clearInterval(timer)
+      process.stderr.write(CLEAR_LINE + SHOW_CURSOR)
+    }
+  }
+}
+
+const printFooter = ({ duration, response }) => {
+  const headers = toPlainHeaders(response?.headers)
+  const time = prettyMs(duration)
+  const size = Number(headers['content-length']) || 0
+  const serverTiming = headers['server-timing']
+  const id = headers['x-request-id']
+  const edgeCacheStatus = headers['cf-cache-status']
+  const unifiedCacheStatus = headers['x-cache-status']
+  const cacheStatus =
+    unifiedCacheStatus === 'MISS' && edgeCacheStatus === 'HIT'
+      ? edgeCacheStatus
+      : unifiedCacheStatus
+  const timestamp = Number(headers['x-timestamp'])
+  const ttl = Number(headers['x-cache-ttl'])
+  const expires = timestamp + ttl - Date.now()
+  const expiredAt =
+    cacheStatus === 'HIT' && Number.isFinite(expires)
+      ? `(${prettyMs(expires)})`
+      : ''
+  const fetchMode = headers['x-fetch-mode']
+  const fetchTime = fetchMode && `(${headers['x-fetch-time']})`
+  const uri = response?.url
+
+  if (process.stdout.isTTY) console.error()
+  console.error(
+    label('success', 'green'),
+    gray(`${prettyBytes(size)} in ${time}`)
+  )
+  console.error()
+
+  if (serverTiming) {
+    console.error('  ', keyValue(green('timing'), serverTiming))
+  }
+  if (cacheStatus) {
+    console.error(
+      '   ',
+      keyValue(green('cache'), `${cacheStatus} ${gray(expiredAt)}`.trim())
+    )
+  }
+  if (fetchMode) {
+    console.error(
+      '    ',
+      keyValue(green('mode'), `${fetchMode} ${gray(fetchTime)}`.trim())
+    )
+  }
+  if (uri) console.error('     ', keyValue(green('uri'), uri))
+  if (id) console.error('      ', keyValue(green('id'), id))
+}
+
+const printFail = error => {
+  if (process.stdout.isTTY) console.error()
+  console.error(
+    label(error.status || 'fail', 'red'),
+    gray(String(error.message).replace(`${error.code}, `, ''))
+  )
+  console.error()
+  const id = error.headers?.['x-request-id']
+  if (id) console.error('    ', keyValue(red('id'), id))
+  if (error.url) console.error('   ', keyValue(red('uri'), error.url))
+  if (error.code) {
+    console.error(
+      '  ',
+      keyValue(
+        red('code'),
+        `${error.code}${error.statusCode ? ` (${error.statusCode})` : ''}`
+      )
+    )
+  }
+  if (error.more) console.error('  ', keyValue(red('more'), error.more))
+}
+
 const showHelp = () => {
   console.log(readFileSync(path.join(__dirname, 'help.txt'), 'utf8'))
   process.exit(0)
@@ -27,10 +233,11 @@ const parseHeaders = input => {
 
 const argv = mri(process.argv.slice(2), {
   alias: { H: 'header' },
+  boolean: ['trace', 'trace-full'],
   string: ['header', 'api-key', 'data', 'file']
 })
 
-const {
+let {
   _: [command, target],
   header,
   help,
@@ -38,18 +245,35 @@ const {
   file,
   'api-key': apiKeyFlag,
   apiKey: apiKeyCamel,
+  trace,
+  'trace-full': traceFull,
   ...flags
 } = argv
 
+const isTrace = trace || traceFull
+
 if (help || !command) showHelp()
 
 const apiKey = apiKeyFlag || apiKeyCamel || process.env.MICROLINK_API_KEY
 const client = create(apiKey ? { apiKey } : {})
 
 if (typeof client[command] !== 'function') {
-  console.error(
-    `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.`
-  )
+  if (!target && URL.canParse(command)) {
+    target = command
+    command = 'metadata'
+  } else {
+    console.error(
+      `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.`
+    )
+    process.exit(1)
+  }
+}
+
+if (
+  isTrace &&
+  (command === 'search' || command === 'function' || command === 'run')
+) {
+  console.error(`\`--trace\` is not supported for \`${command}\`.`)
   process.exit(1)
 }
 
@@ -68,14 +292,23 @@ const invoke = () => {
   return client[command](target, options)
 }
 
-Promise.resolve()
-  .then(invoke)
-  .then(result => {
-    if (typeof result === 'string') console.log(result)
-    else jsome(result)
+const spin = !isTrace && shouldSpin() ? spinner() : null
+
+;(async () => {
+  spin?.start()
+  const started = Date.now()
+  try {
+    const result = await invoke()
+    const duration = Date.now() - started
+    spin?.stop()
+    if (isTrace) printJson(tracePayload({ ...client.last, full: traceFull }))
+    else if (typeof result === 'string') console.log(result)
+    else printJson({ status: 'success', data: result })
+    if (!isTrace) printFooter({ duration, response: client.last.response })
     process.exit(0)
-  })
-  .catch(error => {
-    console.error(error.message)
+  } catch (error) {
+    spin?.stop()
+    printFail(error)
     process.exit(1)
-  })
+  }
+})()

package/package.json

@@ -2,7 +2,7 @@
   "name": "microlink.io",
   "description": "The Microlink API organized into products, each returning a direct result",
   "homepage": "https://github.com/microlinkhq/microlink",
-  "version": "0.0.5",
+  "version": "0.1.1",
   "types": "./src/index.d.ts",
   "main": "./src/index.js",
   "exports": {
@@ -46,7 +46,6 @@
     "@microlink/function": "0.3.0",
     "@microlink/google": "1.0.6",
     "@microlink/mql": "0.18.0",
-    "jsome": "~2.5.0",
     "mri": "~1.2.0"
   },
   "devDependencies": {
@@ -77,5 +76,5 @@
     },
     "directory": "test"
   },
-  "gitHead": "f51f9edb297ebe9d8ac6cdb3208547420cbea6ad"
+  "gitHead": "a12fcd2a7bc918e741a71b50fd2cd93876f4ca8f"
 }

package/src/index.js

@@ -41,6 +41,17 @@ const LIGHTHOUSE_KEYS = ['onlyCategories', 'onlyAudits', 'skipAudits', 'output']
 const isEmpty = obj => Object.keys(obj).length === 0
 
 const create = (ctx = {}) => {
+  const last = {}
+  const request = (...args) => {
+    const [requestUrl, requestOptions] = mql.getApiUrl(...args)
+    last.requestUrl = requestUrl
+    last.requestOptions = requestOptions
+    return mql(...args).then(result => {
+      last.response = result.response
+      return result
+    })
+  }
+
   /**
    * Split the single options bag into the three destinations mql has:
    * `got.headers` (HTTP layer, 3rd arg), `sub` (capability nested keys)
@@ -61,7 +72,7 @@ const create = (ctx = {}) => {
 
   const content = field => (url, options) => {
     const { top, sub, got } = route(options, CONTENT_KEYS)
-    return mql(
+    return request(
       url,
       { ...top, meta: false, data: { [field]: { attr: field, ...sub } } },
       got
@@ -70,7 +81,7 @@ const create = (ctx = {}) => {
 
   const collection = (field, rule) => (url, options) => {
     const { top, sub, got } = route(options, COLLECTION_KEYS)
-    return mql(
+    return request(
       url,
       { ...top, meta: false, data: { [field]: { ...rule, ...sub } } },
       got
@@ -79,7 +90,7 @@ const create = (ctx = {}) => {
 
   const capability = (field, nested) => (url, options) => {
     const { top, sub, got } = route(options, nested)
-    return mql(
+    return request(
       url,
       { ...top, meta: false, [field]: isEmpty(sub) ? true : sub },
       got
@@ -89,7 +100,7 @@ const create = (ctx = {}) => {
   /* Primary media detection (`data.video` / `data.audio`). */
   const media = field => (url, options) => {
     const { top, got } = route(options)
-    return mql(url, { ...top, meta: false, [field]: true }, got).then(
+    return request(url, { ...top, meta: false, [field]: true }, got).then(
       ({ data }) => data[field]
     )
   }
@@ -101,14 +112,14 @@ const create = (ctx = {}) => {
     return fn(code, top, got)(url)
   }
 
-  return {
+  const client = {
     metadata: (url, options) => {
       const { top, got } = route(options)
-      return mql(url, top, got).then(({ data }) => data)
+      return request(url, top, got).then(({ data }) => data)
     },
     logo: (url, options) => {
       const { top, sub, got } = route(options, LOGO_KEYS)
-      return mql(
+      return request(
         url,
         { ...top, meta: isEmpty(sub) ? true : { logo: sub } },
         got
@@ -142,7 +153,7 @@ const create = (ctx = {}) => {
     }),
     extract: (url, rules, options) => {
       const { top, got } = route(options)
-      return mql(url, { ...top, meta: false, data: rules }, got).then(
+      return request(url, { ...top, meta: false, data: rules }, got).then(
         ({ data }) => data
       )
     },
@@ -150,7 +161,7 @@ const create = (ctx = {}) => {
     pdf: capability('pdf', PDF_KEYS),
     embed: (url, options) => {
       const { top, sub, got } = route(options, EMBED_KEYS)
-      return mql(
+      return request(
         url,
         { ...top, meta: false, iframe: isEmpty(sub) ? true : sub },
         got
@@ -158,7 +169,7 @@ const create = (ctx = {}) => {
     },
     technologies: (url, options) => {
       const { top, got } = route(options)
-      return mql(
+      return request(
         url,
         {
           ...top,
@@ -170,7 +181,7 @@ const create = (ctx = {}) => {
     },
     lighthouse: (url, options) => {
       const { top, sub, got } = route(options, LIGHTHOUSE_KEYS)
-      return mql(
+      return request(
         url,
         {
           ...top,
@@ -190,6 +201,9 @@ const create = (ctx = {}) => {
     function: run,
     run
   }
+
+  Object.defineProperty(client, 'last', { value: last })
+  return client
 }
 
 module.exports = create