microlink.io 0.2.0 → 0.5.0

package/bin/config.js

@@ -0,0 +1,61 @@
+'use strict'
+
+const { homedir } = require('os')
+const { join } = require('path')
+const {
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+  unlinkSync,
+  chmodSync
+} = require('fs')
+
+const configDir = () =>
+  join(process.env.XDG_CONFIG_HOME || join(homedir(), '.config'), 'microlink')
+
+const configPath = () => join(configDir(), 'config.json')
+
+const configPathDisplay = () => {
+  const file = configPath()
+  const home = homedir()
+  return file.startsWith(home + '/') ? `~${file.slice(home.length)}` : file
+}
+
+const readConfig = () => {
+  try {
+    return JSON.parse(readFileSync(configPath(), 'utf8'))
+  } catch {
+    return {}
+  }
+}
+
+const writeConfig = data => {
+  mkdirSync(configDir(), { recursive: true, mode: 0o700 })
+  writeFileSync(configPath(), JSON.stringify(data) + '\n', { mode: 0o600 })
+  chmodSync(configPath(), 0o600)
+}
+
+const clearConfig = () => {
+  try {
+    unlinkSync(configPath())
+    return true
+  } catch (error) {
+    if (error.code === 'ENOENT') return false
+    throw error
+  }
+}
+
+const readApiKey = () => {
+  const { apiKey } = readConfig()
+  return typeof apiKey === 'string' && apiKey ? apiKey : undefined
+}
+
+module.exports = {
+  configDir,
+  configPath,
+  configPathDisplay,
+  readConfig,
+  writeConfig,
+  clearConfig,
+  readApiKey
+}

package/bin/help.js

@@ -12,7 +12,10 @@ const cmd = (rest, comment) =>
 const rows = items => items.map(([name, desc]) => col(name, desc)).join('\n')
 
 const CLI = [
-  ['--api-key', 'Microlink API key (defaults to MICROLINK_API_KEY env)'],
+  [
+    '--api-key',
+    'Microlink API key (flag, MICROLINK_API_KEY, or `microlink login`)'
+  ],
   ['--endpoint', 'Microlink API endpoint'],
   ['--header, -H', "Extra request header as 'Name: value' (repeatable)"],
   [
@@ -71,6 +74,23 @@ const COLLECTION = [
 
 const ALIAS = { run: 'function' }
 
+const COMMANDS = {
+  login: {
+    usage: 'login',
+    desc: 'Save an API key from your Microlink account',
+    flags: [],
+    cli: [],
+    examples: [['login', 'open the dashboard and save an API key']]
+  },
+  logout: {
+    usage: 'logout',
+    desc: 'Remove the saved API key from this machine',
+    flags: [],
+    cli: [],
+    examples: [['logout', 'forget the saved API key']]
+  }
+}
+
 const content = (name, desc) => ({
   usage: `${name} <url> [options]`,
   desc,
@@ -222,6 +242,9 @@ const PRODUCTS = {
         'news, images, videos, places, maps, shopping, scholar, patents, autocomplete'
       ],
       ['--limit', 'Maximum number of results'],
+      ['--page', 'Results page (1 default)'],
+      ['--html', 'Fetch HTML for the results page and each result'],
+      ['--markdown', 'Fetch Markdown for the results page and each result'],
       ['--location', 'Country or locale (e.g. es)'],
       ['--period', 'Recency: hour, day, week, month, year'],
       ['--timeout', 'Request timeout']
@@ -235,7 +258,12 @@ const PRODUCTS = {
       [
         'search "open source llm" --type news --period week',
         'news results from the past week'
-      ]
+      ],
+      [
+        'search "the matrix" --markdown',
+        'include Markdown for the SERP and each result'
+      ],
+      ['search "the matrix" --page 2', 'second page of results']
     ]
   },
   function: {
@@ -258,9 +286,18 @@ const productList = Object.entries(PRODUCTS)
   .map(([name, product]) => col(name, product.desc))
   .join('\n')
 
+const commandList = Object.entries(COMMANDS)
+  .map(([name, command]) => col(name, command.desc))
+  .join('\n')
+
 const global = `Usage
 ${cmd('<url> [options]')}
 ${cmd('<product> <url|query> [options]')}
+${cmd('login')}
+${cmd('logout')}
+
+Commands
+${commandList}
 
 Products
 ${productList}
@@ -269,6 +306,7 @@ Options
 ${rows(CLI)}
 
 Examples
+${cmd('login', 'save an API key from your account')}
 ${cmd('https://example.com', 'unified metadata (default)')}
 ${cmd(
   'https://example.com --trace',
@@ -307,15 +345,8 @@ const render = (name, product) => {
     .join('\n')
   const cli = product.cli ?? CLI
   const options = [...product.flags, ...cli]
-  const parts = [
-    'Usage',
-    usage,
-    '',
-    gray(product.desc),
-    '',
-    'Options',
-    rows(options)
-  ]
+  const parts = ['Usage', usage, '', gray(product.desc)]
+  if (options.length > 0) parts.push('', 'Options', rows(options))
   if (product.browser) parts.push('', 'Browser', rows(BROWSER))
   if (product.note) parts.push('', gray(product.note))
   if (product.examples) {
@@ -330,5 +361,6 @@ const render = (name, product) => {
 
 module.exports = command => {
   const name = ALIAS[command] ?? command
+  if (COMMANDS[name]) return render(name, COMMANDS[name])
   return PRODUCTS[name] ? render(name, PRODUCTS[name]) : global
 }

package/bin/index.js

@@ -5,7 +5,9 @@ const { readFileSync } = require('fs')
 const path = require('path')
 const mri = require('mri')
 const helpText = require('./help')
-const { gray, white, green, red, styleText } = require('./style')
+const { readApiKey, clearConfig } = require('./config')
+const login = require('./login')
+const { gray, white, green, red, orange, link, styleText } = require('./style')
 
 const create = require('../src')
 
@@ -15,7 +17,7 @@ const CLEAR_LINE = '\r\u001b[K'
 const FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
 
 const label = (text, color) =>
-  styleText(['inverse', 'bold', color], ` ${text.toUpperCase()} `)
+  styleText(['inverse', 'bold'], color(` ${text.toUpperCase()} `))
 const keyValue = (key, value) => key + ' ' + gray(value)
 
 const prettyMs = ms => {
@@ -63,7 +65,9 @@ const printPretty = (value, indent = 0) => {
   if (typeof value !== 'object') return white(String(value))
 
   const isArray = Array.isArray(value)
-  const keys = isArray ? value : Object.keys(value)
+  const keys = isArray
+    ? value.filter(item => typeof item !== 'function')
+    : Object.keys(value).filter(key => typeof value[key] !== 'function')
   if (keys.length === 0) return gray(isArray ? '[]' : '{}')
 
   const pad = '  '.repeat(indent)
@@ -167,7 +171,7 @@ const printFooter = ({ duration, response }) => {
 
   if (process.stdout.isTTY) console.error()
   console.error(
-    label('success', 'green'),
+    label('success', green),
     gray(`${prettyBytes(size)} in ${time}`)
   )
   console.error()
@@ -187,30 +191,53 @@ const printFooter = ({ duration, response }) => {
       keyValue(green('mode'), `${fetchMode} ${gray(fetchTime)}`.trim())
     )
   }
-  if (uri) console.error('     ', keyValue(green('uri'), uri))
+  if (uri) console.error('     ', keyValue(green('uri'), link(uri)))
   if (id) console.error('      ', keyValue(green('id'), id))
 }
 
+const isClientError = statusCode => statusCode >= 400 && statusCode < 500
+
+/**
+ * The API reports the actionable reason per field under `data`, keeping
+ * `message` as a generic pointer to it.
+ */
+const reasons = error => {
+  const values = Object.values(error.data ?? {}).filter(
+    value => typeof value === 'string'
+  )
+  return values.length > 0
+    ? values
+    : [String(error.message).replace(`${error.code}, `, '')]
+}
+
 const printFail = error => {
+  const color = isClientError(error.statusCode) ? orange : red
+  const status = error.status || 'fail'
+  const [reason, ...rest] = reasons(error)
+  const indent = ' '.repeat(status.length + 2)
   if (process.stdout.isTTY) console.error()
-  console.error(
-    label(error.status || 'fail', 'red'),
-    gray(String(error.message).replace(`${error.code}, `, ''))
-  )
+  console.error(label(status, color), gray(reason))
+  for (const extra of rest) console.error(indent, gray(extra))
   console.error()
   const id = error.headers?.['x-request-id']
-  if (id) console.error('    ', keyValue(red('id'), id))
-  if (error.url) console.error('   ', keyValue(red('uri'), error.url))
+  if (id) console.error('    ', keyValue(color('id'), id))
+  if (error.url) console.error('   ', keyValue(color('uri'), link(error.url)))
   if (error.code) {
     console.error(
       '  ',
       keyValue(
-        red('code'),
+        color('code'),
         `${error.code}${error.statusCode ? ` (${error.statusCode})` : ''}`
       )
     )
   }
-  if (error.more) console.error('  ', keyValue(red('more'), error.more))
+  if (error.more) console.error('  ', keyValue(color('more'), link(error.more)))
+  if (error.statusCode === 429) {
+    console.error(
+      '  ',
+      keyValue(color('hint'), 'run `microlink login` to use an API key')
+    )
+  }
 }
 
 const showHelp = command => {
@@ -248,7 +275,7 @@ const takeHttpHeaders = flags => {
 
 const argv = mri(process.argv.slice(2), {
   alias: { H: 'header' },
-  boolean: ['trace', 'trace-full', 'help'],
+  boolean: ['trace', 'trace-full', 'help', 'html', 'markdown'],
   string: ['header', 'api-key', 'data', 'file', 'endpoint']
 })
 
@@ -263,6 +290,8 @@ let {
   endpoint: endpointFlag,
   trace,
   'trace-full': traceFull,
+  html: htmlFlag,
+  markdown: markdownFlag,
   ...flags
 } = argv
 
@@ -270,69 +299,102 @@ const isTrace = trace || traceFull
 
 if (!command) showHelp()
 
-const apiKey = apiKeyFlag || apiKeyCamel || process.env.MICROLINK_API_KEY
-const endpoint = endpointFlag
-const client = create({
-  ...(apiKey && { apiKey }),
-  ...(endpoint && { endpoint })
-})
+if (command === 'login' || command === 'logout') {
+  if (help) showHelp(command)
+  if (command === 'logout') {
+    console.error(clearConfig() ? 'Logged out.' : 'Already logged out.')
+    process.exit(0)
+  }
+  login().then(
+    () => process.exit(0),
+    error => {
+      console.error(error.message)
+      process.exit(error.code === 'ABORT' ? 130 : 1)
+    }
+  )
+} else {
+  const apiKey =
+    apiKeyFlag || apiKeyCamel || process.env.MICROLINK_API_KEY || readApiKey()
+  const endpoint = endpointFlag
+  const client = create({
+    ...(apiKey && { apiKey }),
+    ...(endpoint && { endpoint })
+  })
 
-if (typeof client[command] !== 'function') {
-  if (!target && URL.canParse(command)) {
-    target = command
-    command = 'metadata'
-  } else if (help) {
-    showHelp()
-  } else {
-    console.error(
-      `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.`
-    )
-    process.exit(1)
+  if (typeof client[command] !== 'function') {
+    if (!target && URL.canParse(command)) {
+      target = command
+      command = 'metadata'
+    } else if (help) {
+      showHelp()
+    } else {
+      console.error(
+        `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.`
+      )
+      process.exit(1)
+    }
   }
-}
 
-if (help || !target) showHelp(command)
+  if (help || !target) showHelp(command)
 
-if (
-  isTrace &&
-  (command === 'search' || command === 'function' || command === 'run')
-) {
-  console.error(`\`--trace\` is not supported for \`${command}\`.`)
-  process.exit(1)
-}
+  if (
+    isTrace &&
+    (command === 'search' || command === 'function' || command === 'run')
+  ) {
+    console.error(`\`--trace\` is not supported for \`${command}\`.`)
+    process.exit(1)
+  }
 
-const options = { ...flags }
-const headers = { ...takeHttpHeaders(options), ...parseHeaders(header) }
-if (Object.keys(headers).length > 0) options.headers = headers
+  const options = { ...flags }
+  const headers = { ...takeHttpHeaders(options), ...parseHeaders(header) }
+  if (Object.keys(headers).length > 0) options.headers = headers
 
-const invoke = () => {
+  let rules
   if (command === 'extract') {
-    return client.extract(target, JSON.parse(data), options)
-  }
-  if (command === 'function' || command === 'run') {
-    const code = readFileSync(path.resolve(file), 'utf8')
-    return client.function(target, code, options)
+    try {
+      rules = JSON.parse(data)
+    } catch {
+      printFail({ message: 'Invalid --data JSON' })
+      process.exit(1)
+    }
   }
-  return client[command](target, options)
-}
 
-const spin = !isTrace && shouldSpin() ? spinner() : null
-
-;(async () => {
-  spin?.start()
-  const started = Date.now()
-  try {
-    const result = await invoke()
-    const duration = Date.now() - started
-    spin?.stop()
-    if (isTrace) printJson(tracePayload({ ...client.last, full: traceFull }))
-    else if (typeof result === 'string') console.log(result)
-    else printJson({ status: 'success', data: result })
-    if (!isTrace) printFooter({ duration, response: client.last.response })
-    process.exit(0)
-  } catch (error) {
-    spin?.stop()
-    printFail(error)
-    process.exit(1)
+  const invoke = () => {
+    if (command === 'extract') {
+      return client.extract(target, rules, options)
+    }
+    if (command === 'function' || command === 'run') {
+      const code = readFileSync(path.resolve(file), 'utf8')
+      return client.function(target, code, options)
+    }
+    if (command === 'search') {
+      return client.search(target, {
+        ...options,
+        ...(htmlFlag && { html: true }),
+        ...(markdownFlag && { markdown: true })
+      })
+    }
+    return client[command](target, options)
   }
-})()
+
+  const spin = !isTrace && shouldSpin() ? spinner() : null
+
+  ;(async () => {
+    spin?.start()
+    const started = Date.now()
+    try {
+      const result = await invoke()
+      const duration = Date.now() - started
+      spin?.stop()
+      if (isTrace) printJson(tracePayload({ ...client.last, full: traceFull }))
+      else if (typeof result === 'string') console.log(result)
+      else printJson({ status: 'success', data: result })
+      if (!isTrace) printFooter({ duration, response: client.last.response })
+      process.exit(0)
+    } catch (error) {
+      spin?.stop()
+      printFail(error)
+      process.exit(1)
+    }
+  })()
+}

package/bin/login.js

@@ -0,0 +1,148 @@
+'use strict'
+
+const { randomBytes } = require('crypto')
+const { spawn } = require('child_process')
+const http = require('http')
+const { writeConfig, readApiKey, configPathDisplay } = require('./config')
+const select = require('./select')
+const { gray } = require('./style')
+
+const TIMEOUT_MS = 5 * 60 * 1000
+
+const dashboardUrl = () =>
+  process.env.MICROLINK_DASHBOARD_URL || 'https://dashboard.microlink.io'
+
+const openUrl = url => {
+  const { platform } = process
+  const child =
+    platform === 'win32'
+      ? spawn('cmd', ['/c', 'start', '""', `"${url}"`], {
+        detached: true,
+        stdio: 'ignore',
+        windowsVerbatimArguments: true
+      })
+      : spawn(platform === 'darwin' ? 'open' : 'xdg-open', [url], {
+        detached: true,
+        stdio: 'ignore'
+      })
+  child.on('error', () => {})
+  child.unref()
+}
+
+const listen = state =>
+  new Promise((resolve, reject) => {
+    let settle
+    const token = new Promise((resolve, reject) => {
+      settle = { resolve, reject }
+    })
+
+    const cors = res => {
+      res.setHeader('Access-Control-Allow-Origin', '*')
+      res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS')
+      res.setHeader('Access-Control-Allow-Headers', 'content-type')
+      res.setHeader('Access-Control-Allow-Private-Network', 'true')
+    }
+
+    const server = http.createServer((req, res) => {
+      cors(res)
+      if (req.method === 'OPTIONS') {
+        res.writeHead(204)
+        res.end()
+        return
+      }
+      if (req.method !== 'POST') {
+        res.writeHead(405)
+        res.end()
+        return
+      }
+      const chunks = []
+      req.on('data', chunk => chunks.push(chunk))
+      req.on('end', () => {
+        try {
+          const body = JSON.parse(Buffer.concat(chunks).toString())
+          if (body.state !== state || typeof body.token !== 'string') {
+            res.writeHead(400)
+            res.end()
+            return
+          }
+          res.writeHead(204)
+          res.end()
+          clearTimeout(timer)
+          settle.resolve(body.token)
+        } catch {
+          res.writeHead(400)
+          res.end()
+        }
+      })
+    })
+
+    const close = () => {
+      clearTimeout(timer)
+      server.close()
+    }
+
+    const timer = setTimeout(() => {
+      close()
+      settle.reject(new Error('Timed out waiting for dashboard authorization'))
+    }, TIMEOUT_MS)
+
+    server.listen(0, '127.0.0.1', () => {
+      resolve({ port: server.address().port, token, close })
+    })
+    server.on('error', reject)
+  })
+
+const fetchKeys = async token => {
+  const res = await fetch(new URL('/api/v1/connect/keys', dashboardUrl()), {
+    headers: { authorization: `Bearer ${token}` }
+  })
+  if (!res.ok) {
+    throw new Error(`Could not load API keys (${res.status})`)
+  }
+  const body = await res.json()
+  return Array.isArray(body) ? body : body.keys
+}
+
+const asChoice = key => ({
+  name: key.label,
+  hint: key.maskedKey,
+  value: key.apiKey
+})
+
+const login = async () => {
+  const state = randomBytes(16).toString('hex')
+  const { port, token: tokenP, close } = await listen(state)
+  const url = new URL('/connect', dashboardUrl())
+  url.searchParams.set('port', String(port))
+  url.searchParams.set('state', state)
+  process.stderr.write(`Opening ${url}\n`)
+  openUrl(url.toString())
+
+  try {
+    const keys = await fetchKeys(await tokenP)
+    if (!keys?.length) {
+      throw new Error(
+        `No API keys on this account. Create a plan at ${dashboardUrl()}/plans`
+      )
+    }
+
+    const choices = keys.map(asChoice)
+    const picked =
+      choices.length === 1
+        ? choices[0]
+        : await select({
+          message: 'Which API key?',
+          choices,
+          current: readApiKey()
+        })
+
+    writeConfig({ apiKey: picked.value })
+    process.stderr.write(
+      `\n${gray('Saved')} ${picked.name} ${gray(`to ${configPathDisplay()}`)}\n`
+    )
+  } finally {
+    close()
+  }
+}
+
+module.exports = login

package/bin/select.js

@@ -0,0 +1,98 @@
+'use strict'
+
+const readline = require('readline')
+const { gray, white, styleText } = require('./style')
+
+const HIDE = '\u001b[?25l'
+const SHOW = '\u001b[?25h'
+const UP_N = n => `\u001b[${n}A`
+
+const label = (choice, current) => {
+  const mark = current && choice.value === current ? gray(' (current)') : ''
+  return `${choice.name} ${gray(`(${choice.hint})`)}${mark}`
+}
+
+const numbered = ({ message, choices, current }) =>
+  new Promise((resolve, reject) => {
+    for (const [i, choice] of choices.entries()) {
+      process.stderr.write(`  ${i + 1}) ${label(choice, current)}\n`)
+    }
+    const rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stderr
+    })
+    rl.question(`${message} `, answer => {
+      rl.close()
+      const choice = choices[Number.parseInt(answer, 10) - 1]
+      if (!choice) reject(new Error('Invalid selection'))
+      else resolve(choice)
+    })
+  })
+
+const arrows = ({ message, choices, current }) => {
+  let index = Math.max(
+    0,
+    choices.findIndex(choice => choice.value === current)
+  )
+  const lines = choices.length + 1
+
+  const draw = first => {
+    if (!first) process.stderr.write(UP_N(lines))
+    process.stderr.write(`${styleText('cyan', '?')} ${white(message)}\n`)
+    for (const [i, choice] of choices.entries()) {
+      const active = i === index
+      const prefix = active ? styleText('cyan', '❯') : ' '
+      const text = active
+        ? styleText('cyan', label(choice, current))
+        : label(choice, current)
+      process.stderr.write(`\u001b[2K${prefix} ${text}\n`)
+    }
+  }
+
+  return new Promise((resolve, reject) => {
+    readline.emitKeypressEvents(process.stdin)
+    process.stdin.setRawMode(true)
+    process.stderr.write(HIDE)
+    draw(true)
+
+    const cleanup = () => {
+      process.stdin.setRawMode(false)
+      process.stdin.off('keypress', onKey)
+      process.stderr.write(SHOW)
+    }
+
+    const onKey = (_str, key) => {
+      if (key.name === 'up') {
+        index = (index - 1 + choices.length) % choices.length
+        draw()
+        return
+      }
+      if (key.name === 'down') {
+        index = (index + 1) % choices.length
+        draw()
+        return
+      }
+      if (key.name === 'return') {
+        cleanup()
+        resolve(choices[index])
+        return
+      }
+      if (key.ctrl && key.name === 'c') {
+        cleanup()
+        reject(Object.assign(new Error('Aborted'), { code: 'ABORT' }))
+      }
+    }
+
+    process.stdin.on('keypress', onKey)
+  })
+}
+
+const select = ({ message, choices, current }) => {
+  if (choices.length === 1) return Promise.resolve(choices[0])
+  if (!process.stdin.isTTY || !process.stdout.isTTY) {
+    return numbered({ message, choices, current })
+  }
+  return arrows({ message, choices, current })
+}
+
+module.exports = select

package/bin/style.js

@@ -1,10 +1,21 @@
 'use strict'
 
 const { styleText } = require('node:util')
+const { default: terminalLink } = require('terminal-link')
 
 const gray = str => styleText('gray', str)
 const white = str => styleText('white', str)
 const green = str => styleText('green', str)
 const red = str => styleText('red', str)
 
-module.exports = { gray, white, green, red, styleText }
+const ORANGE_256 = '\u001b[38;5;208m'
+const DEFAULT_FOREGROUND = '\u001b[39m'
+
+const orange = str =>
+  process.stdout.hasColors?.()
+    ? `${ORANGE_256}${str}${DEFAULT_FOREGROUND}`
+    : String(str)
+
+const link = url => terminalLink.stderr(url, url, { fallback: false })
+
+module.exports = { gray, white, green, red, orange, link, styleText }

package/package.json

@@ -2,7 +2,7 @@
   "name": "microlink.io",
   "description": "The Microlink API organized into products, each returning a direct result",
   "homepage": "https://github.com/microlinkhq/microlink",
-  "version": "0.2.0",
+  "version": "0.5.0",
   "types": "./src/index.d.ts",
   "main": "./src/index.js",
   "exports": {
@@ -14,7 +14,8 @@
     }
   },
   "bin": {
-    "microlink": "bin/index.js"
+    "microlink": "bin/index.js",
+    "microlink.io": "bin/index.js"
   },
   "author": {
     "email": "hello@microlink.io",
@@ -43,10 +44,11 @@
     "seo"
   ],
   "dependencies": {
-    "@microlink/function": "0.3.0",
-    "@microlink/google": "1.0.6",
-    "@microlink/mql": "0.18.0",
-    "mri": "~1.2.0"
+    "mri": "~1.2.0",
+    "terminal-link": "~5.0.0",
+    "@microlink/google": "1.2.0",
+    "@microlink/function": "0.3.2",
+    "@microlink/mql": "0.18.1"
   },
   "devDependencies": {
     "ava": "latest",
@@ -61,9 +63,6 @@
     "bin",
     "src"
   ],
-  "scripts": {
-    "test": "ava && tsd"
-  },
   "preferGlobal": true,
   "license": "MIT",
   "ava": {
@@ -76,5 +75,7 @@
     },
     "directory": "test"
   },
-  "gitHead": "3766602b11599a8284e4aa31815d409e0ffa0788"
-}
+  "scripts": {
+    "test": "ava && tsd"
+  }
+}