microlink.io 0.0.0 → 0.0.2

package/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright © 2026 microlink.io <hello@microlink.io> (microlink.io)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/bin/help.txt

@@ -0,0 +1,47 @@
+Usage
+  $ microlink <product> <url|query> [options]
+
+Products
+  metadata      Unified metadata (title, description, image, ...)
+  logo          Brand logo of the site (--square prefers the square variant)
+  markdown      Page content as Markdown
+  html          Page content as HTML
+  text          Page content as plain text
+  video         Primary video of the page (returns the asset object)
+  audio         Primary audio of the page (returns the asset object)
+  emails        Every email address present on the page
+  links         Every absolute link URL on the page
+  images        Every absolute image URL on the page
+  videos        Every absolute video URL on the page
+  audios        Every absolute audio URL on the page
+  extract       Custom MQL data rules (--data '{"field":{...}}')
+  screenshot    Take a screenshot (returns the asset object)
+  pdf           Generate a PDF (returns the asset object)
+  embed         oEmbed-style embeddable iframe ({ html, scripts })
+  technologies  Detect the tech stack behind the site
+  lighthouse    Run a Lighthouse report
+  search        Google as structured data (query instead of url); --type
+                routes to news, images, videos, places, maps, shopping,
+                scholar, patents or autocomplete
+  function      Run code remotely with browser access (--file ./fn.js);
+                extra flags are injected as variables in the function scope
+
+Options
+  --api-key     Microlink API key (defaults to MICROLINK_API_KEY env)
+  --header, -H  Extra request header as 'Name: value' (repeatable)
+  --data        JSON data rules for the extract command
+  --file        Path to the code file for the function command
+  --help        Show this help
+
+Any other flag is passed as an option to the product, e.g. --fullPage,
+--device 'iPhone 11', --waitUntil networkidle0, --selector article.
+
+Examples
+  $ microlink markdown https://example.com
+  $ microlink screenshot https://example.com --fullPage
+  $ microlink logo https://github.com --square
+  $ microlink links https://example.com
+  $ microlink search "best coffee" --limit 10 --location es
+  $ microlink search "open source llm" --type news --period week
+  $ microlink extract https://microlink.io --data '{"image":{"selector":"meta[property=og:image]","attr":"content","type":"image"}}'
+  $ microlink function https://example.com --file ./fn.js

package/bin/index.js

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+'use strict'
+
+const { readFileSync } = require('fs')
+const path = require('path')
+const jsome = require('jsome')
+const mri = require('mri')
+
+const create = require('../src')
+
+const showHelp = () => {
+  console.log(readFileSync(path.join(__dirname, 'help.txt'), 'utf8'))
+  process.exit(0)
+}
+
+const parseHeaders = input => {
+  const headers = {}
+  for (const item of [].concat(input ?? [])) {
+    const index = String(item).indexOf(':')
+    if (index === -1) continue
+    headers[String(item).slice(0, index).trim().toLowerCase()] = String(item)
+      .slice(index + 1)
+      .trim()
+  }
+  return headers
+}
+
+const argv = mri(process.argv.slice(2), {
+  alias: { H: 'header' },
+  string: ['header', 'api-key', 'data', 'file']
+})
+
+const {
+  _: [command, target],
+  header,
+  help,
+  data,
+  file,
+  'api-key': apiKeyFlag,
+  apiKey: apiKeyCamel,
+  ...flags
+} = argv
+
+if (help || !command) showHelp()
+
+const apiKey = apiKeyFlag || apiKeyCamel || process.env.MICROLINK_API_KEY
+const client = create(apiKey ? { apiKey } : {})
+
+if (typeof client[command] !== 'function') {
+  console.error(
+    `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.`
+  )
+  process.exit(1)
+}
+
+const options = { ...flags }
+const headers = parseHeaders(header)
+if (Object.keys(headers).length > 0) options.headers = headers
+
+const invoke = () => {
+  if (command === 'extract') {
+    return client.extract(target, JSON.parse(data), options)
+  }
+  if (command === 'function' || command === 'run') {
+    const code = readFileSync(path.resolve(file), 'utf8')
+    return client.function(target, code, options)
+  }
+  return client[command](target, options)
+}
+
+Promise.resolve()
+  .then(invoke)
+  .then(result => {
+    if (typeof result === 'string') console.log(result)
+    else jsome(result)
+    process.exit(0)
+  })
+  .catch(error => {
+    console.error(error.message)
+    process.exit(1)
+  })

package/package.json

@@ -1,13 +1,82 @@
 {
   "name": "microlink.io",
-  "version": "0.0.0",
-  "description": "",
-  "main": "index.js",
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
-  "type": "module",
+  "description": "The Microlink API organized into products, each returning a direct result",
+  "homepage": "https://github.com/microlinkhq/microlink",
+  "version": "0.0.2",
+  "types": "./src/index.d.ts",
+  "main": "./src/index.js",
+  "exports": {
+    ".": {
+      "types": "./src/index.d.ts",
+      "require": "./src/index.js",
+      "import": "./src/main.mjs",
+      "default": "./src/main.mjs"
+    }
+  },
+  "bin": {
+    "microlink": "bin/index.js"
+  },
+  "author": {
+    "email": "hello@microlink.io",
+    "name": "microlink.io",
+    "url": "https://microlink.io"
+  },
+  "repository": {
+    "directory": "packages/core",
+    "type": "git",
+    "url": "git+https://github.com/microlinkhq/microlink.git"
+  },
+  "bugs": {
+    "url": "https://github.com/microlinkhq/microlink/issues"
+  },
+  "keywords": [
+    "api",
+    "extract",
+    "lighthouse",
+    "markdown",
+    "metadata",
+    "microlink",
+    "mql",
+    "pdf",
+    "scraper",
+    "screenshot",
+    "seo"
+  ],
+  "dependencies": {
+    "@microlink/function": "0.2.7",
+    "@microlink/google": "1.0.4",
+    "@microlink/mql": "0.17.0",
+    "jsome": "~2.5.0",
+    "mri": "~1.2.0"
+  },
+  "devDependencies": {
+    "ava": "latest",
+    "proxyquire": "latest",
+    "tinyspawn": "latest",
+    "tsd": "latest"
+  },
+  "engines": {
+    "node": ">= 24"
+  },
+  "files": [
+    "bin",
+    "src"
+  ],
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  }
-}
+    "lint": "tsd",
+    "test": "ava"
+  },
+  "preferGlobal": true,
+  "license": "MIT",
+  "ava": {
+    "timeout": "2m"
+  },
+  "tsd": {
+    "compilerOptions": {
+      "module": "node16",
+      "moduleResolution": "node16"
+    },
+    "directory": "test"
+  },
+  "gitHead": "35ba2e58e85c61899d5abd23fca47a4b01ccbebf"
+}

package/src/index.d.ts

@@ -0,0 +1,178 @@
+import createGoogleClient from '@microlink/google'
+
+type GoogleClient = ReturnType<typeof createGoogleClient>
+
+/**
+ * Transport & top-level API query params. Unknown keys fall through
+ * to the API query string, so an index signature is provided.
+ */
+interface Options {
+  apiKey?: string
+  endpoint?: string
+  headers?: Record<string, string>
+  adblock?: boolean
+  animations?: boolean
+  audio?: boolean
+  cacheKey?: string
+  click?: string | string[]
+  colorScheme?: 'light' | 'dark' | 'no-preference'
+  device?: string
+  filename?: string
+  force?: boolean
+  javascript?: boolean
+  meta?: boolean
+  modules?: string | string[]
+  prerender?: boolean | 'auto'
+  proxy?: string
+  retry?: number
+  scripts?: string | string[]
+  staleTtl?: string | number
+  styles?: string | string[]
+  timeout?: number
+  ttl?: string | number
+  video?: boolean
+  viewport?: Record<string, unknown>
+  waitForTimeout?: number
+  waitUntil?: string | string[]
+  [key: string]: unknown
+}
+
+interface ContentOptions extends Options {
+  selector?: string
+  selectorAll?: string | string[]
+  type?: string
+}
+
+interface CollectionOptions extends Options {
+  selector?: string
+  selectorAll?: string | string[]
+  attr?: string
+  type?: string
+}
+
+interface ScreenshotOptions extends Options {
+  fullPage?: boolean
+  type?: 'jpeg' | 'png'
+  overlay?: Record<string, unknown>
+  element?: string
+  omitBackground?: boolean
+  optimizeForSpeed?: boolean
+  codeScheme?: string
+  animated?: boolean
+  palette?: boolean
+}
+
+interface PdfOptions extends Options {
+  format?: string
+  margin?: string | Record<string, string | number>
+  scale?: number
+  landscape?: boolean
+  pageRanges?: string
+  width?: string | number
+  height?: string | number
+  printBackground?: boolean
+}
+
+interface EmbedOptions extends Options {
+  maxWidth?: number
+  maxHeight?: number
+}
+
+interface LogoOptions extends Options {
+  square?: boolean
+}
+
+interface LighthouseOptions extends Options {
+  onlyCategories?: string[]
+  onlyAudits?: string[]
+  skipAudits?: string[]
+  output?: string | string[]
+}
+
+/** Rich asset object (screenshot, pdf, logo, image rules, ...). */
+interface Asset {
+  url: string
+  type?: string
+  size?: number
+  size_pretty?: string
+  width?: number
+  height?: number
+  [key: string]: unknown
+}
+
+interface Metadata {
+  title?: string | null
+  description?: string | null
+  url?: string | null
+  [key: string]: unknown
+}
+
+interface Embed {
+  html: string
+  scripts?: unknown[]
+  [key: string]: unknown
+}
+
+type FunctionInput = string | ((args: Record<string, unknown>) => unknown)
+
+interface FunctionResult<T = unknown> {
+  isFulfilled: boolean
+  value: T
+  profiling: Record<string, unknown>
+  logging: Record<string, unknown>
+}
+
+export declare class MicrolinkError extends Error {
+  status?: string
+  code?: string
+  statusCode?: number
+  description?: string
+  url?: string
+  headers?: Record<string, string>
+  more?: string
+  constructor (props: Record<string, unknown>)
+}
+
+interface MicrolinkClient {
+  metadata (url: string, options?: Options): Promise<Metadata>
+  logo (url: string, options?: LogoOptions): Promise<Asset>
+  markdown (url: string, options?: ContentOptions): Promise<string>
+  html (url: string, options?: ContentOptions): Promise<string>
+  text (url: string, options?: ContentOptions): Promise<string>
+  video (url: string, options?: Options): Promise<Asset>
+  audio (url: string, options?: Options): Promise<Asset>
+  emails (url: string, options?: Options): Promise<string[]>
+  links (url: string, options?: CollectionOptions): Promise<string[]>
+  images (url: string, options?: CollectionOptions): Promise<string[]>
+  videos (url: string, options?: CollectionOptions): Promise<string[]>
+  audios (url: string, options?: CollectionOptions): Promise<string[]>
+  extract (
+    url: string,
+    rules: Record<string, unknown>,
+    options?: Options
+  ): Promise<Record<string, unknown>>
+  screenshot (url: string, options?: ScreenshotOptions): Promise<Asset>
+  pdf (url: string, options?: PdfOptions): Promise<Asset>
+  embed (url: string, options?: EmbedOptions): Promise<Embed>
+  technologies (url: string, options?: Options): Promise<unknown[]>
+  lighthouse (
+    url: string,
+    options?: LighthouseOptions
+  ): Promise<Record<string, unknown>>
+  search: GoogleClient
+  function<T = unknown> (
+    url: string,
+    code: FunctionInput,
+    options?: Options
+  ): Promise<FunctionResult<T>>
+  run: MicrolinkClient['function']
+}
+
+interface create {
+  (options?: Options): MicrolinkClient
+  MicrolinkError: typeof MicrolinkError
+}
+
+declare const create: create
+
+export default create

package/src/index.js

@@ -0,0 +1,195 @@
+'use strict'
+
+const createGoogleClient = require('@microlink/google')
+const fn = require('@microlink/function')
+const mql = require('@microlink/mql')
+
+const CONTENT_KEYS = ['selector', 'selectorAll', 'type']
+
+const COLLECTION_KEYS = ['selector', 'selectorAll', 'attr', 'type']
+
+const SCREENSHOT_KEYS = [
+  'fullPage',
+  'type',
+  'overlay',
+  'element',
+  'omitBackground',
+  'optimizeForSpeed',
+  'codeScheme',
+  'animated',
+  'palette'
+]
+
+const PDF_KEYS = [
+  'format',
+  'margin',
+  'scale',
+  'landscape',
+  'pageRanges',
+  'width',
+  'height',
+  'printBackground'
+]
+
+const EMBED_KEYS = ['maxWidth', 'maxHeight']
+
+const LOGO_KEYS = ['square']
+
+const LIGHTHOUSE_KEYS = ['onlyCategories', 'onlyAudits', 'skipAudits', 'output']
+
+const isEmpty = obj => Object.keys(obj).length === 0
+
+const create = (ctx = {}) => {
+  /**
+   * Split the single options bag into the three destinations mql has:
+   * `got.headers` (HTTP layer, 3rd arg), `sub` (capability nested keys)
+   * and `top` (top-level query params, 2nd arg). Client context is
+   * merged first so per-call options can override it.
+   */
+  const route = (opts, nested = []) => {
+    const top = {}
+    const sub = {}
+    let got
+    for (const [key, value] of Object.entries({ ...ctx, ...opts })) {
+      if (key === 'headers') got = { headers: value }
+      else if (nested.includes(key)) sub[key] = value
+      else top[key] = value
+    }
+    return { top, sub, got }
+  }
+
+  const content = field => (url, options) => {
+    const { top, sub, got } = route(options, CONTENT_KEYS)
+    return mql(
+      url,
+      { ...top, meta: false, data: { [field]: { attr: field, ...sub } } },
+      got
+    ).then(({ data }) => data[field])
+  }
+
+  const collection = (field, rule) => (url, options) => {
+    const { top, sub, got } = route(options, COLLECTION_KEYS)
+    return mql(
+      url,
+      { ...top, meta: false, data: { [field]: { ...rule, ...sub } } },
+      got
+    ).then(({ data }) => data[field] ?? [])
+  }
+
+  const capability = (field, nested) => (url, options) => {
+    const { top, sub, got } = route(options, nested)
+    return mql(
+      url,
+      { ...top, meta: false, [field]: isEmpty(sub) ? true : sub },
+      got
+    ).then(({ data }) => data[field])
+  }
+
+  /* Primary media detection (`data.video` / `data.audio`). */
+  const media = field => (url, options) => {
+    const { top, got } = route(options)
+    return mql(url, { ...top, meta: false, [field]: true }, got).then(
+      ({ data }) => data[field]
+    )
+  }
+
+  const googleClient = createGoogleClient()
+
+  const run = (url, code, options) => {
+    const { top, got } = route(options)
+    return fn(code, top, got)(url)
+  }
+
+  return {
+    metadata: (url, options) => {
+      const { top, got } = route(options)
+      return mql(url, top, got).then(({ data }) => data)
+    },
+    logo: (url, options) => {
+      const { top, sub, got } = route(options, LOGO_KEYS)
+      return mql(
+        url,
+        { ...top, meta: isEmpty(sub) ? true : { logo: sub } },
+        got
+      ).then(({ data }) => data.logo)
+    },
+    markdown: content('markdown'),
+    html: content('html'),
+    text: content('text'),
+    video: media('video'),
+    audio: media('audio'),
+    links: collection('links', { selectorAll: 'a', attr: 'href', type: 'url' }),
+    images: collection('images', {
+      selectorAll: 'img',
+      attr: 'src',
+      type: 'url'
+    }),
+    videos: collection('videos', {
+      selectorAll: ['video[src]', 'video source[src]'],
+      attr: 'src',
+      type: 'url'
+    }),
+    audios: collection('audios', {
+      selectorAll: ['audio[src]', 'audio source[src]'],
+      attr: 'src',
+      type: 'url'
+    }),
+    emails: collection('emails', {
+      selector: 'html',
+      attr: 'html',
+      type: 'email'
+    }),
+    extract: (url, rules, options) => {
+      const { top, got } = route(options)
+      return mql(url, { ...top, meta: false, data: rules }, got).then(
+        ({ data }) => data
+      )
+    },
+    screenshot: capability('screenshot', SCREENSHOT_KEYS),
+    pdf: capability('pdf', PDF_KEYS),
+    embed: (url, options) => {
+      const { top, sub, got } = route(options, EMBED_KEYS)
+      return mql(
+        url,
+        { ...top, meta: false, iframe: isEmpty(sub) ? true : sub },
+        got
+      ).then(({ data }) => data.iframe)
+    },
+    technologies: (url, options) => {
+      const { top, got } = route(options)
+      return mql(
+        url,
+        {
+          ...top,
+          meta: false,
+          insights: { technologies: true, lighthouse: false }
+        },
+        got
+      ).then(({ data }) => data.insights.technologies)
+    },
+    lighthouse: (url, options) => {
+      const { top, sub, got } = route(options, LIGHTHOUSE_KEYS)
+      return mql(
+        url,
+        {
+          ...top,
+          meta: false,
+          insights: {
+            technologies: false,
+            lighthouse: isEmpty(sub) ? true : sub
+          }
+        },
+        got
+      ).then(({ data }) => data.insights.lighthouse)
+    },
+    search: (query, options) => {
+      const { top } = route(options)
+      return googleClient(query, top)
+    },
+    function: run,
+    run
+  }
+}
+
+module.exports = create
+module.exports.MicrolinkError = mql.MicrolinkError

package/src/main.mjs

@@ -0,0 +1,5 @@
+import create from './index.js'
+
+export const MicrolinkError = create.MicrolinkError
+
+export default create