ddg-search 2026.2.15-1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 camo@hiddendj.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,78 @@
+# ddg-search
+
+DuckDuckGo HTML search scraper with multiple output formats. Provides a CLI and small library helpers to fetch result pages, handle pagination, and emit OpenSearch-style structured data.
+
+## Requirements
+- Node.js 22 or newer
+
+## Installation
+- Global CLI (npm): `npm install -g ddg-search`
+- One-off run (npx): `npx ddg-search --help`
+- Project dependency: `npm install ddg-search`
+- Local dev from this repo: `pnpm install` then `pnpm link --global` or `pnpm install -g .` (enable via `corepack enable` if needed)
+
+## CLI usage
+```
+Usage: ddg-search [options] <query>
+
+Search DuckDuckGo and output results in structured formats.
+
+Options:
+  -f, --format <fmt>   Output format (default: json). See formats below.
+  -p, --pages <n>      Maximum pages to scrape, 0 for unlimited (default: 5)
+  -r, --region <code>  Region code, e.g. us-en, uk-en (default: all regions)
+  -t, --time <range>   Time filter: d (day), w (week), m (month), y (year)
+  -h, --help           Show this help message
+
+Formats:
+  json        OpenSearch 1.1 response conventions in JSON
+  jsonl       One JSON object per result line (streaming-friendly)
+  csv         CSV with headers
+  opensearch  OpenSearch 1.1 Atom XML
+  markdown    Numbered markdown list (LLM-friendly)
+  compact     Minimal token format for LLM context windows
+
+Results are written to stdout; progress is written to stderr.
+```
+
+## Examples
+- `ddg-search "node.js tutorial"`
+- `ddg-search -f csv -p 3 "linux kernel"`
+- `ddg-search -f opensearch "rust programming" > results.xml`
+- `ddg-search -f compact "api docs" | llm "summarize these results"`
+- `ddg-search -p 0 "scrape everything"`
+- `ddg-search -r us-en -t w "recent news"`
+- `ddg-search "rust programming" | jq '.items[].link'`
+
+## Programmatic usage
+```js
+import { search, formatJson } from 'ddg-search';
+
+const { results, spelling, zeroClick } = await search('rust programming', {
+  maxPages: 2,
+  region: 'us-en',
+  time: 'w',
+});
+
+// Convert to OpenSearch-style JSON
+const output = formatJson({ results, spelling, zeroClick });
+console.log(output);
+```
+Exports also include `fetchPage`, `parsePage`, and formatters like `formatCsv`, `formatJsonl`, `formatMarkdown`, `formatOpenSearch`, and `formatCompact`.
+
+## Notes
+- DuckDuckGo may present bot-detection. The scraper stops early and returns collected results if that happens.
+- Respect site terms of use and rate-limit your requests; `search()` inserts random delays between pages by default.
+
+## Development
+- Run tests: `pnpm test`
+- Coverage: `pnpm run coverage`
+- Lint: `pnpm run lint`
+- Format check: `pnpm run format`; auto-fix: `pnpm run format:write`
+
+## Links
+- npm: https://www.npmjs.com/package/ddg-search
+- GitHub: https://github.com/camohiddendj/ddg-search
+
+## Contributing
+Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on issues and pull requests.

package/bin/ddg-search.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { main } from '../src/index.js';
+
+main();

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "ddg-search",
+  "version": "2026.2.15-1",
+  "description": "DuckDuckGo HTML search scraper with multiple output formats",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/camohiddendj/ddg-search.git"
+  },
+  "bugs": {
+    "url": "https://github.com/camohiddendj/ddg-search/issues"
+  },
+  "homepage": "https://github.com/camohiddendj/ddg-search#readme",
+  "type": "module",
+  "main": "src/index.js",
+  "files": [
+    "bin",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "bin": {
+    "ddg-search": "bin/ddg-search.js"
+  },
+  "scripts": {
+    "test": "node --test",
+    "coverage": "c8 node --test",
+    "lint": "eslint .",
+    "format": "prettier --check .",
+    "format:write": "prettier --write ."
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "keywords": [
+    "duckduckgo",
+    "search",
+    "cli",
+    "scraper",
+    "opensearch"
+  ],
+  "dependencies": {
+    "cheerio": "^1.2.0"
+  },
+  "devDependencies": {
+    "c8": "^9.1.0",
+    "eslint": "^8.57.1",
+    "eslint-config-prettier": "^9.1.0",
+    "prettier": "^3.3.3"
+  },
+  "packageManager": "pnpm@10.29.3+sha512.498e1fb4cca5aa06c1dcf2611e6fafc50972ffe7189998c409e90de74566444298ffe43e6cd2acdc775ba1aa7cc5e092a8b7054c811ba8c5770f84693d33d2dc"
+}

package/src/args.js

@@ -0,0 +1,57 @@
+import { parseArgs } from 'node:util';
+import { usage } from './usage.js';
+
+const SUPPORTED_FORMATS = ['json', 'jsonl', 'csv', 'opensearch', 'markdown', 'compact'];
+
+export function parseCliArgs(argv = process.argv.slice(2), exitFn = process.exit) {
+  let parsed;
+  try {
+    parsed = parseArgs({
+      allowPositionals: true,
+      args: argv,
+      options: {
+        format: { type: 'string', short: 'f', default: 'json' },
+        pages: { type: 'string', short: 'p' },
+        region: { type: 'string', short: 'r' },
+        time: { type: 'string', short: 't' },
+        help: { type: 'boolean', short: 'h', default: false },
+      },
+    });
+  } catch (e) {
+    console.error(`Error: ${e.message}`);
+    exitFn(1);
+    return { query: '', maxPages: 0, format: 'json', region: '', time: '' };
+  }
+
+  const { values, positionals } = parsed;
+
+  if (values.help || positionals.length === 0) {
+    usage(exitFn);
+  }
+
+  const query = positionals.join(' ');
+  const parsedPages = values.pages != null ? parseInt(values.pages, 10) : 5;
+  const maxPages = parsedPages === 0 ? Infinity : parsedPages;
+  const format = values.format;
+  const region = values.region || '';
+  const time = values.time || '';
+
+  if (!SUPPORTED_FORMATS.includes(format)) {
+    console.error(`Unknown format: ${format}. Supported: ${SUPPORTED_FORMATS.join(', ')}`);
+    exitFn(1);
+  }
+
+  if (Number.isNaN(parsedPages) || parsedPages < 0) {
+    console.error('--pages must be a non-negative integer (0 for unlimited)');
+    exitFn(1);
+  }
+
+  if (time && !['d', 'w', 'm', 'y'].includes(time)) {
+    console.error('Unknown time range: d, w, m, y');
+    exitFn(1);
+  }
+
+  return { query, maxPages, format, region, time };
+}
+
+export { usage };

package/src/cli.js

@@ -0,0 +1,50 @@
+import { parseCliArgs } from './args.js';
+import {
+  formatCompact,
+  formatCsv,
+  formatJson,
+  formatJsonl,
+  formatMarkdown,
+  formatOpenSearch,
+} from './formatters.js';
+import { search } from './search.js';
+
+export async function main(
+  argv = process.argv.slice(2),
+  { searchImpl = search, stdout = process.stdout, exit = process.exit } = {},
+) {
+  const { query, maxPages, format, region, time } = parseCliArgs(argv, exit);
+
+  try {
+    const data = await searchImpl(query, { maxPages, region, time });
+
+    let output;
+    switch (format) {
+      case 'json':
+        output = formatJson(data);
+        break;
+      case 'jsonl':
+        output = formatJsonl(data);
+        break;
+      case 'csv':
+        output = formatCsv(data);
+        break;
+      case 'opensearch':
+        output = formatOpenSearch(data);
+        break;
+      case 'markdown':
+        output = formatMarkdown(data);
+        break;
+      case 'compact':
+        output = formatCompact(data);
+        break;
+      default:
+        throw new Error(`Unsupported format: ${format}`);
+    }
+
+    stdout.write(output + '\n');
+  } catch (err) {
+    console.error(`Error: ${err.message}`);
+    exit(1);
+  }
+}

package/src/constants.js

@@ -0,0 +1,4 @@
+export const USER_AGENT =
+  'Mozilla/5.0 (U; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36';
+
+export const BASE_URL = 'https://html.duckduckgo.com/html/';

package/src/formatters.js

@@ -0,0 +1,169 @@
+import { BASE_URL } from './constants.js';
+
+export function escapeCsv(str) {
+  if (str.includes('"') || str.includes(',') || str.includes('\n')) {
+    return '"' + str.replace(/"/g, '""') + '"';
+  }
+  return str;
+}
+
+export function escapeXml(str) {
+  return str
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;');
+}
+
+export function formatJson(data) {
+  const output = {
+    'opensearch:totalResults': data.results.length,
+    'opensearch:startIndex': 1,
+    'opensearch:itemsPerPage': data.results.length,
+    'opensearch:Query': {
+      role: 'request',
+      searchTerms: data.query,
+    },
+    pagesScraped: data.pagesScraped,
+  };
+
+  if (data.spelling) {
+    output.spelling = data.spelling;
+  }
+
+  if (data.zeroClick) {
+    output.zeroClick = data.zeroClick;
+  }
+
+  output.items = data.results.map((r, i) => ({
+    position: i + 1,
+    title: r.title,
+    link: r.url,
+    description: r.description,
+    displayUrl: r.displayUrl,
+  }));
+
+  return JSON.stringify(output, null, 2);
+}
+
+export function formatJsonl(data) {
+  const lines = [];
+  if (data.zeroClick) {
+    lines.push(JSON.stringify({ type: 'zeroClick', ...data.zeroClick }));
+  }
+  lines.push(
+    ...data.results.map((r, i) =>
+      JSON.stringify({
+        position: i + 1,
+        title: r.title,
+        link: r.url,
+        description: r.description,
+      }),
+    ),
+  );
+  return lines.join('\n');
+}
+
+export function formatCsv(data) {
+  const lines = ['position,title,link,description'];
+  for (let i = 0; i < data.results.length; i++) {
+    const r = data.results[i];
+    lines.push([i + 1, escapeCsv(r.title), escapeCsv(r.url), escapeCsv(r.description)].join(','));
+  }
+  return lines.join('\n');
+}
+
+export function formatOpenSearch(data) {
+  const now = new Date().toISOString();
+  const searchUrl = `${BASE_URL}?q=${encodeURIComponent(data.query)}`;
+
+  let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
+  xml += '<feed xmlns="http://www.w3.org/2005/Atom"\n';
+  xml += '      xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">\n';
+  xml += `  <title>DuckDuckGo: ${escapeXml(data.query)}</title>\n`;
+  xml += `  <link href="${escapeXml(searchUrl)}"/>\n`;
+  xml += `  <updated>${now}</updated>\n`;
+  xml += `  <id>${escapeXml(searchUrl)}</id>\n`;
+  xml += `  <opensearch:totalResults>${data.results.length}</opensearch:totalResults>\n`;
+  xml += '  <opensearch:startIndex>1</opensearch:startIndex>\n';
+  xml += `  <opensearch:itemsPerPage>${data.results.length}</opensearch:itemsPerPage>\n`;
+  xml += `  <opensearch:Query role="request" searchTerms="${escapeXml(data.query)}"/>\n`;
+
+  if (data.zeroClick) {
+    const zc = data.zeroClick;
+    xml += '  <entry>\n';
+    xml += `    <title type="text">${escapeXml(zc.heading)}</title>\n`;
+    xml += `    <link href="${escapeXml(zc.url)}"/>\n`;
+    xml += `    <id>${escapeXml(zc.url)}</id>\n`;
+    xml += `    <summary>${escapeXml(zc.abstract)}</summary>\n`;
+    xml += '    <category term="zeroClick"/>\n';
+    xml += '  </entry>\n';
+  }
+
+  for (const r of data.results) {
+    xml += '  <entry>\n';
+    xml += `    <title>${escapeXml(r.title)}</title>\n`;
+    xml += `    <link href="${escapeXml(r.url)}"/>\n`;
+    xml += `    <id>${escapeXml(r.url)}</id>\n`;
+    xml += `    <summary>${escapeXml(r.description)}</summary>\n`;
+    xml += '  </entry>\n';
+  }
+
+  xml += '</feed>';
+  return xml;
+}
+
+export function formatMarkdown(data) {
+  const lines = [];
+  lines.push(`# Search: ${data.query}`);
+  lines.push(`${data.results.length} results from ${data.pagesScraped} page(s)\n`);
+
+  if (data.spelling) {
+    lines.push(`> **Did you mean:** ${data.spelling.corrected}\n`);
+  }
+
+  if (data.zeroClick) {
+    const zc = data.zeroClick;
+    lines.push(`> **${zc.heading}** — ${zc.abstract}`);
+    const suffix = zc.source ? ` (${zc.source})` : '';
+    lines.push(`> [Read more](${zc.url})${suffix}\n`);
+  }
+
+  for (let i = 0; i < data.results.length; i++) {
+    const r = data.results[i];
+    lines.push(`${i + 1}. [${r.title}](${r.url})`);
+    if (r.description) {
+      lines.push(`   ${r.description}`);
+    }
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+export function formatCompact(data) {
+  const lines = [];
+  lines.push(`query: ${data.query}`);
+  lines.push(`results: ${data.results.length}`);
+  if (data.spelling) {
+    lines.push(`did_you_mean: ${data.spelling.corrected}`);
+  }
+  if (data.zeroClick) {
+    lines.push(`zero_click: ${data.zeroClick.heading}`);
+    lines.push(`    ${data.zeroClick.url}`);
+    lines.push(`    ${data.zeroClick.abstract}`);
+  }
+  lines.push('---');
+
+  for (let i = 0; i < data.results.length; i++) {
+    const r = data.results[i];
+    lines.push(`[${i + 1}] ${r.title}`);
+    lines.push(`    ${r.url}`);
+    if (r.description) {
+      lines.push(`    ${r.description}`);
+    }
+  }
+
+  return lines.join('\n');
+}

package/src/index.js

@@ -0,0 +1,16 @@
+export { BASE_URL, USER_AGENT } from './constants.js';
+export { usage } from './usage.js';
+export { parseCliArgs } from './args.js';
+export { parsePage, isBotDetection } from './parser.js';
+export { fetchPage, randomDelay, search } from './search.js';
+export {
+  escapeCsv,
+  escapeXml,
+  formatCompact,
+  formatCsv,
+  formatJson,
+  formatJsonl,
+  formatMarkdown,
+  formatOpenSearch,
+} from './formatters.js';
+export { main } from './cli.js';

package/src/parser.js

@@ -0,0 +1,87 @@
+import * as cheerio from 'cheerio';
+
+export function isBotDetection(html) {
+  return html.includes('anomaly-modal') || html.includes('challenge-form');
+}
+
+export function parsePage(html) {
+  const $ = cheerio.load(html);
+
+  let spelling = null;
+  const didYouMean = $('#did_you_mean');
+  if (didYouMean.length) {
+    const links = didYouMean.find('a');
+    const correctedLink = links.first();
+    if (correctedLink.length) {
+      spelling = { corrected: correctedLink.text().trim() };
+      const originalLink = links.eq(1);
+      if (originalLink.length) {
+        spelling.original = originalLink.text().trim().replace(/^"|"$/g, '');
+      }
+    }
+  }
+
+  let zeroClick = null;
+  const zciEl = $('.zci-wrapper .zci');
+  if (zciEl.length) {
+    const headingAnchor = zciEl.find('.zci__heading a');
+    const abstractEl = zciEl.find('#zero_click_abstract');
+    const imageEl = abstractEl.find('.zci__image');
+    const sourceLink = abstractEl.find('a q');
+
+    const heading = headingAnchor.text().trim();
+    const url = headingAnchor.attr('href') || '';
+
+    const abstractClone = abstractEl.clone();
+    abstractClone.find('a').remove();
+    const abstract = abstractClone.text().trim();
+
+    if (heading) {
+      zeroClick = { heading, url, abstract };
+      const imageSrc = imageEl.attr('src');
+      if (imageSrc) zeroClick.image = imageSrc;
+      const sourceName = sourceLink.text().trim();
+      if (sourceName) zeroClick.source = sourceName;
+    }
+  }
+
+  const results = [];
+  $('.result.web-result')
+    .not('.result--ad')
+    .not('.result--no-result')
+    .each((_i, el) => {
+      const $el = $(el);
+      const titleEl = $el.find('.result__a');
+      const snippetEl = $el.find('.result__snippet');
+      const urlEl = $el.find('.result__url');
+
+      const title = titleEl.text().trim();
+      const url = titleEl.attr('href') || '';
+      const description = snippetEl.text().trim();
+      const displayUrl = urlEl.text().trim();
+
+      if (title && url) {
+        results.push({ title, url, description, displayUrl });
+      }
+    });
+
+  const noMoreResults = $('.result--no-result').length > 0;
+
+  let nextPageData = null;
+  $('.nav-link').each((_i, el) => {
+    const $form = $(el).find('form');
+    const submitBtn = $form.find('input[type="submit"]');
+    if (submitBtn.val() === 'Next') {
+      nextPageData = {};
+      $form.find('input[type="hidden"]').each((_j, input) => {
+        const $input = $(input);
+        const name = $input.attr('name');
+        if (name) {
+          nextPageData[name] = $input.attr('value') || '';
+        }
+      });
+    }
+  });
+
+  return { results, spelling, zeroClick, noMoreResults, nextPageData };
+}

package/src/search.js

@@ -0,0 +1,84 @@
+import { BASE_URL, USER_AGENT } from './constants.js';
+import { isBotDetection, parsePage } from './parser.js';
+
+export async function fetchPage(url, postData, fetchImpl = fetch) {
+  const opts = {
+    headers: { 'User-Agent': USER_AGENT },
+  };
+
+  if (postData) {
+    opts.method = 'POST';
+    opts.headers['Content-Type'] = 'application/x-www-form-urlencoded';
+    opts.body = new URLSearchParams(postData).toString();
+  }
+
+  const resp = await fetchImpl(url, opts);
+  if (!resp.ok) {
+    throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
+  }
+  return resp.text();
+}
+
+export function randomDelay() {
+  const ms = 800 + Math.random() * 2100;
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function search(
+  query,
+  { maxPages, region, time, fetchImpl = fetch, delay = randomDelay, stderr = process.stderr },
+) {
+  const allResults = [];
+  let spelling = null;
+  let zeroClick = null;
+  let page = 0;
+  const showProgress = stderr.isTTY;
+
+  const params = new URLSearchParams({ q: query });
+  if (region) params.set('kl', region);
+  if (time) params.set('df', time);
+
+  const firstHtml = await fetchPage(`${BASE_URL}?${params}`, null, fetchImpl);
+
+  if (isBotDetection(firstHtml)) {
+    throw new Error('Anti-bot detection triggered on first request. Try again later.');
+  }
+
+  let parsed = parsePage(firstHtml);
+  allResults.push(...parsed.results);
+  spelling = parsed.spelling;
+  zeroClick = parsed.zeroClick;
+  page++;
+
+  if (showProgress) {
+    stderr.write(`\rPage ${page}: ${parsed.results.length} results (${allResults.length} total)`);
+  }
+
+  while (parsed.nextPageData && !parsed.noMoreResults && page < maxPages) {
+    await delay();
+
+    const html = await fetchPage(BASE_URL, parsed.nextPageData, fetchImpl);
+
+    if (isBotDetection(html)) {
+      if (showProgress) {
+        stderr.write('\n');
+        stderr.write('Anti-bot detection hit. Returning results collected so far.\n');
+      }
+      break;
+    }
+
+    parsed = parsePage(html);
+    allResults.push(...parsed.results);
+    page++;
+
+    if (showProgress) {
+      stderr.write(`\rPage ${page}: ${parsed.results.length} results (${allResults.length} total)`);
+    }
+  }
+
+  if (showProgress) {
+    stderr.write('\n');
+  }
+
+  return { results: allResults, spelling, zeroClick, pagesScraped: page, query };
+}

package/src/usage.js

@@ -0,0 +1,33 @@
+export function usage(exitFn = process.exit) {
+  const text = `Usage: ddg-search [options] <query>
+
+Search DuckDuckGo and output results in structured formats.
+
+Options:
+  -f, --format <fmt>   Output format (default: json). See formats below.
+  -p, --pages <n>      Maximum pages to scrape, 0 for unlimited (default: 5)
+  -r, --region <code>  Region code, e.g. us-en, uk-en (default: all regions)
+  -t, --time <range>   Time filter: d (day), w (week), m (month), y (year)
+  -h, --help           Show this help message
+
+Formats:
+  json        OpenSearch 1.1 response conventions in JSON
+  jsonl       One JSON object per result line (streaming-friendly)
+  csv         CSV with headers
+  opensearch  OpenSearch 1.1 Atom XML
+  markdown    Numbered markdown list (AI/LLM-friendly)
+  compact     Minimal token format for LLM context windows
+
+  Results are written to stdout; progress is written to stderr.
+
+Examples:
+  ddg-search "node.js tutorial"
+  ddg-search -f csv -p 3 "linux kernel"
+  ddg-search -f opensearch "rust programming" > results.xml
+  ddg-search -f compact "api docs" | llm "summarize these results"
+  ddg-search -p 0 "scrape everything"
+  ddg-search -r us-en -t w "recent news"
+  ddg-search "rust programming" | jq '.items[].link'`;
+  console.error(text);
+  exitFn(1);
+}