opencode-skills-collection 3.1.0 → 3.1.2

package/bundled-skills/competitor-analysis/scripts/extract_vs_names.mjs

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+
+// Parses "X vs Y" patterns from `browse cloud search` result titles across discovery batch files.
+// Produces a ranked list of candidate competitor names, with an example title each,
+// and attempts to resolve each name to a domain from the result URL pool.
+//
+// Usage: node extract_vs_names.mjs <directory> [--prefix competitor] [--seed "Exa,Tavily,SerpAPI"]
+//
+// Output: newline-delimited JSON to stdout, one object per candidate:
+//   { "name": "serper", "hits": 3, "domain": "serper.dev", "example": "Tavily vs Serper..." }
+
+import { readdirSync, readFileSync } from 'fs';
+import { join } from 'path';
+
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h') || args.length === 0) {
+  console.error(`Usage: node extract_vs_names.mjs <directory> [--prefix <prefix>] [--seed "<csv>"]
+
+Reads all <prefix>_discovery_batch_*.json files, parses "X vs Y" patterns from result
+titles, and outputs a ranked list of candidate competitor names as newline-delimited JSON.
+
+Options:
+  --prefix <prefix>   Batch file prefix (default: "competitor")
+  --seed "<csv>"      Comma-separated list of seed names to exclude from output
+                      (you already know these; want the OTHER side of the comparison)
+  --help, -h          Show this help message`);
+  process.exit(args.includes('--help') || args.includes('-h') ? 0 : 1);
+}
+
+const dir = args[0];
+const prefixIdx = args.indexOf('--prefix');
+const prefix = prefixIdx !== -1 && args[prefixIdx + 1] ? args[prefixIdx + 1] : 'competitor';
+const seedIdx = args.indexOf('--seed');
+const seeds = seedIdx !== -1 && args[seedIdx + 1]
+  ? args[seedIdx + 1].split(',').map(s => s.trim().toLowerCase()).filter(Boolean)
+  : [];
+const seedSet = new Set(seeds);
+
+// Escape regex metacharacters in the user-supplied prefix so a value like
+// "comp.+" matches the literal filename, not as a regex pattern.
+const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const pattern = new RegExp(`^${escapedPrefix}_discovery_batch_.*\\.json$`);
+
+let files;
+try {
+  files = readdirSync(dir).filter(f => pattern.test(f)).sort();
+} catch (err) {
+  console.error(`Error reading directory ${dir}: ${err.message}`);
+  process.exit(1);
+}
+
+if (files.length === 0) {
+  console.error(`No ${prefix}_discovery_batch_*.json files found in ${dir}`);
+  process.exit(1);
+}
+
+const allResults = [];
+for (const f of files) {
+  try {
+    const d = JSON.parse(readFileSync(join(dir, f), 'utf-8'));
+    const rs = Array.isArray(d) ? d : d.results || [];
+    allResults.push(...rs);
+  } catch {}
+}
+
+// Build a lookup of hostname -> candidate root domain from all result URLs.
+// Used later to try to resolve "serper" -> "serper.dev".
+// Exclude any host whose root-base equals a seed name — otherwise a short extracted token
+// like "exa" can match the user's own domain (exa.ai).
+const hostMap = new Map();
+for (const r of allResults) {
+  if (!r.url) continue;
+  try {
+    const h = new URL(r.url).hostname.replace(/^www\./, '');
+    const root = h.split('.').slice(-2).join('.');
+    const rootBase = root.split('.')[0];
+    if (seedSet.has(rootBase)) continue;
+    if (!hostMap.has(root)) hostMap.set(root, h);
+  } catch {}
+}
+
+// Extract names from "X vs Y" patterns.
+const counts = new Map();
+for (const r of allResults) {
+  const title = (r.title || '').toLowerCase();
+  const ms = [...title.matchAll(/\b([a-z][\w.\-]{2,})\s+(?:vs\.?|versus)\s+([a-z][\w.\-]{2,})/g)];
+  for (const m of ms) {
+    for (const raw of [m[1], m[2]]) {
+      const name = raw.replace(/[^a-z0-9.\-]/g, '').trim();
+      if (!name || name.length < 3) continue;
+      if (seedSet.has(name)) continue;
+      // Reject obvious non-product tokens
+      if (['the', 'and', 'for', 'with', 'best', 'top', 'better', 'using', 'choosing'].includes(name)) continue;
+      if (!counts.has(name)) counts.set(name, { name, hits: 0, example: r.title });
+      counts.get(name).hits += 1;
+    }
+  }
+}
+
+// Try to resolve each name to a domain.
+// Strategy:
+//   1. Exact match on rootBase wins outright.
+//   2. Otherwise allow rootBase.startsWith(needle) ONLY when the suffix is a known
+//      branding token (e.g. "serp" → "serpapi.com"). Bidirectional startsWith
+//      was too loose: "serp" matched serpstack.com, "exa" matched example.com.
+//   3. Among multiple suffix matches, prefer the shortest suffix (most specific —
+//      "serp" should match "serpapi" before "serpapilabs"). Deterministic.
+const BRAND_SUFFIXES = ['api','search','app','ai','io','hq','co','dev','tech','cloud','agent','agents','labs','lab'];
+
+function resolveDomain(name) {
+  const needle = name.replace(/\./g, '');
+  let exact = null;
+  let bestSuffix = null; // { host, suffixLen }
+  for (const [root, host] of hostMap.entries()) {
+    const rootBase = root.split('.')[0];
+    if (rootBase === needle) { exact = host; break; }
+    if (rootBase.length > needle.length && rootBase.startsWith(needle)) {
+      const suffix = rootBase.slice(needle.length).replace(/^[\-_]/, '');
+      if (BRAND_SUFFIXES.includes(suffix)) {
+        if (!bestSuffix || suffix.length < bestSuffix.suffixLen) {
+          bestSuffix = { host, suffixLen: suffix.length };
+        }
+      }
+    }
+  }
+  if (exact) return exact;
+  if (bestSuffix) return bestSuffix.host;
+  return null;
+}
+
+const ranked = [...counts.values()]
+  .map(c => ({ ...c, domain: resolveDomain(c.name) }))
+  .sort((a, b) => b.hits - a.hits);
+
+for (const c of ranked) {
+  console.log(JSON.stringify(c));
+}
+
+console.error(`Extracted ${ranked.length} candidate names from ${files.length} batch files`);

package/bundled-skills/competitor-analysis/scripts/gate_candidates.mjs

@@ -0,0 +1,224 @@
+#!/usr/bin/env node
+
+// Category-fit gate. For each candidate URL, fetch the homepage hero via `browse cloud fetch`,
+// extract visible text, and decide whether the candidate is in the same category as
+// the user's company based on include/exclude keyword rules.
+//
+// Usage:
+//   cat urls.txt | node gate_candidates.mjs \
+//     --include "web search api,neural search,retrieval api,semantic search,search for agents" \
+//     --exclude "vector database,observability,analytics,enterprise search appliance,site search widget" \
+//     --concurrency 6
+//
+// Output: newline-delimited JSON to stdout with one object per URL:
+//   { "url": "https://foo.com", "status": "PASS" | "REJECT" | "UNKNOWN",
+//     "matched_includes": [...], "matched_excludes": [...], "title": "...", "hero": "..." }
+
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+import { readFileSync } from 'fs';
+
+// Async execFile so the worker pool actually parallelizes. spawnSync blocks the entire
+// event loop, which silently turns --concurrency N into N=1 — every URL fetched serially
+// regardless of the flag. With promisified execFile, N workers can wait on N pending
+// `browse cloud fetch` processes concurrently.
+const execFileAsync = promisify(execFile);
+
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h')) {
+  console.error(`Usage: cat urls.txt | node gate_candidates.mjs [options]
+
+Reads URLs from stdin (one per line) OR from --input <file>. For each URL, fetches
+the homepage via \`browse cloud fetch --allow-redirects\`, extracts the first N chars of visible
+text (the hero / tagline area), and classifies against include/exclude keyword rules.
+
+Options:
+  --include "<csv>"    Required. Comma-separated keywords; candidate PASSES if any match.
+  --exclude "<csv>"    Comma-separated keywords; candidate REJECTS if any match.
+  --input <file>       Read URLs from file instead of stdin.
+  --concurrency <n>    Max parallel fetches (default: 6).
+  --hero-chars <n>     Chars of visible text to examine (default: 800).
+  --help, -h           Show this help message.`);
+  process.exit(args.includes('--help') || args.includes('-h') ? 0 : 1);
+}
+
+function flag(name) {
+  const i = args.indexOf(name);
+  return i !== -1 ? args[i + 1] : null;
+}
+
+const includes = (flag('--include') || '').split(',').map(s => s.trim().toLowerCase()).filter(Boolean);
+const excludes = (flag('--exclude') || '').split(',').map(s => s.trim().toLowerCase()).filter(Boolean);
+// Floor at 1: `--concurrency 0` or a non-numeric value makes parseInt yield 0/NaN, which would
+// spawn zero workers — the script would exit "successfully" having gated nothing, making
+// discovery look empty with no error. Always run at least one worker.
+const concurrency = Math.max(1, parseInt(flag('--concurrency') || '6', 10) || 0);
+const heroChars = parseInt(flag('--hero-chars') || '800', 10);
+const inputFile = flag('--input');
+
+function stripHtml(html) {
+  const withoutActiveContent = removeElementContent(removeElementContent(html, 'script'), 'style');
+  return withoutActiveContent
+    .replace(/<[^>]*>/g, ' ')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#x27;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function removeElementContent(html, tagName) {
+  let out = '';
+  let cursor = 0;
+  const lower = html.toLowerCase();
+  const openNeedle = `<${tagName}`;
+  const closeNeedle = `</${tagName}`;
+  while (cursor < html.length) {
+    const start = lower.indexOf(openNeedle, cursor);
+    if (start === -1) {
+      out += html.slice(cursor);
+      break;
+    }
+    out += html.slice(cursor, start);
+    const close = lower.indexOf(closeNeedle, start + openNeedle.length);
+    if (close === -1) {
+      cursor = html.length;
+      break;
+    }
+    const closeEnd = html.indexOf('>', close + closeNeedle.length);
+    cursor = closeEnd === -1 ? html.length : closeEnd + 1;
+    out += ' ';
+  }
+  return out;
+}
+
+if (args.includes('--self-test')) {
+  console.assert(stripHtml('<p>A&amp;lt;B</p>') === 'A&lt;B');
+  console.assert(stripHtml('<script>alert(1)</script ignored><p>ok</p>') === 'ok');
+  process.exit(0);
+}
+
+if (includes.length === 0) {
+  console.error('Error: --include is required');
+  process.exit(1);
+}
+
+let urls;
+if (inputFile) {
+  urls = readFileSync(inputFile, 'utf-8').split('\n').map(l => l.trim()).filter(Boolean);
+} else {
+  const stdin = readFileSync(0, 'utf-8');
+  urls = stdin.split('\n').map(l => l.trim()).filter(Boolean);
+}
+
+if (urls.length === 0) {
+  console.error('Error: no URLs provided (pipe via stdin or use --input)');
+  process.exit(1);
+}
+
+// Position-aware classification:
+//   1. Exclude term in <title>        → REJECT (their primary identity is the excluded category)
+//   2. Include term in <title>        → PASS   (their primary identity matches)
+//   3. Include in early hero (200ch)  → PASS iff no exclude in early hero
+//   4. Otherwise                       → REJECT (default conservative)
+// Rationale: <title> is the single strongest signal of what a company sells.
+// Mid/late hero mentions (e.g. "we also support web scraping use cases") shouldn't
+// disqualify a real competitor that self-identifies in its title as a cloud browser.
+function classify(title, heroFull, includes, excludes) {
+  const titleLower = (title || '').toLowerCase();
+  const heroLower = heroFull.toLowerCase();
+  const heroEarly = heroLower.slice(0, 200);
+
+  const incTitle = includes.filter(k => titleLower.includes(k));
+  const excTitle = excludes.filter(k => titleLower.includes(k));
+  const incEarly = includes.filter(k => heroEarly.includes(k));
+  const excEarly = excludes.filter(k => heroEarly.includes(k));
+  const incHero = includes.filter(k => heroLower.includes(k));
+  const excHero = excludes.filter(k => heroLower.includes(k));
+
+  let status, reason;
+  if (incTitle.length > 0 && excTitle.length > 0) {
+    // Hybrid-identity title (e.g. "Browser Automation & Web Scraping API").
+    // Break the tie by the early hero — whichever category has more mentions wins.
+    if (incEarly.length > excEarly.length)       { status = 'PASS';   reason = `title-hybrid→hero200 leans include(${incEarly[0] || incTitle[0]})`; }
+    else if (excEarly.length > incEarly.length)  { status = 'REJECT'; reason = `title-hybrid→hero200 leans exclude(${excEarly[0] || excTitle[0]})`; }
+    else                                          { status = 'PASS';   reason = `title-hybrid→tie, defaulting include(${incTitle[0]})`; }
+  }
+  else if (excTitle.length > 0)                   { status = 'REJECT'; reason = `title→exclude(${excTitle[0]})`; }
+  else if (incTitle.length > 0)                   { status = 'PASS';   reason = `title→include(${incTitle[0]})`; }
+  else if (incEarly.length > 0 && excEarly.length === 0) { status = 'PASS'; reason = `hero200→include(${incEarly[0]})`; }
+  else if (excEarly.length > 0)                   { status = 'REJECT'; reason = `hero200→exclude(${excEarly[0]})`; }
+  else if (incHero.length > 0 && excHero.length === 0)   { status = 'PASS'; reason = `hero→include(${incHero[0]})`; }
+  // Late-hero conflict: both include AND exclude appear in chars 200–800 (nothing in
+  // title or early hero). This is genuine ambiguous signal, not absence — return UNKNOWN
+  // so the candidate surfaces in the user-confirmation bucket at Step 4.5 instead of
+  // being silently dropped as REJECT.
+  else if (incHero.length > 0 && excHero.length > 0)     { status = 'UNKNOWN'; reason = `hero→conflict(include:${incHero[0]}, exclude:${excHero[0]})`; }
+  else                                            { status = 'REJECT'; reason = 'no category signal'; }
+
+  return {
+    status, reason,
+    matched_includes: [...new Set([...incTitle, ...incEarly, ...incHero])],
+    matched_excludes: [...new Set([...excTitle, ...excEarly, ...excHero])],
+  };
+}
+
+async function gateOne(url) {
+  let stdout;
+  try {
+    // --format raw returns the JSON envelope with raw HTML in `.content` (the default
+    // is markdown, which has no <title> tag for the position-aware classifier to read).
+    const r = await execFileAsync('browse', ['cloud', 'fetch', '--allow-redirects', '--format', 'raw', url], {
+      maxBuffer: 4 * 1024 * 1024,
+      timeout: 20000,
+    });
+    stdout = r.stdout;
+  } catch (err) {
+    // Non-zero exit, timeout, or spawn failure all surface here.
+    return { url, status: 'UNKNOWN', reason: `browse cloud fetch failed: ${err.message}`, matched_includes: [], matched_excludes: [], title: '', hero: '' };
+  }
+  let resp;
+  try { resp = JSON.parse(stdout); } catch {
+    return { url, status: 'UNKNOWN', reason: 'non-JSON response', matched_includes: [], matched_excludes: [], title: '', hero: '' };
+  }
+  const html = resp.content || '';
+  const titleM = html.match(/<title[^>]*>([^<]*)<\/title>/i);
+  const title = titleM ? titleM[1].trim() : '';
+  const heroFull = stripHtml(html).slice(0, heroChars);
+  const c = classify(title, heroFull, includes, excludes);
+  return {
+    url,
+    status: c.status,
+    reason: c.reason,
+    matched_includes: c.matched_includes,
+    matched_excludes: c.matched_excludes,
+    title,
+    hero: heroFull.slice(0, 240),
+  };
+}
+
+// Run with bounded concurrency
+const results = [];
+async function runAll() {
+  const queue = [...urls];
+  const workers = Array(Math.min(concurrency, queue.length)).fill(0).map(async () => {
+    while (queue.length > 0) {
+      const u = queue.shift();
+      const r = await gateOne(u);
+      results.push(r);
+      console.log(JSON.stringify(r));
+    }
+  });
+  await Promise.all(workers);
+}
+
+await runAll();
+
+const pass = results.filter(r => r.status === 'PASS').length;
+const reject = results.filter(r => r.status === 'REJECT').length;
+const unknown = results.filter(r => r.status === 'UNKNOWN').length;
+console.error(`\nGate: ${pass} PASS / ${reject} REJECT / ${unknown} UNKNOWN (of ${results.length})`);

package/bundled-skills/competitor-analysis/scripts/list_urls.mjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+// Deduplicates discovery URLs from `browse cloud search` JSON output files.
+// Usage: node list_urls.mjs /tmp [--prefix competitor]
+// Reads all {prefix}_discovery_batch_*.json files, deduplicates by domain,
+// outputs one URL per line to stdout, stats to stderr.
+
+import { readdirSync, readFileSync } from 'fs';
+import { join } from 'path';
+
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h') || args.length === 0) {
+  console.error(`Usage: node list_urls.mjs <directory> [--prefix <prefix>]
+
+Reads all <prefix>_discovery_batch_*.json files from <directory>,
+deduplicates URLs by domain, and outputs one URL per line to stdout.
+
+Options:
+  --prefix <prefix>  Batch file prefix (default: "competitor")
+  --help, -h         Show this help message
+
+Examples:
+  node list_urls.mjs /tmp
+  node list_urls.mjs /tmp --prefix competitor`);
+  process.exit(args.includes('--help') || args.includes('-h') ? 0 : 1);
+}
+
+const dir = args[0];
+const prefixIdx = args.indexOf('--prefix');
+const prefix = prefixIdx !== -1 && args[prefixIdx + 1] ? args[prefixIdx + 1] : 'competitor';
+
+// Escape regex metacharacters in the user-supplied prefix so a value like
+// "comp.+" matches the literal filename, not as a regex pattern.
+const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const pattern = new RegExp(`^${escapedPrefix}_discovery_batch_.*\\.json$`);
+
+let files;
+try {
+  files = readdirSync(dir)
+    .filter(f => pattern.test(f))
+    .sort();
+} catch (err) {
+  console.error(`Error reading directory ${dir}: ${err.message}`);
+  process.exit(1);
+}
+
+if (files.length === 0) {
+  console.error(`No ${prefix}_discovery_batch_*.json files found in ${dir}`);
+  process.exit(1);
+}
+
+// Dedup by hostname, but prefer the site root over a deep link. The first search hit for a
+// domain is often a blog/doc/comparison path; gating + enrichment want the homepage, so when
+// multiple URLs share a host we keep the shallowest path (fewest segments). First-seen host
+// order is preserved (Map.set on an existing key keeps its position).
+const byDomain = new Map(); // hostname -> { url, depth }
+let totalResults = 0;
+
+for (const file of files) {
+  try {
+    const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
+    const results = Array.isArray(data) ? data : (data.results || []);
+    totalResults += results.length;
+
+    for (const result of results) {
+      const url = result.url;
+      if (!url) continue;
+
+      try {
+        const u = new URL(url);
+        const hostname = u.hostname.replace(/^www\./, '');
+        const depth = u.pathname.replace(/\/+$/, '').split('/').filter(Boolean).length;
+        const existing = byDomain.get(hostname);
+        if (!existing || depth < existing.depth) byDomain.set(hostname, { url, depth });
+      } catch {
+        // Skip invalid URLs
+      }
+    }
+  } catch (err) {
+    console.error(`Warning: Failed to parse ${file}: ${err.message}`);
+  }
+}
+
+const urls = [...byDomain.values()].map(v => v.url);
+for (const url of urls) {
+  console.log(url);
+}
+
+console.error(`\n${files.length} files, ${totalResults} total results, ${urls.length} unique domains`);

package/bundled-skills/competitor-analysis/scripts/md_utils.mjs

@@ -0,0 +1,50 @@
+// Shared markdown parsing helpers for competitor-analysis scripts.
+// Used by compile_report.mjs, merge_partials.mjs, and capture_screenshots.mjs.
+
+// Parses YAML-ish frontmatter delimited by `---` lines.
+// Returns an object of fields, or null if no frontmatter delimiter is found.
+export function parseFrontmatter(content) {
+  content = content.replace(/\r\n/g, '\n'); // tolerate CRLF — anchors below assume LF
+  const m = content.match(/^---\n([\s\S]*?)\n---/);
+  if (!m) return null;
+  const fields = {};
+  for (const line of m[1].split('\n')) {
+    const idx = line.indexOf(':');
+    if (idx > 0) {
+      const k = line.slice(0, idx).trim();
+      const v = line.slice(idx + 1).trim().replace(/^["']|["']$/g, '');
+      if (k && v) fields[k] = v;
+    }
+  }
+  return fields;
+}
+
+// Returns the body text after the closing `---` of the frontmatter, trimmed.
+// If no frontmatter is present, returns the full content trimmed — so callers
+// that don't gate on parseFrontmatter still get usable text.
+export function parseBody(content) {
+  content = content.replace(/\r\n/g, '\n'); // tolerate CRLF — anchors below assume LF
+  const m = content.match(/^---\n[\s\S]*?\n---\n([\s\S]*)/);
+  return m ? m[1].trim() : content.trim();
+}
+
+// Splits a markdown body into sections keyed by `## Heading` line.
+// Content before the first `## ` is dropped (matches existing behavior).
+export function parseSections(body) {
+  const sections = {};
+  const lines = body.replace(/\r\n/g, '\n').split('\n');
+  let currentKey = null;
+  let buffer = [];
+  for (const line of lines) {
+    const m = line.match(/^## (.+)$/);
+    if (m) {
+      if (currentKey !== null) sections[currentKey] = buffer.join('\n').trim();
+      currentKey = m[1].trim();
+      buffer = [];
+    } else if (currentKey !== null) {
+      buffer.push(line);
+    }
+  }
+  if (currentKey !== null) sections[currentKey] = buffer.join('\n').trim();
+  return sections;
+}