crawlforge-mcp-server 4.2.12 → 4.6.0

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -13,7 +13,8 @@ const GenerateLLMsTxtSchema = z.object({
     maxPages: z.number().min(10).max(500).optional().default(100).describe('Maximum pages to analyze'),
     detectAPIs: z.boolean().optional().default(true).describe('Whether to detect API endpoints'),
     analyzeContent: z.boolean().optional().default(true).describe('Whether to analyze content types'),
-    checkSecurity: z.boolean().optional().default(true).describe('Whether to check security boundaries'),
+    checkSecurity: z.boolean().optional().default(false).describe('Whether to probe security-sensitive paths (opt-in; sends requests to /admin, /login, etc.)'),
+    probeRateLimit: z.boolean().optional().default(false).describe('Whether to send repeated probe requests to estimate rate limits (opt-in; fires ~5 requests)'),
     respectRobots: z.boolean().optional().default(true).describe('Whether to respect robots.txt')
   }).optional().default({}),
 
@@ -23,7 +24,8 @@ const GenerateLLMsTxtSchema = z.object({
     contactEmail: z.string().email().optional().describe('Contact email for the LLMs.txt'),
     organizationName: z.string().optional().describe('Organization name'),
     customGuidelines: z.array(z.string()).optional().describe('Additional custom guidelines'),
-    customRestrictions: z.array(z.string()).optional().describe('Additional restrictions')
+    customRestrictions: z.array(z.string()).optional().describe('Additional restrictions'),
+    robotsStyle: z.boolean().optional().default(false).describe('Emit legacy robots.txt-style directives instead of spec-compliant llmstxt.org markdown')
   }).optional().default({}),
 
   complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe('Compliance level for generated guidelines'),
@@ -120,9 +122,128 @@ export class GenerateLLMsTxtTool {
   }
 
   /**
-   * Generate standard LLMs.txt content
+   * Generate LLMs.txt content. By default emits spec-compliant llmstxt.org markdown;
+   * set outputOptions.robotsStyle=true for the legacy robots.txt-style directives.
    */
   generateLLMsTxt(analysis, outputOptions, complianceLevel) {
+    if (outputOptions.robotsStyle) {
+      return this.generateRobotsStyleTxt(analysis, outputOptions, complianceLevel);
+    }
+    return this.generateSpecLLMsTxt(analysis, outputOptions);
+  }
+
+  /**
+   * Generate spec-compliant llms.txt per https://llmstxt.org/ :
+   *   # Title
+   *   > one-line summary (blockquote)
+   *   optional detail paragraph(s)
+   *   ## Section
+   *   - [name](url): optional notes
+   */
+  generateSpecLLMsTxt(analysis, outputOptions) {
+    const lines = [];
+    const baseUrl = analysis.metadata.baseUrl;
+    let host = baseUrl;
+    try { host = new URL(baseUrl).hostname; } catch { /* keep baseUrl */ }
+
+    // H1 title (required)
+    const title = outputOptions.organizationName || host;
+    lines.push(`# ${title}`);
+    lines.push('');
+
+    // Blockquote summary (required by spec)
+    const summary = `Site map and key resources for ${baseUrl}, generated to help LLMs locate relevant content.`;
+    lines.push(`> ${summary}`);
+    lines.push('');
+
+    // Optional detail paragraph(s)
+    const details = [];
+    if (analysis.structure?.totalPages) {
+      details.push(`This site has approximately ${analysis.structure.totalPages} discoverable pages.`);
+    }
+    if (outputOptions.contactEmail) {
+      details.push(`Contact: ${outputOptions.contactEmail}.`);
+    }
+    if (Array.isArray(outputOptions.customGuidelines) && outputOptions.customGuidelines.length > 0) {
+      details.push(...outputOptions.customGuidelines);
+    }
+    if (details.length > 0) {
+      lines.push(details.join(' '));
+      lines.push('');
+    }
+
+    // Helper: emit a "## Section" with a list of [name](url) links.
+    const linkLabel = (u) => {
+      try {
+        const p = new URL(u).pathname.replace(/\/+$/, '');
+        if (!p || p === '') return 'Home';
+        const seg = p.split('/').filter(Boolean).pop() || p;
+        return seg.replace(/[-_]/g, ' ').replace(/\.[a-z0-9]+$/i, '').trim() || p;
+      } catch {
+        return u;
+      }
+    };
+    const emitSection = (heading, urls) => {
+      // Coerce to an array: sitemap/sections may arrive as a flat array, a
+      // grouped object ({path: [...]}), or a single value.
+      let arr = [];
+      if (Array.isArray(urls)) {
+        arr = urls;
+      } else if (urls && typeof urls === 'object') {
+        arr = Object.values(urls).flat();
+      }
+      const list = arr
+        .map((u) => (typeof u === 'string' ? u : (u?.url || u?.loc)))
+        .filter(Boolean)
+        .slice(0, 25);
+      if (list.length === 0) return;
+      lines.push(`## ${heading}`);
+      lines.push('');
+      for (const u of list) {
+        lines.push(`- [${linkLabel(u)}](${u})`);
+      }
+      lines.push('');
+    };
+
+    // Sections derived from the categorized site structure.
+    const sections = analysis.structure?.sections || {};
+    const flatten = (cat) => {
+      const v = sections[cat];
+      if (!Array.isArray(v)) return [];
+      // categorizeSections may produce either flat URLs or {path, urls:[...]} groups.
+      return v.flatMap((entry) =>
+        typeof entry === 'string' ? [entry] : (Array.isArray(entry?.urls) ? entry.urls : []));
+    };
+
+    emitSection('Documentation', flatten('documentation'));
+    emitSection('Content', flatten('content'));
+    emitSection('Tools', flatten('tools'));
+    emitSection('Navigation', flatten('navigation'));
+
+    // APIs as their own section.
+    if (Array.isArray(analysis.apis) && analysis.apis.length > 0) {
+      lines.push('## APIs');
+      lines.push('');
+      for (const api of analysis.apis.slice(0, 25)) {
+        const note = api.type ? `: ${api.type}` : '';
+        lines.push(`- [${linkLabel(api.url)}](${api.url})${note}`);
+      }
+      lines.push('');
+    }
+
+    // Fallback: if no categorized sections produced output, list the raw sitemap.
+    const hasBody = lines.some((l) => l.startsWith('## '));
+    if (!hasBody) {
+      emitSection('Pages', analysis.structure?.sitemap || []);
+    }
+
+    return lines.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
+  }
+
+  /**
+   * Generate legacy robots.txt-style content (opt-in via outputOptions.robotsStyle).
+   */
+  generateRobotsStyleTxt(analysis, outputOptions, complianceLevel) {
     const lines = [];
     const baseUrl = analysis.metadata.baseUrl;
 

package/src/tools/research/deepResearch.js

@@ -379,18 +379,57 @@ export class DeepResearchTool {
    * Format research results according to output preferences
    */
   formatResults(results, params) {
-    // Raw evidence mode (no LLM configured): pass through the clean shape
-    // designed for the calling LLM to synthesize.
+    // Raw evidence mode (no LLM configured): apply lightweight formatting so
+    // outputFormat is not silently ignored, and rank sources by credibility.
     if (results.synthesisMode === 'raw_evidence') {
-      return {
+      const rankedSources = (results.sources || [])
+        .slice()
+        .sort((a, b) => (b.credibility || 0) - (a.credibility || 0));
+
+      const base = {
         synthesisMode: 'raw_evidence',
         note: results.note,
-        sources: results.sources,
         researchSummary: results.researchSummary,
         metadata: results.metadata,
         performance: results.performance,
         activityLog: params.includeActivityLog ? results.activityLog : undefined
       };
+
+      switch (params.outputFormat) {
+        case 'summary':
+          return {
+            ...base,
+            sources: rankedSources.slice(0, 5)
+          };
+
+        case 'citations_only':
+          return {
+            ...base,
+            sources: rankedSources.map(s => ({
+              title: s.title,
+              url: s.url,
+              credibility: s.credibility
+            })),
+            citationCount: rankedSources.length,
+            citationSummary: this.generateCitationSummary(rankedSources)
+          };
+
+        case 'conflicts_focus':
+          // Without LLM there is no conflict detection; return ranked sources
+          // with a note so the caller knows what happened.
+          return {
+            ...base,
+            sources: rankedSources,
+            conflictsNote: 'Conflict detection requires an LLM (OPENAI_API_KEY or ANTHROPIC_API_KEY). Sources are ranked by credibility for manual review.'
+          };
+
+        case 'comprehensive':
+        default:
+          return {
+            ...base,
+            sources: rankedSources
+          };
+      }
     }
 
     const formatted = {

package/src/tools/scrape/unifiedScrape.js

@@ -0,0 +1,314 @@
+/**
+ * unifiedScrape — single-fetch, multi-format scraping tool.
+ *
+ * One call, one fetch.  formats[] drives what is returned.
+ * Mirrors the output shape of ScrapeWithActionsTool.generateFormats():
+ *   content.html, content.rawHtml, content.text, content.markdown,
+ *   content.links, content.metadata, content.screenshots, content.json
+ *
+ * onlyMainContent maps to Readability boilerplate removal (same as extractContent).
+ * Partial success: per-format warnings[] never fail the whole call.
+ */
+
+import { z } from 'zod';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+import { fetchAndParse } from '../extract/_fetchAndParse.js';
+import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+import { extractBlockText, readabilityToMarkdown } from '../basic/extractText.js';
+
+// ── Schema ────────────────────────────────────────────────────────────────────
+
+const JsonFormatSchema = z.object({
+  type: z.literal('json'),
+  schema: z.record(z.any()).optional(),
+  prompt: z.string().optional()
+});
+
+const FormatSchema = z.union([
+  z.enum(['markdown', 'html', 'rawHtml', 'text', 'links', 'metadata', 'screenshot']),
+  JsonFormatSchema
+]);
+
+export const UnifiedScrapeSchema = z.object({
+  url: z.string().url(),
+  formats: z.array(FormatSchema).min(1).default(['markdown']),
+  onlyMainContent: z.boolean().optional().default(true),
+  // Pass-through to fetchAndParse
+  timeoutMs: z.number().min(1000).max(60000).optional().default(15000)
+});
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+/**
+ * Extract links from a loaded cheerio $ and the page URL.
+ */
+function extractLinksFromDom($, pageUrl) {
+  const links = [];
+  const seen = new Set();
+  let pageOrigin = '';
+  try { pageOrigin = new URL(pageUrl).origin; } catch { /* ignore */ }
+
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    const text = $(el).text().trim();
+    if (!href) return;
+    try {
+      let absoluteUrl;
+      let isExternal = false;
+      if (href.startsWith('http://') || href.startsWith('https://')) {
+        absoluteUrl = href;
+        isExternal = new URL(href).origin !== pageOrigin;
+      } else if (href.startsWith('#') || href.startsWith('javascript:')) {
+        return;
+      } else {
+        absoluteUrl = new URL(href, pageUrl).toString();
+        isExternal = false;
+      }
+      if (!seen.has(absoluteUrl)) {
+        seen.add(absoluteUrl);
+        links.push({ href: absoluteUrl, text, is_external: isExternal, original_href: href });
+      }
+    } catch { /* skip invalid */ }
+  });
+
+  return {
+    links,
+    total_count: links.length,
+    internal_count: links.filter(l => !l.is_external).length,
+    external_count: links.filter(l => l.is_external).length
+  };
+}
+
+/**
+ * Extract metadata from a loaded cheerio $.
+ */
+function extractMetadataFromDom($, pageUrl) {
+  // JSON-LD
+  const jsonLd = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    try { const raw = $(el).html(); if (raw) jsonLd.push(JSON.parse(raw)); } catch { /* skip */ }
+  });
+
+  // Microdata
+  const microdata = [];
+  $('[itemscope]').each((_, el) => {
+    const $el = $(el);
+    const item = { type: $el.attr('itemtype') || null, properties: {} };
+    $el.find('[itemprop]').each((_, prop) => {
+      const $prop = $(prop);
+      const name = $prop.attr('itemprop');
+      if (!name) return;
+      const tag = ($prop.get(0).tagName || '').toLowerCase();
+      let value;
+      if (tag === 'meta') value = $prop.attr('content');
+      else if (tag === 'a' || tag === 'link') value = $prop.attr('href');
+      else if (tag === 'img') value = $prop.attr('src');
+      else if (tag === 'time') value = $prop.attr('datetime') || $prop.text().trim();
+      else value = $prop.text().trim();
+      if (value) {
+        if (!item.properties[name]) item.properties[name] = [];
+        item.properties[name].push(value);
+      }
+    });
+    microdata.push(item);
+  });
+
+  const title =
+    $('meta[property="og:title"]').attr('content') ||
+    $('title').text().trim() ||
+    $('h1').first().text().trim() || '';
+
+  const ogTags = {};
+  $('meta[property^="og:"]').each((_, el) => {
+    const property = $(el).attr('property');
+    const content = $(el).attr('content');
+    if (property && content) ogTags[property.replace('og:', '')] = content;
+  });
+
+  const twitterTags = {};
+  $('meta[name^="twitter:"]').each((_, el) => {
+    const name = $(el).attr('name');
+    const content = $(el).attr('content');
+    if (name && content) twitterTags[name.replace('twitter:', '')] = content;
+  });
+
+  return {
+    title,
+    description: $('meta[name="description"]').attr('content') || $('meta[property="og:description"]').attr('content') || '',
+    keywords: ($('meta[name="keywords"]').attr('content') || '').split(',').map(k => k.trim()).filter(Boolean),
+    canonical_url: $('link[rel="canonical"]').attr('href') || '',
+    author: $('meta[name="author"]').attr('content') || '',
+    robots: $('meta[name="robots"]').attr('content') || '',
+    viewport: $('meta[name="viewport"]').attr('content') || '',
+    og_tags: ogTags,
+    twitter_tags: twitterTags,
+    json_ld: jsonLd,
+    microdata,
+    url: pageUrl
+  };
+}
+
+// ── Tool class ────────────────────────────────────────────────────────────────
+
+export class UnifiedScrapeTool {
+  constructor(options = {}) {
+    this._extractWithLlm = null;
+    this._extractWithLlmConfig = options.llmConfig || {};
+  }
+
+  /** Lazy-load ExtractWithLlm to avoid pulling in heavy deps unless needed. */
+  async _getExtractWithLlm() {
+    if (!this._extractWithLlm) {
+      const { ExtractWithLlm } = await import('../extract/extractWithLlm.js');
+      this._extractWithLlm = new ExtractWithLlm(this._extractWithLlmConfig);
+    }
+    return this._extractWithLlm;
+  }
+
+  /**
+   * Execute a unified scrape.
+   * @param {object} params - UnifiedScrapeSchema-compatible input
+   * @returns {Promise<object>}
+   */
+  async execute(params) {
+    const validated = UnifiedScrapeSchema.parse(params);
+    const { url, formats, onlyMainContent, timeoutMs } = validated;
+
+    // Single fetch
+    let html, $, finalUrl;
+    try {
+      ({ html, $, finalUrl } = await fetchAndParse(url, {
+        timeoutMs,
+        stripTags: [] // we handle boilerplate ourselves
+      }));
+    } catch (err) {
+      throw new Error(`scrape: fetch failed for ${url}: ${err.message}`);
+    }
+
+    // For onlyMainContent: extract main-content html via Readability once
+    let mainHtml = null;
+    function getMainHtml() {
+      if (mainHtml !== null) return mainHtml;
+      try {
+        const dom = new JSDOM(html, { url: finalUrl });
+        const reader = new Readability(dom.window.document);
+        const article = reader.parse();
+        mainHtml = article ? article.content : html;
+      } catch {
+        mainHtml = html;
+      }
+      return mainHtml;
+    }
+
+    const content = {};
+    const warnings = [];
+
+    for (const fmt of formats) {
+      // JSON format object
+      if (fmt && typeof fmt === 'object' && fmt.type === 'json') {
+        try {
+          const extractWithLlm = await this._getExtractWithLlm();
+          const text = onlyMainContent
+            ? htmlToMarkdown(getMainHtml())
+            : $('body').text().replace(/\s+/g, ' ').trim();
+          const result = await extractWithLlm.execute({
+            content: text,
+            prompt: fmt.prompt || 'Extract structured data from this page content.',
+            schema: fmt.schema,
+            provider: 'auto'
+          });
+          content.json = result.success ? result.data : { error: result.error };
+          if (!result.success) {
+            warnings.push(`json: extraction failed — ${result.error}`);
+          }
+        } catch (err) {
+          content.json = { error: err.message };
+          warnings.push(`json: ${err.message}`);
+        }
+        continue;
+      }
+
+      // String formats
+      switch (fmt) {
+        case 'markdown':
+          try {
+            content.markdown = onlyMainContent
+              ? readabilityToMarkdown(html, finalUrl)
+              : htmlToMarkdown($.html('body') || html);
+          } catch (err) {
+            content.markdown = '';
+            warnings.push(`markdown: ${err.message}`);
+          }
+          break;
+
+        case 'html':
+          try {
+            content.html = onlyMainContent ? getMainHtml() : $.html('body') || html;
+          } catch (err) {
+            content.html = '';
+            warnings.push(`html: ${err.message}`);
+          }
+          break;
+
+        case 'rawHtml':
+          content.rawHtml = html;
+          break;
+
+        case 'text':
+          try {
+            if (onlyMainContent) {
+              // Plain text from Readability main content via cheerio
+              const { load } = await import('cheerio');
+              const $main = load(getMainHtml());
+              $main('script, style').remove();
+              content.text = extractBlockText($main);
+            } else {
+              $('script, style').remove();
+              content.text = extractBlockText($);
+            }
+          } catch (err) {
+            content.text = '';
+            warnings.push(`text: ${err.message}`);
+          }
+          break;
+
+        case 'links':
+          try {
+            content.links = extractLinksFromDom($, finalUrl);
+          } catch (err) {
+            content.links = { links: [], total_count: 0, internal_count: 0, external_count: 0 };
+            warnings.push(`links: ${err.message}`);
+          }
+          break;
+
+        case 'metadata':
+          try {
+            content.metadata = extractMetadataFromDom($, finalUrl);
+          } catch (err) {
+            content.metadata = {};
+            warnings.push(`metadata: ${err.message}`);
+          }
+          break;
+
+        case 'screenshot':
+          // Screenshot requires a browser; not available in the basic scrape path.
+          content.screenshots = [];
+          warnings.push('screenshot: browser screenshots are not available in the scrape tool; use scrape_with_actions for screenshots');
+          break;
+
+        default:
+          warnings.push(`unknown format: ${String(fmt)}`);
+      }
+    }
+
+    return {
+      success: true,
+      url: finalUrl,
+      content,
+      warnings: warnings.length > 0 ? warnings : undefined
+    };
+  }
+}
+
+export default UnifiedScrapeTool;

package/src/tools/search/providers/searxng.js

@@ -117,8 +117,8 @@ export async function searchViaSearxng(opts = {}) {
   return {
     items,
     searchInformation: {
-      totalResults: String(rawResults.length),
-      searchTime: data.answers ? 0 : 0
+      totalResults: rawResults.length,
+      searchTime: 0
     },
     queries: {},
     context: {}

package/src/tools/search/ranking/ResultDeduplicator.js

@@ -545,27 +545,50 @@ export class ResultDeduplicator {
   }
 
   /**
-   * SimHash implementation for content similarity
+   * SimHash implementation for content similarity.
+   * Uses two independent 32-bit FNV-1a hashes (seeded differently) to produce
+   * independent high/low 32-bit words, giving a true 64-bit fingerprint so that
+   * bits 32-63 are not a duplicate of bits 0-31.
    */
   simHash(text, bits = 64) {
     const tokens = text.split(/\s+/);
     const hashBits = new Array(bits).fill(0);
-    
+
     for (const token of tokens) {
-      const hash = this.stringHash(token);
-      
-      for (let i = 0; i < bits; i++) {
-        const bit = (hash >> i) & 1;
-        hashBits[i] += bit ? 1 : -1;
+      const lo = this._fnv1a32(token, 0x811c9dc5);
+      const hi = this._fnv1a32(token, 0x84222325); // different seed
+
+      for (let i = 0; i < 32; i++) {
+        hashBits[i]      += ((lo >>> i) & 1) ? 1 : -1;
+      }
+      for (let i = 0; i < 32; i++) {
+        hashBits[32 + i] += ((hi >>> i) & 1) ? 1 : -1;
       }
     }
-    
+
     // Convert to binary string
     return hashBits.map(bit => bit > 0 ? '1' : '0').join('');
   }
 
   /**
-   * String hash function
+   * FNV-1a 32-bit hash with a configurable offset basis (seed).
+   * @param {string} str
+   * @param {number} seed - 32-bit unsigned offset basis
+   * @returns {number} 32-bit unsigned integer
+   */
+  _fnv1a32(str, seed) {
+    const FNV_PRIME = 0x01000193;
+    let hash = seed >>> 0;
+    for (let i = 0; i < str.length; i++) {
+      hash ^= str.charCodeAt(i);
+      // Multiply by FNV prime using 32-bit arithmetic
+      hash = Math.imul(hash, FNV_PRIME) >>> 0;
+    }
+    return hash;
+  }
+
+  /**
+   * String hash function (kept for hashResults / cache-key use)
    */
   stringHash(str) {
     let hash = 0;

package/src/tools/search/ranking/ResultRanker.js

@@ -191,18 +191,27 @@ export class ResultRanker {
     // Calculate term frequencies
     const termFreqs = this.getTermFrequencies(contentTerms);
 
+    // Build per-term document frequency across all results
+    const docFreqs = {};
+    for (const r of allResults) {
+      const rContent = [r.title || '', r.snippet || '', r.htmlSnippet || ''].join(' ');
+      const rTerms = new Set(this.tokenize(rContent.toLowerCase()));
+      for (const t of rTerms) {
+        docFreqs[t] = (docFreqs[t] || 0) + 1;
+      }
+    }
+
     let score = 0;
     for (const term of queryTerms) {
       const tf = termFreqs[term] || 0;
       if (tf > 0) {
-        // Document frequency (simplified - assume term appears in some docs)
-        const df = Math.min(allResults.length * 0.1, 1); // Conservative estimate
+        const df = docFreqs[term] || 1;
         const idf = Math.log((allResults.length - df + 0.5) / (df + 0.5));
-        
+
         // BM25 formula
         const numerator = tf * (k1 + 1);
         const denominator = tf + k1 * (1 - b + b * (contentLength / avgDocLength));
-        
+
         score += idf * (numerator / denominator);
       }
     }

package/src/tools/search/searchWeb.js

@@ -280,14 +280,14 @@ export class SearchWebTool {
       // Clean up results based on detail level requested
       if (!validated.include_ranking_details) {
         processedResults = processedResults.map(result => {
-          const { rankingDetails, ...cleanResult } = result;
+          const { rankingDetails, finalScore, originalIndex, scores, ...cleanResult } = result;
           return cleanResult;
         });
       }
-      
+
       if (!validated.include_deduplication_details) {
         processedResults = processedResults.map(result => {
-          const { deduplicationInfo, ...cleanResult } = result;
+          const { deduplicationInfo, contentHash, normalizedUrl, titleTokens, ...cleanResult } = result;
           return cleanResult;
         });
       }
@@ -407,10 +407,10 @@ export class SearchWebTool {
     }
 
     if (!validated.include_ranking_details) {
-      processedResults = processedResults.map(({ rankingDetails, ...r }) => r);
+      processedResults = processedResults.map(({ rankingDetails, finalScore, originalIndex, scores, ...r }) => r);
     }
     if (!validated.include_deduplication_details) {
-      processedResults = processedResults.map(({ deduplicationInfo: _d, ...r }) => r);
+      processedResults = processedResults.map(({ deduplicationInfo: _d, contentHash, normalizedUrl, titleTokens, ...r }) => r);
     }
 
     return {

package/src/tools/templates/TemplateRegistry.js

@@ -162,8 +162,9 @@ const TEMPLATES = [
       const stories = [];
       $('tr.athing').each((_, el) => {
         const $row = $(el);
-        const $score = $row.next('.spacer').find('.score');
-        const $subtext = $row.next('.spacer').find('.subtext');
+        // The metadata row (".subtext") is the sibling row immediately after tr.athing.
+        const $subtext = $row.next('tr').find('.subtext');
+        const $score = $subtext.find('.score');
         const $titleLink = $row.find('.titleline > a');
         stories.push({
           id: $row.attr('id'),