freshcontext-mcp 0.2.0 → 0.3.1

package/add-cache.cjs

@@ -0,0 +1,86 @@
+const fs = require('fs');
+let content = fs.readFileSync('./worker/src/worker.ts', 'utf8');
+
+// Tools that need withCache wrapping (github already done)
+// For each: find the async handler start and add withCache open
+// Then find the corresponding catch line and add the closing
+
+const tools = [
+  { name: 'extract_hackernews',  adapter: 'hackernews',    input: 'url' },
+  { name: 'extract_scholar',     adapter: 'scholar',       input: 'url' },
+  { name: 'extract_yc',          adapter: 'yc',            input: 'url' },
+  { name: 'search_repos',        adapter: 'reposearch',    input: 'query' },
+  { name: 'package_trends',      adapter: 'packagetrends', input: 'packages' },
+  { name: 'extract_reddit',      adapter: 'reddit',        input: 'url' },
+  { name: 'extract_producthunt', adapter: 'producthunt',   input: 'url' },
+  { name: 'extract_finance',     adapter: 'finance',       input: 'url' },
+  { name: 'search_jobs',         adapter: 'jobs',          input: 'query' },
+  { name: 'extract_landscape',   adapter: 'landscape',     input: 'topic' },
+];
+
+// We'll work line-by-line for precision
+const lines = content.split('\n');
+const result = [];
+let i = 0;
+
+while (i < lines.length) {
+  const line = lines[i];
+
+  // Check if this line closes a tool handler that needs wrapping
+  // Pattern: "    } catch (err: any) { return ..." followed by "  });"
+  // We add "    }); // end withCache" before the closing  });
+  if (
+    line.match(/^\s+\} catch \(err: any\) \{ return \{ content/) &&
+    lines[i + 1] && lines[i + 1].trim() === '});' &&
+    !(lines[i + 1].includes('end withCache'))
+  ) {
+    result.push(line);
+    result.push('    }); // end withCache');
+    i++;
+    continue;
+  }
+
+  result.push(line);
+  i++;
+}
+
+content = result.join('\n');
+
+// Now add the withCache OPEN for each tool
+// Pattern to find: the }, async ({ INPUT }) => {\n    try {
+// for each tool, but only if not already wrapped
+
+for (const tool of tools) {
+  // Build a specific marker string that is unique per tool
+  // Look for the line: "  }, async ({ url/query/etc }) => {"
+  // followed by next non-empty content being "    try {"
+  // We replace "    try {" with "    return withCache(...); try {"
+  
+  // Find the tool's registerTool call, then find the first "try {" after it
+  const toolMarker = `server.registerTool("${tool.name}"`;
+  const idx = content.indexOf(toolMarker);
+  if (idx === -1) { console.log('Tool not found: ' + tool.name); continue; }
+
+  // From that position, find the first "    try {" that is NOT already preceded by withCache
+  const afterTool = content.slice(idx);
+  const tryIdx = afterTool.indexOf('\n    try {');
+  if (tryIdx === -1) { console.log('No try block for: ' + tool.name); continue; }
+
+  // Check if withCache is already there (look 200 chars before the try)
+  const beforeTry = afterTool.slice(Math.max(0, tryIdx - 200), tryIdx);
+  if (beforeTry.includes('withCache')) {
+    console.log('Already wrapped: ' + tool.name);
+    continue;
+  }
+
+  // Insert the withCache line before "    try {"
+  const absoluteIdx = idx + tryIdx;
+  content = content.slice(0, absoluteIdx) +
+    `\n    return withCache("${tool.adapter}", ${tool.input}, env.CACHE, async () => {` +
+    content.slice(absoluteIdx);
+
+  console.log('Wrapped: ' + tool.name);
+}
+
+fs.writeFileSync('./worker/src/worker.ts', content, 'utf8');
+console.log('\nAll done. Lines: ' + content.split('\n').length);

package/dist/adapters/jobs.js

@@ -1,19 +1,18 @@
 /**
- * Jobs adapter v2 — 4 sources, freshness badges, location + keyword filtering.
+ * Jobs adapter v3 — 5 sources, freshness badges, location + keyword filtering.
  *
  * Sources (all no-auth):
- *   - Remotive       — remote jobs, location filterable
- *   - RemoteOK       — pure remote, salary data, unix timestamps
- *   - The Muse       — structured jobs, location + category
- *   - HN Who is Hiring — community-sourced, raw but real
+ *   - Remotive       — remote tech jobs, salary data
+ *   - RemoteOK       — pure remote, unix timestamps
+ *   - Arbeitnow      — broad jobs API (tech + non-tech, location-aware)
+ *   - The Muse       — structured listings, level info
+ *   - HN Who is Hiring — monthly thread, community-sourced
  *
- * Every listing gets a freshness badge:
+ * Freshness badges on every listing:
  *   🟢 < 7 days     — FRESH, apply now
  *   🟡 7–30 days    — still good
  *   🔴 31–90 days   — apply with caution
  *   ⛔ > 90 days    — likely expired, shown last
- *
- * This adapter was the whole reason freshcontext exists.
  */
 // ─── Freshness Scoring ────────────────────────────────────────────────────────
 function freshnessBadge(dateStr) {
@@ -35,7 +34,11 @@ function matchesLocation(locationField, filterLocation) {
         return true;
     const loc = locationField.toLowerCase();
     const filter = filterLocation.toLowerCase();
-    return loc.includes(filter) || filter.includes(loc) || loc.includes("worldwide") || loc.includes("anywhere") || loc.includes("remote");
+    return (loc.includes(filter) ||
+        filter.includes(loc) ||
+        loc.includes("worldwide") ||
+        loc.includes("anywhere") ||
+        loc.includes("remote"));
 }
 function highlightKeywords(text, keywords) {
     if (!keywords.length)
@@ -55,40 +58,58 @@ export async function jobsAdapter(options) {
     const remoteOnly = options.remoteOnly ?? false;
     const maxAgeDays = options.maxAgeDays ?? 60;
     const keywords = options.keywords ?? [];
-    const perSource = Math.floor(maxLength / 4);
-    const [remotiveRes, remoteOkRes, museRes, hnRes] = await Promise.allSettled([
-        fetchRemotive(query, location, maxAgeDays, keywords, perSource),
-        fetchRemoteOK(query, location, maxAgeDays, keywords, perSource),
-        remoteOnly ? Promise.reject("skipped (remote_only mode)") : fetchMuse(query, location, maxAgeDays, keywords, perSource),
-        fetchHNHiring(query, location, maxAgeDays, keywords, perSource),
+    const [remotiveRes, remoteOkRes, arbeitnowRes, museRes, hnRes] = await Promise.allSettled([
+        fetchRemotive(query, location, maxAgeDays, keywords),
+        fetchRemoteOK(query, location, maxAgeDays, keywords),
+        fetchArbeitnow(query, location, maxAgeDays, keywords, remoteOnly),
+        remoteOnly ? Promise.reject("skipped") : fetchMuse(query, location, maxAgeDays, keywords),
+        fetchHNHiring(query, location, maxAgeDays, keywords),
     ]);
     const pool = [];
+    const sourceStats = {};
     const harvest = (res, label) => {
-        if (res.status === "fulfilled")
+        if (res.status === "fulfilled") {
             pool.push(...res.value.listings);
-        // silently skip rejected sources — don't clutter output
+            sourceStats[label] = res.value.listings.length;
+        }
+        else {
+            sourceStats[label] = 0;
+        }
     };
     harvest(remotiveRes, "Remotive");
     harvest(remoteOkRes, "RemoteOK");
+    harvest(arbeitnowRes, "Arbeitnow");
     harvest(museRes, "The Muse");
-    harvest(hnRes, "HN");
+    harvest(hnRes, "HN Hiring");
     if (!pool.length) {
         return {
-            raw: `No job listings found for "${query}"${location ? ` in ${location}` : ""}.\n\nTips:\n• Try broader terms e.g. "engineer" instead of "senior TypeScript engineer"\n• Set location to "remote" for worldwide results\n• Increase max_age_days`,
+            raw: [
+                `No job listings found for "${query}"${location ? ` in ${location}` : ""}.`,
+                "",
+                "Tips:",
+                "• Try broader terms e.g. \"engineer\" instead of \"senior TypeScript engineer\"",
+                "• Set location to \"remote\" for worldwide results",
+                "• Increase max_age_days (default: 60)",
+                "• Note: FIFO/mining/trades jobs are on specialist boards (myJobsNamibia, SEEK, mining-specific sites) — these sources are tech/remote focused",
+            ].join("\n"),
             content_date: null,
             freshness_confidence: "low",
         };
     }
-    // Sort: freshest first, expired listings last
+    // Sort: freshest first
     pool.sort((a, b) => a.days - b.days);
     const freshCount = pool.filter(l => l.days <= 7).length;
     const goodCount = pool.filter(l => l.days > 7 && l.days <= 30).length;
     const staleCount = pool.filter(l => l.days > 30).length;
+    const sourceSummary = Object.entries(sourceStats)
+        .map(([src, n]) => `${src}:${n}`)
+        .join("  ");
     const header = [
         `# Job Search: "${query}"${location ? ` · ${location}` : ""}${remoteOnly ? " · remote only" : ""}`,
         `Retrieved: ${new Date().toISOString()}`,
         `Found: ${pool.length} listings — 🟢 ${freshCount} fresh  🟡 ${goodCount} recent  🔴 ${staleCount} older`,
-        `⚠️  Listings sorted freshest first. Check the date badge before you apply.`,
+        `Sources: ${sourceSummary}`,
+        `⚠️  Sorted freshest first. Check badge before applying.`,
         keywords.length ? `🔍 Watching for: ${keywords.map(k => `⚡${k}`).join(", ")}` : null,
         "",
     ].filter(Boolean).join("\n");
@@ -107,10 +128,10 @@ export async function jobsAdapter(options) {
     };
 }
 // ─── Source: Remotive ─────────────────────────────────────────────────────────
-async function fetchRemotive(query, location, maxAgeDays, keywords, maxLength) {
+async function fetchRemotive(query, location, maxAgeDays, keywords) {
     const url = `https://remotive.com/api/remote-jobs?search=${encodeURIComponent(query)}&limit=15`;
     const res = await fetch(url, {
-        headers: { "User-Agent": "freshcontext-mcp/0.2.0", "Accept": "application/json" },
+        headers: { "User-Agent": "freshcontext-mcp/0.3.0", "Accept": "application/json" },
     });
     if (!res.ok)
         throw new Error(`Remotive ${res.status}`);
@@ -123,7 +144,7 @@ async function fetchRemotive(query, location, maxAgeDays, keywords, maxLength) {
             return null;
         const text = highlightKeywords([
             `[Remotive] ${j.title} — ${j.company_name}`,
-            `${badge}`,
+            badge,
             `Location: ${j.candidate_required_location || "Remote"} | Type: ${j.job_type || "N/A"}`,
             j.salary ? `Salary: ${j.salary}` : null,
             j.tags?.length ? `Tags: ${j.tags.slice(0, 6).join(", ")}` : null,
@@ -136,22 +157,21 @@ async function fetchRemotive(query, location, maxAgeDays, keywords, maxLength) {
     return { listings };
 }
 // ─── Source: RemoteOK ─────────────────────────────────────────────────────────
-async function fetchRemoteOK(query, location, maxAgeDays, keywords, maxLength) {
+async function fetchRemoteOK(query, location, maxAgeDays, keywords) {
     const tag = query.toLowerCase().replace(/\s+/g, "-");
     const url = `https://remoteok.com/api?tag=${encodeURIComponent(tag)}`;
     const res = await fetch(url, {
-        headers: { "User-Agent": "freshcontext-mcp/0.2.0", "Accept": "application/json" },
+        headers: { "User-Agent": "freshcontext-mcp/0.3.0", "Accept": "application/json" },
     });
     if (!res.ok)
         throw new Error(`RemoteOK ${res.status}`);
     const raw = await res.json();
-    // First element is a legal notice object, skip it
     const jobs = raw.filter(j => j.id && j.position);
     const listings = jobs
         .filter(j => matchesLocation(j.location ?? "Remote", location))
         .map(j => {
         const dateStr = j.date ?? (j.epoch ? new Date(j.epoch * 1000).toISOString() : null);
-        const { badge, days } = freshnessDate(dateStr);
+        const { badge, days } = freshnessBadge(dateStr);
         if (days > maxAgeDays)
             return null;
         const salary = j.salary_min && j.salary_max
@@ -159,7 +179,7 @@ async function fetchRemoteOK(query, location, maxAgeDays, keywords, maxLength) {
             : null;
         const text = highlightKeywords([
             `[RemoteOK] ${j.position} — ${j.company ?? "Unknown"}`,
-            `${badge}`,
+            badge,
             `Location: ${j.location || "Remote Worldwide"}`,
             salary ? `Salary: ${salary}` : null,
             j.tags?.length ? `Tags: ${j.tags.slice(0, 6).join(", ")}` : null,
@@ -171,14 +191,49 @@ async function fetchRemoteOK(query, location, maxAgeDays, keywords, maxLength) {
         .slice(0, 8);
     return { listings };
 }
+// ─── Source: Arbeitnow (NEW) ──────────────────────────────────────────────────
+// Free public API — broader than tech boards. Good for non-remote, non-tech roles.
+async function fetchArbeitnow(query, location, maxAgeDays, keywords, remoteOnly) {
+    const params = new URLSearchParams({ search: query });
+    if (remoteOnly)
+        params.set("remote", "true");
+    const url = `https://arbeitnow.com/api/job-board-api?${params.toString()}`;
+    const res = await fetch(url, {
+        headers: { "User-Agent": "freshcontext-mcp/0.3.0", "Accept": "application/json" },
+    });
+    if (!res.ok)
+        throw new Error(`Arbeitnow ${res.status}`);
+    const data = await res.json();
+    const listings = (data.data ?? [])
+        .filter(j => matchesLocation(j.location ?? "", location))
+        .map(j => {
+        const dateStr = new Date(j.created_at * 1000).toISOString();
+        const { badge, days } = freshnessBadge(dateStr);
+        if (days > maxAgeDays)
+            return null;
+        const text = highlightKeywords([
+            `[Arbeitnow] ${j.title} — ${j.company_name}`,
+            badge,
+            `Location: ${j.location || "Remote"}${j.remote ? " (Remote OK)" : ""}`,
+            j.job_types?.length ? `Type: ${j.job_types.join(", ")}` : null,
+            j.tags?.length ? `Tags: ${j.tags.slice(0, 6).join(", ")}` : null,
+            `Apply: ${j.url}`,
+        ].filter(Boolean).join("\n"), keywords);
+        return { text, days, source: "arbeitnow" };
+    })
+        .filter((l) => l !== null)
+        .slice(0, 8);
+    return { listings };
+}
 // ─── Source: The Muse ─────────────────────────────────────────────────────────
-async function fetchMuse(query, location, maxAgeDays, keywords, maxLength) {
+// Fixed: now uses `name` param for text search instead of `category`
+async function fetchMuse(query, location, maxAgeDays, keywords) {
     const locParam = location && location.toLowerCase() !== "remote"
         ? `&location=${encodeURIComponent(location)}`
         : "&location=Flexible%20%2F%20Remote";
-    const url = `https://www.themuse.com/api/public/jobs?category=${encodeURIComponent(query)}${locParam}&page=0&descending=true`;
+    const url = `https://www.themuse.com/api/public/jobs?name=${encodeURIComponent(query)}${locParam}&page=0&descending=true`;
     const res = await fetch(url, {
-        headers: { "User-Agent": "freshcontext-mcp/0.2.0", "Accept": "application/json" },
+        headers: { "User-Agent": "freshcontext-mcp/0.3.0", "Accept": "application/json" },
     });
     if (!res.ok)
         throw new Error(`The Muse ${res.status}`);
@@ -186,13 +241,13 @@ async function fetchMuse(query, location, maxAgeDays, keywords, maxLength) {
     const listings = (data.results ?? [])
         .map(j => {
         const locationStr = j.locations?.map(l => l.name).join(", ") || "Flexible/Remote";
-        const { badge, days } = freshnessDate(j.publication_date);
+        const { badge, days } = freshnessBadge(j.publication_date);
         if (days > maxAgeDays)
             return null;
         const level = j.levels?.map(l => l.name).join(", ") || null;
         const text = highlightKeywords([
             `[The Muse] ${j.name} — ${j.company?.name ?? "Unknown"}`,
-            `${badge}`,
+            badge,
             `Location: ${locationStr}`,
             level ? `Level: ${level}` : null,
             `Apply: ${j.refs?.landing_page ?? "N/A"}`,
@@ -204,31 +259,46 @@ async function fetchMuse(query, location, maxAgeDays, keywords, maxLength) {
     return { listings };
 }
 // ─── Source: HN Who is Hiring ─────────────────────────────────────────────────
-async function fetchHNHiring(query, location, maxAgeDays, keywords, maxLength) {
-    const searchQ = [query, location, "hiring"].filter(Boolean).join(" ");
-    const url = `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(searchQ)}&tags=comment&hitsPerPage=10`;
-    const res = await fetch(url, { headers: { "User-Agent": "freshcontext-mcp/0.2.0" } });
-    if (!res.ok)
-        throw new Error(`HN ${res.status}`);
-    const data = await res.json();
-    const listings = (data.hits ?? [])
+// Fixed: now searches within the actual monthly "Who is Hiring" thread
+// instead of all HN comments. Uses the parent_id filter to target the thread.
+async function fetchHNHiring(query, location, maxAgeDays, keywords) {
+    // Step 1: Find the most recent "Ask HN: Who is hiring?" thread
+    const threadRes = await fetch(`https://hn.algolia.com/api/v1/search?query=Ask+HN+Who+is+hiring&tags=story&hitsPerPage=5`, { headers: { "User-Agent": "freshcontext-mcp/0.3.0" } });
+    if (!threadRes.ok)
+        throw new Error(`HN thread search ${threadRes.status}`);
+    const threadData = await threadRes.json();
+    // Pick most recent hiring thread (they post monthly)
+    const hiringThread = threadData.hits.find(h => h.title?.toLowerCase().includes("who is hiring"));
+    if (!hiringThread)
+        throw new Error("HN hiring thread not found");
+    // Step 2: Search comments within that thread for the query
+    const searchTerms = [query, location].filter(Boolean).join(" ");
+    const commentsRes = await fetch(`https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(searchTerms)}&tags=comment,story_${hiringThread.objectID}&hitsPerPage=10`, { headers: { "User-Agent": "freshcontext-mcp/0.3.0" } });
+    if (!commentsRes.ok)
+        throw new Error(`HN comments ${commentsRes.status}`);
+    const commentsData = await commentsRes.json();
+    const listings = (commentsData.hits ?? [])
         .filter(h => {
         const t = (h.comment_text ?? "").toLowerCase();
-        return t.includes("hiring") || t.includes("remote") || t.includes("full-time") || t.includes("apply");
+        // Must look like a job post, not a meta comment
+        return t.length > 50 && (t.includes("hiring") || t.includes("remote") ||
+            t.includes("full-time") || t.includes("apply") ||
+            t.includes("|") // common delimiter in HN job posts
+        );
     })
         .map(h => {
-        const { badge, days } = freshnessDate(h.created_at);
+        const { badge, days } = freshnessBadge(h.created_at);
         if (days > maxAgeDays)
             return null;
         const excerpt = (h.comment_text ?? "")
             .replace(/<[^>]+>/g, " ")
             .replace(/\s+/g, " ")
             .trim()
-            .slice(0, 350);
+            .slice(0, 400);
         const text = highlightKeywords([
-            `[HN Hiring] Posted by ${h.author}`,
-            `${badge}`,
-            excerpt + (excerpt.length >= 350 ? "…" : ""),
+            `[HN Hiring · ${hiringThread.title}] by ${h.author}`,
+            badge,
+            excerpt + (excerpt.length >= 400 ? "…" : ""),
             `Source: https://news.ycombinator.com/item?id=${h.objectID}`,
         ].join("\n"), keywords);
         return { text, days, source: "hn" };
@@ -237,7 +307,3 @@ async function fetchHNHiring(query, location, maxAgeDays, keywords, maxLength) {
         .slice(0, 6);
     return { listings };
 }
-// ─── Shared date helper ───────────────────────────────────────────────────────
-function freshnessDate(dateStr) {
-    return freshnessBadge(dateStr ?? null);
-}

package/package.json

@@ -1,55 +1,51 @@
-{
-  "name": "freshcontext-mcp",
-  "mcpName": "io.github.PrinceGabriel-lgtm/freshcontext",
-  "version": "0.2.0",
-  "description": "Real-time web extraction MCP server with freshness timestamps for AI agents",
-  "keywords": [
-    "mcp",
-    "mcp-server",
-    "ai-agents",
-    "llm",
-    "freshness",
-    "web-scraping",
-    "github-analytics",
-    "hackernews",
-    "yc",
-    "typescript",
-    "context",
-    "model-context-protocol"
-  ],
-  "homepage": "https://github.com/PrinceGabriel-lgtm/freshcontext-mcp",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/PrinceGabriel-lgtm/freshcontext-mcp.git"
-  },
-  "license": "MIT",
-  "type": "module",
-  "main": "dist/server.js",
-  "bin": {
-    "freshcontext-mcp": "dist/server.js"
-  },
-  "scripts": {
-    "build": "tsc",
-    "dev": "tsx watch src/server.ts",
-    "start": "node dist/server.js",
-    "inspect": "npx @modelcontextprotocol/inspector tsx src/server.ts",
-    "test": "jest"
-  },
-  "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.0.0",
-    "playwright": "^1.44.0",
-    "zod": "^3.23.0",
-    "dotenv": "^16.4.0"
-  },
-  "devDependencies": {
-    "@types/node": "^20.0.0",
-    "tsx": "^4.0.0",
-    "typescript": "^5.4.0",
-    "jest": "^29.0.0",
-    "@types/jest": "^29.0.0"
-  }
-}
-
-
-
-
+{
+  "name": "freshcontext-mcp",
+  "mcpName": "io.github.PrinceGabriel-lgtm/freshcontext",
+  "version": "0.3.1",
+  "description": "Real-time web extraction MCP server with freshness timestamps for AI agents",
+  "keywords": [
+    "mcp",
+    "mcp-server",
+    "ai-agents",
+    "llm",
+    "freshness",
+    "web-scraping",
+    "github-analytics",
+    "hackernews",
+    "yc",
+    "typescript",
+    "context",
+    "model-context-protocol"
+  ],
+  "homepage": "https://github.com/PrinceGabriel-lgtm/freshcontext-mcp",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/PrinceGabriel-lgtm/freshcontext-mcp.git"
+  },
+  "license": "MIT",
+  "type": "module",
+  "main": "dist/server.js",
+  "bin": {
+    "freshcontext-mcp": "dist/server.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx watch src/server.ts",
+    "start": "node dist/server.js",
+    "inspect": "npx @modelcontextprotocol/inspector tsx src/server.ts",
+    "test": "jest"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "playwright": "^1.44.0",
+    "zod": "^3.23.0",
+    "dotenv": "^16.4.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.4.0",
+    "jest": "^29.0.0",
+    "@types/jest": "^29.0.0"
+  }
+}

package/server.json

@@ -6,14 +6,14 @@
     "url": "https://github.com/PrinceGabriel-lgtm/freshcontext-mcp",
     "source": "github"
   },
-  "version": "0.1.7",
+  "version": "0.3.1",
   "website_url": "https://freshcontext-site.pages.dev",
   "packages": [
     {
       "registry_type": "npm",
       "registry_base_url": "https://registry.npmjs.org",
       "identifier": "freshcontext-mcp",
-      "version": "0.1.7",
+      "version": "0.3.1",
       "transport": {
         "type": "stdio"
       }