@pipeworx/mcp-newsdata 0.1.4 → 0.1.6

package/README.md

@@ -2,13 +2,13 @@
 
 NewsData.io MCP — wraps the NewsData.io global news API (newsdata.io)
 
-Part of [Pipeworx](https://pipeworx.io) — an MCP gateway connecting AI agents to 1684+ live data sources.
+Part of [Pipeworx](https://pipeworx.io) — an MCP gateway connecting AI agents to 1877+ live data sources. This is an independent, unofficial integration — not affiliated with, endorsed by, or published by the upstream provider.
 
 ## Tools
 
 | Tool | Description |
 |------|-------------|
-| `latest_news` | Get the latest global news headlines and articles — world news, breaking news, and business/financial/stock-market news. Filter by keyword, country (2-letter, e.g. "us"), category (business, technology, politics, sports, health, science), and language. IMPORTANT: for stock-market / financial-market / economy / "world market news" questions, ALWAYS pass category: "business" — it returns real market-news outlets and filters out low-quality SEO/crypto-promo articles. Returns article title, description, link, source, publish date, category, and country. Paginate via the nextPage token. Examples: latest_news({ query: "stock market", category: "business" }) for world market news; latest_news({ query: "election", country: "us", category: "politics" }). |
+| `latest_news` | Get news headlines and articles — world news, breaking news, and business/financial/stock-market news. Filter by keyword, country (2-letter, e.g. "us"), category (business, technology, politics, sports, health, science), and language. IMPORTANT: for stock-market / financial-market / economy / "world market news" questions, ALWAYS pass category: "business" — it returns real market-news outlets and filters out low-quality SEO/crypto-promo articles. Returns article title, description, link, source, publish date, category, and country. Paginate via the nextPage token. WITHOUT from_date/to_date this covers only roughly the LAST 48 HOURS (NewsData.io's rolling "latest" feed) — asking about an older event with no date window silently returns today's unrelated articles instead of an empty result. For a PAST date range, pass from_date/to_date (YYYY-MM-DD): this routes to NewsData.io's archive endpoint, which requires a plan tier above the shared platform key's default — if the account cannot serve it, the call returns a `filter_gap` field naming the unsupported window instead of silently substituting the latest 48h. Examples: latest_news({ query: "stock market", category: "business" }) for current world market news; latest_news({ query: "election", country: "us", category: "politics" }); latest_news({ query: "Houthi Yemen attack", from_date: "2026-08-01", to_date: "2026-08-15" }) for an August event. |
 | `crypto_news` | Get the latest cryptocurrency news headlines and articles. Filter by keyword, coin ticker(s) (e.g. "btc,eth"), and language. Returns article title, description, link, source, publish date, category, and country. Example: crypto_news({ coin: "btc,eth", query: "etf" }) |
 | `news_sources` | List available global news sources. Filter by country (2-letter), category, and language. Returns source id, name, url, category, country, and language — useful for discovering which outlets cover a given region or topic. Example: news_sources({ country: "us", category: "technology" }) |
 
@@ -56,7 +56,7 @@ directly, instead of just this one's:
 }
 ```
 
-Both URLs reach the same gateway and the same 1684+ data sources. The
+Both URLs reach the same gateway and the same 1877+ data sources. The
 only difference is which pack's tools are listed **directly**; `ask_pipeworx`
 reaches all of them from either one.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipeworx/mcp-newsdata",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "NewsData.io MCP — wraps the NewsData.io global news API (newsdata.io)",
   "type": "module",
   "main": "src/index.ts",
@@ -26,7 +26,7 @@
     "@cloudflare/workers-types": "^4.20260405.1"
   },
   "pipeworx": {
-    "sourceHash": "v1-67e6f65fa18f2efea30dc86b384652fae8ea5e6cd116ea91615ad81be3fef742",
-    "sourceCommit": "eaa02971997fbfee27712ea66d7d5fd840d008ea"
+    "sourceHash": "v1-cc46b5452257982114d1624bb3b6c9f72020b0320b506803f242d709c09965eb",
+    "sourceCommit": "774de7f685a75e02d0b828d24a7b94794102152c"
   }
 }

package/server.json

@@ -3,7 +3,7 @@
   "name": "io.github.pipeworx-io/newsdata",
   "title": "Newsdata",
   "description": "NewsData.io MCP — wraps the NewsData.io global news API (newsdata.io)",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "websiteUrl": "https://pipeworx.io/packs/newsdata",
   "repository": {
     "url": "https://github.com/pipeworx-io/mcp-newsdata",

package/src/index.ts

@@ -295,12 +295,28 @@ async function httpErrorMessage(res: Response, name: string): Promise<string> {
   // internal-host-class.ts; no-op for every third-party upstream, which is why
   // this touches 481 packs' error text and changes none of it.
   return markInternalOrigin(
-    `${name}: ${res.status}${detailSuffix(await readDetail(res))}`,
+    `${name}: ${res.status}${detailSuffix(await readDetail(res))}${responseHints(res)}`,
     res.url,
     res.status,
   );
 }
 
+/**
+ * Facts about a FAILED response that live in its headers, not its body, appended
+ * so the gateway's classifier (which only ever sees this string) can tell a limit
+ * from a wall from a refusal (fleet #3477). Both are standard, both are absent on
+ * almost every response, and neither is invented: `Retry-After` is the server
+ * saying "come back later" (the one thing that makes a 403 a throttle), and
+ * Cloudflare marks a bot challenge with `cf-mitigated: challenge`.
+ */
+function responseHints(res: Response): string {
+  const hints: string[] = [];
+  const retryAfter = res.headers.get('retry-after');
+  if (retryAfter) hints.push(`retry-after: ${retryAfter.slice(0, 40)}`);
+  if ((res.headers.get('cf-mitigated') ?? '').toLowerCase() === 'challenge') hints.push('Cloudflare bot challenge (cf-mitigated: challenge)');
+  return hints.length ? ` (${hints.join('; ')})` : '';
+}
+
 /**
  * Just the upstream's own explanation — no name, no status.
  *
@@ -670,6 +686,254 @@ function pickMessage(node: unknown, depth: number): string | null {
 function collapse(s: string): string {
   return s.replace(/\s+/g, ' ').trim();
 }
+
+// ── Bounded readers — peak memory, not just wall time (fleet #3322) ──────
+//
+// `parseJson` above does `await res.text()` then `JSON.parse(raw)`, so a
+// multi-megabyte upstream body is held TWICE at peak: once as the raw string
+// and once as the object graph (which for an array of small objects runs
+// several times the byte size). Nothing in `workers/gateway` can see that
+// peak — it happens inside the pack, before any gateway code holds a payload —
+// which is why the gateway-side payload caps of #3293 and #3320 did not move
+// the 503/1102 rate at all.
+//
+// Measured on the live gateway, 2026-10-19, sha fa5de5c34, direct tools/call:
+//   statcan_list_cubes  8,049,566 B   <- one upstream array, no limit argument
+//   gnomad_gene BRCA1   2,732,706 B
+// Five of those in flight at the quality runner's width is ~40 MB of parse
+// peak against a 128 MB isolate; Cloudflare kills the isolate and every
+// request in flight returns 503/1102, which is why the victims look random.
+//
+// `readJsonArrayPrefix` is the fix for the dominant shape: an upstream that
+// returns one enormous top-level JSON array with no server-side paging. It
+// parses elements off the stream and DISCARDS the consumed bytes, so peak
+// retained memory is one element plus one chunk rather than the whole body,
+// and it cancels the body as soon as it has what it was asked for.
+
+/** Default ceiling for a single upstream JSON body read into an isolate.
+ *  Chosen so the known tail (statcan 8.0 MB, gnomad 2.7 MB) is bounded well
+ *  below the point where a handful of concurrent reads exhausts 128 MB. */
+const DEFAULT_MAX_JSON_BYTES = 1_500_000;
+
+interface JsonArrayPrefix<T> {
+  /** The decoded elements, at most `maxItems` of them. */
+  items: T[];
+  /** True when the upstream array had more elements than we read. */
+  truncated: boolean;
+  /** Bytes actually pulled off the socket — the number that bounds memory. */
+  bytesRead: number;
+  /** `Content-Length` if the upstream declared one, else null. */
+  declaredBytes: number | null;
+}
+
+/**
+ * Read the FIRST `maxItems` elements of a top-level JSON array off a response
+ * stream without ever materialising the whole body.
+ *
+ * Use this instead of `parseJson` when the upstream returns one big array and
+ * offers no `limit`/`offset` of its own. Peak retained memory is one element
+ * plus the current chunk; the body is cancelled as soon as `maxItems` is
+ * reached, so the remaining megabytes are never transferred at all.
+ *
+ *     const { items, truncated } = await readJsonArrayPrefix<Cube>(
+ *       res, 'StatCan WDS', { maxItems: limit + offset });
+ *
+ * `maxBytes` is a second, independent backstop for a pathological element (a
+ * single 50 MB object): hitting it stops the read and sets `truncated`, rather
+ * than throwing, because a partial answer plus an explicit truncated flag is
+ * more useful to a caller than an error.
+ */
+async function readJsonArrayPrefix<T>(
+  res: Response,
+  name: string,
+  opts: { maxItems: number; maxBytes?: number },
+): Promise<JsonArrayPrefix<T>> {
+  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_JSON_BYTES;
+  const declaredRaw = Number(res.headers.get('content-length'));
+  const declaredBytes = Number.isFinite(declaredRaw) && declaredRaw > 0 ? declaredRaw : null;
+
+  // No streaming body available (a mock, or an already-buffered response):
+  // fall back to the whole-text path so behaviour stays correct, just unbounded.
+  if (!res.body) {
+    const all = await parseJson<T[]>(res, name);
+    if (!Array.isArray(all)) throw new Error(`upstream_down: ${name} returned ${typeof all} where a JSON array was expected.`);
+    return {
+      items: all.slice(0, opts.maxItems),
+      truncated: all.length > opts.maxItems,
+      bytesRead: declaredBytes ?? 0,
+      declaredBytes,
+    };
+  }
+
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  const items: T[] = [];
+
+  let buf = '';
+  let scan = 0;
+  let bytesRead = 0;
+  let started = false; // consumed the opening '['
+  let arrayEnded = false;
+  let depth = 0;
+  let inStr = false;
+  let esc = false;
+  let elemStart = -1;
+  let truncated = false;
+
+  const push = (text: string) => {
+    const t = text.trim();
+    if (!t) return;
+    items.push(JSON.parse(t) as T);
+  };
+
+  try {
+    outer: while (items.length < opts.maxItems && !arrayEnded) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      bytesRead += value.byteLength;
+      buf += decoder.decode(value, { stream: true });
+      if (bytesRead > maxBytes) { truncated = true; break; }
+
+      for (; scan < buf.length; scan++) {
+        const ch = buf[scan];
+
+        if (inStr) {
+          if (esc) { esc = false; continue; }
+          if (ch === '\\') { esc = true; continue; }
+          if (ch === '"') inStr = false;
+          continue;
+        }
+
+        if (!started) {
+          if (ch === '[') { started = true; continue; }
+          if (ch.trim() === '') continue;
+          throw new Error(
+            `upstream_down: ${name} returned a body that does not begin with a JSON array (saw ${JSON.stringify(ch)}). ` +
+              'This reader is only for upstreams that answer with one top-level array.',
+          );
+        }
+
+        if (ch === '"') { if (depth === 0 && elemStart < 0) elemStart = scan; inStr = true; continue; }
+
+        if (ch === '{' || ch === '[') {
+          if (depth === 0 && elemStart < 0) elemStart = scan;
+          depth++;
+          continue;
+        }
+
+        if (ch === '}' || ch === ']') {
+          if (depth > 0) {
+            depth--;
+            if (depth === 0 && elemStart >= 0) {
+              push(buf.slice(elemStart, scan + 1));
+              elemStart = -1;
+              if (items.length >= opts.maxItems) { truncated = true; scan++; break outer; }
+            }
+            continue;
+          }
+          // depth 0 and a closing bracket: the end of the top-level array.
+          if (elemStart >= 0) { push(buf.slice(elemStart, scan)); elemStart = -1; }
+          arrayEnded = true;
+          break outer;
+        }
+
+        if (ch === ',' && depth === 0) {
+          if (elemStart >= 0) {
+            push(buf.slice(elemStart, scan));
+            elemStart = -1;
+            if (items.length >= opts.maxItems) { truncated = true; scan++; break outer; }
+          }
+          continue;
+        }
+
+        // A bare scalar element (number / true / false / null).
+        if (depth === 0 && elemStart < 0 && ch.trim() !== '') elemStart = scan;
+      }
+
+      // Drop everything already consumed. THIS is what bounds memory: without
+      // it `buf` grows to the full body and the reader is streaming in name only.
+      if (elemStart >= 0) {
+        buf = buf.slice(elemStart);
+        scan -= elemStart;
+        elemStart = 0;
+      } else {
+        buf = buf.slice(scan);
+        scan = 0;
+      }
+    }
+  } finally {
+    // Cancel rather than drain: the point is that the remaining megabytes are
+    // never transferred. A cancel on an already-finished body is a no-op.
+    await reader.cancel().catch(() => {});
+  }
+
+  return { items, truncated, bytesRead, declaredBytes };
+}
+
+/**
+ * `parseJson`, with a hard ceiling on how many bytes are read.
+ *
+ * For an upstream whose body is not a single array (so `readJsonArrayPrefix`
+ * does not apply) but which can still answer with megabytes. Unlike the array
+ * reader this one THROWS when the cap is hit, because a truncated prefix of an
+ * arbitrary JSON document is not parseable and there is no partial answer to
+ * hand back. A declared `Content-Length` over the cap is refused before a
+ * single chunk is read.
+ */
+async function parseJsonBounded<T>(
+  res: Response,
+  name: string,
+  maxBytes: number = DEFAULT_MAX_JSON_BYTES,
+): Promise<T> {
+  const declaredRaw = Number(res.headers.get('content-length'));
+  const declared = Number.isFinite(declaredRaw) && declaredRaw > 0 ? declaredRaw : null;
+
+  const tooBig = (n: number) =>
+    new Error(
+      `upstream_down: ${name} answered with ${n.toLocaleString()} bytes, over this pack's ${maxBytes.toLocaleString()}-byte ceiling. ` +
+        'The body is not read past the ceiling on purpose — a few concurrent reads of a body that size exhaust the ' +
+        'Worker isolate and take every other in-flight request down with them. Narrow the request (a smaller range, ' +
+        'a specific id, or a lower limit) and retry.',
+    );
+
+  if (declared !== null && declared > maxBytes) {
+    await res.body?.cancel().catch(() => {});
+    throw tooBig(declared);
+  }
+  if (!res.body) return parseJson<T>(res, name);
+
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  let raw = '';
+  let bytesRead = 0;
+  try {
+    for (;;) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      bytesRead += value.byteLength;
+      if (bytesRead > maxBytes) throw tooBig(bytesRead);
+      raw += decoder.decode(value, { stream: true });
+    }
+    raw += decoder.decode();
+  } finally {
+    await reader.cancel().catch(() => {});
+  }
+
+  if (!raw.trim()) {
+    throw new Error(
+      `upstream_down: ${name} answered HTTP ${res.status} with an EMPTY body where JSON was expected. ` +
+        'Nothing about the request can cause this — it is an upstream fault, and the same call may well work on retry.',
+    );
+  }
+  try {
+    return JSON.parse(raw) as T;
+  } catch {
+    throw new Error(
+      `upstream_down: ${name} answered HTTP ${res.status} with a body that is not valid JSON. ` +
+        `It begins: ${stripMarkup(raw).slice(0, 120) || '(unreadable)'}`,
+    );
+  }
+}
 /**
  * NewsData.io MCP — wraps the NewsData.io global news API (newsdata.io)
  *
@@ -699,7 +963,7 @@ const tools: McpToolExport['tools'] = [
   {
     name: 'latest_news',
     description:
-      'Get the latest global news headlines and articles — world news, breaking news, and business/financial/stock-market news. Filter by keyword, country (2-letter, e.g. "us"), category (business, technology, politics, sports, health, science), and language. IMPORTANT: for stock-market / financial-market / economy / "world market news" questions, ALWAYS pass category: "business" — it returns real market-news outlets and filters out low-quality SEO/crypto-promo articles. Returns article title, description, link, source, publish date, category, and country. Paginate via the nextPage token. Examples: latest_news({ query: "stock market", category: "business" }) for world market news; latest_news({ query: "election", country: "us", category: "politics" }).',
+      'Get news headlines and articles — world news, breaking news, and business/financial/stock-market news. Filter by keyword, country (2-letter, e.g. "us"), category (business, technology, politics, sports, health, science), and language. IMPORTANT: for stock-market / financial-market / economy / "world market news" questions, ALWAYS pass category: "business" — it returns real market-news outlets and filters out low-quality SEO/crypto-promo articles. Returns article title, description, link, source, publish date, category, and country. Paginate via the nextPage token. WITHOUT from_date/to_date this covers only roughly the LAST 48 HOURS (NewsData.io\'s rolling "latest" feed) — asking about an older event with no date window silently returns today\'s unrelated articles instead of an empty result. For a PAST date range, pass from_date/to_date (YYYY-MM-DD): this routes to NewsData.io\'s archive endpoint, which requires a plan tier above the shared platform key\'s default — if the account cannot serve it, the call returns a `filter_gap` field naming the unsupported window instead of silently substituting the latest 48h. Examples: latest_news({ query: "stock market", category: "business" }) for current world market news; latest_news({ query: "election", country: "us", category: "politics" }); latest_news({ query: "Houthi Yemen attack", from_date: "2026-08-01", to_date: "2026-08-15" }) for an August event.',
     inputSchema: {
       type: 'object' as const,
       properties: {
@@ -719,6 +983,14 @@ const tools: McpToolExport['tools'] = [
           type: 'string',
           description: 'Language code to filter news by, e.g. "en", "es", "fr"',
         },
+        from_date: {
+          type: 'string',
+          description: 'Start of a PAST date window, YYYY-MM-DD. Supplying this (or to_date) switches the call from the 48h "latest" feed to NewsData.io\'s archive endpoint, which may require a higher plan tier than the shared platform key has — if so, the response carries `filter_gap` naming the unsupported window rather than silently returning current articles.',
+        },
+        to_date: {
+          type: 'string',
+          description: 'End of a PAST date window, YYYY-MM-DD. See from_date.',
+        },
         page: {
           type: 'string',
           description: 'nextPage token returned from a prior call, to fetch the next page of results',
@@ -862,6 +1134,8 @@ const ARG_ALIASES: Record<string, readonly string[]> = {
   language: ['lang', 'languages', 'language_code'],
   coin: ['coins', 'ticker', 'tickers', 'symbol', 'symbols'],
   page: ['nextPage', 'next_page', 'page_token', 'pageToken', 'cursor'],
+  from_date: ['fromDate', 'start_date', 'startDate', 'since'],
+  to_date: ['toDate', 'end_date', 'endDate', 'until'],
 };
 
 /**
@@ -936,7 +1210,7 @@ async function callTool(name: string, rawArgs: Record<string, unknown>): Promise
   }
 
   const ACCEPTED: Record<string, readonly string[]> = {
-    latest_news: ['query', 'country', 'category', 'language', 'page'],
+    latest_news: ['query', 'country', 'category', 'language', 'page', 'from_date', 'to_date'],
     crypto_news: ['query', 'coin', 'language'],
     news_sources: ['country', 'category', 'language'],
   };
@@ -944,6 +1218,51 @@ async function callTool(name: string, rawArgs: Record<string, unknown>): Promise
 
   switch (name) {
     case 'latest_news': {
+      const fromDate = args.from_date as string | undefined;
+      const toDate = args.to_date as string | undefined;
+
+      // A caller asking for a PAST window gets routed to NewsData.io's
+      // /archive endpoint instead of /latest (which only ever serves a
+      // rolling last-~48h feed and would otherwise silently answer an
+      // August question with October articles — fleet #2911, 7 judged
+      // misses on 3 questions). /archive commonly needs a plan tier above
+      // the shared platform key's default; rather than assume either way,
+      // make the live call and, if the upstream call itself fails (for ANY
+      // reason — a plan-tier rejection, an unsupported date range, a bad
+      // date format), DECLARE the gap instead of falling back to /latest,
+      // which would silently re-introduce exactly this bug.
+      if (fromDate || toDate) {
+        const archiveParams = new URLSearchParams();
+        if (args.query) archiveParams.set('q', args.query as string);
+        if (args.country) archiveParams.set('country', args.country as string);
+        if (args.category) archiveParams.set('category', args.category as string);
+        if (args.language) archiveParams.set('language', args.language as string);
+        if (fromDate) archiveParams.set('from_date', fromDate);
+        if (toDate) archiveParams.set('to_date', toDate);
+        if (args.page) archiveParams.set('page', args.page as string);
+
+        const archiveData = await newsdataGet(apiKey, '/archive', archiveParams);
+        const upstreamError =
+          archiveData && typeof archiveData === 'object' && 'error' in (archiveData as object)
+            ? (archiveData as { error: number; message: string })
+            : archiveData && typeof archiveData === 'object' && (archiveData as NewsListResponse).status === 'error'
+              ? { error: 'api_error', message: JSON.stringify(archiveData) }
+              : null;
+
+        if (upstreamError) {
+          return {
+            filter_gap: `date window (from_date=${fromDate ?? '(none)'}, to_date=${toDate ?? '(none)'}) could not be served — the archive endpoint call failed, most likely because this NewsData.io plan does not include dated/archive search`,
+            window_requested: { from_date: fromDate ?? null, to_date: toDate ?? null },
+            window_served: 'none — no fallback to the 48h latest feed was taken, to avoid silently answering a past-dated question with current articles',
+            upstream_error: upstreamError,
+          };
+        }
+        return {
+          window_requested: { from_date: fromDate ?? null, to_date: toDate ?? null },
+          ...mapArticles(archiveData as NewsListResponse),
+        };
+      }
+
       const params = new URLSearchParams();
       if (args.query) params.set('q', args.query as string);
       if (args.country) params.set('country', args.country as string);

package/src/server.ts

@@ -8,8 +8,9 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
 import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
 import pack from './index.js';
 
+
 const server = new Server(
-  { name: '@pipeworx/mcp-newsdata', version: '0.1.4' },
+  { name: '@pipeworx/mcp-newsdata', version: '0.1.6' },
   { capabilities: { tools: {} } },
 );
 
@@ -24,7 +25,9 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
   const { name, arguments: args } = request.params;
   try {
-    const result = await pack.callTool(name, (args ?? {}) as Record<string, unknown>);
+    const callArgs = { ...((args ?? {}) as Record<string, unknown>) };
+    
+    const result = await pack.callTool(name, callArgs);
     return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
   } catch (err) {
     return {