crawlforge-extractors 1.5.2 → 1.5.3

package/README.md

@@ -40,6 +40,8 @@ const template = registry.get('shopify-product');
 const url = 'https://shop.example.com/products/some-handle';
 const fetchUrl = template.resolveUrl ? template.resolveUrl(url) : url;
 
+// Apply your SSRF policy to fetchUrl (the RETURNED url) and to any redirect the
+// fetch follows — not just to `url`. resolveUrl/listUrl may rewrite the target.
 const body = await (await fetch(fetchUrl)).text();
 const result = await registry.run('shopify-product', body, url, fetchUrl);
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.5.2",
+  "version": "1.5.3",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/body.js

@@ -70,9 +70,18 @@ export async function readBody(response, options = {}) {
   }
 
   // Only the byte-count guard needs a stream. Responses that are already
-  // buffered (and test doubles) are read as-is so callers still get their text.
+  // buffered (and test doubles) have no reader to meter, but a server can omit
+  // or lie about Content-Length, so enforce the cap on the read result too.
   if (!response.body || typeof response.body.getReader !== 'function') {
-    return response.text();
+    const text = await response.text();
+    const size = Buffer.byteLength(text, 'utf8');
+    if (size > maxBytes) {
+      throw new BodyTooLargeError(
+        `Response body too large: ${size} bytes exceeds limit of ${maxBytes} bytes`,
+        { limit: maxBytes, size }
+      );
+    }
+    return text;
   }
 
   const reader = response.body.getReader();

package/src/jsonPath.js

@@ -66,7 +66,7 @@ export function selectJsonPath(root, path) {
   for (const segment of segments) {
     const container = current !== null && typeof current === 'object';
     const key = Array.isArray(current) ? Number(segment) : segment;
-    if (!container || !(key in current)) {
+    if (!container || !Object.hasOwn(current, key)) {
       const at = walked.length === 0 ? 'the result' : `"${walked.join('.')}"`;
       throw new Error(
         `Path "${path}" not found: ${at} has no "${segment}" (${describeOptions(current)})`

package/src/templates.js

@@ -14,7 +14,9 @@
  *
  * Templates do NOT make network calls. The caller fetches the page and passes
  * the body in; that keeps SSRF policy, timeouts and billing with the surface
- * that owns them.
+ * that owns them. resolveUrl/listUrl rewrite or build the target, so the caller
+ * must apply its SSRF policy to the URL they RETURN (and to every redirect the
+ * fetch follows), not just to the URL the caller started with.
  *
  * Two optional hooks let a template read a machine-readable endpoint instead of
  * scraping the rendered page, without taking the fetch into its own hands: