@zetagoaurum-dev/straw 1.1.0 → 1.1.1

package/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.1.1] "Performance Patch" - 2026-02-27
+- **Perf:** Re-engineered the YouTube scraper in Node.js and Python to use the `IOS` InnerTube API directly, injecting localized `visitorData` tokens to seamlessly bypass bot checks and cipher encryption. Video format lists are returned instantaneously for optimal downloading infrastructure.
+- **Fix:** Fixed HTML parser blocking on high-volume deployed servers by upgrading to the direct `POST /youtubei/v1/player` endpoints.
+
 ## [1.1.0] - "Milk Tea" Release - 2026-02-27
 
 ### Changed

package/dist/index.js

@@ -185,19 +185,57 @@ var YouTubeScraper = class {
    * Parses the ytInitialPlayerResponse object embedded in the watch HTML.
    */
   async scrapeVideo(url) {
+    const videoIdMatch = url.match(/(?:youtu\.be\/|youtube\.com\/(?:embed\/|v\/|watch\?v=|watch\?.+&v=))([^"&?\/\s]{11})/);
+    if (!videoIdMatch || !videoIdMatch[1]) {
+      throw new Error("Invalid YouTube URL");
+    }
+    const videoId = videoIdMatch[1];
     const html = await this.client.getText(url, {
-      headers: {
-        "Cookie": "CONSENT=YES+cb.20230501-14-p0.en+FX+430"
-      }
+      headers: { "Cookie": "CONSENT=YES+cb.20230501-14-p0.en+FX+430" }
     });
     const regex = /ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)/;
     const match = html.match(regex);
-    if (!match || !match[1]) {
-      throw new Error("ytInitialPlayerResponse not found. YouTube might have changed their layout or the IP is blocked.");
+    let visitorData = "";
+    let details = {};
+    if (match && match[1]) {
+      const data = JSON.parse(match[1]);
+      details = data?.videoDetails || {};
+      visitorData = data?.responseContext?.visitorData || "";
+    }
+    if (!visitorData) {
+      const vdMatch = html.match(/"visitorData"\s*:\s*"([^"]+)"/);
+      if (vdMatch) visitorData = vdMatch[1];
+    }
+    const payload = {
+      context: {
+        client: {
+          hl: "en",
+          gl: "US",
+          clientName: "IOS",
+          clientVersion: "19.28.1",
+          osName: "iOS",
+          osVersion: "17.5.1",
+          deviceMake: "Apple",
+          deviceModel: "iPhone16,2",
+          visitorData
+        }
+      },
+      videoId
+    };
+    const res = await this.client.request("https://www.youtube.com/youtubei/v1/player", {
+      method: "POST",
+      headers: {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+        "User-Agent": "com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)"
+      },
+      body: JSON.stringify(payload)
+    });
+    const apiData = await res.json();
+    if (!details.title) {
+      details = apiData?.videoDetails || {};
     }
-    const data = JSON.parse(match[1]);
-    const details = data?.videoDetails;
-    const streamingData = data?.streamingData;
+    const streamingData = apiData?.streamingData;
     if (!details) {
       throw new Error("Video details not found inside player response.");
     }

package/dist/index.mjs

@@ -145,19 +145,57 @@ var YouTubeScraper = class {
    * Parses the ytInitialPlayerResponse object embedded in the watch HTML.
    */
   async scrapeVideo(url) {
+    const videoIdMatch = url.match(/(?:youtu\.be\/|youtube\.com\/(?:embed\/|v\/|watch\?v=|watch\?.+&v=))([^"&?\/\s]{11})/);
+    if (!videoIdMatch || !videoIdMatch[1]) {
+      throw new Error("Invalid YouTube URL");
+    }
+    const videoId = videoIdMatch[1];
     const html = await this.client.getText(url, {
-      headers: {
-        "Cookie": "CONSENT=YES+cb.20230501-14-p0.en+FX+430"
-      }
+      headers: { "Cookie": "CONSENT=YES+cb.20230501-14-p0.en+FX+430" }
     });
     const regex = /ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)/;
     const match = html.match(regex);
-    if (!match || !match[1]) {
-      throw new Error("ytInitialPlayerResponse not found. YouTube might have changed their layout or the IP is blocked.");
+    let visitorData = "";
+    let details = {};
+    if (match && match[1]) {
+      const data = JSON.parse(match[1]);
+      details = data?.videoDetails || {};
+      visitorData = data?.responseContext?.visitorData || "";
+    }
+    if (!visitorData) {
+      const vdMatch = html.match(/"visitorData"\s*:\s*"([^"]+)"/);
+      if (vdMatch) visitorData = vdMatch[1];
+    }
+    const payload = {
+      context: {
+        client: {
+          hl: "en",
+          gl: "US",
+          clientName: "IOS",
+          clientVersion: "19.28.1",
+          osName: "iOS",
+          osVersion: "17.5.1",
+          deviceMake: "Apple",
+          deviceModel: "iPhone16,2",
+          visitorData
+        }
+      },
+      videoId
+    };
+    const res = await this.client.request("https://www.youtube.com/youtubei/v1/player", {
+      method: "POST",
+      headers: {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+        "User-Agent": "com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)"
+      },
+      body: JSON.stringify(payload)
+    });
+    const apiData = await res.json();
+    if (!details.title) {
+      details = apiData?.videoDetails || {};
     }
-    const data = JSON.parse(match[1]);
-    const details = data?.videoDetails;
-    const streamingData = data?.streamingData;
+    const streamingData = apiData?.streamingData;
     if (!details) {
       throw new Error("Video details not found inside player response.");
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zetagoaurum-dev/straw",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Enterprise-grade unified JS/TS and Python scraping library for Web, YouTube, and Media (Images, Audio, Video, Documents)",
   "main": "dist/index.js",
   "module": "dist/index.mjs",

package/release.bat

@@ -0,0 +1,4 @@
+git add .
+git commit -m "v1.1.1 Performance Patch (InnerTube API Bypass)"
+git push origin master -f
+npm publish

package/src/scrapers/youtube.ts

@@ -33,23 +33,64 @@ export class YouTubeScraper {
      * Parses the ytInitialPlayerResponse object embedded in the watch HTML.
      */
     public async scrapeVideo(url: string): Promise<YouTubeResult> {
+        const videoIdMatch = url.match(/(?:youtu\.be\/|youtube\.com\/(?:embed\/|v\/|watch\?v=|watch\?.+&v=))([^"&?\/\s]{11})/);
+        if (!videoIdMatch || !videoIdMatch[1]) {
+            throw new Error('Invalid YouTube URL');
+        }
+        const videoId = videoIdMatch[1];
+
         const html = await this.client.getText(url, {
-            headers: {
-                'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'
-            }
+            headers: { 'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430' }
         });
         
-        // Find ytInitialPlayerResponse JSON fragment in the HTML
         const regex = /ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)/;
         const match = html.match(regex);
+        let visitorData = '';
+        let details: any = {};
+        
+        if (match && match[1]) {
+            const data = JSON.parse(match[1]);
+            details = data?.videoDetails || {};
+            visitorData = data?.responseContext?.visitorData || '';
+        }
         
-        if (!match || !match[1]) {
-            throw new Error('ytInitialPlayerResponse not found. YouTube might have changed their layout or the IP is blocked.');
+        if (!visitorData) {
+            const vdMatch = html.match(/"visitorData"\s*:\s*"([^"]+)"/);
+            if (vdMatch) visitorData = vdMatch[1];
         }
 
-        const data = JSON.parse(match[1]);
-        const details = data?.videoDetails;
-        const streamingData = data?.streamingData;
+        const payload = {
+            context: {
+                client: {
+                    hl: 'en',
+                    gl: 'US',
+                    clientName: 'IOS',
+                    clientVersion: '19.28.1',
+                    osName: 'iOS',
+                    osVersion: '17.5.1',
+                    deviceMake: 'Apple',
+                    deviceModel: 'iPhone16,2',
+                    visitorData: visitorData
+                }
+            },
+            videoId: videoId
+        };
+
+        const res = await this.client.request('https://www.youtube.com/youtubei/v1/player', {
+            method: 'POST',
+            headers: {
+                'Accept': 'application/json',
+                'Content-Type': 'application/json',
+                'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+            },
+            body: JSON.stringify(payload)
+        });
+
+        const apiData = await res.json() as any;
+        if (!details.title) {
+            details = apiData?.videoDetails || {};
+        }
+        const streamingData = apiData?.streamingData;
 
         if (!details) {
             throw new Error('Video details not found inside player response.');

package/straw/youtube.py

@@ -8,18 +8,58 @@ class YouTubeScraper:
         self.client = StrawClient(**client_options)
 
     async def scrape_video(self, url: str) -> Dict[str, Any]:
-        headers = {
-            'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'
-        }
+        match = re.search(r'(?:youtu\.be\/|youtube\.com\/(?:embed\/|v\/|watch\?v=|watch\?.+&v=))([^"&?\/\s]{11})', url)
+        if not match:
+            raise Exception("Invalid YouTube URL")
+        video_id = match.group(1)
+
+        headers = {'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'}
         html = await self.client.get_text(url, headers=headers)
+        
+        visitor_data = ""
+        details = {}
+        
+        player_match = re.search(r'ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)', html)
+        if player_match:
+            data_html = json.loads(player_match.group(1))
+            details = data_html.get('videoDetails', {})
+            visitor_data = data_html.get('responseContext', {}).get('visitorData', '')
+            
+        if not visitor_data:
+            vd_match = re.search(r'"visitorData"\s*:\s*"([^"]+)"', html)
+            if vd_match:
+                visitor_data = vd_match.group(1)
 
-        match = re.search(r'ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)', html)
-        if not match:
-            raise Exception("ytInitialPlayerResponse not found. YouTube layout changed or IP blocked.")
+        payload = {
+            "context": {
+                "client": {
+                    "hl": "en",
+                    "gl": "US",
+                    "clientName": "IOS",
+                    "clientVersion": "19.28.1",
+                    "osName": "iOS",
+                    "osVersion": "17.5.1",
+                    "deviceMake": "Apple",
+                    "deviceModel": "iPhone16,2",
+                    "visitorData": visitor_data
+                }
+            },
+            "videoId": video_id
+        }
+
+        api_headers = {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+            'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+        }
 
-        data = json.loads(match.group(1))
-        details = data.get('videoDetails', {})
-        streaming_data = data.get('streamingData', {})
+        response = await self.client.request('POST', 'https://www.youtube.com/youtubei/v1/player', json=payload, headers=api_headers)
+        api_data = response.json()
+        
+        if not details.get('title'):
+            details = api_data.get('videoDetails', {})
+            
+        streaming_data = api_data.get('streamingData', {})
 
         if not details:
             raise Exception("Video details not found inside player response.")

package/test_api.js

@@ -0,0 +1,42 @@
+const undici = require('undici');
+
+async function testInnerTube() {
+  const videoId = '_4j1Abt_AiM';
+  
+  const payload = {
+    context: {
+      client: {
+        hl: 'en',
+        gl: 'US',
+        clientName: 'IOS',
+        clientVersion: '19.28.1',
+        osName: 'iOS',
+        osVersion: '17.5.1',
+        deviceMake: 'Apple',
+        deviceModel: 'iPhone16,2'
+      }
+    },
+    videoId: videoId
+  };
+
+  const res = await undici.request('https://www.youtube.com/youtubei/v1/player', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+    },
+    body: JSON.stringify(payload)
+  });
+
+  const body = await res.body.json();
+  console.log('Full JSON Response Keys:', Object.keys(body));
+  console.log('Raw JSON String (Truncated):', JSON.stringify(body).slice(0, 1000));
+  console.log('Playability:', body.playabilityStatus);
+  console.log('Title:', body.videoDetails?.title);
+  
+  const formats = [...(body.streamingData?.formats || []), ...(body.streamingData?.adaptiveFormats || [])];
+  console.log('Total Formats:', formats.length);
+  
+}
+
+testInnerTube();

package/test_api_clients.js

@@ -0,0 +1,39 @@
+const undici = require('undici');
+
+async function testClient(clientName, clientVersion, userAgent, osName='', osVersion='') {
+  const payload = {
+    context: {
+      client: {
+        hl: 'en',
+        gl: 'US',
+        clientName,
+        clientVersion,
+        osName,
+        osVersion
+      }
+    },
+    videoId: '_4j1Abt_AiM'
+  };
+
+  const res = await undici.request('https://www.youtube.com/youtubei/v1/player', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'User-Agent': userAgent
+    },
+    body: JSON.stringify(payload)
+  });
+
+  const body = await res.body.json();
+  const formats = [...(body.streamingData?.formats || []), ...(body.streamingData?.adaptiveFormats || [])];
+  console.log(`[${clientName}] Playability:`, body.playabilityStatus?.status, '| Formats:', formats.length);
+}
+
+async function runAll() {
+  await testClient('WEB_EMBED', '1.20230209.00.00', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)');
+  await testClient('TVHTML5', '7.20230209.00.00', 'Mozilla/5.0 (Web0S; Linux/SmartTV) AppleWebKit/537.36 (KHTML, like Gecko)');
+  await testClient('ANDROID', '17.31.35', 'com.google.android.youtube/17.31.35 (Linux; U; Android 11)', 'Android', '11');
+  await testClient('IOS', '19.28.1', 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)', 'iOS', '17.5.1');
+}
+
+runAll();

package/test_client.js

@@ -0,0 +1,37 @@
+const { StrawClient } = require('./dist/core/client.js');
+
+async function test() {
+  const client = new StrawClient();
+  const payload = {
+    context: {
+        client: {
+            hl: 'en',
+            gl: 'US',
+            clientName: 'IOS',
+            clientVersion: '19.28.1',
+            osName: 'iOS',
+            osVersion: '17.5.1',
+            deviceMake: 'Apple',
+            deviceModel: 'iPhone16,2'
+        }
+    },
+    videoId: '_4j1Abt_AiM'
+  };
+
+  const res = await client.request('https://www.youtube.com/youtubei/v1/player', {
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+    },
+    body: JSON.stringify(payload)
+  });
+
+  const data = await res.json();
+  console.log(Object.keys(data));
+  if (data.playabilityStatus) {
+    console.log('Playability:', data.playabilityStatus);
+  }
+}
+
+test();

package/test_embed.js

@@ -0,0 +1,26 @@
+const undici = require('undici');
+
+async function testEmbed() {
+  const url = 'https://www.youtube.com/embed/_4j1Abt_AiM';
+  const res = await undici.request(url, {
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+    }
+  });
+  const html = await res.body.text();
+  
+  const regex = /ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)/;
+  const match = html.match(regex);
+  if (match) {
+    const data = JSON.parse(match[1]);
+    const formats = [...(data.streamingData?.formats || []), ...(data.streamingData?.adaptiveFormats || [])];
+    console.log('Embed playability:', data.playabilityStatus?.status);
+    console.log('Formats found:', formats.length);
+  } else {
+    console.log('No ytInitialPlayerResponse found in embed HTML');
+  }
+}
+
+testEmbed();

package/test_html.js

@@ -0,0 +1,26 @@
+const undici = require('undici');
+
+async function testHtml() {
+  const url = 'https://www.youtube.com/watch?v=_4j1Abt_AiM';
+  const res = await undici.request(url, {
+    method: 'GET',
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'
+    }
+  });
+
+  const html = await res.body.text();
+  const match = html.match(/ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)/);
+  if (match) {
+    const data = JSON.parse(match[1]);
+    const formats = [...(data.streamingData?.formats || []), ...(data.streamingData?.adaptiveFormats || [])];
+    console.log('Got HTML Response with Player:', data.playabilityStatus?.status);
+    console.log('Formats:', formats.length);
+  } else {
+    console.log('No ytInitialPlayerResponse found in direct HTML fetching.');
+  }
+}
+
+testHtml();

package/test_visitor.js

@@ -0,0 +1,56 @@
+const undici = require('undici');
+
+async function testVisitor() {
+  const videoId = '_4j1Abt_AiM';
+  const url = `https://www.youtube.com/watch?v=${videoId}`;
+  
+  const htmlRes = await undici.request(url, {
+    method: 'GET',
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/115.0.0.0 Safari/537.36',
+      'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'
+    }
+  });
+
+  const html = await htmlRes.body.text();
+  
+  let visitorData = '';
+  const match = html.match(/"visitorData"\s*:\s*"([^"]+)"/);
+  if (match) visitorData = match[1];
+  
+  console.log('Got Visitor Data:', visitorData);
+
+  const payload = {
+    context: {
+      client: {
+        hl: 'en',
+        gl: 'US',
+        clientName: 'IOS',
+        clientVersion: '19.28.1',
+        osName: 'iOS',
+        osVersion: '17.5.1',
+        deviceMake: 'Apple',
+        deviceModel: 'iPhone16,2',
+        visitorData: visitorData
+      }
+    },
+    videoId: videoId
+  };
+
+  const res = await undici.request('https://www.youtube.com/youtubei/v1/player', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+    },
+    body: JSON.stringify(payload)
+  });
+
+  const body = await res.body.json();
+  const formats = [...(body.streamingData?.formats || []), ...(body.streamingData?.adaptiveFormats || [])];
+  
+  console.log('Target Playability:', body.playabilityStatus?.status);
+  console.log('Target Formats:', formats.length);
+}
+
+testVisitor();

package/test_vr.js

@@ -0,0 +1,27 @@
+const undici = require('undici');
+
+async function testVR() {
+  const payload = {
+    context: {
+      client: {
+        clientName: 'ANDROID_TESTSUITE',
+        clientVersion: '1.9',
+        androidSdkVersion: 30,
+        hl: 'en',
+        gl: 'US',
+        utcOffsetMinutes: 0
+      }
+    },
+    videoId: '_4j1Abt_AiM'
+  };
+  const res = await undici.request('https://www.youtube.com/youtubei/v1/player', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', 'User-Agent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11)' },
+    body: JSON.stringify(payload)
+  });
+  const body = await res.body.json();
+  const formats = [...(body.streamingData?.formats || []), ...(body.streamingData?.adaptiveFormats || [])];
+  console.log('Playability:', body.playabilityStatus?.status);
+  console.log('Formats:', formats.length);
+}
+testVR();

package/test_yt.js

@@ -0,0 +1,17 @@
+const straw = require('./dist/index.js');
+
+async function run() {
+  console.time('YouTube Scrape');
+  const yt = new straw.YouTubeScraper();
+  try {
+    const res = await yt.scrapeVideo('https://youtu.be/_4j1Abt_AiM?si=qJY_gv4F_adBYMYP');
+    console.log('Title:', res.title);
+    console.log('Formats:', res.formats.length);
+    console.log('First format URL (truncated):', res.formats[0]?.url?.substring(0, 100));
+  } catch (e) {
+    console.error('Scrape failed:', e);
+  }
+  console.timeEnd('YouTube Scrape');
+}
+
+run();