@zetagoaurum-dev/straw 1.1.0 → 1.2.0

package/release.bat

@@ -0,0 +1,4 @@
+git add .
+git commit -m "v1.2.0 Deep Metadata & Formats Engine"
+git push origin master -f
+npm publish

package/src/scrapers/youtube.ts

@@ -14,11 +14,18 @@ export interface YouTubeFormats {
 export interface YouTubeResult {
     title: string;
     author: string;
+    subscribers: string;
     description: string;
     views: string;
+    likes: string;
+    comments: string;
     durationSeconds: string;
     thumbnail: string;
-    formats: YouTubeFormats[];
+    formats: {
+        video: YouTubeFormats[];
+        videoOnly: YouTubeFormats[];
+        audio: YouTubeFormats[];
+    };
 }
 
 export class YouTubeScraper {
@@ -33,35 +40,106 @@ export class YouTubeScraper {
      * Parses the ytInitialPlayerResponse object embedded in the watch HTML.
      */
     public async scrapeVideo(url: string): Promise<YouTubeResult> {
+        const videoIdMatch = url.match(/(?:youtu\.be\/|youtube\.com\/(?:embed\/|v\/|watch\?v=|watch\?.+&v=))([^"&?\/\s]{11})/);
+        if (!videoIdMatch || !videoIdMatch[1]) {
+            throw new Error('Invalid YouTube URL');
+        }
+        const videoId = videoIdMatch[1];
+
         const html = await this.client.getText(url, {
-            headers: {
-                'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'
-            }
+            headers: { 'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430', 'Accept-Language': 'en-US,en;q=0.9' }
         });
         
-        // Find ytInitialPlayerResponse JSON fragment in the HTML
         const regex = /ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)/;
         const match = html.match(regex);
+        let visitorData = '';
+        let details: any = {};
+        
+        let initialData: any = {};
+        const dataMatch = html.match(/var ytInitialData\s*=\s*({.*?});(?:<\/script>)/);
+        if (dataMatch && dataMatch[1]) {
+            try { initialData = JSON.parse(dataMatch[1]); } catch(e) {}
+        }
         
-        if (!match || !match[1]) {
-            throw new Error('ytInitialPlayerResponse not found. YouTube might have changed their layout or the IP is blocked.');
+        if (match && match[1]) {
+            const data = JSON.parse(match[1]);
+            details = data?.videoDetails || {};
+            visitorData = data?.responseContext?.visitorData || '';
+        }
+        
+        if (!visitorData) {
+            const vdMatch = html.match(/"visitorData"\s*:\s*"([^"]+)"/);
+            if (vdMatch) visitorData = vdMatch[1];
         }
 
-        const data = JSON.parse(match[1]);
-        const details = data?.videoDetails;
-        const streamingData = data?.streamingData;
+        const payload = {
+            context: {
+                client: {
+                    hl: 'en',
+                    gl: 'US',
+                    clientName: 'IOS',
+                    clientVersion: '19.28.1',
+                    osName: 'iOS',
+                    osVersion: '17.5.1',
+                    deviceMake: 'Apple',
+                    deviceModel: 'iPhone16,2',
+                    visitorData: visitorData
+                }
+            },
+            videoId: videoId
+        };
+
+        const res = await this.client.request('https://www.youtube.com/youtubei/v1/player', {
+            method: 'POST',
+            headers: {
+                'Accept': 'application/json',
+                'Content-Type': 'application/json',
+                'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+            },
+            body: JSON.stringify(payload)
+        });
+
+        const apiData = await res.json() as any;
+        if (!details.title) {
+            details = apiData?.videoDetails || {};
+        }
+        const streamingData = apiData?.streamingData;
 
         if (!details) {
             throw new Error('Video details not found inside player response.');
         }
 
-        const formats: YouTubeFormats[] = [];
+        let subscribers = '';
+        let likes = '';
+        let comments = '';
+
+        try {
+            const secInfo = initialData?.contents?.twoColumnWatchNextResults?.results?.results?.contents?.find((c: any) => c.videoSecondaryInfoRenderer)?.videoSecondaryInfoRenderer;
+            if (secInfo?.owner?.videoOwnerRenderer?.subscriberCountText?.simpleText) {
+                subscribers = secInfo.owner.videoOwnerRenderer.subscriberCountText.simpleText;
+            }
+            
+            const factoids = initialData?.engagementPanels?.find((p: any) => p.engagementPanelSectionListRenderer?.targetId === 'engagement-panel-structured-description')
+                ?.engagementPanelSectionListRenderer?.content?.structuredDescriptionContentRenderer?.items?.find((i: any) => i.videoDescriptionHeaderRenderer)?.videoDescriptionHeaderRenderer?.factoid || [];
+            const likesFactoid = factoids.find((f: any) => f.factoidRenderer?.accessibilityText?.toLowerCase().includes('like'));
+            if (likesFactoid) likes = likesFactoid.factoidRenderer.accessibilityText;
+            
+            const commentsPanel = initialData?.engagementPanels?.find((p: any) => p.engagementPanelSectionListRenderer?.panelIdentifier === 'engagement-panel-comments-section');
+            if (commentsPanel) {
+                comments = commentsPanel.engagementPanelSectionListRenderer.header.engagementPanelTitleHeaderRenderer.contextualInfo?.runs?.[0]?.text || '';
+            }
+        } catch (e) {}
+
+        const video: YouTubeFormats[] = [];
+        const videoOnly: YouTubeFormats[] = [];
+        const audio: YouTubeFormats[] = [];
+
         const rawFormats = [...(streamingData?.formats || []), ...(streamingData?.adaptiveFormats || [])];
         
         for (const format of rawFormats) {
             if (format.url) {
                 const mimeType = format.mimeType || '';
-                formats.push({
+                const formatObj = {
                     url: format.url,
                     mimeType: mimeType,
                     width: format.width,
@@ -70,23 +148,29 @@ export class YouTubeScraper {
                     bitrate: format.bitrate,
                     hasAudio: mimeType.includes('audio/'),
                     hasVideo: mimeType.includes('video/')
-                });
-            } else if (format.signatureCipher) {
-                // To avoid bloatware, we do not implement the complex decipher algorithm here.
-                // Modern APIs usually provide the URL directly for lower qualities or we can fallback to other APIs.
-                // Implementing decipher requires porting youtube-dl's sig logic or using ytdl-core.
-                continue;
+                };
+
+                if (formatObj.hasVideo && formatObj.hasAudio) video.push(formatObj);
+                else if (formatObj.hasVideo) videoOnly.push(formatObj);
+                else if (formatObj.hasAudio) audio.push(formatObj);
             }
         }
 
         return {
             title: details.title || '',
             author: details.author || '',
+            subscribers: subscribers,
             description: details.shortDescription || '',
             views: details.viewCount || '0',
+            likes: likes,
+            comments: comments,
             durationSeconds: details.lengthSeconds || '0',
             thumbnail: details.thumbnail?.thumbnails?.[details.thumbnail.thumbnails.length - 1]?.url || '',
-            formats
+            formats: {
+                video,
+                videoOnly,
+                audio
+            }
         };
     }
 }

package/straw/youtube.py

@@ -8,38 +8,133 @@ class YouTubeScraper:
         self.client = StrawClient(**client_options)
 
     async def scrape_video(self, url: str) -> Dict[str, Any]:
-        headers = {
-            'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'
-        }
+        match = re.search(r'(?:youtu\.be\/|youtube\.com\/(?:embed\/|v\/|watch\?v=|watch\?.+&v=))([^"&?\/\s]{11})', url)
+        if not match:
+            raise Exception("Invalid YouTube URL")
+        video_id = match.group(1)
+
+        headers = {'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430', 'Accept-Language': 'en-US,en;q=0.9'}
         html = await self.client.get_text(url, headers=headers)
+        
+        visitor_data = ""
+        details = {}
+        initial_data = {}
+        
+        player_match = re.search(r'ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)', html)
+        if player_match:
+            try:
+                data_html = json.loads(player_match.group(1))
+                details = data_html.get('videoDetails', {})
+                visitor_data = data_html.get('responseContext', {}).get('visitorData', '')
+            except:
+                pass
+                
+        data_match = re.search(r'var ytInitialData\s*=\s*({.*?});(?:<\/script>)', html)
+        if data_match:
+            try:
+                initial_data = json.loads(data_match.group(1))
+            except:
+                pass
+            
+        if not visitor_data:
+            vd_match = re.search(r'"visitorData"\s*:\s*"([^"]+)"', html)
+            if vd_match:
+                visitor_data = vd_match.group(1)
 
-        match = re.search(r'ytInitialPlayerResponse\s*=\s*({.*?});(?:var|<\/script>)', html)
-        if not match:
-            raise Exception("ytInitialPlayerResponse not found. YouTube layout changed or IP blocked.")
+        payload = {
+            "context": {
+                "client": {
+                    "hl": "en",
+                    "gl": "US",
+                    "clientName": "IOS",
+                    "clientVersion": "19.28.1",
+                    "osName": "iOS",
+                    "osVersion": "17.5.1",
+                    "deviceMake": "Apple",
+                    "deviceModel": "iPhone16,2",
+                    "visitorData": visitor_data
+                }
+            },
+            "videoId": video_id
+        }
 
-        data = json.loads(match.group(1))
-        details = data.get('videoDetails', {})
-        streaming_data = data.get('streamingData', {})
+        api_headers = {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+            'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+        }
+
+        response = await self.client.request('POST', 'https://www.youtube.com/youtubei/v1/player', json=payload, headers=api_headers)
+        api_data = response.json()
+        
+        if not details.get('title'):
+            details = api_data.get('videoDetails', {})
+            
+        streaming_data = api_data.get('streamingData', {})
 
         if not details:
             raise Exception("Video details not found inside player response.")
 
-        formats = []
+        subscribers = ""
+        likes = ""
+        comments = ""
+        
+        try:
+            contents = initial_data.get('contents', {}).get('twoColumnWatchNextResults', {}).get('results', {}).get('results', {}).get('contents', [])
+            for c in contents:
+                sec_info = c.get('videoSecondaryInfoRenderer')
+                if sec_info:
+                    stext = sec_info.get('owner', {}).get('videoOwnerRenderer', {}).get('subscriberCountText', {}).get('simpleText')
+                    if stext: subscribers = stext
+
+            panels = initial_data.get('engagementPanels', [])
+            for p in panels:
+                sr = p.get('engagementPanelSectionListRenderer', {})
+                if sr.get('targetId') == 'engagement-panel-structured-description':
+                    items = sr.get('content', {}).get('structuredDescriptionContentRenderer', {}).get('items', [])
+                    for i in items:
+                        factoids = i.get('videoDescriptionHeaderRenderer', {}).get('factoid', [])
+                        for f in factoids:
+                            acc = f.get('factoidRenderer', {}).get('accessibilityText', '')
+                            if 'like' in acc.lower():
+                                likes = acc
+                                
+                if sr.get('panelIdentifier') == 'engagement-panel-comments-section':
+                    runs = sr.get('header', {}).get('engagementPanelTitleHeaderRenderer', {}).get('contextualInfo', {}).get('runs', [])
+                    if runs:
+                        comments = runs[0].get('text', '')
+        except:
+            pass
+
+        video_combined = []
+        video_only = []
+        audio_only = []
+        
         raw_formats = streaming_data.get('formats', []) + streaming_data.get('adaptiveFormats', [])
 
         for f in raw_formats:
             if 'url' in f:
                 mime_type = f.get('mimeType', '')
-                formats.append({
+                has_audio = 'audio/' in mime_type
+                has_video = 'video/' in mime_type
+                
+                f_obj = {
                     'url': f['url'],
                     'mimeType': mime_type,
                     'width': f.get('width'),
                     'height': f.get('height'),
                     'quality': f.get('qualityLabel') or f.get('quality'),
                     'bitrate': f.get('bitrate'),
-                    'hasAudio': 'audio/' in mime_type,
-                    'hasVideo': 'video/' in mime_type
-                })
+                    'hasAudio': has_audio,
+                    'hasVideo': has_video
+                }
+                
+                if has_video and has_audio:
+                    video_combined.append(f_obj)
+                elif has_video:
+                    video_only.append(f_obj)
+                elif has_audio:
+                    audio_only.append(f_obj)
 
         thumbnails = details.get('thumbnail', {}).get('thumbnails', [])
         best_thumbnail = thumbnails[-1]['url'] if thumbnails else ''
@@ -47,9 +142,16 @@ class YouTubeScraper:
         return {
             'title': details.get('title', ''),
             'author': details.get('author', ''),
+            'subscribers': subscribers,
             'description': details.get('shortDescription', ''),
             'views': details.get('viewCount', '0'),
+            'likes': likes,
+            'comments': comments,
             'durationSeconds': details.get('lengthSeconds', '0'),
             'thumbnail': best_thumbnail,
-            'formats': formats
+            'formats': {
+                'video': video_combined,
+                'videoOnly': video_only,
+                'audio': audio_only
+            }
         }

package/test_aqz.js

@@ -0,0 +1,54 @@
+const undici = require('undici');
+const fs = require('fs');
+
+async function testNextApi() {
+  const videoId = 'aqz-KE-bpKQ';
+  const url = `https://www.youtube.com/watch?v=${videoId}`;
+  
+  const htmlRes = await undici.request(url, {
+    method: 'GET',
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/115.0.0.0 Safari/537.36',
+      'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'
+    }
+  });
+
+  const html = await htmlRes.body.text();
+  let visitorData = '';
+  const vdMatch = html.match(/"visitorData"\s*:\s*"([^"]+)"/);
+  if (vdMatch) visitorData = vdMatch[1];
+
+  const payload = {
+    context: {
+      client: {
+        hl: 'en',
+        gl: 'US',
+        clientName: 'IOS',
+        clientVersion: '19.28.1',
+        osName: 'iOS',
+        osVersion: '17.5.1',
+        deviceMake: 'Apple',
+        deviceModel: 'iPhone16,2',
+        visitorData: visitorData
+      }
+    },
+    videoId: videoId
+  };
+
+  const res = await undici.request('https://www.youtube.com/youtubei/v1/player', {
+    method: 'POST',
+    headers: {
+      'Accept': 'application/json',
+      'Content-Type': 'application/json',
+      'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+    },
+    body: JSON.stringify(payload)
+  });
+
+  const apiData = await res.body.json();
+  fs.writeFileSync('player_api_dump.json', JSON.stringify(apiData, null, 2));
+  console.log('Saved dump to player_api_dump.json, Playable:', apiData?.playabilityStatus?.status);
+  console.log('Formats:', apiData?.streamingData?.formats?.length || 0, 'Adaptive', apiData?.streamingData?.adaptiveFormats?.length || 0);
+}
+
+testNextApi().catch(console.error);

package/test_extract.js

@@ -0,0 +1,41 @@
+const fs = require('fs');
+
+const data = JSON.parse(fs.readFileSync('ytInitialData_dump.json', 'utf8'));
+
+let subscribers = '';
+let likes = '';
+let comments = '';
+
+try {
+  const videoPrimaryInfo = data?.contents?.twoColumnWatchNextResults?.results?.results?.contents?.find(c => c.videoPrimaryInfoRenderer)?.videoPrimaryInfoRenderer;
+  const videoSecondaryInfo = data?.contents?.twoColumnWatchNextResults?.results?.results?.contents?.find(c => c.videoSecondaryInfoRenderer)?.videoSecondaryInfoRenderer;
+  
+  // Subscribers
+  if (videoSecondaryInfo?.owner?.videoOwnerRenderer?.subscriberCountText?.simpleText) {
+      subscribers = videoSecondaryInfo.owner.videoOwnerRenderer.subscriberCountText.simpleText;
+  }
+  
+  // Try to get likes from factoids (modern UI)
+  const factoids = data?.engagementPanels?.find(p => p.engagementPanelSectionListRenderer?.targetId === 'engagement-panel-structured-description')
+                    ?.engagementPanelSectionListRenderer?.content?.structuredDescriptionContentRenderer?.items?.find(i => i.videoDescriptionHeaderRenderer)?.videoDescriptionHeaderRenderer?.factoid || [];
+                    
+  const likesFactoid = factoids.find(f => f.factoidRenderer?.accessibilityText?.toLowerCase().includes('like'));
+  if (likesFactoid) {
+      likes = likesFactoid.factoidRenderer.accessibilityText;
+  }
+  
+  // Try to get comments
+  // Usually comments count is harder to find in initialData without scrolling, but let's check engagement panels
+  const commentsPanel = data?.engagementPanels?.find(p => p.engagementPanelSectionListRenderer?.panelIdentifier === 'engagement-panel-comments-section');
+  if (commentsPanel) {
+      comments = commentsPanel.engagementPanelSectionListRenderer.header.engagementPanelTitleHeaderRenderer.contextualInfo?.runs?.[0]?.text || '';
+  }
+  
+
+} catch (e) {
+  console.error(e);
+}
+
+console.log('Subscribers:', subscribers);
+console.log('Likes:', likes);
+console.log('Comments:', comments);

package/test_metadata.js

@@ -0,0 +1,33 @@
+const fs = require('fs');
+const undici = require('undici');
+
+async function extractMetadata() {
+  const videoId = '_4j1Abt_AiM';
+  const url = `https://www.youtube.com/watch?v=${videoId}`;
+  
+  const htmlRes = await undici.request(url, {
+    method: 'GET',
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/115.0.0.0 Safari/537.36',
+      'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430',
+      'Accept-Language': 'en-US,en;q=0.9'
+    }
+  });
+
+  const html = await htmlRes.body.text();
+  const match = html.match(/var ytInitialData\s*=\s*({.*?});(?:<\/script>)/);
+  
+  if (match) {
+    const data = JSON.parse(match[1]);
+    fs.writeFileSync('ytInitialData_dump.json', JSON.stringify(data, null, 2));
+    console.log('Saved ytInitialData to ytInitialData_dump.json');
+    
+    // Attempt to drill down to find metadata.
+    // Subscribers usually in secondaryResults or owner item
+    
+  } else {
+    console.log('ytInitialData not found!');
+  }
+}
+
+extractMetadata().catch(console.error);

package/test_next.js

@@ -0,0 +1,53 @@
+const undici = require('undici');
+const fs = require('fs');
+
+async function testNextApi() {
+  const videoId = '_4j1Abt_AiM';
+  const url = `https://www.youtube.com/watch?v=${videoId}`;
+  
+  const htmlRes = await undici.request(url, {
+    method: 'GET',
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/115.0.0.0 Safari/537.36',
+      'Cookie': 'CONSENT=YES+cb.20230501-14-p0.en+FX+430'
+    }
+  });
+
+  const html = await htmlRes.body.text();
+  let visitorData = '';
+  const vdMatch = html.match(/"visitorData"\s*:\s*"([^"]+)"/);
+  if (vdMatch) visitorData = vdMatch[1];
+
+  const payload = {
+    context: {
+      client: {
+        hl: 'en',
+        gl: 'US',
+        clientName: 'IOS',
+        clientVersion: '19.28.1',
+        osName: 'iOS',
+        osVersion: '17.5.1',
+        deviceMake: 'Apple',
+        deviceModel: 'iPhone16,2',
+        visitorData: visitorData
+      }
+    },
+    videoId: videoId
+  };
+
+  const res = await undici.request('https://www.youtube.com/youtubei/v1/next', {
+    method: 'POST',
+    headers: {
+      'Accept': 'application/json',
+      'Content-Type': 'application/json',
+      'User-Agent': 'com.google.ios.youtube/19.28.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X; en_US)'
+    },
+    body: JSON.stringify(payload)
+  });
+
+  const apiData = await res.body.json();
+  fs.writeFileSync('next_api_dump.json', JSON.stringify(apiData, null, 2));
+  console.log('Saved dump to next_api_dump.json');
+}
+
+testNextApi().catch(console.error);

package/tests/test.py

@@ -24,9 +24,13 @@ async def run_tests():
         print("2. Testing YouTube Scraper...")
         yt = YouTubeScraper()
         yt_res = await yt.scrape_video("https://www.youtube.com/watch?v=aqz-KE-bpKQ")
-        print(f"YouTube Scraper Output: Title = {yt_res['title']}")
-        print(f"YouTube Scraper Output: Duration = {yt_res['durationSeconds']} seconds")
-        print(f"YouTube Scraper Output: Found {len(yt_res['formats'])} formats")
+        print(f"YouTube Scraper Output: Title = {yt_res.get('title')}")
+        print(f"YouTube Scraper Output: Subscribers = {yt_res.get('subscribers')}")
+        print(f"YouTube Scraper Output: Likes = {yt_res.get('likes')}")
+        print(f"YouTube Scraper Output: Comments = {yt_res.get('comments')}")
+        print(f"YouTube Scraper Output: Duration = {yt_res.get('durationSeconds')} seconds")
+        formats = yt_res.get('formats', {})
+        print(f"YouTube Scraper Output: Found {len(formats.get('video', []))} video, {len(formats.get('videoOnly', []))} video-only, and {len(formats.get('audio', []))} audio formats")
         await yt.client.close()
 
         print("\n" + "-" * 33)

package/tests/test.ts

@@ -19,9 +19,12 @@ async function runTests() {
         const ytClient = straw.youtube();
         // Use a generic test video like Big Buck Bunny
         const ytResult = await ytClient.scrapeVideo('https://www.youtube.com/watch?v=aqz-KE-bpKQ');
-        console.log(`YouTube Scraper Output: Title = ${ytResult.title}`);
-        console.log(`YouTube Scraper Output: Duration = ${ytResult.durationSeconds} seconds`);
-        console.log(`YouTube Scraper Output: Found ${ytResult.formats.length} formats`);
+        console.log('YouTube Scraper Output: Title =', ytResult.title);
+  console.log('YouTube Scraper Output: Subscribers =', ytResult.subscribers);
+  console.log('YouTube Scraper Output: Likes =', ytResult.likes);
+  console.log('YouTube Scraper Output: Comments =', ytResult.comments);
+  console.log('YouTube Scraper Output: Duration =', ytResult.durationSeconds, 'seconds');
+  console.log(`YouTube Scraper Output: Found ${ytResult.formats.video.length} video (combined), ${ytResult.formats.videoOnly.length} video-only, and ${ytResult.formats.audio.length} audio formats.`);
 
         console.log('\n---------------------------------');