twittercrawler 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1230a596110d9376fa6243651b2994f2d0228afb
-  data.tar.gz: f93449ae2abbc85227da02665f31ea6848712615
+  metadata.gz: 8b2a8f686154fcb64246ed8c3bef02f324df3892
+  data.tar.gz: f8219382c56cfacad13e7fbd55a25e156280976f
 SHA512:
-  metadata.gz: 2dd0c8b7e41565e4f3c71bfbbabfbe22b8f9cbec6a3bde803270ec62e11c591145e1e0bd4263a392627341f6ac96b8c51c89d39da9a90f9bf36465e41d69a66f
-  data.tar.gz: bbdb93efc8b9450b6da96f31c90d1446781ced81d5a2adf6e7540cb09b983d9bd5a1be12f844bca71dd36894e1836c8bfc05a810df4d088f6062ac19cc4ede49
+  metadata.gz: f21e73e1bd854174182fee749153be4499c2b2f6742ae6ca09aa1e85e2d100e9eab9e2af3898400aa0f0bb6d9b071292cf4393d80b45cfaf37d0e22780a0834b
+  data.tar.gz: 7a8c376ffcaaeb0ae55964251dd7a0efaf5bece5fd81a03c99eed231784bc29d1f5c02855b31ee06931faa97e3da988146210afecfbb6a0304497fd0684f12f1

data/lib/twitter_parser.rb

@@ -22,7 +22,8 @@ class TwitterParser
         favorite_count: get_favorite_count,
         reply_count: get_reply_count,
         mention_names: get_mentions[0],
-        mention_uids: get_mentions[1]
+        mention_uids: get_mentions[1],
+        time_collected: Time.now
       }
     end
   end

data/lib/twittercrawler.rb

@@ -2,14 +2,14 @@ require 'requestmanager'
 require 'selenium-webdriver'
 require 'pry'
 require 'nokogiri'
+require 'curb'
 
 load 'twitter_parser.rb'
 
 class TwitterCrawler
-  def initialize(search_term, operator, requests, cm_hash)
+  def initialize(search_term, operator, cm_hash)
     @search_term = search_term
     @operator = operator
-    @requests = requests
     @output = Array.new
 
     # Handle crawler manager info
@@ -26,45 +26,56 @@ class TwitterCrawler
     end
   end
 
-  def crawl
-    @requests.get_page("https://twitter.com/search?f=tweets&q="+gen_query)
-    scroll_down(0)
-    get_tweets
-    @requests.close_all_browsers
+  # Parse the tweets into html
+  def parse_tweets(tweets)
+    return tweets.map do |tweet|
+      parser = TwitterParser.new(tweet.to_html)
+      parser.parse_tweet
+    end
   end
 
-  # Get the tweets on the page
-  def get_tweets
-    browser = @requests.get_most_recent_browser[1].first
-    tweets = browser.find_elements(class: "tweet")
-
-    # Parse each tweet
-    tweets.each do |tweet|
-      # Parse tweet
-      tweet_html = tweet.attribute("innerHTML")
-      parser = TwitterParser.new(tweet_html)
-      parsed_tweet = parser.parse_tweet
-
-      # Report results
-      if parsed_tweet
-        report_results([parsed_tweet], parsed_tweet[:tweet_link])
-      end
+  # Generate the query url for Twitter
+  def gen_query_url(start_tweet, end_tweet)
+    # Base query url
+    query_url = "https://twitter.com/i/search/timeline?f=tweets&vertical=news&q="+gen_query+"&src=typd&include_available_features=1&include_entities=1"
+
+    # Gen query URL
+    if start_tweet && end_tweet
+      query_url += "&max_position=TWEET-"+start_tweet+"-"+end_tweet
     end
+    return query_url
   end
 
-  # Scroll down to the bottom
-  def scroll_down(last_tweet_num)
-    # Scroll down to last tweet
-    browser = @requests.get_most_recent_browser[1].first
-    tweets = browser.find_elements(class: "tweet")
-    tweets[tweets.length-2].location_once_scrolled_into_view
-
-    # Check if it should be rerun
-    sleep(1)
-    tweet_count = browser.find_elements(class: "tweet").length
-    if tweet_count > last_tweet_num
-      scroll_down(tweet_count)
-    end   
+  # Query tweets
+  def query_tweets(start_tweet, end_tweet)
+    # Run Query and parse results
+    c = Curl::Easy.perform(gen_query_url(start_tweet, end_tweet))
+    curl_items = JSON.parse(c.body_str)
+    tweets = Nokogiri::HTML.parse(curl_items["items_html"]).css(".tweet") if curl_items["items_html"]
+
+    # Save results
+    parsed_tweets = parse_tweets(tweets)
+    report_results(parsed_tweets, "Saving "+parsed_tweets.length.to_s+" tweets")
+    
+    # Recurse when needed
+    if !parsed_tweets.empty?
+      start_tweet, end_tweet = get_tweet_range(parsed_tweets, end_tweet)
+      query_tweets(start_tweet, end_tweet)
+    end
+  end
+
+  # Get the ID for a tweet
+  def get_tweet_id(tweet)
+    return tweet[:tweet_link].split("/").last
+  end
+  
+  # Get start and end tweets
+  def get_tweet_range(parsed_tweets, end_tweet)
+    if end_tweet # Keeep latest tweet as same
+      return get_tweet_id(parsed_tweets.last), end_tweet
+    else # Get updated start tweet
+      return get_tweet_id(parsed_tweets.last), get_tweet_id(parsed_tweets.first)
+    end
   end
 
   # Figure out how to report results
@@ -98,3 +109,4 @@ class TwitterCrawler
   end
 end
 
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twittercrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-03 00:00:00.000000000 Z
+date: 2017-02-07 00:00:00.000000000 Z
 dependencies: []
 description: Crawls Twitter
 email: shidash@shidash.com