twittercrawler 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b578c69314b844603dcc13e8968ce0ad5f11024b
-  data.tar.gz: 0f7df31fc66bad9e79d890c50b43ffcaccee4f63
+  metadata.gz: 25d1d3c957757eeca51247c704e42f9c009a8d09
+  data.tar.gz: 55bcea247d9d2c926b6fdafe51c876de6ed30049
 SHA512:
-  metadata.gz: 617241a871ac10423a4096a71b282217a8e53ce182b09df1cefe1ea3e625611a9f55b139cdd6429607d32583c49edf6e92c33c897e3c83f050c3bfbead1be7d5
-  data.tar.gz: 3f1039aa7975a23b8f284ea3f255c15ce2344d9196ca48d48f0a8d78a737d84eac1cc590135bf553e7360b34b20916cec2901d3ee325ba98263c9ab4be24be04
+  metadata.gz: a2bf6e5315fae1c511f63d94dd928c99998d7ae791983837fc9564e5e070ee4ca8afe73c2a5d959855e2e38706688db5e511abf208873bb1981bb3d2341835eb
+  data.tar.gz: 8495e3060c561ee3eb4885a6e8686a64256cd8a85a17562110030eff6c2da66765487f67e72370dd31ae07c350cd4633bdd798756c9abdac4f7addc69aa5908d

data/lib/twittercrawler.rb

@@ -6,11 +6,15 @@ require 'nokogiri'
 load 'twitter_parser.rb'
 
 class TwitterCrawler
-  def initialize(search_term, operator, requests)
+  def initialize(search_term, operator, requests, cm_hash)
     @search_term = search_term
     @operator = operator
     @requests = requests
     @output = Array.new
+
+    # Handle crawler manager info
+    @cm_url = cm_hash[:crawler_manager_url] if cm_hash
+    @selector_id = cm_hash[:selector_id] if cm_hash
   end
 
   # Generate advanced query
@@ -36,9 +40,13 @@ class TwitterCrawler
 
     # Parse each tweet
     tweets.each do |tweet|
+      # Add tweet
       tweet_html = tweet.attribute("innerHTML")
       parser = TwitterParser.new(tweet_html)
-      @output.push(parser.parse_tweet)
+      parsed_tweet = parser.parse_tweet
+
+      # Report results
+      report_results([pared_tweet], parsed_tweet[:tweet_link])
     end
   end
 
@@ -57,6 +65,31 @@ class TwitterCrawler
     end   
   end
 
+  # Figure out how to report results
+  def report_results(results, link)
+    if @cm_url
+      report_incremental(results, link)
+    else
+      report_batch(results)
+    end
+  end
+
+  # Report all results in one JSON
+  def report_batch(results)
+    results.each do |result|
+      @output.push(result)
+    end
+  end
+
+  # Report results back to Harvester incrementally
+  def report_incremental(results, link)
+    curl_url = @cm_url+"/relay_results"
+    c = Curl::Easy.http_post(curl_url,
+                             Curl::PostField.content('selector_id', @selector_id),
+                             Curl::PostField.content('status_message', "Collected " + link),
+                             Curl::PostField.content('results', JSON.pretty_generate(results)))
+  end
+
   # Generate JSON for output
   def gen_json
     JSON.pretty_generate(@output)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twittercrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - M. C. McGrath