harvesterreporter 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f73fb3f8cd5bded2829d4d442174aebe3847a4d9
+  data.tar.gz: 1b39ee438b65279bd66847eb2493a75cf2afc961
+SHA512:
+  metadata.gz: 61cb9222dc77d8a5acbcaf8d534bdae945428b9e5512ab2812c16a51abd90553453242afbc6c867bee5d1034caf958035ef502e218ab3dd11019ec11c06a6493
+  data.tar.gz: 47f20585fbfce28c29c5205ea4b12fa407e5c1c757af05b7beb4cb89bbf714144b855a0d1d58f3ad8a78c54b4a9f7e52bf27dfd66a248f4d14bf6371e8a30bec

data/lib/harvesterreporter.rb

@@ -0,0 +1,42 @@
+require 'curb'
+require 'json'
+
+class HarvesterReporter
+  def initialize(cm_hash = nil)
+    @cm_url = cm_hash[:crawler_manager_url] if cm_hash
+    @selector_id = cm_hash[:selector_id] if cm_hash
+
+    # Output array for when Harvester isn't used
+    @output = Array.new
+  end
+
+  # Figure out how to report results
+  def report_results(results, link)
+    if @cm_url
+      report_incremental(results, link)
+    else
+      report_batch(results)
+    end
+  end
+
+  # Report all results in one JSON
+  def report_batch(results)
+    results.each do |result|
+      @output.push(result)
+    end
+  end
+
+  # Report results back to Harvester incrementally
+  def report_incremental(results, link)
+    curl_url = @cm_url+"/relay_results"
+    c = Curl::Easy.http_post(curl_url,
+                             Curl::PostField.content('selector_id', @selector_id),
+                             Curl::PostField.content('status_message', "Collected " + link),
+                             Curl::PostField.content('results', JSON.pretty_generate(results)))
+  end
+
+  # Output JSON
+  def gen_json
+    return JSON.pretty_generate(@output)
+  end
+end

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: harvesterreporter
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-06 00:00:00.000000000 Z
+dependencies: []
+description: Incremental result reporting for Transparency Toolkit
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/harvesterreporter.rb
+homepage: https://github.com/TransparencyToolkit/HarvesterReporter
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Reports results from crawlers to Harvester
+test_files: []
+has_rdoc: