generalscraper 0.0.26 → 0.0.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85a128c0ae855a52ee3842ba40807b7adc84a43a
-  data.tar.gz: c7b8aa3648f3735c0f69ac7f91884bbd04a2450a
+  metadata.gz: 0fe81408deb27c96a9a67231be5953c8ddf11b23
+  data.tar.gz: 1297c834ccdae5daf1ef1b8664f555f6481995f4
 SHA512:
-  metadata.gz: 54c19ac3c90c3b99b9be0115945c1f59ad71951285a5d779090f4ec91c3e4fd2fbf995c4393a27cc0d73d0efee515d62182b2851cb6205f7f4dfe2a0f17fb84d
-  data.tar.gz: 0ea35567e7c33f9b46ab678f2bf3231d8a7b79a883e749728de0eaa9d46e772e91cc4346bc4ce3c8ff4f9a4855b0acd06cc338e63920c1228a7e5b2ef5f36837
+  metadata.gz: b5ae3df4ea3ea229dc7e89e55bb4e56f86e6d2bc93b38e064d38bb772b1a02fad6fe1ec58698e88c97714c269ca4c9bef5a95d8522c2edf4443141d71f6c9fae
+  data.tar.gz: e63d78332f8decb130f1968939752d8464a5eb6be4d5da151403b3b7687e426128e3670f0ee37c64472b5371b1d5d971bd01b9dfaeb1b86409b0d994734d5be0

data/lib/generalscraper.rb

@@ -6,6 +6,7 @@ require 'pry'
 
 load 'parse_page.rb'
 load 'captcha.rb'
+load 'translate_page.rb'
 
 class GeneralScraper
   include ParsePage

data/lib/translate_page.rb

@@ -0,0 +1,75 @@
+require 'nokogiri'
+require 'requestmanager'
+require 'json'
+
+class TranslatePage
+  def initialize(urls, requests)
+    @urls = urls
+    @requests = requests
+    @output = Array.new
+  end
+
+  # Setup browser for translate
+  def setup_browser
+    @requests.get_page("https://translate.google.com")
+    return @requests.get_most_recent_browser[1][0]
+  end
+
+  # First request
+  def first_request(url, browser)
+    # Enter URL into translate form
+    translate_form = browser.find_element(id: "source")
+    translate_form.send_keys(url)
+
+    # Click the button to translate to a particular language
+    click_button = browser.find_elements(:xpath, "//*[@value='es']").last
+    click_button.click
+
+    # Press Translate button, then switch back to orginal
+    browser.find_element(id: "gt-submit").click
+  end
+
+  # Next request
+  def nth_request(url, browser)
+    browser.switch_to.default_content
+    form_element = browser.find_element(name: "q")
+    form_element.clear
+    form_element.send_keys(url)
+    form_element.submit
+  end
+
+  # Translate the pages
+  def translate
+    browser = setup_browser
+
+    # Go through each link
+    counter = 0
+    @urls.each do |url|
+      # Run translate on each page
+      if counter == 0
+        first_request(url, browser)
+        counter+=1
+      else
+        nth_request(url, browser)
+      end
+
+      # Get html
+      @output.push({url: url, html: get_iframe_html(browser)})
+    end
+
+    # Clean up
+    @requests.close_all_browsers
+    return @output
+  end
+
+  # Get iframe
+  def get_iframe_html(browser)
+    sleep(3)
+    browser.find_element(id: "anno2").click
+
+    # Get HTML inside the iframe
+    browser.switch_to.frame(0)
+    iframe_html = browser.find_element(class: "os-linux").attribute("innerHTML")
+    return iframe_html
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: generalscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.26
+  version: 0.0.27
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-30 00:00:00.000000000 Z
+date: 2017-02-18 00:00:00.000000000 Z
 dependencies: []
 description: Scrapes Google
 email: shidash@shidash.com
@@ -19,6 +19,7 @@ files:
 - lib/captcha.rb
 - lib/generalscraper.rb
 - lib/parse_page.rb
+- lib/translate_page.rb
 homepage: https://github.com/TransparencyToolkit/generalscraper
 licenses:
 - GPL