strigil 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8ef92f9f1864c48161d4cc1095f76e25e9fb3c8d
-  data.tar.gz: 78c6e7f7cf9c967700570dec8e24b8c658cb809c
+  metadata.gz: 14e5b5afe824abec6ddcae00f102eb785b55c033
+  data.tar.gz: 2674268125891b755c502b7d0321e352c99c66a8
 SHA512:
-  metadata.gz: e43012eed26881ef3b460c181193b32f815f0c5ad264ba776a011892e6122078bcdeb2714b190d2c535e93f0a4a3cddcda8080db3d2f15e36d0da58ae7417760
-  data.tar.gz: f61ce715184dd3fd6fa9e69b1f0969c3f550a35a45517dc19f86b960186fddf7125d6d4bca1f213109d531f4536acb07d7a8e0f47d3853d8cb8d13ccaf477f6a
+  metadata.gz: 2107c4baebc045cf2034bc18aec704065dae8677a880db227cf227d3fc934ba04f2ef035adddee0a1d5cba9b163df83ac391b752135dc261cc4a64d7705f78ca
+  data.tar.gz: '025218138f22d2332c3e89ef1a59c511759a49b3ecabb636e8f49cb650a0e9e342e835b8751beaa0c9422c6ec12ffdd53c779de8a613ed29b3a8f04dc11d993a'

data/lib/strigil.rb

@@ -1,8 +1,19 @@
 class Strigil
   def self.engage(user)
-    client = Strigil::StrigilClient.new(user)
+    client_config
+
+    puts "Initializing client..."
+    client = Strigil::StrigilClient.new(
+      username: @username,
+      password: @password,
+      browser: @browser,
+      target: user
+    )
+
+    puts "Initializing comments store..."
     comments = Strigil::Comments.new
 
+    puts "Beginning scrape. This may take a while."
     processing = true
     while processing == true
       comments.add_comments(client.pull_comments)
@@ -22,6 +33,27 @@ class Strigil
       end
     end
   end
+
+  private
+
+  def self.client_config
+    puts "Valid Reddit account details are neccessary to scrape correctly."
+    puts "This information is _not_ sent to any third party - it is simply"
+    puts "used to properly configure how user comments are displayed in"
+    puts "order to scrape them correctly. Feel free to use a throwaway."
+    puts "Username:"
+    print "> "
+    @username = gets.chomp
+    puts "Password:"
+    print "> "
+    @password = gets.chomp
+    puts "Do you have Chrome or Firefox installed?"
+    puts "Type either 'chrome' or 'firefox' without quotes."
+    @browser = gets.chomp
+  end
+end
+
+class ConfigurationError < StandardError
 end
 
 require 'strigil/comment'

data/lib/strigil/comments_parser.rb

@@ -1,10 +1,12 @@
 class Strigil::CommentsParser
 
-  def self.parse(comments)
+  def self.parse(client)
+    comments = get_comments(client)
+
     comments.map do |comment|
       Strigil::Comment.new(
-        author: comment.attribute("data-author"),
-        subreddit: comment.attribute("data-subreddit"),
+        author: get_author(comment),
+        subreddit: get_subreddit(comment),
         permalink: get_permalink(comment),
         timestamp: get_timestamp(comment),
         text: get_text(comment)
@@ -14,19 +16,28 @@ class Strigil::CommentsParser
 
   private
 
+  def self.get_comments(client)
+    client.find_elements(class: "comment")
+  end
+
+  def self.get_author(comment)
+    comment.attribute("data-author")
+  end
+
+  def self.get_subreddit(comment)
+    comment.attribute("data-subreddit")
+  end
+
   def self.get_permalink(comment)
-    entry = get_entry(comment)
-    entry.find_element(class: "bylink").attribute(:href)
+    'https://reddit.com/r/' + comment.attribute("data-permalink")
   end
 
   def self.get_timestamp(comment)
-    entry = get_entry(comment)
-    entry.find_element(tag_name: "time").attribute(:title)
+    get_entry(comment).find_element(tag_name: "time").attribute(:title)
   end
 
   def self.get_text(comment)
-    entry = get_entry(comment)
-    entry.find_element(class: "usertext-body").text
+    get_entry(comment).find_element(class: "usertext-body").text
   end
 
   def self.get_entry(comment)

data/lib/strigil/strigil_client.rb

@@ -4,14 +4,20 @@ require 'webdrivers'
 class Strigil::StrigilClient
   attr_reader :client
 
-  def initialize(user)
-    @client = Selenium::WebDriver.for :chrome
-    client.navigate.to "https://reddit.com/u/#{user}/comments"
+  def initialize(params)
+    target = params[:target]
+    username = params[:username]
+    password = params[:password]
+
+    @client = Selenium::WebDriver.for(params[:browser].to_sym)
+
+    login_and_make_legacy(username, password)
+
+    client.navigate.to "https://reddit.com/u/#{target}/comments"
   end
 
   def pull_comments
-    raw_comments = client.find_elements(class: "comment")
-    Strigil::CommentsParser.parse(raw_comments)
+    Strigil::CommentsParser.parse(client)
   end
 
   def close
@@ -26,6 +32,46 @@ class Strigil::StrigilClient
     end
   end
 
+  private
+
+  def login_and_make_legacy(username, password)
+    login_to_reddit(username, password)
+    set_legacy_preference
+  end
+
+  def login_to_reddit(username, password)
+    client.navigate.to 'https://reddit.com/login'
+
+    login_panel = client.find_element(id: "login-form")
+
+    username_field = login_panel.find_element(id: "user_login")
+    password_field = login_panel.find_element(id: "passwd_login")
+    login_button = login_panel.find_element(tag_name: "button", class: "c-btn")
+
+    username_field.send_keys(username)
+    password_field.send_keys(password)
+
+    login_button.click
+
+    wait = Selenium::WebDriver::Wait.new(timeout: 5)
+    wait.until do
+      client.find_element(id: "header-bottom-right").find_element(link_text: username)
+    end
+  end
+
+  def set_legacy_preference
+    client.navigate.to 'https://reddit.com/prefs/'
+
+    legacy_box = client.find_element(id: "profile_opt_out")
+
+    unless legacy_box.attribute(:checked)
+      legacy_box.click
+
+      save_button = client.find_element(class: "save-preferences")
+      save_button.click
+    end
+  end
+
 end
 
 class EndOfQueueError < StandardError

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: strigil
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Harry Stebbins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-03 00:00:00.000000000 Z
+date: 2018-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver