strigil 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8ef92f9f1864c48161d4cc1095f76e25e9fb3c8d
+  data.tar.gz: 78c6e7f7cf9c967700570dec8e24b8c658cb809c
+SHA512:
+  metadata.gz: e43012eed26881ef3b460c181193b32f815f0c5ad264ba776a011892e6122078bcdeb2714b190d2c535e93f0a4a3cddcda8080db3d2f15e36d0da58ae7417760
+  data.tar.gz: f61ce715184dd3fd6fa9e69b1f0969c3f550a35a45517dc19f86b960186fddf7125d6d4bca1f213109d531f4536acb07d7a8e0f47d3853d8cb8d13ccaf477f6a

data/lib/strigil.rb

@@ -0,0 +1,30 @@
+class Strigil
+  def self.engage(user)
+    client = Strigil::StrigilClient.new(user)
+    comments = Strigil::Comments.new
+
+    processing = true
+    while processing == true
+      comments.add_comments(client.pull_comments)
+
+      begin
+        client.iterate
+      rescue EndOfQueueError
+        processing = false
+
+        client.close
+        comments.save
+
+        puts "Pulled #{comments.log.size} comments."
+        puts "JSON data stored in ./data/comments.json"
+
+        comments.clear
+      end
+    end
+  end
+end
+
+require 'strigil/comment'
+require 'strigil/comments'
+require 'strigil/comments_parser'
+require 'strigil/strigil_client'

data/lib/strigil/comment.rb

@@ -0,0 +1,32 @@
+require 'time'
+
+class Strigil::Comment
+  attr_reader :author, :subreddit, :permalink, :timestamp, :text
+
+  def initialize(params={})
+    @author = params[:author]
+    @subreddit = params[:subreddit]
+    @permalink = params[:permalink]
+    @timestamp = Time.parse(params[:timestamp])
+    @text = params[:text]
+  end
+
+  def to_json
+    {
+      author: author,
+      subreddit: subreddit,
+      permalink: permalink,
+      timestamp: timestamp,
+      text: text
+    }.to_json
+  end
+
+  def display
+    %{
+      posted by #{author} on /r/#{subreddit}
+      #{timestamp} | #{permalink}
+      -------------------------------------
+      #{text}
+    }
+  end
+end

data/lib/strigil/comments.rb

@@ -0,0 +1,42 @@
+require 'fileutils'
+
+class Strigil::Comments
+  attr_accessor :log
+
+  def initialize
+    @log = []
+  end
+
+  def add_comments(comments)
+    @log += comments
+  end
+
+  def add_comment(comment)
+    @log.push(comment)
+  end
+
+  def save
+    unless File.exist?('./data/')
+      FileUtils.mkdir('./data/')
+    end
+    File.open("./data/comments.json", 'w') do |f|
+      f.write(log_to_json)
+    end
+  end
+
+  def clear
+    @log = []
+  end
+
+  private
+
+  def log_to_json
+    result = '['
+    log.each do |comment|
+      result << comment.to_json
+      result << ',' unless comment == log.last
+    end
+    result << ']'
+    result 
+  end
+end

data/lib/strigil/comments_parser.rb

@@ -0,0 +1,35 @@
+class Strigil::CommentsParser
+
+  def self.parse(comments)
+    comments.map do |comment|
+      Strigil::Comment.new(
+        author: comment.attribute("data-author"),
+        subreddit: comment.attribute("data-subreddit"),
+        permalink: get_permalink(comment),
+        timestamp: get_timestamp(comment),
+        text: get_text(comment)
+      )
+    end
+  end
+
+  private
+
+  def self.get_permalink(comment)
+    entry = get_entry(comment)
+    entry.find_element(class: "bylink").attribute(:href)
+  end
+
+  def self.get_timestamp(comment)
+    entry = get_entry(comment)
+    entry.find_element(tag_name: "time").attribute(:title)
+  end
+
+  def self.get_text(comment)
+    entry = get_entry(comment)
+    entry.find_element(class: "usertext-body").text
+  end
+
+  def self.get_entry(comment)
+    comment.find_element(class: "entry")
+  end
+end

data/lib/strigil/strigil_client.rb

@@ -0,0 +1,32 @@
+require 'selenium-webdriver'
+require 'webdrivers'
+
+class Strigil::StrigilClient
+  attr_reader :client
+
+  def initialize(user)
+    @client = Selenium::WebDriver.for :chrome
+    client.navigate.to "https://reddit.com/u/#{user}/comments"
+  end
+
+  def pull_comments
+    raw_comments = client.find_elements(class: "comment")
+    Strigil::CommentsParser.parse(raw_comments)
+  end
+
+  def close
+    client.close
+  end
+
+  def iterate
+    begin
+      client.find_element(link_text: "next ›").click
+    rescue
+      raise EndOfQueueError
+    end
+  end
+
+end
+
+class EndOfQueueError < StandardError
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: strigil
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Harry Stebbins
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-04-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.11'
+- !ruby/object:Gem::Dependency
+  name: webdrivers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: 
+email: hcstebbins@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/strigil.rb
+- lib/strigil/comment.rb
+- lib/strigil/comments.rb
+- lib/strigil/comments_parser.rb
+- lib/strigil/strigil_client.rb
+homepage: https://github.com/stebbins/strigil
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Strigil is a gem for easily scraping a Reddit user's comment history into
+  a JSON file.
+test_files: []