tilde-scraper 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 233e57dca85aa06d060975d41dc93b3c42b4f8b5b780ea98bd3bc55a240fa821
+  data.tar.gz: b7bdc67f83cf7a408ab46c18c37ac6043ed9d9efe27c46cce90b8643b7389509
+SHA512:
+  metadata.gz: 4b141b97282e2f2559fd7374c0a6267c11bdb0590635addc743868b17eb44de27908ce15d325ab774d3f133c9cbc1a3686944bd2df1962c2cf65520ade595102
+  data.tar.gz: f55175e57c0273c2ce319c9d4dc8bc347c57e5499799b63f9a378d5d1e89001f80a1d4369f89733b96b26341a58769113c41e8f0c0ef2a0517426295c842995f

data/bin/tilde-scraper

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../lib/tilde_scraper.rb'
+
+TildeScraper::CommandLineInterface.new.run

data/config/enviornment.rb

@@ -0,0 +1,16 @@
+require 'open-uri'
+require 'pry'
+require 'nokogiri'
+
+
+require_relative '../lib/tilde_scraper/concerns/memorable.rb'
+require_relative '../lib/tilde_scraper/topic/topic.rb'
+require_relative '../lib/tilde_scraper/topic/link_topic.rb'
+require_relative '../lib/tilde_scraper/topic/text_topic.rb'
+require_relative '../lib/tilde_scraper/topic/topic'
+require_relative '../lib/tilde_scraper/api.rb'
+require_relative '../lib/tilde_scraper/cli.rb'
+require_relative '../lib/tilde_scraper/comment.rb'
+require_relative '../lib/tilde_scraper/group.rb'
+require_relative '../lib/tilde_scraper/page.rb'
+require_relative '../lib/tilde_scraper/scraper.rb'

data/lib/tilde_scraper.rb

@@ -0,0 +1,4 @@
+module TildeScraper
+    @@page_id = 0
+end
+require_relative '../config/enviornment.rb'

data/lib/tilde_scraper/api.rb

@@ -0,0 +1,41 @@
+module TildeScraper
+  @@page_id = 0
+  def self.get_page(url)
+      data = TildeScraper::Scraper.scrape_page(url)
+      #Set page_id in page data hash
+      data[0][:page_id] = @@page_id
+      #Create page object
+      page = TildeScraper::Page.create(data[0])
+
+      #Set page_id in all topic data hashes
+      #Set group in all hashes if applicible
+      data[1].each do |topic_hash|
+        if page.group
+          topic_hash[:group] = page.group
+        end
+        topic_hash[:page_id] = @@page_id
+      end
+      @@page_id += 1;
+      #Create topic objects
+      TildeScraper::Topic.create_from_array(data[1])
+      page
+  end
+
+  def self.get_page_with_comments(url)
+    page = get_page(url)
+    #Create comments for each topic
+    page.topics.each do |topic|
+      get_comments(topic.comment_link)
+    end
+  end
+
+  def self.get_groups
+    TildeScraper::Group.all.clear
+    TildeScraper::Group.create_from_array(TildeScraper::Scraper.scrape_groups("/groups"))
+  end
+
+  def self.get_comments(url)
+      comment_array = TildeScraper::Scraper.scrape_comments(url)
+      TildeScraper::Comment.create_from_array(comment_array)
+  end
+end

data/lib/tilde_scraper/cli.rb

@@ -0,0 +1,122 @@
+class TildeScraper::CommandLineInterface
+  def run
+    front_page
+    help
+    input = nil
+    until input == "exit"
+      print "Please Enter A Command: "
+      input = gets.strip.downcase
+      case input.split(" ").first
+      when "help"
+        help
+      when "exit"
+        puts "Goodbye"
+      when "frontpage"
+        front_page
+        page_list
+      when "groups"
+        groups
+      when "group"
+        group(input.split(" ")[1])
+      when "page"
+        page
+      when "list"
+        page_list
+      when "next"
+        next_page
+      when "prev"
+        prev_page
+      when "view"
+        view(input.split(" ")[1])
+      when "comments"
+        comments(input.split(" ")[1])
+      else
+        puts "Invalid command"
+      end
+    end
+  end
+
+  def view(index_string)
+    index = validate_index(index_string, @page.topics.length)
+    return nil if !index
+    @page.topics[index].display_content
+  end
+
+  def next_page
+    if @page.next_link
+      @page = TildeScraper::get_page(@page.next_link)
+      page_list
+    else
+      puts "Last page"
+    end
+  end
+
+  def prev_page
+    if @page.next_link
+      @page = TildeScraper::get_page(@page.prev_link)
+      page_list
+    else
+      puts "No previous page"
+    end
+  end
+
+  def groups
+    if TildeScraper::Group.all.length == 0
+      TildeScraper::get_groups
+    end
+    TildeScraper::Group.display
+  end
+
+  def group(index_string)
+    if TildeScraper::Group.all.length == 0
+      TildeScraper::get_groups
+    end
+    index = validate_index(index_string, TildeScraper::Group.all.length)
+    return nil if !index
+    @page = TildeScraper::get_page(TildeScraper::Group.all[index].get_url)
+    page_list
+  end
+
+  def front_page
+    @page = TildeScraper::get_page("https://tildes.net")
+  end
+
+  def comments(index_string)
+    index = validate_index(index_string, @page.topics.length)
+    return nil if !index
+    topic = @page.topics[index]
+    if topic.comments.length == 0
+      TildeScraper::get_comments(topic.comment_link)
+    end
+    TildeScraper::Comment.display_page(topic.comment_link)
+  end
+
+  def page_list
+    if !@page
+      puts "No page selected"
+    else
+      @page.display
+    end
+  end
+
+  def help
+    puts "To view this list, type: help"
+    puts "To view groups: groups"
+    puts "To view group page: group [index]"
+    puts "To return to front page: frontpage"
+    puts "To view topics of current page: list"
+    puts "To view next page: next"
+    puts "To view prev page: prev"
+    puts "To view submission contents: view [index]"
+    puts "To view submission comments: comments [index]"
+  end
+
+  private
+  def validate_index(index_string, max)
+    if !index_string || index_string.match(/\D/) || !index_string.to_i.between?(1, max)
+      puts "Invalid index"
+      return nil
+    end
+    index_string.to_i - 1
+  end
+end

data/lib/tilde_scraper/comment.rb

@@ -0,0 +1,69 @@
+class TildeScraper::Comment
+  extend TildeScraper::Memorable::ClassMethods
+  include TildeScraper::Memorable::InstanceMethods
+
+  attr_accessor :children, :text, :author, :votes, :age, :url, :level
+
+  @@all = []
+
+  def initialize(attributes)
+    add_attributes(attributes.reject { |key, val| key == :children })
+  end
+
+  def self.create_from_array(array)
+    array.map do |comment_hash|
+      comment = create(comment_hash.reject { |key, val| key == :children })
+      comment.children = self.create_from_array(comment_hash[:children])
+      comment
+    end
+  end
+
+  def self.find_by_url(url)
+    all.select { |comment| comment.url == url }
+  end
+
+  def self.find_top_by_url(url)
+    all.select { |comment| comment.url == url && comment.level == 0}
+  end
+
+  def display(indent = 0)
+    indent(indent)
+    puts self.author
+    display_text(indent)
+    indent(indent)
+    puts "Votes: " + self.votes if self.votes
+    puts "-" * 10
+  end
+
+  def display_text(indent = 0)
+    self.text.split("\n").each do |line|
+      indent(indent)
+      puts line
+    end
+  end
+
+  def self.display_page(url)
+    display(find_top_by_url(url))
+  end
+
+  def self.display(array, indent = 0)
+    #binding.pry
+    array.each do |comment|
+      comment.display(indent)
+      display(comment.children, indent + 1)
+    end
+  end
+
+  def self.all_top
+    @@all.select { |comment| comment.level == 0 }
+  end
+
+  def self.all
+    @@all
+  end
+
+  private
+  def indent(n)
+    print "\t" * n
+  end
+end

data/lib/tilde_scraper/concerns/memorable.rb

@@ -0,0 +1,25 @@
+module TildeScraper::Memorable
+  module ClassMethods
+    def create(attributes)
+      topic = new(attributes)
+      self.all << topic
+      topic
+    end
+
+    def create_from_array(array)
+      array.map do |hash|
+        create(hash)
+      end
+    end
+  end
+
+  module InstanceMethods
+    def initialize(attributes)
+      self.add_attributes(attributes)
+    end
+
+    def add_attributes(attributes)
+      attributes.each { |key, value| self.send("#{key}=", value) }
+    end
+  end
+end

data/lib/tilde_scraper/group.rb

@@ -0,0 +1,29 @@
+class TildeScraper::Group
+  extend TildeScraper::Memorable::ClassMethods
+  include TildeScraper::Memorable::InstanceMethods
+
+  attr_accessor :name, :description, :subs
+
+  @@all = []
+
+  def self.all
+    @@all
+  end
+
+  def display
+    puts name
+    puts description + "    " + subs
+  end
+
+  def get_url
+    "https://tildes.net/#{name}"
+  end
+
+  def self.display
+    all.each.with_index(1) do |group, index|
+      puts index
+      group.display
+      puts ""
+    end
+  end
+end

data/lib/tilde_scraper/page.rb

@@ -0,0 +1,45 @@
+class TildeScraper::Page
+  attr_accessor :url, :next_link, :prev_link, :order, :period, :page_id, :group
+
+  extend TildeScraper::Memorable::ClassMethods
+  include TildeScraper::Memorable::InstanceMethods
+
+  @@all = []
+
+  def self.all
+    @@all
+  end
+
+  def initialize(attributes)
+    super(attributes)
+    get_query
+  end
+
+  def topics
+    TildeScraper::Topic.find_by_page_id(page_id)
+  end
+
+  def display
+    topics.each.with_index(1) do |topic, index|
+      puts "#{index}. #{topic.type}post"
+      topic.display
+      puts ""
+    end
+  end
+
+  private
+  def get_query
+    url_array = url.split("?")
+    query_hash = {order: "Activity", period: "All Time"}
+    group = url.scan(/~\w*/)
+    query_hash[:group] = group.length == 1 ? group.first : nil
+    if url_array.length == 2
+      query_hash = url_array[1].split("&").reduce(query_hash) do |hash, var|
+        var = var.split("=")
+        hash[var[0].to_sym] = var[1] unless var[0] == "before" || var[0] == "after"
+        hash
+      end
+    end
+    add_attributes(query_hash)
+  end
+end

data/lib/tilde_scraper/scraper.rb

@@ -0,0 +1,80 @@
+class TildeScraper::Scraper
+  BASE_URL = "https://tildes.net"
+  #Returns an array with two elements.
+  #the first a hash containing general page info
+  #the secound an array of hashes containing topic info
+  def self.scrape_page(url)
+    doc = open_url(url)
+    output_array = []
+    output_array << {
+      url: url,
+    }
+    page_buttons = doc.css("a.page-item").each do |button|
+      button_name = button.text
+      output_array[0]["#{button_name.downcase}_link".to_sym] = button.attribute("href").value
+    end
+
+    topics = doc.css("article.topic")
+    output_array << topics.map do |topic|
+      title = topic.css("h1.topic-title a")
+      metadata = topic.css("div.topic-metadata")
+      info = {
+        title: title.text,
+        comment_count: topic.css("div.topic-info-comments").text.strip,
+        comment_link: topic.css("div.topic-info-comments a").attribute("href").value.split(" ").first,
+        group: metadata.css("span.topic-group").text,
+        word_count: metadata.css("span.topic-content-metadata").text.split(" ")[0],
+        age: topic.css("time.time-responsive").attribute("data-abbreviated").value,
+        votes: topic.css("div.topic-voting span.topic-voting-votes").text
+      }
+      topic_text = topic.css(".topic-text-excerpt")
+      topic_text = topic_text.children.reject { |el| el.name == "summary" }
+      if topic_text.length > 0
+        info[:topic_text] = topic_text.reduce("") { |s, el| s + el.text}.strip
+      else
+        info[:link] = title.attribute("href").value
+      end
+      info
+    end
+    output_array
+  end
+
+  def self.scrape_groups(url)
+    doc = open_url(BASE_URL + url)
+    out = doc.css("tr.group-level-0").map do |group|
+      {
+        name: group.css("a").text,
+        description: group.css("p").text,
+        subs: group.css("span.group-subscription-count").text.split(" ").first
+      }
+    end
+    out
+  end
+
+  def self.scrape_comments(url)
+    doc = open_url(BASE_URL + url)
+    comments = doc.css("#comments")
+    array = scrape_children(comments, url)
+    array
+  end
+
+  private
+  def self.scrape_children(top_comment, url, level = 0)
+    comments = top_comment.css("> li > article").map do |comment|
+      comment_info = comment.css("> div.comment-itself").first
+      hash = {
+        text: comment_info.css("div.comment-text").text.strip,
+        author: comment_info.css("a.link-user").text,
+        votes: comment_info.css("div.comment-votes").text.split(" ").first,
+        level: level,
+        url: url,
+        children: scrape_children(comment.css("> ol.comment-tree-replies"), url, level + 1)
+      }
+      hash
+    end
+    comments
+  end
+  def self.open_url(url)
+    Nokogiri::HTML(open(url))
+  end
+end

data/lib/tilde_scraper/topic/link_topic.rb

@@ -0,0 +1,15 @@
+class TildeScraper::LinkTopic < TildeScraper::Topic
+  attr_accessor :link
+
+  def type
+    "link"
+  end
+
+  def content
+    link
+  end
+
+  def display_content
+    puts "Topic Link: #{link}"
+  end
+end

data/lib/tilde_scraper/topic/text_topic.rb

@@ -0,0 +1,15 @@
+class TildeScraper::TextTopic < TildeScraper::Topic
+  attr_accessor :topic_text
+
+  def type
+    "text"
+  end
+
+  def content
+    topic_text
+  end
+
+  def display_content
+    puts topic_text
+  end
+end

data/lib/tilde_scraper/topic/topic.rb

@@ -0,0 +1,44 @@
+class TildeScraper::Topic
+  attr_accessor :title, :group, :word_count, :age, :votes, :comment_link, :comment_count, :page_id
+
+  extend TildeScraper::Memorable::ClassMethods
+  include TildeScraper::Memorable::InstanceMethods
+
+  @@all = []
+
+  def self.find_by_page_id(id)
+    all.select { |topic| topic.page_id == id }
+  end
+
+  def self.all
+    @@all
+  end
+
+  def self.create_from_array(array)
+    array.each do |attributes|
+      if attributes.keys.include?(:topic_text)
+        TildeScraper::TextTopic.create(attributes)
+      elsif attributes.keys.include?(:link)
+        TildeScraper::LinkTopic.create(attributes)
+      else
+        binding.pry
+        raise TopicError
+      end
+    end
+  end
+
+  def comments
+    TildeScraper::Comment.find_by_url(comment_link)
+  end
+
+  def display
+    puts "#{title} Votes:#{votes}"
+    puts "#{group} WC:#{word_count} #{age} #{comment_count}"
+  end
+
+  class TopicError < StandardError
+    def message
+      "Topic is neither text topic nor link topic"
+    end
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: tilde-scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Noah Evans
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-10-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+description: Scrapes the website tildes.net, can scrape topics on a page, contents
+  or link of a topic, top level groups, and comments on a topic
+email: noah@nevans.me
+executables:
+- tilde-scraper
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/tilde-scraper
+- config/enviornment.rb
+- lib/tilde_scraper.rb
+- lib/tilde_scraper/api.rb
+- lib/tilde_scraper/cli.rb
+- lib/tilde_scraper/comment.rb
+- lib/tilde_scraper/concerns/memorable.rb
+- lib/tilde_scraper/group.rb
+- lib/tilde_scraper/page.rb
+- lib/tilde_scraper/scraper.rb
+- lib/tilde_scraper/topic/link_topic.rb
+- lib/tilde_scraper/topic/text_topic.rb
+- lib/tilde_scraper/topic/topic.rb
+homepage: http://rubygems.org/gems/tildes-scraper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Scrapes the website tildes.net
+test_files: []