tilde-scraper 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 233e57dca85aa06d060975d41dc93b3c42b4f8b5b780ea98bd3bc55a240fa821
-  data.tar.gz: b7bdc67f83cf7a408ab46c18c37ac6043ed9d9efe27c46cce90b8643b7389509
+  metadata.gz: 0eebcc9c37f1a557411b9f036f34d4f6f2a71e7d9d13dd6848380a5b219aa9f5
+  data.tar.gz: 313dc490897b207cf45ec4004c7eb9be21f85fe8400f2b6221a510e696845b15
 SHA512:
-  metadata.gz: 4b141b97282e2f2559fd7374c0a6267c11bdb0590635addc743868b17eb44de27908ce15d325ab774d3f133c9cbc1a3686944bd2df1962c2cf65520ade595102
-  data.tar.gz: f55175e57c0273c2ce319c9d4dc8bc347c57e5499799b63f9a378d5d1e89001f80a1d4369f89733b96b26341a58769113c41e8f0c0ef2a0517426295c842995f
+  metadata.gz: 8274f406e2c357242a58f452b9f0b2a3fadc9a0297b1deab6d94f25feb58da0ef9522f4b177d21998f8e44d3e832932cb00d1b871f452d385d29015da09475f9
+  data.tar.gz: b44a96b15fe3d44d26e9c3acde82a5dcad4689f84d64f246fbe588a5c5edef5b69e617aa2606606e825242271f67f98ced110b7e62ec16c676a44712557d008b

data/lib/tilde_scraper.rb

@@ -1,4 +1,3 @@
 module TildeScraper
-    @@page_id = 0
 end
 require_relative '../config/enviornment.rb'

data/lib/tilde_scraper/api.rb

@@ -1,12 +1,12 @@
 module TildeScraper
   @@page_id = 0
+  #Scrapes a page at url, creates topic objects for each topic, and returns a page object
   def self.get_page(url)
       data = TildeScraper::Scraper.scrape_page(url)
       #Set page_id in page data hash
       data[0][:page_id] = @@page_id
       #Create page object
       page = TildeScraper::Page.create(data[0])
-
       #Set page_id in all topic data hashes
       #Set group in all hashes if applicible
       data[1].each do |topic_hash|
@@ -21,6 +21,7 @@ module TildeScraper
       page
   end
 
+  #Scrapes a page for topics, and scrapes each topic's comments, returns a page object
   def self.get_page_with_comments(url)
     page = get_page(url)
     #Create comments for each topic
@@ -29,11 +30,13 @@ module TildeScraper
     end
   end
 
+  #Scrapes the group page for first level groups and returns an array of group objects
   def self.get_groups
     TildeScraper::Group.all.clear
-    TildeScraper::Group.create_from_array(TildeScraper::Scraper.scrape_groups("/groups"))
+    TildeScraper::Group.create_from_array(TildeScraper::Scraper.scrape_groups("https://tildes.net/groups"))
   end
 
+  #Scrapes a topic page and returns an array of comment objects
   def self.get_comments(url)
       comment_array = TildeScraper::Scraper.scrape_comments(url)
       TildeScraper::Comment.create_from_array(comment_array)

data/lib/tilde_scraper/comment.rb

@@ -47,7 +47,6 @@ class TildeScraper::Comment
   end
 
   def self.display(array, indent = 0)
-    #binding.pry
     array.each do |comment|
       comment.display(indent)
       display(comment.children, indent + 1)

data/lib/tilde_scraper/scraper.rb

@@ -1,5 +1,4 @@
 class TildeScraper::Scraper
-  BASE_URL = "https://tildes.net"
   #Returns an array with two elements.
   #the first a hash containing general page info
   #the secound an array of hashes containing topic info
@@ -21,7 +20,7 @@ class TildeScraper::Scraper
       info = {
         title: title.text,
         comment_count: topic.css("div.topic-info-comments").text.strip,
-        comment_link: topic.css("div.topic-info-comments a").attribute("href").value.split(" ").first,
+        comment_link: "https://tildes.net" + topic.css("div.topic-info-comments a").attribute("href").value.split(" ").first,
         group: metadata.css("span.topic-group").text,
         word_count: metadata.css("span.topic-content-metadata").text.split(" ")[0],
         age: topic.css("time.time-responsive").attribute("data-abbreviated").value,
@@ -40,7 +39,7 @@ class TildeScraper::Scraper
   end
 
   def self.scrape_groups(url)
-    doc = open_url(BASE_URL + url)
+    doc = open_url(url)
     out = doc.css("tr.group-level-0").map do |group|
       {
         name: group.css("a").text,
@@ -52,7 +51,7 @@ class TildeScraper::Scraper
   end
 
   def self.scrape_comments(url)
-    doc = open_url(BASE_URL + url)
+    doc = open_url(url)
     comments = doc.css("#comments")
     array = scrape_children(comments, url)
     array

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tilde-scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Noah Evans
@@ -52,8 +52,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.12'
-description: Scrapes the website tildes.net, can scrape topics on a page, contents
-  or link of a topic, top level groups, and comments on a topic
+description: Scrapes the website tildes.net. Can scrape topics on a page, top level
+  groups, and comments on a topic. Also has a very basic cli for browsing
 email: noah@nevans.me
 executables:
 - tilde-scraper