vbulletin_scraper 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75834d520af865ec1aef5d413b34d39015c51ac6
-  data.tar.gz: 458f44485c20b301bb24cae79dcd2bbd309a2fb5
+  metadata.gz: 864d58a8ed37ef7a43ea97ecfe7a6d6ac1e1c245
+  data.tar.gz: f25f3b3b12a2a1f12d9654ac7a9a257d52397f03
 SHA512:
-  metadata.gz: b27b992806780f92751cdf860c2df713e9ac6cf2bde735a197f703e9bc4c873b114feac089058b5d88bd4fa9333faef2face7546f28a4a298b9810de341c2863
-  data.tar.gz: 20e582041beb4f28793e4cabb6e651aa92b533daad3a57b22cf6b0239ac2abc18b6c8b867ea37d4e4f86e53233944720777562251f33bd5ca9e530a89c647784
+  metadata.gz: 81f5afbd989d699a161c367328a38b037beb45750900b030605ef6685569cc54ef2c4feff07866930b04b47ce7826f5557e82bfb1ca9771c3977e4f52a0eb9d4
+  data.tar.gz: 10d86908627c6375001e4f93ac4a17aa14ecd1f184c7d56afac31ec9b0e7d7de5574e231f5c82808f468ea3729973f3f0018f96b9f97511e01cefb78c40e9e04

data/README.md

@@ -1,8 +1,6 @@
 # VbulletinScraper
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/vbulletin_scraper`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+VbulletinScraper is a Ruby gem designed to parse compatible vBulletin forums, threads, and posts.  Currently only vBulletin forums v4.x are supported.
 
 ## Installation
 
@@ -20,19 +18,118 @@ Or install it yourself as:
 
     $ gem install vbulletin_scraper
 
-## Usage
 
-TODO: Write usage instructions here
+## Available Functions
+
+### Forums
+
+    is_valid_vbulletin
+    get_vbulletin_version
+    get_forum_url
+    get_forum_title
+    
+### Threads (Topics)
+
+    is_valid_vbulletin
+    get_vbulletin_version
+    get_current_page_number
+    get_total_page_count
+    get_vbulletin_topic_id
+    get_topic_url
+    get_topic_title
+    get_posts
+    
+### Posts
 
-## Development
+    get_vbulletin_post_id
+    get_post_author
+    get_post_content_raw
+    get_post_content
+    get_post_submit_datetime
+    get_quotes
+    get_post_permalink
+
+### Quotes
+
+    get_quote_author
+    get_quote_content
+
+
+## Usage
 
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+At the top of your Ruby file:
+
+    require 'vbulletin_scraper'
+    
+Example usage:
+
+    forumScraper = VbulletinScraper::V4::ForumScraper.new('http://someforum.com')
+    isVbulletinForum = forumScraper.is_valid_vbulletin
+    vbulletinVersion = forumScraper.get_vbulletin_version
+    forumTitle = forumScraper.get_forum_title
+    forumUrl = forumScraper.get_forum_url
+    
+    topicScraper = VbulletinScraper::V4::TopicScraper.new('http://someforum.com/forum/some-thread-section/1234-a-sample-thread')
+    topicTitle = topicScraper.get_topic_title
+    topicUrl = topicScraper.get_topic_url
+    posts = topicScraper.get_posts
+    
+    posts.each do |post|
+        postScraper = VbulletinScraper::V4::PostScraper.new(post.to_s)
+        vbulletinPostId = postScraper.get_vbulletin_post_id
+        postAuthor = postScraper.get_post_author
+        postContent = postScraper.get_post_content
+        postSubmitDate = postScraper.get_post_submit_datetime
+        postPermalink = postScraper.get_post_permalink
+        quotes = postScraper.get_quotes
+        
+        quotes.each do |quote|
+            quoteScraper = VbulletinScraper::V4::QuoteScraper.new(quote.to_s)
+            quoteAuthor = quoteScraper.get_quote_author
+            quoteContent = quoteScraper.get_quote_content
+        end
+    end
+
+To grab multiple pages of posts (starting with latest):
+
+    url = 'http://someforum.com/forum/some-thread-section/1234-a-sample-thread'
+    currPage = 9999
+    postCounter = 0
+    maxPosts = 10
+    stop = false
+    
+    until stop do
+        currPageUrl = url + '?page=' + currPage.to_s
+        currPageScraper = VbulletinScraper::V4::TopicScraper.new(currPageUrl)
+        currPagePosts = currPageScraper.get_posts
+        
+        currPagePosts.each do |post|
+            postScraper = VbulletinScraper::V4::PostScraper.new(post.to_s)    
+            
+            # etc...
+            
+            postCounter += 1
+            
+            if postCounter >= maxPosts
+                stop = true
+            end
+        end
+        
+        if currPage > 1
+            currPage -= 1
+        else
+            stop = true
+        end
+    end
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/vbulletin_scraper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+1. Fork it ( https://github.com/bendrick92/vbulletin_scraper/fork )
+2. Create your feature branch (git checkout -b my-new-feature)
+3. Commit your changes (git commit -am 'Add some feature')
+4. Push to the branch (git push origin my-new-feature)
+5. Create a new Pull Request
 
 
 ## License

data/lib/vbulletin_scraper/V4/post_scraper.rb

@@ -57,16 +57,20 @@ module VbulletinScraper
                 rawDateTimeString = get_item_by_selector('.date')
                 if rawDateTimeString != nil
                     rawDateTimeString = get_raw_text(rawDateTimeString.text)
-                    if rawDateTimeString.include? 'Yesterday'
-                        rawDateTimeString = rawDateTimeString.gsub('Yesterday', '')
-                        formattedDateTimeString = Date.yesterday.strftime(dateFormat) + rawDateTimeString
-                        submitDateTime = DateTime.strptime(formattedDateTimeString, dateTimeFormat)
-                    elsif rawDateTimeString.include? 'Today'
-                        rawDateTimeString = rawDateTimeString.gsub('Today', '')
-                        formattedDateTimeString = Date.today.strftime(dateFormat) + rawDateTimeString
-                        submitDateTime = DateTime.strptime(formattedDateTimeString, dateTimeFormat)
-                    else
-                        submitDateTime = DateTime.strptime(rawDateTimeString, dateTimeFormat)
+                    begin
+                        if rawDateTimeString.include? 'Yesterday'
+                            rawDateTimeString = rawDateTimeString.gsub('Yesterday', '')
+                            formattedDateTimeString = Date.yesterday.strftime(dateFormat) + rawDateTimeString
+                            submitDateTime = DateTime.strptime(formattedDateTimeString, dateTimeFormat)
+                        elsif rawDateTimeString.include? 'Today'
+                            rawDateTimeString = rawDateTimeString.gsub('Today', '')
+                            formattedDateTimeString = Date.today.strftime(dateFormat) + rawDateTimeString
+                            submitDateTime = DateTime.strptime(formattedDateTimeString, dateTimeFormat)
+                        else
+                            submitDateTime = DateTime.strptime(rawDateTimeString, dateTimeFormat)
+                        end
+                    rescue ArgumentError
+                        submitDateTime = nil
                     end
                     return submitDateTime
                 end

data/lib/vbulletin_scraper/version.rb

@@ -1,3 +1,3 @@
 module VbulletinScraper
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vbulletin_scraper
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Ben Walters
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-09 00:00:00.000000000 Z
+date: 2017-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri