generalscraper 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef817246382783da5c99aaf13b1d1a03ebc87ce6
-  data.tar.gz: 9c41b5cf1c937e4ce9826ba7bd619e380277d309
+  metadata.gz: 84330f1a3a5c18dd9b2d8b200b141b5ba7c85827
+  data.tar.gz: 5fe4f81df62c28962565340858984e3d6b6f8a86
 SHA512:
-  metadata.gz: 73967c97043bbd9c79849b036285cf41a187f654a456f53d56b1ecea95042f2db0020eecba0a48366390b6d08f28ae95c8c179815608def58691e38520363a55
-  data.tar.gz: 2d9a966ecc6a6389a730044fd7c94c6a9588fdaa8297d8bea11ac048c30321c8e9b7c76fca0c4bb56e1e1408669e8a27c628e367c8c236dc06ebb3a85470071f
+  metadata.gz: 58b032cb6a3f33f4e5f1d972147952906ca95be1fbbbf181afad95336cc4a24b7585733305e8eefe9bce38da77cb989510a41020b9d418d93b68c7ef2fc8c1e1
+  data.tar.gz: 401093d88ca984e4a3fdc5f39f3ec35be9441fdff15e23fd434d623203ec22a9687f98d5fcfdd021c90662eb791f8ec34e57d45d59d08b5364b626a1961d6865

data/lib/generalscraper.rb

@@ -2,6 +2,7 @@ require 'mechanize'
 require 'json'
 require 'nokogiri'
 require 'open-uri'
+require 'uploadconvert'
 
 class GeneralScraper
   def initialize(scrapesite, input, table)
@@ -64,11 +65,19 @@ class GeneralScraper
         end
       end
       if @table == false
-        pagehash[:page] = html.css("body").text
+        if url.include? ".pdf"
+          `wget -P public/uploads #{url}`
+          path = url.split("/")
+          u = UploadConvert.new("public/uploads/" + path[path.length-1].chomp.strip)
+          pdfparse = JSON.parse(u.handleDoc)
+          pdfparse.each{|k, v| pagehash[k] = v}
+        else
+          pagehash[:text] = html.css("body").text
+        end
       end
       @output.push(pagehash)
     rescue
-      puts "URL: " + url
+      
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: generalscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-15 00:00:00.000000000 Z
+date: 2014-05-18 00:00:00.000000000 Z
 dependencies: []
 description: Scrapes all pages on a site you specify including terms you specify.
 email: shidash@shidash.com