effscraper 0.0.1

data/lib/effscraper.rb

@@ -0,0 +1,67 @@
+require 'nokogiri'
+require 'open-uri'
+require 'json'
+require 'uploadconvert'
+
+class EFFScraper
+  def initialize(url)
+    @url = url
+    @casearray = Array.new
+  end
+
+  # Scrapes all documents in case
+  def scrapeCase
+    html = Nokogiri::HTML(open(@url))
+
+    # Get number of pages to scrape
+    if html.css("li.pager-current")[0]
+      count = html.css("li.pager-current")[0].text.split(" ")
+      n = count[2].to_i
+    else
+      n = 1
+    end
+
+    # Go through pages and scrape them
+    for i in 1..n
+      if i > 1
+        link = "https://eff.org" + html.css("li.pager-next")[0].css("a")[0]["href"]
+        html = Nokogiri::HTML(open(link))
+      end
+      
+      scrapePage(html)
+    end
+    
+    JSON.pretty_generate(@casearray)
+  end
+
+  # Scrapes each page of documents
+  def scrapePage(html)
+    items = html.css("div.view-content")[0]
+    
+    items.css("li").each do |l|
+      dochash = Hash.new
+
+      # Gets link to document and file
+      l.css("a").each do |a|
+        if a.text == "[PDF]"
+          dochash[:url] = a["href"]
+          `wget #{dochash[:url]}` 
+          path = dochash[:url].split("/")
+          dochash[:path] = path[path.length-1].chomp.strip
+        end
+      end
+     print 
+
+      # Get date and title                                                      
+      dochash[:doc_date] = l.css("span.date-display-single").text
+      dochash[:title] = l.css("a")[1].text
+
+      # Extract metadata and text
+      u = UploadConvert.new(dochash[:path])
+      metadata = u.extractMetadataPDF
+      metadata.each{|k, v| dochash[k] = v}
+      dochash[:text] = u.detectPDFType
+      @casearray.push(dochash)
+    end
+  end
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: effscraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-13 00:00:00.000000000 Z
+dependencies: []
+description: Scrapes EFF court documents then extracts the plaintext and metadata.
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/effscraper.rb
+homepage: https://github.com/Shidash/EFFScraper
+licenses:
+- GPL
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Scrapes, extracts text, extracts metadata from EFF court documents
+test_files: []
+has_rdoc: