acluscraper 0.0.1

data/lib/acluscraper.rb

@@ -0,0 +1,60 @@
+require 'nokogiri'
+require 'open-uri'
+require 'json'
+require 'uploadconvert'
+
+class ACLUScraper
+  def initialize(url)
+    @url = url
+    @casearray = Array.new
+  end
+
+  # Get all the case documents
+  def scrapeCase
+    html = Nokogiri::HTML(open(@url))
+    prevdate = ""
+
+    html.css("tbody").each do |t|
+      t.css("tr").each do |r|
+        if !r.css("a").empty?
+          dochash = Hash.new
+          
+          # Get date for filing
+          if r.css("td")[0].text == "\u00a0"
+            dochash[:date] = prevdate
+          else
+            prevdate = r.css("td")[0].text.to_s
+            dochash[:date] = r.css("td")[0].text.to_s
+          end
+
+          a = r.css("a")
+          dochash[:title] = a.text
+
+          # Get URL
+          if a[0]["href"].to_s.include? "https://"
+            dochash[:url] = a[0]["href"]
+          else
+            dochash[:url] = "https://www.aclu.org" + a[0]["href"]
+          end
+          
+          # Download documents
+          `wget #{dochash[:url]}`
+          path = dochash[:url].split("/")
+          dochash[:path] = path[path.length-1].chomp.strip
+
+          # Extract metadata and text
+          begin
+            u = UploadConvert.new(dochash[:path])
+            metadata = u.extractMetadataPDF
+            metadata.each{|k, v| dochash[k] = v}
+            dochash[:text] = u.detectPDFType
+            @casearray.push(dochash)
+          rescue
+          end
+        end
+      end
+    end
+    
+    JSON.pretty_generate(@casearray)
+  end
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: acluscraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-19 00:00:00.000000000 Z
+dependencies: []
+description: Scrapes ACLU court documents then extracts the plaintext and metadata.
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/acluscraper.rb
+homepage: https://github.com/Shidash/ACLUScraper
+licenses:
+- GPL
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Scrapes, extracts text, extracts metadata from ACLU court documents
+test_files: []
+has_rdoc: