coa-op-scraper 0.2.2

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,22 @@
+source "http://rubygems.org"
+
+gem 'nokogiri'
+gem 'rails'
+
+group :development, :test do
+  gem 'rspec-rails'
+end
+
+group :test do
+  gem 'timecop'
+  gem 'vcr' #, '~> 2.0.rc'
+  gem 'fakeweb'
+end
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rdoc" #, "~> 3.12"
+  gem "bundler" #, "~> 1.0.0"
+  gem "jeweler" #, "~> 1.8.4"
+end

data/Gemfile.lock

@@ -0,0 +1,116 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    actionmailer (3.2.11)
+      actionpack (= 3.2.11)
+      mail (~> 2.4.4)
+    actionpack (3.2.11)
+      activemodel (= 3.2.11)
+      activesupport (= 3.2.11)
+      builder (~> 3.0.0)
+      erubis (~> 2.7.0)
+      journey (~> 1.0.4)
+      rack (~> 1.4.0)
+      rack-cache (~> 1.2)
+      rack-test (~> 0.6.1)
+      sprockets (~> 2.2.1)
+    activemodel (3.2.11)
+      activesupport (= 3.2.11)
+      builder (~> 3.0.0)
+    activerecord (3.2.11)
+      activemodel (= 3.2.11)
+      activesupport (= 3.2.11)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activeresource (3.2.11)
+      activemodel (= 3.2.11)
+      activesupport (= 3.2.11)
+    activesupport (3.2.11)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    arel (3.0.2)
+    builder (3.0.4)
+    diff-lcs (1.1.3)
+    erubis (2.7.0)
+    fakeweb (1.3.0)
+    git (1.2.5)
+    hike (1.2.1)
+    i18n (0.6.1)
+    jeweler (1.8.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+      rdoc
+    journey (1.0.4)
+    json (1.7.6)
+    mail (2.4.4)
+      i18n (>= 0.4.0)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.19)
+    multi_json (1.5.0)
+    nokogiri (1.5.6)
+    polyglot (0.3.3)
+    rack (1.4.4)
+    rack-cache (1.2)
+      rack (>= 0.4)
+    rack-ssl (1.3.2)
+      rack
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rails (3.2.11)
+      actionmailer (= 3.2.11)
+      actionpack (= 3.2.11)
+      activerecord (= 3.2.11)
+      activeresource (= 3.2.11)
+      activesupport (= 3.2.11)
+      bundler (~> 1.0)
+      railties (= 3.2.11)
+    railties (3.2.11)
+      actionpack (= 3.2.11)
+      activesupport (= 3.2.11)
+      rack-ssl (~> 1.3.2)
+      rake (>= 0.8.7)
+      rdoc (~> 3.4)
+      thor (>= 0.14.6, < 2.0)
+    rake (10.0.3)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec-core (2.12.2)
+    rspec-expectations (2.12.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.12.1)
+    rspec-rails (2.12.2)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 2.12.0)
+      rspec-expectations (~> 2.12.0)
+      rspec-mocks (~> 2.12.0)
+    sprockets (2.2.2)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    thor (0.16.0)
+    tilt (1.3.3)
+    timecop (0.5.9)
+    treetop (1.4.12)
+      polyglot
+      polyglot (>= 0.3.1)
+    tzinfo (0.3.35)
+    vcr (2.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  fakeweb
+  jeweler
+  nokogiri
+  rails
+  rdoc
+  rspec-rails
+  timecop
+  vcr

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Don Cruse
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# Texas COA Op Scraper - a gem for Texas courts of appeals
+
+### What's this about?
+
+This gem understands how to parse the opinion lists released by each of
+Texas's fourteen intermediate courts of appeals.
+
+Opinion releases are announced on a separate webpage for each court of
+appeals.  Some courts use a legacy system; others have shifted to the new
+TAMES system employed by the Texas Supreme Court.
+
+### Why does this gem exist?
+
+It was developed as part of the TexApp.org project ([github](http://github.com/texapp)),
+which aims to ensure that Texas's court of appeals opinions are available in
+a reliable — and citable — location available to the general public,
+members of the bar, and the court system itself.
+
+In Texas, unpublished decisions of intermediate courts of appeals are precedential.
+Yet litigants do not always have a reliable way to locate or cite this authority.
+In the past, it was possible to use a well-crafted Google search to locate
+relevant opinions (a technique discussed in [this 2009 blog post](http://www.scotxblog.com/practice-notes/researching-unpublished-coa-opinions-in-texas/)). But with the courts'
+new TAMES system, these Google searches no longer work.  The TAMES system does provide
+many of these opinions in an online archive, but its URLs are prohitively long and
+complex to include in any printed legal brief.
+
+### How can I use this?
+
+This gem can be folded into the application of your choice to store 
+information about these opinions or queue up downloads of the opinions
+themselves.  It does not contain code related to data storage or any
+interface to a file storage service.  Those implementation details
+are up to you.
+
+The simplest way to use this gem is to specify a particular court of appeals
+(using its two-digit numberical notation, like "03" for the Third Court) and a
+particular date on which you want to check for opinions.  The gem will then
+determine the correct URL to use, check that page, and parse what is found to
+retrieve the metadata for each opinion released on that date.  What you get back
+is a list of that metadata.
+
+The data for each opinion is a simple hash.  The overall
+set of results is just an array of those hashes, or an empty array if no
+results were found for that page. Here is an example of the hash for one opinion:
+
+> { :author_string => "Opinion by Justice Pemberton", 
+    :opinion_urls => {"html"=>"/opinions/htmlopinion.asp?OpinionId=20764", 
+                      "pdf"=>"/opinions/PDFOpinion.asp?OpinionId=20764"},
+    :disposition => "AFFIRMED:", 
+    :panel_string => "(Before Chief Justice Jones, Justices Pemberton and Henson)", 
+    :release_date => Fri, 20 Jan 2012, 
+    :case_style => "Janeen Denise Smith v. The State of Texas", 
+    :origin => "Appeal from County Court at Law No. 1 of Caldwell County", 
+    :docket_no => "03-10-00725-CR", 
+    :docket_page_url => "/opinions/case.asp?FilingID=15750" }
+
+It's up to you to write code that does something interesting with that metadata —
+such as storing it or downloading the opinion PDFs themselves (as is being done
+for TexApp.org).
+
+## Copyright
+
+Copyright (c) 2013 Don Cruse. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "coa-op-scraper"
+  gem.homepage = "http://github.com/doncruse/coa-op-scraper"
+  gem.license = "(c)2013 Don Cruse"
+  gem.summary = "A scraper for intermediate Texas appellate opinions"
+  gem.description = "A scraper for intermediate appellate opinions"
+  gem.email = "doncruse@gmail.com"
+  gem.authors = ["Don Cruse"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+task :default => :spec
+

data/VERSION

@@ -0,0 +1 @@
+0.2.2

data/coa-op-scraper.gemspec

@@ -0,0 +1,77 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "coa-op-scraper"
+  s.version = "0.2.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Don Cruse"]
+  s.date = "2013-02-09"
+  s.description = "A scraper for intermediate appellate opinions"
+  s.email = "doncruse@gmail.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "coa-op-scraper.gemspec",
+    "lib/coa_docket_no.rb",
+    "lib/coa_op_scraper.rb",
+    "lib/legacy.rb",
+    "lib/tames.rb",
+    "spec/scrapers/legacy_scraper_spec.rb",
+    "spec/scrapers/main_spec.rb",
+    "spec/scrapers/tames_scraper_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/support/vcr.rb",
+    "spec/vcr/legacy/02-2003-02-20.json",
+    "spec/vcr/legacy/03-2013-01-10.json",
+    "spec/vcr/tames/01-2012-01-19.json",
+    "spec/vcr/tames/12-2003-01-31.json",
+    "spec/vcr/tames/14-2005-01-20.json"
+  ]
+  s.homepage = "http://github.com/doncruse/coa-op-scraper"
+  s.licenses = ["(c)2013 Don Cruse"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.23"
+  s.summary = "A scraper for intermediate Texas appellate opinions"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_runtime_dependency(%q<rails>, [">= 0"])
+      s.add_development_dependency(%q<rspec-rails>, [">= 0"])
+      s.add_development_dependency(%q<rdoc>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+    else
+      s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<rails>, [">= 0"])
+      s.add_dependency(%q<rspec-rails>, [">= 0"])
+      s.add_dependency(%q<rdoc>, [">= 0"])
+      s.add_dependency(%q<bundler>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<rails>, [">= 0"])
+    s.add_dependency(%q<rspec-rails>, [">= 0"])
+    s.add_dependency(%q<rdoc>, [">= 0"])
+    s.add_dependency(%q<bundler>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+  end
+end
+

data/lib/coa_docket_no.rb

@@ -0,0 +1,81 @@
+module CoaOpScraper
+  class CoaDocketNo
+    attr_accessor :no
+    # Encapsulating the logic of working with COA docket numbers.
+    # Note: Distinct from knowing if a valid docket number was actually used
+
+    def initialize(no)
+      parts = no.split("-")
+      if parts.count == 4 and (parts.last == "CR" or parts.last == "CV")
+        @no = no
+      else
+        @no = nil
+      end
+    end
+
+    def valid?
+      !@no.nil?
+    end
+
+    def to_s
+      self.fixed_length
+    end
+
+    def fixed_length
+      if self.valid?
+        (coa,year,number,type_suffix) = @no.split("-")
+        [padded(coa,2), padded(year,2), padded(number,5), type_suffix].join('-')
+      else
+        ""
+      end
+    end
+
+    def without_type
+      self.fixed_length.sub("-CR","").sub("-CV","")
+    end
+
+    #####################################
+    # For accessing pieces
+
+    def coa_number
+      self.canonical.split("-")[0]
+    end
+
+    def year_number
+      self.canonical.split("-")[1]
+    end
+
+    def case_number
+      self.canonical.split("-")[2]
+    end
+
+    def civil?
+      self.canonical.split("-")[3] == "CV"
+    end
+
+    def criminal?
+      self.canonical.split("-")[3] == "CR"
+    end
+
+    ###################################
+    # Standardizing how used internally
+
+    def for_database_key
+      self.without_type
+    end
+    # because the -CV/-CR suffix is not relevant to uniqueness
+
+    def for_web_urls
+      self.fixed_length
+    end
+
+    def canonical
+      self.fixed_length
+    end
+
+    protected
+    def padded(number,digits)
+      sprintf("%0#{digits}d", number)
+    end
+  end
+end

data/lib/coa_op_scraper.rb

@@ -0,0 +1,92 @@
+# encoding=utf-8
+module CoaOpScraper
+  require 'legacy'
+  require 'tames'
+  require 'coa_docket_no'
+
+  require 'date'
+  require 'open-uri'
+  require 'active_support/core_ext'
+
+  # The Texas appellate websites are sometimes fragile. 
+  # These sleep intervals should give ample time between requests.
+  HISTORICAL_THROTTLE = 10
+  CURRENT_THROTTLE = 5
+
+  @@check_weekends = FALSE
+
+  # A court's placement in one of these two hashes tells you about the webpage format
+  # currently used by that court.
+  TAMES_COAS = [ "01", "03", "04", "05", "06", "09", "11", "12", "14" ]
+  LEGACY_COAS = [ "02", "07", "08", "10", "13" ]
+
+  ############################################################
+  # This is the easiest method to use here.  Feed it a COA value
+  # (in the form "03", for example) and the date for which you
+  # want the results (in the form of a Ruby date object).
+  #
+  def self.scrape_one_opinion_list(coa,target_date)
+    doc = self.retrieve_list_for_coa_for_date(coa,target_date)
+    if CoaOpScraper::TAMES_COAS[coa]
+      CoaOpScraper::Tames.parse_opinion_list(doc)
+    elsif CoaOpScraper::LEGACY_COAS[coa]
+      CoaOpScraper::Legacy.parse_opinion_list(doc)
+    end
+  end
+
+  ############################################################
+  # These methods would be useful to populate a queue of opinion
+  # lists to check later.
+  # 
+  # The #urls_for_historical_range method will, as expected,
+  # compute a list of the URLs that are appropriate (excluding
+  # weekends by default).
+  #
+  # The #parse_coa_opinion_list_at method will take a coa number
+  # and a URL and return back a list of the results.
+
+  def self.urls_for_historical_range(coa, start_date, end_date)
+    result = []
+    (start_date .. end_date).each do |target_date|
+      next unless @@check_weekends or target_date.weekday?
+      result << self.url_for_coa_for_date(coa, target_date)
+    end
+    result
+  end # returns an array of URLs
+
+  def self.parse_coa_opinion_list_at(coa, url)
+    self.scrape_one_opinion_list(coa, url) || []
+  end # takes a URL, returns a list of the opinion data
+
+protected
+
+  def self.url_for_coa_for_date(coa,date)
+    if CoaOpScraper::TAMES_COAS.include?(coa)
+      CoaOpScraper::Tames.url_for_coa_for_date(coa,date)
+    elsif CoaOpScraper::LEGACY_COAS.include?(coa)
+      CoaOpScraper::Legacy.url_for_coa_for_date(coa,date)
+    end
+  end
+
+  def self.retrieve_list_for_coa_for_date(coa,date)
+    url = self.url_for_coa_for_date(coa,date)
+    open(url)
+  end
+end
+
+# This is required (and helpful) to parse Texas court docket pages
+class String
+  def nbsp_strip
+    strip.gsub(/\u00a0$/,"").gsub(/^\u00a0/,"").strip
+  end # gets rid of some pesky unicode found on Texas OCA sites
+
+  def strip_both_ends
+    nbsp_strip.nbsp_strip.reverse.nbsp_strip.nbsp_strip.reverse
+  end
+end
+
+class Date
+  def weekday?
+    !self.saturday? and !self.sunday?
+  end
+end