justiz 0.0.1 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 79087f670423816d367f2030964da9fb891ebe1f
+  data.tar.gz: f312cfac2918d6821bf9dd0efafc953e3c2c4f1c
+SHA512:
+  metadata.gz: 47e19fb2ccae3d712dcb17970061050baf6b4725584c51575a1adfd8438bcffb62861a001d593fb2be1bbfb79221b774c385a06e02f41cc870f6cebd6823ec8b
+  data.tar.gz: 502813f3a6b29869650375235d5c9f309e5740f3b8ec11c641f44631cda1c933700efaa14bc74986f7c4e6c37cfd222d707083276b110275a17e439e3683eaa1

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+.rvmrc

data/.rvmrc

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 2.0.0" > .rvmrc
+environment_id="ruby-2.0.0-p195@justiz"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.20.13 (stable)" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  for __hook in "${rvm_path:-$HOME/.rvm}/hooks/after_use"*
+  do
+    if [[ -f "${__hook}" && -x "${__hook}" && -s "${__hook}" ]]
+    then \. "${__hook}" || true
+    fi
+  done
+  unset __hook
+  if (( ${rvm_use_flag:=1} >= 2 )) # display only when forced
+  then
+    if [[ $- == *i* ]] # check for interactive shells
+    then printf "%b" "Using: \E[32m$GEM_HOME\E[0m" # show the user the ruby and gemset they are using in green
+    else printf "%b" "Using: $GEM_HOME" # don't use colors in non-interactive shells
+    fi
+  fi
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

data/justiz.gemspec

@@ -25,4 +25,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 2.6"
   spec.add_development_dependency "awesome_print"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "vcr"
 end

data/lib/justiz/address.rb

@@ -0,0 +1,26 @@
+module Justiz
+  class Address
+    attr_reader :text, :city, :plz, :street
+
+    def initialize(text)
+      @text = text
+      parse
+    end
+
+    private
+
+    def parse
+      parts = text.split(/\s*,\s*/)
+      if parts.length > 1
+        @street = parts.slice(0, parts.length - 1).join(", ")
+      end
+      @plz, @city = parse_city(parts.last)
+    end
+
+    def parse_city(string)
+      if string.match(/\s*([0-9]{5})\s*(.*)/i)
+        [$1, $2]
+      end
+    end
+  end
+end

data/lib/justiz/contact.rb

@@ -1,4 +1,22 @@
+require 'ostruct'
+
 module Justiz
-  class Contact < Hash
+  class Contact < OpenStruct
+    # std fields: court, location, post, phone, fax, justiz_id, url, email
+
+    def id
+      # too many duplicates
+      #[court, justiz_id].compact.join("")
+      # currently no duplicates
+      [court, email].compact.join("")
+    end
+
+    def location_address
+      Address.new(self[:location])
+    end
+
+    def post_address
+      Address.new(self[:post])
+    end
   end
 end

data/lib/justiz/scraper/courts.rb

@@ -3,78 +3,94 @@ module Justiz
     class Courts
 
       def court_types
-        select_options(home_page, 'gerausw')
+        home_page.options_of 'gerausw'
       end
 
       def states
-        select_options(home_page, 'landausw')
+        home_page.options_of 'landausw'
       end
 
-      def scrape(court_type, state)
-        page = load_page(court_type, state)
-        # if we reach limit on ALL, query each subtype
-        if court_type == 'ALL' && limit_warning?(page)
-            court_types.map do |court_type, description|
-              next if court_type == 'ALL'
-              page = load_page(court_type, state)
-              if limit_warning?(page)
-                puts(STDERR, "Warning: State #{state} has too many contacts of #{description}[#{court_type}]")
-              end
-              parse_page(page)
-            end.flatten.compact.uniq
-        else
-          parse_page(page)
-        end
+      def contacts
+        states.keys.map do |state|
+          contacts_for(state)
+        end.flatten.compact
+      end
+
+      def contacts_for(state)
+        page = load_page('ALL', state)
+        return page.contacts unless page.limit_warning?
+
+        # do each type separately hoping to avoid limit warning
+        court_types.keys.map do |court_type|
+          contacts_of_type(court_type, state)
+        end.flatten.compact.uniq
+      end
+
+      def contacts_of_type(type, state)
+        load_page(type, state, with_warning: true).contacts
       end
 
       private
 
       def home_page
-        @home_page ||= agent.get('http://www.justizadressen.nrw.de/og.php?MD=nrw')
+        @home_page ||= Page.new(agent.get('http://www.justizadressen.nrw.de/og.php?MD=nrw'))
       end
 
-      def load_page(court_type, state)
+      def load_page(court_type, state, options = {})
         form = home_page.forms[2]
         form['gerausw'] = court_type
         form['landausw'] = state
-        agent.submit(form, form.buttons_with(name: 'suchen1').first)
+        page = Page.new(agent.submit(form, form.buttons_with(name: 'suchen1').first))
+        if options[:with_warning] && page.limit_warning?
+          puts(STDERR, "Warning: State #{state} has too many contacts of #{court_type}")
+        end
+        page
       end
 
       def agent
         @agent ||= Justiz::Scraper::Agent.new
       end
+    end
 
-      def parse_page(page)
-        rows = page.search('tr').map { |tr| tr.search('td').to_a }
-        contact_rows = rows.find_all { |row| row.length == 3 }
-        contact_rows.map do |court, addresses, phones|
-          addresses = AddressTd.new(addresses)
-          phones = AddressTd.new(phones)
-
-          Justiz::Contact.new.merge court: court.text.strip,
-                                    location: addresses.lieferanschrift,
-                                    post: addresses.postfach,
-                                    phone: phones.telefone,
-                                    fax: phones.fax,
-                                    justiz_id: phones.justiz_id,
-                                    url: phones.url,
-                                    email: phones.email
+    class Page < SimpleDelegator
+      def limit_warning?
+        # avoid invalid UTF-8 errors by force encoding.
+        search('p').find do |p|
+          p.text.force_encoding("ISO-8859-15") =~ /Ihre Suchanfrage ergab mehr als/i
         end
       end
 
-      def limit_warning?(page)
-        # avoid invalid UTF-8 errors by force encoding.
-        page.search('p').find {|p| p.text.force_encoding("ISO-8859-15") =~ /Ihre Suchanfrage ergab mehr als/i}
+      # return hash of options of select field, exclude ALL value
+      def options_of(name)
+        search("[name='#{name}'] > option").inject({}) do |memo, node|
+          memo[node['value']] = node.text unless node['value'] == 'ALL'
+          memo
+        end
       end
 
-      def select_options(page, name)
-        page.search("[name='#{name}'] > option").inject({}) do |memo, node|
-          memo[node['value']] = node.text
-          memo
+      def contacts
+        @contacts ||= parse_contacts
+      end
+
+      def parse_contacts
+        rows = search('tr').map { |tr| tr.search('td').to_a }
+        contact_rows = rows.find_all { |row| row.length == 3 }
+        contact_rows.map do |court, addresses, kontakt|
+          addresses = AddressTd.new(addresses)
+          kontakt = KontaktTd.new(kontakt)
+
+          Justiz::Contact.new(court: court.text.strip,
+                              location: addresses.lieferanschrift,
+                              post: addresses.postfach,
+                              phone: kontakt.telefone,
+                              fax: kontakt.fax,
+                              justiz_id: kontakt.justiz_id,
+                              url: kontakt.url,
+                              email: kontakt.email)
         end
       end
 
-      class AddressTd
+      class Td
         attr_reader :texts
 
         def initialize(node)
@@ -82,18 +98,27 @@ module Justiz
           @texts = nodes.map { |n| n.text.strip }.find_all { |t| !blank?(t) }
         end
 
-        def telefone
-          same_line('Telefon:')
+        private
+
+        def blank?(something)
+          something.to_s !~ /[^[:space:]]/
         end
 
-        def fax
-          same_line('Fax:')
+        def next_line(name)
+          reg = Regexp.new(name, true)
+          line = texts.find_index { |text| text.match(reg) }
+          line && texts[line + 1]
         end
 
-        def justiz_id
-          same_line('XJustiz-ID:')
+        def same_line(name)
+          reg = Regexp.new("#{name}(.*)", true)
+          text = texts.map { |t| t.match(reg) }.compact.first
+          text = text[1].strip if text
+          text if !blank?(text)
         end
+      end
 
+      class AddressTd < Td
         def lieferanschrift
           next_line('Lieferanschrift')
         end
@@ -101,32 +126,32 @@ module Justiz
         def postfach
           next_line('Postanschrift')
         end
+      end
 
-        def url
-          next_line('URL')
-        end
+      class KontaktTd < Td
+        attr_reader :url
 
-        def email
-          next_line('Mail')
+        def initialize(node)
+          super
+          if (a = node.search('a').first)
+            @url = a['href']
+          end
         end
 
-        private
+        def telefone
+          same_line('Telefon:')
+        end
 
-        def blank?(something)
-          something.to_s !~ /[^[:space:]]/
+        def fax
+          same_line('Fax:')
         end
 
-        def next_line(name)
-          reg = Regexp.new(name, true)
-          line = texts.find_index { |text| text.match(reg) }
-          line && texts[line + 1]
+        def justiz_id
+          same_line('XJustiz-ID:')
         end
 
-        def same_line(name)
-          reg = Regexp.new("#{name}(.*)", true)
-          text = texts.map { |t| t.match(reg) }.compact.first
-          text = text[1].strip if text
-          text if !blank?(text)
+        def email
+          next_line('Mail')
         end
       end
     end

data/lib/justiz/version.rb

@@ -1,3 +1,3 @@
 module Justiz
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/justiz.rb

@@ -2,6 +2,7 @@ require "justiz/version"
 require 'justiz/cli'
 require 'justiz/scraper/agent'
 require 'justiz/scraper/courts'
+require 'justiz/address'
 require 'justiz/contact'
 
 module Justiz

data/spec/lib/address_spec.rb

@@ -0,0 +1,11 @@
+require_relative '../../spec/lib/spec_helper'
+
+describe Justiz::Address do
+  context "simple address" do
+    subject { Justiz::Address.new('Schlossbezirk 3, 76131 Karlsruhe') }
+
+    it { expect(subject.city).to eq 'Karlsruhe' }
+    it { expect(subject.street).to eq 'Schlossbezirk 3' }
+    it { expect(subject.plz).to eq '76131' }
+  end
+end

data/spec/lib/scraper/courts_spec.rb

@@ -1,13 +1,10 @@
 # encoding: utf-8
 
-require 'rspec'
-require 'justiz'
-require 'awesome_print'
+require_relative '../../lib/spec_helper'
 
 describe Justiz::Scraper::Courts do
   it "should have court types" do
     types = {
-        "ALL" => "-- alle Gerichte/Behörden --",
         "AG" => "Amtsgerichte",
         "LG" => "Landgerichte",
         "OLG" => "Oberlandesgerichte",
@@ -29,12 +26,13 @@ describe Justiz::Scraper::Courts do
         "JVA" => "Vollzugseinrichtungen",
         "AND" => "Sonstige Justizbehörden"
     }
-    expect(subject.court_types).to eq(types)
+    VCR.use_cassette 'courts/homepage' do
+      expect(subject.court_types).to eq(types)
+    end
   end
 
   it "should have states" do
     types = {
-        "ALL" => "-- Auswahl über PLZ/Ort --",
         "BRD" => "Bundesgerichte/-behörden",
         "BW" => "Baden-Württemberg",
         "BAY" => "Bayern",
@@ -53,56 +51,156 @@ describe Justiz::Scraper::Courts do
         "SH" => "Schleswig-Holstein",
         "TH" => "Thüringen"
     }
-    expect(subject.states).to eq(types)
+    VCR.use_cassette 'courts/homepage' do
+      expect(subject.states).to eq(types)
+    end
   end
 
-  context "Bundesgerichte" do
-    it "should find all Bundesgerichte" do
-      contacts = subject.scrape('ALL', 'BRD')
-      #ap contacts
-      expect(contacts.count).to eq(12)
+  context "entry counts" do
+    context "Bundesgerichte" do
+      it "should find all Bundesgerichte" do
+        VCR.use_cassette 'courts/all_brd' do
+          contacts = subject.contacts_for('BRD')
+          #ap contacts
+          expect(contacts.count).to eq(12)
+        end
+      end
+    end
+
+
+    context "NRW" do
+      it "should find all NRW" do
+        VCR.use_cassette 'courts/all_nrw' do
+          contacts = subject.contacts_for('NRW')
+          #ap contacts
+          expect(contacts.count).to eq(513)
+        end
+      end
+    end
+
+    context "search all" do
+      it "should find all entries" do
+        VCR.use_cassette 'courts/all_all' do
+          original = {
+              "BRD" => "Bundesgerichte/-behörden 12",
+              "BW" => "Baden-Württemberg 430",
+              "BAY" => "Bayern 348",
+              "B" => "Berlin 38",
+              "BRA" => "Brandenburg 64",
+              "BRE" => "Bremen 19",
+              "HH" => "Hamburg 35",
+              "HES" => "Hessen 115",
+              "MV" => "Mecklenburg-Vorpommern 55",
+              "NS" => "Niedersachsen 305",
+              "NRW" => "Nordrhein-Westfalen 513",
+              "RPF" => "Rheinland-Pfalz 101",
+              "SAA" => "Saarland 32",
+              "SAC" => "Sachsen 79",
+              "SAH" => "Sachsen-Anhalt 69",
+              "SH" => "Schleswig-Holstein 58",
+              "TH" => "Thüringen 63"
+          }
+          states = subject.states
+          total = 0
+          states.keys.each do |state|
+            count = subject.contacts_for(state).count
+            states[state] += " #{count}"
+            total += count
+          end
+          expect(states).to eq(original)
+          expect(total).to eq(2336)
+        end
+      end
     end
   end
 
+  context "id" do
+    it "dumps non-unique ids" do
+      msg = []
+      VCR.use_cassette 'courts/all_all' do
+        subject.contacts.inject({}) do |memo, c|
+          k = c.id
+          if memo.has_key?(k)
+            msg << "#{memo[k]}\n#{c}"
+          else
+            memo[k] = c
+          end
+          memo
+        end
+        expect(msg).to eq([])
+      end
+    end
 
-  context "NRW" do
-    it "should find all NRW" do
-      contacts = subject.scrape('ALL', 'NRW')
-      #ap contacts
-      expect(contacts.count).to eq(513)
+    it "should have unique court names" do
+      VCR.use_cassette 'courts/all_all' do
+        ids = subject.contacts.map(&:id)
+        expect(ids.uniq.length).to eq(ids.length)
+      end
     end
   end
 
-  context "search all" do
-    it "should find all entries" do
-      # as of 13.6.2013
-      original = {
-          "ALL" => "-- Auswahl über PLZ/Ort -- 0",
-          "BRD" => "Bundesgerichte/-behörden 12",
-          "BW" => "Baden-Württemberg 430",
-          "BAY" => "Bayern 348",
-          "B" => "Berlin 38",
-          "BRA" => "Brandenburg 64",
-          "BRE" => "Bremen 19",
-          "HH" => "Hamburg 35",
-          "HES" => "Hessen 115",
-          "MV" => "Mecklenburg-Vorpommern 55",
-          "NS" => "Niedersachsen 305",
-          "NRW" => "Nordrhein-Westfalen 513",
-          "RPF" => "Rheinland-Pfalz 101",
-          "SAA" => "Saarland 32",
-          "SAC" => "Sachsen 79",
-          "SAH" => "Sachsen-Anhalt 69",
-          "SH" => "Schleswig-Holstein 58",
-          "TH" => "Thüringen 63"
-      }
-      states = subject.states
-      states.keys.each do |state|
-        count = subject.scrape('ALL', state).count
-        states[state] += " #{count}"
+
+  context "contact details" do
+    it "parses first url from multiple" do
+      VCR.use_cassette 'courts/sg_b' do
+        contacts = subject.contacts_of_type('SG', 'B')
+        expect(contacts.length).to eq(2)
+        expect(contacts.first.url).to eq('http://www.berlin.de/lsg')
+      end
+    end
+
+    it "should have full location addresses" do
+      VCR.use_cassette 'courts/all_all' do
+        failed = false
+        subject.contacts.each do |contact|
+          a = contact.location_address
+          unless a.street && a.plz && a.city
+            ap a
+            failed = true
+          end
+        end
+        expect(failed).to eq(false)
+      end
+    end
+
+    it "should have plz & city post addresses" do
+      VCR.use_cassette 'courts/all_all' do
+        failed = false
+        subject.contacts.each do |contact|
+          a = contact.location_address
+          unless a.plz && a.city
+            ap a
+            failed = true
+          end
+        end
+        expect(failed).to eq(false)
+      end
+    end
+
+    xit "dumps contact in csv format" do
+      require 'csv'
+      VCR.use_cassette 'courts/all_all' do
+        CSV.open("address.csv", "wb") do |csv|
+          subject.contacts.each do |c|
+            l = c.location_address
+            p = c.post_address
+            csv << [c.justiz_id, c.court,
+                    l.street, l.plz, l.city,
+                    p.street, p.plz, p.city,
+                    c.phone, c.fax, c.email, c.url]
+          end
+        end
+      end
+    end
+
+    it "should return Address" do
+      VCR.use_cassette 'courts/all_brd' do
+        contact = subject.contacts_for('BRD').first
+        expect(contact.location_address).to be_a(Justiz::Address)
+        expect(contact.post_address).to be_a(Justiz::Address)
       end
-      expect(states).to eq(original)
     end
   end
 end
 
+

data/spec/lib/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'rspec'
+require 'justiz'
+require 'awesome_print'
+require 'vcr'
+
+VCR.configure do |c|
+  c.cassette_library_dir = 'spec/vcr'
+  c.hook_into :webmock
+end