justiz 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in justiz.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Mike Park
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Justiz
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'justiz'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install justiz
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/justiz

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'justiz'
+
+Justiz::Cli.start

data/justiz.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'justiz/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "justiz"
+  spec.version       = Justiz::VERSION
+  spec.authors       = ["Mike Park"]
+  spec.email         = ["mikep@quake.net"]
+  spec.description   = %q{Extracts contact data.}
+  spec.summary       = %q{Extract contact data from http://www.justizadressen.nrw.de/}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = ['justiz']
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "mechanize"
+  spec.add_dependency "thor"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 2.6"
+  spec.add_development_dependency "awesome_print"
+end

data/lib/justiz/cli.rb

@@ -0,0 +1,10 @@
+require 'thor'
+
+module Justiz
+  class Cli < Thor
+    desc "hello NAME", "say hello to NAME"
+    def hello(name)
+      puts "Hello #{name}"
+    end
+  end
+end

data/lib/justiz/contact.rb

@@ -0,0 +1,4 @@
+module Justiz
+  class Contact < Hash
+  end
+end

data/lib/justiz/scraper/agent.rb

@@ -0,0 +1,17 @@
+require 'mechanize'
+require 'logger'
+
+module Justiz
+  module Scraper
+    class Agent < Mechanize
+      def initialize
+        super do |config|
+          config.default_encoding = 'UTF-8'
+          config.force_default_encoding = true
+        end
+        #self.log = Logger.new STDOUT
+        self.user_agent_alias = 'Mac Safari'
+      end
+    end
+  end
+end

data/lib/justiz/scraper/courts.rb

@@ -0,0 +1,134 @@
+module Justiz
+  module Scraper
+    class Courts
+
+      def court_types
+        select_options(home_page, 'gerausw')
+      end
+
+      def states
+        select_options(home_page, 'landausw')
+      end
+
+      def scrape(court_type, state)
+        page = load_page(court_type, state)
+        # if we reach limit on ALL, query each subtype
+        if court_type == 'ALL' && limit_warning?(page)
+            court_types.map do |court_type, description|
+              next if court_type == 'ALL'
+              page = load_page(court_type, state)
+              if limit_warning?(page)
+                puts(STDERR, "Warning: State #{state} has too many contacts of #{description}[#{court_type}]")
+              end
+              parse_page(page)
+            end.flatten.compact.uniq
+        else
+          parse_page(page)
+        end
+      end
+
+      private
+
+      def home_page
+        @home_page ||= agent.get('http://www.justizadressen.nrw.de/og.php?MD=nrw')
+      end
+
+      def load_page(court_type, state)
+        form = home_page.forms[2]
+        form['gerausw'] = court_type
+        form['landausw'] = state
+        agent.submit(form, form.buttons_with(name: 'suchen1').first)
+      end
+
+      def agent
+        @agent ||= Justiz::Scraper::Agent.new
+      end
+
+      def parse_page(page)
+        rows = page.search('tr').map { |tr| tr.search('td').to_a }
+        contact_rows = rows.find_all { |row| row.length == 3 }
+        contact_rows.map do |court, addresses, phones|
+          addresses = AddressTd.new(addresses)
+          phones = AddressTd.new(phones)
+
+          Justiz::Contact.new.merge court: court.text.strip,
+                                    location: addresses.lieferanschrift,
+                                    post: addresses.postfach,
+                                    phone: phones.telefone,
+                                    fax: phones.fax,
+                                    justiz_id: phones.justiz_id,
+                                    url: phones.url,
+                                    email: phones.email
+        end
+      end
+
+      def limit_warning?(page)
+        # avoid invalid UTF-8 errors by force encoding.
+        page.search('p').find {|p| p.text.force_encoding("ISO-8859-15") =~ /Ihre Suchanfrage ergab mehr als/i}
+      end
+
+      def select_options(page, name)
+        page.search("[name='#{name}'] > option").inject({}) do |memo, node|
+          memo[node['value']] = node.text
+          memo
+        end
+      end
+
+      class AddressTd
+        attr_reader :texts
+
+        def initialize(node)
+          nodes = node.children.to_a
+          @texts = nodes.map { |n| n.text.strip }.find_all { |t| !blank?(t) }
+        end
+
+        def telefone
+          same_line('Telefon:')
+        end
+
+        def fax
+          same_line('Fax:')
+        end
+
+        def justiz_id
+          same_line('XJustiz-ID:')
+        end
+
+        def lieferanschrift
+          next_line('Lieferanschrift')
+        end
+
+        def postfach
+          next_line('Postanschrift')
+        end
+
+        def url
+          next_line('URL')
+        end
+
+        def email
+          next_line('Mail')
+        end
+
+        private
+
+        def blank?(something)
+          something.to_s !~ /[^[:space:]]/
+        end
+
+        def next_line(name)
+          reg = Regexp.new(name, true)
+          line = texts.find_index { |text| text.match(reg) }
+          line && texts[line + 1]
+        end
+
+        def same_line(name)
+          reg = Regexp.new("#{name}(.*)", true)
+          text = texts.map { |t| t.match(reg) }.compact.first
+          text = text[1].strip if text
+          text if !blank?(text)
+        end
+      end
+    end
+  end
+end

data/lib/justiz/version.rb

@@ -0,0 +1,3 @@
+module Justiz
+  VERSION = "0.0.1"
+end

data/lib/justiz.rb

@@ -0,0 +1,8 @@
+require "justiz/version"
+require 'justiz/cli'
+require 'justiz/scraper/agent'
+require 'justiz/scraper/courts'
+require 'justiz/contact'
+
+module Justiz
+end

data/spec/lib/contact_spec.rb

@@ -0,0 +1,5 @@
+require 'rspec'
+require 'justiz'
+
+describe Justiz::Contact do
+end

data/spec/lib/justiz_spec.rb

@@ -0,0 +1,10 @@
+require 'rspec'
+
+describe 'My behaviour' do
+
+  it 'should do something' do
+
+    #To change this template use File | Settings | File Templates.
+    true.should == false
+  end
+end

data/spec/lib/scraper/courts_spec.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+
+require 'rspec'
+require 'justiz'
+require 'awesome_print'
+
+describe Justiz::Scraper::Courts do
+  it "should have court types" do
+    types = {
+        "ALL" => "-- alle Gerichte/Behörden --",
+        "AG" => "Amtsgerichte",
+        "LG" => "Landgerichte",
+        "OLG" => "Oberlandesgerichte",
+        "STA" => "Staatsanwaltschaften",
+        "MG" => "Mahngerichte",
+        "FAM" => "Familiengerichte",
+        "INS" => "Insolvenzgerichte",
+        "HREG" => "Handels-/Genossenschaftsregister",
+        "VREG" => "Vereinsregistergerichte",
+        "PREG" => "Partnerschaftsregistergerichte",
+        "ZVG" => "Zwangsversteigerungsgerichte",
+        "ZVK" => "Zentrale Vollstreckungsgerichte",
+        "SG" => "Sozialgerichte",
+        "VG" => "Verwaltungsgerichte",
+        "ARB" => "Arbeitsgerichte",
+        "FG" => "Finanzgerichte",
+        "VFG" => "Verfassungsgerichte",
+        "JM" => "Justizministerium",
+        "JVA" => "Vollzugseinrichtungen",
+        "AND" => "Sonstige Justizbehörden"
+    }
+    expect(subject.court_types).to eq(types)
+  end
+
+  it "should have states" do
+    types = {
+        "ALL" => "-- Auswahl über PLZ/Ort --",
+        "BRD" => "Bundesgerichte/-behörden",
+        "BW" => "Baden-Württemberg",
+        "BAY" => "Bayern",
+        "B" => "Berlin",
+        "BRA" => "Brandenburg",
+        "BRE" => "Bremen",
+        "HH" => "Hamburg",
+        "HES" => "Hessen",
+        "MV" => "Mecklenburg-Vorpommern",
+        "NS" => "Niedersachsen",
+        "NRW" => "Nordrhein-Westfalen",
+        "RPF" => "Rheinland-Pfalz",
+        "SAA" => "Saarland",
+        "SAC" => "Sachsen",
+        "SAH" => "Sachsen-Anhalt",
+        "SH" => "Schleswig-Holstein",
+        "TH" => "Thüringen"
+    }
+    expect(subject.states).to eq(types)
+  end
+
+  context "Bundesgerichte" do
+    it "should find all Bundesgerichte" do
+      contacts = subject.scrape('ALL', 'BRD')
+      #ap contacts
+      expect(contacts.count).to eq(12)
+    end
+  end
+
+
+  context "NRW" do
+    it "should find all NRW" do
+      contacts = subject.scrape('ALL', 'NRW')
+      #ap contacts
+      expect(contacts.count).to eq(513)
+    end
+  end
+
+  context "search all" do
+    it "should find all entries" do
+      # as of 13.6.2013
+      original = {
+          "ALL" => "-- Auswahl über PLZ/Ort -- 0",
+          "BRD" => "Bundesgerichte/-behörden 12",
+          "BW" => "Baden-Württemberg 430",
+          "BAY" => "Bayern 348",
+          "B" => "Berlin 38",
+          "BRA" => "Brandenburg 64",
+          "BRE" => "Bremen 19",
+          "HH" => "Hamburg 35",
+          "HES" => "Hessen 115",
+          "MV" => "Mecklenburg-Vorpommern 55",
+          "NS" => "Niedersachsen 305",
+          "NRW" => "Nordrhein-Westfalen 513",
+          "RPF" => "Rheinland-Pfalz 101",
+          "SAA" => "Saarland 32",
+          "SAC" => "Sachsen 79",
+          "SAH" => "Sachsen-Anhalt 69",
+          "SH" => "Schleswig-Holstein 58",
+          "TH" => "Thüringen 63"
+      }
+      states = subject.states
+      states.keys.each do |state|
+        count = subject.scrape('ALL', state).count
+        states[state] += " #{count}"
+      end
+      expect(states).to eq(original)
+    end
+  end
+end
+

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: justiz
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Mike Park
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Extracts contact data.
+email:
+- mikep@quake.net
+executables:
+- justiz
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/justiz
+- justiz.gemspec
+- lib/justiz.rb
+- lib/justiz/cli.rb
+- lib/justiz/contact.rb
+- lib/justiz/scraper/agent.rb
+- lib/justiz/scraper/courts.rb
+- lib/justiz/version.rb
+- spec/lib/contact_spec.rb
+- spec/lib/justiz_spec.rb
+- spec/lib/scraper/courts_spec.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Extract contact data from http://www.justizadressen.nrw.de/
+test_files:
+- spec/lib/contact_spec.rb
+- spec/lib/justiz_spec.rb
+- spec/lib/scraper/courts_spec.rb