lobbyliste 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a4ec8ab1a342db01916730a29a6061d70dd9e4f0
+  data.tar.gz: b65292668e6d0256df3c1b517a84157838a1a0c8
+SHA512:
+  metadata.gz: d119a69036498b8e8d124bb1a1a6244a5904ffb9576c68ae08afe7078318601bb6040678ba9a0598bf223f0dc9265bc157bb366e9eb1a6b46e93d798edd69910
+  data.tar.gz: fd4984edf6a3ef34632ca714911a871a52ea5160858188dc7c235e7fb7fdab682827f16ce1e7691ce359525cd63f77f8f86b37f73170009a92e039c8eb02f3b5

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.idea/

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - "2.0"
+  - "2.1"
+  - "2.2"
+  - "2.3.0"
+
+before_install:
+  - gem install bundler -v 1.11.2

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,49 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at max@kopfueber.org. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in lobbyliste.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 DarthMax
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,76 @@
+# Lobbyliste
+
+[![Build Status](https://travis-ci.org/FHG-IMW/lobbyliste.svg?branch=master)](https://travis-ci.org/FHG-IMW/lobbyliste) [![Documentation Status](https://inch-ci.org/github/FHG-IMW/lobbyliste.svg?branch=master)](https://inch-ci.org/github/FHG-IMW/lobbyliste/suggestions?branch=master)
+
+This gem crawls and parses the the list of lobbyists which is published as a PDF by the German Bundestag.
+Our goal is to provide a simple and easy to maintain parser.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'lobbyliste'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install lobbyliste
+
+*NOTE: This gem requires JAVA to be installed. We use [PDFBox](https://pdfbox.apache.org/) for PDF extraction as this currently seems to be the best alternative*
+
+## Usage
+
+```ruby
+require 'lobbyliste'
+
+list = Lobbyliste.fetch_and_parse
+organisation = list.organisations.first
+
+organisation.name #=> 1219. Deutsche Stiftung für interreligiösen und interkulturellen Dialog e. V.
+
+organisation.people.map {|person| person.name} #=> ["Claudius Groß", "Markus Hoymann", "Thomas M. Schimmel"]
+
+organisation.tags #=> ["Kultur", "Religion"]
+
+organisation.abbreviations #=> []
+
+address = organisation.address
+puts address.full_address
+# 1219. Deutsche Stiftung für interreligiösen und #interkulturellen Dialog e. V.
+# Hinter der katholischen Kirche 3
+# 10117 Berlin
+# Deutschland
+# Tel: +4930 51057773
+# Fax: +4930 51057785
+# Email: schimmel@1219.eu
+# http://www.1219.eu
+```
+
+### CLI
+
+You can also use this gem on your comandline. It will dump the complete list as JSON
+
+For example to create a gziped json file run:
+
+```bash
+$ lobbyliste | gzip > lobbyliste.json.gz 
+```
+
+
+# Special Thanks
+
+- [Sebastian Vollnhals (@yetzt)](https://github.com/yetzt) - for his excellent node based scraper for the lobbyliste (https://github.com/yetzt/scraper-lobbyliste) from which many lines were reused.
+
+# Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/lobbyliste. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "lobbyliste"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/lobbyliste

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "lobbyliste"
+
+list = Lobbyliste.fetch_and_parse
+puts list.to_json

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/lobbyliste/address.rb

@@ -0,0 +1,79 @@
+module Lobbyliste
+
+  # This class represents addresses found in the lobbylist.
+  class Address
+    # @return [String] organisation name (the bold part)
+    attr_reader :name
+
+    # @return [String] Everything that is not part of the name or any other field
+    attr_reader :address
+
+    # @return [String] Postcode
+    attr_reader :postcode
+
+    # @return [String] City
+    attr_reader :city
+
+    # @return [String] the country, default: "Germany"
+    attr_reader :country
+
+    # @return [String] the telephone number if given (german numbers are automatically prefixed with +49)
+    attr_reader :tel
+
+    # @return [String] the fax number if given (german numbers are automatically prefixed with +49)
+    attr_reader :fax
+
+    # @return [String] website url
+    attr_reader :website
+
+    # @return [String] contact email address
+    attr_reader :email
+
+    # @return [Symbol] address type, :primary for 1. address, :secondary for all others
+    attr_reader :type
+
+    def initialize(name, address, postcode, city, country, tel, fax, website, email, type)
+      @name = name
+      @address = address
+      @postcode = postcode
+      @city = city
+      @country = country
+      @tel = tel
+      @fax = fax
+      @website = website
+      @email = email
+      @type=type
+    end
+
+    # @return String pretty formated address of all existing address fields
+    def full_address
+      full_address = [
+        @name,
+        @address,
+        [@postcode,@city].reject(&:nil?).join(" "),
+        @country,
+      ]
+
+      full_address << "Tel: #{@tel}" if @tel
+      full_address << "Fax: #{@fax}" if @fax
+      full_address << "Email: #{@email}" if @email
+      full_address << @website if @website
+      full_address.reject(&:nil?).join("\n")
+    end
+
+    def to_json(*a)
+      {
+          name: name,
+          address: address,
+          postcode: postcode,
+          city: city,
+          country: country,
+          tel: tel,
+          fax: fax,
+          email: email,
+          website: website,
+          type: type.to_s
+      }.to_json(*a)
+    end
+  end
+end

data/lib/lobbyliste/core_ext/string.rb

@@ -0,0 +1,5 @@
+class String
+  def squish
+    self.dup.gsub(/[[:space:]]+/, ' ').strip
+  end
+end

data/lib/lobbyliste/downloader.rb

@@ -0,0 +1,71 @@
+require 'open-uri'
+require 'nokogiri'
+
+module Lobbyliste
+
+  # This class finds the lobbyliste pdf on the Bundestag website, downloads it and extracts the pdf content
+  class Downloader
+
+    # @return [String] raw content of pdf file
+    def pdf_data
+      retrieve_pdf unless @pdf_data
+      @pdf_data
+    end
+
+
+    # @return [String] extracted content of pdf file
+    def text_data
+      extract_pdf unless @text_data
+      @text_data
+    end
+
+    # @return [String] extracted content of pdf file in html format
+    def html_data
+      extract_pdf unless @html_data
+      @html_data
+    end
+
+    private
+
+    # Since this link changes with every new version we download the Lobbyliste website and try to extract the link
+    # @return [String] the link to the Lobbyliste pdf
+    def pdf_link
+      website = Nokogiri::HTML(open("https://www.bundestag.de/dokumente/lobbyliste"))
+      link = website.css(".inhalt a[title^='Aktuelle Fassung']").first
+
+      raise "Could no find PDF link on the website!" unless link
+      "https://bundestag.de#{link['href']}"
+    end
+
+    def retrieve_pdf
+      @pdf_data = open(pdf_link) {|f| f.read}
+    end
+
+
+    def extract_pdf
+      pdf_file = Tempfile.new(["lobbyliste",".pdf"])
+      pdf_file.write(pdf_data)
+      pdf_file.rewind
+
+      @text_data = run_extraction(pdf_file)
+      @html_data = run_extraction(pdf_file,true)
+    ensure
+      pdf_file.close
+      pdf_file.unlink
+    end
+
+    def run_extraction(pdf_file,html=false)
+      tmp_file = Tempfile.new(["lobbyliste"])
+      status = system("/usr/bin/java -jar #{jar_path} ExtractText #{pdf_file.path} #{html ? "-html":""} #{tmp_file.path} > /dev/null 2>&1")
+      raise "PDF extraction failed" unless status
+      return tmp_file.read
+    ensure
+      tmp_file.close
+      tmp_file.unlink
+    end
+
+    def jar_path
+      File.join(File.dirname(File.expand_path(__FILE__)), '../../ext/pdfbox.jar')
+    end
+  end
+end

data/lib/lobbyliste/factories/address_factory.rb

@@ -0,0 +1,129 @@
+module Lobbyliste
+  module Factories
+    # This class is used to build an address from raw data
+    # Since it is to hard to separate the na,e and address data without markup,
+    # we use the html data to accomplish that
+    class AddressFactory
+
+      # @return [Lobbyliste::Address]
+      def self.build(name,raw_data, type=:primary)
+        factory = new(name,raw_data,type)
+        ::Lobbyliste::Address.new(
+          factory.name,
+          factory.address,
+          factory.postcode,
+          factory.city,
+          factory.country,
+          factory.tel,
+          factory.fax,
+          factory.website,
+          factory.email,
+          factory.type
+        )
+      end
+
+      attr_reader :name, :tel, :fax, :website, :email, :country, :postcode, :city, :type
+
+      def initialize(name,raw_data,type=:primary)
+        @name = name
+        @raw_data = raw_data
+
+        @address = []
+        @tel = nil
+        @fax = nil
+        @website = nil
+        @email = nil
+        @country = "Deutschland"
+        @postcode = nil
+        @city = nil
+        @type=type
+
+        parse
+      end
+
+      def parse
+        @raw_data.each_with_index do |line,i|
+          case label(line,i)
+            when :addr then @address << line
+            when :tel then extract_tel_fax(line)
+            when :postcode then extract_postcode_city(line)
+            when :email then extract_email(line)
+            when :website then extract_website(line)
+            when :country then @country = line
+            else next
+          end
+        end
+      end
+
+      def address
+        @address.join(", ")
+      end
+
+      private
+
+      def label(line,i)
+        # this line is part of the name
+        return :name if name.include?(line)
+
+        return :tel if line.match(/^(Tel\.|Fax): /)
+        return :email if   line.match(/^E\-Mail\: /)
+        return :website if  line.match(/^Internet\:/)
+
+        # international postcodes
+        return :postcode if line.match(/^\d{5,7}\s(.+)$/)
+        # UK postcodes
+        return :postcode if line.match(/^([A-Z0-9]{3}\s?[A-Z0-9]{3})$/)
+
+        # if the line looks like an address
+        return :addr if  line.match(/(c\/o|^postfach\b|[Ss]tr(aße|\.)?\b|[Aa]llee\b|[Pp]latz\b|[Gg]asse\b|[Ww]eg\b|\b([0-9]+\-)?[0-9]+\s?[a-zA-Z]*$|[Vv]orstand|[Ss]ekretär|[Gg]eschäfts)/)
+        # if the previous line ended with e.V. the next line is addr
+        return :addr if @raw_data[i-1].match(/(e\.\s?V\.|\([A-Z]+\)$)/)
+
+        return :country if %w(Niederlande Belgien Schweiz Luxemburg Dänemark Österreich Tschechien Polen USA Israel Russland).include?(line)
+        # a single Word with a capital letter is probably a country
+        return :country if i > 3 && line.match(/^[A-Z][a-zA-Zöüä?]+$/)
+        return :country if line == "Vereinigtes Königreich"
+
+        :addr
+      end
+
+      def extract_tel_fax(line)
+        _tel = line.match(/Tel\.\: ((\(?\d+\)? )?(\d+\s?)+\d+)/)
+        if _tel
+          @tel = _tel[1].gsub(/[\(|\)]/,"").gsub(/^0{1,2}/,"+49")
+        end
+
+        _fax = line.match(/Fax\: ((\(?\d+\)? )?(\d+\s?)+\d+)/)
+        if _fax
+          @fax = _fax[1].gsub(/[\(|\)]/,"").gsub(/^0{1,2}/,"+49")
+        end
+      end
+
+      def extract_postcode_city(line)
+        _postcode_city = line.match(/(\d{5,7})\s?(.+)?/)
+        if _postcode_city
+          @postcode = _postcode_city[1]
+          @city = _postcode_city[2]
+        else
+          _uk_postcode = line.match(/^([A-Z0-9]{3}\s?[A-Z0-9]{3})$/)
+          @postcode = _uk_postcode[1] if _uk_postcode
+        end
+      end
+
+      def extract_website(line)
+        _website = line.match(/^Internet\:\s?(.+)$/)
+        if _website
+          @website = _website[1]
+        end
+      end
+
+      def extract_email(line)
+        _email = line.match(/^E\-Mail\:\s?(.+)$/)
+        if _email
+          @email = _email[1]
+        end
+      end
+
+    end
+  end
+end

data/lib/lobbyliste/factories/list_factory.rb

@@ -0,0 +1,179 @@
+module Lobbyliste
+  module Factories
+    # This class is used to build the list from raw data
+    class ListFactory
+      attr_reader :data
+
+      # @return [Lobbyliste::List]
+      def self.build(text_data,html_data)
+        factory = new(text_data,html_data)
+        ::Lobbyliste::List.new(
+          factory.organisations,
+          factory.tags,
+          factory.abbreviations,
+          factory.last_update
+        )
+      end
+
+      def initialize(text_data,html_data)
+        @text_data = text_data
+        @html_data = html_data
+
+        @lines = text_data.each_line.to_a.map(&:chomp)
+
+        @organisations = nil
+        @tags = nil
+        @abbreviations = nil
+        @names = nil
+      end
+
+      def organisations
+        return @organisations if @organisations
+
+        @organisations = organisations_data.map do |organisation_data|
+          id = organisation_data[0].to_i
+          name = names[id]
+          tags = tags_for_organisation(id)
+          abbreviations = abbreviations_for_organisation(id)
+          ::Lobbyliste::Factories::OrganisationFactory.build(name,organisation_data,tags,abbreviations)
+        end
+      end
+
+      def names
+        extract_names unless @names
+        @names
+      end
+
+
+      def tags
+        return @tags if @tags
+
+        tags = Hash.new{|h,k| h[k] = []}
+        tag_data = extract_tag_data
+
+        current_tag = "A"
+        tag_data.each do |line|
+          if line.match(/^[A-ZÄÖÜ][a-zäöüß]+$/) && [current_tag[0],current_tag[0].next].include?(line[0])
+            current_tag = line
+          elsif line.match(/^\– \d+/)
+            id = line.match(/^\– (\d+)/)[1].to_i
+            tags[current_tag] << id
+          end
+        end
+
+        @tags = tags
+      end
+
+
+      def abbreviations
+        return @abbreviations if @abbreviations
+        abbreviations = Hash.new{|h,k| h[k] = []}
+        current_abbr = "A"
+        extract_abbreviation_data.each do |line|
+          if line.match(/^[A-ZÄÖÜ][A-ZÄÖÜa-zäöüß]+$/) && [current_abbr[0],current_abbr[0].next].include?(line[0])
+            current_abbr = line
+          elsif line.match(/^\– \d+/)
+            id = line.match(/^\– (\d+)/)[1].to_i
+            abbreviations[current_abbr] << id
+          end
+        end
+
+        @abbreviations = abbreviations
+      end
+
+      def last_update
+        date = @text_data.match /^Stand: (\d\d\.\d\d\.\d\d\d\d)/
+        Date.parse(date[1])
+      end
+
+      private
+
+
+        def organisations_data
+          start_lines = []
+          end_line = nil
+
+          @lines.each_with_index do |line,i|
+            if possible_organisation_id?(line) && begin_organisation?(@lines[i+1])
+              start_lines << i
+            elsif line == "Stichwortverzeichnis"
+              end_line = i - 1
+              break
+            end
+          end
+
+          organisations_data = start_lines.each_cons(2).map do |a,b|
+            @lines[a..b-1]
+          end
+
+          organisations_data.
+              push(@lines[start_lines.last..end_line]).
+              map { |data| data.reject {|line| ignored_line?(line)} }
+        end
+
+
+
+        def extract_tag_data
+          start_line = @lines.index {|line| line == "Stichwortverzeichnis"}
+          @lines.
+            drop(start_line+1).
+            take_while {|line| !(line == "Verzeichnis der anderen Namensformen")}.
+            reject {|line| ignored_line?(line)}
+        end
+
+        def extract_abbreviation_data
+          start_line = @lines.index {|line| line == "Verzeichnis der anderen Namensformen"}
+          @lines.
+            drop(start_line+1).
+            reject {|line| ignored_line?(line)}
+        end
+
+        def tags_for_organisation(organisation_id)
+
+          tags.
+              select {|_,organisation_ids| organisation_ids.include?(organisation_id)}.
+              map(&:first)
+        end
+
+        def abbreviations_for_organisation(organisation_id)
+          abbreviations.
+              select {|_,organisation_ids| organisation_ids.include?(organisation_id)}.
+              map(&:first)
+        end
+
+
+        def extract_names
+          names = {}
+
+          regexp = Regexp.compile(/<p><b>(\d+)\n<\/b>N a m e u n d S i t z \, 1 \. A d r e s s e\n<\/p>\n<p><b>(.*?)\n<\/b>/m)
+
+          @html_data.to_enum(:scan, regexp).each do
+            match = Regexp.last_match
+            names[match[1].to_i] = CGI.unescape_html(match[2].gsub("\n"," "))
+          end
+
+          @names = names
+        end
+
+
+        def ignored_line?(line)
+          regexps = [
+                      /^– \d+ –$/,
+                      /^Aktuelle Fassung der öffentlichen Liste/,
+                      /^Die Zahlen verweisen auf die fortlaufenden Nummern im Hauptteil/,
+                      /^\n$/
+
+          ]
+          regexps.any? {|regexp| line.match(regexp)}
+        end
+
+        def possible_organisation_id?(line)
+          line =~ /^\d+$/
+        end
+
+        def begin_organisation?(line)
+          line =~/^N a m e u n d S i t z \, 1 \. A d r e s s e$/
+        end
+    end
+  end
+end

data/lib/lobbyliste/factories/organisation_factory.rb

@@ -0,0 +1,113 @@
+module Lobbyliste
+  module Factories
+
+    # This class is used to build an organisation from raw data
+    class OrganisationFactory
+
+      # @return [Lobbyliste::Organisation]
+      def self.build(name, raw_data,tags,abbreviations)
+        factory = new(name, raw_data)
+        ::Lobbyliste::Organisation.new(
+            factory.id,
+            factory.name,
+            factory.address,
+            factory.additional_address,
+            factory.address_at_bt_br,
+            factory.people,
+            factory.interests,
+            factory.members,
+            factory.associated_organisations,
+            tags,
+            abbreviations
+        )
+      end
+
+      attr_reader :name
+
+      def initialize(name,raw_data)
+        @name = name
+        @raw_data = raw_data
+      end
+
+      def id
+        @raw_data.first.to_i
+      end
+
+
+      def address
+        data = read_section("N a m e u n d S i t z , 1 . A d r e s s e")
+        AddressFactory.build(name, data, :primary)
+      end
+
+      def additional_address
+        data = read_section("W e i t e r e A d r e s s e")
+        return nil if data[0] == "–"
+        AddressFactory.build(name, data, :secondary)
+      end
+
+      def address_at_bt_br
+        data = read_section("A n s c h r i f t a m S i t z v o n B T u n d B R g")
+        return nil if data[0] == "–" || data[0].match(/\(s\. Abschnitt/)
+        AddressFactory.build(name, data, :secondary)
+      end
+
+      def people
+        data = read_section("V o r s t a n d u n d G e s c h ä f t s f ü h r u n g")
+        data.concat read_section("V e r b a n d s v e r t r e t e r / - i n n e n")
+        data.reject! {|line| ignored_person_line?(line)}
+
+        data.map { |person| PersonFactory.build(person) }.uniq.reject(&:nil?)
+      end
+
+      def interests
+        read_section("I n t e r e s s e n b e r e i c h").join(" ").gsub(/(- )(?=[a-z])/,"")
+      end
+
+      def members
+        read_section("M i t g l i e d e r z a h l")[0].to_i || nil
+      end
+
+      def associated_organisations
+        read_section("A n z a h l d e r a n g e s c h l o s s e n e n O r g a n i s a t i o n e n")[0].to_i || nil
+      end
+
+
+
+      private
+        def new_section?(line)
+           line =~ /^([a-zA-Z\d\,\.\-\/äöüß]\s){3,}\w$/
+        end
+
+        def read_section(section)
+          start_line = @raw_data.index {|line| line == section}
+          return [] unless start_line
+
+          @raw_data.drop(start_line+1).take_while {|line| !new_section?(line)}
+        end
+
+        def ignored_person_line?(line)
+          [
+            /^–$/,
+            /\(s\. Abschnitt/,
+            /\:$/,
+            /^GdW$/,
+            /^Forschung$/,
+            /^des Verwaltungsrats$/,
+            /^Schatzmeister$/,
+            /^Kinder- u\. Jugendmed\.$/,
+            /^u\. Kinderchirurgen$/,
+            /^Finanzen & Recht I$/,
+            /^Geschäftsführ(er(in)?|ung)$/,
+            /^gleichzeitig Verbandsdirektor^/,
+            /^(stellvertretender )?Vorsitzender?$/,
+            /^weitere Vorstandsmitglieder$/,
+            /^Managementgesellschaft des DZVhÄ/,
+            /^Besonderer Vertreter nach § 30/,
+            /^Sektretär$/,
+            /^Alleingesellschafter: Ev\.Werk für/
+          ].any? {|regexp| line.match(regexp) }
+        end
+
+    end
+  end
+end

data/lib/lobbyliste/factories/person_factory.rb

@@ -0,0 +1,70 @@
+module Lobbyliste
+  module Factories
+
+    # This class is used to build a person from raw line data
+    # It has to split titles from the actual name
+    class PersonFactory
+
+      # A list of regular expressions that try to match any titles that might occur
+      REGEX = [
+         # Prof titles
+        /Prof\.?/,
+        # Dr. titles
+        /(PhD|Dr\.(([a-z]+\.?)+|(\-\w+\.?))*)/,
+        # Dipl. titles
+        /(Magister|(Dipl[\.|\-]+)([\w|ÖöÄäÜüß]*[\.|\-]*)*)/,
+        # other academic abbreviations
+        /\b([a-z|A-Z]{2,}\.)+/,
+        # h.c.
+        /h\.\s?c\./,
+        # general abbreviations
+        /\(?\b(M\.?Sc|B\.?Sc|B\.?sc|FH|fh|BA|Ba|TH|Th|VWA|univ|US|PD|RA|CCM|LD|Ing\.|OTL|CISA|CIA|CISM|CRISC|StB|vBP|StD|habil|med)\b\)?/,
+        # job titles
+        /((^\s*und)?(Mathematiker|(Bundes)?Vorstand|^Arzt|Assistent|Generalstabs[aä]rzt|Augenoptikermeister|Bankkauf|Bau-Ing\.|Betriebswirtschaftslehre|Mediziner|agrar|Schneidwerkzeugmechanikermeister|Studienassessor|Gesundheitsökonom|Syndikus|Straßenbaumeister|^Bruder|Pater|Prälat|Theologe|Jurist|\bmed|Ministerpräsident|(Medizin)?pädagog|Verwaltungs-Wirt|Veterinär|^Kauf|Generalleutnant|General|Generalarzt|Ergotherapeut|Fregattenkapitän|Agrarbiolog|Amtsanw[aä]lt|Apotheker|(Freie )?Architekt|(Berg-)?Assessor|Betriebswirt|^Bischof( von [a-z]+)?|Botschafter|(vereidigter )?Buchprüfer|Bundesbankoberamtsrat|Bundesinnungsmeister|Bundesminister|Bundespräsident|Bundestagspräsident|Bürgermeister|Chefapotheker|Diakon|Dompropst|Augenoptiker|Optometrist|Biolog|Brennmeister|Chemiker|((Kommunikations|Grafik)[\-\s]?)?Designer|Finanzwirt|Forstwirt|Geograph|Geolog|Geophysiker|Handelslehrer|Holzwirt|Informatiker|Jurist|Kauf|Mathematiker|Medizinpädagog|Meteorolog|physiker|Politolog|Psycholog|Pädagog|Rechtspfleger|Restaurator|Sachverständig|Sozialpädagog|Sozialwirt|Sozialwissenschaftler|Soziolog|Stomatolog|Verwaltungsbetriebswirt|Verwaltungswirt|Verwaltungswissenschaftler|Volkswirt|Wirtschafts(ing)?|Wirtschaftsjurist|Ökonom|Übersetzer|Domkapitular|Probst|Finanzfachwirt|(Forst|Bau)assessor|Hauptfeldwebel|Hauptmann|Honorar|Honorargeneralkonsul|Honorarkonsul|Justizminister|Kapitän(leutnant)?|Konsul|Land(es)?rat|Landwirtschaftsmeister|Lohnsteuerberater|Landwirtschaftsdirektor|Luftverkehrskauf|Magistratsr[aä]t|BreigGen|Generalmajor|Major|Minist|Ministerialdirektor|Ministerialdirigent|Monsignore|Notar|Oberamtsanw[aä]lt|Oberbürgermeister|Oberfeldarzt|Obermeister|Oberst|Oberstaatsanw[aä]lt|Oberstabsboots|Oberstabsfeldwebel|Oberstleutnant|Oberstudiendirektor|Staatssekretär|Parlamentarischer|Parlamentspräsident|Patentanw[aä]lt|(Landesjugend)?[Pp]farrer|Politikwissenschaftler|Privatdozent|Priv(\\.|at)-?Dozent|RA|Ran|Fachanwalt( für [a-z]+)?|Realschulrektor|Rechtsreferent|Regierungsamts(rat)?|Regionspräsident|Revieroberjäger|(Vorsitzender? )?Richter(in)?( am [a-zA-Z]+)?|Senator|Staats(minister|sekretär)|Stabsfeldwebel|Stabshauptmann|(Steuer|Wirtschafts)(berater|prüfer)|Stuckateurmeister|Studiendirektor|Studienrat|Uni(v(ersitäts)?)?|Verleger|Rechtsjournalist|Rechtsjurist|Veterinärdirektor|Vizepräsident des Verwaltungsgerichts|Visuelle Kommunikation|Zahntechnikermeister)(er|e|\s?in|mann|frau)?)(\s?am\s(Finanzgericht|Bundesverwaltungsgerichtshof|Verwaltungsgerichtshof))?(\bD\.)?(\s?a\.D\.)?([\.\-,\s\/]+|$)/,
+        # Beamtenbezeichnung?
+        /^D\.\s/,
+        # more abbreviations
+        /\b[A-ZÜÄÖ]{2,}\b/,
+        # Everything after first colon
+        /,.*$/
+      ]
+
+
+      # @return [Lobbylist::Person] builds a new person, might be nil if the line does not represent a person
+      def self.build(raw_data)
+        factory = new(raw_data)
+        factory.is_person? ? ::Lobbyliste::Person.new(factory.name,factory.titles) : nil
+      end
+
+      def initialize(raw_data)
+        @raw_data = raw_data
+        @name = nil
+      end
+
+      def name
+        return @name if @name
+
+        @name = @raw_data.dup
+        REGEX.each do |regex|
+          @name = clean(@name.gsub(regex,""))
+        end
+
+        @name
+      end
+
+      def titles
+        @raw_data.gsub(name,"").split(", ").map(&:squish).reject{|x| x==""}
+      end
+
+      def is_person?
+        !name.nil? && name.length > 2 && name.include?(" ")
+      end
+
+      private
+
+      def clean(string)
+        string.gsub(/^(\s*[,-:\(\)\|\.])*/,"").squish
+      end
+    end
+  end
+end

data/lib/lobbyliste/factories.rb

@@ -0,0 +1,9 @@
+require 'lobbyliste/factories/list_factory'
+require 'lobbyliste/factories/organisation_factory'
+require 'lobbyliste/factories/address_factory'
+require 'lobbyliste/factories/person_factory'
+
+module Lobbyliste
+  module Factories
+  end
+end

data/lib/lobbyliste/list.rb

@@ -0,0 +1,34 @@
+module Lobbyliste
+
+  # This class represents an instance of the parsed lobbylist.
+  class List
+
+    # @return [Array] list of organisations
+    attr_reader :organisations
+
+    # @return [Hash] keys are the tags, values are Arrays of organisation ids
+    attr_reader :tags
+
+    # @return [Hash] keys are the abbreviations, values are Arrays of organisation ids
+    attr_reader :abbreviations
+
+    # @return [Date] the date when the document was last updated
+    attr_reader :last_update
+
+    def initialize(organisations, tags, abbreviations, last_update)
+      @organisations = organisations
+      @tags=tags
+      @abbreviations = abbreviations
+      @last_update = last_update
+    end
+
+    def to_json(*a)
+      {
+          organisations: organisations,
+          tags: tags,
+          abbreviations: abbreviations,
+          last_update: last_update
+      }.to_json(*a)
+    end
+  end
+end

data/lib/lobbyliste/organisation.rb

@@ -0,0 +1,71 @@
+module Lobbyliste
+  # Class to encapsulate an organisation
+  class Organisation
+
+    # @return [Integer] the organisation id of the organisation. This number is not fix and may change with every new document version
+    attr_reader :id
+
+    # @return [String] the organisations name
+    attr_reader :name
+
+    # @return [Lobbyliste::Address] the primary Address of the organisation
+    attr_reader :address
+
+    # @return [Lobbyliste::Address] the address stated under "Weitere Addresse"
+    attr_reader :additional_address
+
+    # @return [Lobbyliste::Address] the address stated under "Anschrift am Sitz von BT und BRg"
+    attr_reader :address_at_bt_br
+
+    # @return [Array] List of {Lobbyliste::Person} which includes all members stated under "Vorstand und Geschäftsführung" and "Verbandsvertreter/-innen"
+    attr_reader :people
+
+    # @return [String] interests as stated under "Interessenbereich"
+    attr_reader :interests
+
+    # @return [Integer] number of members as stated under "Mitgleiderzahl"
+    attr_reader :members
+
+    # @return [Integer] number of associated organisations as stated under "Anzahl der angeschlossenen Organisationen"
+    attr_reader :associated_organisations
+
+    # @return [Array] list of tags
+    attr_reader :tags
+
+    # @return [Array] list of abbreviations
+    attr_reader :abbreviations
+
+    def initialize(id, name, address, additional_address, address_at_bt_br, people, interests, members, associated_organisations,tags,abbreviations)
+      @id = id
+      @name = name
+      @address = address
+      @additional_address = additional_address
+      @address_at_bt_br = address_at_bt_br
+      @people = people
+      @interests = interests
+      @members = members
+      @associated_organisations = associated_organisations
+      @tags = tags
+      @abbreviations = abbreviations
+    end
+
+    # @return [Array] list of all known addresses
+    def addresses
+      [address,additional_address,address_at_bt_br].reject(&:nil?)
+    end
+
+    def to_json(*a)
+      {
+          id: id,
+          name: name,
+          addresses: addresses,
+          people: people,
+          interests: interests,
+          members: members,
+          associated_organisations: associated_organisations,
+          tags: tags,
+          abbreviations: abbreviations
+      }.to_json(*a)
+    end
+  end
+end

data/lib/lobbyliste/person.rb

@@ -0,0 +1,27 @@
+module Lobbyliste
+  # Class to encapsulate a person.
+  class Person
+
+    # @return [String] the persons name (hopefully) stripped of all titles
+    attr_reader :name
+
+    # @return [Array] list of all titles (job, academic, positions)
+    attr_reader :titles
+
+    def initialize(name, titles)
+      @name = name
+      @titles = titles
+    end
+
+    def ==(other)
+      name==other.name && titles==other.titles
+    end
+
+    def to_json(*a)
+      {
+          name: name,
+          titles: titles
+      }.to_json(*a)
+    end
+  end
+end

data/lib/lobbyliste/version.rb

@@ -0,0 +1,3 @@
+module Lobbyliste
+  VERSION = "0.1.0"
+end

data/lib/lobbyliste.rb

@@ -0,0 +1,19 @@
+require "lobbyliste/version"
+require "lobbyliste/factories"
+require "lobbyliste/list"
+require "lobbyliste/organisation"
+require "lobbyliste/address"
+require "lobbyliste/person"
+require "lobbyliste/downloader"
+require "lobbyliste/core_ext/string"
+require 'json'
+
+module Lobbyliste
+
+  # Download the PDF and parse it
+  # @return [Lobbyliste::Liste]
+  def self.fetch_and_parse
+    downloader = Lobbyliste::Downloader.new
+    Lobbyliste::Factories::ListFactory.build(downloader.text_data,downloader.html_data)
+  end
+end

data/lobbyliste.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'lobbyliste/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "lobbyliste"
+  spec.version       = Lobbyliste::VERSION
+  spec.authors       = ["DarthMax"]
+  spec.email         = ["max@kopfueber.org"]
+
+  spec.summary       = %q{Ruby crawler for the list of lobbyists published by German Bundestag}
+  spec.description   = %q{This gem crawls and parses the the list of lobbyists which is published as a PDF by the German Bundestag. }
+  spec.homepage      = "https://github.com/DarthMax/lobbyliste"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = ["lobbyliste"]
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'nokogiri'
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "mocha"
+  spec.add_development_dependency "vcr"
+  spec.add_development_dependency "webmock"
+end

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification
+name: lobbyliste
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- DarthMax
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-06-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'This gem crawls and parses the the list of lobbyists which is published
+  as a PDF by the German Bundestag. '
+email:
+- max@kopfueber.org
+executables:
+- lobbyliste
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/lobbyliste
+- bin/setup
+- ext/pdfbox.jar
+- lib/lobbyliste.rb
+- lib/lobbyliste/address.rb
+- lib/lobbyliste/core_ext/string.rb
+- lib/lobbyliste/downloader.rb
+- lib/lobbyliste/factories.rb
+- lib/lobbyliste/factories/address_factory.rb
+- lib/lobbyliste/factories/list_factory.rb
+- lib/lobbyliste/factories/organisation_factory.rb
+- lib/lobbyliste/factories/person_factory.rb
+- lib/lobbyliste/list.rb
+- lib/lobbyliste/organisation.rb
+- lib/lobbyliste/person.rb
+- lib/lobbyliste/version.rb
+- lobbyliste.gemspec
+homepage: https://github.com/DarthMax/lobbyliste
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Ruby crawler for the list of lobbyists published by German Bundestag
+test_files: []
+has_rdoc: