hdo-storting-importer 0.0.1

data/.gitignore

@@ -0,0 +1,16 @@
+Gemfile.lock
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+InstalledFiles
+_yardoc
+coverage
+doc/
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "folketingparser"]
+	path = folketingparser
+	url = git://gitorious.org/nuug/folketingparser.git

data/.travis.yml

@@ -0,0 +1,7 @@
+rvm:
+  - 1.9.3
+notifications:
+  recipients:
+    - jari.bakken@gmail.com
+  irc: "irc.freenode.net#holderdeord"
+script: "bundle exec rake"

data/Gemfile

@@ -0,0 +1,8 @@
+source :rubygems
+
+gem "builder"
+gem "nokogiri"
+gem "socksify"
+gem "aruba"
+gem "rake"
+gem "rest-client"

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Holder De Ord
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,14 @@
+What
+====
+
+Convert and import XML from data.stortinget.no
+
+Usage
+=====
+
+    $ bin/hdo-converter --app-root /src/hdo-site all
+
+Caveats
+=======
+
+Right now we're executing script/import from hdo-site to perform the import. That means running this with bundler may fail if the gems don't match. Don't do that.

data/Rakefile

@@ -0,0 +1,11 @@
+require 'rake'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new
+
+require 'cucumber/rake/task'
+Cucumber::Rake::Task.new
+
+task :default => [:spec, :cucumber]
+
+require "bundler/gem_tasks"

data/bin/hdo-converter

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+require 'hdo/storting_importer'
+require 'optparse'
+
+Hdo::StortingImporter::CLI.new(ARGV).execute

data/features/import.feature

@@ -0,0 +1,85 @@
+Feature: Import data
+    Scenario: Import districts
+      When I run `hdo-converter --only-print districts`
+      Then the stdout should contain:
+      """
+      <?xml version="1.0" encoding="UTF-8"?>
+      <districts>
+        <district>
+          <externalId>Ak</externalId>
+          <name>Akershus</name>
+        </district>
+        <district>
+          <externalId>AA</externalId>
+          <name>Aust-Agder</name>
+        </district>
+        <district>
+          <externalId>Bu</externalId>
+          <name>Buskerud</name>
+        </district>
+        <district>
+          <externalId>Fi</externalId>
+          <name>Finnmark</name>
+        </district>
+        <district>
+          <externalId>He</externalId>
+          <name>Hedmark</name>
+        </district>
+        <district>
+          <externalId>Ho</externalId>
+          <name>Hordaland</name>
+        </district>
+        <district>
+          <externalId>MR</externalId>
+          <name>Møre og Romsdal</name>
+        </district>
+        <district>
+          <externalId>NT</externalId>
+          <name>Nord-Trøndelag</name>
+        </district>
+        <district>
+          <externalId>No</externalId>
+          <name>Nordland</name>
+        </district>
+        <district>
+          <externalId>Op</externalId>
+          <name>Oppland</name>
+        </district>
+        <district>
+          <externalId>Os</externalId>
+          <name>Oslo</name>
+        </district>
+        <district>
+          <externalId>Ro</externalId>
+          <name>Rogaland</name>
+        </district>
+        <district>
+          <externalId>SF</externalId>
+          <name>Sogn og Fjordane</name>
+        </district>
+        <district>
+          <externalId>ST</externalId>
+          <name>Sør-Trøndelag</name>
+        </district>
+        <district>
+          <externalId>Te</externalId>
+          <name>Telemark</name>
+        </district>
+        <district>
+          <externalId>Tr</externalId>
+          <name>Troms</name>
+        </district>
+        <district>
+          <externalId>VA</externalId>
+          <name>Vest-Agder</name>
+        </district>
+        <district>
+          <externalId>Ve</externalId>
+          <name>Vestfold</name>
+        </district>
+        <district>
+          <externalId>Øs</externalId>
+          <name>Østfold</name>
+        </district>
+      </districts>
+      """

data/features/support/env.rb

@@ -0,0 +1,7 @@
+require 'aruba'
+require 'aruba/cucumber'
+
+Before do
+  @aruba_timeout_seconds = 10
+  ENV['VOTE_COUNT'] = '5'
+end

data/hdo-storting-importer.gemspec

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/hdo/storting_importer/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jari Bakken"]
+  gem.email         = ["jari.bakken@gmail.com"]
+  gem.description   = %q{Gem to process data from data.stortinget.no}
+  gem.summary       = %q{Gem to process data from data.stortinget.no}
+  gem.homepage      = "http://github.com/holderdeord/hdo-storting-importer"
+
+  gem.files         = `git ls-files`.split($\).reject { |e| e =~ /^(data|folketingparser)/ }
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "hdo-storting-importer"
+  gem.require_paths = ["lib"]
+  gem.version       = Hdo::StortingImporter::VERSION
+end

data/lib/hdo/storting_importer/api_data_source.rb

@@ -0,0 +1,62 @@
+module Hdo
+  module StortingImporter
+    class ApiDataSource < DataSource
+
+      def initialize(url)
+        @log = Logger.new(STDERR)
+        @resource = RestClient::Resource.new(URI.parse(url))
+      end
+
+      def representatives(period = DEFAULT_PERIOD)
+        fetch "eksport/representanter/?StortingsPeriodeId=#{period}"
+      end
+
+      def representatives_today
+        fetch 'eksport/dagensrepresentanter/'
+      end
+
+      def parties(session_id = DEFAULT_SESSION)
+        fetch "eksport/partier/?sesjonid=#{session_id}"
+      end
+
+      def committees(session_id = DEFAULT_SESSION)
+        fetch "eksport/komiteer/?SesjonId=#{session_id}"
+      end
+
+      def districts
+        fetch "eksport/fylker"
+      end
+
+      def categories
+        fetch "eksport/emner"
+      end
+
+      def issues(session_id = DEFAULT_SESSION)
+        fetch "eksport/saker?sesjonid=#{session_id}"
+      end
+
+      def votes_for(issue_id)
+        fetch "eksport/voteringer?sakid=#{issue_id}"
+      end
+
+      def propositions_for(vote_id)
+        fetch "eksport/voteringsforslag/?voteringid=#{vote_id}"
+      end
+
+      def vote_results_for(vote_id)
+        fetch "eksport/voteringsresultat/?voteringid=#{vote_id}"
+      end
+
+      private
+
+      def fetch(path)
+        sub_resource = @resource[path]
+        @log.info "parsing #{sub_resource}"
+
+        parse sub_resource.get
+      end
+
+    end
+  end
+end
+

data/lib/hdo/storting_importer/category.rb

@@ -0,0 +1,55 @@
+module Hdo
+  module StortingImporter
+    class Category
+      include IvarEquality
+
+      attr_reader :external_id, :name
+      attr_accessor :children
+
+      def self.from_storting_doc(doc)
+        doc.css("emne_liste > emne").map { |xt| from_storting_node(xt) }
+      end
+
+      def self.from_storting_node(node)
+        cat = Category.new node.css("id").first.text, node.css("navn").first.text
+
+        subnodes = node.css("underemne_liste > emne")
+        subnodes.map do |subnode|
+          cat.children << Category.new(subnode.css("id").first.text, subnode.css("navn").first.text)
+        end
+
+        cat
+      end
+
+      def self.from_hdo_node(node)
+        external_id = node.css("externalId").first.text
+        name        = node.css("name").first.text
+
+        cat = new external_id, name
+        cat.children = node.css("subcategories category").map { |e| from_hdo_node(e) }
+
+        cat
+      end
+
+      def initialize(external_id, name)
+        @external_id = external_id
+        @name        = name
+        @children    = []
+      end
+
+      def to_hdo_xml(builder = Util.builder)
+        builder.category do |cat|
+          cat.externalId external_id
+          cat.name name
+
+          if children.any?
+            cat.subcategories do |sub|
+              children.each { |child| child.to_hdo_xml(sub) }
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/cli.rb

@@ -0,0 +1,141 @@
+module Hdo
+  module StortingImporter
+    class CLI
+      def initialize(args)
+        @log = Logger.new(STDERR)
+        @cmd, @options = parse(args)
+      end
+
+      def execute
+        import @cmd.to_sym
+      end
+
+      private
+
+      def parse(args)
+        options = {:source => (has_submodule? ? 'disk' : 'api')}
+
+        parser = OptionParser.new do |opt|
+          opt.banner = "Usage: #{$0} <import-type> [options]"
+
+          opt.on("--help", "You're looking at it.") { puts opt; exit; }
+          opt.on("--only-print", "Don't run import, only print generated XML.") { options[:only_print] = true }
+          opt.on("--except WHAT", 'Ignore this comma separated list of entities from import.') { |s| options[:ignore] = s.split(",").map(&:strip).map(&:to_sym) }
+          opt.on("--app-root APP_ROOT", 'Path to clone of git://github.com/holderdeord/hdo-site.git') { |path| options[:app_root] = path }
+          opt.on("--app-url APP_URL", 'URL to hdo-site') { |url| options[:app_url] = url }
+          opt.on("--source SOURCE ", 'Where to take data from [disk|api]') { |source| options[:source] = source }
+          opt.on("--socks PROXY", 'host:port for SOCKS proxy') do |proxy|
+            require 'socksify'
+            host, port = proxy.split ":"
+            TCPSocket.socks_server = host
+            TCPSocket.socks_port = Integer(port)
+          end
+        end
+
+        parser.parse!(args)
+
+        cmd = args.shift
+
+        unless cmd
+          puts(parser)
+          exit 1
+        end
+
+        [cmd, options]
+      end
+
+      def data_source
+        @data_source ||= (
+          ds = if @options[:source] == "disk"
+                 DiskDataSource.new(File.join(StortingImporter.root, 'folketingparser/rawdata/data.stortinget.no'))
+               elsif @options[:source] == "api"
+                 ApiDataSource.new("http://data.stortinget.no/")
+               else
+                 raise ArgumentError, "invalid source: #{@options[:source].inspect}"
+               end
+
+          ParsingDataSource.new(ds)
+        )
+      end
+
+      def importer
+        @importer ||= (
+          if @options[:app_root]
+            ScriptImporter.new(@options[:app_root])
+          else
+            raise ArgumentError, "app-root not given, can't import"
+          end
+        )
+      end
+
+      def import(what)
+        case what
+        when :dld
+          import_dld
+        when :promises
+          import_promises
+        when :all
+          import_all
+        else
+          import_docs converter.xml_for(what.to_sym)
+        end
+      end
+
+      def import_all
+        ignore = Array(@options[:ignore])
+
+        import_docs converter.xml_for(:parties) unless ignore.include?(:parties)
+        import_docs converter.xml_for(:committees) unless ignore.include?(:committees)
+        import_docs converter.xml_for(:districts) unless ignore.include?(:districts)
+        import_docs converter.xml_for(:representatives) unless ignore.include?(:representatives)
+        import_docs converter.xml_for(:categories) unless ignore.include?(:categories)
+        import_docs converter.xml_for(:issues) unless ignore.include?(:issues)
+        import_docs converter.xml_for(:votes) unless ignore.include?(:votes)
+
+        import_dld unless ignore.include?(:dld)
+        import_promises unless ignore.include?(:promises)
+      end
+
+      def converter
+        @converter ||= Converter.new(data_source)
+      end
+
+      def import_dld
+        if has_submodule?
+          print_or_import File.read(File.join(StortingImporter.root, 'data/dld-issues.xml'))
+          print_or_import File.read(File.join(StortingImporter.root, 'folketingparser/data/votering-2011-04-04-dld-hdo.xml'))
+        else
+          $stderr.puts "folketingparser not found, skipping DLD votes and issues (run `git submodule update --init` if you need this)"
+        end
+      end
+
+      def import_promises
+        csvs = Dir[File.join(StortingImporter.root, 'data/promises-*.csv')].sort_by { |e| File.basename(e) }
+        csvs.each do |path|
+          print_or_import PromiseConverter.new(path).xml
+        end
+      end
+
+      def import_docs(docs)
+        docs = [docs] unless docs.kind_of?(Array)
+
+        docs.each do |doc|
+          print_or_import doc.to_s
+        end
+      end
+
+      def print_or_import(xml)
+        if @options[:only_print]
+          puts xml
+        else
+          importer.import xml
+        end
+      end
+
+      def has_submodule?
+        File.exist?(File.join(StortingImporter.root, 'folketingparser/data'))
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/committee.rb

@@ -0,0 +1,35 @@
+module Hdo
+  module StortingImporter
+    class Committee
+      include IvarEquality
+
+      attr_reader :external_id, :name
+
+      def self.from_storting_doc(doc)
+        doc.css("komiteer_liste komite").map do |node|
+          from_storting_node(node)
+        end
+      end
+
+      def self.from_storting_node(node)
+        new node.css("id").first.text, node.css("navn").first.text
+      end
+
+      def self.from_hdo_node(node)
+        new node.css("externalId").first.text, node.css("name").first.text
+      end
+
+      def initialize(external_id, name)
+        @external_id = external_id
+        @name = name
+      end
+
+      def to_hdo_xml(builder = Util.builder)
+        builder.committee do |com|
+          com.externalId external_id
+          com.name name
+        end
+      end
+    end
+  end
+end

data/lib/hdo/storting_importer/converter.rb

@@ -0,0 +1,45 @@
+module Hdo
+  module StortingImporter
+    class Converter
+
+      def initialize(data_source)
+        @data_source = data_source
+        @cache = {}
+      end
+
+      def xml_for(name)
+        Util.builder do |xml|
+          xml.instruct!
+          xml.__send__(name) do |builder|
+            data_for(name).each do |obj|
+              obj.to_hdo_xml(builder)
+            end
+          end
+        end
+      end
+
+      private
+
+      def data_for(name)
+        case name
+        when :votes
+          # not ideal
+          issue_ids = fetch(:issues).map { |e| e.external_id }
+          issue_ids.map { |id| @data_source.votes_for(id) }.flatten.uniq_by { |e| e.external_id }
+        when :districts
+          fetch(name).sort_by { |e| e.name }
+        when :representatives
+          data = fetch(:representatives_today) + fetch(:representatives)
+          data.uniq_by { |e| e.external_id }.sort_by { |e| e.external_id }
+        else
+          fetch(name)
+        end
+      end
+
+      def fetch(name)
+        @cache[name] ||= @data_source.__send__(name)
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/core_ext/enumerable.rb

@@ -0,0 +1,12 @@
+module Enumerable
+  def uniq_by(&blk)
+    res = {}
+    
+    each do |e|
+      res[yield(e)] ||= e
+    end
+    
+    res.values
+  end
+end
+

data/lib/hdo/storting_importer/data_source.rb

@@ -0,0 +1,22 @@
+module Hdo
+  module StortingImporter
+    class DataSource
+
+      class NotFoundError < StandardError
+      end
+
+      DEFAULT_SESSION = '2011-2012'
+      DEFAULT_PERIOD = '2009-2013'
+
+      private
+
+      def parse(str)
+        doc = Nokogiri.XML(str)
+        doc.remove_namespaces!
+
+        doc.first_element_child
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/disk_data_source.rb

@@ -0,0 +1,61 @@
+module Hdo
+  module StortingImporter
+    class DiskDataSource < DataSource
+
+      def initialize(root)
+        @root = Pathname.new(root)
+        @log = Logger.new(STDERR)
+      end
+
+      def parties(session_id = DEFAULT_SESSION)
+        fetch "eksport/partier/index.html?sesjonid=#{session_id}"
+      end
+
+      def committees(session_id = DEFAULT_SESSION)
+        fetch "eksport/komiteer/index.html?SesjonId=#{session_id}"
+      end
+
+      def districts
+        fetch "eksport/fylker/index.html"
+      end
+
+      def categories
+        fetch "eksport/emner/index.html"
+      end
+
+      def issues(session_id = DEFAULT_SESSION)
+        fetch "eksport/saker/index.html?sesjonid=#{session_id}"
+      end
+
+      def representatives(period = DEFAULT_PERIOD)
+        fetch "eksport/representanter/index.html?StortingsPeriodeId=#{period}"
+      end
+
+      def representatives_today
+        fetch 'eksport/dagensrepresentanter/index.html'
+      end
+
+      def votes_for(issue_id)
+        fetch "eksport/voteringer/index.html?sakid=#{issue_id}"
+      end
+
+      def propositions_for(vote_id)
+        fetch "eksport/voteringsforslag/index.html?voteringid=#{vote_id}"
+      end
+
+      def vote_results_for(vote_id)
+        fetch "eksport/voteringsresultat/index.html?voteringid=#{vote_id}"
+      end
+
+      private
+
+      def fetch(path)
+        subpath = @root.join(path)
+        @log.info "parsing #{subpath}"
+
+        parse subpath.read
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/district.rb

@@ -0,0 +1,36 @@
+module Hdo
+  module StortingImporter
+    class District
+      include IvarEquality
+
+      attr_reader :external_id, :name
+
+      def self.from_storting_doc(doc)
+        doc.css("fylker_liste fylke").map do |node|
+          from_storting_node(node)
+        end
+      end
+
+      def self.from_storting_node(node)
+        new node.css("id").first.text, node.css("navn").first.text
+      end
+
+      def self.from_hdo_node(node)
+        new node.css("externalId").first.text, node.css("name").first.text
+      end
+
+      def initialize(external_id, name)
+        @external_id = external_id
+        @name = name
+      end
+
+      def to_hdo_xml(builder = Util.builder)
+        builder.district do |d|
+          d.externalId external_id
+          d.name name
+        end
+      end
+
+    end
+  end
+end