hdo-storting-importer 0.0.1

data/lib/hdo/storting_importer/issue.rb

@@ -0,0 +1,90 @@
+module Hdo
+  module StortingImporter
+    class Issue
+      include IvarEquality
+
+      attr_reader :external_id, :summary, :description, :type, :status, :last_update,
+                  :reference, :document_group, :committee, :categories
+
+      def self.from_storting_doc(doc)
+        doc.css("saker_liste sak").map do |node|
+          from_storting_node(node)
+        end
+      end
+
+      def self.from_storting_node(node)
+        external_id    = node.xpath("./id").first.text
+        summary        = node.css("korttittel").first.text
+        description    = Util.remove_newlines(node.css("tittel").first.text)
+        type           = node.css("type").first.text
+        status         = node.css("status").first.text
+        last_update    = node.css("sist_oppdatert_dato").first.text
+        reference      = node.css("henvisning").first.text
+        document_group = node.css("dokumentgruppe").first.text
+
+        committee_node = node.css("komite").first
+        if committee_node && committee_node['nil'] != "true"
+          committee = committee_node.css("navn").first.text
+        end
+
+        xcategories = node.css("emne")
+        if xcategories.any?
+          categories = xcategories.map { |xt| xt.css("navn").first.text }
+        end
+
+        new(external_id, summary, description, type, status, last_update, reference, document_group, committee, categories)
+      end
+
+      def self.from_hdo_node(node)
+        external_id    = node.css("externalId").first.text
+        summary        = node.css("summary").first.text
+        description    = node.css("description").first.text
+        type           = node.css("type").first.text
+        status         = node.css("status").first.text
+        last_update    = node.css("lastUpdate").first.text
+        reference      = node.css("reference").first.text
+        document_group = node.css("documentGroup").first.text
+        committee      = node.css("committee").first.text
+        categories     = node.css("categories category").map { |e| e.text }
+
+        new external_id, summary, description, type, status, last_update, reference, document_group, committee, categories
+      end
+
+      def initialize(external_id, summary, description, type, status, last_update,
+                     reference, document_group, committee, categories)
+        @external_id    = external_id
+        @summary        = summary
+        @description    = description
+        @type           = type
+        @status         = status
+        @last_update    = last_update
+        @reference      = reference
+        @document_group = document_group
+        @committee      = committee
+        @categories     = categories || []
+      end
+
+      def to_hdo_xml(builder = Util.builder)
+        builder.issue do |i|
+          i.externalId external_id
+          i.summary summary
+          i.description description
+          i.type type
+          i.status status
+          i.lastUpdate last_update
+          i.reference reference
+          i.documentGroup document_group
+          i.committee(committee) if committee
+
+          if categories.any?
+            i.categories do |cats|
+              categories.each { |e| cats.category e }
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/ivar_equality.rb

@@ -0,0 +1,22 @@
+module Hdo
+  module StortingImporter
+    module IvarEquality
+
+      def ==(other)
+        other.kind_of?(self.class) && __ivars__ == other.__ivars__
+      end
+      alias_method :eql?, :==
+
+      def hash
+        __ivars__.hash ^ self.class.hash
+      end
+
+      def __ivars__
+        res = {}
+        instance_variables.each { |e| res[e] = instance_variable_get(e) }
+
+        res
+      end
+    end
+  end
+end

data/lib/hdo/storting_importer/parsing_data_source.rb

@@ -0,0 +1,59 @@
+module Hdo
+  module StortingImporter
+    class ParsingDataSource < DataSource
+      def initialize(delegate_data_source)
+        @data_source = delegate_data_source
+      end
+
+      def representatives(period = DEFAULT_PERIOD)
+        Representative.from_storting_doc @data_source.representatives(period)
+      end
+
+      def representatives_today
+        Representative.from_storting_doc @data_source.representatives_today
+      end
+
+      def parties(session_id = DEFAULT_SESSION)
+        Party.from_storting_doc @data_source.parties(session_id)
+      end
+
+      def committees(session_id = DEFAULT_SESSION)
+        Committee.from_storting_doc @data_source.committees(session_id)
+      end
+
+      def districts
+        District.from_storting_doc @data_source.districts
+      end
+
+      def categories
+        Category.from_storting_doc @data_source.categories
+      end
+
+      def issues(session_id = DEFAULT_SESSION)
+        Issue.from_storting_doc @data_source.issues(session_id)
+      end
+
+      def votes_for(issue_id)
+        votes = Vote.from_storting_doc @data_source.votes_for(issue_id)
+
+        votes.each do |vote|
+          vote.add_storting_propositions @data_source.propositions_for(vote.external_id)
+          if vote.personal?
+            vote.add_storting_results @data_source.vote_results_for(vote.external_id)
+          end
+        end
+
+        votes
+      end
+
+      def propositions_for(vote_id)
+        raise NotImplementedError, 'result of #votes_for includes propositions'
+      end
+
+      def vote_results_for(vote_id)
+        raise NotImplementedError, 'result of #votes_for includes vote results'
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/party.rb

@@ -0,0 +1,31 @@
+module Hdo
+  module StortingImporter
+    class Party
+      include IvarEquality
+
+      attr_reader :external_id, :name
+
+      def self.from_storting_doc(doc)
+        doc.css("partier_liste parti").map do |node|
+          new node.css("id").first.text, node.css("navn").first.text
+        end
+      end
+
+      def self.from_hdo_node(node)
+        new node.css("externalId").first.text, node.css("name").first.text
+      end
+
+      def initialize(external_id, name)
+        @external_id = external_id
+        @name        = name
+      end
+
+      def to_hdo_xml(builder = Util.builder)
+        builder.party do |party|
+          party.externalId external_id
+          party.name name
+        end
+      end
+    end
+  end
+end

data/lib/hdo/storting_importer/promise.rb

@@ -0,0 +1,49 @@
+require 'csv'
+
+module Hdo
+  module StortingImporter
+    class Promise
+      attr_reader :party, :body, :general, :categories, :source, :page
+      alias_method :general?, :general
+
+      def self.from_csv(str)
+        # cleanup
+        str.gsub!(/\bFrp\b/, "FrP")
+        str.gsub!(/\bKrf\b/, "KrF")
+        str.gsub!(/\bSP\b/, "Sp")
+
+        rows = CSV.parse(
+          str,
+          headers: [:party, :body, :general, :categories, :source, :page],
+          col_sep: ";",
+          skip_blanks: true,
+          return_headers: false,
+        )
+
+        rows.map do |e|
+          pr = e.to_hash
+          next if pr[:party].to_s.strip == "Parti" # header row
+
+          new(
+            pr[:party].to_s.strip,
+            pr[:body].to_s.strip,
+            pr[:general].to_s.downcase == 'ja',
+            pr[:categories].split(",").map(&:upcase).map(&:strip),
+            pr[:source].to_s.strip,
+            pr[:page].to_s.strip
+            )
+        end.compact
+      end
+
+      def initialize(party, body, general, categories, source, page)
+        @party      = party
+        @body       = body
+        @general    = general
+        @categories = categories
+        @source     = source
+        @page       = page
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/promise_converter.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+
+require 'csv'
+require 'builder'
+
+module Hdo
+  module StortingImporter
+    class PromiseConverter
+      def initialize(exported_spreadsheet)
+        content = File.read(File.expand_path(exported_spreadsheet), encoding: "ISO-8859-1").encode("UTF-8")
+        @promises = Promise.from_csv(content)
+      end
+
+      def xml
+        builder = Builder::XmlMarkup.new :indent => 2
+        builder.instruct!
+
+        builder.promises do |promises|
+          @promises.each do |data|
+            next if data.body == "Løftetekst" || data.body.nil? || data.body.empty?
+            add_promise(promises, data)
+          end
+        end
+
+        builder.target!
+      end
+
+      def add_promise(promises, data)
+        promises.promise do |promise|
+          promise.party data.party.strip
+          promise.general data.general
+          promise.categories do |categories|
+            data.categories.each do |name|
+              categories.category name
+            end
+          end
+
+          promise.source [data.source, data.page].join(":")
+          promise.body data.body.strip
+        end
+      rescue
+        STDERR.puts data.inspect
+        raise
+      end
+    end
+
+  end
+end

data/lib/hdo/storting_importer/representative.rb

@@ -0,0 +1,106 @@
+module Hdo
+  module StortingImporter
+    class Representative
+      include IvarEquality
+
+      attr_reader :external_id, :first_name, :last_name, :date_of_birth, :date_of_death,
+                  :district, :party, :committees, :period, :gender
+
+      attr_accessor :vote_result
+
+      def self.from_storting_doc(doc)
+        nodes = doc.css("dagensrepresentant")
+        nodes += doc.css("representant")
+
+        nodes.map { |e| from_storting_node(e) }
+      end
+
+      def self.from_storting_node(node)
+        district_node = node.css("fylke navn").first
+        district      = district_node ? district_node.text : ''
+
+        party_node = node.css("parti navn").first
+        party      = party_node ? party_node.text : ''
+
+        committees = node.css("komite").map { |c| c.css("navn").text.strip }
+        period     = '2011-2012' # FIXME
+
+        new(
+          node.css("id").first.text,
+          node.css("fornavn").first.text,
+          node.css("etternavn").first.text,
+          node.css("kjoenn").first.text == "mann" ? 'M' : 'F',
+          node.css("foedselsdato").first.text,
+          node.css("doedsdato").first.text,
+          district,
+          party,
+          committees,
+          period
+        )
+      end
+
+      def self.from_hdo_node(node)
+        district_node = node.css("district").first
+        district      = district_node ? district_node.text : ''
+
+        party_node = node.css("party").first
+        party      = party_node ? party_node.text : ''
+
+        rep = new node.css("externalId").first.text,
+                  node.css("firstName").first.text,
+                  node.css("lastName").first.text,
+                  node.css("gender").first.text,
+                  node.css("dateOfBirth").first.text,
+                  node.css("dateOfDeath").first.text,
+                  district,
+                  party,
+                  node.css("committees committee").map { |e| e.text.strip },
+                  node.css("period").first.text
+
+        result_node = node.css("voteResult").first
+        rep.vote_result = result_node.text if result_node
+
+        rep
+      end
+
+      def initialize(external_id, first_name, last_name, gender, date_of_birth, date_of_death, district, party, committees, period)
+        @external_id   = external_id
+        @first_name    = first_name
+        @last_name     = last_name
+        @gender        = gender
+        @date_of_birth = date_of_birth
+        @date_of_death = date_of_death
+        @district      = district
+        @party         = party
+        @committees    = committees
+        @period        = period
+
+        @vote_result   = nil
+      end
+
+      def to_hdo_xml(builder = Util.builder)
+        builder.representative do |rep|
+          rep.externalId external_id
+          rep.firstName first_name
+          rep.lastName last_name
+          rep.gender gender
+          rep.dateOfBirth date_of_birth
+          rep.dateOfDeath date_of_death
+          rep.district district
+          rep.party party
+
+          rep.committees do |coms|
+            committees.each { |e| coms.committee e }
+          end
+
+          rep.period period
+
+          if vote_result
+            rep.voteResult vote_result
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/hdo/storting_importer/script_importer.rb

@@ -0,0 +1,20 @@
+module Hdo
+  module StortingImporter
+    class ScriptImporter
+      
+      def initialize(app_root)
+        @app_root = app_root 
+      end
+      
+      def import(xml)
+        Tempfile.open("storting2hdo") do |f|
+          f << xml
+          f.close
+
+          Dir.chdir(@app_root) { system "script/import", f.path }
+        end
+      end
+      
+    end
+  end
+end

data/lib/hdo/storting_importer/util.rb

@@ -0,0 +1,26 @@
+module Hdo
+  module StortingImporter
+    module Util
+      module_function
+
+      def builder
+        xml = Builder::XmlMarkup.new :indent => 2
+
+        if block_given?
+          yield xml
+          return xml.target!
+        end
+
+        xml
+      end
+
+      def remove_newlines(str)
+        str.gsub(/\r?\n/, '')
+      end
+
+      def remove_invalid_html(str)
+        str.gsub("<\\p>", "")
+      end
+    end
+  end
+end

data/lib/hdo/storting_importer/version.rb

@@ -0,0 +1,5 @@
+module Hdo
+  module StortingImporter
+    VERSION = "0.0.1"
+  end
+end

data/lib/hdo/storting_importer/vote.rb

@@ -0,0 +1,171 @@
+module Hdo
+  module StortingImporter
+    class Vote
+      include IvarEquality
+
+      attr_reader :external_id, :external_issue_id, :personal, :enacted, :subject,
+                  :method, :result_type, :time, :counts
+      attr_accessor :propositions, :representatives
+
+      alias_method :personal?, :personal
+      alias_method :enacted?, :enacted
+
+      def self.from_storting_doc(doc)
+        issue_id = doc.css("sak_id").first.text
+
+        doc.css("sak_votering").map do |vote_node|
+          vote_id     = vote_node.css("votering_id").text
+          personal    = vote_node.css("personlig_votering").text == "true"
+          enacted     = vote_node.css("vedtatt").text == "true"
+          subject     = vote_node.css("votering_tema").text
+          method      = vote_node.css("votering_metode").text
+          result_type = vote_node.css("votering_resultat_type").text
+          time        = vote_node.css("votering_tid").text
+
+          forc     = Integer(vote_node.css("antall_for").text)
+          againstc = Integer(vote_node.css("antall_mot").text)
+          absentc  = Integer(vote_node.css("antall_ikke_tilstede").text)
+
+          # settes til -1 ved personlig_votering=false
+          forc = 0 if forc < 0
+          againstc = 0 if againstc < 0
+          absentc = 0 if absentc < 0
+
+          new vote_id, issue_id, personal, enacted, subject, method, result_type, time, forc, againstc, absentc
+        end
+      end
+
+      def self.from_hdo_node(node)
+        external_id       = node.css("externalId").first.text
+        external_issue_id = node.css("externalIssueId").first.text
+        for_count         = node.css("counts for").first.text
+        against_count     = node.css("counts against").first.text
+        absent_count      = node.css("counts absent").first.text
+        personal          = node.css("personal").first.text == 'true'
+        enacted           = node.css("enacted").first.text == 'true'
+        subject           = node.css("subject").first.text
+        method            = node.css("method").first.text
+        result_type       = node.css("resultType").first.text
+        time              = node.css("time").first.text
+
+        vote = new external_id, external_issue_id, personal, enacted, subject, method, result_type, time, for_count, against_count, absent_count
+        vote.propositions = node.css("propositions proposition").map { |e| Proposition.from_hdo_node(e) }
+        vote.representatives = node.css("representatives representative").map { |e| Representative.from_hdo_node(e) }
+
+        vote
+      end
+
+      def initialize(external_id, external_issue_id, personal, enacted, subject, method, result_type, time, for_count, against_count, absent_count)
+        @external_id       = external_id
+        @external_issue_id = external_issue_id
+        @personal          = personal
+        @enacted           = enacted
+        @subject           = subject
+        @method            = method
+        @result_type       = result_type
+        @time              = time
+        @counts            = Counts.new(Integer(for_count), Integer(against_count), Integer(absent_count))
+
+        @propositions    = []
+        @representatives = []
+      end
+
+      def add_storting_propositions(node)
+        @propositions += node.css("voteringsforslag").map do |n|
+          rep_node = n.css("forslag_levert_av_representant").first
+          if rep_node && rep_node['nil'] != 'true'
+            delivered_by = Representative.from_storting_node(rep_node)
+          else
+            delivered_by = nil
+          end
+
+          Proposition.new n.css("forslag_id").first.text,
+                          n.css("forslag_betegnelse").first.text,
+                          n.css("forslag_paa_vegne_av_tekst").first.text,
+                          Util.remove_invalid_html(n.css("forslag_tekst").first.text),
+                          delivered_by
+
+        end
+      end
+
+      def add_storting_results(node)
+        @representatives += node.css("representant_voteringsresultat").map do |n|
+          rep = Representative.from_storting_node(n.css("representant").first)
+          rep.vote_result = case n.css("votering").text
+                            when 'for'
+                              'for'
+                            when 'mot'
+                             'against'
+                            when 'ikke_tilstede'
+                             'absent'
+                            else
+                              raise "unexpected vote: #{vote_result.inspect}"
+                            end
+
+          rep
+        end
+      end
+
+      def to_hdo_xml(builder = Util.builder)
+        builder.vote do |vote|
+          vote.externalId external_id
+          vote.externalIssueId external_issue_id
+          vote.counts do |c|
+            c.for counts.for
+            c.against counts.against
+            c.absent counts.absent
+          end
+          vote.personal personal?
+          vote.enacted enacted?
+          vote.subject subject
+          vote.method method
+          vote.resultType result_type
+          vote.time time
+
+          vote.representatives do |reps|
+            representatives.each do |rep|
+              rep.to_hdo_xml(reps)
+            end
+          end
+
+          vote.propositions do |props|
+            propositions.each do |prop|
+              prop.to_hdo_xml(props)
+            end
+          end
+        end
+      end
+
+      class Counts < Struct.new(:for, :against, :absent)
+      end
+
+      class Proposition < Struct.new(:external_id, :description, :on_behalf_of, :body, :delivered_by)
+        def self.from_hdo_node(node)
+          external_id  = node.css("externalId").first.text
+          description  = node.css("description").first.text
+          on_behalf_of = node.css("onBehalfOf").first.text
+          body         = node.css("body").first.text
+
+          delivered_by_node = node.css("deliveredBy representative").first
+          delivered_by = Representative.from_hdo_node(delivered_by_node) if delivered_by_node
+
+          new external_id, description, on_behalf_of, body, delivered_by
+        end
+
+        def to_hdo_xml(builder)
+          builder.proposition do |pr|
+            pr.externalId external_id
+            pr.description description
+            pr.onBehalfOf on_behalf_of
+            pr.body body
+
+            pr.deliveredBy do |db|
+              delivered_by.to_hdo_xml(db) if delivered_by
+            end
+          end
+        end
+      end # Proposition
+
+    end # Vote
+  end
+end

data/lib/hdo/storting_importer.rb

@@ -0,0 +1,41 @@
+module Hdo
+  module StortingImporter
+    def self.root
+      @root ||= File.expand_path("../../..", __FILE__)
+    end
+  end
+end
+
+require 'nokogiri'
+require 'restclient'
+require 'tempfile'
+require 'pathname'
+require 'open-uri'
+require 'erb'
+require 'logger'
+
+require 'hdo/storting_importer/core_ext/enumerable'
+require 'hdo/storting_importer/ivar_equality'
+require 'hdo/storting_importer/util'
+
+require 'hdo/storting_importer/data_source'
+require 'hdo/storting_importer/disk_data_source'
+require 'hdo/storting_importer/api_data_source'
+require 'hdo/storting_importer/parsing_data_source'
+require 'hdo/storting_importer/script_importer'
+
+require 'hdo/storting_importer/category'
+require 'hdo/storting_importer/committee'
+require 'hdo/storting_importer/district'
+require 'hdo/storting_importer/issue'
+require 'hdo/storting_importer/party'
+require 'hdo/storting_importer/promise'
+require 'hdo/storting_importer/representative'
+require 'hdo/storting_importer/vote'
+
+require 'hdo/storting_importer/converter'
+require 'hdo/storting_importer/promise_converter'
+
+require 'hdo/storting_importer/cli'
+
+