hdo-storting-importer 0.0.1 → 0.0.2

data/lib/hdo/storting_importer/party.rb

@@ -5,12 +5,32 @@ module Hdo
 
       attr_reader :external_id, :name
 
+      def self.type_name
+        'party'
+      end
+
+      def self.description
+        'a political party'
+      end
+
+      def self.xml_example(builder = Util.builder)
+        new("DEM", "Democratic Party").to_hdo_xml(builder)
+      end
+
+      def self.fields
+        [EXTERNAL_ID_FIELD, Field.new(:name, true, :string, 'The name of the party.')]
+      end
+
       def self.from_storting_doc(doc)
         doc.css("partier_liste parti").map do |node|
           new node.css("id").first.text, node.css("navn").first.text
         end
       end
 
+      def self.from_hdo_doc(doc)
+        doc.css("parties > party").map { |e| from_hdo_node(e) }
+      end
+
       def self.from_hdo_node(node)
         new node.css("externalId").first.text, node.css("name").first.text
       end

data/lib/hdo/storting_importer/promise.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 require 'csv'
 
 module Hdo
@@ -6,6 +8,47 @@ module Hdo
       attr_reader :party, :body, :general, :categories, :source, :page
       alias_method :general?, :general
 
+      def self.type_name
+        'promise'
+      end
+
+      def self.description
+        'a party promise'
+      end
+
+      def self.fields
+        [
+          Field.new(:party, true, :string, 'The external id of the party.'),
+          Field.new(:general, true, :boolean, "Whether this is considered a general promise (i.e. can be ambigious whether it has been fulfilled)."),
+          Field.new(:categories, true, :list, "List of category names (matching names imported in <a href='#input-format-category'>&lt;category&gt;</a>)"),
+          Field.new(:source, true, :string, "The source of the promise. (TODO: this should always be a URL)"),
+          Field.new(:body, true, :string, "The body text of the promise."),
+        ]
+      end
+
+      def self.example
+        new("H", "Stille strengere krav til orden og oppførsel for å hindre at uro ødelegger undervisningen.", true, ["GRUNNSKOLE"], "PP", 8)
+      end
+
+      def self.xml_example(builder = Util.builder)
+        example.to_hdo_xml(builder)
+      end
+
+      def self.from_hdo_doc(doc)
+        doc.css("promises > promise").map { |e| from_hdo_node(e) }
+      end
+
+      def self.from_hdo_node(node)
+        source, page = node.css("source").first.text.split(":")
+
+        new node.css("party").first.text,
+            node.css("body").first.text,
+            node.css("general").first.text == "true",
+            node.css("categories > category").map { |e| e.text },
+            source,
+            page
+      end
+
       def self.from_csv(str)
         # cleanup
         str.gsub!(/\bFrp\b/, "FrP")
@@ -28,7 +71,7 @@ module Hdo
             pr[:party].to_s.strip,
             pr[:body].to_s.strip,
             pr[:general].to_s.downcase == 'ja',
-            pr[:categories].split(",").map(&:upcase).map(&:strip),
+            pr[:categories].to_s.split(",").map(&:upcase).map(&:strip),
             pr[:source].to_s.strip,
             pr[:page].to_s.strip
             )
@@ -44,6 +87,18 @@ module Hdo
         @page       = page
       end
 
+      def to_hdo_xml(builder = Util.builder)
+        builder.promise do |promise|
+          promise.party party
+          promise.general general?
+          promise.categories do |cats|
+            categories.each { |e| cats.category e }
+          end
+          promise.source [source, page].join(":")
+          promise.body body
+        end
+      end
+
     end
   end
 end

data/lib/hdo/storting_importer/promise_converter.rb

@@ -18,31 +18,12 @@ module Hdo
         builder.promises do |promises|
           @promises.each do |data|
             next if data.body == "Løftetekst" || data.body.nil? || data.body.empty?
-            add_promise(promises, data)
+            data.to_hdo_xml(promises)
           end
         end
 
         builder.target!
       end
-
-      def add_promise(promises, data)
-        promises.promise do |promise|
-          promise.party data.party.strip
-          promise.general data.general
-          promise.categories do |categories|
-            data.categories.each do |name|
-              categories.category name
-            end
-          end
-
-          promise.source [data.source, data.page].join(":")
-          promise.body data.body.strip
-        end
-      rescue
-        STDERR.puts data.inspect
-        raise
-      end
     end
-
   end
 end

data/lib/hdo/storting_importer/representative.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 module Hdo
   module StortingImporter
     class Representative
@@ -8,6 +10,47 @@ module Hdo
 
       attr_accessor :vote_result
 
+      def self.type_name
+        'representative'
+      end
+
+      def self.description
+        'a member of parliament'
+      end
+
+      def self.example
+        new(
+          'ADA',
+          'André Oktay',
+          'Dahl',
+          'M',
+          '1975-07-07T00:00:00',
+          '0001-01-01T00:00:00',
+          'Akershus',
+          'Høyre',
+          ['Justiskomiteen'],
+          '2011-2012'
+        )
+      end
+
+      def self.xml_example(builder = Util.builder)
+        example.to_hdo_xml(builder)
+      end
+
+      def self.fields
+        [
+          EXTERNAL_ID_FIELD,
+          Field.new(:firstName, true, :string, 'The first name of the representative.'),
+          Field.new(:lastName, true, :string, 'The last name of the representative.'),
+          Field.new(:period, true, :string, "An identifier for the period the representative is elected for."),
+          Field.new(:district, true, :string, "The electoral district the representative belongs to. Must match the 'name' field of the district type."),
+          Field.new(:party, true, :string, "The name of the representative's party."),
+          Field.new(:committee, true, :list, "A (possibly empty) list of committees the representative is a member of. This should match the 'name' field of the committee type."),
+          Field.new(:dateOfBirth, true, :string, "The representative's birth date."),
+          Field.new(:dateOfDeath, false, :string, "The representative's death date."),
+        ]
+      end
+
       def self.from_storting_doc(doc)
         nodes = doc.css("dagensrepresentant")
         nodes += doc.css("representant")
@@ -39,6 +82,10 @@ module Hdo
         )
       end
 
+      def self.from_hdo_doc(doc)
+        doc.css("representatives > representative").map { |e| from_hdo_node e }
+      end
+
       def self.from_hdo_node(node)
         district_node = node.css("district").first
         district      = district_node ? district_node.text : ''

data/lib/hdo/storting_importer/version.rb

@@ -1,5 +1,5 @@
 module Hdo
   module StortingImporter
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

data/lib/hdo/storting_importer/vote.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 module Hdo
   module StortingImporter
     class Vote
@@ -10,6 +12,47 @@ module Hdo
       alias_method :personal?, :personal
       alias_method :enacted?, :enacted
 
+      def self.type_name
+        'vote'
+      end
+
+      def self.description
+        'a parliamentary vote'
+      end
+
+      def self.fields
+        [
+          EXTERNAL_ID_FIELD,
+          Field.new(:externalIssueId, true, :string, "The id (matching the issue's externalId) of the issue being voted on."),
+          Field.new(:counts, true, :element, "An element with <for>, <against> and <absent> counts (see example)."),
+          Field.new(:enacted, true, :boolean, "Whether the proposal was enacted."),
+          Field.new(:subject, true, :string, "The subject of the vote."),
+          Field.new(:method, true, :string, "??"),
+          Field.new(:resultType, true, :string, "??"),
+          Field.new(:time, true, :string, "The timestamp for the vote."),
+          Field.new(:representatives, true, :element, "An element with each representative's vote. The element should contain a set of <a href='#input-format-representative'>&lt;representative&gt;</a> elements with an extra subnode 'voteResult', where valid values are 'for', 'against', 'absent'. See example."),
+          Field.new(:propositions, false, :element, "An element with each proposition being voted over. The element should contain a set of <a href='#input-format-proposition'>&lt;proposition&gt;</a> elements. See example."),
+        ]
+      end
+
+      def self.example
+        vote = new('2175', '51448', true, false, 'Forslag 24 - 26 på vegne av Per Olaf Lundteigen', 'ikke_spesifisert', 'ikke_spesifisert', '2012-04-12T16:37:27.053', 2, 96, 71)
+
+        rep = Representative.example
+        rep.vote_result = 'for'
+        vote.representatives << rep
+
+        prop = Vote::Proposition.example
+
+        vote.propositions << prop
+
+        vote
+      end
+
+      def self.xml_example(builder = Util.builder)
+        example.to_hdo_xml(builder)
+      end
+
       def self.from_storting_doc(doc)
         issue_id = doc.css("sak_id").first.text
 
@@ -35,6 +78,10 @@ module Hdo
         end
       end
 
+      def self.from_hdo_doc(doc)
+        doc.css("votes > vote").map { |e| from_hdo_node(e) }
+      end
+
       def self.from_hdo_node(node)
         external_id       = node.css("externalId").first.text
         external_issue_id = node.css("externalIssueId").first.text
@@ -140,6 +187,32 @@ module Hdo
       end
 
       class Proposition < Struct.new(:external_id, :description, :on_behalf_of, :body, :delivered_by)
+        def self.type_name
+          'proposition'
+        end
+
+        def self.example
+          new('1234', 'description', 'on behalf of', 'body', Representative.example)
+        end
+
+        def self.description
+          'a proposition being voted over'
+        end
+
+        def self.fields
+          [
+            EXTERNAL_ID_FIELD,
+            Field.new(:description, true, :string, 'A short description of the proposition.'),
+            Field.new(:deliveredBy, true, :string, "The representative that delivered the proposition. The element should contain a <a href='#input-format-representative'>&lt;representative&gt;</a> element."),
+            Field.new(:onBehalfOf, true, :string, "Description of who is behind the proposition."),
+            Field.new(:body, true, :string, "The full text of the proposition."),
+          ]
+        end
+
+        def self.xml_example(builder = Util.builder)
+          example.to_hdo_xml(builder)
+        end
+
         def self.from_hdo_node(node)
           external_id  = node.css("externalId").first.text
           description  = node.css("description").first.text

data/lib/hdo/storting_importer.rb

@@ -3,6 +3,9 @@ module Hdo
     def self.root
       @root ||= File.expand_path("../../..", __FILE__)
     end
+
+    Field = Struct.new(:name, :required, :type, :description)
+    EXTERNAL_ID_FIELD = Field.new(:externalId, false, :string, 'An optional external id, matching potential id fields in the input data. This is useful if you want to reimport previous data without creating duplicates.')
   end
 end
 
@@ -22,7 +25,6 @@ require 'hdo/storting_importer/data_source'
 require 'hdo/storting_importer/disk_data_source'
 require 'hdo/storting_importer/api_data_source'
 require 'hdo/storting_importer/parsing_data_source'
-require 'hdo/storting_importer/script_importer'
 
 require 'hdo/storting_importer/category'
 require 'hdo/storting_importer/committee'

data/spec/hdo/storting_importer/category_spec.rb

@@ -64,13 +64,37 @@ module Hdo
           XML
       end
 
-      it 'can deserialize HDO XML' do
+      it 'can deserialize a HDO XML node' do
         orig = Category.new("5", "ARBEIDSLIV")
         orig.children << Category.new("3", "LØNN")
 
         Category.from_hdo_node(parse(orig.to_hdo_xml)).should == orig
       end
 
+      it 'can deserialize a HDO XML doc' do
+        orig = Category.new("5", "ARBEIDSLIV")
+        orig.children << Category.new("3", "LØNN")
+
+        Category.from_hdo_doc(parse("<categories>#{orig.to_hdo_xml}</categories>")).should == [orig]
+      end
+
+      it 'has a type name' do
+        Category.type_name.should == 'category'
+      end
+
+      it 'has a description' do
+        Category.description.should be_kind_of(String)
+      end
+
+      it 'has an XML example' do
+        Category.xml_example.should be_kind_of(String)
+      end
+
+      it 'has a list of fields' do
+        Category.fields.should_not be_empty
+      end
+
+
     end
   end
 end

data/spec/hdo/storting_importer/committee_spec.rb

@@ -35,11 +35,32 @@ module Hdo
         XML
       end
 
-      it 'can deserialize HDO XML' do
+      it 'can deserialize a HDO XML node' do
         com = Committee.new("ARBSOS", 'Arbeids- og sosialkomiteen')
         Committee.from_hdo_node(parse(com.to_hdo_xml)).should == com
       end
 
+      it 'can deserialize a HDO XML doc' do
+        com = Committee.new("ARBSOS", 'Arbeids- og sosialkomiteen')
+        Committee.from_hdo_doc(parse("<committees>#{com.to_hdo_xml}</committees>")).should == [com]
+      end
+
+      it 'has a type name' do
+        Committee.type_name.should == 'committee'
+      end
+
+      it 'has a description' do
+        Committee.description.should be_kind_of(String)
+      end
+
+      it 'has an XML example' do
+        Committee.xml_example.should be_kind_of(String)
+      end
+
+      it 'has a list of fields' do
+        Committee.fields.should_not be_empty
+      end
+
 
     end
   end

data/spec/hdo/storting_importer/district_spec.rb

@@ -43,11 +43,32 @@ module Hdo
         XML
       end
 
-      it 'can deserialize HDO XML' do
+      it 'can deserialize a HDO XML node' do
         orig = District.new("Ak", "Akershus")
         District.from_hdo_node(parse(orig.to_hdo_xml)).should == orig
       end
 
+      it 'can deserialize a HDO XML doc' do
+        orig = District.new("Ak", "Akershus")
+        District.from_hdo_doc(parse("<districts>#{orig.to_hdo_xml}</districts>")).should == [orig]
+      end
+
+      it 'has a type name' do
+        District.type_name.should == 'district'
+      end
+
+      it 'has a description' do
+        District.description.should be_kind_of(String)
+      end
+
+      it 'has an XML example' do
+        District.xml_example.should be_kind_of(String)
+      end
+
+      it 'has a list of fields' do
+        District.fields.should_not be_empty
+      end
+
 
     end
   end

data/spec/hdo/storting_importer/issue_spec.rb

@@ -5,21 +5,6 @@ module Hdo
   module StortingImporter
     describe Issue do
 
-      def create_issue
-        Issue.new(
-          "53520",
-          "Inngåelse av avtale om opprettelse av sekretariatet for Den nordlige dimensjons partnerskap for helse og livskvalitet (NDPHS)",
-          "Samtykke til inngåelse av avtale av 25. november 2011 om opprettelse av sekretariatet for Den nordlige dimensjons partnerskap for helse og livskvalitet (NDPHS)",
-          "alminneligsak",
-          "mottatt",
-          "2012-04-20T00:00:00",
-          "Prop. 90 S (2011-2012)",
-          "proposisjon",
-          "Transport- og kommunikasjonskomiteen",
-          ['UTENRIKSSAKER', 'TRAKTATER', 'NORDISK SAMARBEID']
-        )
-      end
-
       it 'builds issues from Storting XML list' do
         xml = <<-XML
         <?xml version="1.0" encoding="utf-8"?>
@@ -85,7 +70,7 @@ module Hdo
       end
 
       it 'can serialize as HDO XML' do
-        create_issue.to_hdo_xml.should == <<-XML
+        Issue.example.to_hdo_xml.should == <<-XML
 <issue>
   <externalId>53520</externalId>
   <summary>Inngåelse av avtale om opprettelse av sekretariatet for Den nordlige dimensjons partnerskap for helse og livskvalitet (NDPHS)</summary>
@@ -105,11 +90,32 @@ module Hdo
 XML
       end
 
-      it 'can deserialize HDO XML' do
-        orig = create_issue
+      it 'can deserialize an HDO XML node' do
+        orig = Issue.example
         Issue.from_hdo_node(parse(orig.to_hdo_xml)).should == orig
       end
 
+      it 'can deserialize an HDO XML doc' do
+        orig = Issue.example
+        Issue.from_hdo_doc(parse("<issues>#{orig.to_hdo_xml}</issues>")).should == [orig]
+      end
+
+      it 'has a type name' do
+        Issue.type_name.should == 'issue'
+      end
+
+      it 'has a description' do
+        Issue.description.should be_kind_of(String)
+      end
+
+      it 'has an XML example' do
+        Issue.xml_example.should be_kind_of(String)
+      end
+
+      it 'has a list of fields' do
+        Issue.fields.should_not be_empty
+      end
+
     end
   end
 end

data/spec/hdo/storting_importer/party_spec.rb

@@ -41,11 +41,31 @@ module Hdo
         XML
       end
 
-      it 'can deserialize HDO XML' do
+      it 'can deserialize a HDO XML node' do
         orig = Party.new('Sp', 'Senterpartiet')
         Party.from_hdo_node(parse(orig.to_hdo_xml)).should == orig
       end
 
+      it 'can deserialize a HDO XML doc' do
+        orig = Party.new('Sp', 'Senterpartiet')
+        Party.from_hdo_doc(parse("<parties>#{orig.to_hdo_xml}</parties>")).should == [orig]
+      end
+
+      it 'has a type name' do
+        Party.type_name.should == 'party'
+      end
+
+      it 'has a description' do
+        Party.description.should be_kind_of(String)
+      end
+
+      it 'has an XML example' do
+        Party.xml_example.should be_kind_of(String)
+      end
+
+      it 'has a list of fields' do
+        Party.fields.should_not be_empty
+      end
 
     end
   end

data/spec/hdo/storting_importer/promise_spec.rb

@@ -33,6 +33,62 @@ module Hdo
         prom.page.should == '10'
       end
 
+      it 'serializes to HDO XML' do
+        Promise.example.to_hdo_xml.should == <<-XML
+<promise>
+  <party>H</party>
+  <general>true</general>
+  <categories>
+    <category>GRUNNSKOLE</category>
+  </categories>
+  <source>PP:8</source>
+  <body>Stille strengere krav til orden og oppførsel for å hindre at uro ødelegger undervisningen.</body>
+</promise>
+XML
+      end
+
+      it 'deserializes from HDO XML' do
+        promises = Promise.from_hdo_doc(parse(<<-XML))
+        <promises>
+          <promise>
+            <party>H</party>
+            <general>true</general>
+            <categories>
+              <category>GRUNNSKOLE</category>
+            </categories>
+            <source>PP:8</source>
+            <body>Stille strengere krav til orden og oppførsel for å hindre at uro ødelegger undervisningen.</body>
+          </promise>
+        </promises>
+        XML
+
+        promises.size.should == 1
+        promise = promises.first
+
+        promise.party.should == "H"
+        promise.should be_general
+        promise.categories.should == ["GRUNNSKOLE"]
+        promise.source.should == "PP"
+        promise.page.should == "8"
+        promise.body.should == "Stille strengere krav til orden og oppførsel for å hindre at uro ødelegger undervisningen."
+      end
+
+      it 'has a description' do
+        Promise.description.should be_kind_of(String)
+      end
+
+      it 'has fields' do
+        Promise.fields.should_not be_empty
+      end
+
+      it 'has a type name' do
+        Promise.type_name.should == 'promise'
+      end
+
+      it 'has a an XML example' do
+        Promise.xml_example.should be_kind_of(String)
+      end
+
     end
   end
 end

data/spec/hdo/storting_importer/representative_spec.rb

@@ -4,10 +4,6 @@ module Hdo
   module StortingImporter
     describe Representative do
 
-      def create_representative
-        Representative.new('ADA', 'André Oktay', 'Dahl', 'M', '1975-07-07T00:00:00', '0001-01-01T00:00:00', 'Akershus', 'Høyre', ['Justiskomiteen'], '2011-2012')
-      end
-
       it "builds representatives from the Storting XML list" do
         xml = <<-XML
         <?xml version="1.0" encoding="utf-8"?>
@@ -52,7 +48,7 @@ module Hdo
       end
 
       it 'converts itself into HDO XML' do
-        rep = create_representative
+        rep = Representative.example
         rep.to_hdo_xml.should == <<-XML
 <representative>
   <externalId>ADA</externalId>
@@ -71,11 +67,32 @@ module Hdo
         XML
       end
 
-      it 'can deserialize HDO XML' do
-        rep = create_representative
+      it 'can deserialize a HDO XML node' do
+        rep = Representative.example
         Representative.from_hdo_node(parse(rep.to_hdo_xml)).should == rep
       end
 
+      it 'can deserialize a HDO XML doc' do
+        rep = Representative.example
+        Representative.from_hdo_doc(parse("<representatives>#{rep.to_hdo_xml}</representatives>")).should == [rep]
+      end
+
+      it 'has a type name' do
+        Representative.type_name.should == 'representative'
+      end
+
+      it 'has a description' do
+        Representative.description.should be_kind_of(String)
+      end
+
+      it 'has an XML example' do
+        Representative.xml_example.should be_kind_of(String)
+      end
+
+      it 'has a list of fields' do
+        Representative.fields.should_not be_empty
+      end
+
     end
   end
 end