hdo-storting-importer 0.0.1 → 0.0.2

data/Gemfile

@@ -1,8 +1,3 @@
 source :rubygems
 
-gem "builder"
-gem "nokogiri"
-gem "socksify"
-gem "aruba"
-gem "rake"
-gem "rest-client"
+gemspec

data/README.md

@@ -1,14 +1,13 @@
 What
 ====
 
-Convert and import XML from data.stortinget.no
+Convert XML from data.stortinget.no to HDO XML
 
 Usage
 =====
 
-    $ bin/hdo-converter --app-root /src/hdo-site all
+    $ hdo-converter --help
+    $ hdo-converter categories folketingparser/rawdata/data.stortinget.no/eksport/emner/index.html
+    $ hdo-converter promises all-promises.csv | script/import promises
 
-Caveats
-=======
 
-Right now we're executing script/import from hdo-site to perform the import. That means running this with bundler may fail if the gems don't match. Don't do that.

data/features/convert.feature

@@ -0,0 +1,189 @@
+Feature: Import data
+    Scenario: Import districts
+      Given a file named "fylker.xml" with:
+      """
+      <?xml version="1.0" encoding="utf-8"?>
+      <fylker_oversikt xmlns:i="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://data.stortinget.no">
+        <versjon>1.0</versjon>
+        <fylker_liste>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>AA</id>
+            <navn>Aust-Agder</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>VA</id>
+            <navn>Vest-Agder</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Ak</id>
+            <navn>Akershus</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Bu</id>
+            <navn>Buskerud</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Fi</id>
+            <navn>Finnmark</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>He</id>
+            <navn>Hedmark</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Ho</id>
+            <navn>Hordaland</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>MR</id>
+            <navn>Møre og Romsdal</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>No</id>
+            <navn>Nordland</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Op</id>
+            <navn>Oppland</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Os</id>
+            <navn>Oslo</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Ro</id>
+            <navn>Rogaland</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>SF</id>
+            <navn>Sogn og Fjordane</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Te</id>
+            <navn>Telemark</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Tr</id>
+            <navn>Troms</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>NT</id>
+            <navn>Nord-Trøndelag</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>ST</id>
+            <navn>Sør-Trøndelag</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Ve</id>
+            <navn>Vestfold</navn>
+          </fylke>
+          <fylke>
+            <versjon>1.0</versjon>
+            <id>Øs</id>
+            <navn>Østfold</navn>
+          </fylke>
+        </fylker_liste>
+      </fylker_oversikt>
+      """
+      When I run `hdo-converter districts fylker.xml`
+      Then the stdout should contain:
+      """
+      <?xml version="1.0" encoding="UTF-8"?>
+      <districts>
+        <district>
+          <externalId>AA</externalId>
+          <name>Aust-Agder</name>
+        </district>
+        <district>
+          <externalId>VA</externalId>
+          <name>Vest-Agder</name>
+        </district>
+        <district>
+          <externalId>Ak</externalId>
+          <name>Akershus</name>
+        </district>
+        <district>
+          <externalId>Bu</externalId>
+          <name>Buskerud</name>
+        </district>
+        <district>
+          <externalId>Fi</externalId>
+          <name>Finnmark</name>
+        </district>
+        <district>
+          <externalId>He</externalId>
+          <name>Hedmark</name>
+        </district>
+        <district>
+          <externalId>Ho</externalId>
+          <name>Hordaland</name>
+        </district>
+        <district>
+          <externalId>MR</externalId>
+          <name>Møre og Romsdal</name>
+        </district>
+        <district>
+          <externalId>No</externalId>
+          <name>Nordland</name>
+        </district>
+        <district>
+          <externalId>Op</externalId>
+          <name>Oppland</name>
+        </district>
+        <district>
+          <externalId>Os</externalId>
+          <name>Oslo</name>
+        </district>
+        <district>
+          <externalId>Ro</externalId>
+          <name>Rogaland</name>
+        </district>
+        <district>
+          <externalId>SF</externalId>
+          <name>Sogn og Fjordane</name>
+        </district>
+        <district>
+          <externalId>Te</externalId>
+          <name>Telemark</name>
+        </district>
+        <district>
+          <externalId>Tr</externalId>
+          <name>Troms</name>
+        </district>
+        <district>
+          <externalId>NT</externalId>
+          <name>Nord-Trøndelag</name>
+        </district>
+        <district>
+          <externalId>ST</externalId>
+          <name>Sør-Trøndelag</name>
+        </district>
+        <district>
+          <externalId>Ve</externalId>
+          <name>Vestfold</name>
+        </district>
+        <district>
+          <externalId>Øs</externalId>
+          <name>Østfold</name>
+        </district>
+      </districts>
+      """

data/hdo-storting-importer.gemspec

@@ -14,4 +14,10 @@ Gem::Specification.new do |gem|
   gem.name          = "hdo-storting-importer"
   gem.require_paths = ["lib"]
   gem.version       = Hdo::StortingImporter::VERSION
+
+  gem.add_runtime_dependency "builder"
+  gem.add_runtime_dependency "nokogiri"
+  gem.add_runtime_dependency "aruba"
+  gem.add_runtime_dependency "rake"
+  gem.add_runtime_dependency "rest-client"
 end

data/lib/hdo/storting_importer/category.rb

@@ -6,10 +6,45 @@ module Hdo
       attr_reader :external_id, :name
       attr_accessor :children
 
+      def self.type_name
+        'category'
+      end
+
+      def self.description
+        'a parliamentary category, used to categorize issues and promises'
+      end
+
+      def self.fields
+        [
+          EXTERNAL_ID_FIELD,
+          Field.new(:name, true, :string, 'The name of the category.'),
+          Field.new(:subcategories, false, 'list<category>', 'A list of subcategories.'),
+        ]
+      end
+
+      def self.xml_example(builder = Util.builder)
+        cat = new("5", "Employment")
+        cat.children << new("6", "Wages")
+
+        cat.to_hdo_xml(builder)
+      end
+
+      #
+      # Deserialize from a Storting XML document (<emne_liste><emne>...</emne></emne_liste>)
+      #
+      # @return [Array<Category>]
+      #
+
       def self.from_storting_doc(doc)
         doc.css("emne_liste > emne").map { |xt| from_storting_node(xt) }
       end
 
+      #
+      # Deserialize form a Storting XML node (<emne>...</emne>)
+      #
+      # @return [Category]
+      #
+
       def self.from_storting_node(node)
         cat = Category.new node.css("id").first.text, node.css("navn").first.text
 
@@ -21,6 +56,24 @@ module Hdo
         cat
       end
 
+      #
+      # Deserialize from a HDO XML document (<categories><category>...</category></categories>)
+      #
+      # @param [Nokogiri::XML::Element]
+      # @return [Array<Category>]
+      #
+
+
+      def self.from_hdo_doc(doc)
+        doc.css("categories > category").map { |node| from_hdo_node(node) }
+      end
+
+      #
+      # Deserialize from a HDO XML node
+      #
+      # @return [Category]
+      #
+
       def self.from_hdo_node(node)
         external_id = node.css("externalId").first.text
         name        = node.css("name").first.text
@@ -37,6 +90,10 @@ module Hdo
         @children    = []
       end
 
+      #
+      # Serialize as HDO XML
+      #
+
       def to_hdo_xml(builder = Util.builder)
         builder.category do |cat|
           cat.externalId external_id

data/lib/hdo/storting_importer/cli.rb

@@ -3,139 +3,118 @@ module Hdo
     class CLI
       def initialize(args)
         @log = Logger.new(STDERR)
-        @cmd, @options = parse(args)
+        @type, @files, @options = parse(args)
       end
 
       def execute
-        import @cmd.to_sym
+        case @type
+        when :dld_issues
+          puts read_dld_issues
+        when :dld_votes
+          puts read_dld_votes
+        when :promises
+          puts read_promises
+        else
+          puts read_type(@type, class_for_type(@type))
+        end
+      rescue Errno::EPIPE
+        # ignored
       end
 
       private
 
+      TYPE_TO_CLASS = {
+        :categories      => Category,
+        :issues          => Issue,
+        :districts       => District,
+        :committees      => Committee,
+        :parties         => Party,
+        :representatives => Representative,
+        :votes           => Vote
+      }
+
+      def class_for_type(type)
+        TYPE_TO_CLASS[type] or raise ArgumentError, "unknown type: #{type.inspect}"
+      end
+
+      def read_type(plural, klass)
+        objs = @files.map do |e|
+          doc = Nokogiri::XML.parse(File.read(e))
+          doc.remove_namespaces!
+
+          klass.from_storting_doc(doc)
+        end.flatten
+
+        str = Util.builder do |xml|
+          xml.instruct!
+          xml.__send__(plural) do |builder|
+            objs.each { |e| e.to_hdo_xml(builder) }
+          end
+        end
+
+        str
+      end
+
       def parse(args)
-        options = {:source => (has_submodule? ? 'disk' : 'api')}
+        options = {}
 
         parser = OptionParser.new do |opt|
-          opt.banner = "Usage: #{$0} <import-type> [options]"
-
+          types = TYPE_TO_CLASS.keys + [:dld_issues, :dld_votes, :promises]
+          opt.banner = "Usage: #{$0} <#{types.join '|'}> <file(s)>"
           opt.on("--help", "You're looking at it.") { puts opt; exit; }
-          opt.on("--only-print", "Don't run import, only print generated XML.") { options[:only_print] = true }
-          opt.on("--except WHAT", 'Ignore this comma separated list of entities from import.') { |s| options[:ignore] = s.split(",").map(&:strip).map(&:to_sym) }
-          opt.on("--app-root APP_ROOT", 'Path to clone of git://github.com/holderdeord/hdo-site.git') { |path| options[:app_root] = path }
-          opt.on("--app-url APP_URL", 'URL to hdo-site') { |url| options[:app_url] = url }
-          opt.on("--source SOURCE ", 'Where to take data from [disk|api]') { |source| options[:source] = source }
-          opt.on("--socks PROXY", 'host:port for SOCKS proxy') do |proxy|
-            require 'socksify'
-            host, port = proxy.split ":"
-            TCPSocket.socks_server = host
-            TCPSocket.socks_port = Integer(port)
-          end
         end
 
         parser.parse!(args)
 
-        cmd = args.shift
+        type, files = args
 
-        unless cmd
+        if type.nil?
           puts(parser)
           exit 1
         end
 
-        [cmd, options]
+        [type.to_sym, Array(files), options]
       end
 
-      def data_source
-        @data_source ||= (
-          ds = if @options[:source] == "disk"
-                 DiskDataSource.new(File.join(StortingImporter.root, 'folketingparser/rawdata/data.stortinget.no'))
-               elsif @options[:source] == "api"
-                 ApiDataSource.new("http://data.stortinget.no/")
-               else
-                 raise ArgumentError, "invalid source: #{@options[:source].inspect}"
-               end
-
-          ParsingDataSource.new(ds)
-        )
-      end
+      def read_dld_issues
+        doc = Nokogiri::XML.parse(File.read(File.join(StortingImporter.root, 'data/dld-issues.xml')))
+        issues = Issue.from_hdo_doc doc
 
-      def importer
-        @importer ||= (
-          if @options[:app_root]
-            ScriptImporter.new(@options[:app_root])
-          else
-            raise ArgumentError, "app-root not given, can't import"
+        Util.builder do |xml|
+          xml.instruct!
+          xml.issues do |issues_builder|
+            issues.each { |i| i.to_hdo_xml(issues_builder) }
           end
-        )
-      end
-
-      def import(what)
-        case what
-        when :dld
-          import_dld
-        when :promises
-          import_promises
-        when :all
-          import_all
-        else
-          import_docs converter.xml_for(what.to_sym)
         end
       end
 
-      def import_all
-        ignore = Array(@options[:ignore])
-
-        import_docs converter.xml_for(:parties) unless ignore.include?(:parties)
-        import_docs converter.xml_for(:committees) unless ignore.include?(:committees)
-        import_docs converter.xml_for(:districts) unless ignore.include?(:districts)
-        import_docs converter.xml_for(:representatives) unless ignore.include?(:representatives)
-        import_docs converter.xml_for(:categories) unless ignore.include?(:categories)
-        import_docs converter.xml_for(:issues) unless ignore.include?(:issues)
-        import_docs converter.xml_for(:votes) unless ignore.include?(:votes)
+      def read_dld_votes
+        doc = Nokogiri::XML.parse(File.read(File.join(StortingImporter.root, 'folketingparser/data/votering-2011-04-04-dld-hdo.xml')))
+        votes = Vote.from_hdo_doc doc
 
-        import_dld unless ignore.include?(:dld)
-        import_promises unless ignore.include?(:promises)
-      end
-
-      def converter
-        @converter ||= Converter.new(data_source)
-      end
-
-      def import_dld
-        if has_submodule?
-          print_or_import File.read(File.join(StortingImporter.root, 'data/dld-issues.xml'))
-          print_or_import File.read(File.join(StortingImporter.root, 'folketingparser/data/votering-2011-04-04-dld-hdo.xml'))
-        else
-          $stderr.puts "folketingparser not found, skipping DLD votes and issues (run `git submodule update --init` if you need this)"
-        end
-      end
-
-      def import_promises
-        csvs = Dir[File.join(StortingImporter.root, 'data/promises-*.csv')].sort_by { |e| File.basename(e) }
-        csvs.each do |path|
-          print_or_import PromiseConverter.new(path).xml
+        Util.builder do |xml|
+          xml.instruct!
+          xml.votes do |votes_builder|
+            votes.each { |v| v.to_hdo_xml(votes_builder) }
+          end
         end
       end
 
-      def import_docs(docs)
-        docs = [docs] unless docs.kind_of?(Array)
-
-        docs.each do |doc|
-          print_or_import doc.to_s
+      def read_promises
+        csvs = @files.any? ? @files : Dir[File.join(StortingImporter.root, 'data/promises-*.csv')].sort_by { |e| File.basename(e) }
+        content = ''
+        csvs.each do |csv|
+          content << File.read(File.expand_path(csv), encoding: "ISO-8859-1").encode("UTF-8")
         end
-      end
 
-      def print_or_import(xml)
-        if @options[:only_print]
-          puts xml
-        else
-          importer.import xml
+        Util.builder do |xml|
+          xml.instruct!
+          xml.promises do |promises|
+            Promise.from_csv(content).each { |e| e.to_hdo_xml(promises) }
+          end
         end
       end
 
-      def has_submodule?
-        File.exist?(File.join(StortingImporter.root, 'folketingparser/data'))
-      end
-
     end
   end
 end

data/lib/hdo/storting_importer/committee.rb

@@ -5,6 +5,22 @@ module Hdo
 
       attr_reader :external_id, :name
 
+      def self.type_name
+        'committee'
+      end
+
+      def self.description
+        'a parliamentary committe'
+      end
+
+      def self.xml_example(builder = Util.builder)
+        new("ARBSOS", "Arbeids- og sosialkomiteen").to_hdo_xml(builder)
+      end
+
+      def self.fields
+        [EXTERNAL_ID_FIELD, Field.new(:name, true, :string, 'The name of the committee.')]
+      end
+
       def self.from_storting_doc(doc)
         doc.css("komiteer_liste komite").map do |node|
           from_storting_node(node)
@@ -15,6 +31,10 @@ module Hdo
         new node.css("id").first.text, node.css("navn").first.text
       end
 
+      def self.from_hdo_doc(doc)
+        doc.css("committees > committee").map { |e| from_hdo_node e }
+      end
+
       def self.from_hdo_node(node)
         new node.css("externalId").first.text, node.css("name").first.text
       end

data/lib/hdo/storting_importer/district.rb

@@ -5,6 +5,22 @@ module Hdo
 
       attr_reader :external_id, :name
 
+      def self.type_name
+        'district'
+      end
+
+      def self.description
+        'an electoral district'
+      end
+
+      def self.xml_example(builder = Util.builder)
+        new("Db", "Duckburg").to_hdo_xml(builder)
+      end
+
+      def self.fields
+        [EXTERNAL_ID_FIELD, Field.new(:name, true, :string, 'The name of the electoral district.')]
+      end
+
       def self.from_storting_doc(doc)
         doc.css("fylker_liste fylke").map do |node|
           from_storting_node(node)
@@ -15,6 +31,10 @@ module Hdo
         new node.css("id").first.text, node.css("navn").first.text
       end
 
+      def self.from_hdo_doc(doc)
+        doc.css("districts > district").map { |e| from_hdo_node(e) }
+      end
+
       def self.from_hdo_node(node)
         new node.css("externalId").first.text, node.css("name").first.text
       end

data/lib/hdo/storting_importer/issue.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 module Hdo
   module StortingImporter
     class Issue
@@ -6,6 +8,48 @@ module Hdo
       attr_reader :external_id, :summary, :description, :type, :status, :last_update,
                   :reference, :document_group, :committee, :categories
 
+      def self.type_name
+        'issue'
+      end
+
+      def self.description
+        'a parliament issue'
+      end
+
+      def self.example
+        new(
+          "53520",
+          "Inngåelse av avtale om opprettelse av sekretariatet for Den nordlige dimensjons partnerskap for helse og livskvalitet (NDPHS)",
+          "Samtykke til inngåelse av avtale av 25. november 2011 om opprettelse av sekretariatet for Den nordlige dimensjons partnerskap for helse og livskvalitet (NDPHS)",
+          "alminneligsak",
+          "mottatt",
+          "2012-04-20T00:00:00",
+          "Prop. 90 S (2011-2012)",
+          "proposisjon",
+          "Transport- og kommunikasjonskomiteen",
+          ['UTENRIKSSAKER', 'TRAKTATER', 'NORDISK SAMARBEID']
+        )
+      end
+
+      def self.xml_example(builder = Util.builder)
+        example.to_hdo_xml(builder)
+      end
+
+      def self.fields
+        [
+          EXTERNAL_ID_FIELD,
+          Field.new(:summary,       true,  :string, 'A (preferably one-line) summary of the issue.'),
+          Field.new(:description,   true,  :string, 'A longer description of the issue.'),
+          Field.new(:type,          true,  :string, 'The type of issue.'),
+          Field.new(:status,        true,  :string, 'The status of the issue.'),
+          Field.new(:lastUpdate,    true,  :string, 'The time the issue was last updated in the parliament.'),
+          Field.new(:reference,     true,  :string, 'A reference.'),
+          Field.new(:documentGroup, true,  :string, 'What document group this issue belongs to.'),
+          Field.new(:committee,     false, :string, "What committee this issue belongs to. Should match the 'name' field in the committee type."),
+          Field.new(:categories,    false, 'list',  "List of categories (matching the 'name' field of the category type).")
+        ]
+      end
+
       def self.from_storting_doc(doc)
         doc.css("saker_liste sak").map do |node|
           from_storting_node(node)
@@ -33,6 +77,13 @@ module Hdo
         end
 
         new(external_id, summary, description, type, status, last_update, reference, document_group, committee, categories)
+      rescue
+        puts lnode
+        raise
+      end
+
+      def self.from_hdo_doc(doc)
+        doc.css("issues > issue").map { |e| from_hdo_node(e) }
       end
 
       def self.from_hdo_node(node)