hdo-storting-importer 0.0.8 → 0.0.9

data/Gemfile

@@ -1,3 +1,3 @@
 source :rubygems
 
-gemspec
+gemspec

data/Rakefile

@@ -1,6 +1,5 @@
 require 'rake'
 require 'rspec/core/rake_task'
-require 'pry'
 
 RSpec::Core::RakeTask.new
 

data/features/convert.feature

@@ -107,83 +107,101 @@ Feature: Import data
       When I run `hdo-converter districts fylker.xml`
       Then the stdout should contain:
       """
-      <?xml version="1.0" encoding="UTF-8"?>
-      <districts>
-        <district>
-          <externalId>AA</externalId>
-          <name>Aust-Agder</name>
-        </district>
-        <district>
-          <externalId>VA</externalId>
-          <name>Vest-Agder</name>
-        </district>
-        <district>
-          <externalId>Ak</externalId>
-          <name>Akershus</name>
-        </district>
-        <district>
-          <externalId>Bu</externalId>
-          <name>Buskerud</name>
-        </district>
-        <district>
-          <externalId>Fi</externalId>
-          <name>Finnmark</name>
-        </district>
-        <district>
-          <externalId>He</externalId>
-          <name>Hedmark</name>
-        </district>
-        <district>
-          <externalId>Ho</externalId>
-          <name>Hordaland</name>
-        </district>
-        <district>
-          <externalId>MR</externalId>
-          <name>Møre og Romsdal</name>
-        </district>
-        <district>
-          <externalId>No</externalId>
-          <name>Nordland</name>
-        </district>
-        <district>
-          <externalId>Op</externalId>
-          <name>Oppland</name>
-        </district>
-        <district>
-          <externalId>Os</externalId>
-          <name>Oslo</name>
-        </district>
-        <district>
-          <externalId>Ro</externalId>
-          <name>Rogaland</name>
-        </district>
-        <district>
-          <externalId>SF</externalId>
-          <name>Sogn og Fjordane</name>
-        </district>
-        <district>
-          <externalId>Te</externalId>
-          <name>Telemark</name>
-        </district>
-        <district>
-          <externalId>Tr</externalId>
-          <name>Troms</name>
-        </district>
-        <district>
-          <externalId>NT</externalId>
-          <name>Nord-Trøndelag</name>
-        </district>
-        <district>
-          <externalId>ST</externalId>
-          <name>Sør-Trøndelag</name>
-        </district>
-        <district>
-          <externalId>Ve</externalId>
-          <name>Vestfold</name>
-        </district>
-        <district>
-          <externalId>Øs</externalId>
-          <name>Østfold</name>
-        </district>
-      </districts>
+      [
+        {
+          "kind": "hdo#district",
+          "externalId": "AA",
+          "name": "Aust-Agder"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "VA",
+          "name": "Vest-Agder"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Ak",
+          "name": "Akershus"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Bu",
+          "name": "Buskerud"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Fi",
+          "name": "Finnmark"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "He",
+          "name": "Hedmark"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Ho",
+          "name": "Hordaland"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "MR",
+          "name": "Møre og Romsdal"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "No",
+          "name": "Nordland"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Op",
+          "name": "Oppland"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Os",
+          "name": "Oslo"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Ro",
+          "name": "Rogaland"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "SF",
+          "name": "Sogn og Fjordane"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Te",
+          "name": "Telemark"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Tr",
+          "name": "Troms"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "NT",
+          "name": "Nord-Trøndelag"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "ST",
+          "name": "Sør-Trøndelag"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Ve",
+          "name": "Vestfold"
+        },
+        {
+          "kind": "hdo#district",
+          "externalId": "Øs",
+          "name": "Østfold"
+        }
+      ]
       """

data/hdo-storting-importer.gemspec

@@ -22,4 +22,9 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "rest-client"
   gem.add_runtime_dependency "unicode_utils"
   gem.add_runtime_dependency "multi_json"
+  gem.add_runtime_dependency "yajl-ruby"
+  gem.add_runtime_dependency "jschematic", ">= 0.1.0"
+
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'rspec'
 end

data/lib/hdo/storting_importer/category.rb

@@ -3,25 +3,12 @@ module Hdo
     class Category
       include IvarEquality
       include Inspectable
+      include HasJsonSchema
 
       attr_reader :external_id, :name
       attr_accessor :children
 
-      def self.type_name
-        'category'
-      end
-
-      def self.description
-        'a parliamentary category, used to categorize issues and promises'
-      end
-
-      def self.fields
-        [
-          EXTERNAL_ID_FIELD,
-          Field.new(:name, true, :string, 'The name of the category.'),
-          Field.new(:subcategories, false, 'list<category>', 'A list of subcategories.'),
-        ]
-      end
+      schema_path StortingImporter.lib.join("hdo/storting_importer/schema/category.json").to_s
 
       def self.example
         cat = new("5", "Employment")
@@ -30,8 +17,8 @@ module Hdo
         cat
       end
 
-      def self.xml_example(builder = Util.builder)
-        example.to_hdo_xml(builder)
+      def self.json_example
+        Util.json_pretty example
       end
 
       #
@@ -61,32 +48,11 @@ module Hdo
         cat
       end
 
-      #
-      # Deserialize from a HDO XML document (<categories><category>...</category></categories>)
-      #
-      # @param [Nokogiri::XML::Element]
-      # @return [Array<Category>]
-      #
-
-
-      def self.from_hdo_doc(doc)
-        doc.css("categories > category").map { |node| from_hdo_node(node) }
-      end
-
-      #
-      # Deserialize from a HDO XML node
-      #
-      # @return [Category]
-      #
-
-      def self.from_hdo_node(node)
-        external_id = node.css("externalId").first.text
-        name        = node.css("name").first.text
+      def self.from_hash(data)
+        obj = new data['externalId'], data['name']
+        obj.children = Array(data['subCategories']).map { |e| from_hash(e) }
 
-        cat = new external_id, name
-        cat.children = node.css("subcategories category").map { |e| from_hdo_node(e) }
-
-        cat
+        obj
       end
 
       def initialize(external_id, name)
@@ -99,23 +65,17 @@ module Hdo
         short_inspect_string :include => [:external_id, :name]
       end
 
-      #
-      # Serialize as HDO XML
-      #
+      def to_hash
+        h = {
+          :kind       => self.class.kind,
+          :externalId => @external_id,
+          :name       => @name
+        }
 
-      def to_hdo_xml(builder = Util.builder)
-        builder.category do |cat|
-          cat.externalId external_id
-          cat.name name
+        h[:subCategories] = @children.map { |e| e.to_hash } if @children.any?
 
-          if children.any?
-            cat.subcategories do |sub|
-              children.each { |child| child.to_hdo_xml(sub) }
-            end
-          end
-        end
+        h
       end
-
     end
   end
 end

data/lib/hdo/storting_importer/cli.rb

@@ -44,21 +44,14 @@ module Hdo
           klass.from_storting_doc(doc)
         end.flatten
 
-        str = Util.builder do |xml|
-          xml.instruct!
-          xml.__send__(plural) do |builder|
-            objs.each { |e| e.to_hdo_xml(builder) }
-          end
-        end
-
-        str
+        Util.json_pretty objs
       end
 
       def parse(args)
         options = {}
 
         parser = OptionParser.new do |opt|
-          types = TYPE_TO_CLASS.keys + [:dld_issues, :dld_votes, :promises]
+          types = TYPE_TO_CLASS.keys + [:dld_issues, :dld_votes, :promises, :any]
           opt.banner = "Usage: #{$0} <#{types.join '|'}> <file(s)>"
           opt.on("--help", "You're looking at it.") { puts opt; exit; }
         end
@@ -76,31 +69,15 @@ module Hdo
       end
 
       def read_dld_issues
-        doc = Nokogiri::XML.parse(File.read(File.join(StortingImporter.root, 'data/dld-issues.xml')))
-        issues = Issue.from_hdo_doc doc
-
-        Util.builder do |xml|
-          xml.instruct!
-          xml.issues do |issues_builder|
-            issues.each { |i| i.to_hdo_xml(issues_builder) }
-          end
-        end
+        Util.json_pretty Issue.from_json(StortingImporter.root.join('data/dld-issues.json').read)
       end
 
       def read_dld_votes
-        doc = Nokogiri::XML.parse(File.read(File.join(StortingImporter.root, 'folketingparser/data/votering-2011-04-04-dld-hdo.xml')))
-        votes = Vote.from_hdo_doc doc
-
-        Util.builder do |xml|
-          xml.instruct!
-          xml.votes do |votes_builder|
-            votes.each { |v| v.to_hdo_xml(votes_builder) }
-          end
-        end
+        Util.json_pretty Vote.from_json StortingImporter.root.join('data/dld-votes.json').read
       end
 
       def read_promises
-        csvs = @files.any? ? @files : Dir[File.join(StortingImporter.root, 'data/promises-*.csv')].sort_by { |e| File.basename(e) }
+        csvs = @files.any? ? @files : Dir[StortingImporter.root.join('data/promises-*.csv').to_s].sort_by { |e| File.basename(e) }
         content = ''
         csvs.each do |csv|
           if csv =~ /^http/
@@ -110,12 +87,7 @@ module Hdo
           end
         end
 
-        Util.builder do |xml|
-          xml.instruct!
-          xml.promises do |promises|
-            Promise.from_csv(content).each { |e| e.to_hdo_xml(promises) }
-          end
-        end
+        Util.json_pretty Promise.from_csv(content)
       end
 
     end

data/lib/hdo/storting_importer/committee.rb

@@ -1,29 +1,20 @@
 module Hdo
   module StortingImporter
     class Committee
+      include HasJsonSchema
       include IvarEquality
 
       attr_reader :external_id, :name
       alias_method :short_inspect, :inspect
 
-      def self.type_name
-        'committee'
-      end
-
-      def self.description
-        'a parliamentary committe'
-      end
+      schema_path StortingImporter.lib.join("hdo/storting_importer/schema/committee.json").to_s
 
       def self.example
         new "ARBSOS", "Arbeids- og sosialkomiteen"
       end
 
-      def self.xml_example(builder = Util.builder)
-        example.to_hdo_xml(builder)
-      end
-
-      def self.fields
-        [EXTERNAL_ID_FIELD, Field.new(:name, true, :string, 'The name of the committee.')]
+      def self.json_example
+        Util.json_pretty example
       end
 
       def self.from_storting_doc(doc)
@@ -36,12 +27,8 @@ module Hdo
         new node.css("id").first.text, node.css("navn").first.text
       end
 
-      def self.from_hdo_doc(doc)
-        doc.css("committees > committee").map { |e| from_hdo_node e }
-      end
-
-      def self.from_hdo_node(node)
-        new node.css("externalId").first.text, node.css("name").first.text
+      def self.from_hash(hash)
+        new hash['externalId'], hash['name']
       end
 
       def initialize(external_id, name)
@@ -49,12 +36,14 @@ module Hdo
         @name = name
       end
 
-      def to_hdo_xml(builder = Util.builder)
-        builder.committee do |com|
-          com.externalId external_id
-          com.name name
-        end
+      def to_hash
+        {
+          :kind       => self.class.kind,
+          :externalId => @external_id,
+          :name       => @name
+        }
       end
+
     end
   end
 end

data/lib/hdo/storting_importer/converter.rb

@@ -7,15 +7,10 @@ module Hdo
         @cache = {}
       end
 
-      def xml_for(name)
-        Util.builder do |xml|
-          xml.instruct!
-          xml.__send__(name) do |builder|
-            data_for(name).each do |obj|
-              obj.to_hdo_xml(builder)
-            end
-          end
-        end
+      def json_for(name, opts = nil)
+        obj = data_for(name)
+        
+        Yajl::Encoder.encode(obj, opts && opts[:pretty])
       end
 
       private

data/lib/hdo/storting_importer/district.rb

@@ -1,29 +1,20 @@
 module Hdo
   module StortingImporter
     class District
+      include HasJsonSchema
       include IvarEquality
 
       attr_reader :external_id, :name
       alias_method :short_inspect, :inspect
 
-      def self.type_name
-        'district'
-      end
-
-      def self.description
-        'an electoral district'
-      end
+      schema_path StortingImporter.lib.join("hdo/storting_importer/schema/district.json").to_s
 
       def self.example
         new("Db", "Duckburg")
       end
 
-      def self.xml_example(builder = Util.builder)
-        example.to_hdo_xml(builder)
-      end
-
-      def self.fields
-        [EXTERNAL_ID_FIELD, Field.new(:name, true, :string, 'The name of the electoral district.')]
+      def self.json_example
+        Util.json_pretty example
       end
 
       def self.from_storting_doc(doc)
@@ -36,12 +27,8 @@ module Hdo
         new node.css("id").first.text, node.css("navn").first.text
       end
 
-      def self.from_hdo_doc(doc)
-        doc.css("districts > district").map { |e| from_hdo_node(e) }
-      end
-
-      def self.from_hdo_node(node)
-        new node.css("externalId").first.text, node.css("name").first.text
+      def self.from_hash(hash)
+        new hash.fetch('externalId'), hash.fetch('name')
       end
 
       def initialize(external_id, name)
@@ -49,11 +36,12 @@ module Hdo
         @name = name
       end
 
-      def to_hdo_xml(builder = Util.builder)
-        builder.district do |d|
-          d.externalId external_id
-          d.name name
-        end
+      def to_hash
+        {
+          :kind       => self.class.kind,
+          :externalId => @external_id,
+          :name       => @name
+        }
       end
 
     end

data/lib/hdo/storting_importer/fusion_table.rb

@@ -0,0 +1,52 @@
+module Hdo
+  module StortingImporter
+    #
+    # Simple query public Fusion Tables (onle need API key for auth).
+    # If we need to write to the table programatically, see https://gist.github.com/3265043.
+    #
+
+    class FusionTable
+      def initialize(api_key)
+        @api_key  = api_key
+      end
+
+      def query(sql, opts = {})
+        resp = fix_request_failure do
+          RestClient.get("https://www.googleapis.com/fusiontables/v1/query", :params => {:sql => sql, :key => @api_key})
+        end
+
+        data = MultiJson.decode(resp)
+
+        if opts[:rows]
+          data.fetch('rows')
+        else
+          cols = data.fetch('columns')
+          rows = data.fetch('rows')
+
+          rows.map do |row|
+            res = {}
+            cols.each_with_index { |col, idx| res[col] = row[idx] }
+
+            res
+          end
+        end
+      end
+
+      def columns_for(table_id)
+        resp = fix_request_failure do
+          RestClient.get("https://www.googleapis.com/fusiontables/v1/tables/#{table_id}/columns", :params => {:key => @api_key})
+        end
+
+        data = MultiJson.decode(resp)
+        data['items']
+      end
+
+      def fix_request_failure(&blk)
+        yield
+      rescue RestClient::RequestFailed => ex
+        raise unless ex.respond_to?(:http_body)
+        raise "#{ex.message}: #{ex.http_body}"
+      end
+    end
+  end
+end

data/lib/hdo/storting_importer/has_json_schema.rb

@@ -0,0 +1,85 @@
+module Hdo
+  module StortingImporter
+    class ValidationError < StandardError
+    end
+
+    #
+    # Includer must define these methods:
+    #
+    #   .schema_path
+    #   #from_hash(hash)
+    #   #to_hash
+    #
+
+    module HasJsonSchema
+      def self.schemas
+        @schemas ||= []
+      end
+
+      def self.included(base)
+        base.extend ClassMethods
+      end
+
+      def to_json(*args)
+        to_hash.to_json(*args)
+      end
+
+      def as_json(*args)
+        to_hash
+      end
+
+      module ClassMethods
+        attr_reader :schema
+
+        def schema_path(path)
+          @schema = MultiJson.decode(open(path).read)
+          HasJsonSchema.schemas << @schema
+        end
+
+        def schema
+          @schema or raise "schema must be set with #{self}.schema_path"
+        end
+
+        def kind
+          @kind ||= schema['properties']['kind']['default']
+        end
+
+        def description
+          @description ||= schema['description']
+        end
+
+        def properties
+          @properties ||= (
+            schema['properties'].map do |name, data|
+              Field.new(name, !!data['required'], data['type'], data['description'] || 'unknown')
+            end
+          )
+        end
+
+        # TODO: remove #fields usage
+        alias_method :fields, :properties
+
+        def from_json(str)
+          data = MultiJson.decode(str)
+
+          case data
+          when Array
+            data.map { |e| from_hash validate!(e) }
+          when Hash
+            from_hash validate!(data)
+          else
+            raise TypeError, "expected Array or Hash, got #{data.inspect}:#{data.class}"
+          end
+        end
+
+        def validate!(e)
+          Jschematic.validate!(e, schema, :context => HasJsonSchema.schemas, :debug => true)
+          e
+        rescue Jschematic::ValidationError => ex
+          raise ValidationError, "#{ex.message}: #{e.inspect}"
+        end
+      end
+
+    end
+  end
+end