hdo-storting-importer 0.0.8 → 0.0.9

data/lib/hdo/storting_importer/issue.rb

@@ -3,19 +3,14 @@
 module Hdo
   module StortingImporter
     class Issue
+      include HasJsonSchema
       include IvarEquality
       include Inspectable
 
       attr_reader :external_id, :summary, :description, :type, :status, :last_update,
                   :reference, :document_group, :committee, :categories
 
-      def self.type_name
-        'issue'
-      end
-
-      def self.description
-        'a parliament issue'
-      end
+      schema_path StortingImporter.lib.join("hdo/storting_importer/schema/issue.json").to_s
 
       def self.example
         new(
@@ -32,23 +27,8 @@ module Hdo
         )
       end
 
-      def self.xml_example(builder = Util.builder)
-        example.to_hdo_xml(builder)
-      end
-
-      def self.fields
-        [
-          EXTERNAL_ID_FIELD,
-          Field.new(:summary,       true,  :string, 'A (preferably one-line) summary of the issue.'),
-          Field.new(:description,   true,  :string, 'A longer description of the issue.'),
-          Field.new(:type,          true,  :string, 'The type of issue.'),
-          Field.new(:status,        true,  :string, 'The status of the issue.'),
-          Field.new(:lastUpdate,    true,  :string, 'The time the issue was last updated in the parliament.'),
-          Field.new(:reference,     true,  :string, 'A reference.'),
-          Field.new(:documentGroup, true,  :string, 'What document group this issue belongs to.'),
-          Field.new(:committee,     false, :string, "What committee this issue belongs to. Should match the 'name' field in the committee type."),
-          Field.new(:categories,    false, 'list',  "List of categories (matching the 'name' field of the category type).")
-        ]
+      def self.json_example
+        Util.json_pretty example
       end
 
       def self.from_storting_doc(doc)
@@ -83,23 +63,17 @@ module Hdo
         raise
       end
 
-      def self.from_hdo_doc(doc)
-        doc.css("issues > issue").map { |e| from_hdo_node(e) }
-      end
-
-      def self.from_hdo_node(node)
-        external_id    = node.css("externalId").first.text
-        summary        = node.css("summary").first.text
-        description    = node.css("description").first.text
-        type           = node.css("type").first.text
-        status         = node.css("status").first.text
-        last_update    = node.css("lastUpdate").first.text
-        reference      = node.css("reference").first.text
-        document_group = node.css("documentGroup").first.text
-        committee      = node.css("committee").first.text
-        categories     = node.css("categories category").map { |e| e.text }
-
-        new external_id, summary, description, type, status, last_update, reference, document_group, committee, categories
+      def self.from_hash(hash)
+        new hash.fetch('externalId'),
+            hash.fetch('summary'),
+            hash.fetch('description'),
+            hash.fetch('type'),
+            hash.fetch('status'),
+            hash.fetch('lastUpdate'),
+            hash.fetch('reference'),
+            hash.fetch('documentGroup'),
+            hash.fetch('committee'),
+            hash.fetch('categories')
       end
 
       def initialize(external_id, summary, description, type, status, last_update,
@@ -120,25 +94,20 @@ module Hdo
         short_inspect_string :include => [:external_id, :summary]
       end
 
-      def to_hdo_xml(builder = Util.builder)
-        builder.issue do |i|
-          i.externalId external_id
-          i.summary summary
-          i.description description
-          i.type type
-          i.status status
-          i.lastUpdate last_update
-          i.reference reference
-          i.documentGroup document_group
-          i.committee(committee) if committee
-
-          if categories.any?
-            i.categories do |cats|
-              categories.each { |e| cats.category e }
-            end
-          end
-        end
-
+      def to_hash
+        {
+          :kind           => self.class.kind,
+          :externalId     => @external_id,
+          :summary        => @summary,
+          :description    => @description,
+          :type           => @type,
+          :status         => @status,
+          :lastUpdate     => @last_update,
+          :reference      => @reference,
+          :documentGroup  => @document_group,
+          :committee      => @committee,
+          :categories     => @categories
+        }
       end
 
     end

data/lib/hdo/storting_importer/party.rb

@@ -1,29 +1,20 @@
 module Hdo
   module StortingImporter
     class Party
+      include HasJsonSchema
       include IvarEquality
 
       attr_reader :external_id, :name
       alias_method :short_inspect, :inspect
 
-      def self.type_name
-        'party'
-      end
-
-      def self.description
-        'a political party'
-      end
+      schema_path StortingImporter.lib.join("hdo/storting_importer/schema/party.json").to_s
 
       def self.example
         new("DEM", "Democratic Party")
       end
 
-      def self.xml_example(builder = Util.builder)
-        example.to_hdo_xml(builder)
-      end
-
-      def self.fields
-        [EXTERNAL_ID_FIELD, Field.new(:name, true, :string, 'The name of the party.')]
+      def self.json_example
+        Util.json_pretty example
       end
 
       def self.from_storting_doc(doc)
@@ -32,12 +23,8 @@ module Hdo
         end
       end
 
-      def self.from_hdo_doc(doc)
-        doc.css("parties > party").map { |e| from_hdo_node(e) }
-      end
-
-      def self.from_hdo_node(node)
-        new node.css("externalId").first.text, node.css("name").first.text
+      def self.from_hash(hash)
+        new hash.fetch('externalId'), hash.fetch('name')
       end
 
       def initialize(external_id, name)
@@ -49,12 +36,14 @@ module Hdo
         Util.unescape_param @external_id
       end
 
-      def to_hdo_xml(builder = Util.builder)
-        builder.party do |party|
-          party.externalId external_id
-          party.name name
-        end
+      def to_hash
+        {
+          :kind       => self.class.kind,
+          :externalId => @external_id,
+          :name       => @name
+        }
       end
+
     end
   end
 end

data/lib/hdo/storting_importer/promise.rb

@@ -5,49 +5,25 @@ require 'csv'
 module Hdo
   module StortingImporter
     class Promise
+      include HasJsonSchema
+      include IvarEquality
+
       attr_reader :party, :body, :general, :categories, :source, :page
+      attr_accessor :external_id
       alias_method :general?, :general
       alias_method :short_inspect, :inspect
 
-      def self.type_name
-        'promise'
-      end
-
-      def self.description
-        'a party promise'
-      end
-
-      def self.fields
-        [
-          Field.new(:party, true, :string, 'The external id of the party.'),
-          Field.new(:general, true, :boolean, "Whether this is considered a general promise (i.e. can be ambigious whether it has been fulfilled)."),
-          Field.new(:categories, true, :list, "List of category names (matching names imported in <a href='#input-format-category'>&lt;category&gt;</a>)"),
-          Field.new(:source, true, :string, "The source of the promise. (TODO: this should always be a URL)"),
-          Field.new(:body, true, :string, "The body text of the promise."),
-        ]
-      end
+      schema_path StortingImporter.lib.join("hdo/storting_importer/schema/promise.json").to_s
 
       def self.example
-        new("H", "Stille strengere krav til orden og oppførsel for å hindre at uro ødelegger undervisningen.", true, ["GRUNNSKOLE"], "PP", 8)
-      end
+        pr = new("H", "Stille strengere krav til orden og oppførsel for å hindre at uro ødelegger undervisningen.", true, ["GRUNNSKOLE"], "PP", 8)
+        pr.external_id = "1"
 
-      def self.xml_example(builder = Util.builder)
-        example.to_hdo_xml(builder)
+        pr
       end
 
-      def self.from_hdo_doc(doc)
-        doc.css("promises > promise").map { |e| from_hdo_node(e) }
-      end
-
-      def self.from_hdo_node(node)
-        source, page = node.css("source").first.text.split(":")
-
-        new node.css("party").first.text,
-            node.css("body").first.text,
-            node.css("general").first.text == "true",
-            node.css("categories > category").map { |e| e.text },
-            source,
-            page
+      def self.json_example
+        Util.json_pretty example
       end
 
       def self.from_csv(str)
@@ -79,34 +55,63 @@ module Hdo
         end.compact
       end
 
+      # doesn't really belong here - data source?
       def self.from_fusion_table(table_id, api_key)
-        url = "https://www.googleapis.com/fusiontables/v1/query"
+        table = FusionTable.new(api_key)
+
+        column_names = table.columns_for(table_id).map { |e| e['name'] }.map { |e| e.inspect }.join(",")
+        row_count = Integer(table.query("select count(rowid) from #{table_id}", :rows => true).flatten.first)
 
-        resp = RestClient.get(url, :params => {:sql => "select * from #{table_id}", :key => api_key})
-        MultiJson.decode(resp).fetch('rows').map { |data| new(*data) }
-      rescue RestClient::RequestFailed => ex
-        raise "#{ex.message}: #{ex.http_body}"
+        # do this in batches of 100
+        limit = 100
+        result = []
+
+        (0..row_count).step(limit) do |offset|
+          sql = "SELECT #{column_names},\"rowid\" FROM #{table_id} OFFSET #{offset} LIMIT #{limit}"
+          result.concat table.query(sql, :rows => true).map { |data| new(*data) }
+        end
+
+        result
+      end
+
+      def self.from_hash(hash)
+        pr = new hash.fetch('party'),
+                 hash.fetch('body'),
+                 hash.fetch('general'),
+                 hash.fetch('categories'),
+                 hash.fetch('source'),
+                 hash.fetch('page')
+
+        pr.external_id = hash['externalId']
+
+        pr
       end
 
       def initialize(party, body, general, categories, source, page)
-        @party      = party
-        @body       = body
-        @general    = general
-        @categories = clean_categories(categories)
-        @source     = source
-        @page       = page
+        @party       = party.strip
+        @body        = body.strip
+        @general     = general
+        @categories  = clean_categories(categories)
+        @source      = source.strip
+        @page        = page
+
+        @external_id = nil
       end
 
-      def to_hdo_xml(builder = Util.builder)
-        builder.promise do |promise|
-          promise.party party
-          promise.general general?
-          promise.categories do |cats|
-            categories.each { |e| cats.category e }
-          end
-          promise.source [source, page].join(":")
-          promise.body body
-        end
+      def to_hash
+        h = {
+          :kind       => self.class.kind,
+          :party      => @party,
+          :general    => @general,
+          :categories => @categories,
+          :source     => @source,
+          :page       => @page,
+          :body       => @body
+        }
+
+        h[:externalId] = @external_id if @external_id
+
+        h
       end
 
       private

data/lib/hdo/storting_importer/proposition.rb

@@ -0,0 +1,61 @@
+module Hdo
+  module StortingImporter
+    class Proposition
+      include HasJsonSchema
+      include IvarEquality
+      include Inspectable
+
+      attr_reader :external_id, :description, :on_behalf_of, :body, :delivered_by
+
+      schema_path StortingImporter.lib.join("hdo/storting_importer/schema/proposition.json").to_s
+
+      def self.example
+        new('1234', 'description', 'on behalf of', 'body', Representative.example)
+      end
+
+      def self.json_example
+        Util.json_pretty example
+      end
+
+      def self.from_hash(hash)
+        arr = [
+          hash.fetch('externalId'),
+          hash.fetch('description'),
+          hash.fetch('onBehalfOf'),
+          hash.fetch('body')
+        ]
+
+        delivered_by = hash['deliveredBy']
+        arr << Representative.from_hash(delivered_by) if delivered_by
+
+        new(*arr)
+      end
+
+      def initialize(external_id, description, on_behalf_of, body, delivered_by)
+        @external_id  = external_id
+        @description  = description
+        @on_behalf_of = on_behalf_of
+        @body         = body
+        @delivered_by = delivered_by
+      end
+
+      def short_inspect
+        short_inspect_string :include => [:external_id, :description, :on_behalf_of]
+      end
+
+      def to_hash
+        h = {
+          :kind        => self.class.kind,
+          :externalId  => @external_id,
+          :description => @description,
+          :onBehalfOf  => @on_behalf_of,
+          :body        => @body
+        }
+
+        h[:deliveredBy] = @delivered_by.to_hash if @delivered_by
+
+        h
+      end
+    end # Proposition
+  end
+end

data/lib/hdo/storting_importer/representative.rb

@@ -3,6 +3,7 @@
 module Hdo
   module StortingImporter
     class Representative
+      include HasJsonSchema
       include IvarEquality
       include Inspectable
 
@@ -11,13 +12,7 @@ module Hdo
 
       attr_accessor :vote_result
 
-      def self.type_name
-        'representative'
-      end
-
-      def self.description
-        'a member of parliament'
-      end
+      schema_path StortingImporter.lib.join("hdo/storting_importer/schema/representative.json").to_s
 
       def self.example
         new(
@@ -34,22 +29,8 @@ module Hdo
         )
       end
 
-      def self.xml_example(builder = Util.builder)
-        example.to_hdo_xml(builder)
-      end
-
-      def self.fields
-        [
-          EXTERNAL_ID_FIELD,
-          Field.new(:firstName, true, :string, 'The first name of the representative.'),
-          Field.new(:lastName, true, :string, 'The last name of the representative.'),
-          Field.new(:period, true, :string, "An identifier for the period the representative is elected for."),
-          Field.new(:district, true, :string, "The electoral district the representative belongs to. Must match the 'name' field of the district type."),
-          Field.new(:party, true, :string, "The name of the representative's party."),
-          Field.new(:committee, true, :list, "A (possibly empty) list of committees the representative is a member of. This should match the 'name' field of the committee type."),
-          Field.new(:dateOfBirth, true, :string, "The representative's birth date."),
-          Field.new(:dateOfDeath, false, :string, "The representative's death date."),
-        ]
+      def self.json_example
+        Util.json_pretty example
       end
 
       def self.from_storting_doc(doc)
@@ -83,32 +64,21 @@ module Hdo
         )
       end
 
-      def self.from_hdo_doc(doc)
-        doc.css("representatives > representative").map { |e| from_hdo_node e }
-      end
-
-      def self.from_hdo_node(node)
-        district_node = node.css("district").first
-        district      = district_node ? district_node.text : ''
-
-        party_node = node.css("party").first
-        party      = party_node ? party_node.text : ''
-
-        rep = new node.css("externalId").first.text,
-                  node.css("firstName").first.text,
-                  node.css("lastName").first.text,
-                  node.css("gender").first.text,
-                  node.css("dateOfBirth").first.text,
-                  node.css("dateOfDeath").first.text,
-                  district,
-                  party,
-                  node.css("committees committee").map { |e| e.text.strip },
-                  node.css("period").first.text
-
-        result_node = node.css("voteResult").first
-        rep.vote_result = result_node.text if result_node
-
-        rep
+      def self.from_hash(hash)
+        v = new hash.fetch('externalId'),
+                hash.fetch('firstName'),
+                hash.fetch('lastName'),
+                hash.fetch('gender'),
+                hash.fetch('dateOfBirth'),
+                hash.fetch('dateOfDeath'),
+                hash.fetch('district'),
+                hash.fetch('party'),
+                hash.fetch('committees'),
+                hash.fetch('period')
+
+        v.vote_result = hash['voteResult']
+
+        v
       end
 
       def initialize(external_id, first_name, last_name, gender, date_of_birth, date_of_death, district, party, committees, period)
@@ -134,27 +104,24 @@ module Hdo
         Util.unescape_param @external_id
       end
 
-      def to_hdo_xml(builder = Util.builder)
-        builder.representative do |rep|
-          rep.externalId external_id
-          rep.firstName first_name
-          rep.lastName last_name
-          rep.gender gender
-          rep.dateOfBirth date_of_birth
-          rep.dateOfDeath date_of_death
-          rep.district district
-          rep.party party
-
-          rep.committees do |coms|
-            committees.each { |e| coms.committee e }
-          end
-
-          rep.period period
-
-          if vote_result
-            rep.voteResult vote_result
-          end
-        end
+      def to_hash
+        h = {
+          :kind        => self.class.kind,
+          :externalId  => @external_id,
+          :firstName   => @first_name,
+          :lastName    => @last_name,
+          :gender      => @gender,
+          :dateOfBirth => @date_of_birth,
+          :dateOfDeath => @date_of_death,
+          :district    => @district,
+          :party       => @party,
+          :committees  => @committees,
+          :period      => @period
+        }
+
+        h[:voteResult] = @vote_result if @vote_result
+
+        h
       end
 
     end

data/lib/hdo/storting_importer/schema/category.json

@@ -0,0 +1,28 @@
+{
+  "id": "hdo#category",
+  "description": "a parliamentary category, used to categorize issues and promises",
+  "type":"object",
+  "properties": {
+    "kind": {
+      "type": "string",
+      "description": "This is always 'hdo#category'",
+      "default": "hdo#category",
+      "required": true
+    },
+    "externalId": {
+      "type": "string"
+    },
+    "name": {
+      "description": "The name of the category",
+      "type": "string",
+      "required": true
+    },
+    "subCategories": {
+      "description": "A list of subcategories.",
+      "type":"array",
+      "items":{
+        "$ref": "hdo#category"
+      }
+    }
+  }
+}

data/lib/hdo/storting_importer/schema/committee.json

@@ -0,0 +1,21 @@
+{
+  "id": "hdo#committee",
+  "description": "a parliamentary committe",
+  "type":"object",
+  "properties": {
+    "kind": {
+        "type": "string",
+        "description": "This is always 'hdo#committee'",
+        "default": "hdo#committee",
+        "required": true
+    },
+    "externalId": {
+      "type": "string"
+    },
+    "name": {
+      "description": "The name of the committee",
+      "type": "string",
+      "required": true
+    }
+  }
+}

data/lib/hdo/storting_importer/schema/district.json

@@ -0,0 +1,21 @@
+{
+  "id": "hdo#district",
+  "description": "an electoral district",
+  "type":"object",
+  "properties": {
+    "kind": {
+        "type": "string",
+        "description": "This is always 'hdo#district'",
+        "default": "hdo#district",
+        "required": true
+    },
+    "externalId": {
+      "type": "string"
+    },
+    "name": {
+      "description": "The name of the district",
+      "type": "string",
+      "required": true
+    }
+  }
+}

data/lib/hdo/storting_importer/schema/issue.json

@@ -0,0 +1,62 @@
+{
+  "id": "hdo#issue",
+  "description": "a parliament issue",
+  "type":"object",
+  "properties": {
+    "kind": {
+      "type": "string",
+      "description": "This is always 'hdo#issue'",
+      "default": "hdo#issue",
+      "required": true
+    },
+    "externalId": {
+      "type": "string",
+      "description": "",
+      "required": false
+    },
+    "summary": {
+      "type": "string",
+      "description": "A (preferably one-line) summary of the issue.",
+      "required": true
+    },
+    "description": {
+      "type": "string",
+      "description": "A longer description of the issue.",
+      "required": true
+    },
+    "type": {
+      "type": "string",
+      "description": "The type of issue.",
+      "required": true
+    },
+    "status": {
+      "type": "string",
+      "description": "The status of the issue.",
+      "required": true
+    },
+    "lastUpdate": {
+      "type": "string",
+      "description": "The date the issue was last updated in the parliament.",
+      "required": true
+    },
+    "reference": {
+      "type": "string",
+      "description": "A reference",
+      "required": true
+    },
+    "documentGroup": {
+      "type": "string",
+      "description": "What document group this issue belongs to.",
+      "required": true
+      },
+    "committee": {
+      "type": "string",
+      "description": "What committee this issue belongs to. Should match the 'name' field in the committee type."
+    },
+    "categories": {
+      "type": "array",
+      "description": "List of categories (matching the 'name' field of the category type).",
+      "items": { "type": "string" }
+    }
+  }
+}