hdo-storting-importer 0.1.5 → 0.1.6

data/hdo-storting-importer.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "unicode_utils"
   gem.add_runtime_dependency "multi_json"
   gem.add_runtime_dependency "yajl-ruby"
+  gem.add_runtime_dependency "roo"
   gem.add_runtime_dependency "jschematic", ">= 0.1.0"
 
   gem.add_development_dependency 'pry'

data/lib/hdo/storting_importer/fusion_table.rb

@@ -7,7 +7,7 @@ module Hdo
 
     class FusionTable
       def initialize(api_key)
-        @api_key  = api_key
+        @api_key = api_key
       end
 
       def query(sql, opts = {})

data/lib/hdo/storting_importer/promise.rb

@@ -1,6 +1,6 @@
 # encoding: UTF-8
 
-require 'csv'
+require 'roo'
 
 module Hdo
   module StortingImporter
@@ -8,8 +8,7 @@ module Hdo
       include HasJsonSchema
       include IvarEquality
 
-      attr_reader :party, :body, :general, :categories, :source, :page
-      attr_accessor :external_id
+      attr_reader :external_id, :party, :body, :general, :categories, :source, :page, :date
       alias_method :general?, :general
       alias_method :short_inspect, :inspect
 
@@ -23,7 +22,8 @@ module Hdo
           true,
           ["GRUNNSKOLE"],
           "PP",
-          8
+          8,
+          '2009-06-01'
         )
       end
 
@@ -31,23 +31,70 @@ module Hdo
         Util.json_pretty example
       end
 
-      # doesn't really belong here - data source?
-      def self.from_fusion_table(table_id, api_key)
-        table = FusionTable.new(api_key)
-
-        column_names = table.columns_for(table_id).map { |e| e['name'] }.map { |e| e.inspect }.join(",")
-        row_count = Integer(table.query("select count(rowid) from #{table_id}", :rows => true).flatten.first)
-
-        # do this in batches of 100
-        limit = 100
-        result = []
-
-        (0..row_count).step(limit) do |offset|
-          sql = "SELECT #{column_names},\"rowid\" FROM #{table_id} OFFSET #{offset} LIMIT #{limit}"
-          result.concat table.query(sql, :rows => true).map { |data| new(*data) }
+      def self.from_xlsx(path)
+        table  = Excelx.new(path)
+        raise "empty spreadsheet" unless table.first_row
+
+        errors = []
+
+        # 0 is empty, 1 is header
+        promises = 2.upto(table.last_row).map { |idx|
+          data = table.row(idx)
+
+          Hdo::StortingImporter.logger.info "promise row: #{data.inspect}"
+
+          external_id = data.fetch(0).to_i.to_s
+          date        = data.fetch(1)
+          party       = data.fetch(2)
+          body        = data.fetch(3)
+          general     = data.fetch(4).to_s.strip.downcase
+          categories  = data.fetch(5)
+          source      = data.fetch(6)
+          page        = data.fetch(7)
+
+          unless %w[ja nei].include?(general)
+            errors << "row #{external_id}: unexpected value #{general.inspect} for general"
+            next
+          end
+
+          if page.to_s !~ /^\d+(\.\d+)?$/ || page.to_i == 0
+            errors << "row #{external_id}: unexpected value #{page.inspect} for page"
+            next
+          end
+
+          unless party
+            errors << "row #{external_id}: party missing"
+            next
+          end
+
+          unless body =~ /\.$/
+            body << "."
+          end
+
+          promise = new external_id,
+                        party,
+                        body,
+                        general.downcase == "ja",
+                        categories,
+                        source,
+                        Integer(page),
+                        date
+
+          begin
+            promise.validate!
+          rescue Hdo::StortingImporter::ValidationError => ex
+            errors << "row #{external_id}: #{ex.message}"
+            next
+          end
+
+          promise
+        }.compact
+
+        if errors.any?
+          raise "found errors:\n#{errors.join("\n")}"
         end
 
-        result
+        promises
       end
 
       def self.from_hash(hash)
@@ -57,12 +104,13 @@ module Hdo
                  hash['general'],
                  hash['categories'],
                  hash['source'],
-                 hash['page']
+                 hash['page'],
+                 hash['date']
 
         pr
       end
 
-      def initialize(external_id, party, body, general, categories, source, page)
+      def initialize(external_id, party, body, general, categories, source, page, date)
         @external_id = external_id
         @party       = strip_if_string(party)
         @body        = strip_if_string(body)
@@ -70,10 +118,11 @@ module Hdo
         @categories  = clean_categories(categories)
         @source      = strip_if_string(source)
         @page        = page
+        @date        = date
       end
 
       def to_hash
-        h = {
+        {
           'kind'       => self.class.kind,
           'externalId' => @external_id,
           'party'      => @party,
@@ -81,10 +130,9 @@ module Hdo
           'categories' => @categories,
           'source'     => @source,
           'page'       => @page,
-          'body'       => @body
+          'body'       => @body,
+          'date'       => @date
         }
-
-        h
       end
 
       private

data/lib/hdo/storting_importer/schema/promise.json

@@ -32,13 +32,23 @@
     },
     "source": {
       "type": "string",
-      "description": "The source of the promise. (TODO: this should always be a URL)",
+      "description": "The source of the promise.",
       "required": true
     },
+    "page": {
+      "type": "number",
+      "description": "What page in the source the promise was found on"
+    },
     "body": {
       "type": "string",
       "description": "The body text of the promise.",
       "required": true
+    },
+    "date": {
+      "type": "string",
+      "description": "The date the promise was made.",
+      "format": "date",
+      "required": true
     }
   }
 }

data/lib/hdo/storting_importer/schema/representative.json

@@ -55,7 +55,7 @@
     "voteResult": {
       "type": "string",
       "description": "If this representative object is part of a hdo#vote, this property describes how the representative voted. Valid values are for|against|absent.",
-      "format": "for|against|absent"
+      "pattern": "for|against|absent"
     }
   }
 }

data/lib/hdo/storting_importer/version.rb

@@ -1,5 +1,5 @@
 module Hdo
   module StortingImporter
-    VERSION = "0.1.5"
+    VERSION = "0.1.6"
   end
 end

data/spec/hdo/storting_importer/promise_spec.rb

@@ -18,23 +18,24 @@ module Hdo
           "categories": ["GRUNNSKOLE"],
           "source": "PP",
           "page": 8,
-          "body": "Stille strengere krav til orden og oppførsel for å hindre at uro ødelegger undervisningen."
+          "body": "Stille strengere krav til orden og oppførsel for å hindre at uro ødelegger undervisningen.",
+          "date": "2009-06-01"
         }
         JSON
       end
 
       it 'strips trailing space from the body' do
-        promise = Promise.new("1", "Party", "Body   ", true, ["æøå"], "PP", 8)
+        promise = Promise.new("1", "Party", "Body   ", true, ["æøå"], "PP", 8, '2012-06-01')
         promise.body.should == "Body"
       end
 
       it 'correctly upcases non-ASCII category names' do
-        promise = Promise.new("1", "Party", "Body", true, ["æøå"], "PP", 8)
+        promise = Promise.new("1", "Party", "Body", true, ["æøå"], "PP", 8, '2012-06-01')
         promise.categories.should == ["ÆØÅ"]
       end
 
       it 'ignores empty categories' do
-        promise = Promise.new("1", "Party", "Body", true, ["FOO", ""], "PP", 8)
+        promise = Promise.new("1", "Party", "Body", true, ["FOO", ""], "PP", 8, '2012-06-01')
         promise.categories.should == ["FOO"]
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hdo-storting-importer
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
   prerelease: 
 platform: ruby
 authors:
@@ -139,6 +139,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: jschematic
   requirement: !ruby/object:Gem::Requirement