clasrip 0.1.0 → 0.1.1

data/bin/clasrip

@@ -1,37 +1,33 @@
 #!/usr/bin/env ruby
-$:.push("./lib")
 
-if $0 == __FILE__
-  require "clasrip"
-  require "clasrip/sql"
-  require "date"
+require "clasrip"
+require "clasrip/sql"
+require "date"
 
-  if ARGV.size >= 2 and ARGV[0] == "rip"
-    scraper = Clasrip::Scraper.new(1971, Date.today.year + 1)
-	sql = Clasrip::SQL.new(ARGV[1])
-	
-	last_record = Clasrip::SQL::Classification.last
-	if last_record != nil
-      last_record = last_record.attributes
-      last_record.delete(:id)
-	  last_record[:date_of_classification] = last_record[:date_of_classification].iso8601
-    
-      date = Clasrip::SQL::Classification.last.date_of_classification
-      scraper.set_date(date.year, date.month-1, 1)
-      #puts "Set date to: #{scraper.get_date}"
+if ARGV.size >= 2 and ARGV[0] == "rip"
+  scraper = Clasrip::Scraper.new(1971, Date.today.year + 1)
+  sql = Clasrip::SQL.new(ARGV[1])
 
-      print "Finding last record (#{last_record[:title]})... "
-	  scraper.each do |record|
-		break if record == last_record
-	  end
-	  puts "Found!"
-	end
+  last_record = Clasrip::SQL::Classification.last
+  if last_record != nil
+    last_record = last_record.attributes
+    last_record.delete(:id)
+    last_record[:date_of_classification] = last_record[:date_of_classification].iso8601
+  
+    date = Clasrip::SQL::Classification.last.date_of_classification
+    scraper.set_date(date.year, date.month-1, 1)
 
-	date = scraper.get_date
-	scraper.each do |record|
-		puts "(#{record[:date_of_classification]}) #{record[:title]} [#{record[:classification]}]"
-		sql.add_record(record)
-	end
+    print "Finding last record (#{last_record[:title]})... "
+    scraper.each do |record|
+      break if record == last_record
+    end
+    puts "Found!"
+  end
+
+  date = scraper.get_date
+  scraper.each do |record|
+    puts "(#{record[:date_of_classification]}) #{record[:title]} [#{record[:classification]}]"
+    sql.add_record(record)
   end
 end
 

data/clasrip.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "clasrip"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brendan Molloy"]
-  s.date = "2012-02-25"
+  s.date = "2012-02-26"
   s.description = "A scraper for classification.gov.au"
   s.email = "brendan@bbqsrc.net"
   s.executables = ["clasrip"]

data/lib/clasrip.rb

@@ -5,7 +5,7 @@ module Clasrip
   module Version
     MAJOR = 0
     MINOR = 1
-    PATCH = 0
+    PATCH = 1
     BUILD = nil
     def self.to_s
       [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
@@ -111,6 +111,16 @@ module Clasrip
       end
     end
     
+    def ensure_correct_encoding(s)
+      s.force_encoding("utf-8")
+      return s if s.valid_encoding?
+
+      puts ("Invalid: " + s)
+      s.encode!("utf-8", "iso-8859-1")
+      raise "Could not enforce UTF-8 encoding: '#{s}'" unless s.valid_encoding?
+      s
+    end
+
     def new_enum
       @records = Enumerator.new do |y|
         @dates[0].each do |first_date|
@@ -133,6 +143,9 @@ module Clasrip
             parse_table(table).each do |record|
               form = get_classification(record[:original_url]) or next
               record.merge!(parse_classification(form))
+              record.each_pair do |k,v|
+                record[k] = ensure_correct_encoding(v)
+              end
               y << record
             end
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: clasrip
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-25 00:00:00.000000000 Z
+date: 2012-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70095153397460 !ruby/object:Gem::Requirement
+  requirement: &70149198193640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 1.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *70095153397460
+  version_requirements: *70149198193640
 - !ruby/object:Gem::Dependency
   name: data_mapper
-  requirement: &70095153396980 !ruby/object:Gem::Requirement
+  requirement: &70149198192960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *70095153396980
+  version_requirements: *70149198192960
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &70095153396500 !ruby/object:Gem::Requirement
+  requirement: &70149198192360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70095153396500
+  version_requirements: *70149198192360
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &70095153395980 !ruby/object:Gem::Requirement
+  requirement: &70149198191660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '3.12'
   type: :development
   prerelease: false
-  version_requirements: *70095153395980
+  version_requirements: *70149198191660
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70095153395480 !ruby/object:Gem::Requirement
+  requirement: &70149198190640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70095153395480
+  version_requirements: *70149198190640
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70095153394980 !ruby/object:Gem::Requirement
+  requirement: &70149198206040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 1.8.3
   type: :development
   prerelease: false
-  version_requirements: *70095153394980
+  version_requirements: *70149198206040
 description: A scraper for classification.gov.au
 email: brendan@bbqsrc.net
 executables:
@@ -112,7 +112,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3829822103400611157
+      hash: -1633970238896182400
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: