eeml 0.0.25 → 0.0.29

data/.gitignore

@@ -6,3 +6,4 @@ eeml.gemspec
 client_config.yml
 coverage/
 .*.swp
+gem_deps_manual/

data/CHANGELOG

@@ -1,3 +1,11 @@
+commit 750283de4b56996f05adae5bc02f992c56af4ef5
+Author: Paul Bellamy <paul@connectedenvironments.com>
+Date:   Mon Jan 17 10:30:15 2011 +0000
+
+    Version 0.0.29
+    
+      Fix for JSON parsing when receiving bad data
+
 commit c4e0c43b3600256942fddd26d4e77d33549299cb
 Author: Levent Ali <lebreeze@gmail.com>
 Date:   Fri Oct 8 11:18:09 2010 +0000

data/ci/build_hudson.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+set -e
+set -u
+set -x
+
+echo "Build script started $0 at `date`"
+echo "dir is `pwd`"
+outdir=build_artefacts
+
+mkdir -p $outdir
+
+custom_gem_dir=./gem_deps_manual
+export GEM_PATH=$custom_gem_dir
+
+set +e
+~/sysadmin/show_platform/show_platform.sh > $outdir/platform.txt 2>&1
+set -e
+
+#this only really needs to be done initially, or on change to dependencies
+gem install --install-dir $custom_gem_dir -y mocha jeweler libxml-ruby lightcsv yajl-ruby --no-rdoc --no-ri
+
+set +e
+rake test > $outdir/rspec.log
+spec_rc=$?
+set -e
+
+exit $spec_rc
+

data/lib/eeml/csv_parser_v1.rb

@@ -2,8 +2,8 @@ module Eeml
   class CsvParserV1
 
     def self.make_environment_from_csv(csv_content)
-      csv = LightCsv.parse(csv_content)
-      environment = Environment.new
+      csv = LightCsv.parse(strip_content(csv_content))
+      environment = Environment.new(:csv_version => 'v1')
 
       raise(CsvEncodingError, "Currently Pachube can only accept csv for your most recent set of values. You have submitted #{csv.size} rows of data.") unless csv.size == 1
 
@@ -12,6 +12,5 @@ module Eeml
       end
       environment
     end
-
   end
 end

data/lib/eeml/csv_parser_v2.rb

@@ -4,8 +4,10 @@ module Eeml
   class CsvParserV2
 
     def self.make_environment_from_csv(csv_content)
-      csv = LightCsv.parse(csv_content)
-      environment = Environment.new
+      csv = LightCsv.parse(strip_content(csv_content))
+      environment = Environment.new(:csv_version => 'v2')
+
+      csv = strip_content(csv)
 
       csv.each do |datastream|
         raise(CsvEncodingError, "CSV is invalid. Double check you are properly encoding line breaks.") if datastream.length == 3 && datastream[1].include?('\n')
@@ -14,6 +16,5 @@ module Eeml
       end
       environment
     end
-
   end
 end

data/lib/eeml/environment.rb

@@ -99,6 +99,7 @@ module Eeml
     attr_accessor :identifier, :updated, :creator
     attr_accessor :title, :description, :feed_url, :website, :email, :icon, :status
     attr_accessor :location
+    attr_accessor :csv_version
     attr_accessor :datastreams
     attr_accessor :tags
     attr_writer :private
@@ -116,6 +117,7 @@ module Eeml
       @email = options[:email]
       @icon = options[:icon]
       @private = options[:private]
+      @csv_version = options[:csv_version]
       @tags = []
     end
 
@@ -144,8 +146,8 @@ module Eeml
     end
 
     # Create a new Environment object from a CSV string
-    def self.new_from_csv(csv_str, version)
-      EnvironmentBuilder.build_from_csv(csv_str, version)
+    def self.new_from_csv(csv_str, version = nil)
+      EnvironmentBuilder.build_from_csv(csv_str, detect_csv_version(version, csv_str))
     end
   
     def update_datastreams_from_csv_values!(csv_values)
@@ -178,6 +180,16 @@ module Eeml
         return false
       end
     end
+
+    private
+
+    def self.detect_csv_version(version, csv)
+      return version if version
+      parsed_csv = LightCsv.parse(strip_content(csv))
+      return :v2 if parsed_csv.size >= 2
+      return :v1 if parsed_csv.size == 1 && parsed_csv.first.length != 2
+      raise(CsvEncodingError, "CSV version could not be detected")
+    end
     
   end
 

data/lib/eeml/environment_builder.rb

@@ -35,13 +35,15 @@ module Eeml
       if json["version"].to_i == Constants::EEML['0.5.0'][:version].to_i
         parser = JsonEnvironmentParserV005.new
       elsif json["version"] == Constants::JSON_API['1.0.0'][:version]
-	parser = JsonEnvironmentParserV100.new
+        parser = JsonEnvironmentParserV100.new
       elsif json["version"].to_f == Constants::JSON_API['0.6-alpha'][:version].to_f
         parser = JsonEnvironmentParserV006.new
       else
-        raise JSON::ParserError, "Invalid version specification. Permitted versions are #{Constants::EEML['0.5.0'][:version]}, #{Constants::JSON_API["0.6-alpha"][:version]} and #{Constants::JSON_API["1.0.0"][:version]}"
+        raise # raise something so we jump to the rescue block below
       end
       return parser.make_environment_from_hash(json)
+    rescue
+      raise JSON::ParserError, "Invalid version specification. Permitted versions are #{Constants::EEML['0.5.0'][:version]}, #{Constants::JSON_API["0.6-alpha"][:version]} and #{Constants::JSON_API["1.0.0"][:version]}"
     end
 
     def self.build_from_csv(csv_str, version = :v1)

data/lib/eeml/helpers.rb

@@ -0,0 +1,7 @@
+def strip_content(content)
+  if content.kind_of? Array
+    content.collect { |i| strip_content(i) }.delete_if { |i| i == [] }
+  else
+    content.respond_to?(:strip) ? content.strip : content
+  end
+end

data/lib/eeml/json_environment_parser_v005.rb

@@ -24,7 +24,7 @@ module Eeml
     end
 
     def make_environment_from_json(json_str)
-      env_hash = JSON.parse(json_str)
+      env_hash = JSON.parse(strip_content(json_str))
       return make_environment_from_hash(env_hash)
     end
 
@@ -44,7 +44,7 @@ module Eeml
 
     def buildDatastreams(env_hash)
       datastreams_hash = env_hash["datastreams"]
-      return if datastreams_hash.nil?
+      return [] if datastreams_hash.nil?
       datastreams = []
       datastreams_hash.each do |datastream_hash|
         datastream = DataStream.new

data/lib/eeml/json_environment_parser_v006.rb

@@ -24,7 +24,7 @@ module Eeml
     end
 
     def make_environment_from_json(json_str)
-      env_hash = JSON.parse(json_str)
+      env_hash = JSON.parse(strip_content(json_str))
       return make_environment_from_hash(env_hash)
     end
 
@@ -44,7 +44,7 @@ module Eeml
 
     def buildDatastreams(env_hash)
       datastreams_hash = env_hash["datastreams"]
-      return if datastreams_hash.nil?
+      return [] if datastreams_hash.nil?
       datastreams = []
       datastreams_hash.each do |datastream_hash|
         datastream = DataStream.new

data/lib/eeml/json_environment_parser_v100.rb

@@ -26,7 +26,7 @@ module Eeml
     end
 
     def make_environment_from_json(json_str)
-      env_hash = JSON.parse(json_str)
+      env_hash = JSON.parse(strip_content(json_str))
       return make_environment_from_hash(env_hash)
     end
 
@@ -46,7 +46,7 @@ module Eeml
 
     def buildDatastreams(env_hash)
       datastreams_hash = env_hash["datastreams"]
-      return if datastreams_hash.nil?
+      return [] if datastreams_hash.nil?
       datastreams = []
       datastreams_hash.each do |datastream_hash|  
       datastream = DataStream.new

data/lib/eeml/libxml_eeml_parser_v005.rb

@@ -37,7 +37,7 @@ module Eeml
       #TODO: are these configurations per-thread?  If they're global (e.g. class variables) then we shouldn't be setting them here.
       XML.default_line_numbers=true
       
-      parser = XML::Parser.string(xml_str)
+      parser = XML::Parser.string(strip_content(xml_str))
       begin
         doc = parser.parse
       rescue XML::Error => e

data/lib/eeml/libxml_eeml_parser_v051.rb

@@ -37,7 +37,7 @@ module Eeml
       #TODO: are these configurations per-thread?  If they're global (e.g. class variables) then we shouldn't be setting them here.
       XML.default_line_numbers=true
       
-      parser = XML::Parser.string(xml_str)
+      parser = XML::Parser.string(strip_content(xml_str))
       begin
         doc = parser.parse
       rescue XML::Error => e

data/lib/eeml.rb

@@ -8,6 +8,7 @@ require 'yajl/json_gem'
 require 'lightcsv'
 require 'eeml/constants'
 require 'eeml/exceptions'
+require 'eeml/helpers'
 require 'eeml/environment'
 require 'eeml/environment_builder'
 require 'time'
@@ -15,7 +16,7 @@ require 'time'
 module Eeml
 
   # library version number
-  VERSION = '0.0.25'
+  VERSION = '0.0.29'
 
   # TODO: put in some configuration file, not here
   LOCAL_EEML_SCHEMA_LOCATION = 'schemas/eeml/005.xsd'

data/test/test_csv_parser_v1.rb

@@ -10,6 +10,20 @@ class TestCsvParserV1 < Test::Unit::TestCase
     assert_equal "Currently Pachube can only accept csv for your most recent set of values. You have submitted 2 rows of data.", exception.message
   end
 
+  test "should strip whitespace before parsing" do
+    csv = <<-EOF
+         20,10n5,fsa,432,6
+      
+      
+    EOF
+    environment = CsvParserV1.make_environment_from_csv(csv)
+    assert_equal 5, environment.datastreams.length
+    csv.split(',').each_with_index do |value, index|
+      assert_nil environment.datastreams[index].identifier
+      assert_equal value.strip, environment.datastreams[index].value
+    end
+  end
+
   test "should parse and return Environment object with datastreams" do
     csv = "20,10n5,fsa,432,6"
     environment = CsvParserV1.make_environment_from_csv(csv)

data/test/test_csv_parser_v2.rb

@@ -18,6 +18,24 @@ class TestCsvParserV2 < Test::Unit::TestCase
     assert_equal "CSV is invalid. Incorrect number of fields.", exception.message
   end
 
+  test "should strip whitespace before parsing" do
+    csv = <<-EOF
+
+       20,10
+
+       5,6
+      
+
+    EOF
+    environment = CsvParserV2.make_environment_from_csv(csv)
+    assert_equal 2, environment.datastreams.length
+    assert_equal "20", environment.datastreams.first.identifier
+    assert_equal "10", environment.datastreams.first.value
+
+    assert_equal "5", environment.datastreams.last.identifier
+    assert_equal "6", environment.datastreams.last.value
+  end
+
   test "should parse and return Environment object with datastreams" do
     csv = "20,10\n5,6"
     environment = CsvParserV2.make_environment_from_csv(csv)

data/test/test_environment.rb

@@ -95,8 +95,6 @@ class TestEnvironment < Test::Unit::TestCase
     #TODO: arguably, fetcher should also have rejected huge string
   end
 
-  # should this actually pass? Location is essentially a datastream in it's own right, so a feed
-  # that only specified location should probably be alright.
   test "passes if no datastreams" do 
     original_xml = File.read('test/data/doc_1.xml')
     test_xml = remove_nodes_called('data', original_xml)
@@ -104,6 +102,7 @@ class TestEnvironment < Test::Unit::TestCase
     environment = Environment.new_from_eeml(test_xml)
 
     assert_equal('title here', environment.title)
+    assert_equal([], environment.datastreams)
   end
 
   test "fail if any datastream has no value" do
@@ -380,6 +379,27 @@ class TestEnvironment < Test::Unit::TestCase
     end
   end
 
+  test "passes if no datastreams from json" do 
+    original_hash = JSON.parse(File.read('test/data/doc_1.json'))
+    original_hash.delete('datastreams')
+    test_json = original_hash.to_json
+
+    environment = Environment.new_from_json(test_json)
+
+    assert_equal('title here', environment.title)
+    assert_equal([], environment.datastreams)
+  end
+
+  test "parses a json with version missing and raises appropriate error" do
+    #original_hash = JSON.parse(File.read('test/data/doc_1.json'))
+    #original_hash.delete('version')
+    test_json = '[0,0,0,"9.44","83.68","1013.64"]'
+
+    assert_raises(JSON::ParserError) do
+      environment = Environment.new_from_json(test_json)
+    end
+  end
+
   test "parses a v1.0.0 input file ok" do
     env = create_env_from_json_v100_file_one
     assert_not_nil env
@@ -444,6 +464,17 @@ class TestEnvironment < Test::Unit::TestCase
     end
   end
 
+  test "passes in v1.0.0 if no datastreams from json" do 
+    original_hash = JSON.parse(File.read('test/data/doc_1_v1-0-0.json'))
+    original_hash.delete('datastreams')
+    test_json = original_hash.to_json
+
+    environment = Environment.new_from_json(test_json)
+
+    assert_equal('title here', environment.title)
+    assert_equal([], environment.datastreams)
+  end
+
   test "parses environment level tags from v1.0.0 json" do
     env = create_env_from_json_file('test/data/environment_tags.json')
     assert_equal 3, env.datastreams.size
@@ -515,6 +546,17 @@ class TestEnvironment < Test::Unit::TestCase
     end
   end
 
+  test "passes in v6 if no datastreams from json" do 
+    original_hash = JSON.parse(File.read('test/data/doc_1_v6.json'))
+    original_hash.delete('datastreams')
+    test_json = original_hash.to_json
+
+    environment = Environment.new_from_json(test_json)
+
+    assert_equal('title here', environment.title)
+    assert_equal([], environment.datastreams)
+  end
+
   test "captures `private` attribute when parsing old json" do
     env = create_env_from_json_file("test/data/doc_1_private.json")
     assert_not_nil env
@@ -593,6 +635,50 @@ class TestEnvironment < Test::Unit::TestCase
     assert_equal values, env.values_quick
   end
 
+  test "creating via csv should accept version v1" do
+    environment = Environment.new_from_csv("1,2,3,4", :v1)
+    assert_equal 4, environment.datastreams.length
+  end
+
+  test "creating via csv should accept version v2" do
+    csv = <<-CSV
+1,2
+3,4
+CSV
+    environment = Environment.new_from_csv(csv, :v2)
+    assert_equal 2, environment.datastreams.length
+    assert_equal ['1', '3'], environment.datastreams.map(&:identifier).sort
+  end
+
+  test "creating via csv should attempt to detect version" do
+    csv = <<-CSV
+1,2
+3,4
+CSV
+    environment = Environment.new_from_csv(csv)
+    assert_equal 2, environment.datastreams.length
+    assert_equal ['1', '3'], environment.datastreams.map(&:identifier).sort
+  end
+
+  test "creating via csv should assume v1 if one row and one entry" do
+    environment = Environment.new_from_csv("nought")
+    assert_equal 1, environment.datastreams.length
+    assert_equal "nought", environment.datastreams.first.values.first.value
+  end
+
+  test "creating via csv should assume v1 if one row and 3+ entry" do
+    environment = Environment.new_from_csv("nought,two,free")
+    assert_equal 3, environment.datastreams.length
+    assert_equal ["free", "nought", "two"], environment.datastreams.collect(&:values).flatten.map(&:value).sort
+  end
+
+  test "creating via csv should raise an exception if it cannot detect the version" do
+    exception = assert_raises CsvEncodingError do
+      Environment.new_from_csv("1,2")
+    end
+    assert_equal "CSV version could not be detected", exception.message
+  end
+
   # --- convenience stuff - don't put tests under here ------------------------------
 
   def create_env_from_xml_string(string)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: eeml
 version: !ruby/object:Gem::Version 
-  hash: 45
+  hash: 37
   prerelease: false
   segments: 
   - 0
   - 0
-  - 25
-  version: 0.0.25
+  - 29
+  version: 0.0.29
 platform: ruby
 authors: 
 - Neill Bogie
@@ -18,7 +18,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-29 00:00:00 +01:00
+date: 2011-01-17 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -85,6 +85,7 @@ files:
 - Manifest
 - README
 - Rakefile
+- ci/build_hudson.sh
 - example.rb
 - lib/blank.rb
 - lib/eeml.rb
@@ -94,6 +95,7 @@ files:
 - lib/eeml/environment.rb
 - lib/eeml/environment_builder.rb
 - lib/eeml/exceptions.rb
+- lib/eeml/helpers.rb
 - lib/eeml/json_environment_parser_v005.rb
 - lib/eeml/json_environment_parser_v006.rb
 - lib/eeml/json_environment_parser_v100.rb
@@ -165,10 +167,10 @@ signing_key:
 specification_version: 3
 summary: Simple little library for programmatically manipulating EEML documents.
 test_files: 
-- test/test_libxml_eeml_parser_v005.rb
+- test/libxml_test_helper.rb
 - test/test_libxml_test_helper.rb
+- test/test_libxml_eeml_parser_v005.rb
 - test/test_csv_parser_v2.rb
+- test/test_helper.rb
 - test/test_csv_parser_v1.rb
 - test/test_environment.rb
-- test/libxml_test_helper.rb
-- test/test_helper.rb