datapackage 0.0.2 → 0.0.3

data/README.md

@@ -15,7 +15,7 @@ The library is intending to support:
 
 Add the gem into your Gemfile:
 
-        gem 'datapackage.rb', :git => "git://github.com/theodi/datapackage.rb.git"
+    gem 'datapackage.rb', :git => "git://github.com/theodi/datapackage.rb.git"
 
 Or:
 
@@ -65,15 +65,22 @@ To expose more detail on errors and warnings:
 
     messages = package.validate() # or package.validate(:datapackage)
 
-This returns an object with two keys: `:errors` and `:warnings`. These are arrays of messages.
+This returns an object with two keys: `:errors` and `:warnings`. The values of these keys are arrays of message object. 
+Message objects are formatted as follows:
 
-Warnings might include notes on missing metadata elements (e.g. package `licenses`) which are not required by the DataPackage specification 
-but which SHOULD be included.
+    {
+        :type => :metadata|:integrity,
+        :message => "message for user",
+        :fragment => "/path/to/responsible/element"
+    }
 
 It is possible to treat all warnings as errors by performing strict validation:
 
     package.valid?(true)
 
+Examples of warnings might include notes on missing metadata elements (e.g. package `licenses`) which are not required by the 
+DataPackage specification but which SHOULD be included.
+
 Warnings are currently generated for:
 
 * Missing `README.md` files from packages

data/bin/datapackage

@@ -12,23 +12,25 @@ end
 
 package = DataPackage::Package.new(ARGV[0])
 
-messages = package.validate
+messages = package.validate(:simpledataformat)
 
 if messages[:errors].length == 0
-    puts "Package #{package.name} is valid"
+    puts "Package #{package.name} is VALID"
 else
-    puts "Package #{package.name} is invalid"
-    puts "Fix the following errors: "
-    messages[:errors].each_with_index do |error, idx|
-        msg = error[:message] if error.class == Hash
-        msg = error if error.class != Hash
-        puts "#{idx+1}. #{msg}"
-    end        
+    puts "Package #{package.name} is INVALID"
+    puts "Errors: "
+    messages[:errors].each_with_index do |msg, idx|
+        puts "#{idx+1}. #{msg[:type]} error. #{msg[:fragment]}"
+        puts "  #{msg[:message]}"
+    end    
 end
 
 if !messages[:warnings].empty?
     puts "Warnings: "
-    messages[:warnings].each_with_index do |warning, idx|
-        puts "#{idx+1}. #{warning}"
+    messages[:warnings].each_with_index do |msg, idx|
+        puts "#{idx+1}. #{msg[:type]} warning. #{msg[:fragment]}"
+        puts "  #{msg[:message]}"
     end    
 end
+
+exit( messages[:errors].length )

data/lib/datapackage/validator.rb

@@ -3,6 +3,8 @@ module DataPackage
     #Base class for validators
     class Validator
                 
+        attr_reader :messages
+        
         def Validator.create(profile, opts={})
             if profile == :simpledataformat
                 return SimpleDataFormatValidator.new(profile, opts)
@@ -19,45 +21,41 @@ module DataPackage
         end
         
         def valid?(package, strict=false)
-            messages = validate( package )
-            return messages[:errors].empty? if !strict
-            return messages[:errors].empty? && messages[:warnings].empty? 
+            validate( package )
+            return @messages[:errors].empty? if !strict
+            return @messages[:errors].empty? && @messages[:warnings].empty? 
         end
         
         def validate( package )
-            return validate_integrity( package, validate_with_schema(package) )
+            @messages = {:errors=>[], :warnings=>[]}
+            validate_with_schema( package )
+            validate_integrity( package )
+            return @messages
         end
 
+        protected 
+
         def validate_with_schema(package)
             schema = load_schema(@schema_name)
-            messages = {
-                :errors => JSON::Validator.fully_validate(schema, package.metadata, :errors_as_objects => true),
-                :warnings => [] 
-            }
-            validate_metadata(package, messages)
-            return messages
+            messages = JSON::Validator.fully_validate(schema, package.metadata, :errors_as_objects => true)
+            @messages[:errors] += messages.each {|msg| msg[:type] = :metadata  } 
+            validate_metadata(package)
         end
 
-        def validate_integrity(package, messages={ :errors=>[], :warnings=>[] } )            
-            package.resources.each do |resource|
-                validate_resource(package, resource, messages)
-            end
-            
-            messages
+        def validate_integrity(package )            
+            package.resources.each_with_index do |resource, idx|
+                validate_resource( package, resource, "#/resources/#{idx}" )
+            end            
         end
-
-        protected 
-        
+                
         #implement to perform additional validation on metadata
-        def validate_metadata(package, messages)
+        def validate_metadata( package )
         end
         
         #implement for per-resource validation
-        def validate_resource(package, resource, messages)
+        def validate_resource( package, resource, path )
         end
-        
-        protected
-        
+                
         def load_schema(profile)
             if @opts[:schema] && @opts[:schema][profile]
                 if !File.exists?( @opts[:schema][profile] )
@@ -71,6 +69,22 @@ module DataPackage
             end
             return JSON.parse( File.read( schema_file ) )
         end
+        
+        def add_error(type, message, fragment=nil)
+            @messages[:errors] << create_message(type, message, fragment)
+        end
+        
+        def add_warning(type, message, fragment=nil)
+            @messages[:warnings] << create_message(type, message, fragment)
+        end
+        
+        def create_message(type, message, fragment=nil)
+            { :message => message, :type => type, :fragment => fragment }              
+        end
+
+        def rebase(base, fragment)
+            return fragment.gsub("#/", base)
+        end
                 
         private
         
@@ -89,17 +103,17 @@ module DataPackage
             super(:datapackage, opts)
         end        
         
-        def validate_metadata(package, messages)
+        def validate_metadata(package)
             #not required, but recommended
             prefix = "The package does not include a"
-            messages[:warnings] << "#{prefix} 'licenses' property" if package.licenses.empty?
-            messages[:warnings] << "#{prefix} 'datapackage_version' property" unless package.datapackage_version 
-            messages[:warnings] << "#{prefix} README.md file" unless package.resource_exists?( package.resolve("README.md") )            
+            add_warning( :metadata, "#{prefix} 'licenses' property", "#/") if package.licenses.empty?
+            add_warning( :metadata, "#{prefix} 'datapackage_version' property", "#/") unless package.datapackage_version
+            add_warning( :integrity, "#{prefix} README.md file" ) unless package.resource_exists?( package.resolve("README.md") ) 
         end
         
-        def validate_resource(package, resource, messages)
+        def validate_resource( package, resource, path )
             if !package.resource_exists?( package.resolve_resource( resource ) )
-                messages[:errors] << "Resource #{resource["url"] || resource["path"]} does not exist"
+                add_error( :integrity, "Missing resource #{resource["url"] || resource["path"]}", path)
             end            
         end
         
@@ -114,63 +128,94 @@ module DataPackage
             @csvddf_schema = load_schema("csvddf-dialect")
         end
         
-        def validate_resource(package, resource, messages)
-            super(package, resource, messages)
+        def validate_resource(package, resource, path)
+            super(package, resource, path)
             
             if !csv?(resource)
-                messages[:errors] << "#{resource["name"]} is not a CSV file"
-            else                
-                if !resource["schema"]
-                    messages[:errors] << "#{resource["name"]} does not have a schema"
+                add_error(:integrity, "#{resource["name"]} is not a CSV file", path )
+            else  
+                schema = resource["schema"]              
+                if !schema
+                    add_error(:metadata, "#{resource["name"]} does not have a schema", path )
                 else
-                    messages[:errors] +=
-                        JSON::Validator.fully_validate(@jsontable_schema, 
-                            resource["schema"], :errors_as_objects => true)                                                           
-                end            
+                    messages = JSON::Validator.fully_validate(@jsontable_schema, schema, :errors_as_objects => true)
+                    @messages[:errors] += adjust_messages(messages, :metadata, path + "/schema/")                                                                                   
+                end  
+                          
                 if resource["dialect"]
-                    messages[:errors] +=
-                        JSON::Validator.fully_validate(@csvddf_schema, 
-                            resource["dialect"], :errors_as_objects => true)                                
+                    messages = JSON::Validator.fully_validate(@csvddf_schema, resource["dialect"], :errors_as_objects => true)
+                    @messages[:errors] += adjust_messages(messages, :metadata, path + "/dialect")
                 end
                 
-                if resource["schema"] && resource["schema"]["fields"]
-                    fields = resource["schema"]["fields"]
-                    declared_fields = fields.map{ |f| f["name"] }
-                    headers = headers(package, resource)
-                    
-                    #set algebra to finding fields missing from schema and/or CSV file
-                    missing_fields = declared_fields - headers
-                    if missing_fields != []
-                        messages[:errors] << 
-                            "Declared schema has fields not present in CSV file (#{missing_fields.join(",")})"
+                if package.resource_exists?( package.resolve_resource( resource ) )
+                    if resource["schema"] && resource["schema"]["fields"]
+                        fields = resource["schema"]["fields"]
+                        declared_fields = fields.map{ |f| f["name"] }
+                        headers = headers(package, resource, path)
+                        
+                        #set algebra to finding fields missing from schema and/or CSV file
+                        missing_fields = declared_fields - headers
+                        if missing_fields != []
+                            add_error( :integrity, 
+                                "Declared schema has fields not present in CSV file (#{missing_fields.join(",")})", 
+                                path+"/schema/fields")
+                        end
+                        undeclared_fields = headers - declared_fields
+                        if undeclared_fields != []
+                            add_error( :integrity, 
+                                "CSV file has fields missing from schema (#{undeclared_fields.join(",")})", 
+                                path+"/schema/fields")
+                        end                    
                     end
-                    undeclared_fields = headers - declared_fields
-                    if undeclared_fields != []
-                        messages[:errors] << "CSV file has fields missing from schema (#{undeclared_fields.join(",")})"
-                    end                    
                 end
-                
             end
             
         end
-        
+                
         def csv?(resource)
             resource["mediatype"] == "text/csv" ||
             resource["format"] == "csv"       
         end  
         
-        def headers(package, resource)
+        def headers(package, resource, path)
             headers = []
+            #Using built-in CSV parser here as its more permissive than fastercsv
+            #Lets us provide options to tweak the parsing                
             opts = dialect_to_csv_options(resource["dialect"])
-            CSV.open( package.resolve_resource(resource), "r", opts) do |csv|
-                headers = csv.shift
+            begin
+                CSV.open( package.resolve_resource(resource), "r", opts) do |csv|
+                    headers = csv.shift
+                end
+            rescue => e
+                add_error( :integrity, "Parse error for #{package.resolve_resource(resource)}: #{e}", path)
             end
             return headers
         end
                             
         def dialect_to_csv_options(dialect)
-            return {}
+            return {} unless dialect
+            #supplying defaults here just in case the dialect is invalid
+            delimiter = dialect["delimiter"] || ","
+            delimiter = delimiter + " " if !dialect["skipinitialspace"]  
+            return {
+                :col_sep => delimiter,
+                :row_sep => ( dialect["lineterminator"] || :auto ),
+                :quote_char => ( dialect["quotechar"] || '"')
+            }
+        end
+        
+        private
+        
+        #adjust message structure returned by JSON::Validator to add out type and
+        #adjust fragment references when we're using sub-schemas
+        def adjust_messages(messages, type, path)
+            messages.each do |msg| 
+                msg[:type]= type
+                msg[:fragment] = rebase( path , msg[:fragment] )
+            end 
+            messages                               
         end
+        
     end
         
 end

data/lib/datapackage/version.rb

@@ -1,3 +1,3 @@
 module DataPackage
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datapackage
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors: