json_schema 0.0.7 → 0.0.9

data/README.md

@@ -2,6 +2,15 @@
 
 A JSON Schema V4 and Hyperschema V4 parser and validator.
 
+Validate some data based on a JSON Schema:
+
+```
+gem install json_schema
+validate-schema schema.json data.json
+```
+
+## Programmatic
+
 ``` ruby
 require "json"
 require "json_schema"

data/bin/validate-schema

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require "json"
+require "optparse"
+require_relative "../lib/commands/validate_schema"
+
+def print_usage!
+  $stderr.puts "Usage: validate-schema <schema> <data>, ..."
+  $stderr.puts "       validate-schema -d <data>, ..."
+end
+
+command = Commands::ValidateSchema.new
+
+OptionParser.new { |opts|
+  opts.on("-d", "--detect", "Detect schema from $schema") do
+    command.detect = true
+
+    # mix in common schemas for convenience
+    command.extra_schemas += ["schema.json", "hyper-schema.json"].
+      map { |f| File.expand_path(f, __FILE__ + "/../../schemas") }
+  end
+  opts.on("-s", "--schema SCHEMA", "Additional schema to use for references") do |s|
+    command.extra_schemas << s
+  end
+}.parse!
+
+success = command.run(ARGV.dup)
+
+if success
+  command.messages.each { |m| $stdout.puts(m) }
+elsif !command.errors.empty?
+  command.errors.each { |e| $stderr.puts(File.basename(__FILE__) + ": " + e) }
+  exit(1)
+else
+  print_usage!
+  exit(1)
+end

data/lib/commands/validate_schema.rb

@@ -0,0 +1,102 @@
+require_relative "../json_schema"
+
+module Commands
+  class ValidateSchema
+    attr_accessor :detect
+    attr_accessor :extra_schemas
+
+    attr_accessor :errors
+    attr_accessor :messages
+
+    def initialize
+      @detect = false
+      @extra_schemas = []
+
+      @errors = []
+      @messages = []
+    end
+
+    def run(argv)
+      return false if !initialize_store
+
+      if !detect
+        return false if !(schema_file = argv.shift)
+        return false if !(schema = parse(schema_file))
+      end
+
+      # if there are no remaining files in arguments, also a problem
+      return false if argv.count < 1
+
+      argv.each do |data_file|
+        return false if !check_file(data_file)
+        data = JSON.parse(File.read(data_file))
+
+        if detect
+          if !(schema_uri = data["$schema"])
+            @errors = ["#{data_file}: No $schema tag for detection."]
+            return false
+          end
+
+          if !(schema = @store.lookup_uri(schema_uri))
+            @errors = ["#{data_file}: Unknown $schema, try specifying one with -s."]
+            return false
+          end
+        end
+
+        valid, errors = schema.validate(data)
+
+        if valid
+          @messages += ["#{data_file} is valid."]
+        else
+          errors = ["Invalid."] + errors.map { |e| e.message }
+          @errors += errors.map { |e| "#{data_file}: #{e}" }
+        end
+      end
+
+      @errors.empty?
+    end
+
+    private
+
+    def check_file(file)
+      if !File.exists?(file)
+        @errors = ["#{file}: No such file or directory."]
+        false
+      else
+        true
+      end
+    end
+
+    def initialize_store
+      @store = JsonSchema::DocumentStore.new
+      extra_schemas.each do |extra_schema|
+        if !(extra_schema = parse(extra_schema))
+          return false
+        end
+        @store.add_uri_reference(extra_schema.uri, extra_schema)
+      end
+      true
+    end
+
+    def parse(file)
+      return nil if !check_file(file)
+
+      parser = JsonSchema::Parser.new
+      if !(schema = parser.parse(JSON.parse(File.read(file))))
+        @errors = ["Schema is invalid."] + parser.errors.map { |e| e.message }
+        @errors.map! { |e| "#{file}: #{e}" }
+        return nil
+      end
+
+      expander = JsonSchema::ReferenceExpander.new
+      if !expander.expand(schema, store: @store)
+        @errors = ["Could not expand schema references."] +
+          expander.errors.map { |e| e.message }
+        @errors.map! { |e| "#{file}: #{e}" }
+        return nil
+      end
+
+      schema
+    end
+  end
+end

data/lib/json_reference.rb

@@ -1,8 +1,14 @@
-require "json_pointer"
 require "uri"
+require_relative "json_pointer"
 
 module JsonReference
+  def self.reference(ref)
+    Reference.new(ref)
+  end
+
   class Reference
+    include Comparable
+
     attr_accessor :pointer
     attr_accessor :uri
 
@@ -14,12 +20,22 @@ module JsonReference
         if uri && !uri.empty?
           @uri = URI.parse(uri)
         end
+        @pointer ||= ""
       else
         @pointer = ref
       end
 
-      # normalize pointers by prepending "#"
+      # normalize pointers by prepending "#" and stripping trailing "/"
       @pointer = "#" + @pointer
+      @pointer = @pointer.chomp("/")
+    end
+
+    def <=>(other)
+      to_s <=> other.to_s
+    end
+
+    def inspect
+      "\#<JsonReference::Reference #{to_s}>"
     end
 
     # Given the document addressed by #uri, resolves the JSON Pointer part of

data/lib/json_schema.rb

@@ -1,3 +1,4 @@
+require_relative "json_schema/document_store"
 require_relative "json_schema/parser"
 require_relative "json_schema/reference_expander"
 require_relative "json_schema/schema"

data/lib/json_schema/document_store.rb

@@ -0,0 +1,48 @@
+module JsonSchema
+  # The document store helps resolve URI-based JSON pointers by storing IDs
+  # that we've seen in the schema.
+  #
+  # Each URI tuple also contains a pointer map that helps speed up expansions
+  # that have already happened and handles cyclic dependencies. Store a
+  # reference to the top-level schema before doing anything else.
+  class DocumentStore
+    def initialize
+      @uri_map = {}
+    end
+
+    def add_pointer_reference(uri, path, schema)
+      raise "can't add nil URI" if uri.nil?
+
+      if !@uri_map[uri][:pointer_map].key?(path)
+        @uri_map[uri][:pointer_map][path] = schema
+      end
+    end
+
+    def add_uri_reference(uri, schema)
+      raise "can't add nil URI" if uri.nil?
+
+      # Children without an ID keep the same URI as their parents. So since we
+      # traverse trees from top to bottom, just keep the first reference.
+      if !@uri_map.key?(uri)
+        @uri_map[uri] = {
+          pointer_map: {
+            JsonReference.reference("#").to_s => schema
+          },
+          schema: schema
+        }
+      end
+    end
+
+    def lookup_pointer(uri, pointer)
+      @uri_map[uri][:pointer_map][pointer]
+    end
+
+    def lookup_uri(uri)
+      if @uri_map[uri]
+        @uri_map[uri][:schema]
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/json_schema/parser.rb

@@ -1,4 +1,4 @@
-require "json_reference"
+require_relative "../json_reference"
 
 module JsonSchema
   class Parser
@@ -45,20 +45,38 @@ module JsonSchema
     def build_uri(id, parent_uri)
       # kill any trailing slashes
       if id
-        id = id.chomp("/")
-      end
-
+        # may look like: http://json-schema.org/draft-04/hyper-schema#
+        uri = URI.parse(id)
+        # make sure there is no `#` suffix
+        uri.fragment = nil
+        # if id is defined as absolute, the schema's URI stays absolute
+        if uri.absolute? || uri.path[0] == "/"
+          uri.to_s.chomp("/")
+        # otherwise build it according to the parent's URI
+        elsif parent_uri
+          # make sure we don't end up with duplicate slashes
+          parent_uri = parent_uri.chomp("/")
+          parent_uri + "/" + id
+        else
+          "/"
+        end
       # if id is missing, it's defined as its parent schema's URI
-      if id.nil?
+      elsif parent_uri
         parent_uri
-      # if id is defined as absolute, the schema's URI stays absolute
-      elsif id[0] == "/"
-        id
-      # otherwise build it according to the parent's URI
       else
-        # make sure we don't end up with duplicate slashes
-        parent_uri = parent_uri.chomp("/")
-        parent_uri + "/" + id
+        "/"
+      end
+    end
+
+    def parse_additional_properties(schema)
+      if schema.additional_properties
+        # an object indicates a schema that will be used to parse any
+        # properties not listed in `properties`
+        if schema.additional_properties.is_a?(Hash)
+          schema.additional_properties =
+            parse_data(schema.additional_properties, schema)
+        end
+        # otherwise, leave as boolean
       end
     end
 
@@ -201,8 +219,11 @@ module JsonSchema
       schema.data        = data
       schema.id          = validate_type(schema, [String], "id")
 
+      # any parsed schema is automatically expanded
+      schema.expanded    = true
+
       # build URI early so we can reference it in errors
-      schema.uri = parent ?  build_uri(schema.id, parent.uri) : "/"
+      schema.uri         = build_uri(schema.id, parent ? parent.uri : nil)
 
       schema.title       = validate_type(schema, [String], "title")
       schema.description = validate_type(schema, [String], "description")
@@ -235,7 +256,7 @@ module JsonSchema
 
       # validation: object
       schema.additional_properties =
-        validate_type(schema, BOOLEAN, "additionalProperties")
+        validate_type(schema, BOOLEAN + [Hash], "additionalProperties")
       schema.dependencies       = validate_type(schema, [Hash], "dependencies") || {}
       schema.max_properties     = validate_type(schema, [Integer], "maxProperties")
       schema.min_properties     = validate_type(schema, [Integer], "minProperties")
@@ -256,6 +277,7 @@ module JsonSchema
       schema.path_start = validate_type(schema, [String], "pathStart")
       schema.read_only  = validate_type(schema, BOOLEAN, "readOnly")
 
+      parse_additional_properties(schema)
       parse_all_of(schema)
       parse_any_of(schema)
       parse_one_of(schema)

data/lib/json_schema/reference_expander.rb

@@ -1,35 +1,22 @@
-require "json_schema/parser"
 require "set"
 
 module JsonSchema
   class ReferenceExpander
     attr_accessor :errors
 
-    def expand(schema)
+    def expand(schema, options = {})
       @errors = []
       @schema = schema
-      @store = {}
-      @unresolved_refs = Set.new
-      last_num_unresolved_refs = 0
+      @store  = options[:store] ||= DocumentStore.new
 
-      loop do
-        traverse_schema(schema)
+      @store.add_uri_reference("/", schema)
 
-        # nothing left unresolved, we're done!
-        if @unresolved_refs.count == 0
-          break
-        end
-
-        # a new traversal pass still hasn't managed to resolved anymore
-        # references; we're out of luck
-        if @unresolved_refs.count == last_num_unresolved_refs
-          refs = @unresolved_refs.to_a.join(", ")
-          message = %{Couldn't resolve references (possible circular dependency): #{refs}.}
-          @errors << SchemaError.new(schema, message)
-          break
-        end
+      traverse_schema(schema)
 
-        last_num_unresolved_refs = @unresolved_refs.count
+      refs = unresolved_refs(schema).sort
+      if refs.count > 0
+        message = %{Couldn't resolve references: #{refs.to_a.join(", ")}.}
+        @errors << SchemaError.new(schema, message)
       end
 
       @errors.count == 0
@@ -44,65 +31,101 @@ module JsonSchema
 
     private
 
-    def dereference(schema)
-      ref = schema.reference
+    def dereference(ref_schema, ref_stack)
+      ref = ref_schema.reference
+
+      # detects a reference cycle
+      if ref_stack.include?(ref)
+        message = %{Reference cycle detected: #{ref_stack.sort.join(", ")}.}
+        @errors << SchemaError.new(ref_schema, message)
+        return false
+      end
+
+      new_schema = resolve_reference(ref_schema)
+      return false unless new_schema
+
+      # if the reference resolved to a new reference we need to continue
+      # dereferencing until we either hit a non-reference schema, or a
+      # reference which is already resolved
+      if new_schema.reference && !new_schema.expanded?
+        success = dereference(new_schema, ref_stack + [ref])
+        return false unless success
+      end
+
+      # copy new schema into existing one while preserving parent
+      parent = ref_schema.parent
+      ref_schema.copy_from(new_schema)
+      ref_schema.parent = parent
+
+      true
+    end
+
+    def resolve_pointer(ref_schema, uri_path, resolved_schema)
+      ref = ref_schema.reference
+
+      # we've already evaluated this precise URI/pointer combination before
+      if !(new_schema = @store.lookup_pointer(uri_path, ref.pointer.to_s))
+        data = JsonPointer.evaluate(resolved_schema.data, ref.pointer)
+
+        # couldn't resolve pointer within known schema; that's an error
+        if data.nil?
+          message = %{Couldn't resolve pointer "#{ref.pointer}".}
+          @errors << SchemaError.new(resolved_schema, message)
+          return
+        end
+
+        # parse a new schema and use the same parent node
+        new_schema = Parser.new.parse(data, ref_schema.parent)
+
+        # add the reference into our document store right away; it will
+        # eventually be fully expanded
+        @store.add_pointer_reference(uri_path, ref.pointer.to_s, new_schema)
+      else
+        # insert a clone record so that the expander knows to expand it when
+        # the schema traversal is finished
+        new_schema.clones << ref_schema
+      end
+
+      new_schema
+    end
+
+    def resolve_reference(ref_schema)
+      ref = ref_schema.reference
       uri = ref.uri
 
       if uri && uri.host
         scheme = uri.scheme || "http"
-        message = %{Reference resolution over #{scheme} is not currently supported.}
-        @errors << SchemaError.new(schema, message)
+        # allow resolution if something we've already parsed has claimed the
+        # full URL
+        if @store.lookup_uri(uri.to_s)
+          resolve_uri(ref_schema, uri.to_s)
+        else
+          message =
+            %{Reference resolution over #{scheme} is not currently supported.}
+          @errors << SchemaError.new(ref_schema, message)
+          nil
+        end
       # absolute
       elsif uri && uri.path[0] == "/"
-        resolve(schema, uri.path, ref)
+        resolve_uri(ref_schema, uri.path)
       # relative
       elsif uri
         # build an absolute path using the URI of the current schema
-        schema_uri = schema.uri.chomp("/")
-        resolve(schema, schema_uri + "/" + uri.path, ref)
+        schema_uri = ref_schema.uri.chomp("/")
+        resolve_uri(ref_schema, schema_uri + "/" + uri.path)
       # just a JSON Pointer -- resolve against schema root
       else
-        evaluate(schema, @schema, ref)
-      end
-    end
-
-    def evaluate(schema, schema_context, ref)
-      data = JsonPointer.evaluate(schema_context.data, ref.pointer)
-
-      # couldn't resolve pointer within known schema; that's an error
-      if data.nil?
-        message = %{Couldn't resolve pointer "#{ref.pointer}".}
-        @errors << SchemaError.new(schema_context, message)
-        return
-      end
-
-      # this counts as a resolution
-      @unresolved_refs.delete(ref.to_s)
-
-      # parse a new schema and use the same parent node
-      new_schema = Parser.new.parse(data, schema.parent)
-
-      # mark a new unresolved reference if the schema we got back is also a
-      # reference
-      if new_schema.reference
-        @unresolved_refs.add(new_schema.reference.to_s)
+        resolve_pointer(ref_schema, "/", @schema)
       end
-
-      # copy new schema into existing one while preserving parent
-      parent = schema.parent
-      schema.copy_from(new_schema)
-      schema.parent = parent
-
-      new_schema
     end
 
-    def resolve(schema, uri, ref)
-      if schema_context = @store[uri]
-        evaluate(schema, schema_context, ref)
+    def resolve_uri(ref_schema, uri_path)
+      if schema = @store.lookup_uri(uri_path)
+        resolve_pointer(ref_schema, uri_path, schema)
       else
-        # couldn't resolve, return original reference
-        @unresolved_refs.add(ref.to_s)
-        schema
+        message = %{Couldn't resolve URI: #{uri_path}.}
+        @errors << SchemaError.new(ref_schema, message)
+        nil
       end
     end
 
@@ -116,17 +139,23 @@ module JsonSchema
         schema.pattern_properties.each { |_, s| yielder << s }
         schema.properties.each { |_, s| yielder << s }
 
+        if additional = schema.additional_properties
+          if additional.is_a?(Schema)
+            yielder << additional
+          end
+        end
+
         if schema.not
           yielder << schema.not
         end
 
         # can either be a single schema (list validation) or multiple (tuple
         # validation)
-        if schema.items
-          if schema.items.is_a?(Array)
-            schema.items.each { |s| yielder << s }
+        if items = schema.items
+          if items.is_a?(Array)
+            items.each { |s| yielder << s }
           else
-            yielder << schema.items
+            yielder << items
           end
         end
 
@@ -138,18 +167,40 @@ module JsonSchema
       end
     end
 
-    def traverse_schema(schema)
-      # Children without an ID keep the same URI as their parents. So since we
-      # traverse trees from top to bottom, just keep the first reference.
-      if !@store.key?(schema.uri)
-        @store[schema.uri] = schema
+    def unresolved_refs(schema)
+      # prevent endless recursion
+      return [] unless schema.original?
+
+      schema_children(schema).reduce([]) do |arr, subschema|
+        if !subschema.expanded?
+          arr += [subschema.reference]
+        else
+          arr += unresolved_refs(subschema)
+        end
       end
+    end
+
+    def traverse_schema(schema)
+      @store.add_uri_reference(schema.uri, schema)
 
       schema_children(schema).each do |subschema|
-        if subschema.reference
-          dereference(subschema)
+        if subschema.reference && !subschema.expanded?
+          dereference(subschema, [])
+        end
+
+        # traverse child schemas only if they're the original copy
+        if subschema.expanded? && subschema.original?
+          traverse_schema(subschema)
+        end
+      end
+
+      # after finishing a schema traversal, find all clones and re-hydrate them
+      if schema.original?
+        schema.clones.each do |clone_schema|
+          parent = clone_schema.parent
+          clone_schema.copy_from(schema)
+          clone_schema.parent = parent
         end
-        traverse_schema(subschema)
       end
     end
   end