aspace_client 0.0.1

data/lib/aspace_client/json_schema_utils.rb

@@ -0,0 +1,414 @@
+module JSONSchemaUtils
+
+  def self.fragment_join(fragment, property = nil)
+    fragment = fragment.gsub(/^#\//, "")
+    property = property.gsub(/^#\//, "") if property
+
+    if property && fragment != "" && fragment !~ /\/$/
+      fragment = "#{fragment}/"
+    end
+
+    "#{fragment}#{property}"
+  end
+
+
+  def self.schema_path_lookup(schema, path)
+    if path.is_a? String
+      return self.schema_path_lookup(schema, path.split("/"))
+    end
+
+    if schema.has_key?('properties')
+      schema = schema['properties']
+    end
+
+    if path.length == 1
+      schema[path.first]
+    else
+      if schema[path.first]
+        self.schema_path_lookup(schema[path.first], path.drop(1))
+      else
+        nil
+      end
+    end
+  end
+
+
+
+  SCHEMA_PARSE_RULES =
+    [
+     {
+       :failed_attribute => ['Properties', 'IfMissing', 'ArchivesSpaceSubType'],
+       :pattern => /([A-Z]+: )?The property '.*?' did not contain a required property of '(.*?)'.*/,
+       :do => ->(msgs, message, path, type, property) {
+         if type && type =~ /ERROR/
+           msgs[:errors][fragment_join(path, property)] = ["Property is required but was missing"]
+         else
+           msgs[:warnings][fragment_join(path, property)] = ["Property was missing"]
+         end
+       }
+     },
+
+     {
+       :failed_attribute => ['ArchivesSpaceType'],
+       :pattern => /The property '#(.*?)' was not a well-formed date/,
+       :do => ->(msgs, message, path, property) {
+         msgs[:errors][fragment_join(path)] = ["Not a valid date"]
+       }
+     },
+
+     {
+       :failed_attribute => ['Pattern'],
+       :pattern => /The property '#\/.*?' did not match the regex '(.*?)' in schema/,
+       :do => ->(msgs, message, path, regexp) {
+         msgs[:errors][fragment_join(path)] = ["Did not match regular expression: #{regexp}"]
+       }
+     },
+
+     {
+       :failed_attribute => ['MinLength'],
+       :pattern => /The property '#\/.*?' was not of a minimum string length of ([0-9]+) in schema/,
+       :do => ->(msgs, message, path, length) {
+         msgs[:errors][fragment_join(path)] = ["Must be at least #{length} characters"]
+       }
+     },
+
+     {
+       :failed_attribute => ['MaxLength'],
+       :pattern => /The property '#\/.*?' was not of a maximum string length of ([0-9]+) in schema/,
+       :do => ->(msgs, message, path, length) {
+         msgs[:errors][fragment_join(path)] = ["Must be #{length} characters or fewer"]
+       }
+     },
+
+     {
+       :failed_attribute => ['MinItems'],
+       :pattern => /The property '#\/.*?' did not contain a minimum number of items ([0-9]+) in schema/,
+       :do => ->(msgs, message, path, items) {
+         msgs[:errors][fragment_join(path)] = ["At least #{items} item(s) is required"]
+       }
+     },
+
+     {
+       :failed_attribute => ['Enum'],
+       :pattern => /The property '#\/.*?' value "(.*?)" .*values: (.*) in schema/,
+       :do => ->(msgs, message, path, invalid, valid_set) {
+         msgs[:errors][fragment_join(path)] = ["Invalid value '#{invalid}'.  Must be one of: #{valid_set}"]
+       }
+     },
+     
+     {
+       :failed_attribute => ['ArchivesSpaceDynamicEnum'],
+       :pattern => /The property '#\/.*?' value "(.*?)" .*values: (.*) in schema/,
+       :do => ->(msgs, message, path, invalid, valid_set) {
+         msgs[:attribute_types][fragment_join(path)] = 'ArchivesSpaceDynamicEnum'
+         msgs[:errors][fragment_join(path)] = ["Invalid value '#{invalid}'.  Must be one of: #{valid_set}"]
+       }
+     },
+     {
+       :failed_attribute => ['ArchivesSpaceReadOnlyDynamicEnum'],
+       :pattern => /The property '#\/.*?' value "(.*?)" .*values: (.*) in schema/,
+       :do => ->(msgs, message, path, invalid, valid_set) {
+         msgs[:attribute_types][fragment_join(path)] = 'ArchivesSpaceReadOnlyDynamicEnum'
+         msgs[:errors][fragment_join(path)] = ["Protected read-only list #{path}. Invalid value '#{invalid}'.  Must be one of: #{valid_set}"]
+       }
+     },
+
+     {
+       :failed_attribute => ['Type', 'ArchivesSpaceType'],
+       :pattern => /The property '#\/.*?' of type (.*?) did not match the following type: (.*?) in schema/,
+       :do => ->(msgs, message, path, actual_type, desired_type) {
+         if actual_type !~ /JSONModel/ || message[:failed_attribute] == 'ArchivesSpaceType'
+           # We'll skip JSONModels because the specific problem with the
+           # document will have already been listed separately.
+
+           msgs[:state][fragment_join(path)] ||= []
+           msgs[:state][fragment_join(path)] << desired_type
+
+           if msgs[:state][fragment_join(path)].length == 1
+             msgs[:errors][fragment_join(path)] = ["Must be a #{desired_type} (you provided a #{actual_type})"]
+           else
+             msgs[:errors][fragment_join(path)] = ["Must be one of: #{msgs[:state][fragment_join(path)].join (", ")} (you provided a #{actual_type})"]
+           end
+         end
+
+       }
+     },
+
+     {
+       :failed_attribute => ['custom_validation'],
+       :pattern => /Validation failed for '(.*?)': (.*?) in schema /,
+       :do => ->(msgs, message, path, property, msg) {
+         property = (property && !property.empty?) ? property : nil
+         msgs[:errors][fragment_join(path, property)] = [msg]
+       }
+     },
+
+     {
+       :failed_attribute => ['custom_validation'],
+       :pattern => /Warning generated for '(.*?)': (.*?) in schema /,
+       :do => ->(msgs, message, path, property, msg) {
+         msgs[:warnings][fragment_join(path, property)] = [msg]
+       }
+     },
+
+     {
+       :failed_attribute => ['custom_validation'],
+       :pattern => /Validation error code: (.*?) in schema /,
+       :do => ->(msgs, message, path, error_code) {
+         msgs[:errors]['coded_errors'] = [error_code]
+       }
+     },
+
+
+     # Catch all
+     {
+       :failed_attribute => nil,
+       :pattern => /^(.*)$/,
+       :do => ->(msgs, message, path, msg) {
+         msgs[:errors]['unknown'] = [msg]
+       }
+     }
+    ]
+
+
+  # For a given error, find its list of sub errors.
+  def self.extract_suberrors(errors)
+    errors = Array[errors].flatten
+
+    result = errors.map do |error|
+      if !error[:errors]
+        error
+      else
+        self.extract_suberrors(error[:errors])
+      end
+    end
+
+    result.flatten
+  end
+
+
+
+  # Given a list of error messages produced by JSON schema validation, parse
+  # them into a structured format like:
+  #
+  # {
+  #   :errors => {:attr1 => "(What was wrong with attr1)"},
+  #   :warnings => {:attr2 => "(attr2 not quite right either)"}
+  # }
+  def self.parse_schema_messages(messages, validator)
+
+    messages = self.extract_suberrors(messages)
+
+    msgs = {
+      :errors => {},
+      :warnings => {},
+      # to lookup e.g., msgs[:attribute_types]['extents/0/extent_type'] => 'ArchivesSpaceDynamicEnum'
+      :attribute_types => {},
+      :state => {}              # give the parse rules somewhere to store useful state for a run
+    }
+
+    messages.each do |message|
+
+      SCHEMA_PARSE_RULES.each do |rule|
+        if (rule[:failed_attribute].nil? || rule[:failed_attribute].include?(message[:failed_attribute])) and
+            message[:message] =~ rule[:pattern]
+          rule[:do].call(msgs, message, message[:fragment],
+                         *message[:message].scan(rule[:pattern]).flatten)
+
+          break
+        end
+      end
+
+    end
+
+    msgs.delete(:state)
+    msgs
+  end
+
+
+  # Given a hash representing a record tree, map across the hash and this
+  # model's schema in lockstep.
+  #
+  # Each proc in the 'transformations' array is called with the current node
+  # in the record tree as its first argument, and the part of the schema
+  # that corresponds to it.  Whatever the proc returns is used to replace
+  # the node in the record tree.
+  #
+  def self.map_hash_with_schema(record, schema, transformations = [])
+    return record if not record.is_a?(Hash)
+
+    if schema.is_a?(String)
+      schema = resolve_schema_reference(schema)
+    end
+
+    # Sometimes a schema won't specify anything other than the required type
+    # (like {'type' => 'object'}).  If there's nothing more to check, we're
+    # done.
+    return record if !schema.has_key?("properties")
+
+
+    # Apply transformations to the current level of the tree
+    transformations.each do |transform|
+      record = transform.call(record, schema)
+    end
+
+    # Now figure out how to traverse the remainder of the tree...
+    result = {}
+
+    record.each do |k, v|
+      k = k.to_s
+      properties = schema['properties']
+
+      if properties.has_key?(k) && (properties[k]["type"] == "object")
+        result[k] = self.map_hash_with_schema(v, properties[k], transformations)
+
+      elsif v.is_a?(Array) && properties.has_key?(k) && (properties[k]["type"] == "array")
+
+        # Arrays are tricky because they can either consist of a single type, or
+        # a number of different types.
+
+        if properties[k]["items"]["type"].is_a?(Array)
+          result[k] = v.map {|elt|
+
+            if elt.is_a?(Hash)
+              next_schema = determine_schema_for(elt, properties[k]["items"]["type"])
+              self.map_hash_with_schema(elt, next_schema, transformations)
+            elsif elt.is_a?(Array)
+              raise "Nested arrays aren't supported here (yet)"
+            else
+              elt
+            end
+          }
+
+        # The array contains a single type of object
+        elsif properties[k]["items"]["type"] === "object"
+          result[k] = v.map {|elt| self.map_hash_with_schema(elt, properties[k]["items"], transformations)}
+        else
+          # Just one valid type
+          result[k] = v.map {|elt| self.map_hash_with_schema(elt, properties[k]["items"]["type"], transformations)}
+        end
+
+      elsif (v.is_a?(Hash) || v.is_a?(Array)) && (properties.has_key?(k) && properties[k]["type"].is_a?(Array))
+        # Multiple possible types for this single value
+
+        results = (v.is_a?(Array) ? v : [v]).map {|elt|
+          next_schema = determine_schema_for(elt, properties[k]["type"])
+          self.map_hash_with_schema(elt, next_schema, transformations)
+        }
+
+        result[k] = v.is_a?(Array) ? results : results[0]
+
+      elsif properties.has_key?(k) && JSONModel.parse_jsonmodel_ref(properties[k]["type"])
+        result[k] = self.map_hash_with_schema(v, properties[k]["type"], transformations)
+      else
+        result[k] = v
+      end
+    end
+
+    result
+  end
+
+
+
+  def self.is_blank?(obj)
+    obj.nil? || obj == "" || obj == {}
+  end
+
+
+  def self.drop_empty_elements(obj)
+    if obj.is_a?(Hash)
+      Hash[obj.map do |k, v|
+             v = drop_empty_elements(v)
+             [k, v] if !is_blank?(v)
+           end]
+    elsif obj.is_a?(Array)
+      obj.map {|elt| drop_empty_elements(elt)}.reject {|elt| is_blank?(elt)}
+    else
+      obj
+    end
+  end
+
+
+  # Drop any keys from 'hash' that aren't defined in the JSON schema.
+  #
+  # If drop_readonly is true, also drop any values where the schema has
+  # 'readonly' set to true.  These values are produced by the system for the
+  # client, but are not part of the data model.
+  #
+  def self.drop_unknown_properties(hash, schema, drop_readonly = false)
+    fn = proc do |hash, schema|
+      result = {}
+
+      hash.each do |k, v|
+        if schema["properties"].has_key?(k.to_s) && (!drop_readonly || !schema["properties"][k.to_s]["readonly"])
+          result[k] = v
+        end
+      end
+
+      result
+    end
+
+    hash = drop_empty_elements(hash)
+    map_hash_with_schema(hash, schema, [fn])
+  end
+
+
+  def self.apply_schema_defaults(hash, schema)
+    fn = proc do |hash, schema|
+      result = hash.clone
+
+      schema["properties"].each do |property, definition|
+
+        if definition.has_key?("default") && !hash.has_key?(property.to_s) && !hash.has_key?(property.intern)
+          result[property] = definition["default"]
+        elsif definition['type'] == 'array' && !hash.has_key?(property.to_s) && !hash.has_key?(property.intern)
+          # Array values that weren't provided default to empty
+          result[property] = []
+        end
+
+      end
+
+      result
+    end
+
+    map_hash_with_schema(hash, schema, [fn])
+  end
+
+
+  private
+
+  def self.resolve_schema_reference(schema_reference)
+    # This should be a reference to a different JSONModel type.  Resolve it
+    # and return its schema.
+    ref = JSONModel.parse_jsonmodel_ref(schema_reference)
+    raise "Invalid schema given: #{schema_reference}" if !ref
+
+    JSONModel.JSONModel(ref[0]).schema
+  end
+
+
+  def self.determine_schema_for(elt, possible_schemas)
+    # A number of different types.  Match them up based on the value of the 'jsonmodel_type' property
+    schema_types = possible_schemas.map {|schema| schema.is_a?(Hash) ? schema["type"] : schema}
+
+    jsonmodel_type = elt["jsonmodel_type"]
+
+    if !jsonmodel_type
+      raise JSONModel::ValidationException.new(:errors => {"record" => ["Can't unambiguously match #{elt.inspect} against schema types: #{schema_types.inspect}. " +
+                                                             "Resolve this by adding a 'jsonmodel_type' property to #{elt.inspect}"]})
+    end
+
+    next_schema = schema_types.find {|type|
+      (type.is_a?(String) && type.include?("JSONModel(:#{jsonmodel_type})")) ||
+      (type.is_a?(Hash) && type["jsonmodel_type"] === jsonmodel_type)
+    }
+
+    if next_schema.nil?
+      raise "Couldn't determine type of '#{elt.inspect}'.  Must be one of: #{schema_types.inspect}"
+    end
+
+    next_schema
+  end
+
+
+end

data/lib/aspace_client/jsonmodel.rb

@@ -0,0 +1,342 @@
+require 'json-schema'
+require 'atomic'
+require 'uri'
+require_relative 'jsonmodel_type'
+require_relative 'json_schema_concurrency_fix'
+require_relative 'json_schema_utils'
+require_relative 'asutils'
+require_relative 'validator_cache'
+
+
+module JSONModel
+
+  @@models = {}
+  @@custom_validations = {}
+  @@strict_mode = false
+  @@init_args = {:initialized => false}
+
+  def self.custom_validations
+    @@custom_validations
+  end
+
+  def strict_mode(val)
+    @@strict_mode = val
+  end
+
+
+  def self.strict_mode?
+    @@strict_mode
+  end
+
+
+  class ValidationException < StandardError
+    attr_accessor :invalid_object
+    attr_accessor :errors
+    attr_accessor :warnings
+    attr_accessor :attribute_types
+    attr_accessor :import_context
+    attr_accessor :object_context
+
+    def initialize(opts)
+      @invalid_object = opts[:invalid_object]
+      @errors = opts[:errors]
+      @import_context = opts[:import_context]
+      @object_context = opts[:object_context]
+      @attribute_types = opts[:attribute_types]
+    end
+
+    def to_s
+       msg = { :errors => @errors } 
+       msg[:import_context] = @import_context unless @import_context.nil? 
+       msg[:object_context] = @object_context unless @object_context.nil? 
+       "#<:ValidationException: #{msg.inspect}>"
+    end
+  end
+
+
+  def self.JSONModel(source)
+    if !@@models.has_key?(source.to_s)
+      load_schema(source.to_s)
+    end
+
+    @@models[source.to_s] or raise "JSONModel not found for #{source}"
+  end
+
+
+  def JSONModel(source)
+    JSONModel.JSONModel(source)
+  end
+
+
+  # Yield all known JSONModel classes
+  def models
+    @@models
+  end
+
+
+  def self.repository_for(reference)
+    if reference =~ /^(\/repositories\/[0-9]+)\//
+      return $1
+    else
+      return nil
+    end
+  end
+
+
+  # Parse a URI reference like /repositories/123/archival_objects/500 into
+  # {:id => 500, :type => :archival_object}
+  def self.parse_reference(reference, opts = {})
+    @@models.each do |type, model|
+      id = model.id_for(reference, opts, true)
+      if id
+        return {:id => id, :type => type, :repository => repository_for(reference)}
+      end
+    end
+
+    nil
+  end
+
+
+  def self.destroy_model(type)
+    @@models.delete(type.to_s)
+  end
+
+
+  def self.schema_src(schema_name)
+
+    if schema_name.to_s !~ /\A[0-9A-Za-z_-]+\z/
+      raise "Invalid schema name: #{schema_name}"
+    end
+
+    [*ASUtils.find_local_directories('schemas'),
+     File.join(File.dirname(__FILE__), "schemas")].each do |dir|
+
+      schema = File.join(dir, "#{schema_name}.rb")
+
+      if File.exists?(schema)
+        return File.open(schema).read
+      end
+    end
+
+    nil
+  end
+
+
+  def self.allow_unmapped_enum_value(val, magic_value = 'other_unmapped')
+    if val.is_a? Array
+      val.each { |elt| allow_unmapped_enum_value(elt) }
+    elsif val.is_a? Hash
+      val.each do |k, v|
+        if k == 'enum'
+          v << magic_value
+         else
+          allow_unmapped_enum_value(v)
+        end
+      end
+    end
+  end
+
+
+  def self.load_schema(schema_name)
+    if not @@models[schema_name]
+
+      old_verbose = $VERBOSE
+      $VERBOSE = nil
+      src = schema_src(schema_name)
+
+      return if !src
+
+      entry = eval(src)
+      $VERBOSE = old_verbose
+
+      parent = entry[:schema]["parent"]
+      if parent
+        load_schema(parent)
+
+        base = @@models[parent].schema["properties"].clone
+        properties = ASUtils.deep_merge(base, entry[:schema]["properties"])
+
+        # Maybe we'll eventually want the version of a schema to be
+        # automatically set to max(my_version, parent_version), but for now...
+        if entry[:schema]["version"] < @@models[parent].schema_version
+          raise ("Can't inherit from a JSON schema whose version is newer than ours " +
+                 "(our (#{schema_name}) version: #{entry[:schema]['version']}; " +
+                 "parent (#{parent}) version: #{@@models[parent].schema_version})")
+        end
+
+        entry[:schema]["properties"] = properties
+      end
+
+      # All records have a lock_version property that we use for optimistic concurrency control.
+      entry[:schema]["properties"]["lock_version"] = {"type" => ["integer", "string"], "required" => false}
+
+      # All records must indicate their model type
+      entry[:schema]["properties"]["jsonmodel_type"] = {"type" => "string", "ifmissing" => "error"}
+
+      # All records have audit fields
+      entry[:schema]["properties"]["created_by"] = {"type" => "string", "readonly" => true}
+      entry[:schema]["properties"]["last_modified_by"] = {"type" => "string", "readonly" => true}
+      entry[:schema]["properties"]["user_mtime"] = {"type" => "date-time", "readonly" => true}
+      entry[:schema]["properties"]["system_mtime"] = {"type" => "date-time", "readonly" => true}
+      entry[:schema]["properties"]["create_time"] = {"type" => "date-time", "readonly" => true}
+
+      # Records may include a reference to the repository that contains them
+      entry[:schema]["properties"]["repository"] ||= {
+        "type" => "object",
+        "subtype" => "ref",
+        "readonly" => "true",
+        "properties" => {
+          "ref" => {
+            "type" => "JSONModel(:repository) uri",
+            "ifmissing" => "error",
+            "readonly" => "true"
+          },
+          "_resolved" => {
+            "type" => "object",
+            "readonly" => "true"
+          }
+        }
+      }
+
+
+      if @@init_args && @@init_args[:allow_other_unmapped]
+        allow_unmapped_enum_value(entry[:schema]['properties'])
+      end
+
+      ASUtils.find_local_directories("schemas/#{schema_name}_ext.rb").
+              select {|path| File.exists?(path)}.
+              each do |schema_extension|
+        entry[:schema]['properties'] = ASUtils.deep_merge(entry[:schema]['properties'],
+                                                          eval(File.open(schema_extension).read))
+      end
+
+      self.create_model_for(schema_name, entry[:schema])
+    end
+  end
+
+
+  def self.init(opts = {})
+
+    @@init_args ||= nil
+
+    # Skip initialisation if this model has already been loaded.
+    if @@init_args[:initialized]
+      return true
+    else 
+       @@init_args[:initialized] = true
+    end
+
+    if opts.has_key?(:strict_mode)
+      @@strict_mode = true
+    end
+
+    @@init_args = opts
+
+    if !opts.has_key?(:enum_source)
+      if opts[:client_mode]
+        require_relative 'jsonmodel_client'
+        opts[:enum_source] = JSONModel::Client::EnumSource.new
+      else
+        raise "Required JSONModel.init arg :enum_source was missing"
+      end
+    end
+
+    # Load all JSON schemas from the schemas subdirectory
+    # Create a model class for each one.
+    Dir.glob(File.join(File.dirname(__FILE__),
+                       "schemas",
+                       "*.rb")).sort.each do |schema|
+      schema_name = File.basename(schema, ".rb")
+      load_schema(schema_name)
+    end
+
+    require_relative "validations"
+
+    # For dynamic enums, automatically slot in the 'other_unmapped' string as an allowable value
+    if @@init_args[:allow_other_unmapped]
+      enum_wrapper = Struct.new(:enum_source).new(@@init_args[:enum_source])
+
+      def enum_wrapper.valid?(name, value)
+        value == 'other_unmapped' || enum_source.valid?(name, value)
+      end
+
+      def enum_wrapper.editable?(name)
+        enum_source.editable?(name)
+      end
+
+      def enum_wrapper.values_for(name)
+        enum_source.values_for(name) + ['other_unmapped']
+      end
+
+      def enum_wrapper.default_value_for(name)
+        enum_source.default_value_for(name)
+      end
+
+      @@init_args[:enum_source] = enum_wrapper
+    end
+
+    true
+
+  rescue
+    # If anything went wrong we're not initialised.
+    @@init_args = nil
+
+    raise $!
+  end
+
+
+  def self.enum_values(name)
+    @@init_args[:enum_source].values_for(name)
+  end
+
+
+  def self.enum_default_value(name)
+    @@init_args[:enum_source].default_value_for(name)
+  end
+
+
+  def self.client_mode?
+    @@init_args[:client_mode]
+  end
+
+
+  def self.parse_jsonmodel_ref(ref)
+    if ref.is_a? String and ref =~ /JSONModel\(:([a-zA-Z_\-]+)\) (.*)/
+      [$1.intern, $2]
+    else
+      nil
+    end
+  end
+
+
+  protected
+
+
+  def self.clean_data(data)
+    data = ASUtils.keys_as_strings(data) if data.is_a?(Hash)
+    data = ASUtils.jsonmodels_to_hashes(data)
+
+    data
+  end
+
+
+  # Create and return a new JSONModel class called 'type', based on the
+  # JSONSchema 'schema'
+  def self.create_model_for(type, schema)
+
+    cls = Class.new(JSONModelType)
+    cls.init(type, schema, Array(@@init_args[:mixins]))
+
+    @@models[type] = cls
+  end
+
+
+  def self.init_args
+    @@init_args
+  end
+
+end
+
+
+# Custom JSON schema validations
+require_relative 'archivesspace_json_schema'