isomorfeus-data 2.4.2 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a5f86620e77e947cb68b49795d40a7d696bd7b6e7f50812f5ec7028728f447d
-  data.tar.gz: 02bb60ee8785e484be3dc1d73191d2a61012f2d6d7bc13cafdb7542c6ba8a330
+  metadata.gz: 69be34181734a815ae1468128e569eee76b41d4022217f21042e0b96a63acbd2
+  data.tar.gz: 351403a3931b8a69c44612d52a5f322813ad0e57f9b14383ec4e255b6d3742a9
 SHA512:
-  metadata.gz: 28cd6b56d6134ac8d68215221e9423a745c2e8971d310bdc5e3a2660bebc55b0c05f6c064529b2863cc0e303620300d48370fc18437587af492ddfba692d19c6
-  data.tar.gz: 624fbc993b3f2c8e185f754c1f2dfd737fd2820603d0bac6510305834d9f0b15d4d6e178f06c88d9c04f4e0adc1c8917fb10dc7e004bbc03851047d0ca2ed6c0
+  metadata.gz: bb1f8eed6401abdbc008c6616b21d77e77d91dff6fe53ac19cecea03a313c260fc3aa6fe042aa3d98c46c030adc426eacd76a042730fe562dde5a775e870ee3e
+  data.tar.gz: 1b7fefa06146fd81b733fd4ecf8c04ab4385bab3669875fc9cd0fad2fe1fe170b75db70202d305d344b8bfee02f9741ba9aa68aae96cb5e6629833e6032ed350

data/README.md

@@ -23,8 +23,7 @@ Graphs can easily be implemented with objects.
 
 All classes follow the common principles and the common API above.
 
-- [LucidDocument](docs/data_document.md) - for textual data with fields
-- [LucidObject](docs/data_object.md) - for objects with attributes
+- [LucidObject](docs/data_object.md) - for objects with text_fields or attributes
 - [LucidFile](docs/data_file.md) - for files like images, pdfs, etc.
 - [LucidQuery](docs/data_query.md) - for isomorphic queries.
 
@@ -32,7 +31,6 @@ All classes follow the common principles and the common API above.
 
 isomorfeus-data relies on:
 
-- [isomorfeus-ferret](https://github.com/isomorfeus/isomorfeus-ferret) as storage and index for documents
-- [isomorfeus-hamster](https://github.com/isomorfeus/isomorfeus-hamster) as storage and index for objects
+- [isomorfeus-ferret](https://github.com/isomorfeus/isomorfeus-ferret) as storage and index for objects
 - the Filesystem for files
 - [Oj](https://github.com/ohler55/oj) for serialization

data/lib/isomorfeus/data/config.rb

@@ -62,8 +62,6 @@ module Isomorfeus
       attr_accessor :ferret_path
       attr_accessor :data_documents_path
 
-      attr_accessor :hamster_path
-      attr_accessor :hamster_mapsize
       attr_accessor :data_object_envs_path
       attr_accessor :data_object_idxs_path
     end
@@ -76,11 +74,5 @@ module Isomorfeus
     # documents and indices
     self.ferret_path = File.expand_path(File.join(self.storage_path, 'ferret'))
     self.data_documents_path = File.expand_path(File.join(self.ferret_path, 'documents'))
-
-    # objects, nodes and edges
-    self.hamster_path = File.expand_path(File.join(self.storage_path, 'hamster'))
-    self.hamster_mapsize = 4_294_967_296
-    self.data_object_envs_path = File.expand_path(File.join(self.hamster_path, 'object_envs'))
-    self.data_object_idxs_path = File.expand_path(File.join(self.hamster_path, 'object_idxs'))
   end
 end

data/lib/isomorfeus/data/field_support.rb

@@ -3,7 +3,13 @@ module Isomorfeus
     module FieldSupport
       def self.included(base)
         base.instance_exec do
+          def field_types
+            # determines how the item is stored in the index
+            @field_types ||= {}
+          end
+
           def field_options
+            # options for the index
             @field_options ||= {}
           end
 
@@ -11,8 +17,77 @@ module Isomorfeus
             @field_conditions ||= {}
           end
 
+          def _register_field(name, type, options)
+            field_options[name] = {}
+            field_options[name][:default_boost] = options.delete(:default_boost) if options.key?(:default_boost)
+            field_options[name][:default_boost] = options.delete(:default_boost) if options.key?(:boost)
+            field_options[name][:index] = options.delete(:index) { :no }
+            field_options[name][:store] = options.delete(:store) if options.key?(:store)
+            field_options[name][:compress] = options.delete(:compress) if options.key?(:compress)
+            if field_options[name][:index] == :yes
+              field_options[name][:term_vector] = options.delete(:term_vector) { type == :text_field ? :with_positions_offsets : :no }
+            elsif field_options[name][:index] == :no
+              field_options[name][:term_vector] = :no
+            end
+            field_conditions[name] = options
+            field_types[name] = type
+          end
+
+          def text_field(name, options = {})
+            field(name, :text_field, options)
+          end
+
+          def attribute(name, options = {})
+            field(name, :attribute, options)
+          end
+
+          def object(name, options = {})
+            default_options = { is_a: LucidObject::Mixin, allow_nil: true }
+            field(name, :object, default_options.merge(options))
+          end
+
+          def field?(field_name)
+            field_options.key?(field_name)
+          end
+
+          def text_field?(field_name)
+            field_types[field_name] == :text_field
+          end
+
+          def attribute?(attr_name)
+            field_types[attr_name] == :attribute
+          end
+
+          def object?(obj_name)
+            field_types[obj_name] == :object
+          end
+
           def valid_field?(field_name, val)
-            Isomorfeus::Props::Validator.new(self.name, field_name, val, field_conditions[field_name]).validate!
+            return Isomorfeus::Props::Validator.new(self.name, field_name, val, field_conditions[field_name]).validate! if field?(field_name)
+            false
+          rescue
+            false
+          end
+
+          def valid_text_field?(field_name, val)
+            return valid_field?(field_name, val) if text_field?(field_name)
+            false
+          rescue
+            false
+          end
+
+          def valid_attribute?(attr_name, val)
+            return valid_field?(attr_name, val) if attribute?(attr_name)
+            false
+          rescue
+            false
+          end
+
+          def valid_object?(obj_name, val)
+            return valid_field?(obj_name, val) if object?(obj_name)
+            false
+          rescue
+            false
           end
 
           def validate
@@ -20,14 +95,14 @@ module Isomorfeus
           end
 
           def _validate_field(field_name, val)
-            Isomorfeus.raise_error(message: "#{self.name}: No such field declared: '#{field_name}'!") unless field_conditions.key?(field_name)
+            Isomorfeus.raise_error(message: "#{self.name}: No such text_field, attribute or object declared: '#{field_name}'!") unless field_conditions.key?(field_name)
             Isomorfeus::Props::Validator.new(self.name, field_name, val, field_conditions[field_name]).validated_value
           end
 
           def _validate_fields(fields)
             field_conditions.each_key do |field_name|
               if field_conditions[field_name].key?(:required) && field_conditions[field_name][:required] && !fields.key?(field_name)
-                Isomorfeus.raise_error(message: "Required field #{field_name} not given!")
+                Isomorfeus.raise_error(message: "Required #{field_types[field_name]} #{field_name} not given!")
               end
               fields[field_name] = _validate_field(field_name, fields[field_name])
             end
@@ -42,28 +117,19 @@ module Isomorfeus
           self.class._validate_fields(fields)
         end
 
-        def exclude_fields(*flds)
-          @_excluded_fields = flds
-        end
-
-        def select_fields(*flds)
-          @_selected_fields = flds
-        end
-
         if RUBY_ENGINE == 'opal'
           base.instance_exec do
-            def field(name, options = {})
-              field_options[name] = {}
-              field_options[name][:default_boost] = options.delete(:default_boost) if options.key?(:default_boost)
-              field_options[name][:default_boost] = options.delete(:default_boost) if options.key?(:boost)
-              field_options[name][:index] = options.delete(:index) if options.key?(:index)
-              field_options[name][:store] = options.delete(:store) if options.key?(:store)
-              field_options[name][:compress] = options.delete(:compress) if options.key?(:compress)
-              field_options[name][:term_vector] = options.delete(:term_vector) if options.key?(:term_vector)
-              field_conditions[name] = options
+            def field(name, type, options)
+              _register_field(name, type, options)
 
-              define_method(name) do
-                _get_field(name)
+              if type == :object
+                define_method(name) do
+                  _get_object(name)
+                end
+              else
+                define_method(name) do
+                  _get_field(name)
+                end
               end
 
               define_method("#{name}=") do |val|
@@ -109,6 +175,15 @@ module Isomorfeus
             }
           end
 
+          def _get_object(name)
+            return @_changed_fields[name] if @_changed_fields.key?(name)
+            return @_objects[name] if @_objects.key?(name)
+            path = @_store_path + [name]
+            sid_or_array = Redux.fetch_by_path(*path)
+            return nil if `(sid_or_array === null)`
+            @_objects[name] = Isomorfeus.instance_from_sid(sid_or_array) if sid_or_array
+          end
+
           def fields
             raw_fields = Redux.fetch_by_path(*@_store_path)
             hash = Hash.new(raw_fields)
@@ -118,29 +193,15 @@ module Isomorfeus
 
           def _get_selected_fields
             sel_fields = fields.dup
-            if @_selected_fields && !@_selected_fields.empty?
-              sel_fields.each_key do |fld|
-                unless @_selected_fields.include?(fld) || @_selected_fields.include?(fld)
-                  sel_fields.delete(fld)
-                end
-              end
-            end
-            if @_excluded_fields && !@_excluded_fields.empty?
-              @_excluded_fields.each { |fld| sel_fields.delete(fld) }
+            self.class.field_types.each do |field, type|
+              sel_fields[field] = sel_fields[field]&.sid if type == :object
             end
             sel_fields
           end
         else
           base.instance_exec do
-            def field(name, options = {})
-              field_options[name] = {}
-              field_options[name][:default_boost] = options.delete(:default_boost) if options.key?(:default_boost)
-              field_options[name][:default_boost] = options.delete(:default_boost) if options.key?(:boost)
-              field_options[name][:index] = options.delete(:index) if options.key?(:index)
-              field_options[name][:store] = options.delete(:store) if options.key?(:store)
-              field_options[name][:compress] = options.delete(:compress) if options.key?(:compress)
-              field_options[name][:term_vector] = options.delete(:term_vector) if options.key?(:term_vector)
-              field_conditions[name] = options
+            def field(name, type, options)
+              _register_field(name, type, options)
 
               define_method(name) do
                 @_raw_fields[name]
@@ -160,16 +221,15 @@ module Isomorfeus
 
           def _get_selected_fields
             sel_fields = fields.transform_keys(&:to_s)
-            if @_selected_fields && !@_selected_fields.empty?
-              sel_fields.each_key do |fld|
-                unless @_selected_fields.include?(fld.to_sym) || @_selected_fields.include?(fld)
-                  sel_fields.delete(fld)
-                end
+            ft = self.class.field_types
+            self.class.field_conditions.each do |field, options|
+              field_s = field.to_s
+              if options[:server_only]
+                sel_fields.delete(field_s)
+              elsif ft[field] == :object
+                sel_fields[field_s] = sel_fields[field_s]&.sid
               end
             end
-            if @_excluded_fields && !@_excluded_fields.empty?
-              @_excluded_fields.each { |fld| sel_fields.delete(fld.to_s) }
-            end
             sel_fields
           end
         end

data/lib/isomorfeus/data/generic_class_api.rb

@@ -62,8 +62,12 @@ module Isomorfeus
           Promise.new.resolve(destroy(key: key))
         end
 
-        def load(key:)
-          data = instance_exec(key: key, &@_load_block)
+        def load(key:, _already_loaded: {})
+          data = if @_load_block.arity == 1
+                   instance_exec(key: key, &@_load_block)
+                 else
+                   instance_exec(key: key, _already_loaded: _already_loaded, &@_load_block)
+                 end
           return nil if data.nil?
           return data if data.class.to_s == self.to_s
           Isomorfeus.raise_error(message: "#{self}: execute_load must return a instance of #{self} or nil. Returned was: #{data.class}.")
@@ -95,6 +99,10 @@ module Isomorfeus
         end
       end  # RUBY_ENGINE
 
+      def gen_ref_s(key)
+        "---iso-object-reference---#{self.name}---#{key}---"
+      end
+
       def gen_sid_s(key)
         "[#{self.name}|#{key}]"
       end

data/lib/isomorfeus/data/generic_instance_api.rb

@@ -25,6 +25,10 @@ module Isomorfeus
         "[#{@class_name}|#{@key}]"
       end
 
+      def ref_s
+        self.class.gen_ref_s(@key)
+      end
+
       if RUBY_ENGINE == 'opal'
         def loaded?
           Redux.fetch_by_path(*@_store_path) ? true : false
@@ -100,9 +104,14 @@ module Isomorfeus
           true
         end
 
-        def create
+        def create(_already_saved: {})
           previous_key = self.key
-          instance = instance_exec(&self.class.instance_variable_get(:@_create_block))
+          create_block = self.class.instance_variable_get(:@_create_block)
+          instance = if create_block.arity == 0
+                       instance_exec(&create_block)
+                     else
+                       instance_exec(_already_saved: _already_saved, &create_block)
+                     end
           return nil unless instance
           Isomorfeus.raise_error(message: "#{self.to_s}: execute_create must return self or nil. Returned was: #{instance.class}.") if instance != self
           instance_variable_set(:@previous_key, previous_key) if key != previous_key
@@ -129,9 +138,14 @@ module Isomorfeus
           Promise.new.resolve(reload)
         end
 
-        def save
+        def save(_already_saved: {})
           previous_key = self.key
-          instance = instance_exec(&self.class.instance_variable_get(:@_save_block))
+          save_block = self.class.instance_variable_get(:@_save_block)
+          instance = if save_block.arity == 0
+                       instance_exec(&save_block)
+                     else
+                       instance_exec(_already_saved: _already_saved, &save_block)
+                     end
           return nil unless instance
           Isomorfeus.raise_error(message: "#{self.to_s}: execute_save must return self or nil. Returned was: #{instance.class}.") if instance != self
           instance_variable_set(:@previous_key, previous_key) if key != previous_key

data/lib/isomorfeus/data/handler/generic.rb

@@ -6,7 +6,6 @@ module Isomorfeus
       class Generic < LucidHandler::Base
         # responsible for loading:
         # LucidObject
-        # LucidDocument
         # LucidFile
         # LucidQuery
 

data/lib/isomorfeus/data/object_accelerator.rb

@@ -1,75 +1,140 @@
 module Isomorfeus
   module Data
     class ObjectAccelerator
-      def self.finalize(ham_acc)
-        proc { ham_acc.close_index }
+      def self.finalize(acc)
+        proc { acc.close_store }
       end
 
-      attr_accessor :index
+      attr_reader :object_class, :object_class_name
+      attr_accessor :store
+
+      def initialize(ruby_class)
+        @object_class = ruby_class
+        @object_class_name = ruby_class.name
+        @class_cache = Isomorfeus.production?
+
+        @store_path = File.expand_path(File.join(Isomorfeus.data_documents_path, "#{@object_class_name.underscore}"))
+        open_store
 
-      def initialize(object_class_name, &block)
-        if block_given?
-          res = block.call(self)
-          @index = res unless @index
-        else
-          @index_path = File.expand_path(File.join(Isomorfeus.data_object_idxs_path, object_class_name.underscore))
-          open_index
-        end
         ObjectSpace.define_finalizer(self, self.class.finalize(self))
       end
 
-      def destroy_index
-        close_index
-        FileUtils.rm_rf(@index_path)
+      def object_from_ref(ref, already_loaded)
+        _, iso, type_class_name, key = ref.split('---')
+        raise "not a valid object reference '#{ref}'" unless iso == "iso-object-reference"
+        raise "invalid data class #{type_class_name}" unless Isomorfeus.valid_data_class_name?(type_class_name)
+        type_class = Isomorfeus.cached_data_class(type_class_name)
+        type_class.load(key: key, _already_loaded: already_loaded)
       end
 
-      def close_index
-        @index.close
+      def serialize(obj)
+        Oj.dump(obj, mode: :object, circular: true, class_cache: @class_cache)
       end
 
-      def create_doc(document)
-        @index.add_document(document)
+      def unserialize(v)
+        Oj.load(v, mode: :object, circular: true, class_cache: @class_cache)
       end
 
-      def destroy_doc(key)
-        id = get_doc_id(key)
-        @index.delete(id) if id
-        true
+      def destroy_store
+        close_store
+        FileUtils.rm_rf(@store_path)
+      end
+
+      def close_store
+        @store.close
       end
 
-      def load_doc(key)
-        id = get_doc_id(key)
-        @index.doc(id)&.load&.to_h if id
+      def search_each(query, options, &block)
+        @store.search_each(query, options, &block)
       end
 
-      def save_doc(key, document)
-        id = get_doc_id(key)
+      def each(&block)
+        ft = @object_class.field_types
+        @store.each do |doc|
+          hash = doc.to_h do |k, v|
+            [k, unserialize_or_load(v, ft[k], {})]
+          end
+          block.call hash
+        end
+      end
+
+      def create_object(key, fields, already_saved)
+        ft = @object_class.field_types
+        hash = fields.to_h do |k, v|
+          [k, serialize_or_save(v, ft[k], already_saved)]
+        end
+        @store.add_document(hash.merge!({key: key}))
+      end
+
+      def destroy_object(key)
+        id = get_object_id(key)
+        @store.delete(id) if id
+      end
+
+      def load_object(key: nil, id: nil, already_loaded: {})
+        hash = nil
+        id = get_object_id(key) if key
         if id
-          @index.update(id, document)
-          true
+          ft = @object_class.field_types
+          hash = @store.doc(id)&.to_h do |k, v|
+            [k, unserialize_or_load(v, ft[k], already_loaded)]
+          end
         end
+        hash
       end
 
-      def search_each(query, options, &block)
-        @index.search_each(query, options, &block)
+      def save_object(key, fields, already_saved)
+        id = get_object_id(key)
+        raise "object not created yet" unless id
+        ft = @object_class.field_types
+        hash = fields.to_h do |k, v|
+          [k, serialize_or_save(v, ft[k], already_saved)]
+        end
+        @store.update(id, hash.merge!({key: key}))
+        true
       end
 
       private
 
-      def get_doc_id(key)
+      def unserialize_or_load(v, t, already_loaded)
+        return unserialize(v) if t == :attribute
+        if t == :object && v
+          return nil if v.empty?
+          return object_from_ref(v, already_loaded)
+        end
+        v
+      end
+
+      def serialize_or_save(v, t, already_saved)
+        return serialize(v) if t == :attribute
+        return create_or_save(v, already_saved) if t == :object && v
+        v
+      end
+
+      def create_or_save(v, already_saved)
+        if get_object_id(v.key)
+          v.save(_already_saved: already_saved)
+        else
+          v.create
+        end
+        v.ref_s
+      end
+
+      def get_object_id(key)
         # special characters must be escaped, characters taken from the ferret query parser documentation
         escaped_key = key.gsub(/([\\\&\:\(\)\[\]\{\}\!\"\~\^\|\<\>\=\*\?\+\-\s])/, '\\\\\1')
-        top_docs = @index.search("sid_s_attr:\"#{escaped_key}\"", limit: 1)
+        top_docs = @store.search("key:\"#{escaped_key}\"", limit: 1)
         id = top_docs.hits[0].doc if top_docs.total_hits == 1
       end
 
-      def open_index
-        FileUtils.mkdir_p(@index_path) unless Dir.exist?(@index_path)
-        @index = Isomorfeus::Ferret::Index::Index.new(path: @index_path, key: :sid_s_attr, auto_flush: true, lock_retry_time: 5)
-        @index.field_infos.add_field(:attribute, store: :no, term_vector: :no) unless @index.field_infos[:attribute]
-        @index.field_infos.add_field(:class_name, store: :no, term_vector: :no) unless @index.field_infos[:class_name]
-        @index.field_infos.add_field(:value, store: :no) unless @index.field_infos[:value]
-        @index.field_infos.add_field(:sid_s_attr, store: :yes, term_vector: :no) unless @index.field_infos[:sid_s_attr]
+      def open_store
+        FileUtils.mkdir_p(Isomorfeus.data_documents_path) unless Dir.exist?(Isomorfeus.data_documents_path)
+        field_infos = Isomorfeus::Ferret::Index::FieldInfos.new(store: :yes, index: :yes, term_vector: :no)
+        @store = Isomorfeus::Ferret::Index::Index.new(path: @store_path, key: :key, auto_flush: true, lock_retry_time: 5, field_infos: field_infos)
+        @store.field_infos.add_field(:key, store: :yes, index: :yes, term_vector: :yes) unless @store.field_infos[:key]
+        @object_class.field_options.each do |field, options|
+          @store.field_infos.add_field(field, options) unless @store.field_infos[field]
+        end
       end
     end
   end

data/lib/isomorfeus/data/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Data
-    VERSION = '2.4.2'
+    VERSION = '2.5.0'
   end
 end

data/lib/isomorfeus-data.rb

@@ -9,7 +9,6 @@ require 'isomorfeus-policy'
 require 'isomorfeus-transport'
 require 'isomorfeus-i18n'
 require 'isomorfeus/data/config'
-require 'isomorfeus/data/attribute_support'
 require 'isomorfeus/data/field_support'
 require 'isomorfeus/data/generic_class_api'
 require 'isomorfeus/data/generic_instance_api'
@@ -28,17 +27,11 @@ else
   require 'active_support/core_ext/hash'
 
   require 'isomorfeus-ferret'
-  require 'isomorfeus/data/document_accelerator'
-
-  require 'isomorfeus-hamster'
-  require 'isomorfeus/data/object_expander'
   require 'isomorfeus/data/object_accelerator'
 
   require 'isomorfeus_data/lucid_query_result'
   require 'isomorfeus_data/lucid_object/mixin'
   require 'isomorfeus_data/lucid_object/base'
-  require 'isomorfeus_data/lucid_document/mixin'
-  require 'isomorfeus_data/lucid_document/base'
   require 'isomorfeus_data/lucid_query/mixin'
   require 'isomorfeus_data/lucid_query/base'
   require 'isomorfeus_data/lucid_file/mixin'