terrestrial 0.3.0 → 0.5.0

data/lib/terrestrial/configurations/mapping_config_options_proxy.rb

@@ -0,0 +1,97 @@
+module Terrestrial
+  module Configurations
+    class MappingConfigOptionsProxy
+      def initialize(configuration, mapping_name)
+        @configuration = configuration
+        @mapping_name = mapping_name
+      end
+
+      attr_reader :configuration, :mapping_name
+      private :configuration, :mapping_name
+
+      def relation_name(name)
+        add_override(relation_name: name)
+      end
+      alias_method :table_name, :relation_name
+
+      def subset(subset_name, &block)
+        configuration.add_subset(mapping_name, subset_name, block)
+      end
+
+      def has_many(*args)
+        add_association(:has_many, args)
+      end
+
+      def has_many_through(*args)
+        add_association(:has_many_through, args)
+      end
+
+      def belongs_to(*args)
+        add_association(:belongs_to, args)
+      end
+
+      def fields(field_names)
+        add_override(fields: field_names)
+      end
+
+      def primary_key(field_names)
+        add_override(primary_key: field_names)
+      end
+
+      def use_database_id(&block)
+        add_override(use_database_id: true)
+        block && add_override(database_id_setter: block)
+      end
+
+      def database_owned_field(field_name, &object_setter)
+        configuration.overrides.fetch(mapping_name)[:database_owned_fields_setter_map] ||= {}
+        db_owned_fields = configuration.overrides.fetch(mapping_name).fetch(:database_owned_fields_setter_map)
+
+        db_owned_fields.merge!({field_name => object_setter})
+      end
+
+      def database_default_field(field_name, &object_setter)
+        configuration.overrides.fetch(mapping_name)[:database_default_fields_setter_map] ||= {}
+        db_default_fields = configuration.overrides.fetch(mapping_name).fetch(:database_default_fields_setter_map)
+
+        db_default_fields.merge!({field_name => object_setter})
+      end
+
+      def created_at_timestamp(field_name = Default, &block)
+        add_override(created_at_field: field_name)
+        block && add_override(created_at_setter: block)
+      end
+
+      def updated_at_timestamp(field_name = Default, &block)
+        add_override(updated_at_field: field_name)
+        block && add_override(updated_at_setter: block)
+      end
+
+      def factory(callable)
+        add_override(factory: callable)
+      end
+
+      def class(entity_class)
+        add_override('class': entity_class)
+      end
+
+      def class_name(class_name)
+        add_override(class_name: class_name)
+      end
+
+      def serializer(serializer_func)
+        add_override(serializer: serializer_func)
+      end
+
+      private
+
+      def add_override(*args)
+        configuration.add_override(mapping_name, *args)
+      end
+
+      def add_association(*args)
+        configuration.add_association(mapping_name, *args)
+      end
+    end
+  end
+end

data/lib/terrestrial/deleted_record.rb

@@ -1,20 +1,24 @@
-require "terrestrial/abstract_record"
+require "terrestrial/record"
 
 module Terrestrial
-  class DeletedRecord < AbstractRecord
+  class DeletedRecord < Record
+    def initialize(mapping, attributes, depth)
+      @mapping = mapping
+      @attributes = attributes
+      @depth = depth
+    end
+
+    attr_reader :mapping, :attributes, :depth
+
     def if_delete(&block)
       block.call(self)
       self
     end
 
-    def subset?(_other)
-      false
-    end
-
     protected
 
-    def operation
-      :delete
+    def new_with_attributes(new_attributes)
+      self.class.new(mapping, new_attributes, depth)
     end
   end
 end

data/lib/terrestrial/dirty_map.rb

@@ -13,15 +13,12 @@ module Terrestrial
     end
 
     def load(record)
-      storage.store(hash_key(record), deep_clone(record))
+      storage.store(hash_key(record), record.deep_clone)
       record
     end
 
     def dirty?(record)
-      record_as_loaded = storage.fetch(hash_key(record), NotFound)
-      return true if record_as_loaded == NotFound
-
-      !record.subset?(record_as_loaded)
+      !same_as_loaded?(record) || deleted?(record)
     end
 
     def reject_unchanged_fields(record)
@@ -36,12 +33,23 @@ module Terrestrial
 
     NotFound = Module.new
 
-    def hash_key(record)
-      deep_clone([record.namespace, record.identity])
+    def same_as_loaded?(record)
+      record_as_loaded = storage.fetch(hash_key(record), NotFound)
+
+      if record_as_loaded == NotFound
+        false
+      else
+        record.subset?(record_as_loaded)
+      end
+    end
+
+    def deleted?(record)
+      record.if_delete { return true }
+      return false
     end
 
-    def deep_clone(record)
-      Marshal.load(Marshal.dump(record))
+    def hash_key(record)
+      [record.namespace, record.identity]
     end
   end
 end

data/lib/terrestrial/functional_pipeline.rb

@@ -0,0 +1,64 @@
+module Terrestrial
+  class FunctionalPipeline
+    def self.from_array(steps = [])
+      new(steps.map { |name, func| Step.new(name, func) })
+    end
+
+    def initialize(steps = [])
+      @steps = steps
+    end
+
+    def call(args, &block)
+      result = execution_result([[:input, args]], &block)
+
+      [result.last.last, result]
+    end
+
+    def describe
+      @steps.map(&:name)
+    end
+
+    def append(name, func)
+      self.class.new(@steps + [Step.new(name, func)])
+    end
+
+    def take_until(step_name)
+      step = @steps.detect { |step| step.name == step_name }
+      last_step_index = @steps.index(step)
+      steps = @steps.slice(0..last_step_index)
+
+      self.class.new(steps)
+    end
+
+    def drop_until(step_name)
+      step = @steps.detect { |step| step.name == step_name }
+      first_step_index = @steps.index(step) + 1
+      steps = @steps.slice(first_step_index..-1)
+
+      self.class.new(steps)
+    end
+
+    private
+
+    def execution_result(initial_state, &block)
+      @steps.reduce(initial_state) { |state, step|
+        new_value = step.call(state.last.last)
+        block && block.call(step.name, new_value)
+        state + [ [step.name, new_value] ]
+      }
+    end
+
+    class Step
+      def initialize(name, func)
+        @name = name
+        @func = func
+      end
+
+      attr_reader :name, :func
+
+      def call(*args, &block)
+        func.call(*args, &block)
+      end
+    end
+  end
+end

data/lib/terrestrial/inspection_string.rb

@@ -1,8 +1,13 @@
 module Terrestrial
   module InspectionString
     def inspect
+      original_inspect_string = super
+      # this is kind of a silly way of getting the object id hex string but
+      # multiple Ruby versions have changed how this calculated.
+      hex_object_id = /#{self.class.to_s}:0x([0-9a-f]+)/.match(original_inspect_string)[1]
+
       (
-        ["\#<#{self.class.name}:0x#{sprintf("%014x", (object_id.<<(1)))}"] +
+        ["\#<#{self.class.to_s}:0x#{hex_object_id}"] +
         inspectable_properties.map { |name|
           [
             name,

data/lib/terrestrial/lazy_object_proxy.rb

@@ -1,4 +1,5 @@
 module Terrestrial
+  # TODO: This should do a better job of showing what will be loaded when it is inspected
   class LazyObjectProxy
     include InspectionString
 

data/lib/terrestrial/many_to_many_association.rb

@@ -3,9 +3,10 @@ require "terrestrial/dataset"
 
 module Terrestrial
   class ManyToManyAssociation
-    def initialize(mapping_name:, join_mapping_name:, foreign_key:, key:, proxy_factory:, association_foreign_key:, association_key:, order:)
+    def initialize(mapping_name:, join_mapping_name:, join_dataset:, foreign_key:, key:, proxy_factory:, association_foreign_key:, association_key:, order:)
       @mapping_name = mapping_name
       @join_mapping_name = join_mapping_name
+      @join_dataset = join_dataset
       @foreign_key = foreign_key
       @key = key
       @proxy_factory = proxy_factory
@@ -18,10 +19,18 @@ module Terrestrial
       [mapping_name, join_mapping_name]
     end
 
+    def outgoing_foreign_keys
+      []
+    end
+
+    def local_foreign_keys
+      []
+    end
+
     attr_reader :mapping_name, :join_mapping_name
 
-    attr_reader :foreign_key, :key, :proxy_factory, :association_key, :association_foreign_key, :order
-    private     :foreign_key, :key, :proxy_factory, :association_key, :association_foreign_key, :order
+    attr_reader :join_dataset, :foreign_key, :key, :proxy_factory, :association_key, :association_foreign_key, :order
+    private     :join_dataset, :foreign_key, :key, :proxy_factory, :association_key, :association_foreign_key, :order
 
     def build_proxy(data_superset:, loader:, record:)
       proxy_factory.call(
@@ -37,36 +46,31 @@ module Terrestrial
     end
 
     def eager_superset((superset, join_superset), (associated_dataset))
-      join_data = Dataset.new(
+      subselect_data = Dataset.new(
         join_superset
           .where(foreign_key => associated_dataset.select(association_key))
           .to_a
       )
 
       eager_superset = Dataset.new(
-        superset.where(key => join_data.select(association_foreign_key)).to_a
+        superset.where(key => subselect_data.select(association_foreign_key)).to_a
       )
 
       [
         eager_superset,
-        join_data,
+        subselect_data,
       ]
     end
 
     def build_query((superset, join_superset), parent_record)
-      ids = join_superset
-              .where(foreign_key => foreign_key_value(parent_record))
-              .select(association_foreign_key)
+      subselect_ids = join_superset
+        .where(foreign_key => foreign_key_value(parent_record))
+        .select(association_foreign_key)
 
       order
-        .apply(
-          superset.where(
-            key => ids
-          )
-        )
-        .lazy.map { |record|
-          [record, [foreign_keys(parent_record, record)]]
-        }
+        .apply(superset.where(key => subselect_ids))
+        .lazy
+        .map { |record| [record, [foreign_keys(parent_record, record)]] }
     end
 
     def dump(parent_record, collection, depth, &block)
@@ -102,13 +106,23 @@ module Terrestrial
           depth + depth_modifier,
         )
 
-        fks = foreign_keys(parent_record, record)
+        join_foreign_keys = foreign_keys(parent_record, record)
         join_record_depth = depth + join_record_depth_modifier
 
+        # TODO: This is a bit hard to figure out
+        #
+        # The block defined in GraphSerializer#updated_nodes_recursive (inspect the block to confirm)
+        # join_foreign_keys is the two foreign key values in a hash
+        # the hash is two of the arugments here
+        # first one is normally an object to be serialized but serializing this hash will just return the same hash
+        # second one is the foreign keys that would need to accompany the object
+        #
+        # Passing it twice like this is allows it to go though the GraphSerializer like a regular user defined object
+
         join_records = block.call(
           join_mapping_name,
-          fks,
-          fks,
+          join_foreign_keys, # normally this is the object which gets serialized
+          join_foreign_keys, # normally this is the foreign key data the object doesn't know about
           join_record_depth
         ).flatten(1)
 

data/lib/terrestrial/many_to_one_association.rb

@@ -13,6 +13,14 @@ module Terrestrial
       [mapping_name]
     end
 
+    def outgoing_foreign_keys
+      []
+    end
+
+    def local_foreign_keys
+      [foreign_key]
+    end
+
     attr_reader :mapping_name
 
     attr_reader :foreign_key, :key, :proxy_factory
@@ -42,17 +50,17 @@ module Terrestrial
 
     def dump(parent_record, collection, depth, &block)
       collection
+        .reject(&:nil?)
         .flat_map { |object|
           block.call(mapping_name, object, _foreign_key_does_not_go_here = {}, depth + depth_modifier)
         }
-        .reject(&:nil?)
     end
     alias_method :delete, :dump
 
     def extract_foreign_key(record)
       {
-        foreign_key => (record && record.fetch(key)),
-      }
+        foreign_key => record.fetch(key),
+      }.reject { |_k, v| v.nil? }
     end
 
     private

data/lib/terrestrial/one_to_many_association.rb

@@ -13,6 +13,15 @@ module Terrestrial
     def mapping_names
       [mapping_name]
     end
+
+    def outgoing_foreign_keys
+      [foreign_key]
+    end
+
+    def local_foreign_keys
+      []
+    end
+
     attr_reader :mapping_name
 
     attr_reader :foreign_key, :key, :order, :proxy_factory

data/lib/terrestrial/public_conveniencies.rb

@@ -1,9 +1,11 @@
 require "terrestrial/identity_map"
 require "terrestrial/dirty_map"
-require "terrestrial/upserted_record"
+require "terrestrial/upsert_record"
 require "terrestrial/relational_store"
 require "terrestrial/configurations/conventional_configuration"
 require "terrestrial/inspection_string"
+require "terrestrial/functional_pipeline"
+require "terrestrial/adapters/sequel_postgres_adapter"
 
 module Terrestrial
   class ObjectStore
@@ -31,23 +33,42 @@ module Terrestrial
   end
 
   module PublicConveniencies
-    def config(database_connection)
-      Configurations::ConventionalConfiguration.new(database_connection)
-    end
-
-    def object_store(mappings:, datastore:)
+    def config(database, clock: Time)
       dirty_map = Private.build_dirty_map
       identity_map = Private.build_identity_map
 
-      stores = Hash[mappings.map { |name, _mapping|
+      Configurations::ConventionalConfiguration.new(
+        datastore: Private.datastore_adapter(database),
+        clock: clock,
+        dirty_map: dirty_map,
+        identity_map: identity_map,
+      )
+    end
+
+    def object_store(config:)
+      load_pipeline = Private.build_load_pipeline(
+        dirty_map: config.dirty_map,
+        identity_map: config.identity_map,
+      )
+      dump_pipeline = Private.build_dump_pipeline(
+        dirty_map: config.dirty_map,
+        datastore: config.datastore,
+        clock: config.clock,
+      )
+
+      mappings = config.mappings
+      mapping_names = mappings.keys
+      stores = Hash[mapping_names.map { |mapping_name|
         [
-          name,
-          Private.single_type_store(
+          mapping_name,
+          Private.relational_store(
+            name: mapping_name,
             mappings: mappings ,
-            name: name,
-            datastore: datastore,
-            identity_map: identity_map,
-            dirty_map: dirty_map,
+            datastore: config.datastore,
+            identity_map: config.identity_map,
+            dirty_map: config.dirty_map,
+            load_pipeline: load_pipeline,
+            dump_pipeline: dump_pipeline,
           )
         ]
       }]
@@ -58,24 +79,13 @@ module Terrestrial
     module Private
       module_function
 
-      def single_type_store(mappings:, name:, datastore:, identity_map:, dirty_map:)
-        dataset = datastore[mappings.fetch(name).namespace]
-
+      def relational_store(mappings:, name:, datastore:, identity_map:, dirty_map:, load_pipeline:, dump_pipeline:)
         RelationalStore.new(
           mappings: mappings,
           mapping_name: name,
           datastore: datastore,
-          dataset: dataset,
-          load_pipeline: build_load_pipeline(
-            dirty_map: dirty_map,
-            identity_map: identity_map,
-          ),
-          dump_pipeline: build_dump_pipeline(
-            dirty_map: dirty_map,
-            transaction: datastore.method(:transaction),
-            upsert: method(:upsert_record).curry.call(datastore),
-            delete: method(:delete_record).curry.call(datastore),
-          )
+          load_pipeline: load_pipeline,
+          dump_pipeline: dump_pipeline,
         )
       end
 
@@ -87,10 +97,23 @@ module Terrestrial
         DirtyMap.new(storage)
       end
 
+      def datastore_adapter(datastore)
+        if datastore.is_a?(Terrestrial::Adapters::AbstractAdapter)
+          return datastore
+        end
+
+        case datastore.class.name
+        when "Sequel::Postgres::Database"
+          Adapters::SequelPostgresAdapter.new(datastore)
+        else
+          raise "No adapter found for #{datastore.inspect}"
+        end
+      end
+
       def build_load_pipeline(dirty_map:, identity_map:)
         ->(mapping, record, associated_fields = {}) {
           [
-            record_factory(mapping),
+            ->(record) { Record.new(mapping, record) },
             dirty_map.method(:load),
             ->(record) {
               attributes = record.to_h.select { |k,_v|
@@ -106,64 +129,24 @@ module Terrestrial
         }
       end
 
-      def build_dump_pipeline(dirty_map:, transaction:, upsert:, delete:)
-        ->(records) {
-          [
-            :uniq.to_proc,
-            ->(rs) { rs.select { |r| dirty_map.dirty?(r) } },
-            ->(rs) { rs.map { |r| dirty_map.reject_unchanged_fields(r) } },
-            ->(rs) { rs.sort_by(&:depth) },
-            ->(rs) {
-              transaction.call {
+      def build_dump_pipeline(dirty_map:, datastore:, clock:)
+        Terrestrial::FunctionalPipeline.from_array([
+          [:dedup, :uniq.to_proc],
+          [:sort_by_depth, ->(rs) { rs.sort_by(&:depth) }],
+          [:select_changed, ->(rs) { rs.select { |r| dirty_map.dirty?(r) } }],
+          [:remove_unchanged_fields, ->(rs) { rs.map { |r| dirty_map.reject_unchanged_fields(r) } }],
+          [:save_records, ->(rs) {
+            datastore.transaction {
                 rs.each { |r|
-                  r.if_upsert(&upsert)
-                  .if_delete(&delete)
+                  r.if_upsert(&datastore.method(:upsert))
+                  r.if_delete(&datastore.method(:delete))
                 }
               }
-            },
-            ->(rs) { rs.map { |r| dirty_map.load_if_new(r) } },
-          ].reduce(records) { |agg, operation|
-            operation.call(agg)
-          }
-        }
-      end
-
-      def record_factory(mapping)
-        ->(record_hash) {
-          identity = Hash[
-            mapping.primary_key.map { |field|
-              [field, record_hash.fetch(field)]
             }
-          ]
-
-          UpsertedRecord.new(
-            mapping.namespace,
-            identity,
-            record_hash,
-          )
-        }
-      end
-
-      def upsert_record(datastore, record)
-        row_count = 0
-        unless record.non_identity_attributes.empty?
-          row_count = datastore[record.namespace].
-            where(record.identity).
-            update(record.non_identity_attributes)
-        end
-
-        if row_count < 1
-          row_count = datastore[record.namespace].insert(record.to_h)
-        end
-
-        row_count
-      rescue Object => e
-        raise UpsertError.new(record.namespace, record.to_h, e)
+          ],
+          [:add_new_records_to_dirty_map, ->(rs) { rs.map { |r| dirty_map.load_if_new(r) } }],
+        ])
       end
-
-      def delete_record(datastore, record)
-        datastore[record.namespace].where(record.identity).delete
-      end
-    end
+   end
   end
 end