db_fuel 1.1.0.pre.alpha → 1.1.0.pre.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb563756df642d23e7d268cf0e7b35735300ddfc84da31e0ee11d7e138d41584
-  data.tar.gz: f5720ed0f4649ba82d0c233cf9ccbc4aff5d54d8d9c4267c830467f43f85cdf3
+  metadata.gz: 0d7fce3d73ed5e5c1cfabfe4a578cd00fc6e932c7b32d1f60a7e17968163b247
+  data.tar.gz: 88eab379caeac681a34a8d58cc5ea3d8de51d242638d7707d4938d00a99f490d
 SHA512:
-  metadata.gz: 16c63b0939d36f32cfc72baeefc20fbb88e7285d5c5e8266d337f55362909a6f751f7f2b1c535f6c231a11adba2693807c68e6af1b4d5eaaf4b14a9213589d01
-  data.tar.gz: a5143b01108dc065d98a5e085da06c8e82a5f0c084a0097128219e8c49542818a8ce754ef1af79be23d6493f259f745657bba4e74051cf856264fbe87a76dfde
+  metadata.gz: 91b4c18f3a081c296b005bbd6717f81d9595323dee3c69b7fca1594e4dfdf8063aba2052d54d6182bfa1a1440e3e8150117213961c4c92c010e7637ebb0f1560
+  data.tar.gz: b6e18bc0c58d6472465fe51fe06669bc54cb456526c6af1944de4c66b2e3e59281cf3ea21fbb54a78b9109e186536342b32dd2b15fc4561edc4ce4cc38abbe9b

data/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 New Jobs:
 
+* db_fuel/active_record/find_or_insert
 * db_fuel/active_record/insert
 * db_fuel/active_record/update
 

data/README.md

@@ -24,8 +24,9 @@ Refer to the [Burner](https://github.com/bluemarblepayroll/burner) library for m
 
 ### ActiveRecord Jobs
 
+* **db_fuel/active_record/find_or_insert** [table_name, attributes, debug, primary_key, register, separator, timestamps, unique_attributes]: An extension of the `db_fuel/active_record/insert` job that adds an existence check before sql insertion. The  `unique_attributes` will be converted to WHERE clauses for performing the existence check.
 * **db_fuel/active_record/insert** [table_name, attributes, debug, primary_key, register, separator, timestamps]: This job can take the objects in a register and insert them into a database table.  Attributes defines which object properties to convert to SQL.  Refer to the class and constructor specification for more detail.
-* **db_fuel/active_record/update** [table_name, attributes, debug, register, separator, timestamps, unique_keys]: This job can take the objects in a register and updates them within a database table.  Attributes defines which object properties to convert to SQL SET clauses while unique_keys translate to WHERE clauses.  Refer to the class and constructor specification for more detail.
+* **db_fuel/active_record/update** [table_name, attributes, debug, register, separator, timestamps, unique_attributes]: This job can take the objects in a register and updates them within a database table.  Attributes defines which object properties to convert to SQL SET clauses while unique_attributes translate to WHERE clauses.  Refer to the class and constructor specification for more detail.
 
 ### Dbee Jobs
 
@@ -195,6 +196,49 @@ Notes:
 * Since we specified the `primary_key`, the records' `id` attributes should be set to their respective primary key values.
 * Set `debug: true` to print out each INSERT statement in the output (not for production use.)
 
+#### Inserting Only New Records
+
+Another job `db_fuel/active_record/find_or_insert` allows for an existence check to performed each insertion.  If a record is found then it will not insert the record.  If `primary_key` is set then the existence check will also still set the primary key on the payload's respective object. We can build on the above insert example for only inserting new patients if their chart_number is unique:
+
+````ruby
+pipeline = {
+  jobs: [
+    {
+      name: :load_patients,
+      type: 'b/value/static',
+      register: :patients,
+      value: [
+        { chart_number: 'B0001', first_name: 'Bugs', last_name: 'Bunny' },
+        { chart_number: 'B0002', first_name: 'Babs', last_name: 'Bunny' }
+      ]
+    },
+    {
+      name: 'insert_patients',
+      type: 'db_fuel/active_record/insert',
+      register: :patients,
+      attributes: [
+        { key: :chart_number },
+        { key: :first_name },
+        { key: :last_name }
+      ],
+      table_name: 'patients',
+      primary_key: {
+        key: :id
+      },
+      unique_attributes: [
+        { key: :chart_number }
+      ]
+    }
+  ]
+}
+
+payload = Burner::Payload.new
+
+Burner::Pipeline.make(pipeline).execute(payload: payload)
+````
+
+Now only records where the chart_number does not match an existing record will be inserted.
+
 #### Updating Records
 
 Let's say we now want to update those records' last names:
@@ -219,7 +263,7 @@ pipeline = {
         { key: :last_name }
       ],
       table_name: 'patients',
-      unique_keys: [
+      unique_attributes: [
         { key: :chart_number }
       ]
     }
@@ -235,7 +279,7 @@ Each database record should have been updated with their new respective last nam
 
 Notes:
 
-* The unique_keys translate to WHERE clauses.
+* The `unique_attributes` translate to WHERE clauses.
 * Set `debug: true` to print out each UPDATE statement in the output (not for production use.)
 ## Contributing
 

data/lib/db_fuel.rb

@@ -17,4 +17,7 @@ require 'objectable'
 # General purpose classes used by the main job classes.
 require_relative 'db_fuel/modeling'
 
+# Internal logic used across jobs.
+require_relative 'db_fuel/db_provider'
+
 require_relative 'db_fuel/library'

data/lib/db_fuel/db_provider.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module DbFuel
+  # Intermediate internal API for Arel/ActiveRecord.  There is some overlap in job needs when
+  # it comes to the Arel interface so this class condenses down those needs into this class.
+  class DbProvider # :nodoc: all
+    attr_reader :arel_table
+
+    def initialize(table_name)
+      raise ArgumentError, 'table_name is required' if table_name.to_s.empty?
+
+      @arel_table = ::Arel::Table.new(table_name.to_s)
+
+      freeze
+    end
+
+    def first(object)
+      sql = first_sql(object)
+
+      ::ActiveRecord::Base.connection.exec_query(sql).first
+    end
+
+    def first_sql(object)
+      relation = arel_table.project(Arel.star).take(1)
+      manager  = apply_where(object, relation)
+
+      manager.to_sql
+    end
+
+    def insert_sql(object)
+      insert_manager(object).to_sql
+    end
+
+    def insert(object)
+      manager = insert_manager(object)
+
+      ::ActiveRecord::Base.connection.insert(manager)
+    end
+
+    def update(set_object, where_object)
+      manager = update_manager(set_object, where_object)
+
+      ::ActiveRecord::Base.connection.update(manager)
+    end
+
+    def update_sql(set_object, where_object)
+      update_manager(set_object, where_object).to_sql
+    end
+
+    private
+
+    def update_manager(set_object, where_object)
+      arel_row       = make_arel_row(set_object)
+      update_manager = ::Arel::UpdateManager.new.set(arel_row).table(arel_table)
+
+      apply_where(where_object, update_manager)
+    end
+
+    def apply_where(hash, manager)
+      (hash || {}).inject(manager) do |memo, (key, value)|
+        memo.where(arel_table[key].eq(value))
+      end
+    end
+
+    def insert_manager(object)
+      arel_row = make_arel_row(object)
+
+      ::Arel::InsertManager.new.insert(arel_row)
+    end
+
+    def make_arel_row(row)
+      row.map { |key, value| [arel_table[key], value] }
+    end
+  end
+end

data/lib/db_fuel/library.rb

@@ -7,6 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
+require_relative 'library/active_record/find_or_insert'
 require_relative 'library/active_record/insert'
 require_relative 'library/active_record/update'
 
@@ -16,10 +17,11 @@ require_relative 'library/dbee/range'
 module Burner
   # Open up Burner::Jobs and add registrations for this libraries jobs.
   class Jobs
-    register 'db_fuel/active_record/insert', DbFuel::Library::ActiveRecord::Insert
-    register 'db_fuel/active_record/update', DbFuel::Library::ActiveRecord::Update
+    register 'db_fuel/active_record/find_or_insert', DbFuel::Library::ActiveRecord::FindOrInsert
+    register 'db_fuel/active_record/insert',         DbFuel::Library::ActiveRecord::Insert
+    register 'db_fuel/active_record/update',         DbFuel::Library::ActiveRecord::Update
 
-    register 'db_fuel/dbee/query', DbFuel::Library::Dbee::Query
-    register 'db_fuel/dbee/range', DbFuel::Library::Dbee::Range
+    register 'db_fuel/dbee/query',                   DbFuel::Library::Dbee::Query
+    register 'db_fuel/dbee/range',                   DbFuel::Library::Dbee::Range
   end
 end

data/lib/db_fuel/library/active_record/base.rb

@@ -19,8 +19,8 @@ module DbFuel
         NOW_TYPE   = 'r/value/now'
         UPDATED_AT = :updated_at
 
-        attr_reader :arel_table,
-                    :attribute_renderers,
+        attr_reader :attribute_renderers,
+                    :db_provider,
                     :debug,
                     :resolver
 
@@ -34,19 +34,37 @@ module DbFuel
         )
           super(name: name, register: register)
 
-          @arel_table = ::Arel::Table.new(table_name.to_s)
-          @debug      = debug || false
-
           # set resolver first since make_attribute_renderers needs it.
-          @resolver = Objectable.resolver(separator: separator)
-
-          @attribute_renderers = Burner::Modeling::Attribute
-                                 .array(attributes)
-                                 .map { |a| Burner::Modeling::AttributeRenderer.new(a, resolver) }
+          @resolver            = Objectable.resolver(separator: separator)
+          @attribute_renderers = make_attribute_renderers(attributes)
+          @db_provider         = DbProvider.new(table_name)
+          @debug = debug || false
         end
 
         private
 
+        def make_attribute_renderers(attributes)
+          Burner::Modeling::Attribute
+            .array(attributes)
+            .map { |a| Burner::Modeling::AttributeRenderer.new(a, resolver) }
+        end
+
+        def transform(attribute_renderers, row, time)
+          attribute_renderers.each_with_object({}) do |attribute_renderer, memo|
+            value = attribute_renderer.transform(row, time)
+
+            resolver.set(memo, attribute_renderer.key, value)
+          end
+        end
+
+        def created_at_timestamp_attribute
+          timestamp_attribute(CREATED_AT)
+        end
+
+        def updated_at_timestamp_attribute
+          timestamp_attribute(UPDATED_AT)
+        end
+
         def timestamp_attribute(key)
           Burner::Modeling::Attribute.make(
             key: key,
@@ -61,18 +79,6 @@ module DbFuel
 
           output.detail(message)
         end
-
-        def make_arel_row(row)
-          row.map { |key, value| [arel_table[key], value] }
-        end
-
-        def transform(row, time)
-          attribute_renderers.each_with_object({}) do |attribute_renderer, memo|
-            value = attribute_renderer.transform(row, time)
-
-            resolver.set(memo, attribute_renderer.key, value)
-          end
-        end
       end
     end
   end

data/lib/db_fuel/library/active_record/find_or_insert.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'insert'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job is a slight enhancement to the insert job, in that it will only insert new
+      # records.  It will use the unique_keys to first run a query to see if it exists.
+      # Each unique_key becomes a WHERE clause.  If primary_key is specified and a record is
+      # found then the first record's id will be set to the primary_key.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class FindOrInsert < Insert
+        attr_reader :unique_attribute_renderers
+
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for sql-specific transformers
+        #                before insertion.
+        #
+        #   debug: If debug is set to true (defaults to false) then the SQL statements and
+        #          returned objects will be printed in the output.  Only use this option while
+        #          debugging issues as it will fill up the output with (potentially too much) data.
+        #
+        #   primary_key: If primary_key is present then it will be used to set the object's
+        #                property to the returned primary key from the INSERT statement.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then both created_at
+        #               and updated_at columns will automatically have their values set
+        #               to the current UTC timestamp.
+        #
+        #   unique_attributes: Each key will become a WHERE clause in order check for record
+        #                      existence before insertion attempt.
+        def initialize(
+          name:,
+          table_name:,
+          attributes: [],
+          debug: false,
+          primary_key: nil,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true,
+          unique_attributes: []
+        )
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            primary_key: primary_key,
+            register: register,
+            separator: separator,
+            timestamps: timestamps
+          )
+
+          @unique_attribute_renderers = make_attribute_renderers(unique_attributes)
+        end
+
+        def perform(output, payload)
+          total_inserted = 0
+          total_existed  = 0
+
+          payload[register] = array(payload[register])
+
+          payload[register].each do |row|
+            exists = existence_check_and_mutate(output, row, payload.time)
+
+            if exists
+              total_existed += 1
+              next
+            end
+
+            insert(output, row, payload.time)
+
+            total_inserted += 1
+          end
+
+          output.detail("Total Existed: #{total_existed}")
+          output.detail("Total Inserted: #{total_inserted}")
+        end
+
+        private
+
+        def existence_check_and_mutate(output, row, time)
+          unique_row = transform(unique_attribute_renderers, row, time)
+
+          first_sql = db_provider.first_sql(unique_row)
+          debug_detail(output, "Find Statement: #{first_sql}")
+
+          first_record = db_provider.first(unique_row)
+
+          return false unless first_record
+
+          if primary_key
+            id = resolver.get(first_record, primary_key.column)
+
+            resolver.set(row, primary_key.key, id)
+          end
+
+          debug_detail(output, "Record Exists: #{first_record}")
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/db_fuel/library/active_record/insert.rb

@@ -68,33 +68,41 @@ module DbFuel
           )
 
           @primary_key = Modeling::KeyedColumn.make(primary_key, nullable: true)
-
-          freeze
         end
 
         def perform(output, payload)
           payload[register] = array(payload[register])
 
-          payload[register].each do |row|
-            arel_row       = make_arel_row(transform(row, payload.time))
-            insert_manager = ::Arel::InsertManager.new.insert(arel_row)
+          payload[register].each { |row| insert(output, row, payload.time) }
+        end
 
-            debug_detail(output, "Insert Statement: #{insert_manager.to_sql}")
+        private
 
-            id = ::ActiveRecord::Base.connection.insert(insert_manager)
+        def insert(output, row, time)
+          transformed_row = transform(attribute_renderers, row, time)
+
+          output_sql(output, transformed_row)
+          insert_and_mutate(output, transformed_row, row)
+        end
 
-            resolver.set(row, primary_key.key, id) if primary_key
+        def output_sql(output, row)
+          sql = db_provider.insert_sql(row)
 
-            debug_detail(output, "Insert Return: #{row}")
-          end
+          debug_detail(output, "Insert Statement: #{sql}")
         end
 
-        private
+        def insert_and_mutate(output, row_to_insert, row_to_return)
+          id = db_provider.insert(row_to_insert)
+
+          resolver.set(row_to_return, primary_key.key, id) if primary_key
+
+          debug_detail(output, "Insert Return: #{row_to_return}")
+        end
 
         def timestamp_attributes
           [
-            timestamp_attribute(CREATED_AT),
-            timestamp_attribute(UPDATED_AT)
+            created_at_timestamp_attribute,
+            updated_at_timestamp_attribute
           ]
         end
       end

data/lib/db_fuel/library/active_record/update.rb

@@ -19,7 +19,7 @@ module DbFuel
       # Expected Payload[register] input: array of objects
       # Payload[register] output: array of objects.
       class Update < Base
-        attr_reader :unique_keys
+        attr_reader :unique_attribute_renderers
 
         # Arguments:
         #   name [required]: name of the job within the Burner::Pipeline.
@@ -43,8 +43,8 @@ module DbFuel
         #   timestamps: If timestamps is true (default behavior) then the updated_at column will
         #               automatically have its value set to the current UTC timestamp.
         #
-        #   unique_keys: Each key will become a WHERE clause in order to only update specific
-        #                records.
+        #   unique_attributes: Each key will become a WHERE clause in order to only update specific
+        #                      records.
         def initialize(
           name:,
           table_name:,
@@ -53,7 +53,7 @@ module DbFuel
           register: Burner::DEFAULT_REGISTER,
           separator: '',
           timestamps: true,
-          unique_keys: []
+          unique_attributes: []
         )
           explicit_attributes = Burner::Modeling::Attribute.array(attributes)
 
@@ -68,7 +68,7 @@ module DbFuel
             separator: separator
           )
 
-          @unique_keys = Modeling::KeyedColumn.array(unique_keys)
+          @unique_attribute_renderers = make_attribute_renderers(unique_attributes)
 
           freeze
         end
@@ -79,11 +79,14 @@ module DbFuel
           payload[register] = array(payload[register])
 
           payload[register].each do |row|
-            update_manager = make_update_manager(row, payload.time)
+            set_object   = transform(attribute_renderers, row, payload.time)
+            where_object = transform(unique_attribute_renderers, row, payload.time)
 
-            debug_detail(output, "Update Statement: #{update_manager.to_sql}")
+            sql = db_provider.update_sql(set_object, where_object)
 
-            rows_affected = ::ActiveRecord::Base.connection.update(update_manager)
+            debug_detail(output, "Update Statement: #{sql}")
+
+            rows_affected = db_provider.update(set_object, where_object)
 
             debug_detail(output, "Individual Rows Affected: #{rows_affected}")
 
@@ -95,28 +98,10 @@ module DbFuel
 
         private
 
-        def make_update_manager(row, time)
-          arel_row       = make_arel_row(transform(row, time))
-          unique_values  = make_unique_column_values(row)
-          update_manager = ::Arel::UpdateManager.new.set(arel_row).table(arel_table)
-
-          apply_where(unique_values, update_manager)
-        end
-
-        def make_unique_column_values(row)
-          unique_keys.each_with_object({}) do |unique_key, memo|
-            memo[unique_key.column] = resolver.get(row, unique_key.key)
-          end
-        end
-
-        def apply_where(hash, manager)
-          (hash || {}).inject(manager) do |memo, (key, value)|
-            memo.where(arel_table[key].eq(value))
-          end
-        end
-
         def timestamp_attributes
-          [timestamp_attribute(UPDATED_AT)]
+          [
+            updated_at_timestamp_attribute
+          ]
         end
       end
     end

data/lib/db_fuel/version.rb

@@ -8,5 +8,5 @@
 #
 
 module DbFuel
-  VERSION = '1.1.0-alpha'
+  VERSION = '1.1.0-alpha.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_fuel
 version: !ruby/object:Gem::Version
-  version: 1.1.0.pre.alpha
+  version: 1.1.0.pre.alpha.1
 platform: ruby
 authors:
 - Matthew Ruggio
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-25 00:00:00.000000000 Z
+date: 2020-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -236,8 +236,10 @@ files:
 - db_fuel.gemspec
 - exe/.gitkeep
 - lib/db_fuel.rb
+- lib/db_fuel/db_provider.rb
 - lib/db_fuel/library.rb
 - lib/db_fuel/library/active_record/base.rb
+- lib/db_fuel/library/active_record/find_or_insert.rb
 - lib/db_fuel/library/active_record/insert.rb
 - lib/db_fuel/library/active_record/update.rb
 - lib/db_fuel/library/dbee/base.rb