db_fuel 1.1.0.pre.alpha → 1.2.1.pre.alpha.1

data/lib/db_fuel/library/active_record/find_or_insert.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'upsert'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job is a slight enhancement to the insert job, in that it will only insert new
+      # records.  It will use the unique_keys to first run a query to see if it exists.
+      # Each unique_key becomes a WHERE clause.  If primary_key is specified and a record is
+      # found then the first record's id will be set to the primary_key.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class FindOrInsert < Upsert
+        # attr_reader :unique_attribute_renderers
+
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for SQL-specific transformers
+        #                before insertion.
+        #
+        #   debug: If debug is set to true (defaults to false) then the SQL statements and
+        #          returned objects will be printed in the output.  Only use this option while
+        #          debugging issues as it will fill up the output with (potentially too much) data.
+        #
+        #   primary_key: If primary_key is present then it will be used to set the object's
+        #                property to the returned primary key from the INSERT statement.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then both created_at
+        #               and updated_at columns will automatically have their values set
+        #               to the current UTC timestamp.
+        #
+        #   unique_attributes: Each key will become a WHERE clause in order check for record
+        #                      existence before insertion attempt.
+        def initialize(
+          name:,
+          table_name:,
+          attributes: [],
+          debug: false,
+          primary_key: nil,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true,
+          unique_attributes: []
+        )
+
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            primary_key: primary_key,
+            register: register,
+            separator: separator,
+            timestamps: timestamps,
+            unique_attributes: unique_attributes
+          )
+        end
+
+        def perform(output, payload)
+          total_inserted = 0
+          total_existed  = 0
+
+          payload[register] = array(payload[register])
+
+          payload[register].each do |row|
+            exists = find_record(output, row, payload.time)
+
+            if exists
+              total_existed += 1
+              next
+            end
+
+            insert_record(output, row, payload.time)
+
+            total_inserted += 1
+          end
+
+          output.detail("Total Existed: #{total_existed}")
+          output.detail("Total Inserted: #{total_inserted}")
+        end
+      end
+    end
+  end
+end

data/lib/db_fuel/library/active_record/insert.rb

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'upsert'
 
 module DbFuel
   module Library
@@ -16,8 +16,8 @@ module DbFuel
       #
       # Expected Payload[register] input: array of objects
       # Payload[register] output: array of objects.
-      class Insert < Base
-        attr_reader :primary_key
+      class Insert < Upsert
+        # attr_reader :primary_key
 
         # Arguments:
         #   name [required]: name of the job within the Burner::Pipeline.
@@ -26,7 +26,7 @@ module DbFuel
         #
         #   attributes:  Used to specify which object properties to put into the
         #                SQL statement and also allows for one last custom transformation
-        #                pipeline, in case the data calls for sql-specific transformers
+        #                pipeline, in case the data calls for SQL-specific transformers
         #                before insertion.
         #
         #   debug: If debug is set to true (defaults to false) then the SQL statements and
@@ -54,48 +54,25 @@ module DbFuel
           separator: '',
           timestamps: true
         )
-          explicit_attributes = Burner::Modeling::Attribute.array(attributes)
 
-          attributes = timestamps ? timestamp_attributes + explicit_attributes : explicit_attributes
+          attributes = Burner::Modeling::Attribute.array(attributes)
 
           super(
             name: name,
             table_name: table_name,
             attributes: attributes,
             debug: debug,
+            primary_key: primary_key,
             register: register,
-            separator: separator
+            separator: separator,
+            timestamps: timestamps
           )
-
-          @primary_key = Modeling::KeyedColumn.make(primary_key, nullable: true)
-
-          freeze
         end
 
         def perform(output, payload)
           payload[register] = array(payload[register])
 
-          payload[register].each do |row|
-            arel_row       = make_arel_row(transform(row, payload.time))
-            insert_manager = ::Arel::InsertManager.new.insert(arel_row)
-
-            debug_detail(output, "Insert Statement: #{insert_manager.to_sql}")
-
-            id = ::ActiveRecord::Base.connection.insert(insert_manager)
-
-            resolver.set(row, primary_key.key, id) if primary_key
-
-            debug_detail(output, "Insert Return: #{row}")
-          end
-        end
-
-        private
-
-        def timestamp_attributes
-          [
-            timestamp_attribute(CREATED_AT),
-            timestamp_attribute(UPDATED_AT)
-          ]
+          payload[register].each { |row| insert_record(output, row, payload.time) }
         end
       end
     end

data/lib/db_fuel/library/active_record/update.rb

@@ -7,20 +7,20 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'upsert'
 
 module DbFuel
   module Library
     module ActiveRecord
-      # This job can take the objects in a register and updates them within database table.
+      # This job can take the unique objects in a register and updates them within database table.
       # The attributes translate to SQL SET clauses and the unique_keys translate to
-      # WHERE clauses.
+      # WHERE clauses to find the records to update.
+      # The primary_key is used to update the unique record.
+      # Only one record will be updated per statement.
       #
       # Expected Payload[register] input: array of objects
       # Payload[register] output: array of objects.
-      class Update < Base
-        attr_reader :unique_keys
-
+      class Update < Upsert
         # Arguments:
         #   name [required]: name of the job within the Burner::Pipeline.
         #
@@ -28,13 +28,18 @@ module DbFuel
         #
         #   attributes:  Used to specify which object properties to put into the
         #                SQL statement and also allows for one last custom transformation
-        #                pipeline, in case the data calls for sql-specific transformers
+        #                pipeline, in case the data calls for SQL-specific transformers
         #                before mutation.
         #
         #   debug: If debug is set to true (defaults to false) then the SQL statements and
         #          returned objects will be printed in the output.  Only use this option while
         #          debugging issues as it will fill up the output with (potentially too much) data.
         #
+        #   primary_key [required]: Primary key column for the corresponding table.
+        #                           Used as the WHERE clause for the UPDATE statement.
+        #                           Only one record will be updated at a time
+        #                           using the primary key specified.
+        #
         #   separator: Just like other jobs with a 'separator' option, if the objects require
         #              key-path notation or nested object support, you can set the separator
         #              to something non-blank (like a period for notation in the
@@ -43,33 +48,35 @@ module DbFuel
         #   timestamps: If timestamps is true (default behavior) then the updated_at column will
         #               automatically have its value set to the current UTC timestamp.
         #
-        #   unique_keys: Each key will become a WHERE clause in order to only update specific
-        #                records.
+        #   unique_attributes: Each key will become a WHERE clause in order to only find specific
+        #                      records. The UPDATE statement's WHERE
+        #                      clause will use the primary key specified.
         def initialize(
           name:,
           table_name:,
           attributes: [],
           debug: false,
+          primary_key: nil,
           register: Burner::DEFAULT_REGISTER,
           separator: '',
           timestamps: true,
-          unique_keys: []
+          unique_attributes: []
         )
-          explicit_attributes = Burner::Modeling::Attribute.array(attributes)
 
-          attributes = timestamps ? timestamp_attributes + explicit_attributes : explicit_attributes
+          attributes = Burner::Modeling::Attribute.array(attributes)
 
           super(
             name: name,
             table_name: table_name,
             attributes: attributes,
             debug: debug,
+            primary_key: primary_key,
             register: register,
-            separator: separator
+            separator: separator,
+            timestamps: timestamps,
+            unique_attributes: unique_attributes
           )
 
-          @unique_keys = Modeling::KeyedColumn.array(unique_keys)
-
           freeze
         end
 
@@ -79,11 +86,11 @@ module DbFuel
           payload[register] = array(payload[register])
 
           payload[register].each do |row|
-            update_manager = make_update_manager(row, payload.time)
+            rows_affected = 0
 
-            debug_detail(output, "Update Statement: #{update_manager.to_sql}")
+            first_record = update_record(output, row, payload.time)
 
-            rows_affected = ::ActiveRecord::Base.connection.update(update_manager)
+            rows_affected = 1 if first_record
 
             debug_detail(output, "Individual Rows Affected: #{rows_affected}")
 
@@ -92,32 +99,6 @@ module DbFuel
 
           output.detail("Total Rows Affected: #{total_rows_affected}")
         end
-
-        private
-
-        def make_update_manager(row, time)
-          arel_row       = make_arel_row(transform(row, time))
-          unique_values  = make_unique_column_values(row)
-          update_manager = ::Arel::UpdateManager.new.set(arel_row).table(arel_table)
-
-          apply_where(unique_values, update_manager)
-        end
-
-        def make_unique_column_values(row)
-          unique_keys.each_with_object({}) do |unique_key, memo|
-            memo[unique_key.column] = resolver.get(row, unique_key.key)
-          end
-        end
-
-        def apply_where(hash, manager)
-          (hash || {}).inject(manager) do |memo, (key, value)|
-            memo.where(arel_table[key].eq(value))
-          end
-        end
-
-        def timestamp_attributes
-          [timestamp_attribute(UPDATED_AT)]
-        end
       end
     end
   end

data/lib/db_fuel/library/active_record/update_all.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'upsert'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job can take the objects in a register and updates them within database table.
+      # The attributes translate to SQL SET clauses
+      # and the unique_keys translate to WHERE clauses.
+      # One or more records may be updated at a time.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class UpdateAll < Upsert
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for SQL-specific transformers
+        #                before mutation.
+        #
+        #   debug: If debug is set to true (defaults to false) then the SQL statements and
+        #          returned objects will be printed in the output.  Only use this option while
+        #          debugging issues as it will fill up the output with (potentially too much) data.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then the updated_at column will
+        #               automatically have its value set to the current UTC timestamp.
+        #
+        #   unique_attributes: Each key will become a WHERE clause in order to only update specific
+        #                      records.
+        def initialize(
+          name:,
+          table_name:,
+          attributes: [],
+          debug: false,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true,
+          unique_attributes: []
+        )
+
+          attributes = Burner::Modeling::Attribute.array(attributes)
+
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            primary_key: nil,
+            register: register,
+            separator: separator,
+            timestamps: timestamps,
+            unique_attributes: unique_attributes
+          )
+
+          freeze
+        end
+
+        def perform(output, payload)
+          total_rows_affected = 0
+
+          payload[register] = array(payload[register])
+
+          payload[register].each do |row|
+            where_object = attribute_renderers_set
+                           .transform(unique_attribute_renderers, row, payload.time)
+
+            rows_affected = update(output, row, payload.time, where_object)
+
+            debug_detail(output, "Individual Rows Affected: #{rows_affected}")
+
+            total_rows_affected += rows_affected
+          end
+
+          output.detail("Total Rows Affected: #{total_rows_affected}")
+        end
+      end
+    end
+  end
+end

data/lib/db_fuel/library/active_record/upsert.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'base'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job will insert or update records.
+      # It will use the unique_keys to first run a query to see if it exists.
+      # Each unique_key becomes a WHERE clause. If a record is found it will then
+      # update the found record using the primary key specified.
+      # If a record is updated or created the record's id will be set to the primary_key.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class Upsert < Base
+        attr_reader :primary_key, :timestamps, :unique_attribute_renderers
+
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT OR UPDATE statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for SQL-specific transformers
+        #                before mutation.
+        #
+        #   debug:       If debug is set to true (defaults to false) then the SQL statements and
+        #                returned objects will be printed in the output.  Only use this option while
+        #                debugging issues as it will fill
+        #                up the output with (potentially too much) data.
+        #
+        #   primary_key [required]: Used to set the object's property to the returned primary key
+        #                           from the INSERT statement or used as the
+        #                           WHERE clause for the UPDATE statement.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then the updated_at column will
+        #               automatically have its value set
+        #               to the current UTC timestamp if a record was updated.
+        #               If a record was created the
+        #               created_at and updated_at columns will be set.
+        #
+        #   unique_attributes: Each key will become a WHERE clause in
+        #                      order to check for the existence of a specific record.
+        def initialize(
+          name:,
+          table_name:,
+          primary_key:,
+          attributes: [],
+          debug: false,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true,
+          unique_attributes: []
+        )
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            register: register,
+            separator: separator
+          )
+
+          @primary_key = Modeling::KeyedColumn.make(primary_key, nullable: true)
+
+          @unique_attribute_renderers = attribute_renderers_set
+                                        .make_attribute_renderers(unique_attributes)
+
+          @timestamps = timestamps
+
+          freeze
+        end
+
+        def perform(output, payload)
+          raise ArgumentError, 'primary_key is required' unless primary_key
+
+          total_inserted = 0
+          total_updated  = 0
+
+          payload[register] = array(payload[register])
+
+          payload[register].each do |row|
+            record_updated = insert_or_update(output, row, payload.time)
+
+            if record_updated
+              total_updated += 1
+            else
+              total_inserted += 1
+            end
+          end
+
+          output.detail("Total Updated: #{total_updated}")
+          output.detail("Total Inserted: #{total_inserted}")
+        end
+
+        protected
+
+        def find_record(output, row, time)
+          unique_row = attribute_renderers_set.transform(unique_attribute_renderers, row, time)
+
+          first_sql = db_provider.first_sql(unique_row)
+
+          debug_detail(output, "Find Statement: #{first_sql}")
+
+          first_record = db_provider.first(unique_row)
+
+          id = resolver.get(first_record, primary_key.column)
+
+          resolver.set(row, primary_key.key, id)
+
+          debug_detail(output, "Record Exists: #{first_record}") if first_record
+
+          first_record
+        end
+
+        def insert_record(output, row, time)
+          dynamic_attrs = if timestamps
+                            # doing an INSERT and timestamps should be set
+                            # set the created_at and updated_at fields
+                            attribute_renderers_set.timestamp_created_attribute_renderers
+                          else
+                            attribute_renderers_set.attribute_renderers
+                          end
+
+          set_object = attribute_renderers_set.transform(dynamic_attrs, row, time)
+
+          insert_sql = db_provider.insert_sql(set_object)
+
+          debug_detail(output, "Insert Statement: #{insert_sql}")
+
+          id = db_provider.insert(set_object)
+
+          # add the primary key name and value to row if primary_key was specified
+          resolver.set(row, primary_key.key, id) if primary_key
+
+          debug_detail(output, "Insert Return: #{row}")
+        end
+
+        # Updates only a single record. Lookups primary key to update the record.
+        def update_record(output, row, time)
+          raise ArgumentError, 'primary_key is required' unless primary_key
+
+          first_record = find_record(output, row, time)
+
+          if first_record
+            debug_detail(output, "Record Exists: #{first_record}")
+
+            id = resolver.get(first_record, primary_key.column)
+
+            where_object = { primary_key.key => id }
+
+            # update record using the primary key as the WHERE clause
+            update(output, row, time, where_object)
+          end
+
+          first_record
+        end
+
+        # Updates one or many records depending on where_object passed
+        def update(output, row, time, where_object)
+          dynamic_attrs = if timestamps
+                            # doing an UPDATE and timestamps should be set,
+                            # modify the updated_at field, don't modify the created_at field
+                            attribute_renderers_set.timestamp_updated_attribute_renderers
+                          else
+                            attribute_renderers_set.attribute_renderers
+                          end
+
+          set_object = attribute_renderers_set.transform(dynamic_attrs, row, time)
+
+          update_sql = db_provider.update_sql(set_object, where_object)
+
+          debug_detail(output, "Update Statement: #{update_sql}")
+
+          debug_detail(output, "Update Return: #{row}")
+
+          db_provider.update(set_object, where_object)
+        end
+
+        private
+
+        def insert_or_update(output, row, time)
+          first_record = update_record(output, row, time)
+
+          if first_record
+            first_record
+          else
+            # create the record
+            insert_record(output, row, time)
+            nil
+          end
+        end
+      end
+    end
+  end
+end