db_fuel 1.0.0.pre.alpha → 1.2.0.pre.alpha

data/db_fuel.gemspec

@@ -5,14 +5,14 @@ require './lib/db_fuel/version'
 Gem::Specification.new do |s|
   s.name        = 'db_fuel'
   s.version     = DbFuel::VERSION
-  s.summary     = 'TBD'
+  s.summary     = 'Dbee and ActiveRecord jobs for Burner'
 
   s.description = <<-DESCRIPTION
-    TBD
+    This library adds database-centric jobs to the Burner library.  Burner does not ship with database jobs out of the box.
   DESCRIPTION
 
-  s.authors     = ['Matthew Ruggio']
-  s.email       = ['mruggio@bluemarblepayroll.com']
+  s.authors     = ['Matthew Ruggio', 'John Bosko']
+  s.email       = ['mruggio@bluemarblepayroll.com', 'jbosko@bluemarblepayroll.com']
   s.files       = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   s.bindir      = 'exe'
   s.executables = %w[]
@@ -42,7 +42,7 @@ Gem::Specification.new do |s|
 
   s.add_dependency('activerecord', activerecord_version)
   s.add_dependency('acts_as_hashable', '~>1.2')
-  s.add_dependency('burner', '~>1.0')
+  s.add_dependency('burner', '~>1.2')
   s.add_dependency('dbee', '~>2.1')
   s.add_dependency('dbee-active_record', '~>2.1')
   s.add_dependency('objectable', '~>1.0')
@@ -51,7 +51,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency('pry', '~>0')
   s.add_development_dependency('rake', '~> 13')
   s.add_development_dependency('rspec', '~> 3.8')
-  s.add_development_dependency('rubocop', '~>0.90.0')
+  s.add_development_dependency('rubocop', '~>1.7.0')
   s.add_development_dependency('simplecov', '~>0.18.5')
   s.add_development_dependency('simplecov-console', '~>0.7.0')
   s.add_development_dependency('sqlite3', '~>1')

data/lib/db_fuel.rb

@@ -14,4 +14,10 @@ require 'dbee'
 require 'dbee/providers/active_record_provider'
 require 'objectable'
 
+# General purpose classes used by the main job classes.
+require_relative 'db_fuel/modeling'
+
+# Internal logic used across jobs.
+require_relative 'db_fuel/db_provider'
+
 require_relative 'db_fuel/library'

data/lib/db_fuel/db_provider.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module DbFuel
+  # Intermediate internal API for Arel/ActiveRecord.  There is some overlap in job needs when
+  # it comes to the Arel interface so this class condenses down those needs into this class.
+  class DbProvider # :nodoc: all
+    attr_reader :arel_table
+
+    def initialize(table_name)
+      raise ArgumentError, 'table_name is required' if table_name.to_s.empty?
+
+      @arel_table = ::Arel::Table.new(table_name.to_s)
+
+      freeze
+    end
+
+    def first(object)
+      sql = first_sql(object)
+
+      ::ActiveRecord::Base.connection.exec_query(sql).first
+    end
+
+    def first_sql(object)
+      relation = arel_table.project(Arel.star).take(1)
+      manager  = apply_where(object, relation)
+
+      manager.to_sql
+    end
+
+    def insert_sql(object)
+      insert_manager(object).to_sql
+    end
+
+    def insert(object)
+      manager = insert_manager(object)
+
+      ::ActiveRecord::Base.connection.insert(manager)
+    end
+
+    def update(set_object, where_object)
+      manager = update_manager(set_object, where_object)
+
+      ::ActiveRecord::Base.connection.update(manager)
+    end
+
+    def update_sql(set_object, where_object)
+      update_manager(set_object, where_object).to_sql
+    end
+
+    private
+
+    def update_manager(set_object, where_object)
+      arel_row       = make_arel_row(set_object)
+      update_manager = ::Arel::UpdateManager.new.set(arel_row).table(arel_table)
+
+      apply_where(where_object, update_manager)
+    end
+
+    def apply_where(hash, manager)
+      (hash || {}).inject(manager) do |memo, (key, value)|
+        memo.where(arel_table[key].eq(value))
+      end
+    end
+
+    def insert_manager(object)
+      arel_row = make_arel_row(object)
+
+      ::Arel::InsertManager.new.insert(arel_row)
+    end
+
+    def make_arel_row(row)
+      row.map { |key, value| [arel_table[key], value] }
+    end
+  end
+end

data/lib/db_fuel/library.rb

@@ -7,8 +7,25 @@
 # LICENSE file in the root directory of this source tree.
 #
 
+require_relative 'library/active_record/find_or_insert'
+require_relative 'library/active_record/insert'
+require_relative 'library/active_record/update'
+require_relative 'library/active_record/update_all'
+require_relative 'library/active_record/upsert'
+
 require_relative 'library/dbee/query'
 require_relative 'library/dbee/range'
 
-Burner::Jobs.register('db_fuel/dbee/query', DbFuel::Library::Dbee::Query)
-Burner::Jobs.register('db_fuel/dbee/range', DbFuel::Library::Dbee::Range)
+module Burner
+  # Open up Burner::Jobs and add registrations for this libraries jobs.
+  class Jobs
+    register 'db_fuel/active_record/find_or_insert', DbFuel::Library::ActiveRecord::FindOrInsert
+    register 'db_fuel/active_record/insert',         DbFuel::Library::ActiveRecord::Insert
+    register 'db_fuel/active_record/update',         DbFuel::Library::ActiveRecord::Update
+    register 'db_fuel/active_record/update_all',     DbFuel::Library::ActiveRecord::UpdateAll
+    register 'db_fuel/active_record/upsert',         DbFuel::Library::ActiveRecord::Upsert
+
+    register 'db_fuel/dbee/query',                   DbFuel::Library::Dbee::Query
+    register 'db_fuel/dbee/range',                   DbFuel::Library::Dbee::Range
+  end
+end

data/lib/db_fuel/library/active_record/base.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job can take the objects in a register and insert them into a database table.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class Base < Burner::JobWithRegister
+        CREATED_AT = :created_at
+        NOW_TYPE   = 'r/value/now'
+        UPDATED_AT = :updated_at
+
+        attr_reader :attribute_renderers,
+                    :db_provider,
+                    :debug,
+                    :resolver,
+                    :attribute_renderers_set
+
+        def initialize(
+          name:,
+          table_name:,
+          attributes: [],
+          debug: false,
+          register: Burner::DEFAULT_REGISTER,
+          separator: ''
+        )
+          super(name: name, register: register)
+
+          @resolver = Objectable.resolver(separator: separator)
+          @attribute_renderers_set = Modeling::AttributeRendererSet.new(attributes: attributes,
+                                                                        resolver: resolver)
+          @db_provider = DbProvider.new(table_name)
+          @debug = debug || false
+        end
+
+        private
+
+        def debug_detail(output, message)
+          return unless debug
+
+          output.detail(message)
+        end
+      end
+    end
+  end
+end

data/lib/db_fuel/library/active_record/find_or_insert.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'upsert'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job is a slight enhancement to the insert job, in that it will only insert new
+      # records.  It will use the unique_keys to first run a query to see if it exists.
+      # Each unique_key becomes a WHERE clause.  If primary_key is specified and a record is
+      # found then the first record's id will be set to the primary_key.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class FindOrInsert < Upsert
+        # attr_reader :unique_attribute_renderers
+
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for SQL-specific transformers
+        #                before insertion.
+        #
+        #   debug: If debug is set to true (defaults to false) then the SQL statements and
+        #          returned objects will be printed in the output.  Only use this option while
+        #          debugging issues as it will fill up the output with (potentially too much) data.
+        #
+        #   primary_key: If primary_key is present then it will be used to set the object's
+        #                property to the returned primary key from the INSERT statement.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then both created_at
+        #               and updated_at columns will automatically have their values set
+        #               to the current UTC timestamp.
+        #
+        #   unique_attributes: Each key will become a WHERE clause in order check for record
+        #                      existence before insertion attempt.
+        def initialize(
+          name:,
+          table_name:,
+          attributes: [],
+          debug: false,
+          primary_key: nil,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true,
+          unique_attributes: []
+        )
+
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            primary_key: primary_key,
+            register: register,
+            separator: separator,
+            timestamps: timestamps,
+            unique_attributes: unique_attributes
+          )
+        end
+
+        def perform(output, payload)
+          total_inserted = 0
+          total_existed  = 0
+
+          payload[register] = array(payload[register])
+
+          payload[register].each do |row|
+            exists = find_record(output, row, payload.time)
+
+            if exists
+              total_existed += 1
+              next
+            end
+
+            insert_record(output, row, payload.time)
+
+            total_inserted += 1
+          end
+
+          output.detail("Total Existed: #{total_existed}")
+          output.detail("Total Inserted: #{total_inserted}")
+        end
+      end
+    end
+  end
+end

data/lib/db_fuel/library/active_record/insert.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'upsert'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job can take the objects in a register and insert them into a database table.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class Insert < Upsert
+        # attr_reader :primary_key
+
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for SQL-specific transformers
+        #                before insertion.
+        #
+        #   debug: If debug is set to true (defaults to false) then the SQL statements and
+        #          returned objects will be printed in the output.  Only use this option while
+        #          debugging issues as it will fill up the output with (potentially too much) data.
+        #
+        #   primary_key: If primary_key is present then it will be used to set the object's
+        #                property to the returned primary key from the INSERT statement.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then both created_at
+        #               and updated_at columns will automatically have their values set
+        #               to the current UTC timestamp.
+        def initialize(
+          name:,
+          table_name:,
+          attributes: [],
+          debug: false,
+          primary_key: nil,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true
+        )
+
+          attributes = Burner::Modeling::Attribute.array(attributes)
+
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            primary_key: primary_key,
+            register: register,
+            separator: separator,
+            timestamps: timestamps
+          )
+        end
+
+        def perform(output, payload)
+          payload[register] = array(payload[register])
+
+          payload[register].each { |row| insert_record(output, row, payload.time) }
+        end
+      end
+    end
+  end
+end

data/lib/db_fuel/library/active_record/update.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'upsert'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job can take the unique objects in a register and updates them within database table.
+      # The attributes translate to SQL SET clauses and the unique_keys translate to
+      # WHERE clauses to find the records to update.
+      # The primary_key is used to update the unique record.
+      # Only one record will be updated per statement.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class Update < Upsert
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for SQL-specific transformers
+        #                before mutation.
+        #
+        #   debug: If debug is set to true (defaults to false) then the SQL statements and
+        #          returned objects will be printed in the output.  Only use this option while
+        #          debugging issues as it will fill up the output with (potentially too much) data.
+        #
+        #   primary_key [required]: Primary key column for the corresponding table.
+        #                           Used as the WHERE clause for the UPDATE statement.
+        #                           Only one record will be updated at a time
+        #                           using the primary key specified.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then the updated_at column will
+        #               automatically have its value set to the current UTC timestamp.
+        #
+        #   unique_attributes: Each key will become a WHERE clause in order to only find specific
+        #                      records. The UPDATE statement's WHERE
+        #                      clause will use the primary key specified.
+        def initialize(
+          name:,
+          table_name:,
+          attributes: [],
+          debug: false,
+          primary_key: nil,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true,
+          unique_attributes: []
+        )
+
+          attributes = Burner::Modeling::Attribute.array(attributes)
+
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            primary_key: primary_key,
+            register: register,
+            separator: separator,
+            timestamps: timestamps,
+            unique_attributes: unique_attributes
+          )
+
+          freeze
+        end
+
+        def perform(output, payload)
+          total_rows_affected = 0
+
+          payload[register] = array(payload[register])
+
+          payload[register].each do |row|
+            rows_affected = 0
+
+            first_record = update_record(output, row, payload.time)
+
+            rows_affected = 1 if first_record
+
+            debug_detail(output, "Individual Rows Affected: #{rows_affected}")
+
+            total_rows_affected += rows_affected
+          end
+
+          output.detail("Total Rows Affected: #{total_rows_affected}")
+        end
+      end
+    end
+  end
+end