db_fuel 1.0.0.pre.alpha → 1.2.0.pre.alpha

data/lib/db_fuel/library/active_record/update_all.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'upsert'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job can take the objects in a register and updates them within database table.
+      # The attributes translate to SQL SET clauses
+      # and the unique_keys translate to WHERE clauses.
+      # One or more records may be updated at a time.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class UpdateAll < Upsert
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for SQL-specific transformers
+        #                before mutation.
+        #
+        #   debug: If debug is set to true (defaults to false) then the SQL statements and
+        #          returned objects will be printed in the output.  Only use this option while
+        #          debugging issues as it will fill up the output with (potentially too much) data.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then the updated_at column will
+        #               automatically have its value set to the current UTC timestamp.
+        #
+        #   unique_attributes: Each key will become a WHERE clause in order to only update specific
+        #                      records.
+        def initialize(
+          name:,
+          table_name:,
+          attributes: [],
+          debug: false,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true,
+          unique_attributes: []
+        )
+
+          attributes = Burner::Modeling::Attribute.array(attributes)
+
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            primary_key: nil,
+            register: register,
+            separator: separator,
+            timestamps: timestamps,
+            unique_attributes: unique_attributes
+          )
+
+          freeze
+        end
+
+        def perform(output, payload)
+          total_rows_affected = 0
+
+          payload[register] = array(payload[register])
+
+          payload[register].each do |row|
+            where_object = attribute_renderers_set
+                           .transform(unique_attribute_renderers, row, payload.time)
+
+            rows_affected = update(output, row, payload.time, where_object)
+
+            debug_detail(output, "Individual Rows Affected: #{rows_affected}")
+
+            total_rows_affected += rows_affected
+          end
+
+          output.detail("Total Rows Affected: #{total_rows_affected}")
+        end
+      end
+    end
+  end
+end

data/lib/db_fuel/library/active_record/upsert.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'base'
+
+module DbFuel
+  module Library
+    module ActiveRecord
+      # This job will insert or update records.
+      # It will use the unique_keys to first run a query to see if it exists.
+      # Each unique_key becomes a WHERE clause. If a record is found it will then
+      # update the found record using the primary key specified.
+      # If a record is updated or created the record's id will be set to the primary_key.
+      #
+      # Expected Payload[register] input: array of objects
+      # Payload[register] output: array of objects.
+      class Upsert < Base
+        attr_reader :primary_key, :timestamps, :unique_attribute_renderers
+
+        # Arguments:
+        #   name [required]: name of the job within the Burner::Pipeline.
+        #
+        #   table_name [required]: name of the table to use for the INSERT OR UPDATE statements.
+        #
+        #   attributes:  Used to specify which object properties to put into the
+        #                SQL statement and also allows for one last custom transformation
+        #                pipeline, in case the data calls for SQL-specific transformers
+        #                before mutation.
+        #
+        #   debug:       If debug is set to true (defaults to false) then the SQL statements and
+        #                returned objects will be printed in the output.  Only use this option while
+        #                debugging issues as it will fill
+        #                up the output with (potentially too much) data.
+        #
+        #   primary_key [required]: Used to set the object's property to the returned primary key
+        #                           from the INSERT statement or used as the
+        #                           WHERE clause for the UPDATE statement.
+        #
+        #   separator: Just like other jobs with a 'separator' option, if the objects require
+        #              key-path notation or nested object support, you can set the separator
+        #              to something non-blank (like a period for notation in the
+        #              form of: name.first).
+        #
+        #   timestamps: If timestamps is true (default behavior) then the updated_at column will
+        #               automatically have its value set
+        #               to the current UTC timestamp if a record was updated.
+        #               If a record was created the
+        #               created_at and updated_at columns will be set.
+        #
+        #   unique_attributes: Each key will become a WHERE clause in
+        #                      order to check for the existence of a specific record.
+        def initialize(
+          name:,
+          table_name:,
+          primary_key:,
+          attributes: [],
+          debug: false,
+          register: Burner::DEFAULT_REGISTER,
+          separator: '',
+          timestamps: true,
+          unique_attributes: []
+        )
+          super(
+            name: name,
+            table_name: table_name,
+            attributes: attributes,
+            debug: debug,
+            register: register,
+            separator: separator
+          )
+
+          @primary_key = Modeling::KeyedColumn.make(primary_key, nullable: true)
+
+          @unique_attribute_renderers = attribute_renderers_set
+                                        .make_attribute_renderers(unique_attributes)
+
+          @timestamps = timestamps
+
+          freeze
+        end
+
+        def perform(output, payload)
+          raise ArgumentError, 'primary_key is required' unless primary_key
+
+          total_inserted = 0
+          total_updated  = 0
+
+          payload[register] = array(payload[register])
+
+          payload[register].each do |row|
+            record_updated = insert_or_update(output, row, payload.time)
+
+            if record_updated
+              total_updated += 1
+            else
+              total_inserted += 1
+            end
+          end
+
+          output.detail("Total Updated: #{total_updated}")
+          output.detail("Total Inserted: #{total_inserted}")
+        end
+
+        protected
+
+        def find_record(output, row, time)
+          unique_row = attribute_renderers_set.transform(unique_attribute_renderers, row, time)
+
+          first_sql = db_provider.first_sql(unique_row)
+
+          debug_detail(output, "Find Statement: #{first_sql}")
+
+          first_record = db_provider.first(unique_row)
+
+          id = resolver.get(first_record, primary_key.column)
+
+          resolver.set(row, primary_key.key, id)
+
+          debug_detail(output, "Record Exists: #{first_record}") if first_record
+
+          first_record
+        end
+
+        def insert_record(output, row, time)
+          dynamic_attrs = if timestamps
+                            # doing an INSERT and timestamps should be set
+                            # set the created_at and updated_at fields
+                            attribute_renderers_set.timestamp_created_attribute_renderers
+                          else
+                            attribute_renderer_set.attribute_renderers
+                          end
+
+          set_object = attribute_renderers_set.transform(dynamic_attrs, row, time)
+
+          insert_sql = db_provider.insert_sql(set_object)
+
+          debug_detail(output, "Insert Statement: #{insert_sql}")
+
+          id = db_provider.insert(set_object)
+
+          # add the primary key name and value to row if primary_key was specified
+          resolver.set(row, primary_key.key, id) if primary_key
+
+          debug_detail(output, "Insert Return: #{row}")
+        end
+
+        # Updates only a single record. Lookups primary key to update the record.
+        def update_record(output, row, time)
+          raise ArgumentError, 'primary_key is required' unless primary_key
+
+          first_record = find_record(output, row, time)
+
+          if first_record
+            debug_detail(output, "Record Exists: #{first_record}")
+
+            id = resolver.get(first_record, primary_key.column)
+
+            where_object = { primary_key.key => id }
+
+            # update record using the primary key as the WHERE clause
+            update(output, row, time, where_object)
+          end
+
+          first_record
+        end
+
+        # Updates one or many records depending on where_object passed
+        def update(output, row, time, where_object)
+          dynamic_attrs = if timestamps
+                            # doing an UPDATE and timestamps should be set,
+                            # modify the updated_at field, don't modify the created_at field
+                            attribute_renderers_set.timestamp_updated_attribute_renderers
+                          else
+                            attribute_renderer_set.attribute_renderers
+                          end
+
+          set_object = attribute_renderers_set.transform(dynamic_attrs, row, time)
+
+          update_sql = db_provider.update_sql(set_object, where_object)
+
+          debug_detail(output, "Update Statement: #{update_sql}")
+
+          debug_detail(output, "Update Return: #{row}")
+
+          db_provider.update(set_object, where_object)
+        end
+
+        private
+
+        def insert_or_update(output, row, time)
+          first_record = update_record(output, row, time)
+
+          if first_record
+            first_record
+          else
+            # create the record
+            insert_record(output, row, time)
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_fuel/library/dbee/base.rb

@@ -16,6 +16,11 @@ module DbFuel
                     :provider,
                     :query
 
+        # Arguments:
+        # - model:    Dbee Model configuration
+        # - query:    Dbee Query configuration
+        # - register: Name of the register to use for gathering the IN clause values and where
+        #             to store the resulting recordset.
         def initialize(
           name:,
           model: {},

data/lib/db_fuel/library/dbee/query.rb

@@ -12,8 +12,8 @@ require_relative 'base'
 module DbFuel
   module Library
     module Dbee
-      # Execute a Dbee Query against a Dbee Model and store the resulting records in the designated
-      # payload register.
+      # Executes a Dbee Query against a Dbee Model and stores the resulting records
+      # in the designated payload register.
       #
       # Expected Payload[register] input: nothing
       # Payload[register] output: array of objects.

data/lib/db_fuel/library/dbee/range.rb

@@ -23,6 +23,15 @@ module DbFuel
                     :key_path,
                     :resolver
 
+        # Arguments:
+        # - key:       Specifies which key to use to aggregate a list of values for within
+        #              the specified register's dataset.
+        # - key_path:  Specifies the Dbee identifier (column) to use for the IN filter.
+        # - model:     Dbee Model configuration
+        # - query:     Dbee Query configuration
+        # - register:  Name of the register to use for gathering the IN clause values and where
+        #              to store the resulting recordset.
+        # - separator: Character to use to split the key-path for nested object support.
         def initialize(
           name:,
           key:,
@@ -58,20 +67,24 @@ module DbFuel
           array(payload[register]).map { |o| resolver.get(o, key) }.compact
         end
 
-        def dynamic_filter(payload)
+        def dynamic_filters(payload)
           values = map_values(payload)
 
-          {
-            type: :equals,
-            key_path: key_path,
-            value: values,
-          }
+          return [] if values.empty?
+
+          [
+            {
+              type: :equals,
+              key_path: key_path,
+              value: values,
+            }
+          ]
         end
 
         def compile_dbee_query(payload)
           ::Dbee::Query.make(
             fields: query.fields,
-            filters: query.filters + [dynamic_filter(payload)],
+            filters: query.filters + dynamic_filters(payload),
             limit: query.limit,
             sorters: query.sorters
           )

data/lib/db_fuel/modeling.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'modeling/keyed_column'
+require_relative 'modeling/attribute_renderer_set'

data/lib/db_fuel/modeling/attribute_renderer_set.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module DbFuel
+  module Modeling
+    # Creates attribute renderers based on attributes passed.
+    # Also constains methods to transform attribute renderers
+    # and include timestamp attributes if needed.
+    class AttributeRendererSet
+      CREATED_AT = :created_at
+      NOW_TYPE   = 'r/value/now'
+      UPDATED_AT = :updated_at
+
+      attr_reader :attribute_renderers, :resolver
+
+      def initialize(attributes: [], resolver: nil)
+        raise ArgumentError, 'resolver is required' unless resolver
+
+        @resolver            = resolver
+        @attribute_renderers = make_attribute_renderers(attributes)
+
+        freeze
+      end
+
+      # Adds the attributes for created_at and updated_at to the currrent attribute renderers.
+      def timestamp_created_attribute_renderers
+        timestamp_attributes = [created_at_timestamp_attribute, updated_at_timestamp_attribute]
+
+        timestamp_attributes.map do |a|
+          Burner::Modeling::AttributeRenderer.new(a, resolver)
+        end + attribute_renderers
+      end
+
+      # Adds the attribute for updated_at to the currrent attribute renderers.
+      def timestamp_updated_attribute_renderers
+        timestamp_attributes = [updated_at_timestamp_attribute]
+
+        timestamp_attributes.map do |a|
+          Burner::Modeling::AttributeRenderer.new(a, resolver)
+        end + attribute_renderers
+      end
+
+      def make_attribute_renderers(attributes)
+        Burner::Modeling::Attribute
+          .array(attributes)
+          .map { |a| Burner::Modeling::AttributeRenderer.new(a, resolver) }
+      end
+
+      def transform(attribute_renderers, row, time)
+        attribute_renderers.each_with_object({}) do |attribute_renderer, memo|
+          value = attribute_renderer.transform(row, time)
+
+          resolver.set(memo, attribute_renderer.key, value)
+        end
+      end
+
+      private
+
+      def created_at_timestamp_attribute
+        timestamp_attribute(CREATED_AT)
+      end
+
+      def updated_at_timestamp_attribute
+        timestamp_attribute(UPDATED_AT)
+      end
+
+      def timestamp_attribute(key)
+        Burner::Modeling::Attribute.make(
+          key: key,
+          transformers: [
+            { type: NOW_TYPE }
+          ]
+        )
+      end
+    end
+  end
+end