huginn_acumen_product_agent 1.7.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1025b857902f5c14f649104a892ec5d882201920933ca4f010b45cab3a25f480
-  data.tar.gz: 999d68b437f373c47b70f69f5d884c52a30c6788f46262beb226b24c2200bba0
+  metadata.gz: 30c918a4763fbc8b84145f83215cf46d9b162adeb32d025d206d7f7fa51ded41
+  data.tar.gz: 77b5d58d8de8f15e3908b920b04882fb0706974009c9fa1ea1fe93bb66b7f1d6
 SHA512:
-  metadata.gz: 46ea5a9bd225a32c1af84c9a14becb87b71a4c01038a85199e6dfc9bdea0f1073fef090738839f4d2cbc2a5d3b3b81e2d13da67cc9f6952dbd9542b7647a3014
-  data.tar.gz: c08f0f45ed92defb6704b6b821053859c959c0a4061c5f58b094bac7c5bb5ca3c7e1bf0bda93ff2117d11134533945d00a6b65df1a6ed1214c2ac242a88c2782
+  metadata.gz: e2fe59ce492e910b0ca2c063938141a9f0ef6b64639fff28f34c3e01d577e5388d6007e1a135a51f01579677e60f2ee0e79e3aa2a449bfffba0e44242258953a
+  data.tar.gz: 92563dd015ec11f237a6a91d740d3ff682ddcb73e6598f1768c3dfa51c2c4e75cb8355c3ee7228a5f63a0f1470a483d21adb5bd324283f4efa45b7c77ccada50

data/lib/huginn_acumen_product_agent.rb

@@ -1,6 +1,13 @@
 require 'huginn_agent'
 
-HuginnAgent.load 'huginn_acumen_product_agent/concerns/acumen_product_query_concern'
+HuginnAgent.load 'huginn_acumen_product_agent/concerns/acumen_query_concern'
+HuginnAgent.load 'huginn_acumen_product_agent/concerns/alternate_products_query_concern'
+HuginnAgent.load 'huginn_acumen_product_agent/concerns/inv_product_query_concern'
+HuginnAgent.load 'huginn_acumen_product_agent/concerns/prod_mkt_query_concern'
+HuginnAgent.load 'huginn_acumen_product_agent/concerns/product_categories_query_concern'
+HuginnAgent.load 'huginn_acumen_product_agent/concerns/product_contributors_query_concern'
+
 HuginnAgent.load 'huginn_acumen_product_agent/acumen_client'
+HuginnAgent.load 'huginn_acumen_product_agent/acumen_agent_error'
 
 HuginnAgent.register 'huginn_acumen_product_agent/acumen_product_agent'

data/lib/huginn_acumen_product_agent/acumen_agent_error.rb

@@ -0,0 +1,9 @@
+class AcumenAgentError < StandardError
+  attr_reader :thing
+  def initialize(scope, message, data, original_error)
+    @scope = scope
+    @data = data
+    @original_error = original_error
+    super(message)
+  end
+end

data/lib/huginn_acumen_product_agent/acumen_product_agent.rb

@@ -3,7 +3,12 @@
 module Agents
     class AcumenProductAgent < Agent
         include WebRequestConcern
-        include AcumenProductQueryConcern
+        include AcumenQueryConcern
+        include InvProductQueryConcern
+        include ProdMktQueryConcern
+        include ProductContributorsQueryConcern
+        include ProductCategoriesQueryConcern
+        include AlternateProductsQueryConcern
 
         default_schedule '12h'
 
@@ -11,27 +16,86 @@ module Agents
         default_schedule 'never'
 
         description <<-MD
-      Huginn agent for retrieving sane ACUMEN product data.
+        Huginn agent for retrieving sane ACUMEN product data.
 
-      ## Agent Options
-      The following outlines the available options in this agent
+        ## Agent Options
+        The following outlines the available options in this agent
 
-      ### Acumen Connection
-      * endpoint: The root URL for the Acumen API
-      * site_code: The site code from Acumen
-      * password: The Acumen API password
+        ### Acumen Connection
+        * endpoint: The root URL for the Acumen API
+        * site_code: The site code from Acumen
+        * password: The Acumen API password
 
-      ### Variant Settings
-      * physical_formats: A list of the formats associated with a physical product
-      * digital_formats: A list of the formats associated with a digital product
+        ### Format Options
+        * digital_formats: A list of the formats associated with a digital product
 
-      ### Product Attributes
-      * attribute_to_property: An optional map linking Acumen attributes to Schema.org
-        product properties.
+        ### Product Attributes
+        * attribute_to_property: An optional map linking Acumen attributes to Schema.org
+          product properties.
 
-      ### Other Options
-      * ignore_skus: An optional array of Acumen product skus that will be intentionally
-        excluded from any output.
+        ### Other Options
+        * ignore_skus: An optional array of Acumen product skus that will be intentionally
+          excluded from any output.
+
+        ### Event Output
+        This agent will output one of two event types during processing:
+
+        *  Product bundles
+        *  Processing Errors
+
+        The product bundle payload will be structured as:
+
+        ```
+        {
+          products: [ { ... }, { ... }, ... ],
+          status: 200
+        }
+        ```
+
+        The processing error payload will be structured as:
+
+        ```
+        {
+          status: 500,
+          scope: '[Process Name]',
+          message: '[Error Message]',
+          data: { ... },
+          trace: [ ... ]
+        }
+        ```
+
+        ### Payload Status
+
+        `status: 200`: Indicates a true success. The agent has output the full
+        range of expected data.
+
+        `status: 206`: Indicates a partial success. The products within the bundle
+        are vaild, but the bundle _may_ be missing products that were somehow invalid.
+
+        `status: 500`: Indicates a processing error. This may represent a complete
+        process failure, but may also be issued in parallel to a `202` payload.
+
+        Because this agent receives an array of Product IDs as input, errors will be issued in
+        such a way that product processing can recover when possible. Errors that occur within
+        a specific product bundle will emit an error event, but the agent will then move
+        forward processing the next bundle.
+
+        For example, if this agent receives two products as input (`A` and `B`), and we fail to
+        load the Inv_Product record for product `A`, the agent would emit an error payload of:
+
+        ```
+        {
+          status: 500,
+          scope: 'Fetch Inv_Product Data',
+          message: 'Failed to lookup Inv_Product record for Product A',
+          data: { product_id: 123 },
+          trace: [ ... ]
+        }
+        ```
+
+        The goal of this approach is to ensure the agent outputs as much data as reasonably possible
+        with each execution. If there is an error in the Paperback version of a title, that shouldn't
+        prevent this agent from returning the Hardcover version.
 
         MD
 
@@ -40,7 +104,6 @@ module Agents
                 'endpoint' => 'https://example.com',
                 'site_code' => '',
                 'password' => '',
-                'physical_formats' => [],
                 'digital_formats' => [],
                 'attribute_to_property' => {},
             }
@@ -59,10 +122,6 @@ module Agents
                 errors.add(:base, 'password is a required field')
             end
 
-            unless options['physical_formats'].present?
-                errors.add(:base, "physical_formats is a required field")
-            end
-
             unless options['digital_formats'].present?
                 errors.add(:base, "digital_formats is a required field")
             end
@@ -72,9 +131,9 @@ module Agents
             end
 
             if options['ignore_skus']
-              unless options['ignore_skus'].is_a?(Array)
-                  errors.add(:base, "if provided, ignore_skus must be an array")
-              end
+                unless options['ignore_skus'].is_a?(Array)
+                    errors.add(:base, "if provided, ignore_skus must be an array")
+                end
             end
         end
 
@@ -95,13 +154,14 @@ module Agents
         private
 
         def handle(event)
+            # Process agent options
             endpoint = interpolated['endpoint']
             site_code = interpolated['site_code']
             password = interpolated['password']
-            physical_formats = interpolated['physical_formats']
             digital_formats = interpolated['digital_formats']
-            ignore_skus = interpolated['ignore_skus'] ? interpolated['ignore_skus'] : []
+            ignored_skus = interpolated['ignore_skus'] ? interpolated['ignore_skus'] : []
 
+            # Configure the Acumen Client
             auth = {
                 'site_code' => site_code,
                 'password' => password,
@@ -110,33 +170,62 @@ module Agents
             client = AcumenClient.new(faraday, auth)
 
             ids = event.payload['ids']
-            products = get_products_by_ids(client, ids)
-            products = get_product_variants(client, products, physical_formats, digital_formats)
-            products = get_master_products_by_id(client, products)
-            products = get_product_categories(client, products)
-            products = get_product_contributors(client, products)
-
-            # map attributes
-            products.map do |product|
-                map_attributes(product)
 
-                product['model'].each do |model|
-                    map_attributes(model)
-                end
+            # Load Products
+            fetch_product_bundles(client, ids, digital_formats, ignored_skus)
+        end
 
-                product
-            end
+        private
+
+        # Returns an array of Product objects for the provided product_ids.
+        # Each object is a merged representation of all the individual Acumen tables
+        # that make up a product record with fields mapped to the schema.org/Product
+        # object definition.
+        def fetch_products(acumen_client, product_ids, digital_format_list)
+            products = fetch_inv_product_data(acumen_client, product_ids, digital_format_list)
+            products = fetch_product_marketing(acumen_client, products)
+            products = fetch_product_contributors(acumen_client, products)
+            products = fetch_product_categories(acumen_client, products)
 
             products.each do |product|
-                unless ignore_skus.include?(product['sku'])
-                  create_event payload: product
-                end
+                map_attributes(product)
             end
+
+            return products
         end
 
-        private
+        # Returns an array of product bundles for the provided `products` array.
+        # Each bundle will contain an array of all the product definitions for each
+        # format of a given title.
+        #
+        # NOTE: The generated bundles will contain both active and inactive products
+        # to facilitate product deletion in external systems.
+        def fetch_product_bundles(acumen_client, product_ids, digital_format_list, ignored_skus)
 
+          begin
+            alternate_ids_map = fetch_alternate_format_ids(acumen_client, product_ids)
+
+            bundles = product_ids.map do |id|
+              bundle_ids = alternate_ids_map[id]
+              bundle_ids.append(id) unless bundle_ids.include?(id)
+              bundle = fetch_products(acumen_client, bundle_ids.sort, digital_format_list)
+
+              # Filter out any products that are explicitly ignored by SKU
+              bundle.select { |p| !ignored_skus.include?(p['sku']) }
+
+              create_event payload: { products: bundle, status: 200 }
+            end
+          rescue AcumenAgentError => e
+            issue_error(e)
+          end
+        end
+
+        # Maps additional Acumen attributes to the `additionalProperty` array
+        # NOTE:  Attributes mapped in this way will be _removed_ from the
+        # `acumenAttributes` array.
         def map_attributes(product)
+
+
             attribute_to_property = interpolated['attribute_to_property']
             attributes = product['acumenAttributes']
 

data/lib/huginn_acumen_product_agent/concerns/acumen_query_concern.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This module contains the baseline utility methods used in the more specific
+# data concerns
+module AcumenQueryConcern
+  extend ActiveSupport::Concern
+
+  UNIT_MAP = {
+    'oz.' => 'OZ',
+    'Inches (US)' => 'INH',
+  }
+
+  protected
+
+  # Maps Acumen XML data to a hash object as specified in the provided `field_map`
+  # The field map is a hash of { source_field: target_field }
+  def response_mapper(data, field_map)
+    result = {}
+
+    field_map.each do |source_field, target_field|
+      result[target_field] = get_field_value(data, source_field)
+    end
+
+    return result
+  end
+
+  # Utility function to retrieve a value from an XML field
+  def get_field_value(data, field_name)
+    data[field_name]['__content__'] if data[field_name]
+  end
+
+  # Returns a quantitative field value (e.g. weight) as a Schema.org/QuantitativeValue
+  # object
+  def get_quantitative_value(value, unit)
+    {
+      '@type' => 'QuantitativeValue',
+      'value' => value,
+      'unitText' => unit,
+      'unitCode' => (UNIT_MAP[unit] if unit),
+    } if value
+  end
+
+  # Emits an error payload event to facilitate better debugging/logging
+  # NOTE: The `error` here is expected to be an instance of AcumenAgentError
+  def issue_error(error, status = 500)
+    # NOTE: Status is intentionally included on the top-level payload so that other
+    # agents can look for a `payload[:status]` of either 200 or 500 to distinguish
+    # between success and failure states
+    create_event payload: {
+      status: status,
+      scope: error.scope,
+      message: error.message,
+      original_error: error.original_error,
+      data: error.data,
+      trace: error.original_error.backtrace,
+    }
+  end
+end

data/lib/huginn_acumen_product_agent/concerns/agent_error_concern.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module AgentErrorConcern
+  extend ActiveSupport::Concern
+
+  def issue_error(error, status = 500)
+    # NOTE: Status is intentionally included on the top-level payload so that other
+    # agents can look for a `payload[:status]` of either 200 or 500 to distinguish
+    # between success and failure states
+    create_event payload: {
+      status: status,
+      scope: error.scope,
+      message: error.message,
+      original_error: error.original_error,
+      data: error.data,
+      trace: error.original_error.backtrace,
+    }
+  end
+end

data/lib/huginn_acumen_product_agent/concerns/alternate_products_query_concern.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# This module is responsible for reading/processing data from the Product_Link
+# table. This table defines the link between product records.
+#
+# It's important to note that this table defines links between related products
+# as well as product formats. The `Alt_Format` field determines whether the linked
+# product is an alternate format for a given title.
+#
+# For the purposes of this concern, we only care about Product_Link records where
+# the `Alt_Format` field is `true`
+module AlternateProductsQueryConcern
+  extend AcumenQueryConcern
+
+  # Fetches the Inv_Product.ID for alternate formats of the provided product_ids
+  def fetch_alternate_format_ids(acumen_client, product_ids)
+    begin
+      link_data = acumen_client.get_linked_products(product_ids)
+
+      links = process_alternate_format_response(link_data)
+
+      return map_alternate_format_links(links, product_ids)
+    rescue => error
+      issue_error(AcumenAgentError.new(
+        'fetch_alternate_format_ids',
+        'Failed attempting to lookup alternate products',
+        product_ids,
+        error
+      ))
+    end
+  end
+
+  # This function parses the raw data returned from the Product_Link table
+  # The resulting array contains the set alternate format IDs associated with a
+  # single product
+  def process_alternate_format_response(raw_data)
+    results = []
+    raw_data.map do |link|
+
+      begin
+        mapped = response_mapper(link, {
+          'Product_Link.Link_From_ID' => 'from_id',
+          'Product_Link.Link_To_ID' => 'to_id',
+          'Product_Link.Alt_Format' => 'alt_format',
+        })
+
+        if mapped['alt_format'].to_s != '0' && !mapped.in?(results)
+          results.push(mapped)
+        end
+
+      rescue => error
+        issue_error(AcumenAgentError.new(
+          'process_alternate_format_response',
+          'Failed while processing alternate format links',
+          raw_data,
+          error
+        ))
+      end
+    end
+
+    return results
+  end
+
+  # Returns a map that ties each provided `product_id` to an array of IDs for its
+  # other formats
+  def map_alternate_format_links(links, product_ids)
+    results = {}
+
+    product_ids.each do |id|
+
+      begin
+        alternates = links.select { |l| l['from_id'] == id }
+        results[id] = alternates.map { |l| l['to_id'] }
+
+      rescue => error
+        issue_error(AcumenAgentError.new(
+          'map_alternate_format_links',
+          'Failed while mapping alternate format links',
+          { id: id, links: links },
+          error
+        ))
+      end
+    end
+
+    return results
+  end
+end