iron_bank 3.4.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 195ba10f5c39626868e1d281972ff45e4618f1e72b0d87684aa9c65d1ed1fa10
-  data.tar.gz: acd56eae404df5027aceeb8081b542da811178a71ff56924531b4d2280fa045d
+  metadata.gz: ddab957161d110b39c87104391a5883a5e22091186a0e368ebcd8b03d8d7b4ad
+  data.tar.gz: 372c3ecc6be5c37ff50cf08085451244df1c49fa1dae6de9ed4301de18c05511
 SHA512:
-  metadata.gz: 4db1e53ef118e2b496a1ae41ea0f880458386767e57a562de235f0e734416df59f441c7f1928aca06cfa3904c49623985a794f5b3bf72121391488c2dc1a1da4
-  data.tar.gz: 25c9457c4716177d3a21d0f6b993a433558a323a6ba8f63e4cca881660ddca92408a16226d6ae4e42f05f6d280c6340c42c2667a8a2c131afc0a3adee9acd05d
+  metadata.gz: adda95d1a9f68e9aa4819244ad322b8b71b8ae19596b8c48ee704a559c835c1646fe4affd9d0b3bafac275641b9fc1f581be7887606657e0d8268c4c08f8303d
+  data.tar.gz: a069124becac1289cb2e43e58ae7cf8fc47727a5b0b7860551c46ff6d6531d48e145eab296884bcc1a3a33df7da0660e3bbea8f1309a8d8c404021b7828f8d66

data/.env.example

@@ -3,3 +3,4 @@ ZUORA_CLIENT_SECRET=secret
 ZUORA_AUTH_TYPE=token
 ZUORA_DOMAIN=rest.apisandbox.zuora.com
 ZUORA_TENANT_ID=00000
+ZUORA_EXCLUDED_FIELDS_FILE=excluded_fields.yml

data/.gitignore

@@ -15,3 +15,6 @@
 
 # byebug history
 .byebug_history
+
+# do not version specific excluded fields
+excluded_fields.yml

data/.reek.yml

@@ -13,6 +13,7 @@ detectors:
       - IronBank::Configuration#client_id
       - IronBank::Configuration#client_secret
       - IronBank::Configuration#domain
+      - IronBank::Configuration#excluded_fields
       - IronBank::Configuration#export_directory
       - IronBank::Configuration#logger
       - IronBank::Configuration#middlewares
@@ -54,13 +55,14 @@ detectors:
 
   NestedIterators:
     exclude:
-      - IronBank::Schema#self.import
+      - IronBank::Authentications::Cookie#connection
+      - IronBank::Authentications::Token#connection
+      - IronBank::Client#connection
+      - IronBank::Describe::ExcludedFields#remove_last_failure_fields
       - IronBank::Local#where
       - IronBank::LocalRecords#export
       - IronBank::Resources::ProductRatePlanChargeTier#self.load_records
-      - IronBank::Client#connection
-      - IronBank::Authentications::Token#connection
-      - IronBank::Authentications::Cookie#connection
+      - IronBank::Schema#self.import
 
   NilCheck:
     enabled: false
@@ -71,22 +73,16 @@ detectors:
       - IronBank::Configuration
 
   TooManyStatements:
+    max_statements: 10
     exclude:
-      - IronBank::Associations::ClassMethods#with_many
-      - IronBank::Associations::ClassMethods#with_one
-      - IronBank::Authentications::Cookie#connection
-      - IronBank::Authentications::Token#connection
       - IronBank::Client#connection
-      - IronBank::Configuration#schema_directory=
-      - IronBank::Configuration#initialize
-      - IronBank::Error#message_from_body
-      - IronBank::Queryable#find_each
-      - IronBank::Queryable#where
       - IronBank::Utils#camelize
-      - IronBank::Utils#underscore
-      - IronBank::Resources::ProductRatePlanChargeTier#self.load_records
       - IronBank::Error#self.from_response
 
+  UncommunicativeVariableName:
+    exclude:
+      - IronBank::Describe::ExcludedFields#valid_query?
+
   UtilityFunction:
     exclude:
       - IronBank::Cacheable::ClassMethods#cache

data/.rubocop.yml

@@ -56,6 +56,10 @@ Style/OptionalArguments:
     - lib/iron_bank/queryable.rb
     - lib/iron_bank/resource.rb
 
+Style/RegexpLiteral:
+  Exclude:
+    - lib/iron_bank/describe/excluded_fields.rb
+
 Style/RescueStandardError:
   Exclude:
     - lib/iron_bank/csv.rb

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    iron_bank (3.4.0)
+    iron_bank (4.0.0)
       faraday (~> 0)
       faraday_middleware (~> 0)
       nokogiri (~> 1)

data/README.md

@@ -132,7 +132,7 @@ https://github.com/zendesk/iron_bank.
 
 - `AutoPay` field on the `Invoice` object is not queryable using ZOQL despite
   the metadata showing `<selectable>true</selectable>`, hence it has been added
-  to the `exclude_fields` method.
+  to the `excluded_fields` method.
 - Exporting the local records through `IronBank::LocalRecords.export` can take a
   long time (especially the `ProductRatePlanChargeTier` records). In case the
   task fails because the export is still processing, you can manually download

data/Rakefile

@@ -21,16 +21,31 @@ task default: %i[rubocop reek spec]
 
 desc "Export the Zuora Schema using the Describe API"
 task :export_schema do
+  setup_iron_bank
+  IronBank::Schema.export
+end
+
+desc "Query Zuora for fields that we should not use in a query"
+task :excluded_fields, [:filename] do |_t, args|
+  require "psych"
+  setup_iron_bank
+
+  destination = args[:filename] || IronBank.configuration.excluded_fields_file
+  fields      = IronBank::Schema.excluded_fields
+
+  File.open(destination, "w") { |file| file.write(Psych.dump(fields)) }
+end
+
+# Helper function to set up an `IronBank::Client` instance
+def setup_iron_bank
   require "dotenv/load"
   require "iron_bank"
 
-  # Set up the client
-  IronBank.client = IronBank::Client.new(
-    domain:        ENV["ZUORA_DOMAIN"],
-    client_id:     ENV["ZUORA_CLIENT_ID"],
-    client_secret: ENV["ZUORA_CLIENT_SECRET"],
-    auth_type:     ENV["ZUORA_AUTH_TYPE"]
-  )
-
-  IronBank::Schema.export
+  IronBank.configure do |config|
+    config.client_id            = ENV["ZUORA_CLIENT_ID"]
+    config.client_secret        = ENV["ZUORA_CLIENT_SECRET"]
+    config.auth_type            = ENV.fetch("ZUORA_AUTH_TYPE", "token")
+    config.domain               = ENV["ZUORA_DOMAIN"]
+    config.excluded_fields_file = ENV["ZUORA_EXCLUDED_FIELDS_FILE"]
+  end
 end

data/bin/console

@@ -6,14 +6,12 @@ require "bundler/setup"
 require "iron_bank"
 require "pry-byebug"
 
-credentials = {
-  domain:        ENV["ZUORA_DOMAIN"],
-  client_id:     ENV["ZUORA_CLIENT_ID"],
-  client_secret: ENV["ZUORA_CLIENT_SECRET"],
-  auth_type:     ENV["ZUORA_AUTH_TYPE"]
-}
-
-# If credentials are present in environment variables
-IronBank.client = IronBank::Client.new(credentials) if credentials.values.all?
+IronBank.configure do |config|
+  config.client_id            = ENV["ZUORA_CLIENT_ID"]
+  config.client_secret        = ENV["ZUORA_CLIENT_SECRET"]
+  config.auth_type            = ENV.fetch("ZUORA_AUTH_TYPE", "token")
+  config.domain               = ENV["ZUORA_DOMAIN"]
+  config.excluded_fields_file = ENV["ZUORA_EXCLUDED_FIELDS_FILE"]
+end
 
 Pry.start

data/lib/iron_bank/configuration.rb

@@ -25,6 +25,9 @@ module IronBank
     # Cache store instance, optionally used by certain resources.
     attr_accessor :cache
 
+    # File path for excluded fields (when querying using ZOQL)
+    attr_accessor :excluded_fields_file
+
     # Directory where the XML describe files are located.
     attr_reader :schema_directory
 
@@ -78,5 +81,21 @@ module IronBank
     def retry_options
       @retry_options ||= IronBank::Client::DEFAULT_RETRY_OPTIONS
     end
+
+    def excluded_fields
+      return {} unless excluded_fields_file
+
+      unless File.exist?(excluded_fields_file)
+        IronBank.logger.warn "File does not exist: #{excluded_fields_file}"
+
+        return {}
+      end
+
+      @excluded_fields ||= begin
+        Psych.load_file(excluded_fields_file).tap do |fields|
+          raise "Excluded fields must be a hash" unless fields.is_a?(Hash)
+        end
+      end
+    end
   end
 end

data/lib/iron_bank/describe/excluded_fields.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module IronBank
+  module Describe
+    # Returns an array of non-queryable fields for the given object in the
+    # current Zuora tenant, despites Zuora clearly marking these fields as
+    # `<selectable>true</true>` in their Describe API... /rant
+    #
+    class ExcludedFields
+      extend Forwardable
+
+      FAULT_FIELD_MESSAGES = Regexp.union(
+        # Generic fault field
+        /invalid field for query: \w+\.(\w+)/,
+        # Invoice Bill Run is not selectable in ZOQL
+        /Cannot use the (BillRunId) field in the select clause/,
+        # Invoice Body, implemented as a separate call
+        /Can only query one invoice (body) at a time/,
+        # Catalog tier should only query the price field
+        /use Price or (DiscountAmount) or (DiscountPercentage)/,
+        # Catalog plan currencies, implemented as a separate call
+        /When querying for (active currencies)/,
+        # Catalog charge rollover balance
+        /You can only query (RolloverBalance) in particular/,
+        # (Subscription) charge should only query the price field
+        %r{
+          (OveragePrice),
+          \ Price,
+          \ (IncludedUnits),
+          \ (DiscountAmount)
+          \ or\ (DiscountPercentage)
+        }x
+      ).freeze
+
+      private_class_method :new
+
+      def self.call(object_name:)
+        new(object_name).call
+      end
+
+      def call
+        remove_last_failure_fields until valid_query?
+
+        excluded_fields
+      end
+
+      private
+
+      attr_reader :object_name, :excluded_fields, :last_failed_fields
+
+      def_delegators "IronBank.logger", :info
+
+      def initialize(object_name)
+        @object_name        = object_name
+        @excluded_fields    = []
+        @last_failed_fields = nil
+      end
+
+      def object
+        IronBank::Resources.const_get(object_name)
+      end
+
+      def remove_last_failure_fields
+        query_fields = object.query_fields
+
+        failed_fields = query_fields.select do |field|
+          last_failed_fields.any? { |failed| field.casecmp?(failed) }
+        end
+
+        @excluded_fields += failed_fields
+
+        # Remove the field for the next query
+        query_fields.delete_if { |field| failed_fields.include?(field) }
+      end
+
+      def valid_query?
+        object.first
+        info "Successful query for #{object_name}"
+
+        true
+      rescue IronBank::InternalServerError => e
+        @last_failed_fields = extract_fields_from_exception(e)
+
+        false
+      end
+
+      def extract_fields_from_exception(exception)
+        message = exception.message
+
+        unless FAULT_FIELD_MESSAGES.match(message)
+          raise "Could not parse error message: #{message}"
+        end
+
+        failed_fields = Regexp.last_match.
+                        captures.
+                        compact.
+                        map { |capture| capture.delete(" ") }
+
+        info "Invalid fields '#{failed_fields}' for #{object_name} query"
+
+        failed_fields
+      end
+    end
+  end
+end

data/lib/iron_bank/metadata.rb

@@ -4,22 +4,24 @@ module IronBank
   # Metadata to provide accessors to Zuora resources.
   #
   module Metadata
-    # Can be overriden to exclude specific fields for a given resource, see
-    # `Account` class for an example
-    def exclude_fields
-      []
+    def excluded_fields
+      return [] unless (fields = IronBank.configuration.excluded_fields)
+
+      # Return the field for the given resource name
+      # (where the module is extended from)
+      fields.fetch(object_name, [])
     end
 
     def fields
       return [] unless schema
 
-      @fields ||= schema.fields.map(&:name) - exclude_fields
+      @fields ||= schema.fields.map(&:name) - excluded_fields
     end
 
     def query_fields
       return [] unless schema
 
-      @query_fields ||= schema.query_fields - exclude_fields
+      @query_fields ||= schema.query_fields - excluded_fields
     end
 
     def schema

data/lib/iron_bank/resources/account.rb

@@ -9,18 +9,6 @@ module IronBank
     # method (usually a credit card, but PayPal is also accepted by Zuora).
     #
     class Account < Resource
-      # Tenants without credit memo activated cannot query these fields BUT they
-      # are still described as `selectable` through the metadata.
-      #
-      # Similarly, accounts with `TaxExemptStatus` set to `No` cannot query
-      # the `TaxExemptEntityUseCode` related fields.
-      def self.exclude_fields
-        %w[
-          TaxExemptEntityUseCode
-          TotalDebitMemoBalance
-          UnappliedCreditMemoAmount
-        ]
-      end
       with_schema
 
       # Contacts

data/lib/iron_bank/resources/amendment.rb

@@ -7,6 +7,7 @@ module IronBank
     #
     class Amendment < Resource
       with_schema
+
       with_one :subscription
     end
   end

data/lib/iron_bank/resources/contact.rb

@@ -7,6 +7,7 @@ module IronBank
     #
     class Contact < Resource
       with_schema
+
       with_one :account
     end
   end

data/lib/iron_bank/resources/import.rb

@@ -6,6 +6,7 @@ module IronBank
     #
     class Import < Resource
       with_schema
+
       with_many :usages
     end
   end

data/lib/iron_bank/resources/invoice.rb

@@ -6,19 +6,11 @@ module IronBank
     # holds many invoice items.
     #
     class Invoice < Resource
-      # These fields are declared as `<selectable>true</selectable>` but Zuora
-      # returns a QueryError when trying to query an invoice with them. Also,
-      # the `Body` field can only be retrieved for a single invoice at a time.
-      def self.exclude_fields
-        %w[
-          AutoPay
-          BillRunId
-          BillToContactSnapshotId
-          Body
-          RegenerateInvoicePDF
-          SoldToContactSnapshotId
-        ]
+      # See the comment for the instance method `#body`
+      def self.excluded_fields
+        super + %w[Body]
       end
+
       with_schema
 
       with_one :account

data/lib/iron_bank/resources/invoice_adjustment.rb

@@ -6,6 +6,7 @@ module IronBank
     #
     class InvoiceAdjustment < Resource
       with_schema
+
       with_one :account
       with_one :invoice
     end

data/lib/iron_bank/resources/invoice_item.rb

@@ -5,12 +5,6 @@ module IronBank
     # An invoice item holds a charge that is billed to a customer.
     #
     class InvoiceItem < Resource
-      def self.exclude_fields
-        %w[
-          AppliedToChargeNumber
-          Balance
-        ]
-      end
       with_schema
 
       with_one :invoice

data/lib/iron_bank/resources/invoice_payment.rb

@@ -6,6 +6,7 @@ module IronBank
     #
     class InvoicePayment < Resource
       with_schema
+
       with_one :invoice
       with_one :payment
     end

data/lib/iron_bank/resources/payment_method.rb

@@ -8,6 +8,7 @@ module IronBank
     #
     class PaymentMethod < Resource
       with_schema
+
       with_one :account
     end
   end

data/lib/iron_bank/resources/product.rb

@@ -8,6 +8,7 @@ module IronBank
       with_schema
       with_local_records
       with_cache
+
       with_many :product_rate_plans, alias: :plans
     end
   end

data/lib/iron_bank/resources/product_rate_plan.rb

@@ -6,10 +6,10 @@ module IronBank
     # charges. It represents what a customer is subscribing to.
     #
     class ProductRatePlan < Resource
-      # Zuora limitation: we cannot query for `ActiveCurrencies` with any other
-      # fields, so instead we define a separate `#active_currencies` method.
-      def self.exclude_fields
-        %w[ActiveCurrencies]
+      # NOTE: Zuora doesn't let us query for more than one product rate plan
+      #       `ActiveCurrencies` at a time
+      def self.excluded_fields
+        super + %w[ActiveCurrencies]
       end
 
       with_schema

data/lib/iron_bank/resources/product_rate_plan_charge.rb

@@ -10,11 +10,6 @@ module IronBank
     # is *at least* one tier per active currency.
     #
     class ProductRatePlanCharge < Resource
-      # `DiscountClass` seems to not be a queryable field, despite the result
-      # from the describe call for product rate plan charge.
-      def self.exclude_fields
-        %w[DiscountClass]
-      end
       with_schema
       with_local_records
       with_cache

data/lib/iron_bank/resources/product_rate_plan_charge_tier.rb

@@ -6,20 +6,6 @@ module IronBank
     # charge.
     #
     class ProductRatePlanChargeTier < Resource
-      # These fields are declared as `<selectable>true</selectable>` but only
-      # for the `export` context (ie., via ZOQL export). To let live queries go
-      # through when the local records are not exported, we only allow querying
-      # the non-discount tiers.
-      def self.exclude_fields
-        %w[
-          Active
-          IncludedUnits
-          OveragePrice
-          DiscountAmount
-          DiscountPercentage
-        ]
-      end
-
       with_schema
       with_local_records
       with_cache

data/lib/iron_bank/resources/rate_plan_charge.rb

@@ -5,18 +5,10 @@ module IronBank
     # A rate plan charge belongs to a subscription rate plan.
     #
     class RatePlanCharge < Resource
-      def self.exclude_fields
-        %w[
-          DiscountAmount
-          DiscountClass
-          DiscountPercentage
-          IncludedUnits
-          OveragePrice
-          Price
-          RevenueRecognitionRuleName
-          RolloverBalance
-        ]
+      def self.excluded_fields
+        super + %w[RolloverBalance]
       end
+
       with_schema
       with_cache
 
@@ -25,6 +17,10 @@ module IronBank
       with_one :rate_plan, alias: :plan
 
       with_many :rate_plan_charge_tiers, alias: :tiers
+
+      def rollover_balance
+        remote[:rollover_balance] || reload.remote[:rollover_balance]
+      end
     end
   end
 end

data/lib/iron_bank/resources/rate_plan_charge_tier.rb

@@ -6,17 +6,6 @@ module IronBank
     # billed for, in his/her currency.
     #
     class RatePlanChargeTier < Resource
-      # Excluding 'DiscountAmount' & 'DiscountPercentage', as Zuora stores these
-      # values in the 'Price' field.
-      def self.exclude_fields
-        %w[
-          Currency
-          DiscountAmount
-          DiscountPercentage
-          IncludedUnits
-          OveragePrice
-        ]
-      end
       with_schema
       with_cache
 

data/lib/iron_bank/resources/taxation_item.rb

@@ -6,13 +6,6 @@ module IronBank
     # item.
     #
     class TaxationItem < Resource
-      def self.exclude_fields
-        %w[
-          Balance
-          CreditAmount
-          PaymentAmount
-        ]
-      end
       with_schema
 
       with_one :invoice

data/lib/iron_bank/schema.rb

@@ -34,6 +34,15 @@ module IronBank
       @import = nil
     end
 
+    def self.excluded_fields
+      @excluded_fields ||= begin
+        IronBank::Resources.constants.each.with_object({}) do |resource, fields|
+          fields[resource.to_s] =
+            IronBank::Describe::ExcludedFields.call(object_name: resource)
+        end
+      end
+    end
+
     def export
       tenant.objects.compact.each do |object|
         begin

data/lib/iron_bank/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module IronBank
-  VERSION     = "3.4.0"
+  VERSION     = "4.0.0"
   API_VERSION = "v1"
 end

data/lib/iron_bank.rb

@@ -80,6 +80,7 @@ require "iron_bank/authentications/cookie"
 require "iron_bank/authentications/token"
 require "iron_bank/client"
 require "iron_bank/describe/field"
+require "iron_bank/describe/excluded_fields"
 require "iron_bank/describe/object"
 require "iron_bank/describe/related"
 require "iron_bank/describe/tenant"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iron_bank
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Mickael Pham
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-03 00:00:00.000000000 Z
+date: 2019-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -287,6 +287,7 @@ files:
 - lib/iron_bank/collection.rb
 - lib/iron_bank/configuration.rb
 - lib/iron_bank/csv.rb
+- lib/iron_bank/describe/excluded_fields.rb
 - lib/iron_bank/describe/field.rb
 - lib/iron_bank/describe/object.rb
 - lib/iron_bank/describe/related.rb