miscellany 0.1.0

data/lib/miscellany/active_record/custom_preloaders.rb

@@ -0,0 +1,59 @@
+# Example Usage (Preloading items with reference to the same Polymorphic Object):
+#
+# has_many :related_objects, -> (self) { where(poly_type: self.poly_type, poly_id: self.poly_id) }, preloader: 'RelatedObjectPreloader'
+#
+# class RelatedObjectPreloader < ActiveRecord::Associations::Preloader::Association
+#   def run(preloader)
+#     @preloaded_records = []
+#     owners.group_by(&:poly_type).each do |type, owner_group|
+#       ids = owner_group.map(&:poly_id)
+#
+#       ids_to_priors = {}
+#       scope = Poly.scope_for_association.where(poly_type: self.poly_type, poly_id: self.poly_id)
+#       scope.find_each do |pa|
+#         ids_to_priors[pa.poly_id] ||= []
+#         ids_to_priors[pa.poly_id] << pa
+#         @preloaded_records << pa
+#       end
+#
+#       owner_group.each do |owner|
+#         priors = ids_to_priors[owner.poly_id] || []
+#
+#         association = owner.association(reflection.name)
+#         association.loaded!
+#         association.target = priors
+#
+#         # association.set_inverse_instance(record)
+#       end
+#     end
+#   end
+# end
+#
+module Miscellany
+  module CustomPreloaders
+    module AssociationBuilderExtension
+      def self.build(model, reflection); end
+
+      def self.valid_options
+        [:preloader]
+      end
+    end
+
+    module PreloaderExtension
+      def preloader_for(reflection, owners)
+        cust_preloader = reflection.options[:preloader]
+        if cust_preloader.present?
+          cust_preloader = cust_preloader.constantize if cust_preloader.is_a?(String)
+          cust_preloader
+        else
+          super
+        end
+      end
+    end
+
+    def self.install
+      ActiveRecord::Associations::Builder::Association.extensions << AssociationBuilderExtension
+      ActiveRecord::Associations::Preloader.prepend(CustomPreloaders::PreloaderExtension)
+    end
+  end
+end

data/lib/miscellany/batch_processor.rb

@@ -0,0 +1,41 @@
+module Miscellany
+  # An array that "processes" after so many items are added.
+  #
+  # Example Usage:
+  #   batches = BatchProcessor.new(of: 1000) do |batch|
+  #     # Process the batch somehow
+  #   end
+  #   enumerator_of_some_kind.each { |item| batches << item }
+  #   batches.flush
+  class BatchProcessor
+    attr_reader :batch_size
+
+    def initialize(of: 1000, &blk)
+      @batch_size = of
+      @block = blk
+      @current_batch = []
+    end
+
+    def <<(item)
+      @current_batch << item
+      process_batch if @current_batch.count >= batch_size
+    end
+
+    def add_all(items)
+      items.each do |i|
+        self << i
+      end
+    end
+
+    def flush
+      process_batch if @current_batch.present?
+    end
+
+    protected
+
+    def process_batch
+      @block.call(@current_batch)
+      @current_batch = []
+    end
+  end
+end

data/lib/miscellany/batching_csv_processor.rb

@@ -0,0 +1,86 @@
+module Miscellany
+  class BatchingCsvProcessor
+    attr_accessor :csv, :file_name
+
+    class RowError < StandardError; end
+
+    def initialize(csv, file_name: nil)
+      @csv = csv
+      @file_name = file_name
+    end
+
+    def process_in_batches(&blk)
+      batch = BatchProcessor.new(&blk)
+      CSV.new(csv, headers: true, header_converters: :symbol).each.with_index do |row, line|
+        row[:line_number] = line + 1
+        next unless validate_line(row)
+
+        batch << row
+      end
+      batch.flush
+    end
+
+    def get_row_errors(row); end
+
+    def find_or_init(_row)
+      raise NotImplementedError
+    end
+
+    def apply_row_to_model(_row, _instance)
+      raise NotImplementedError
+    end
+
+    def log_line_error(message, line_number, **kwargs)
+      raise NotImplementedError
+    end
+
+    def validate_line(row)
+      errors = get_row_errors(row) || []
+      if errors.present?
+        log_line_error(errors[0], row[:line_number])
+        false
+      else
+        true
+      end
+    end
+
+    def batch_rows_to_models(rows)
+      rows.map { |row| build_model_from_row(row) }.reject! { |inst| inst.nil? || !inst.changed? }
+    end
+
+    def build_model_from_row(row)
+      model = find_or_init(row)
+      apply_row_to_model(row, model)
+      return nil if model.respond_to?(:discarded_at) && model.discarded_at.present? && !model.persisted?
+
+      model
+    rescue ActiveRecord::RecordNotFound, RowError => err
+      log_line_error(err.message, row[:line_number], exception: err)
+      nil
+    rescue StandardError => err
+      log_line_error('An Internal Error Occurred', row[:line_number], exception: err)
+      Raven.capture_exception(err)
+      nil
+    end
+
+    def map_defined_columns(row, map)
+      newh = {}
+      map.each do |newk, oldk|
+        next unless row.include?(oldk)
+
+        newh[newk] = row[oldk]
+      end
+      newh
+    end
+
+    def self.file_matches?(file)
+      header = file.readline.strip.split(',')
+      file.try(:rewind)
+      headers_match?(header)
+    end
+
+    def self.headers_match?(headers)
+      (self::HEADERS - headers).empty?
+    end
+  end
+end

data/lib/miscellany/controller/http_error_handling.rb

@@ -0,0 +1,62 @@
+module Miscellany
+  module HttpErrorHandling
+    extend ActiveSupport::Concern
+
+    class UnauthorizedError < StandardError; end
+
+    class HttpError < StandardError
+      attr_accessor :status, :extra
+      def initialize(arg = nil, status: nil, message: '', **extra)
+        if arg.is_a?(Numeric)
+          raise ArgumentError, ':status supplied multiple times' if status.present?
+
+          status = arg
+        elsif arg.present?
+          raise ArgumentError, ':message supplied multiple times' if message.present?
+
+          message = arg
+        end
+        super(message)
+        @status = status
+        @extra = extra
+      end
+    end
+
+    module ClassMethods
+      def http_error(status = 400, message = nil, &blk)
+        message = blk if blk.present?
+        lambda { |err|
+          # This is necessary to ensure that HttpErrors are never handled as StandardErrors
+          if err.is_a?(HttpError)
+            render_http_error(err)
+          else
+            render_http_error(err, status: status, message: message)
+          end
+        }
+      end
+
+      def rescue_with_http_error(*errors, status: nil, message: nil, **kwargs)
+        rescue_from(*errors, with: http_error(status, message), **kwargs)
+      end
+    end
+
+    included do
+      rescue_from HttpError do |err|
+        render_http_error(err)
+      end
+
+      rescue_with_http_error UnauthorizedError, status: 401
+    end
+
+    def render_http_error(err, status: nil, message: nil)
+      status = err.status if err.is_a?(HttpError) && status.nil?
+      status ||= 400
+      message ||= err.message
+      message.message.call(err) if message.is_a?(Proc)
+      response_json = { status: status }
+      response_json[:message] = message if message.present?
+      response_json.merge!(err.extra)
+      render json: response_json, status: status
+    end
+  end
+end

data/lib/miscellany/controller/json_uploads.rb

@@ -0,0 +1,41 @@
+module Miscellany
+  module JsonUploads
+    extend ActiveSupport::Concern
+
+    # This hackery allows using JSON params along with file-uploads
+    # On the frontend, use MultiPart form data, and add _parameters as an entry
+    def params
+      if @_params.nil?
+        params_obj = @_params = super
+        phash = params_obj.instance_variable_get(:@parameters)
+        if phash[:_parameters].present?
+          json_layer = JSON.parse(phash.delete(:_parameters))
+          shared_keys = phash.keys & json_layer.keys
+          main_layer = phash.slice(*shared_keys)
+          merge_params(json_layer, main_layer)
+          phash.merge!(json_layer)
+        end
+      end
+      @_params
+    end
+
+    private
+
+    def merge_params(base, layer)
+      if base.is_a?(Array) && (layer.is_a?(Hash) || layer.is_a?(Array))
+        base.each_with_index.map do |v, i|
+          over_key = layer.is_a?(Hash) ? i.to_s : i
+          merge_params(v, layer[over_key])
+        end
+      elsif base.is_a?(Hash) && layer.is_a?(Hash)
+        base.each do |k, v|
+          base[k] = merge_params(v, layer[k])
+        end
+        layer.merge(base)
+      else
+        Rails.logger.warn 'Duplicate parameter passed' if base.present? && layer.present?
+        layer.presence || base
+      end
+    end
+  end
+end

data/lib/miscellany/controller/sliced_response.rb

@@ -0,0 +1,271 @@
+module Miscellany
+  module SlicedResponse
+    extend ActiveSupport::Concern
+
+    include HttpErrorHandling
+
+    def slice_results(queryset, **kwargs)
+      @sliced_data = sliced_json(queryset, **kwargs) { |x| x }
+    end
+
+    # rubocop:disable Metrics/ParameterLists
+    def sliced_json(
+      queryset, slice_params = params,
+      max_size: 50, default_size: 25, allow_all: false,
+      default_sort: nil, valid_sorts: nil,
+      &blk
+    )
+      normalized_sorts = normalize_sort_options(valid_sorts || queryset.column_names, default: default_sort)
+
+      slice = Slice.build(
+        queryset, slice_params,
+        default_page_size: default_size,
+        item_transformer: blk,
+        max_size: max_size,
+        default_size: default_size,
+        allow_all: allow_all,
+        valid_sorts: normalized_sorts,
+      )
+
+      slice.render_json
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    def as_sliced_json(queryset, slice: nil, total_count: nil, &blk)
+      slice = Slice.build(queryset, slice, total_count: total_count, item_transformer: blk)
+      slice.render_json
+    end
+
+    def bearcat_as_sliced_json(*args, transform: nil, **kwargs, &blk)
+      bearcat_exec = ->(slice) {
+        response = blk.call({
+          per_page: slice.page_size,
+          page: slice.page_number,
+        })
+
+        slice.rendered_json[:page_count] = response.page_count
+
+        response
+      }
+      sliced_json(bearcat_exec, *args, valid_sorts: {}, **kwargs, allow_all: false, &transform)
+    end
+
+    private
+
+    def normalize_sort_options(sorts, default: nil)
+      norm_sorts = { }
+
+      sorts.each do |s|
+        if s.is_a?(Hash)
+          s.each do |k,v|
+            sort_hash = normalize_sort(v, key: k)
+            norm_sorts[k] = sort_hash
+          end
+        else
+          sort_hash = normalize_sort(s)
+          norm_sorts[sort_hash[:column]] = sort_hash
+          # default ||= sort_hash
+        end
+      end
+
+      if default.present?
+        norm_default = normalize_sort(default)
+        reference = norm_sorts[norm_default[:key].to_s] || norm_default
+        norm_sorts[:default] = {
+          key: reference[:key],
+          column: reference[:column],
+          order: norm_default[:order] || reference[:order],
+        }
+      end
+
+      norm_sorts
+    end
+
+    def normalize_sort(sort, key: nil)
+      sort = sort.to_s if sort.is_a?(Symbol)
+      if sort.is_a?(String)
+        m = sort.match(/(\w+)(?: (ASC|DESC)(!?))?/)
+        sort = { column: m[1], order: m[2], force_order: m[3].present? }.compact
+      elsif sort.is_a?(Proc)
+        sort = { column: sort }
+      end
+      sort[:key] = key || sort[:column]
+      sort.compact
+    end
+
+    class Slice
+      attr_accessor :slice_start, :slice_end
+      attr_accessor :page_size, :page_number
+      attr_accessor :sort
+
+      attr_reader :items
+      attr_reader :options
+
+      def initialize(items, options={})
+        @items = items
+        @options = options
+      end
+
+      # rubocop:disable Metrics/PerceivedComplexity
+      def self.build(queryset, arg, options={})
+        new(queryset, options).tap do |slice|
+          if arg.is_a?(Array)
+            slice_bounds = arg
+          elsif arg.is_a?(String)
+            slice_bounds = arg.split(':').map(&:to_i)
+          elsif arg.is_a?(Range)
+            slice_bounds = arg.minmax
+          elsif arg.respond_to? :[]
+            if arg[:slice] == 'all' || arg[:page] == 'all'
+              slice_bounds = [0, -1]
+            elsif arg[:slice].present?
+              slice_bounds = arg[:slice].split(':').map(&:to_i)
+            else
+              page_size = slice[:page_size] = (arg[:page_size] || options[:default_page_size]).to_i
+              page_number = slice[:page_number] = (arg[:page] || 1).to_i
+              slice_bounds = [(page_number - 1) * page_size, page_number * page_size]
+            end
+
+            slice[:sort] = parse_and_validate_sorts(arg[:sort], options[:valid_sorts])
+          end
+
+          if slice_bounds.present?
+            slice[:slice_start] = slice_bounds[0]
+            slice[:slice_end] = slice_bounds[1]
+          end
+
+          if slice[:slice_end] == -1
+            raise HttpErrorHandling::HttpError, message: "cannot request whole collection" unless options[:allow_all]
+          else
+            if options[:max_size] && (slice[:slice_end] - slice[:slice_start]) > [options[:max_size], options[:default_size]].max
+              raise HttpErrorHandling::HttpError, message: "cannot request more than #{options[:max_size]} objects"
+            end
+          end
+        end
+      end
+      # rubocop:enable Metrics/PerceivedComplexity
+
+      def [](key)
+        send(key)
+      end
+
+      def []=(key, val)
+        send(:"#{key}=", val)
+      end
+
+      def render_json
+        return @rendered_json if defined?(@rendered_json)
+
+        json = @rendered_json = {
+          slice_start: slice_start,
+          slice_end: slice_end,
+        }
+
+        rendered_items
+
+        if page_size.present?
+          json[:page] ||= page_number
+          json[:page_size] ||= page_size
+        end
+
+        if total_item_count.present?
+          json[:total_count] ||= total_item_count
+          json[:page_count] ||= (total_item_count.to_f / page_size).ceil if page_size.present?
+        end
+
+        if sort.present?
+          json[:sort] ||= sort.map do |s|
+            "#{s[:key] || s[:column]} #{s[:order] || 'ASC'}"
+          end.join(', ')
+        end
+
+        json[:slice_start] ||= 0
+        json[:slice_end] ||= total_item_count
+
+        json[:items] = rendered_items
+
+        json
+      end
+      def rendered_json; render_json; end
+
+      def self.parse_and_validate_sorts(sortstr, sorts_map, silent_failure: true)
+        (sortstr || '').split(',').map do |s|
+          m = s.strip.match(/^(\w+)(?: (ASC|DESC))?$/)
+
+          if m.nil?
+            next if silent_failure
+            raise HttpErrorHandling::HttpError, message: 'Could not parse sort parameter'
+          end
+
+          resolved_sort = sorts_map[m[1]]
+          unless resolved_sort.present?
+            next if silent_failure
+            raise HttpErrorHandling::HttpError, message: 'Could not parse sort parameter'
+          end
+
+          sort = resolved_sort.dup
+          sort[:order] = m[2] if m[2].present? && !sort[:force_order]
+          sort
+        end.compact.presence || [sorts_map[:default]].compact
+      end
+
+      private
+
+      def rendered_items
+        ritems = sliced_items
+        ritems = ritems.map(&options[:item_transformer]) if options[:item_transformer]
+        ritems
+      end
+
+      def total_item_count
+        @total_item_count ||= options[:total_count] || (items.respond_to?(:count) && items.count) || nil
+      end
+
+      def sliced_items
+        @sliced_items ||= begin
+          if items.is_a?(Array)
+            items
+          elsif items.is_a?(Proc)
+            items.call(self)
+          elsif items.is_a?(ActiveRecord::Relation)
+            offset, limit = slice_bounds
+            limit -= offset unless limit.nil?
+            apply_ar_sort(items).limit(limit).offset(offset)
+          end
+        end
+      end
+
+      def slice_bounds
+        [slice_start, slice_end == -1 ? nil : slice_end]
+      end
+
+      def apply_ar_sort(qset)
+        if sort.present?
+          sorts = [ *Array(self.sort) ]
+          sorts << options[:valid_sorts][:default] if options.dig(:valid_sorts, :default).present?
+
+          sorts.reduce(qset) do |qset, sort|
+            if sort[:column].is_a?(Proc)
+              sort[:column].call(qset, sort[:order] || 'ASC')
+            else
+              qset.order(sort[:column] => sort[:order] || 'ASC')
+            end
+          end
+        else
+          qset
+        end
+      end
+    end
+
+    module JbuilderTemplateExt
+      def partial!(*args, **kwargs, &blk)
+        kwargs[:block] = blk if blk.present?
+        super(*args, **kwargs)
+      end
+    end
+
+    def self.install_extensions
+      ::JbuilderTemplate.prepend JbuilderTemplateExt
+    end
+  end
+end