worker_tools 0.1.2 → 1.0.0

data/lib/worker_tools/csv_input.rb

@@ -54,6 +54,7 @@ module WorkerTools
     def csv_input_columns_check(csv_rows_enum)
       # override and return true if you do not want this check to be performed
       return csv_input_columns_array_check(csv_rows_enum) if csv_input_columns.is_a?(Array)
+
       csv_input_columns_hash_check(csv_rows_enum)
     end
 
@@ -61,7 +62,9 @@ module WorkerTools
       expected_columns_length = csv_input_columns.length
       actual_columns_length = csv_rows_enum.first.length
       return if expected_columns_length == actual_columns_length
-      raise "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+
+      msg = "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+      raise Errors::WrongNumberOfColumns, msg
     end
 
     def csv_input_columns_hash_check(csv_rows_enum)
@@ -73,7 +76,9 @@ module WorkerTools
 
     def csv_input_columns_hash_check_duplicates(names)
       dups = names.group_by(&:itself).select { |_, v| v.count > 1 }.keys
-      raise "The file contains duplicated columns: #{dups}" if dups.present?
+      return unless dups.present?
+
+      raise Errors::DuplicatedColumns, "The file contains duplicated columns: #{dups}"
     end
 
     def csv_input_columns_hash_check_missing(actual_names, expected_names)
@@ -81,7 +86,7 @@ module WorkerTools
         matchable = name.is_a?(String) ? csv_input_header_normalized(name) : name
         actual_names.any? { |n| case n when matchable then true end } # rubocop does not like ===
       end
-      raise "Some columns are missing: #{missing}" unless missing.empty?
+      raise Errors::MissingColumns, "Some columns are missing: #{missing}" unless missing.empty?
     end
 
     def csv_input_csv_options
@@ -105,6 +110,7 @@ module WorkerTools
     #     =>  { tenant: 1, area: 0}
     def csv_input_mapping_order(header_names)
       return csv_input_columns.map.with_index { |n, i| [n, i] }.to_h if csv_input_columns.is_a?(Array)
+
       csv_input_mapping_order_for_hash(header_names)
     end
 
@@ -115,6 +121,7 @@ module WorkerTools
         h[k] = filtered_column_names.index { |n| case n when matchable then true end }
       end
       return mapping unless csv_input_include_other_columns
+
       csv_input_mapping_order_with_other_columns(mapping, filtered_column_names)
     end
 
@@ -131,7 +138,7 @@ module WorkerTools
     end
 
     def csv_rows_enum
-      @csv_rows_enum ||= CSV.foreach(csv_input_file_path, csv_input_csv_options)
+      @csv_rows_enum ||= CSV.foreach(csv_input_file_path, **csv_input_csv_options)
     end
 
     def csv_input_headers_present
@@ -168,12 +175,14 @@ module WorkerTools
 
         @rows_enum.with_index.each do |values, index|
           next if index.zero? && @headers_present
+
           yield values_to_row(values)
         end
       end
 
       def values_to_row(values)
         return values_to_row_according_to_mapping(values) if @mapping_order
+
         values_to_row_according_to_position(values)
       end
 

data/lib/worker_tools/csv_output.rb

@@ -2,13 +2,6 @@ require 'csv'
 
 module WorkerTools
   module CsvOutput
-    # if defined, this file will be written to this destination (regardless
-    # of whether the model saves the file as well)
-    def csv_output_target
-      # Ex: Rails.root.join('shared', 'foo', 'bar.csv')
-      false
-    end
-
     def csv_output_entries
       raise "csv_output_entries has to be defined in #{self}"
     end
@@ -27,33 +20,18 @@ module WorkerTools
       raise "csv_output_column_headers has to be defined in #{self}"
     end
 
-    # rubocop:disable Lint/UnusedMethodArgument
     def csv_output_row_values(entry)
-      # Ex:
-      # {
-      #   foo: entry.foo,
-      #   bar: entry.bar
-      # }.values_at(*csv_output_column_headers.keys)
-      raise "csv_output_row_values has to be defined in #{self}"
-    end
-    # rubocop:enable Lint/UnusedMethodArgument
-
-    def cvs_output_target_folder
-      File.dirname(csv_output_target)
-    end
-
-    def csv_output_target_file_name
-      File.basename(csv_output_target)
-    end
-
-    def csv_ouput_ensure_target_folder
-      FileUtils.mkdir_p(cvs_output_target_folder) unless File.directory?(cvs_output_target_folder)
+      entry.values_at(*csv_output_column_headers.keys)
     end
 
     def csv_output_tmp_file
       @csv_output_tmp_file ||= Tempfile.new(['output', '.csv'])
     end
 
+    def csv_output_file_name
+      "#{model_kind}.csv"
+    end
+
     def csv_output_col_sep
       ';'
     end
@@ -62,21 +40,29 @@ module WorkerTools
       Encoding::UTF_8
     end
 
+    def csv_output_write_mode
+      'wb'
+    end
+
+    def csv_output_csv_options
+      { col_sep: csv_output_col_sep, encoding: csv_output_encoding }
+    end
+
     def csv_output_insert_headers(csv)
       csv << csv_output_column_headers.values if csv_output_column_headers
     end
 
+    def csv_output_add_attachment
+      model.add_attachment(csv_output_tmp_file, file_name: csv_output_file_name, content_type: 'text/csv')
+    end
+
     def csv_output_write_file
-      CSV.open(csv_output_tmp_file, 'wb', col_sep: csv_output_col_sep, encoding: csv_output_encoding) do |csv|
+      CSV.open(csv_output_tmp_file, csv_output_write_mode, **csv_output_csv_options) do |csv|
         csv_output_insert_headers(csv)
         csv_output_entries.each { |entry| csv << csv_output_row_values(entry) }
       end
-      csv_output_write_target if csv_output_target
-    end
 
-    def csv_output_write_target
-      csv_ouput_ensure_target_folder
-      FileUtils.cp(csv_output_tmp_file.path, csv_output_target)
+      csv_output_add_attachment
     end
   end
 end

data/lib/worker_tools/errors.rb

@@ -0,0 +1,10 @@
+module WorkerTools
+  Error = Class.new(StandardError)
+
+  module Errors
+    Invalid = Class.new(Error)
+    WrongNumberOfColumns = Class.new(Invalid)
+    DuplicatedColumns = Class.new(Invalid)
+    MissingColumns = Class.new(Invalid)
+  end
+end

data/lib/worker_tools/recorder.rb

@@ -23,34 +23,42 @@ module WorkerTools
     end
 
     def record_fail(error)
-      record "ID #{model.id} - Error"
       record(error, :error)
-      model.information = information
       model.save!(validate: false)
     end
 
-    def add_log(message, level = :info)
-      logger.public_send(level, format_log_message(message))
+    def add_log(message, level = nil)
+      attrs = default_message_attrs(message, level)
+      logger.public_send(attrs[:level], format_message(attrs[:message]))
     end
 
-    def add_info(message)
-      @information ||= ''
-      information << "#{format_info_message(message)}\n"
+    def add_note(message, level = nil)
+      attrs = default_message_attrs(message, level)
+      model.notes.push(level: attrs[:level], message: attrs[:message])
     end
 
     def record(message, level = :info)
       add_log(message, level)
-      add_info(message)
+      add_note(message, level)
     end
 
-    def format_log_message(message)
+    def level_from_message_type(message)
+      return :error if message.is_a?(Exception)
+
+      :info
+    end
+
+    def format_message(message)
       return error_to_text(message, log_error_trace_lines) if message.is_a?(Exception)
+
       message
     end
 
-    def format_info_message(message)
-      return error_to_text(message, info_error_trace_lines) if message.is_a?(Exception)
-      message
+    def default_message_attrs(message, level)
+      {
+        message: format_message(message),
+        level: level || level_from_message_type(message)
+      }
     end
 
     def logger

data/lib/worker_tools/slack_error_notifier.rb

@@ -5,7 +5,7 @@ module WorkerTools
     def with_wrapper_slack_error_notifier(&block)
       block.yield
     rescue StandardError => e
-      slack_error_notify(e) if slack_error_notifier_enabled
+      slack_error_notify(e) if slack_error_notifier_enabled && slack_error_notifiable?(e)
       raise
     end
 
@@ -13,6 +13,10 @@ module WorkerTools
       Rails.env.production?
     end
 
+    def slack_error_notifiable?(error)
+      error.is_a?(StandardError)
+    end
+
     def slack_error_notifier_emoji
       ':red_circle:'
     end
@@ -83,7 +87,7 @@ module WorkerTools
 
     def slack_error_notifier_attachments_fields
       [
-        { title: 'Application', value: Rails.application.class.parent_name, short: true },
+        { title: 'Application', value: Rails.application.class.module_parent_name, short: true },
         { title: 'Environment', value: Rails.env, short: true }
       ]
     end

data/lib/worker_tools/utils/hash_with_indifferent_access_type.rb

@@ -0,0 +1,9 @@
+module WorkerTools
+  module Utils
+    class HashWithIndifferentAccessType < ActiveRecord::Type::Json
+      def deserialize(value)
+        HashWithIndifferentAccess.new(super)
+      end
+    end
+  end
+end

data/lib/worker_tools/utils/serialized_array_type.rb

@@ -0,0 +1,19 @@
+module WorkerTools
+  module Utils
+    class SerializedArrayType < ActiveRecord::Type::Json
+      def initialize(type: nil)
+        @type = type
+      end
+
+      def deserialize(value)
+        super(value)&.map { |d| @type.deserialize(d) }
+      end
+
+      def serialize(value)
+        raise 'not an array' unless value.is_a?(Array)
+
+        super value
+      end
+    end
+  end
+end

data/lib/worker_tools/version.rb

@@ -1,3 +1,3 @@
 module WorkerTools
-  VERSION = '0.1.2'.freeze
+  VERSION = '1.0.0'.freeze
 end

data/lib/worker_tools/xlsx_input.rb

@@ -47,7 +47,9 @@ module WorkerTools
     end
 
     def xlsx_input_header_normalized(name)
-      name.to_s.strip.downcase
+      # some elements return obj.to_s => nil
+      # for example [#<Roo::Excelx::Cell::Empty:0x0000000af8d4c8 ... @value=nil>]
+      name&.to_s&.strip&.downcase || ''
     end
 
     # Allows for some basic cleanup of the values, such as applying strip to
@@ -59,6 +61,7 @@ module WorkerTools
     def xlsx_input_columns_check(xlsx_rows_enum)
       # override and return true if you do not want this check to be performed
       return xlsx_input_columns_array_check(xlsx_rows_enum) if xlsx_input_columns.is_a?(Array)
+
       xlsx_input_columns_hash_check(xlsx_rows_enum)
     end
 
@@ -66,7 +69,9 @@ module WorkerTools
       expected_columns_length = xlsx_input_columns.length
       actual_columns_length = xlsx_rows_enum.first.length
       return if expected_columns_length == actual_columns_length
-      raise "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+
+      msg = "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+      raise Errors::WrongNumberOfColumns, msg
     end
 
     def xlsx_input_columns_hash_check(xlsx_rows_enum)
@@ -78,7 +83,9 @@ module WorkerTools
 
     def xlsx_input_columns_hash_check_duplicates(names)
       dups = names.group_by(&:itself).select { |_, v| v.count > 1 }.keys
-      raise "The file contains duplicated columns: #{dups}" if dups.present?
+      return unless dups.present?
+
+      raise Errors::DuplicatedColumns, "The file contains duplicated columns: #{dups}"
     end
 
     def xlsx_input_columns_hash_check_missing(actual_names, expected_names)
@@ -86,7 +93,7 @@ module WorkerTools
         matchable = name.is_a?(String) ? xlsx_input_header_normalized(name) : name
         actual_names.any? { |n| case n when matchable then true end } # rubocop does not like ===
       end
-      raise "Some columns are missing: #{missing}" unless missing.empty?
+      raise Errors::MissingColumns, "Some columns are missing: #{missing}" unless missing.empty?
     end
 
     # Compares the first row (header names) with the xlsx_input_columns hash to find
@@ -97,6 +104,7 @@ module WorkerTools
     #     =>  { tenant: 1, area: 0}
     def xlsx_input_mapping_order(header_names)
       return xlsx_input_columns.map.with_index { |n, i| [n, i] }.to_h if xlsx_input_columns.is_a?(Array)
+
       xlsx_input_mapping_order_for_hash(header_names)
     end
 
@@ -107,6 +115,7 @@ module WorkerTools
         h[k] = filtered_column_names.index { |n| case n when matchable then true end }
       end
       return mapping unless xlsx_input_include_other_columns
+
       xlsx_input_mapping_order_with_other_columns(mapping, filtered_column_names)
     end
 
@@ -155,6 +164,7 @@ module WorkerTools
 
         @rows_enum.with_index.each do |values, index|
           next if index.zero? # headers
+
           yield values_to_row(values)
         end
       end

data/lib/worker_tools/xlsx_output.rb

@@ -1,26 +1,8 @@
 require 'rubyXL'
 module WorkerTools
   module XlsxOutput
-    # if defined, this file will be written to this destination (regardless
-    # of whether the model saves the file as well)
-    def xlsx_output_target
-      # Ex: Rails.root.join('shared', 'foo', 'bar.xlsx')
-      raise "xlsx_output_target has to be defined in #{self}"
-    end
-
-    def xlsx_output_content
-      {
-        sheet1: {
-          label: 'Sheet 1',
-          headers: xlsx_output_column_headers,
-          rows: xlsx_output_values,
-          column_style: xlsx_output_column_format
-        }
-      }
-    end
-
-    def xlsx_output_values
-      raise "xlsx_output_values has to be defined in #{self}"
+    def xlsx_output_entries
+      raise "xlsx_output_entries has to be defined in #{self}"
     end
 
     def xlsx_output_column_headers
@@ -37,6 +19,21 @@ module WorkerTools
       raise "xlsx_output_column_headers has to be defined in #{self}"
     end
 
+    def xlsx_output_content
+      {
+        sheet1: {
+          label: 'Sheet 1',
+          headers: xlsx_output_column_headers,
+          rows: xlsx_output_entries.lazy.map { |entry| xlsx_output_row_values(entry) },
+          column_style: xlsx_output_column_format
+        }
+      }
+    end
+
+    def xlsx_output_row_values(entry)
+      entry.values_at(*xlsx_output_column_headers.keys)
+    end
+
     def xlsx_output_column_format
       # These columns are used to set the headers, also
       # to set the row values depending on your implementation.
@@ -51,16 +48,9 @@ module WorkerTools
       {}
     end
 
-    def xlsx_output_target_folder
-      @xlsx_output_target_folder ||= File.dirname(xlsx_output_target)
-    end
-
-    def xlsx_ensure_output_target_folder
-      FileUtils.mkdir_p(xlsx_output_target_folder) unless File.directory?(xlsx_output_target_folder)
-    end
-
-    def xlsx_insert_headers(spreadsheet, headers)
+    def xlsx_output_insert_headers(spreadsheet, headers)
       return unless headers
+
       iterator =
         if headers.is_a? Hash
           headers.values
@@ -72,53 +62,71 @@ module WorkerTools
       end
     end
 
-    def xlsx_insert_rows(spreadsheet, rows, headers)
+    def xlsx_output_insert_rows(spreadsheet, rows, headers)
       rows.each_with_index do |row, row_index|
-        xlsx_iterators(row, headers).each_with_index do |value, col_index|
+        xlsx_output_iterators(row, headers).each_with_index do |value, col_index|
           spreadsheet.add_cell(row_index + 1, col_index, value.to_s)
         end
       end
     end
 
-    def xlsx_iterators(iterable, compare_hash = nil)
+    def xlsx_output_iterators(iterable, compare_hash = nil)
       if iterable.is_a? Hash
         raise 'parameter compare_hash should be a hash, too.' if compare_hash.nil? || !compare_hash.is_a?(Hash)
+
         iterable.values_at(*compare_hash.keys)
       else
         iterable
       end
     end
 
-    def xlsx_style_columns(spreadsheet, styles, headers)
+    def xlsx_output_style_columns(spreadsheet, styles, headers)
       return false unless headers
 
-      xlsx_iterators(styles, headers).each_with_index do |format, index|
+      xlsx_output_iterators(styles, headers).each_with_index do |format, index|
         next unless format
+
         spreadsheet.change_column_width(index, format[:width])
         spreadsheet.change_text_wrap(index, format[:text_wrap])
       end
       true
     end
 
-    def xlsx_write_sheet(workbook, sheet_content, index)
+    def xlsx_output_tmp_file
+      @xlsx_output_tmp_file ||= Tempfile.new(['output', '.xlsx'])
+    end
+
+    def xlsx_output_file_name
+      "#{model_kind}.xlsx"
+    end
+
+    def xlsx_output_add_attachment
+      model.add_attachment(
+        xlsx_output_tmp_file,
+        file_name: xlsx_output_file_name,
+        content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+      )
+    end
+
+    def xlsx_output_write_sheet(workbook, sheet_content, index)
       sheet = workbook.worksheets[index]
       sheet = workbook.add_worksheet(sheet_content[:label]) if sheet.nil?
 
       sheet.sheet_name = sheet_content[:label]
-      xlsx_style_columns(sheet, sheet_content[:column_style], sheet_content[:headers])
-      xlsx_insert_headers(sheet, sheet_content[:headers])
-      xlsx_insert_rows(sheet, sheet_content[:rows], sheet_content[:headers])
+      xlsx_output_style_columns(sheet, sheet_content[:column_style], sheet_content[:headers])
+      xlsx_output_insert_headers(sheet, sheet_content[:headers])
+      xlsx_output_insert_rows(sheet, sheet_content[:rows], sheet_content[:headers])
     end
 
-    def xlsx_write_output_target
-      xlsx_ensure_output_target_folder
-
+    def xlsx_output_write_file
       book = RubyXL::Workbook.new
       xlsx_output_content.each_with_index do |(_, object), index|
-        xlsx_write_sheet(book, object, index)
+        xlsx_output_write_sheet(book, object, index)
       end
 
-      book.write xlsx_output_target
+      book.write xlsx_output_tmp_file
+
+      xlsx_output_add_attachment
     end
   end
 end

data/lib/worker_tools.rb

@@ -1,4 +1,4 @@
-Dir[File.join(__dir__, 'worker_tools/*.rb')].each { |path| require path }
+Dir[File.join(__dir__, 'worker_tools/**/*.rb')].sort.each { |path| require path }
 
 module WorkerTools
 end

data/worker_tools.gemspec

@@ -1,4 +1,4 @@
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'worker_tools/version'
 
@@ -16,6 +16,7 @@ Gem::Specification.new do |spec|
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the "allowed_push_host"
   # to allow pushing to a single host or delete this section to allow pushing to any host.
   raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.' unless spec.respond_to?(:metadata)
+
   spec.metadata['allowed_push_host'] = 'https://rubygems.org'
 
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
@@ -26,7 +27,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport'
-  spec.add_dependency 'rocketchat-notifier', '>= 0.1.2'
   spec.add_dependency 'roo'
   spec.add_dependency 'rubyXL'
   spec.add_dependency 'slack-notifier'
@@ -39,7 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'mocha'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rubocop', '0.52.1'
+  spec.add_development_dependency 'rubocop', '0.71.0'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'sqlite3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worker_tools
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 1.0.0
 platform: ruby
 authors:
 - fsainz
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-03-18 00:00:00.000000000 Z
+date: 2022-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rocketchat-notifier
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.1.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: roo
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +184,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.52.1
+        version: 0.71.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.52.1
+        version: 0.71.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -234,7 +220,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - fernando.sainz@i22.de
 executables: []
@@ -244,6 +230,7 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
@@ -252,11 +239,15 @@ files:
 - bin/setup
 - lib/worker_tools.rb
 - lib/worker_tools/basics.rb
+- lib/worker_tools/benchmark.rb
+- lib/worker_tools/counters.rb
 - lib/worker_tools/csv_input.rb
 - lib/worker_tools/csv_output.rb
+- lib/worker_tools/errors.rb
 - lib/worker_tools/recorder.rb
-- lib/worker_tools/rocketchat_error_notifier.rb
 - lib/worker_tools/slack_error_notifier.rb
+- lib/worker_tools/utils/hash_with_indifferent_access_type.rb
+- lib/worker_tools/utils/serialized_array_type.rb
 - lib/worker_tools/version.rb
 - lib/worker_tools/xlsx_input.rb
 - lib/worker_tools/xlsx_output.rb
@@ -266,7 +257,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -281,9 +272,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: A collection of modules to help writing common worker tasks)
 test_files: []