worker_tools 0.2.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a53ae86156768dd1b1f80601a50f8c6926918d3f625be933173f7d4513c0f2d8
-  data.tar.gz: 48dfc8c5430311e2b6dae3d5bfbb55d6d2ce0444d3c5e5dd5a9cf044e42afdf7
+  metadata.gz: 2d73219c29b55493f0be76ce047dbe7f72259f4efaab15d1528230bdfc1920b5
+  data.tar.gz: a8ef47af79183aa6e6b465a0cbc6c7396ccccf3cb92574c386b956ebd3415056
 SHA512:
-  metadata.gz: 95ac0fe0d5255b995a5c7a0193edf6a56eb5d376234e1f6088f6c97ad03ea6e760d499c62c2c711f5ad2b453f5c9f7b5309f9bea2acfb50b9c60621b83ed3800
-  data.tar.gz: e2a09d6b64d813a0296f81d5ef995ed49178957bfdbf67349e5b2d811ec84e98b08850e9565f875f02a6b105f9ae9769cfd177d660b2a4d98bf0b2ae03ecf77c
+  metadata.gz: 30287518cdce7bb9536d802fffdaa54b893bfcc811b03f1c226ea59b45787c0f5b4090b82796984e5c77741679bd035620cd3e38aecb4ebb0f1dcac9de50cb16
+  data.tar.gz: 18c1c3ff22097f710cea838c6cf4a8058c35343d7f76312e063771ef2dc02c6ad0fbdace866c1b3dc9e21ab3545ea91f041ff16f4d540402be601c44c834d0dc

data/CHANGELOG.md

@@ -0,0 +1,48 @@
+# Changelog
+
+## [1.0.0] - 2022-05-20
+
+Compared to 0.2.1
+
+### New
+
+- Namespaced errors and non failure logic
+- Support for status running and complete_with_warnings
+- Benchmark wrapper
+- Counters wrapper
+- Notes instead of information field
+- Filter for slack errors (`slack_error_notifiable`)
+- Model attachments convention
+- Complete specification of csv open arguments
+
+### BREAKING CHANGES
+
+Instead of writing the final csv or xlsx to a folder, the gem assumes that the model provides an add_attachment method.
+
+Both csv and xlsx output modules use entry hashes for content (`csv_output_entries`, `xlsx_output_entries`). The mapper methods `csv_output_row_values` and `xlsx_output_row_values` do not need (in most cases) to be defined, there is a default now. See the complete examples in the README.
+
+These methods were renamed
+
+- `xlsx_output_values` => `xlsx_output_row_values`
+- `xlsx_insert_headers` => `xlsx_output_insert_headers`
+- `xlsx_insert_rows` => ` xlsx_output_insert_rows`
+- `xlsx_iterators` => `xlsx_output_iterators`
+- `xlsx_style_columns` => `xlsx_output_style_columns`
+- `xlsx_write_sheet` => `xlsx_output_write_sheet`
+
+These methods were removed
+
+- `add_info` in favor of `add_note`
+- `create_model_if_not_available`, a model is always created.
+- `format_log_message`, `format_info_message` in favor of `format_message`
+- `csv_output_target`
+- `cvs_output_target_folder`
+- `csv_output_target_file_name`
+- `csv_ouput_ensure_target_folder`
+- `csv_output_write_target`
+- `xlsx_output_target`
+- `xlsx_output_target_folder`
+- `xlsx_ensure_output_target_folder`
+- `xlsx_write_output_target`
+
+[1.0.0]: https://github.com/i22-digitalagentur/worker-tools/compare/0.2.1...1.0.0

data/README.md

@@ -1,13 +1,53 @@
 # WorkerTools
 
-[![Build Status](https://travis-ci.org/i22-digitalagentur/worker-tools.svg?branch=master)](https://travis-ci.org/i22-digitalagentur/worker-tools)
+[![Build Status][build-badge]][build-url]
+[![MIT License][license-shield]][license-url]
+[![Release][release-shield]][release-url]
+![Maintenance][maintained-shield]
+
+<br>
+
+<details open="open">
+  <summary>Table of Contents</summary>
+  <ol>
+    <li>
+      <a href="#about-the-project">About The Project</a>
+    </li>
+    <li>
+      <a href="#installation">Installation</a>
+    </li>
+    <li><a href="#conventions">Conventions</a></li>
+    <li><a href="#module-basics">Module 'Basics'</a></li>
+    <li><a href="#module-recorder">Module 'Recorder'</a></li>
+    <li><a href="#module-slackerrornotifier">Module 'SlackErrorNotifier'</a></li>
+    <li><a href="#wrappers">Wrappers</a></li>
+    <li><a href="#module-notes">Module 'Notes'</a></li>
+    <li><a href="#attachments">Attachments</a></li>
+    <li>
+      <a href="#complete-examples">Complete Examples</a>
+      <ul>
+        <li><a href="#xlsx-input-example">XLSX Input Example</a></li>
+        <li><a href="#csv-input-example">CSV Input Example</a></li>
+        <li><a href="#csv-output-example">CSV Output Example</a></li>
+        <li><a href="#xlsx-output-example">XLSX Output Example</a></li>
+      </ul>
+    </li>
+    <li><a href="#changelog">Changelog</a></li>
+    <li><a href="#requirements">Requirements</a></li>
+    <li><a href="#contributing">Contributing</a></li>
+    <li><a href="#license">License</a></li>
+    <li><a href="#acknowledgement">Acknowledgement</a></li>
+  </ol>
+</details>
+
+## About The Project
 
 WorkerTools is a collection of modules meant to speed up how we write background tasks following a few basic patterns. The structure of plain independent modules with limited abstraction allows to define and override a few methods according to your needs without requiring a deep investment in the library.
 
 These modules provide some features and conventions to address the following points with little configuration on your part:
 
 - How to save the state the task.
-- How to save information relevant to the admins / customers.
+- How to save notes relevant to the admins / customers.
 - How to log the details
 - How to handle exceptions and send notifications
 - How to process CSV files (as input and output)
@@ -32,13 +72,20 @@ Or install it yourself as:
 
 ## Conventions
 
-Most of the modules require an ActiveRecord model to keep track of the state, information, and files related to the job. The class of this model is typically an Import, Export, Report.. or something more generic like a JobEntry.
+Most of the modules require an ActiveRecord model to keep track of the state, notes, and files related to the job. The class of this model is typically an Import, Export, Report.. or something more generic like a JobEntry.
 
 An example of this model for an Import using Paperclip would be something like this:
 
 ```ruby
 class Import < ApplicationRecord
-  enum state: { waiting: 0, complete: 1, failed: 2 }
+  enum state: %w[
+    waiting
+    complete
+    complete_with_warnings
+    failed
+    running
+  ].map { |e| [e, e] }.to_h
+
   enum kind: { foo: 0, bar: 1 }
 
   has_attached_file :attachment
@@ -48,7 +95,9 @@ class Import < ApplicationRecord
   end
 ```
 
-The state `complete` and `failed` are used by the modules. Both `state` and `kind` could be an enum or just a string field.  Whether you have one, none or many attachments, and which library you use to handle it's up to you.
+The state `complete` and `failed` are used by the modules. Both `state` and `kind` could be an enum or just a string field. Whether you have one, none or many attachments, and which library you use to handle it's up to you.
+
+The state `complete_with_warnings` indicates that the model contains notes that did not lead to a failure but should get some attention. By default those levels are `warning` and `errors` and can be customized.
 
 In this case the migration would be something like this:
 
@@ -56,8 +105,8 @@ In this case the migration would be something like this:
   def change
     create_table :imports do |t|
       t.integer :kind, null: false
-      t.integer :state, default: 0, null: false
-      t.text :information
+      t.string :state, default: 'waiting', null: false
+      t.json :notes, default: []
       t.json  :options, default: {}
       t.json :meta, default: {}
 
@@ -67,7 +116,7 @@ In this case the migration would be something like this:
 
       t.timestamps
     end
- ```
+```
 
 ## Module 'Basics'
 
@@ -96,12 +145,13 @@ end
 
 The basics module contains a `perform` method, which is the usual entry point for ApplicationJob and Sidekiq. It can receive the id of the model, the model instance, or nothing, in which case it will attempt to create this model on its own.
 
+By default errors subclassed from WorkerTools::Errors::Invalid (such as those related to wrong headers in the input modules) will not raise and mark the model as failed. The method `non_failure_error?` lets you modifiy this behaviour.
 
 ## Module 'Recorder'
 
-Provides some methods to manage a log and the `information` field of the model. The main methods are `add_info`, `add_log`, and `record` (which both logs and appends the message to the information field). See all methods in [recorder](/lib/worker_tools/recorder.rb)
+Provides some methods to manage a log and the `notes` field of the model. The main methods are `add_info`, `add_log`, and `record` (which both logs and appends the message to the notes field). See all methods in [recorder](/lib/worker_tools/recorder.rb)
 
-This module has a _recoder_ wrapper that will register the exception details into the log and information field in case of error:
+This module has a _recoder_ wrapper that will register the exception details into the log and notes field in case of error:
 
 ```ruby
 class MyImporter
@@ -127,24 +177,43 @@ If you only want the logger functions, without worrying about persisting a model
 
 ## Module SlackErrorNotifier
 
-[slack_error_notifier](/lib/worker_tools/slack_error_notifier.rb)
+Provides a Slack error notifier wrapper. To do this, you need to define SLACK_NOTIFIER_WEBHOOK as well as SLACK_NOTIFIER_CHANNEL. Then you need to include the SlackErrorNotifier module in your class and append slack_error_notifier to your wrappers. Below you can see an example.
+
+```ruby
+  class MyImporter
+    include WorkerTools::SlackErrorNotifier
+
+    wrappers :slack_error_notifier
+
+    def perform
+      with_wrapper_logger do
+        # do stuff
+      end
+    end
+  end
+```
+
+See all methods in [slack_error_notifier](/lib/worker_tools/slack_error_notifier.rb)
 
 ## Module CSV Input
 
-[csv_input](/lib/worker_tools/csv_input.rb)
+See all methods in [csv_input](/lib/worker_tools/csv_input.rb)
 
 ## Module CSV Output
 
-[csv_output](/lib/worker_tools/csv_output.rb)
+See all methods in [csv_output](/lib/worker_tools/csv_output.rb)
 
 ## Module XLSX Input
 
-[xlsx_input](/lib/worker_tools/xlsx_input.rb)
+See all methods in [xlsx_input](/lib/worker_tools/xlsx_input.rb)
 
+## Module XLSX Output
+
+See all methods in [xlsx_output](/lib/worker_tools/xlsx_output.rb)
 
 ## Wrappers
 
-In the [basics module](/lib/worker_tools/basics.rb), `perform` calls your custom method `run` to do the actual work of the task, and wraps it around any methods expecting a block that you might have had defined using `wrappers`.  That gives us a systematic way to add logic depending on the output of `run` and any exceptions that might arise, such as logging the error and context, sending a chat notification, retrying under some circumstances, etc.
+In the [basics module](/lib/worker_tools/basics.rb), `perform` calls your custom method `run` to do the actual work of the task, and wraps it around any methods expecting a block that you might have had defined using `wrappers`. That gives us a systematic way to add logic depending on the output of `run` and any exceptions that might arise, such as logging the error and context, sending a chat notification, retrying under some circumstances, etc.
 
 The following code
 
@@ -200,13 +269,14 @@ def perform(model_id)
 end
 ```
 
-## Counters
+## Counter
 
 There is a counter wrapper that you can use to add custom counters to the meta attribute. To do this, you need to complete the following tasks:
+
 - include WorkerTools::Counters to your class
 - add :counters to the wrappers method props
 - call counters method with your custom counters
-You can see an example below. After that, you can access your custom counters via the meta attribute.
+  You can see an example below. After that, you can access your custom counters via the meta attribute.
 
 ```ruby
 class MyImporter
@@ -219,13 +289,15 @@ class MyImporter
   end
 
   def example_foo_counter_methods
-    self.foo = 0
-
+    # you can use the increment helper
     10.times { increment_foo }
 
-    puts foo # foo == 10
+    # the counter works like a regular accessor, you can read it and modify it
+    # directly
+    self.bar = 100
+    puts bar # => 100
   end
-  
+
   # ..
 end
 ```
@@ -236,7 +308,7 @@ There is a benchmark wrapper that you can use to record the benchmark. The only
 
 ```ruby
 class MyImporter
-  include WorkerTools::CustomBenchmark
+  include WorkerTools::Benchmark
   wrappers :benchmark
 
   def run
@@ -247,11 +319,235 @@ class MyImporter
 end
 ```
 
+## Module 'Notes'
+
+If you use ActiveRecord you may need to modify the serializer as well as deserializer from the note attribute. After that you can easily serialize hashes and array of hashes with indifferent access. For that purpose the gem provides two utility methods. (HashWithIndifferentAccessType, SerializedArrayType). There is an example of how you can use it.
+
+```ruby
+  class ServiceTask < ApplicationRecord
+
+    attribute :notes, SerializedArrayType.new(type: HashWithIndifferentAccessType.new)
+  end
+```
+
+See all methods in [utils](/lib/worker_tools/utils)
+
+## Attachments
+
+The modules that generate a file expect the model to provide an `add_attachment` method with following signature:
+
+```ruby
+  def add_attachment(file, file_name: nil, content_type: nil)
+    # your logic
+  end
+```
+
+You can skip this convention by overwriting the module related method, for example after including `CsvOutput`
+
+```ruby
+def csv_output_add_attachment
+  # default implementation
+  # model.add_attachment(csv_output_tmp_file, file_name: csv_output_file_name, content_type: 'text/csv')
+
+  # your method
+  ftp_upload(csv_output_tmp_file)
+end
+```
+
+## Complete Examples
+
+### XLSX Input Example
+
+```ruby
+class XlsxInputExample
+  include Sidekiq::Worker
+  include WorkerTools::Basics
+  include WorkerTools::Recorder
+  include WorkerTools::XlsxInput
+
+  wrappers %i[basics recorder]
+
+  def model_class
+    Import
+  end
+
+  def model_kind
+    'xlsx_input_example'
+  end
+
+  def run
+    xlsx_input_foreach.each { |row| SomeModel.create!(row) }
+  end
+
+  def xlsx_input_columns
+    {
+      foo: 'Your Foo',
+      bar: 'Your Bar'
+    }
+  end
+end
+```
+
+### CSV Input Example
+
+```ruby
+class CsvInputExample
+  include Sidekiq::Worker
+  include WorkerTools::Basics
+  include WorkerTools::Recorder
+  include WorkerTools::CsvInput
+
+  wrappers %i[basics recorder]
+
+  def model_class
+    Import
+  end
+
+  def model_kind
+    'csv_input_example'
+  end
+
+  def csv_input_columns
+    {
+      flavour: 'Flavour',
+      number: 'Number'
+    }
+  end
+
+  def run
+    csv_input_foreach.map { |row| do_something row_to_attributes(row) }
+  end
+
+  def row_to_attributes(row)
+    {
+      flavour: row['flavour'].downcase,
+      number: row['number'].to_i * 10
+    }
+  end
+end
+```
+
+### CSV Output Example
+
+```ruby
+# More complex example with CsvOutput
+class CsvOutputExample
+  include Sidekiq::Worker
+  include WorkerTools::Basics
+  include WorkerTools::CsvOutput
+  include WorkerTools::Recorder
+
+  wrappers %i[basics recorder]
+
+  def model_class
+    Report
+  end
+
+  def model_kind
+    'csv_out_example'
+  end
+
+  def model_file_name
+    "#{model_kind}-#{Date.current}.csv"
+  end
+
+  def run
+    csv_output_write_file
+  end
+
+  def csv_output_column_headers
+    @csv_output_column_headers ||= {
+      foo: 'Foo',
+      bar: 'Bar'
+    }
+  end
+
+  def csv_output_entries
+    @csv_output_entries ||= User.includes(...).lazy.map do |user|
+      {
+        foo: user.foo,
+        bar: user.bar
+      }
+    end
+  end
+
+end
+```
+
+### XLSX Output Example
+
+```ruby
+# ExampleXlsxOutput
+class XlsxOutputExample
+  include Sidekiq::Worker
+  include WorkerTools::Basics
+  include WorkerTools::Recorder
+  include WorkerTools::XlsxOutput
+
+  wrappers %i[basics recorder]
+
+  def model_class
+    Export
+  end
+
+  def model_kind
+    'xlsx_output_example'
+  end
+
+  def run
+    xlsx_output_write_file
+  end
+
+  def xlsx_output_column_headers
+    @xlsx_output_column_headers ||= {
+      foo: 'Foo',
+      bar: 'Bar'
+    }
+  end
+
+  def xlsx_output_entries
+    @xlsx_output_entries ||= SomeArray.map do |entry|
+      {
+        foo: user.foo,
+        bar: user.bar
+      }
+    end
+  end
+end
+```
+
+## Changelog
+
+See [CHANGELOG](CHANGELOG.md)
+
+## Requirements
+
+- ruby > 2.3.1
+
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/i22-digitalagentur/worker_tools.
+Contributions are what make the open source community such an amazing place to be learn, inspire, and create. Any contributions you make are **greatly appreciated**.
 
+1. Fork the Project
+2. Create your Feature Branch (`git checkout -b feature/new_feature`)
+3. Commit your Changes (`git commit -m 'feat: Add new feature'`)
+4. Push to the Branch (`git push origin feature/new_feature`)
+5. Open a Pull Request
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+The gem is available under the MIT License. See `LICENSE` for more information.
+
+## Acknowledgement
+
+- [Img Shields](https://shields.io)
+
+<!--shield-styles-->
+
+[build-badge]: https://travis-ci.org/i22-digitalagentur/worker-tools.svg?branch=master
+[build-url]: https://travis-ci.org/i22-digitalagentur/worker-tools
+[maintained-shield]: https://img.shields.io/badge/Maintained%3F-yes-green.svg?style=flat
+[release-shield]: https://img.shields.io/github/release/i22-digitalagentur/coverage-badge-creator.svg?style=flat
+[release-url]: https://github.com/i22-digitalagentur/worker-tools/releases/
+[license-shield]: https://img.shields.io/badge/License-MIT-yellow.svg?style=flat
+[license-url]: https://github.com/i22-digitalagentur/worker-tools/blob/master/LICENSE

data/lib/worker_tools/basics.rb

@@ -6,7 +6,6 @@ module WorkerTools
 
     included do
       attr_writer :model
-      attr_accessor :information
 
       def self.wrappers(*args)
         @wrappers ||= args.flatten
@@ -49,27 +48,30 @@ module WorkerTools
     end
 
     def with_wrapper_basics(&block)
+      save_state_without_validate('running')
       block.yield
       finalize
     # this time we do want to catch Exception to attempt to handle some of the
     # critical errors.
     # rubocop:disable Lint/RescueException
-    rescue Exception
+    rescue Exception => e
+      return finalize if non_failure_error?(e)
+
       # rubocop:enable Lint/RescueException
-      model.state = 'failed'
-      model.save!(validate: false)
+      save_state_without_validate('failed')
       raise
     end
 
     def finalize
-      model.update!(
-        state: 'complete',
-        information: information
-      )
+      mark_with_warnings = model.notes.any? do |note|
+        complete_with_warnings_note_levels.include?(note.with_indifferent_access[:level].to_s)
+      end
+
+      model.update!(state: mark_with_warnings ? :complete_with_warnings : :complete)
     end
 
-    def create_model_if_not_available
-      false
+    def complete_with_warnings_note_levels
+      %w[error warning]
     end
 
     def model
@@ -83,13 +85,23 @@ module WorkerTools
       send(current_wrapper_symbol) { with_wrappers(wrapper_symbols, &block) }
     end
 
+    def non_failure_error?(error)
+      error.is_a?(WorkerTools::Errors::Invalid)
+      # or add your list
+      # [WorkerTools::Errors::Invalid, SomeOtherError].any? { |k| e.is_a?(k) }
+    end
+
     private
 
+    def save_state_without_validate(state)
+      model.state = state
+      model.save!(validate: false)
+    end
+
     def find_model
       @model_id ||= nil
       return @model_id if @model_id.is_a?(model_class)
       return model_class.find(@model_id) if @model_id
-      raise 'Model not available' unless create_model_if_not_available
 
       t = model_class.new
       t.kind = model_kind if t.respond_to?(:kind=)

data/lib/worker_tools/benchmark.rb

@@ -1,12 +1,12 @@
 module WorkerTools
-  module CustomBenchmark
+  module Benchmark
     extend ActiveSupport::Concern
 
     included do
       attr_accessor :benchmark
 
       def with_wrapper_benchmark(&block)
-        @benchmark = Benchmark.measure(&block)
+        @benchmark = ::Benchmark.measure(&block)
 
         model.meta['duration'] = @benchmark.real.round if model.respond_to?(:meta)
       end

data/lib/worker_tools/counters.rb

@@ -14,15 +14,14 @@ module WorkerTools
 
       def self.add_counter_methods
         @counters.each do |name|
-          define_method name do
-            model.meta[name]
-          end
-          define_method "#{name}=" do |value|
-            model.meta[name] = value
-          end
-          define_method "increment_#{name}" do
-            model.meta[name] += 1
-          end
+          # ex `inserts`
+          define_method(name) { model.meta[name] }
+
+          # ex `inserts=`
+          define_method("#{name}=") { |value| model.meta[name] = value }
+
+          # ex `increment_inserts`
+          define_method("increment_#{name}") { model.meta[name] += 1 }
         end
       end
 

data/lib/worker_tools/csv_input.rb

@@ -63,7 +63,8 @@ module WorkerTools
       actual_columns_length = csv_rows_enum.first.length
       return if expected_columns_length == actual_columns_length
 
-      raise "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+      msg = "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+      raise Errors::WrongNumberOfColumns, msg
     end
 
     def csv_input_columns_hash_check(csv_rows_enum)
@@ -75,7 +76,9 @@ module WorkerTools
 
     def csv_input_columns_hash_check_duplicates(names)
       dups = names.group_by(&:itself).select { |_, v| v.count > 1 }.keys
-      raise "The file contains duplicated columns: #{dups}" if dups.present?
+      return unless dups.present?
+
+      raise Errors::DuplicatedColumns, "The file contains duplicated columns: #{dups}"
     end
 
     def csv_input_columns_hash_check_missing(actual_names, expected_names)
@@ -83,7 +86,7 @@ module WorkerTools
         matchable = name.is_a?(String) ? csv_input_header_normalized(name) : name
         actual_names.any? { |n| case n when matchable then true end } # rubocop does not like ===
       end
-      raise "Some columns are missing: #{missing}" unless missing.empty?
+      raise Errors::MissingColumns, "Some columns are missing: #{missing}" unless missing.empty?
     end
 
     def csv_input_csv_options

data/lib/worker_tools/csv_output.rb

@@ -2,13 +2,6 @@ require 'csv'
 
 module WorkerTools
   module CsvOutput
-    # if defined, this file will be written to this destination (regardless
-    # of whether the model saves the file as well)
-    def csv_output_target
-      # Ex: Rails.root.join('shared', 'foo', 'bar.csv')
-      false
-    end
-
     def csv_output_entries
       raise "csv_output_entries has to be defined in #{self}"
     end
@@ -27,33 +20,18 @@ module WorkerTools
       raise "csv_output_column_headers has to be defined in #{self}"
     end
 
-    # rubocop:disable Lint/UnusedMethodArgument
     def csv_output_row_values(entry)
-      # Ex:
-      # {
-      #   foo: entry.foo,
-      #   bar: entry.bar
-      # }.values_at(*csv_output_column_headers.keys)
-      raise "csv_output_row_values has to be defined in #{self}"
-    end
-    # rubocop:enable Lint/UnusedMethodArgument
-
-    def cvs_output_target_folder
-      File.dirname(csv_output_target)
-    end
-
-    def csv_output_target_file_name
-      File.basename(csv_output_target)
-    end
-
-    def csv_ouput_ensure_target_folder
-      FileUtils.mkdir_p(cvs_output_target_folder) unless File.directory?(cvs_output_target_folder)
+      entry.values_at(*csv_output_column_headers.keys)
     end
 
     def csv_output_tmp_file
       @csv_output_tmp_file ||= Tempfile.new(['output', '.csv'])
     end
 
+    def csv_output_file_name
+      "#{model_kind}.csv"
+    end
+
     def csv_output_col_sep
       ';'
     end
@@ -62,21 +40,29 @@ module WorkerTools
       Encoding::UTF_8
     end
 
+    def csv_output_write_mode
+      'wb'
+    end
+
+    def csv_output_csv_options
+      { col_sep: csv_output_col_sep, encoding: csv_output_encoding }
+    end
+
     def csv_output_insert_headers(csv)
       csv << csv_output_column_headers.values if csv_output_column_headers
     end
 
+    def csv_output_add_attachment
+      model.add_attachment(csv_output_tmp_file, file_name: csv_output_file_name, content_type: 'text/csv')
+    end
+
     def csv_output_write_file
-      CSV.open(csv_output_tmp_file, 'wb', col_sep: csv_output_col_sep, encoding: csv_output_encoding) do |csv|
+      CSV.open(csv_output_tmp_file, csv_output_write_mode, **csv_output_csv_options) do |csv|
         csv_output_insert_headers(csv)
         csv_output_entries.each { |entry| csv << csv_output_row_values(entry) }
       end
-      csv_output_write_target if csv_output_target
-    end
 
-    def csv_output_write_target
-      csv_ouput_ensure_target_folder
-      FileUtils.cp(csv_output_tmp_file.path, csv_output_target)
+      csv_output_add_attachment
     end
   end
 end

data/lib/worker_tools/errors.rb

@@ -0,0 +1,10 @@
+module WorkerTools
+  Error = Class.new(StandardError)
+
+  module Errors
+    Invalid = Class.new(Error)
+    WrongNumberOfColumns = Class.new(Invalid)
+    DuplicatedColumns = Class.new(Invalid)
+    MissingColumns = Class.new(Invalid)
+  end
+end

data/lib/worker_tools/recorder.rb

@@ -1,6 +1,5 @@
 module WorkerTools
   module Recorder
-
     def with_wrapper_recorder(&block)
       block.yield
     # this time we do want to catch Exception to attempt to handle some of the
@@ -24,38 +23,44 @@ module WorkerTools
     end
 
     def record_fail(error)
-      record "ID #{model.id} - Error"
       record(error, :error)
-      model.information = information
       model.save!(validate: false)
     end
 
-    def add_log(message, level = :info)
-      logger.public_send(level, format_log_message(message))
+    def add_log(message, level = nil)
+      attrs = default_message_attrs(message, level)
+      logger.public_send(attrs[:level], format_message(attrs[:message]))
     end
 
-    def add_info(message)
-      @information ||= ''
-      information << "#{format_info_message(message)}\n"
+    def add_note(message, level = nil)
+      attrs = default_message_attrs(message, level)
+      model.notes.push(level: attrs[:level], message: attrs[:message])
     end
 
     def record(message, level = :info)
       add_log(message, level)
-      add_info(message)
+      add_note(message, level)
     end
 
-    def format_log_message(message)
-      return error_to_text(message, log_error_trace_lines) if message.is_a?(Exception)
+    def level_from_message_type(message)
+      return :error if message.is_a?(Exception)
 
-      message
+      :info
     end
 
-    def format_info_message(message)
-      return error_to_text(message, info_error_trace_lines) if message.is_a?(Exception)
+    def format_message(message)
+      return error_to_text(message, log_error_trace_lines) if message.is_a?(Exception)
 
       message
     end
 
+    def default_message_attrs(message, level)
+      {
+        message: format_message(message),
+        level: level || level_from_message_type(message)
+      }
+    end
+
     def logger
       @logger ||= Logger.new(File.join(log_directory, log_file_name))
     end

data/lib/worker_tools/slack_error_notifier.rb

@@ -5,7 +5,7 @@ module WorkerTools
     def with_wrapper_slack_error_notifier(&block)
       block.yield
     rescue StandardError => e
-      slack_error_notify(e) if slack_error_notifier_enabled
+      slack_error_notify(e) if slack_error_notifier_enabled && slack_error_notifiable?(e)
       raise
     end
 
@@ -13,6 +13,10 @@ module WorkerTools
       Rails.env.production?
     end
 
+    def slack_error_notifiable?(error)
+      error.is_a?(StandardError)
+    end
+
     def slack_error_notifier_emoji
       ':red_circle:'
     end

data/lib/worker_tools/utils/hash_with_indifferent_access_type.rb

@@ -0,0 +1,9 @@
+module WorkerTools
+  module Utils
+    class HashWithIndifferentAccessType < ActiveRecord::Type::Json
+      def deserialize(value)
+        HashWithIndifferentAccess.new(super)
+      end
+    end
+  end
+end

data/lib/worker_tools/utils/serialized_array_type.rb

@@ -0,0 +1,19 @@
+module WorkerTools
+  module Utils
+    class SerializedArrayType < ActiveRecord::Type::Json
+      def initialize(type: nil)
+        @type = type
+      end
+
+      def deserialize(value)
+        super(value)&.map { |d| @type.deserialize(d) }
+      end
+
+      def serialize(value)
+        raise 'not an array' unless value.is_a?(Array)
+
+        super value
+      end
+    end
+  end
+end

data/lib/worker_tools/version.rb

@@ -1,3 +1,3 @@
 module WorkerTools
-  VERSION = '0.2.2'.freeze
+  VERSION = '1.0.0'.freeze
 end

data/lib/worker_tools/xlsx_input.rb

@@ -47,7 +47,9 @@ module WorkerTools
     end
 
     def xlsx_input_header_normalized(name)
-      name.to_s.strip.downcase
+      # some elements return obj.to_s => nil
+      # for example [#<Roo::Excelx::Cell::Empty:0x0000000af8d4c8 ... @value=nil>]
+      name&.to_s&.strip&.downcase || ''
     end
 
     # Allows for some basic cleanup of the values, such as applying strip to
@@ -68,7 +70,8 @@ module WorkerTools
       actual_columns_length = xlsx_rows_enum.first.length
       return if expected_columns_length == actual_columns_length
 
-      raise "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+      msg = "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+      raise Errors::WrongNumberOfColumns, msg
     end
 
     def xlsx_input_columns_hash_check(xlsx_rows_enum)
@@ -80,7 +83,9 @@ module WorkerTools
 
     def xlsx_input_columns_hash_check_duplicates(names)
       dups = names.group_by(&:itself).select { |_, v| v.count > 1 }.keys
-      raise "The file contains duplicated columns: #{dups}" if dups.present?
+      return unless dups.present?
+
+      raise Errors::DuplicatedColumns, "The file contains duplicated columns: #{dups}"
     end
 
     def xlsx_input_columns_hash_check_missing(actual_names, expected_names)
@@ -88,7 +93,7 @@ module WorkerTools
         matchable = name.is_a?(String) ? xlsx_input_header_normalized(name) : name
         actual_names.any? { |n| case n when matchable then true end } # rubocop does not like ===
       end
-      raise "Some columns are missing: #{missing}" unless missing.empty?
+      raise Errors::MissingColumns, "Some columns are missing: #{missing}" unless missing.empty?
     end
 
     # Compares the first row (header names) with the xlsx_input_columns hash to find

data/lib/worker_tools/xlsx_output.rb

@@ -1,26 +1,8 @@
 require 'rubyXL'
 module WorkerTools
   module XlsxOutput
-    # if defined, this file will be written to this destination (regardless
-    # of whether the model saves the file as well)
-    def xlsx_output_target
-      # Ex: Rails.root.join('shared', 'foo', 'bar.xlsx')
-      raise "xlsx_output_target has to be defined in #{self}"
-    end
-
-    def xlsx_output_content
-      {
-        sheet1: {
-          label: 'Sheet 1',
-          headers: xlsx_output_column_headers,
-          rows: xlsx_output_values,
-          column_style: xlsx_output_column_format
-        }
-      }
-    end
-
-    def xlsx_output_values
-      raise "xlsx_output_values has to be defined in #{self}"
+    def xlsx_output_entries
+      raise "xlsx_output_entries has to be defined in #{self}"
     end
 
     def xlsx_output_column_headers
@@ -37,6 +19,21 @@ module WorkerTools
       raise "xlsx_output_column_headers has to be defined in #{self}"
     end
 
+    def xlsx_output_content
+      {
+        sheet1: {
+          label: 'Sheet 1',
+          headers: xlsx_output_column_headers,
+          rows: xlsx_output_entries.lazy.map { |entry| xlsx_output_row_values(entry) },
+          column_style: xlsx_output_column_format
+        }
+      }
+    end
+
+    def xlsx_output_row_values(entry)
+      entry.values_at(*xlsx_output_column_headers.keys)
+    end
+
     def xlsx_output_column_format
       # These columns are used to set the headers, also
       # to set the row values depending on your implementation.
@@ -51,15 +48,7 @@ module WorkerTools
       {}
     end
 
-    def xlsx_output_target_folder
-      @xlsx_output_target_folder ||= File.dirname(xlsx_output_target)
-    end
-
-    def xlsx_ensure_output_target_folder
-      FileUtils.mkdir_p(xlsx_output_target_folder) unless File.directory?(xlsx_output_target_folder)
-    end
-
-    def xlsx_insert_headers(spreadsheet, headers)
+    def xlsx_output_insert_headers(spreadsheet, headers)
       return unless headers
 
       iterator =
@@ -73,15 +62,15 @@ module WorkerTools
       end
     end
 
-    def xlsx_insert_rows(spreadsheet, rows, headers)
+    def xlsx_output_insert_rows(spreadsheet, rows, headers)
       rows.each_with_index do |row, row_index|
-        xlsx_iterators(row, headers).each_with_index do |value, col_index|
+        xlsx_output_iterators(row, headers).each_with_index do |value, col_index|
           spreadsheet.add_cell(row_index + 1, col_index, value.to_s)
         end
       end
     end
 
-    def xlsx_iterators(iterable, compare_hash = nil)
+    def xlsx_output_iterators(iterable, compare_hash = nil)
       if iterable.is_a? Hash
         raise 'parameter compare_hash should be a hash, too.' if compare_hash.nil? || !compare_hash.is_a?(Hash)
 
@@ -91,10 +80,10 @@ module WorkerTools
       end
     end
 
-    def xlsx_style_columns(spreadsheet, styles, headers)
+    def xlsx_output_style_columns(spreadsheet, styles, headers)
       return false unless headers
 
-      xlsx_iterators(styles, headers).each_with_index do |format, index|
+      xlsx_output_iterators(styles, headers).each_with_index do |format, index|
         next unless format
 
         spreadsheet.change_column_width(index, format[:width])
@@ -103,25 +92,41 @@ module WorkerTools
       true
     end
 
-    def xlsx_write_sheet(workbook, sheet_content, index)
+    def xlsx_output_tmp_file
+      @xlsx_output_tmp_file ||= Tempfile.new(['output', '.xlsx'])
+    end
+
+    def xlsx_output_file_name
+      "#{model_kind}.xlsx"
+    end
+
+    def xlsx_output_add_attachment
+      model.add_attachment(
+        xlsx_output_tmp_file,
+        file_name: xlsx_output_file_name,
+        content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+      )
+    end
+
+    def xlsx_output_write_sheet(workbook, sheet_content, index)
       sheet = workbook.worksheets[index]
       sheet = workbook.add_worksheet(sheet_content[:label]) if sheet.nil?
 
       sheet.sheet_name = sheet_content[:label]
-      xlsx_style_columns(sheet, sheet_content[:column_style], sheet_content[:headers])
-      xlsx_insert_headers(sheet, sheet_content[:headers])
-      xlsx_insert_rows(sheet, sheet_content[:rows], sheet_content[:headers])
+      xlsx_output_style_columns(sheet, sheet_content[:column_style], sheet_content[:headers])
+      xlsx_output_insert_headers(sheet, sheet_content[:headers])
+      xlsx_output_insert_rows(sheet, sheet_content[:rows], sheet_content[:headers])
     end
 
-    def xlsx_write_output_target
-      xlsx_ensure_output_target_folder
-
+    def xlsx_output_write_file
       book = RubyXL::Workbook.new
       xlsx_output_content.each_with_index do |(_, object), index|
-        xlsx_write_sheet(book, object, index)
+        xlsx_output_write_sheet(book, object, index)
       end
 
-      book.write xlsx_output_target
+      book.write xlsx_output_tmp_file
+
+      xlsx_output_add_attachment
     end
   end
 end

data/lib/worker_tools.rb

@@ -1,4 +1,4 @@
-Dir[File.join(__dir__, 'worker_tools/*.rb')].each { |path| require path }
+Dir[File.join(__dir__, 'worker_tools/**/*.rb')].sort.each { |path| require path }
 
 module WorkerTools
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worker_tools
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 1.0.0
 platform: ruby
 authors:
 - fsainz
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-21 00:00:00.000000000 Z
+date: 2022-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -230,6 +230,7 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
@@ -242,8 +243,11 @@ files:
 - lib/worker_tools/counters.rb
 - lib/worker_tools/csv_input.rb
 - lib/worker_tools/csv_output.rb
+- lib/worker_tools/errors.rb
 - lib/worker_tools/recorder.rb
 - lib/worker_tools/slack_error_notifier.rb
+- lib/worker_tools/utils/hash_with_indifferent_access_type.rb
+- lib/worker_tools/utils/serialized_array_type.rb
 - lib/worker_tools/version.rb
 - lib/worker_tools/xlsx_input.rb
 - lib/worker_tools/xlsx_output.rb
@@ -268,7 +272,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: A collection of modules to help writing common worker tasks)