worker_tools 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7b0794f3ef2c1157bf4254ec78087218ff50dcc4
+  data.tar.gz: 28cb0e939304f7d81becb39e6d223493b059c60e
+SHA512:
+  metadata.gz: 78e98bd9a94c25f990f809b72d60257121467875154c2bda55a1b7ef65e25b7ecaba0faffebd932f22c775a213eb554886d87aab8080aab3458f3aaf8bc71ae8
+  data.tar.gz: 2d910f18319f729d0b05ced99d6fc24886308439e12c01ea2a58a09ee1320e4ee83bd3d2e0f36aa1915fff11fab7d578d5134805de0da7cd3c7b7fefe2ebcd51

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.DS_Store

data/.rubocop.yml

@@ -0,0 +1,42 @@
+# See all options in these files:
+  # https://github.com/bbatsov/rubocop/blob/master/config/default.yml
+  # https://github.com/bbatsov/rubocop/blob/master/config/enabled.yml
+  # https://github.com/bbatsov/rubocop/blob/master/config/disabled.yml
+
+# run bundle exec rubocop -D to see what cop is reporting
+
+# If you want to skip the check for line length in a specific case do:
+# rubocop:disable LineLength
+# superlong-line-with-unavoidable-excesive-length
+# rubocop:enable LineLength
+
+AllCops:
+  TargetRubyVersion: 2.3
+  Exclude:
+    - bin/*
+    - Rakefile
+
+Documentation:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - test/**/*
+    - ./*.gemspec
+
+Metrics/LineLength:
+  Max: 120
+  Exclude:
+    - test/**/*
+
+Metrics/MethodLength:
+  Exclude:
+    - test/**/*
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+# Although casecmp is more performant, downcase is more readable
+# 'Foo'.downcase.eql?('foo') vs 'Foo'.casecmp('foo').zero?
+Performance/Casecmp:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.14.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in worker_tools.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 i22 Digitalagentur GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,211 @@
+# WorkerTools
+
+[![Build Status](https://travis-ci.org/i22-digitalagentur/worker-tools.svg?branch=master)](https://travis-ci.org/i22-digitalagentur/worker-tools)
+
+WorkerTools is a collection of modules meant to speed up how we write background tasks following a few basic patterns. The structure of plain independent modules with limited abstraction allows to define and override a few methods according to your needs without requiring a deep investment in the library.
+
+These modules provide some features and conventions to address the following points with little configuration on your part:
+
+- How to save the state the task.
+- How to save information relevant to the admins / customers.
+- How to log the details
+- How to handle exceptions and send notifications
+- How to process CSV files (as input and output)
+- How to process XLXS files (as input, output coming next)
+- How to set options
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'worker_tools'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install worker_tools
+
+## Conventions
+
+Most of the modules require an ActiveRecord model to keep track of the state, information, and files related to the job. The class of this model is typically an Import, Export, Report.. or something more generic like a JobEntry.
+
+An example of this model for an Import using Paperclip would be something like this:
+
+```ruby
+class Import < ApplicationRecord
+  enum state: { waiting: 0, complete: 1, failed: 2 }
+  enum kind: { foo: 0, bar: 1 }
+
+  has_attached_file :attachment
+
+  validates :kind, presence: true
+  validates :state, presence: true
+  end
+```
+
+The state `complete` and `failed` are used by the modules. Both `state` and `kind` could be an enum or just a string field.  Whether you have one, none or many attachments, and which library you use to handle it's up to you.
+
+In this case the migration would be something like this:
+
+```ruby
+  def change
+    create_table :imports do |t|
+      t.integer :kind, null: false
+      t.integer :state, default: 0, null: false
+      t.text :information
+      t.json  :options, default: {}
+
+      t.string :attachment_file_name
+      t.integer :attachment_file_size
+      t.string :attachment_content_type
+
+      t.timestamps
+    end
+ ```
+
+## Module 'Basics'
+
+the [basics module](/lib/worker_tools/basics.rb) takes care of finding or creating the model, marking it as completed or failed, and calling any flow control wrappers around `run` that had been specified. (See wrappers)
+
+A simple example would be as follows:
+
+```ruby
+class MyImporter
+  include WorkerTools::Basic
+  wrappers :basics
+
+  def model_class
+    Import
+  end
+
+  def model_kind
+    'foo'
+  end
+
+  def run
+    # do stuff
+  end
+end
+```
+
+The basics module contains a `perform` method, which is the usual entry point for ApplicationJob and Sidekiq. It can receive the id of the model, the model instance, or nothing, in which case it will attempt to create this model on its own.
+
+
+## Module 'Recorder'
+
+Provides some methods to manage a log and the `information` field of the model. The main methods are `add_info`, `add_log`, and `record` (which both logs and appends the message to the information field). See all methods in [recorder](/lib/worker_tools/recorder.rb)
+
+This module has a _recoder_ wrapper that will register the exception details into the log and information field in case of error:
+
+```ruby
+class MyImporter
+  include WorkerTools::Basic
+  wrappers :basics, :recorder
+  # ...
+end
+```
+
+If you only want the logger functions, without worrying about persisting a model, you can use the `logger` wrapper and include the module as a stand alone component (without the basics module), like this:
+
+```ruby
+  class StandAloneWithLogging
+    include WorkerTools::Recorder
+
+    def perform
+      with_wrapper_logger do
+        # do stuff
+      end
+    end
+  end
+```
+
+## Module RocketchatErrorNotifier
+
+[recorder](/lib/worker_tools/rocketchat_error_notifier.rb)
+
+## Module CSV Input
+
+[csv_input](/lib/worker_tools/csv_input.rb)
+
+## Module CSV Output
+
+[csv_output](/lib/worker_tools/csv_output.rb)
+
+## Module XLSX Input
+
+[xlsx_input](/lib/worker_tools/xlsx_input.rb)
+
+
+## Wrappers
+
+In the [basics module](/lib/worker_tools/basics.rb), `perform` calls your custom method `run` to do the actual work of the task, and wraps it around any methods expecting a block that you might have had defined using `wrappers`.  That gives us a systematic way to add logic depending on the output of `run` and any exceptions that might arise, such as logging the error and context, sending a chat notification, retrying under some circumstances, etc.
+
+The following code
+
+```ruby
+class MyImporter
+  include WorkerTools::Basic
+  wrappers :basics
+
+  def run
+    # do stuff
+  end
+
+  # ..
+end
+```
+
+is internally handled as
+
+```ruby
+def perform(model_id)
+  # set model
+  with_wrapper_basics do
+    run
+  end
+end
+```
+
+where this wrapper method looks like
+
+```ruby
+def with_wrapper_basics(&block)
+  block.yield # calls run
+  # marks the import as complete
+  rescue Exception
+  # marks the import as failed
+  raise
+end
+```
+
+if we also add a wrapper to send notifications, such as
+`wrappers :basics, :rocketchat_error_notifier`
+
+the resulting nested calls would look like
+
+```ruby
+def perform(model_id)
+  # set model
+  with_wrapper_basics do
+    with_wrapper_rocketchat_error_notifier do
+      run
+    end
+  end
+end
+```
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/i22-digitalagentur/worker_tools.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "worker_tools"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/worker_tools/basics.rb

@@ -0,0 +1,96 @@
+require 'active_support/concern'
+
+module WorkerTools
+  module Basics
+    extend ActiveSupport::Concern
+
+    included do
+      attr_writer :model
+      attr_accessor :information
+
+      def self.wrappers(*args)
+        @wrappers ||= args.flatten
+      end
+
+      def self.read_wrappers
+        @wrappers || []
+      end
+    end
+
+    def model_class
+      # Ex: Import
+      raise "model_class has to be defined in #{self}"
+    end
+
+    def model_kind
+      # Ex: 'sdom'
+      raise "model_kind has to be defined in #{self}"
+    end
+
+    def run
+      raise "run has to be defined in #{self}"
+    end
+
+    def perform(model_id = nil)
+      @model_id = model_id
+      with_wrappers(wrapper_methods) do
+        run
+      end
+    end
+
+    def wrapper_methods
+      self.class.read_wrappers.map do |wrapper|
+        symbolized_method = "with_wrapper_#{wrapper}".to_sym
+        raise "Missing wrapper #{wrapper}" unless respond_to?(symbolized_method)
+        symbolized_method
+      end
+    end
+
+    def with_wrapper_basics(&block)
+      block.yield
+      finalize
+    # this time we do want to catch Exception to attempt to handle some of the
+    # critical errors.
+    # rubocop:disable Lint/RescueException
+    rescue Exception
+      # rubocop:enable Lint/RescueException
+      model.state = 'failed'
+      model.save!(validate: false)
+      raise
+    end
+
+    def finalize
+      model.update_attributes!(
+        state: 'complete',
+        information: information
+      )
+    end
+
+    def create_model_if_not_available
+      false
+    end
+
+    def model
+      @model ||= find_model
+    end
+
+    def with_wrappers(wrapper_symbols, &block)
+      return yield if wrapper_symbols.blank?
+      current_wrapper_symbol = wrapper_symbols.shift
+      send(current_wrapper_symbol) { with_wrappers(wrapper_symbols, &block) }
+    end
+
+    private
+
+    def find_model
+      @model_id ||= nil
+      return @model_id if @model_id.is_a?(model_class)
+      return model_class.find(@model_id) if @model_id
+      raise 'Model not available' unless create_model_if_not_available
+      t = model_class.new
+      t.kind = model_kind if t.respond_to?(:kind=)
+      t.save!(validate: false)
+      t
+    end
+  end
+end

data/lib/worker_tools/csv_input.rb

@@ -0,0 +1,191 @@
+require 'csv'
+
+module WorkerTools
+  module CsvInput
+    # If an array is provided, the names will be used as the row keys, the row
+    # values will be assign according to the columns order.
+    #
+    # Ex: %w(tenant segment area)
+    # row => {
+    #   tenant: _value_at_first_column_,
+    #   segment: _value_at_second_column_,
+    #   area: _value_at_third_column_
+    # }
+    #
+    # If a hash if provided, the keys will turn into the row keys, the values
+    # will be used to find the corresponding columns (the order in the csv won't
+    # affect the import)
+    #
+    # Ex: { tenant: 'Mandant', segment: 'Segment', area: 'Bereich')
+    # row => {
+    #   tenant: _value_at_column_Mandant,
+    #   segment: _value_at_column_Segment,
+    #   area: _value_at_column_Bereich
+    # }
+    #
+    # The name of the column is filtered using the csv_input_header_normalized
+    # method, which takes care of extra spaces and looks for a case insentive
+    # match (so 'Bereich' matches ' Bereich', 'bereich', etc.). You can override
+    # that method as well.
+    #
+    # Besides matching the columns using strings, it is possible to use a regular
+    # expression or a proc:
+    # {
+    #   tenant: 'Mandant',
+    #   segment: /Segment/i,
+    #   area: ->(name) { name.downcase == 'area' }
+    # }
+    def csv_input_columns
+      raise "csv_input_columns has to be defined in #{self}"
+    end
+
+    def csv_input_header_normalized(name)
+      name = name.to_s.strip
+      name = name.downcase if csv_input_header_normalize?
+      name
+    end
+
+    # Allows for some basic cleanup of the values, such as applying strip to
+    # the strings.
+    def cvs_input_value_cleanup(value)
+      value.is_a?(String) ? value.strip : value
+    end
+
+    def csv_input_columns_check(csv_rows_enum)
+      # override and return true if you do not want this check to be performed
+      return csv_input_columns_array_check(csv_rows_enum) if csv_input_columns.is_a?(Array)
+      csv_input_columns_hash_check(csv_rows_enum)
+    end
+
+    def csv_input_columns_array_check(csv_rows_enum)
+      expected_columns_length = csv_input_columns.length
+      actual_columns_length = csv_rows_enum.first.length
+      return if expected_columns_length == actual_columns_length
+      raise "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+    end
+
+    def csv_input_columns_hash_check(csv_rows_enum)
+      expected_names = csv_input_columns.values
+      filtered_actual_names = csv_rows_enum.first.map { |n| csv_input_header_normalized(n) }
+      csv_input_columns_hash_check_duplicates(filtered_actual_names)
+      csv_input_columns_hash_check_missing(filtered_actual_names, expected_names)
+    end
+
+    def csv_input_columns_hash_check_duplicates(names)
+      dups = names.group_by(&:itself).select { |_, v| v.count > 1 }.keys
+      raise "The file contains duplicated columns: #{dups}" if dups.present?
+    end
+
+    def csv_input_columns_hash_check_missing(actual_names, expected_names)
+      missing = expected_names.reject do |name|
+        matchable = name.is_a?(String) ? csv_input_header_normalized(name) : name
+        actual_names.any? { |n| case n when matchable then true end } # rubocop does not like ===
+      end
+      raise "Some columns are missing: #{missing}" unless missing.empty?
+    end
+
+    def csv_input_csv_options
+      # Ex: { col_sep: ';', encoding: Encoding::ISO_8859_1 }
+      { col_sep: ';' }
+    end
+
+    def csv_input_include_other_columns
+      false
+    end
+
+    def csv_input_header_normalize?
+      true
+    end
+
+    # Compares the first row (header names) with the csv_input_columns hash to find
+    # the corresponding positions.
+    #
+    # Ex: csv_input_columns: {tenant: 'Mandant', area: 'Bereich'}
+    #     headers: ['Bereich', 'Mandant']
+    #     =>  { tenant: 1, area: 0}
+    def csv_input_mapping_order(header_names)
+      return csv_input_columns.map.with_index { |n, i| [n, i] }.to_h if csv_input_columns.is_a?(Array)
+      csv_input_mapping_order_for_hash(header_names)
+    end
+
+    def csv_input_mapping_order_for_hash(header_names)
+      filtered_column_names = header_names.map { |n| csv_input_header_normalized(n) }
+      mapping = csv_input_columns.each_with_object({}) do |(k, v), h|
+        matchable = v.is_a?(String) ? csv_input_header_normalized(v) : v
+        h[k] = filtered_column_names.index { |n| case n when matchable then true end }
+      end
+      return mapping unless csv_input_include_other_columns
+      csv_input_mapping_order_with_other_columns(mapping, filtered_column_names)
+    end
+
+    def csv_input_mapping_order_with_other_columns(mapping, filtered_column_names)
+      positions_taken = mapping.values
+      filtered_column_names.each_with_index do |header, index|
+        mapping[header.to_sym] = index unless positions_taken.include?(index)
+      end
+      mapping
+    end
+
+    def csv_input_file_path
+      model.attachment.path.to_s
+    end
+
+    def csv_rows_enum
+      @csv_rows_enum ||= CSV.foreach(csv_input_file_path, csv_input_csv_options)
+    end
+
+    def csv_input_headers_present
+      true
+    end
+
+    def csv_input_foreach
+      @csv_input_foreach ||= begin
+        csv_input_columns_check(csv_rows_enum)
+
+        CsvInputForeach.new(
+          rows_enum: csv_rows_enum,
+          input_columns: csv_input_columns,
+          mapping_order: csv_input_mapping_order(csv_rows_enum.first),
+          cleanup_method: method(:cvs_input_value_cleanup),
+          headers_present: csv_input_headers_present
+        )
+      end
+    end
+
+    class CsvInputForeach
+      include Enumerable
+
+      def initialize(rows_enum:, input_columns:, mapping_order:, cleanup_method:, headers_present:)
+        @rows_enum = rows_enum
+        @input_columns = input_columns
+        @mapping_order = mapping_order
+        @cleanup_method = cleanup_method
+        @headers_present = headers_present
+      end
+
+      def each
+        return enum_for(:each) unless block_given?
+
+        @rows_enum.with_index.each do |values, index|
+          next if index.zero? && @headers_present
+          yield values_to_row(values)
+        end
+      end
+
+      def values_to_row(values)
+        return values_to_row_according_to_mapping(values) if @mapping_order
+        values_to_row_according_to_position(values)
+      end
+
+      def values_to_row_according_to_mapping(values)
+        @mapping_order.each_with_object(HashWithIndifferentAccess.new) do |(k, v), h|
+          h[k] = @cleanup_method.call(values[v])
+        end
+      end
+
+      def values_to_row_according_to_position(values)
+        @input_columns.map.with_index { |c, i| [c, @cleanup_method.call(values[i])] }.to_h.with_indifferent_access
+      end
+    end
+  end
+end

data/lib/worker_tools/csv_output.rb

@@ -0,0 +1,82 @@
+require 'csv'
+
+module WorkerTools
+  module CsvOutput
+    # if defined, this file will be written to this destination (regardless
+    # of whether the model saves the file as well)
+    def csv_output_target
+      # Ex: Rails.root.join('shared', 'foo', 'bar.csv')
+      false
+    end
+
+    def csv_output_entries
+      raise "csv_output_entries has to be defined in #{self}"
+    end
+
+    def csv_output_column_headers
+      # These columns are used to set the headers, also
+      # to set the row values depending on your implementation.
+      #
+      # To ignore them set it to _false_
+      #
+      # Ex:
+      # @csv_output_column_headers ||= {
+      #   foo: 'Foo Header',
+      #   bar: 'Bar Header'
+      # }
+      raise "csv_output_column_headers has to be defined in #{self}"
+    end
+
+    # rubocop:disable Lint/UnusedMethodArgument
+    def csv_output_row_values(entry)
+      # Ex:
+      # {
+      #   foo: entry.foo,
+      #   bar: entry.bar
+      # }.values_at(*csv_output_column_headers.keys)
+      raise "csv_output_row_values has to be defined in #{self}"
+    end
+    # rubocop:enable Lint/UnusedMethodArgument
+
+    def cvs_output_target_folder
+      File.dirname(csv_output_target)
+    end
+
+    def csv_output_target_file_name
+      File.basename(csv_output_target)
+    end
+
+    def csv_ouput_ensure_target_folder
+      FileUtils.mkdir_p(cvs_output_target_folder) unless File.directory?(cvs_output_target_folder)
+    end
+
+    def csv_output_tmp_file
+      @csv_output_tmp_file ||= Tempfile.new(['output', '.csv'])
+    end
+
+    def csv_output_col_sep
+      ';'
+    end
+
+    def csv_output_encoding
+      Encoding::UTF_8
+    end
+
+    def csv_output_insert_headers(csv)
+      csv << csv_output_column_headers.values if csv_output_column_headers
+    end
+
+    def csv_output_write_file
+      CSV.open(csv_output_tmp_file, 'wb', col_sep: csv_output_col_sep, encoding: csv_output_encoding) do |csv|
+        csv_output_insert_headers(csv)
+        csv_output_entries.each { |entry| csv << csv_output_row_values(entry) }
+      end
+      csv_output_write_target if csv_output_target
+    end
+
+    def csv_output_write_target
+      csv_ouput_ensure_target_folder
+      FileUtils.cp(csv_output_tmp_file.path, csv_output_target)
+    end
+  end
+end

data/lib/worker_tools/recorder.rb

@@ -0,0 +1,81 @@
+module WorkerTools
+  module Recorder
+    def with_wrapper_recorder(&block)
+      block.yield
+    # this time we do want to catch Exception to attempt to handle some of the
+    # critical errors.
+    # rubocop:disable Lint/RescueException
+    rescue Exception => e
+      # rubocop:enable Lint/RescueException
+      record_fail(e)
+      raise
+    end
+
+    def with_wrapper_logger(&block)
+      block.yield
+    # this time we do want to catch Exception to attempt to handle some of the
+    # critical errors.
+    # rubocop:disable Lint/RescueException
+    rescue Exception => e
+      # rubocop:enable Lint/RescueException
+      add_log(e, :error)
+      raise
+    end
+
+    def record_fail(error)
+      record "ID #{model.id} - Error"
+      record(error, :error)
+      model.information = information
+      model.save!(validate: false)
+    end
+
+    def add_log(message, level = :info)
+      logger.public_send(level, format_log_message(message))
+    end
+
+    def add_info(message)
+      @information ||= ''
+      information << "#{format_info_message(message)}\n"
+    end
+
+    def record(message, level = :info)
+      add_log(message, level)
+      add_info(message)
+    end
+
+    def format_log_message(message)
+      return error_to_text(message, log_error_trace_lines) if message.is_a?(Exception)
+      message
+    end
+
+    def format_info_message(message)
+      return error_to_text(message, info_error_trace_lines) if message.is_a?(Exception)
+      message
+    end
+
+    def logger
+      @logger ||= Logger.new(File.join(log_directory, log_file_name))
+    end
+
+    def log_directory
+      Rails.root.join('log')
+    end
+
+    def log_file_name
+      "#{self.class.name.underscore.tr('/', '_')}.log"
+    end
+
+    def log_error_trace_lines
+      20
+    end
+
+    def info_error_trace_lines
+      20
+    end
+
+    def error_to_text(error, trace_lines = 20)
+      txt = "Error: #{error.message} (#{error.class})"
+      txt << "Backtrace:\n#{error.backtrace[0, trace_lines].join("\n\t")}"
+    end
+  end
+end

data/lib/worker_tools/rocketchat_error_notifier.rb

@@ -0,0 +1,68 @@
+module WorkerTools
+  module RocketchatErrorNotifier
+    def with_wrapper_rocketchat_error_notifier(&block)
+      block.yield
+    rescue StandardError => e
+      rocketchat_error_notify(e) if rocketchat_error_notifier_enabled
+      raise
+    end
+
+    def rocketchat_error_notifier_enabled
+      Rails.env.production?
+    end
+
+    def rocketchat_error_notifier_emoji
+      ':red_circle:'
+    end
+
+    def rocketchat_error_notifier_receivers
+      # Ex: '@all'
+    end
+
+    def rocketchat_error_notifier_event
+      'Worker Error Notifier'
+    end
+
+    def rocketchat_error_notifier_title
+      # Example with a link:
+      #
+      # For urls a default_url_options[:host] might be necessary.
+      # In this example I just copy it from existing action_mailer defaults.
+      #
+      # import = rocketchat_error_notifier_model
+      # host = Rails.application.config.action_mailer.default_url_options[:host]
+      # url = Rails.application.routes.url_helpers.import_url(import, host: host, protocol: :https)
+      # kind = I18n.t(import.kind, scope: 'import.kinds')
+      # text = "##{import.id} *#{kind}*"
+      # "[#{text}](#{url})"
+      klass = model.class.model_name.human
+      kind = I18n.t("activerecord.attributes.#{model.class.name.underscore}.kinds.#{model.kind}")
+      "#{klass} #{kind} ##{model.id}"
+    end
+
+    def rocketchat_error_notifier_error_details(error)
+      details = "#{error.class}: #{error.message}\n"
+      details << error.backtrace[0..10].join("\n")
+    end
+
+    def rocketchat_error_notifier_message
+      message = []
+      message << rocketchat_error_notifier_receivers
+      message << rocketchat_error_notifier_title
+      message.compact.join(' - ')
+    end
+
+    def rocketchat_error_notifier_attachment(error)
+      { collapsed: true, title: 'Error', text: rocketchat_error_notifier_error_details(error) }
+    end
+
+    def rocketchat_error_notify(error)
+      RocketChatNotifier.notify(
+        rocketchat_error_notifier_message,
+        emoji: rocketchat_error_notifier_emoji,
+        event: "#{rocketchat_error_notifier_event} (#{Rails.env})",
+        attachment: rocketchat_error_notifier_attachment(error)
+      )
+    end
+  end
+end

data/lib/worker_tools/version.rb

@@ -0,0 +1,3 @@
+module WorkerTools
+  VERSION = '0.1.1'.freeze
+end

data/lib/worker_tools/xlsx_input.rb

@@ -0,0 +1,169 @@
+require 'roo'
+
+module WorkerTools
+  module XlsxInput
+    # If an array is provided, the names will be used as the row keys, the row
+    # values will be assign according to the columns order.
+    #
+    # Ex: %w(tenant segment area)
+    # row => {
+    #   tenant: _value_at_first_column_,
+    #   segment: _value_at_second_column_,
+    #   area: _value_at_third_column_
+    # }
+    #
+    # If a hash if provided, the keys will turn into the row keys, the values
+    # will be used to find the corresponding columns (the order in the spreadsheet
+    # won't affect the import)
+    #
+    # Ex: { tenant: 'Mandant', segment: 'Segment', area: 'Bereich')
+    # row => {
+    #   tenant: _value_at_column_Mandant,
+    #   segment: _value_at_column_Segment,
+    #   area: _value_at_column_Bereich
+    # }
+    #
+    # The name of the column is filtered using the xlsx_input_header_normalized
+    # method, which takes care of extra spaces and looks for a case insentive
+    # match (so 'Bereich' matches ' Bereich', 'bereich', etc.). You can override
+    # that method as well.
+    #
+    # Besides matching the columns using strings, it is possible to use a regular
+    # expression or a proc:
+    # {
+    #   tenant: 'Mandant',
+    #   segment: /Segment/i,
+    #   area: ->(name) { name.downcase == 'area' }
+    # }
+    def xlsx_input_columns
+      raise "xlsx_input_columns has to be defined in #{self}"
+    end
+
+    # If true, the rows will append those columns that don't belong to the
+    # xlsx_input_columns list. Useful when the spreadsheet contains some fixed
+    # columns and a number of variable ones.
+    def xlsx_input_include_other_columns
+      false
+    end
+
+    def xlsx_input_header_normalized(name)
+      name.to_s.strip.downcase
+    end
+
+    # Allows for some basic cleanup of the values, such as applying strip to
+    # the strings.
+    def xlsx_input_value_cleanup(value)
+      value.is_a?(String) ? value.strip : value
+    end
+
+    def xlsx_input_columns_check(xlsx_rows_enum)
+      # override and return true if you do not want this check to be performed
+      return xlsx_input_columns_array_check(xlsx_rows_enum) if xlsx_input_columns.is_a?(Array)
+      xlsx_input_columns_hash_check(xlsx_rows_enum)
+    end
+
+    def xlsx_input_columns_array_check(xlsx_rows_enum)
+      expected_columns_length = xlsx_input_columns.length
+      actual_columns_length = xlsx_rows_enum.first.length
+      return if expected_columns_length == actual_columns_length
+      raise "The number of columns (#{actual_columns_length}) is not the expected (#{expected_columns_length})"
+    end
+
+    def xlsx_input_columns_hash_check(xlsx_rows_enum)
+      expected_names = xlsx_input_columns.values
+      filtered_actual_names = xlsx_rows_enum.first.map { |n| xlsx_input_header_normalized(n) }
+      xlsx_input_columns_hash_check_duplicates(filtered_actual_names)
+      xlsx_input_columns_hash_check_missing(filtered_actual_names, expected_names)
+    end
+
+    def xlsx_input_columns_hash_check_duplicates(names)
+      dups = names.group_by(&:itself).select { |_, v| v.count > 1 }.keys
+      raise "The file contains duplicated columns: #{dups}" if dups.present?
+    end
+
+    def xlsx_input_columns_hash_check_missing(actual_names, expected_names)
+      missing = expected_names.reject do |name|
+        matchable = name.is_a?(String) ? xlsx_input_header_normalized(name) : name
+        actual_names.any? { |n| case n when matchable then true end } # rubocop does not like ===
+      end
+      raise "Some columns are missing: #{missing}" unless missing.empty?
+    end
+
+    # Compares the first row (header names) with the xlsx_input_columns hash to find
+    # the corresponding positions.
+    #
+    # Ex: xlsx_input_columns: {tenant: 'Mandant', area: 'Bereich'}
+    #     headers: ['Bereich', 'Mandant']
+    #     =>  { tenant: 1, area: 0}
+    def xlsx_input_mapping_order(header_names)
+      return xlsx_input_columns.map.with_index { |n, i| [n, i] }.to_h if xlsx_input_columns.is_a?(Array)
+      xlsx_input_mapping_order_for_hash(header_names)
+    end
+
+    def xlsx_input_mapping_order_for_hash(header_names)
+      filtered_column_names = header_names.map { |n| xlsx_input_header_normalized(n) }
+      mapping = xlsx_input_columns.each_with_object({}) do |(k, v), h|
+        matchable = v.is_a?(String) ? xlsx_input_header_normalized(v) : v
+        h[k] = filtered_column_names.index { |n| case n when matchable then true end }
+      end
+      return mapping unless xlsx_input_include_other_columns
+      xlsx_input_mapping_order_with_other_columns(mapping, filtered_column_names)
+    end
+
+    def xlsx_input_mapping_order_with_other_columns(mapping, filtered_column_names)
+      positions_taken = mapping.values
+      filtered_column_names.each_with_index do |header, index|
+        mapping[header] = index unless positions_taken.include?(index)
+      end
+      mapping
+    end
+
+    def xlsx_input_file_path
+      model.attachment.path.to_s
+    end
+
+    def xlsx_rows_enum
+      @xlsx_rows_enum ||= begin
+        spreadsheet = Roo::Excelx.new(xlsx_input_file_path)
+        spreadsheet.each_row_streaming(sheet: spreadsheet.sheets.first, pad_cells: true)
+      end
+    end
+
+    def xlsx_input_foreach
+      @xlsx_input_foreach ||= begin
+        xlsx_input_columns_check(xlsx_rows_enum)
+
+        XlsxInputForeach.new(
+          rows_enum: xlsx_rows_enum,
+          mapping_order: xlsx_input_mapping_order(xlsx_rows_enum.first),
+          cleanup_method: method(:xlsx_input_value_cleanup)
+        )
+      end
+    end
+
+    class XlsxInputForeach
+      include Enumerable
+
+      def initialize(rows_enum:, mapping_order:, cleanup_method:)
+        @rows_enum = rows_enum
+        @mapping_order = mapping_order
+        @cleanup_method = cleanup_method
+      end
+
+      def each
+        return enum_for(:each) unless block_given?
+
+        @rows_enum.with_index.each do |values, index|
+          next if index.zero? # headers
+          yield values_to_row(values)
+        end
+      end
+
+      def values_to_row(values)
+        @mapping_order.each_with_object(HashWithIndifferentAccess.new) do |(k, v), h|
+          h[k] = @cleanup_method.call(values[v].try(:value))
+        end
+      end
+    end
+  end
+end

data/lib/worker_tools.rb

@@ -0,0 +1,4 @@
+Dir[File.join(__dir__, 'worker_tools/*.rb')].each { |path| require path }
+
+module WorkerTools
+end

data/worker_tools.gemspec

@@ -0,0 +1,43 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'worker_tools/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'worker_tools'
+  spec.version       = WorkerTools::VERSION
+  spec.authors       = ['fsainz']
+  spec.email         = ['fernando.sainz@i22.de']
+
+  spec.summary       = 'A collection of modules to help writing common worker tasks)'
+  # spec.description   = %q{TODO: Write a longer description or delete this line.}
+  spec.homepage      = 'https://github.com/i22-digitalagentur/worker-tools'
+  spec.license       = 'MIT'
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the "allowed_push_host"
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.' unless spec.respond_to?(:metadata)
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'rocketchat-notifier', '>= 0.1.2'
+  spec.add_dependency 'roo'
+
+  spec.add_development_dependency 'activerecord'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'database_cleaner'
+  spec.add_development_dependency 'm'
+  spec.add_development_dependency 'minitest'
+  spec.add_development_dependency 'mocha'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rubocop', '0.52.1'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'sqlite3'
+end

metadata

@@ -0,0 +1,259 @@
+--- !ruby/object:Gem::Specification
+name: worker_tools
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- fsainz
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-06-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rocketchat-notifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: m
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.52.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.52.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- fernando.sainz@i22.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/worker_tools.rb
+- lib/worker_tools/basics.rb
+- lib/worker_tools/csv_input.rb
+- lib/worker_tools/csv_output.rb
+- lib/worker_tools/recorder.rb
+- lib/worker_tools/rocketchat_error_notifier.rb
+- lib/worker_tools/version.rb
+- lib/worker_tools/xlsx_input.rb
+- worker_tools.gemspec
+homepage: https://github.com/i22-digitalagentur/worker-tools
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A collection of modules to help writing common worker tasks)
+test_files: []