itrp-export-monitor 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YmVjNzQ1ZDYzMWE0MGI1MjA1ZGI0NDc3NjVlNGFjZTZjYjM3YmQxMw==
+  data.tar.gz: !binary |-
+    YmU3ZGE5NTE1ZDYzYzUyZjgyN2I0ZjY0OGFjN2QxMGJmOTZjNzU0Zg==
+SHA512:
+  metadata.gz: !binary |-
+    MmQ4YmZiMDUyZTg3YTZlZDMzZWY1ODBlZjA5YjFiMWFhNTJjMDM4ZDQ0OGU5
+    OWQ1OGUwYzc3NDUxZGQ5ZTZkOTk4YjJhNzBmOGIxZjIwZGU5ZDVlOTRhY2Zl
+    YjIzMDNiMzM2MWVjOWViOTkwN2YxY2QxYjE1ZDY1YmI4MWJkYmQ=
+  data.tar.gz: !binary |-
+    YWM5ZjZlYjFhZDEyZDgzMzc3MmExMTM2MDRiMjJkODc3ZWM0MTJhZmIzMTRj
+    ZTRlMzExNTZmNDIxOTk2Y2QxNGNjMjE4OTJlZmNmN2ZkZWM0ZjMyYmFhMWIz
+    MjhjZWEwNTFlMzYzYzliYjM2ZjgzMWU4MGFlZGUxNWRlYmFlMGY=

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in itrp-client.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,69 @@
+PATH
+  remote: .
+  specs:
+    itrp-export-monitor (1.0.0)
+      active_support
+      clacks (>= 1.0.0)
+      gem_config
+      i18n
+      itrp-client
+      rubyzip
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    active_support (3.0.0)
+      activesupport (= 3.0.0)
+    activesupport (3.0.0)
+    addressable (2.3.5)
+    clacks (1.0.4)
+      mail
+    crack (0.4.1)
+      safe_yaml (~> 0.9.0)
+    diff-lcs (1.2.5)
+    docile (1.1.0)
+    gem_config (0.2.4)
+    i18n (0.6.5)
+    itrp-client (1.0.1)
+      activesupport
+      gem_config
+      mime-types
+    mail (2.5.4)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.25)
+    multi_json (1.8.2)
+    polyglot (0.3.3)
+    rake (10.1.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    rubyzip (1.1.0)
+    safe_yaml (0.9.7)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    treetop (1.4.15)
+      polyglot
+      polyglot (>= 0.3.1)
+    webmock (1.16.0)
+      addressable (>= 2.2.7)
+      crack (>= 0.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  itrp-export-monitor!
+  rake
+  rspec
+  simplecov
+  webmock

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013 ITRP Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,322 @@
+# ITRP Export Monitor
+
+
+The itrp-export-monitor gem makes it easy to monitor a mailbox
+receiving [Scheduled Exports](http://help.itrp.com/help/export_fields) from ITRP
+and to store the incoming export files on disk or forward them to an FTP server.
+
+This readme will take you through all the steps of setting up an Export Monitor:
+
+* [Install Ruby, bundler and create a Gemfile](#installation)
+* [Generate an Export Monitor](#generate-an-export-monitor)
+* [Customize the configuration](#configuration)
+* [Start the Export Monitor](#start-the-export-monitor)
+* [Monitor the Export Monitor](#monitor-the-export-monitor)
+
+
+Installation
+------------
+
+1. [Install Ruby 1.9.3 or higher](https://www.ruby-lang.org/en/downloads/)
+2. Create a *root directory* for the ITRP Export Monitor and go to that directory, e.g.
+   ```
+   $ mkdir /usr/local/itrp_exports
+   $ cd /usr/local/itrp_exports
+   ```
+3. Install [Bundler](http://bundler.io/)
+   ```
+   $ gem install bundler
+   ```
+4. Create a [`Gemfile`](http://bundler.io/v1.3/gemfile.html) in the *root directory* with the following contents:
+   ```
+   source 'https://rubygems.org'
+
+   gem 'itrp-export-monitor'
+
+   ```
+5. Finally download the gems using bundler:
+   ```
+   $ bundle
+   ```
+
+Generate an Export Monitor
+--------------------------
+
+An Export Monitor is basically a ruby file that [configures](#configuration) the monitor
+and then fires it up to start looking for finished exports in the mailbox.
+
+To help you create the ruby file, the following generator is available from the *root directory*:
+```
+$ itrp-export-monitor generate[<export ID>,<email address>,<imap password>]
+```
+
+The **export ID** is the unique identifier of the Scheduled Export in ITRP
+and can be found in the address bar of the browser when you view
+the Scheduled Export in ITRP at `https://<your domain>.itrp.com/exports`.
+
+The **email address** is the email address where the export file is sent to.
+This is the email address of the user defined in the *Run as* field.
+**We strongly recommend to [create a separate mailbox](setup-a-mailbox) for the export monitor.**
+
+The **password** is the password with which the IMAP server can be accessed for this email address.
+
+The default configuration is set up to work with [GMail](http://mail.google.com) (in English)
+and copies completed export files to `/tmp/exports`.
+
+Below is an example of the generated configuration:
+
+```
+$ cd /usr/local/itrp_exports
+$ itrp-export-monitor generate[777,test.my.export@gmail.com,easy_to_gu3ss]
+```
+
+**/usr/local/itrp_exports/export_monitor.777.rb**
+
+```
+require 'itrp/export/monitor'
+
+# the location where all the run-time information on the Export Monitor is stored
+BASE_DIR = "/usr/local/itrp_exports/export_monitor_777"
+FileUtils.mkpath "#{BASE_DIR}/log"
+
+Itrp::Export::Monitor.configure do |export|
+  export.root =       BASE_DIR
+  export.logger =     Logger.new("#{BASE_DIR}/log/export_monitor.777.log")
+  export.ids =        [777]
+  export.unzip =      true
+  export.sub_dirs =   false
+
+  export.to = '/tmp/exports'
+  # export.to_ftp =        'ftp.mycompany.com'
+  # export.to_ftp_dir =    'my/exports'
+  # export.ftp_user_name = 'user'
+  # export.ftp_password =  'secret'
+
+  export.imap_address =    'imap.googlemail.com'
+  export.imap_port =       993
+  export.imap_user_name =  'test.my.export@gmail.com'
+  export.imap_password =   'easy_to_gu3ss'
+  export.imap_ssl =        true
+  export.imap_mailbox =    'INBOX'
+  export.imap_archive =    '[Gmail]/All Mail'
+end
+
+Itrp::Export::Monitor.run
+```
+
+
+Configuration
+-------------
+
+Before you [start the Export Monitor](#start-the-export-monitor)
+you need to customize the [generated configuration](#generate-an-export-monitor).
+
+The Export Monitor configuration is defined using a block:
+```
+Itrp::Export::Monitor.configure do |export|
+  export.root = '/usr/local/...'
+  export.ids =    [777, 779]
+  ...
+end
+```
+
+All options available:
+
+* _logger_:         The Ruby Logger instance, default: `Logger.new(STDOUT)`
+* _daemonize_:      Set to `true` to run in daemon mode; not available on Windows (default: `false`)
+* _exit_when_idle_: Number of minutes after which the service stops when no Export mails are found (default: `-1` no exit)
+* _root_:           **required** The root directory to store Export Monitor logs, pids and downloads
+* _id/ids_:         **required** The id(s) of the Scheduled Exports to monitor, e.g. `[777]`
+* _unzip_:          Unzip the CSV files, default: `true`.
+* _sub_dirs_:       Place each CSV file in a different subdirectory based on the export type (default: `false`)
+* _to_:             Directory to store export files on local disk, e.g. `'/tmp/people_export'`
+* _to_ftp_:         The address of the FTP server to sent the completed downloads to, e.g. `'ftp.mycompany.com'`
+* _to_ftp_dir_:     The subdirectory on the FTP server, e.g. `'my/downloads'` (default: `'.'`)
+* _ftp_user_name_:  The user name to access the FTP server
+* _ftp_password_:   The password to access the FTP server
+* _imap_address_:   **required** The address of the IMAP mail server (default: `'imap.googlemail.com'`)
+* _imap_port_:      The port of the IMAP mail server (default: `993`)
+* _imap_ssl_:       Set to +false+ to disabled SSL (default: `true`)
+* _imap_user_name_: **required** The user name to access the IMAP server
+* _imap_password_:  **required** The password to access the IMAP server
+* _imap_mailbox_:   The mailbox to monitor for ITRP export mails (default: `'INBOX'`)
+* _imap_archive_:   The archive mailbox to store the processed ITRP export mails (default: `'[Gmail]/All Mail'`)
+* _on_exception_:   A Proc that takes an exception and the mail as an argument: `Proc.new{ |ex, mail| ... }`.
+  All exceptions will also be logged as errors in the logfile.
+* _csv_row_sep_:    Set the CSV row separator (default: `:auto`, i.e. windows/unix newline)
+* _csv_col_sep_:    Set the CSV column separator (default: `','`)
+* _csv_quote_char_: Set the CSV quote character, at most 1 character (default: `'"'`)
+* _csv_value_proc_: Provide a procedure to change values before adding them to the CSV, e.g.
+  `Proc.new{ |value| value.gsub(/\r?\n/, ' ')`
+
+
+Start the Export Monitor
+------------------------
+
+To start an Export Monitor simply run the following command:
+
+```
+$ cd /usr/local/itrp_exports
+$ bundle exec ruby export_monitor.777.rb
+```
+
+If the configuration is correct, the Export Monitor will startup and keep on running until it
+receives a *QUIT* signal (by pressing `<ctrl>-C`).
+
+If the Export Monitor stops running immediately the configuration is probably incorrect.
+The log file will contain the details on what went wrong, see:
+```
+$ less /usr/local/itrp_exports/export_monitor_777/log/export_monitor.777.log
+```
+
+If the Export Monitor is running, but the exports are not picked up and processed as expected, also check
+the log file first.
+
+On startup the following directory structure will be created in the *export.root* directory:
+
+```
+/usr
+  /local
+    /itrp_exports
+      /export_monitor_777
+        /downloads
+          ...
+        /log
+          export_monitor.777.log
+        /pids
+          export_monitor.777.pid
+        /tmp
+          clacks_config.export_monitor.777.rb
+```
+
+The `downloads` subdirectory contains the downloaded csv/zip files. These files are not
+deleted automatically, so it is advisable to [monitor the disk usage](#disk-usage).
+
+The `log` directory contains the log file of the Export Monitor.
+
+The `pids` directory contains the pid file of the Export Monitor.
+
+The `tmp` directory contains temporary files to run the Export Monitor and should be left alone.
+
+
+Monitoring the Export Monitor
+-----------------------------
+
+The process ID of the Export Monitor will be stored in the `<root dir>/pids/export_monitor.<id>.pid` file.
+Tools like [Monit](http://mmonit.com/monit/) can be used to make sure the Export Monitor is always running.
+
+When an incoming export could not be processed correctly, an error is logged in the logfile
+which is located in `<root dir>/log/export_monitor.<id>.log`. You should either watch this logfile for
+errors, or define a custom *on_exception* handler to take the [appropriate action](#recovering-from-errors).
+
+Below is an example of an *on_exception* handler sending a mail to a systems mailbox using
+the [mail gem](https://github.com/mikel/mail).
+
+```
+Itrp::Export::Monitor.configure do |export|
+  ...
+  export.on_exception = Proc.new do |ex, mail|
+    Mail.deliver do
+      from    'export.monitor@mycompany.example.com'
+      to      'sysadmin@mycompany.example.com'
+      subject "Unable to process incoming export mail: #{mail.original.subject}"
+      body    ex.message
+    end
+  end
+end
+```
+
+#### Recovering from errors
+
+When an export mail could not be processed correctly:
+
+1. the error is logged in the logfile (once)
+2. the *on_exception* handler is called (once)
+3. the mail is left in the *imap_mailbox*
+
+After that the mail will not be processed again unless the Export Monitor is restarted.
+
+#### Disk usage
+
+All export files that are downloaded are kept in the `<export.root>/downloads` directory.
+These files are not deleted automatically, so you might want to add a job to cleanup this
+directory every month/year depending on your setup.
+
+#### Not a real service?
+
+The `exit_when_idle` option may be used to stop the Export Monitor when there are no new
+export mails coming in for a couple of minutes. This may be useful in case you want to
+fire up the Export Monitor at a scheduled time using the cron-tab or Windows Task Scheduler.
+
+Other considerations
+--------------------
+
+#### Setup a mailbox
+
+It is best to create a separate user in ITRP with a corresponding mailbox for each Export Monitor.
+
+For example: if the Export Monitor will monitor the Weekly Full People, you could create a
+[GMail](http://mail.google.com) for *people.monitor.mycompany@gmail.com*. Then create a new
+[ITRP User](http://developer.itrp.com/v1/general/getting_started/) using that email address.
+
+That's it. Now you are ready to [generate an Export Monitor](#generate-an-export-monitor).
+
+#### Not a dedicated mailbox?
+
+The Export Monitor will search all mails in the *imap_mailbox* for export mails sent by ITRP.
+When an export mail is found and the export ID matches one of the *ids* in the
+[configuration](#configuration), the mail is processed. Other mails in the same *imap_mailbox*
+are left alone.
+
+If there are a lot of mails kept in the mailbox the processing may slow down.
+It is advisable to create a separate user in ITRP with the Account Administrator role and
+it's own mailbox for processing exports.
+
+That user should then be selected as the *Run as* user in the Scheduled Export.
+
+#### IMAP
+
+All options starting with the *imap* prefix are used to access the mailbox to monitor incoming mails.
+By default the configuration is setup for [GMail](http://mail.google.com).
+
+Contact the System Administrator of your mailbox in case you are not sure how to setup the IMAP
+configuration.
+
+#### Multiple Scheduled Exports
+
+To monitor and process multiple Scheduled Exports simply provide all the export IDs to monitor
+in the [configuration](#configuration):
+
+```
+export.ids = [777, 785, 786]
+```
+
+#### Forwarding export mails
+
+Forwarded mails will not be processed as the Export Monitor depends on the ITRP
+[mail message headers](http://developer.itrp.com/v1/export/#downloading-an-export-file) to be available in the mail.
+When a mail is forwarded these headers may not be found and the export mail is not processed.
+
+#### Export files that are partially copied
+
+To prevent issues with partial files the export monitor will append `.in_progress` to export files
+that are being copied/FTP'd. Once the copy is completed, the file is renamed and the `.in_progress` suffix
+is removed.
+
+Another way to prevent issues with partially copied files is to try to obtain a write lock on the file
+before processing the export file. The OS will return an error when the file is not completely copied.
+
+#### CSV not fully supported?
+
+Some systems, like [SAP BI](http://scn.sap.com/thread/3200938), cannot handle well-defined CSV files.
+As a workaround the Export Monitor can rewrite the CSV files in a different format. To do so, take a look
+at the [configuration](#configuration).
+
+The most common issue is that new-lines within a value are not handled correctly when the new-line is also used
+as a row separator. One way to deal with that is to replace the new-lines in the values with a different value:
+
+```
+Itrp::Export::Monitor.configure do |export|
+  export.csv_value_proc = Proc.new{ |value| value.gsub(/\r?\n/, ' ') }
+end
+```

data/bin/itrp-export-monitor

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+require 'rake'
+require 'bundler'
+
+raise "Bundler is required. Please install bundler with 'gem install bundler'" unless defined?(Bundler)
+
+# Example:
+#   itrp-export-monitor generate[<export_id>]
+
+# init dependencies
+Bundler.setup
+
+# init rake
+Rake.application.init
+
+# load the rake tasks
+gem_dir = File.expand_path('..',File.dirname(__FILE__))
+load "#{gem_dir}/lib/itrp/export/monitor/tasks/itrp_export_monitor.rake"
+
+# invoke the given task
+Rake.application.invoke_task("itrp_export_monitor:#{ARGV[0]}")

data/itrp-export-monitor.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'itrp/export/monitor/version'
+
+Gem::Specification.new do |spec|
+  spec.name                  = 'itrp-export-monitor'
+  spec.version               = Itrp::Export::Monitor::VERSION
+  spec.platform              = Gem::Platform::RUBY
+  spec.required_ruby_version = '>= 1.9.3'
+  spec.authors               = ['ITRP']
+  spec.email                 = %q{developers@itrp.com}
+  spec.description           = %q{Monitor a mailbox and store Scheduled ITRP Exports to disk or FTP.}
+  spec.summary               = %q{The itrp-export-monitor gem makes it easy to monitor a mailbox receiving Scheduled Exports from ITRP and to store the incoming export files on disk or forward it to an FTP server.}
+  spec.homepage              = %q{https://github.com/itrp/itrp-export-monitor}
+  spec.license               = 'MIT'
+
+  spec.files = Dir.glob("lib/**/*") + %w(LICENSE.txt README.md Gemfile Gemfile.lock itrp-export-monitor.gemspec)
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  spec.test_files    = `git ls-files -- {test,spec}/*`.split("\n")
+  spec.require_paths = ['lib']
+  spec.rdoc_options = ['--charset=UTF-8']
+
+  spec.add_runtime_dependency 'gem_config'
+  spec.add_runtime_dependency 'itrp-client'
+  spec.add_runtime_dependency 'active_support'
+  spec.add_runtime_dependency 'rubyzip'
+  spec.add_runtime_dependency 'clacks', '>= 1.0.0'
+  spec.add_dependency 'i18n'
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'webmock'
+  spec.add_development_dependency 'simplecov'
+end

data/lib/itrp/export/monitor.rb

@@ -0,0 +1,118 @@
+require 'gem_config'
+require 'logger'
+require 'mail'
+
+require 'itrp/client' # this will load CA Certificate bundle
+require 'clacks'
+
+require 'itrp/export/monitor/service'
+require 'itrp/export/monitor/version'
+require 'itrp/export/monitor/mail'
+require 'itrp/export/monitor/exchange'
+
+require 'active_support/core_ext/module/aliasing.rb'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/object/try.rb'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Itrp
+  module Export
+
+    # Configuration for ITRP Export Monitor:
+    #   Itrp::Export::Monitor.configure do |export|
+    #     export.name           = 'people_export'
+    #     export.root           = File.expand_path('../exports', __FILE__)
+    #     export.id             = 8713
+    #     export.to             = "#{export.root}/people"
+    #     export.imap_user_name = 'test.exports@gmail.com'
+    #     export.imap_password  = 'secret'
+    #     ...
+    #   end
+    #
+    # Start the ITRP Export Monitor:
+    #   Itrp::Export::Monitor.run
+    #
+    # All options available:
+    #  - logger:         The Ruby Logger instance, default: Logger.new(STDOUT)
+    #  - daemonize:      Set to +true+ to run in daemon mode; not available on Windows (default: false)
+    #  - name:           *required* The name of the export
+    #  - root:           *required* The root directory to store export monitor logs, pids and downloads
+    #  - id/ids:         *required* The id(s) of the scheduled exports to monitor
+    #  - to:             Location to store export files (default = <root>/ready)
+    #  - to_ftp:         The address of the FTP server to sent the completed downloads to
+    #  - to_ftp_dir:     The subdirectory on the FTP server (default = '.')
+    #  - ftp_user_name:  The user name to access the FTP server
+    #  - ftp_password:   The password to access the FTP server
+    #  - imap_address:   The address of the IMAP mail server (default: 'imap.googlemail.com')
+    #  - imap_port:      The port of the IMAP mail server (default: 993)
+    #  - imap_ssl:       Set to +false+ to disabled SSL (default: true)
+    #  - imap_user_name: *required* The user name to access the IMAP server
+    #  - imap_password:  *required* The password to access the IMAP server
+    #  - imap_mailbox:   The mailbox to monitor for ITRP export mails (default: 'INBOX')
+    #  - imap_archive:   The archive mailbox to store the processed ITRP export mails (default: '[Gmail]/All Mail')
+    #  - on_exception:   A Proc that takes an exception and the mail as an argument: Proc.new{ |ex, mail| ... }
+    module Monitor
+      include GemConfig::Base
+
+      with_configuration do
+        has :logger, classes: ::Logger, default: ::Logger.new(STDOUT)
+
+        has :daemonize, classes: [TrueClass, FalseClass], default: false
+        has :exit_when_idle, classes: Fixnum, default: -1
+        has :root, classes: String
+        has :id, classes: Fixnum
+        has :ids, classes: Array
+        has :unzip, classes: [TrueClass, FalseClass], default: true
+        has :sub_dirs, classes: [TrueClass, FalseClass], default: false
+
+        has :to, classes: String
+
+        has :to_ftp, classes: String
+        has :to_ftp_dir, classes: String, default: '.'
+        has :ftp_user_name, classes: String
+        has :ftp_password, classes: String
+
+        has :imap_address, classes: String, default: 'imap.googlemail.com'
+        has :imap_port, classes: Fixnum, default: 993
+        has :imap_ssl, classes: [TrueClass, FalseClass], default: true
+        has :imap_user_name, classes: String
+        has :imap_password, classes: String
+
+        has :imap_mailbox, classes: String, default: 'INBOX'
+        has :imap_archive, classes: String, default: '[Gmail]/All Mail'
+
+        has :on_exception, classes: Proc
+
+        has :csv_row_sep, classes: String
+        has :csv_col_sep, classes: String
+        has :csv_quote_char, classes: String
+        has :csv_value_proc, classes: Proc
+      end
+
+      class << self
+
+        def run
+          # create the export monitor as a singleton
+          @service = Itrp::Export::Monitor::Service.new
+          # generate clacks file
+          clacks_config_filename = @service.generate_clacks_config
+          # start clacks with the generated config
+          args = ['-c', clacks_config_filename]
+          args << '-D' if @service.option(:daemonize)
+          Clacks::Command.new(args).exec
+          # return the singleton instance
+          @service
+        end
+
+        def process(mail)
+          @service.process(mail)
+        end
+
+        def logger
+          configuration.logger
+        end
+      end
+
+    end
+  end
+end

data/lib/itrp/export/monitor/exchange.rb

@@ -0,0 +1,139 @@
+require 'fileutils'
+require 'net/ftp'
+require 'zip'
+require 'csv'
+
+module Itrp
+  module Export
+    module Monitor
+
+      class Exchange
+        attr_accessor :options, :fullpath, :basename
+
+        def initialize(export_filename, options)
+          @options = options
+          @options[:to_ftp_dir].gsub!('\\', '/')
+          @fullpath = export_filename
+          @basename = File.basename(export_filename)
+          @logger = @options[:logger]
+        end
+
+        def transfer
+          files = transfer_files
+          local_transfer(files) unless option(:to).blank?
+          ftp_transfer(files) unless option(:to_ftp).blank?
+        end
+
+        private
+
+        def option(key)
+          @options[key]
+        end
+
+        # return an hash with files that need to be transferred {<local file name>: <remote file path>}
+        def transfer_files
+          # unzip if needed
+          files = option(:unzip) && @fullpath =~ /\.zip$/ ? unzip : {@fullpath => @basename}
+          if option(:unzip)
+            # prepend the export type as a subdirectory in the target path
+            files.each_value{ |target| target.insert(0, "#{target[/.*-(.*)\.csv$/, 1]}/")} if option(:sub_dirs)
+            # convert the CSV files
+            files.each_key{ |source| convert_csv(source) } unless [:csv_row_sep, :csv_col_sep, :csv_quote_char, :csv_value_proc].all?{ |csv_option| option(csv_option).blank? }
+          end
+          files
+        end
+
+        # unzip all files to tmp directory
+        def unzip
+          files = {}
+          unzip_dir = @fullpath[0..-5]
+          FileUtils.mkpath(unzip_dir)
+          Zip::File.open(@fullpath) do |zipfile|
+            zipfile.each do |entry|
+              next unless entry.file?
+              full_source_path = "#{unzip_dir}/#{entry.name}"
+              entry.extract(full_source_path) { true } # true for overwrite
+              files[full_source_path] = entry.name
+            end
+          end
+          files
+        end
+
+        # Convert the CSV files using the CSV options
+        def convert_csv(original_csv)
+          csv_options = {}
+          csv_options[:col_sep] = option(:csv_col_sep) unless option(:csv_col_sep).blank?
+          csv_options[:row_sep] = option(:csv_row_sep) unless option(:csv_row_sep).blank?
+          csv_options[:quote_char] = option(:csv_quote_char) unless option(:csv_quote_char).blank?
+          value_converter = option(:csv_value_proc)
+
+          converted_csv = "#{original_csv}.converting"
+          CSV.open(converted_csv, 'wb', csv_options) do |csv|
+            CSV.foreach(original_csv) do |row|
+              row = row.map{ |value| value.blank? ? '' : value_converter.call(value) } if value_converter
+              csv << row
+            end
+          end
+          FileUtils.remove(original_csv)
+          FileUtils.move(converted_csv, original_csv)
+        end
+
+        def local_transfer(files)
+          files.each do |full_source_path, relative_target_path|
+            full_target_path = "#{option(:to)}/#{relative_target_path}"
+            local_copy(full_source_path, full_target_path)
+          end
+          @logger.info { "Copied #{files.size} file(s) from '#{@fullpath}' to '#{option(:to)}'" }
+        end
+
+        # copy a local file and make sure the directories are created
+        def local_copy(source, target)
+          FileUtils.mkpath(File.dirname(target))
+          FileUtils.copy(source, "#{target}.in_progress")
+          FileUtils.move("#{target}.in_progress", target)
+        end
+
+        def ftp_transfer(files)
+          Net::FTP.open(option(:to_ftp), option(:ftp_user_name), option(:ftp_password)) do |ftp|
+            files.each do |full_source_path, relative_target_path|
+              ftp_copy(ftp, full_source_path, "#{option(:to_ftp_dir)}/#{relative_target_path}")
+            end
+          end
+          @logger.info { "Copied #{files.size} file(s) from '#{@fullpath}' to '#{option(:to_ftp)}/#{option(:to_ftp_dir)}'" }
+        end
+
+        # copy a file from the local disk to a remote FTP server
+        # it is possible to use a path in the remote file, e.g. 'dir1/dir2/remote_file.txt'
+        def ftp_copy(ftp, local_file, remote_file)
+          in_ftp_dir(ftp, File.dirname(remote_file)) do |_ftp|
+            basename = File.basename(remote_file)
+            _ftp.putbinaryfile(local_file, "#{basename}.in_progress")
+            _ftp.rename("#{basename}.in_progress", basename)
+          end
+        end
+
+        # move to the given subdirectory (create directories on the fly) and yield
+        # the ftp directory will be reset afterwards
+        def in_ftp_dir(ftp, path, &block)
+          if path.blank? || path == '.'
+            yield ftp
+          else
+            pwd = ftp.pwd
+            begin
+              # move to the directory, ignore / at start or end, and paths with only dots like . and ..
+              path.split('/').reject{|dir| dir.blank? || dir =~ /\.+/}.each do |dir|
+                begin ftp.mkdir(dir); rescue ::Exception => e; end
+                ftp.chdir(dir)
+              end
+              yield ftp
+            ensure
+              ftp.chdir(pwd)
+            end
+          end
+        end
+
+      end
+    end
+  end
+end
+

data/lib/itrp/export/monitor/mail.rb

@@ -0,0 +1,43 @@
+module Itrp
+  module Export
+    module Monitor
+      class Mail
+        def initialize(mail)
+          @mail = mail
+        end
+
+        def original
+          @mail
+        end
+
+        # value of mail header: X-ITRP-ExportID, e.g. 2
+        def export_id
+          @export_id ||= @mail.header['X-ITRP-ExportID'].try(:value).try(:to_i)
+        end
+
+        # value of mail header: X-ITRP-Export, e.g. '0fad4fc0fd4a0130ad2a12313b0e50759969ab71899d2bb1d3e3d8f66e6e5133'
+        def token
+          @token ||= @mail.header['X-ITRP-Export'].try(:value)
+        end
+
+        # First hyperlink in the text, e.g. https://itrp.amazonaws.com/exports/20130911/wdc/20130911-195545-affected_slas.csv?AWSAccessKeyId=AKIA&Signature=du%2B23ZUsrLng%3D&Expires=1379102146
+        def download_uri
+          return nil unless self.export_id
+          # the first match from https:// until a space or the end of the line
+          @download_uri ||= @mail.text_part.body.decoded[/(https?:\/\/[^\s$]+)/, 1]
+        end
+
+        # the filename of the csv or zip file
+        def filename
+          return nil if self.download_uri.blank?
+          @filename ||= self.download_uri[/\/([^\/]+\.(?:csv|zip))\?/, 1]
+        end
+
+        # ignore the message
+        def ignore
+          @mail.skip_deletion
+        end
+      end
+    end
+  end
+end

data/lib/itrp/export/monitor/service.rb

@@ -0,0 +1,153 @@
+require 'fileutils'
+require 'open-uri'
+
+module Itrp
+  module Export
+    module Monitor
+
+      class Service
+        def initialize
+          @failed_exports = Set.new
+          @missing_export_ids = Set.new
+          @options = Itrp::Export::Monitor.configuration.current
+          @options[:ids] = (@options[:ids] || []) + [@options[:id]].flatten.compact.map(&:to_i)
+          [:root, :ids, :imap_user_name, :imap_password].each do |required_option|
+            raise ::Itrp::Exception.new("Missing required configuration option #{required_option}") if option(required_option).blank?
+          end
+          [:sub_dirs, :csv_row_sep, :csv_col_sep, :csv_quote_char, :csv_value_proc].each do |unzip_dependent_option|
+            raise ::Itrp::Exception.new("Configuration option #{unzip_dependent_option} is only available when unzip is true") unless option(unzip_dependent_option).blank?
+          end unless @options[:unzip]
+          raise ::Itrp::Exception.new("Configuration option csv_quote_char must be 1 character long") unless option(:csv_quote_char).blank? || option(:csv_quote_char).length == 1
+          @logger = @options[:logger]
+          create_exit_when_idle_timer
+        end
+
+        # Retrieve an option
+        def option(key)
+          @options[key]
+        end
+
+        def process(mail)
+          mail = Itrp::Export::Monitor::Mail.new(mail)
+          return if @failed_exports.include?(mail.download_uri)
+
+          if option(:ids).include?(mail.export_id)
+            begin
+              kill_exit_when_idle_timer
+              @logger.info { "Processing ITRP Export mail:\n  Subject: #{mail.original.subject}\n  Export ID: #{mail.export_id}\n  Token: #{mail.token}\n  URI: #{mail.download_uri}" }
+              store_export(mail)
+            rescue ::Exception => ex
+              @failed_exports.add(mail.download_uri)
+              @logger.error { "Processing of mail '#{mail.original.subject}' failed: #{ex.message}\n  #{ex.backtrace.join("\n  ")}" }
+              handle_exception(ex, mail)
+              mail.ignore # leave mail in the mailbox
+            ensure
+              create_exit_when_idle_timer
+            end
+          else
+            mail.ignore # leave mail in the mailbox
+            unless @missing_export_ids.include?(mail.export_id)
+              @missing_export_ids.add(mail.export_id)
+              @logger.info { mail.export_id ? "Skipping mail. ITRP Export ID #{mail.export_id} not configured for monitoring" : "Skipping mail. Not an ITRP Export mail: #{mail.original.subject}" }
+            end
+          end
+        end
+
+        # Generate a clacks config file based on the export config
+        def generate_clacks_config
+          clacks_config_filename = "#{dir(:tmp)}/clacks_config.#{monitor_id}.rb"
+          log_device = @logger.instance_variable_get(:@logdev)
+          log_file = log_device.respond_to?(:filename) && log_device.filename ? File.expand_path(log_device.filename) : "#{dir(:log)}/#{monitor_id}.log"
+          File.open(clacks_config_filename, 'w') do |clacks_config|
+            clacks_config.write(<<EOF)
+# -- DO NOT EDIT --
+# Generated by the Export Monitor
+
+pid "#{dir(:pids)}/#{monitor_id}.pid"
+stdout_path "#{log_file}"
+stderr_path "#{log_file}"
+
+imap({
+  address:    '#{option(:imap_address)}',
+  port:       #{option(:imap_port)},
+  user_name:  '#{option(:imap_user_name)}',
+  password:   '#{option(:imap_password)}',
+  enable_ssl: #{option(:imap_ssl)}
+})
+
+find_options({
+  mailbox:           '#{option(:imap_mailbox)}',
+  archivebox:        '#{option(:imap_archive)}',
+  keys:              'FROM ITRP HEADER X-ITRP-ExportID ""',
+  delete_after_find: true # Note that only the processed export mails will be deleted
+})
+
+on_mail do |mail|
+  Itrp::Export::Monitor.process(mail)
+end
+EOF
+          end
+          clacks_config_filename
+        end
+
+        private
+
+        def store_export(mail)
+          # download export file to the downloads directory
+          export_file_name = download_export(mail)
+          # and tranfer the files
+          Itrp::Export::Monitor::Exchange.new(export_file_name, @options).transfer
+        end
+
+        def download_export(mail)
+          local_filename = "#{dir(:downloads)}/#{mail.filename}"
+          File.open(local_filename, 'wb') { |f| f.write(open(mail.download_uri).read) }
+          local_filename
+        end
+
+        def dir(subdir)
+          directory = File.expand_path(subdir.to_s, option(:root))
+          FileUtils.mkpath(directory)
+          directory
+        end
+
+        def monitor_id
+          @monitor_id ||= "export_monitor.#{option(:ids).map(&:to_s).join('.')}"
+        end
+
+        def handle_exception(ex, mail)
+          proc = option(:on_exception)
+          if proc
+            begin
+              proc.call(ex, mail)
+            rescue ::Exception => another_exception
+              @logger.error { "Exception occurred in exception handling: #{another_exception.message}\n  #{another_exception.backtrace.join("\n  ")}" }
+            end
+          end
+        end
+
+        def kill_exit_when_idle_timer
+          @exit_when_idle_timer.try(:kill)
+          @exit_when_idle_timer = nil
+        end
+
+        # reset timer to exit when the export monitor has been idle for a given nr of minutes
+        def create_exit_when_idle_timer
+          timeout_in_seconds = option(:exit_when_idle).to_i * 60
+          return unless timeout_in_seconds > 0
+          begin
+            kill_exit_when_idle_timer # make sure only 1 timer thread is running at a time
+            @exit_when_idle_timer = Thread.new do
+              sleep timeout_in_seconds
+              stop_signal = (Signal.list.keys & ['QUIT', 'INT']).first
+              Process.kill(stop_signal, Process.pid)
+            end
+          rescue ::Exception => ex
+            @logger.error { "Unable to schedule timer to exit when idle in #{timeout_in_seconds} seconds: #{ex.message}\n  #{ex.backtrace.join("\n  ")}" }
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/itrp/export/monitor/tasks/itrp_export_monitor.rake

@@ -0,0 +1,67 @@
+namespace :itrp_export_monitor do
+
+  GENERATE_HELP = %(Generate a ruby file to configure and start the ITRP Export Monitor:\n  itrp-export-monitor generate[<export id>,<email address>,<IMAP password>])
+  desc GENERATE_HELP
+  task :generate, [:export_id, :email_address, :imap_password] do |t, args|
+    check_required_args(args, [:export_id, :email_address, :imap_password], GENERATE_HELP)
+    monitor_name = "export_monitor.#{args.export_id}"
+    File.open("#{Dir.pwd}/#{monitor_name}.rb", 'w'){ |f| f.write(<<EOF) }
+require 'itrp/export/monitor'
+
+# the location where all the run-time information on the export monitor is stored
+BASE_DIR = "#{Dir.pwd}/#{monitor_name.gsub('.', '_')}"
+FileUtils.mkpath "\#{BASE_DIR}/log"
+
+Itrp::Export::Monitor.configure do |export|
+  export.root =       BASE_DIR
+  export.logger =     Logger.new("\#{BASE_DIR}/log/#{monitor_name}.log")
+  export.ids =        [#{args.export_id}]
+  export.unzip =      true
+  export.sub_dirs =   false
+
+  export.to = '/tmp/exports'
+  # export.to_ftp =        'ftp.mycompany.com'
+  # export.to_ftp_dir =    'my/exports'
+  # export.ftp_user_name = 'user'
+  # export.ftp_password =  'secret'
+
+  export.imap_address =    'imap.googlemail.com'
+  export.imap_port =       993
+  export.imap_user_name =  '#{args.email_address}'
+  export.imap_password =   '#{args.imap_password}'
+  export.imap_ssl =        true
+  export.imap_mailbox =    'INBOX'
+  export.imap_archive =    '[Gmail]/All Mail'
+end
+
+Itrp::Export::Monitor.run
+EOF
+    $stdout.puts "\nGenerated '#{Dir.pwd}/#{monitor_name}.rb'."
+    $stdout.puts "\nEdit the file and:"
+    $stdout.puts " - fill in the IMAP details to connect to the mailbox that receives the ITRP Export mails"
+    $stdout.puts " - specify the directory or FTP server to sent the export files to"
+    $stdout.puts "\nStart the export monitor as follows:"
+    $stdout.puts " $ bundle exec ruby #{monitor_name}.rb"
+    $stdout.puts "\nFor more information and all available options look at the itrp-export-monitor gem documentation online.\n\n"
+  end
+
+  #desc %(List all scheduled exports:\n  itrp-export-monitor list['<api-token>'])
+  #task :list, [:api_token] do |t, args|
+  #  check_required_args(args, [:api_token])
+  #  $stdout.puts "Searching for scheduled exports..."
+  #  Itrp::Client.new(api_token: args.api_token).each('/exports')
+  #  $stdout.puts "Account #{args.name} created successfully. The login instructions for #{account.url} are mailed to #{args.primary_email}."
+  #end
+
+  # Helper methods
+
+  # test if required arguments are provided
+  def check_required_args(args, fields, help)
+    missing = fields.select{|field| args.send(field) == nil || args.send(field) == '' }
+    unless missing.empty?
+      $stderr.puts "\n#{help}"
+      $stderr.puts "\nMissing required arguments: #{missing.join(', ')}"
+      exit(1)
+    end
+  end
+end

data/lib/itrp/export/monitor/version.rb

@@ -0,0 +1,7 @@
+module Itrp
+  module Export
+    module Monitor
+      VERSION = "1.0.0"
+    end
+  end
+end

metadata

@@ -0,0 +1,213 @@
+--- !ruby/object:Gem::Specification
+name: itrp-export-monitor
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- ITRP
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: gem_config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: itrp-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: active_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: clacks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Monitor a mailbox and store Scheduled ITRP Exports to disk or FTP.
+email: developers@itrp.com
+executables:
+- itrp-export-monitor
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/itrp/export/monitor.rb
+- lib/itrp/export/monitor/service.rb
+- lib/itrp/export/monitor/version.rb
+- lib/itrp/export/monitor/mail.rb
+- lib/itrp/export/monitor/tasks/itrp_export_monitor.rake
+- lib/itrp/export/monitor/exchange.rb
+- LICENSE.txt
+- README.md
+- Gemfile
+- Gemfile.lock
+- itrp-export-monitor.gemspec
+- bin/itrp-export-monitor
+homepage: https://github.com/itrp/itrp-export-monitor
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: The itrp-export-monitor gem makes it easy to monitor a mailbox receiving
+  Scheduled Exports from ITRP and to store the incoming export files on disk or forward
+  it to an FTP server.
+test_files: []