hyperflow-amqp-executor 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: aa8e10ac439e7a475bd1a019a7bf9b911238a648
+  data.tar.gz: 50c7432942f8e4c29ecd3039ceb54c2a81ccbfbc
+SHA512:
+  metadata.gz: 3aa7281cea2f3ff3e7dd9f164d4fe27bf798e85ce706c107c9fcf662733c0789b9ef03484597ef656669d1405de1cdb31281e0ebecdd9639258f4ad8769d6bf7
+  data.tar.gz: c46df737e23a3acca22bac4519b72791b9560fe9321cac81abaa672fa265615ea1b2b845a8f50863140a359c03573eae84eefe88f1c67dcd13092300eb9c9996

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,21 @@
+source "https://rubygems.org"
+# Add dependencies required to use your gem here.
+
+#ruby "2.1.0"
+
+gem 'fog', "~> 1.18"
+gem 'unf', "~> 0.1" # for AWS fog 
+gem 'recursive-open-struct', "~> 0.4"
+gem 'amqp', "~> 1.1"
+gem 'deep_merge', "~> 1.0"
+gem 'httpclient', "~> 2.6"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0"
+  gem "jeweler", "~> 1.8"
+  gem "pry"
+end

data/Gemfile.lock

@@ -0,0 +1,110 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.3.8)
+    amq-protocol (1.9.2)
+    amqp (1.5.0)
+      amq-protocol (>= 1.9.2)
+      eventmachine
+    builder (3.2.2)
+    coderay (1.1.0)
+    deep_merge (1.0.1)
+    eventmachine (1.0.7)
+    excon (0.31.0)
+    faraday (0.8.9)
+      multipart-post (~> 1.2.0)
+    fog (1.19.0)
+      builder
+      excon (~> 0.31.0)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5)
+      ruby-hmac
+    formatador (0.2.5)
+    git (1.2.9.1)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
+      oauth2
+    hashie (3.4.1)
+    highline (1.7.2)
+    httpclient (2.6.0.1)
+    i18n (0.7.0)
+    jeweler (1.8.8)
+      builder
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      github_api (= 0.10.1)
+      highline (>= 1.6.15)
+      nokogiri (= 1.5.10)
+      rake
+      rdoc
+    json (1.8.2)
+    jwt (1.4.1)
+    method_source (0.8.2)
+    mime-types (2.5)
+    minitest (5.6.1)
+    multi_json (1.11.0)
+    multi_xml (0.5.5)
+    multipart-post (1.2.0)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    nokogiri (1.5.10)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (1.6.0)
+    rake (10.4.2)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    recursive-open-struct (0.6.3)
+    ruby-hmac (0.4.0)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.2.1)
+    shoulda-matchers (2.8.0)
+      activesupport (>= 3.0.0)
+    slop (3.6.0)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  amqp (~> 1.1)
+  bundler (~> 1.0)
+  deep_merge (~> 1.0)
+  fog (~> 1.18)
+  httpclient (~> 2.6)
+  jeweler (~> 1.8)
+  pry
+  rdoc (~> 3.12)
+  recursive-open-struct (~> 0.4)
+  shoulda
+  unf (~> 0.1)

data/LICENSE.txt

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2013-2015 Kamil Figiela (kfigiela@agh.edu.pl)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,119 @@
+# hyperflow-amqp-executor
+
+Job executor for [Hyperflow](http://github.com/dice-cyfronet/hyperflow) workflow engine.
+
+## Usage
+
+Executor may be configured in two ways:
+
+ * by using YAML configuration file (see `examples/settings.yml`) - path to that file should be passed as only argument
+ * via environment variables:
+   * `STORAGE` = cloud | nfs | local | plgdata | gridftp, defaults to `cloud` (Amazon S3)
+   * `AWS_ACCESS_KEY_ID` (for S3 cloud storage)
+   * `AWS_SECRET_ACCESS_KEY` (for S3 cloud storage)
+   * `AMQP_URL` – address of AMQP queue
+   * `AMQP_QUEUE` – name of AMQP queue (defaults to `hyperflow.jobs`)
+   * `THREADS` (defaults to number of cores or 1 if it couldn't be determined)
+   * `X509_USER_PROXY` – user proxy certificate for PLGData and GridFTP storage
+   * `PLGDATA_ENDPOINT` – endpoint of PLGData service (defaults to: `https://data.plgrid.pl`)
+
+To execute jobs:
+  
+`$ hyperflow-amqp-executor`
+
+To collect some metrics:
+  
+`$ hyperflow-amqp-metric-collector`
+
+## Supported data storage services
+
+### cloud: Cloud object data storage service
+
+In this scenario, input and output files of jobs are stored in cloud object data storage service such as [Amazon S3](http://aws.amazon.com/s3/). In that case executor does the follwing:
+
+* Create temporary directory
+* Download input files
+* Execute job in temporary directory
+* Upload output files
+* Remove temporary directory (anyway, app should remove temporary files itself)
+
+Each task needs to provide some options:
+
+* `s3_bucket` - name of Amazon S3 bucket,
+* `s3_prefix` - path prefix in bucket,
+* *(optionally)* cloud_storage – same hash as in config file to use other than default storage provider.
+
+### nfs: (Slow) network file system
+
+This case is similar to previous one, but executor will copy files from and copy results back to locally available filesystem. It may be NFS, SSHFS or other file system where working directly on remote data is not recommended.
+
+Job options:
+
+* `workdir` - working directory (files will be copied to local temporary workdir for task processing).
+
+### local: Local/network file system
+
+In this scenario we assume that job is executed in shared directory available on execution node. It may be NFS/SMB/etc. share or local disk for single node deployments. There is no stage-in or stage-out phase, job is executed directly in specified directory, so that job must not leave any temporary files.
+
+Job options:
+
+* `workdir` - working directory (tasks will be executed in this working directory).
+
+### plgdata: PL-Grid Data
+
+Behaves like *cloud* storage, but uses [PL-Grid Data](https://data.plgrid.pl) service as a backend. It requires path of user proxy certificate in `$X509_USER_PROXY`. Does not have any external requirements, however in PLGData file uploads are limited to 256 MiB per file.
+
+Each task needs to provide some options:
+
+* `prefix` – path of working directory on the storage infrastructure (probably something like /people/plgyourlogin/workflowdir).
+
+### gridftp: GridFTP
+
+Behaves like *cloud* storage, but uses [GridFTP](https://www.globus.org/toolkit/docs/latest-stable/gridftp/) service as a backend. It requires path of user proxy certificate in `$X509_USER_PROXY`. Requires `globus-url-copy` which is part of Globus Toolkit installed.
+
+Each task needs to provide some options:
+
+* `prefix` – path of working directory on the storage infrastructure (probably something like gsiftp://example.com/people/plgyourlogin/workflowdir).
+
+## Execution event monitoring
+
+Executor publishes events for monitoring purpose by `hyperflow.events` exchange created on AMQP server. The exchange is topic type, so one may subscribe for specific event type. To request all messages request wildcard routing key `#`. Routing keys are as follows:
+
+* `executor.ready`
+* `job.{job-id}.started`
+* `job.{job-id}.finished`
+* `job.{job-id}.stage-in.started` - not applicable for `local` storage
+* `job.{job-id}.stage-in.finished` - not applicable for `local` storage
+* `job.{job-id}.execution.started`
+* `job.{job-id}.execution.finished`
+* `job.{job-id}.stage-out.started` - not applicable for `local` storage
+* `job.{job-id}.stage-out.finished` - not applicable for `local` storage
+
+Each event published in JSON provides:
+
+* `executor`: UUID executor id 
+* `timestamp`: UNIX timestamp (float in UTC timezone)
+* `type`: Like routing key, but without job id part.
+
+Events related to jobs also are provided with:
+
+* `id`: Job id (AMQP correlation id)
+* `thread`: Executor thread id (random looking number)
+
+Stage finish events send some additional info: `job.*.stage-in.finished` and `job.*.stage-out.finished` provide `time` and `bytes` for transfer time and data size respectively. The event of `job.*.execution.finished` provides `executable` name, `metrics` and `exit_status`.
+
+
+## Contributing to hyperflow-amqp-executor
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+Copyright © 2013-2015 Kamil Figiela (kfigiela@agh.edu.pl). Distributed under MIT License. See LICENSE.txt for further details.
+

data/Rakefile

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "hyperflow-amqp-executor"
+  gem.homepage = "http://github.com/kfigiela/hyperflow-amqp-executor"
+  gem.license = "MIT"
+  gem.summary = %Q{AMQP job executor for Hyperflow workflow engine}
+  gem.description = %Q{AMQP job executor for Hyperflow workflow engine (http://github.com/dice-cyfronet/hyperflow)}
+  gem.email = "kamil.figiela@gmail.com"
+  gem.authors = ["Kamil Figiela"]
+  gem.executables = %w{hyperflow-amqp-executor hyperflow-amqp-metric-collector}
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+# require 'rake/testtask'
+# Rake::TestTask.new(:test) do |test|
+#   test.libs << 'lib' << 'test'
+#   test.pattern = 'test/**/test_*.rb'
+#   test.verbose = true
+# end
+#
+#
+# task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "hyperflow-amqp-executor #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/bin/hyperflow-amqp-executor

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+require_relative '../lib/hyperflow-amqp-executor'
+
+include Executor
+
+task_queue_name = (ENV['AMQP_QUEUE'] or "hyperflow.jobs")
+
+Executor::id = SecureRandom.uuid
+Executor::logger.info "Starting worker #{Executor.id}"
+
+Executor::settings = Executor::Settings.load(ARGV.first)
+
+Executor::logger.info "Running #{Executor::settings.threads} worker threads"
+EM.threadpool_size = Executor::settings.threads
+
+EM.run do
+  AMQP.connect(Executor::settings.amqp_url) do |connection|
+    Executor::logger.info "Connected to AMQP broker... "
+
+    channel  = AMQP::Channel.new(connection)
+    channel.prefetch(Executor::settings.threads)
+
+    Executor::events_exchange = channel.topic('hyperflow.events', durable: true)
+    queue = channel.queue(task_queue_name,   durable: true)
+
+    queue.subscribe(ack: true) do |header, payload|
+      begin
+        job_data = RecursiveOpenStruct.new(JSON.parse(payload), recurse_over_arrays: true)
+        job = Job.new(header.correlation_id, job_data)
+        op = -> {
+          begin
+            job.run
+          rescue Exception => e
+            Executor::logger.error "[#{header.correlation_id}] Error running job: #{e}"
+            Executor::logger.debug "[#{header.correlation_id}] Backtrace\n#{e.backtrace.join("\n")}"
+            {exit_status: -2, exceptions: [e]}
+          end
+        }
+        cb = ->(output){
+          channel.default_exchange.publish(JSON.dump(output), content_type: 'application/json', routing_key: header.reply_to, correlation_id: header.correlation_id, mandatory: true)
+          header.ack
+        }
+        EM.defer(op, cb)
+      rescue JSON::ParserError
+        Executor::logger.error "[#{header.correlation_id}] Invalid JSON"
+        header.ack
+      end
+    end
+
+    Executor::publish_event('executor.ready', 'executor.ready', hostname: `hostname -f`)
+
+    Signal.trap("INT") {
+      connection.close { EM.stop }
+    }
+  end
+end

data/bin/hyperflow-amqp-metric-collector

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require "amqp"
+require "recursive-open-struct"
+require "json"
+
+EM.run do
+  AMQP.connect(ENV['AMQP_URL']) do |connection|
+    warn "Connected to AMQP broker..."
+
+    channel  = AMQP::Channel.new(connection)
+    queue    = channel.queue("", auto_delete: true, durable: false)
+    queue.bind("hyperflow.events", routing_key: "job.*.finished")
+
+    queue.subscribe do |payload|
+      data = RecursiveOpenStruct.new(JSON.parse(payload))
+      if data.metrics
+        puts [Time.now,
+              data.executor,
+              data.thread,
+              data.job,
+              data.executable,
+              data.metrics.execution,
+              # data.metrics.stage_in,
+              # data.metrics.stage_out,
+              data.metrics.input_size,
+              data.metrics.output_size,
+              # data.metrics.timestamps['job.started'],
+              # data.metrics.timestamps['stage_in.started'],
+              # data.metrics.timestamps['stage_in.finished'],
+              # data.metrics.timestamps['execution.started'],
+              # data.metrics.timestamps['execution.finished'],
+              # data.metrics.timestamps['stage_out.started'],
+              # data.metrics.timestamps['stage_out.finished'],
+              # data.metrics.timestamps['job.finished']
+           ].map {|v| if v.nil? then "NA" else v end }.join("\t")
+      else
+        warn payload
+      end
+    end
+    Signal.trap("INT") { connection.close { EM.stop } }
+  end
+end

data/example/settings.yml

@@ -0,0 +1,19 @@
+# RabbitMQ URL, defaults to ENV['AMQP_URL']
+amqp_url: <%= ENV['AMQP_URL'] %>
+
+# Execution storage scenario: cloud | nfs | local | plgdata
+# This setting may be overriden for each task
+storage: <%= ENV['STORAGE'] || 'cloud' %> 
+
+# Number of execution threads (defaults to: ENV['THREADS'], `nprocs` if available or 1)
+threads: <%= Executor::cpu_count %>
+
+# Cloud storage settings, the following hash is passed directyl to Fog::Storage, see http://fog.io/storage/ for further reference
+# Defaults to AWS S3 with credentials passed in ENV
+cloud_storage:
+  provider: AWS
+  aws_access_key_id:        <%= ENV['AWS_ACCESS_KEY_ID'] %>
+  aws_secret_access_key:    <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
+
+plgdata:
+  proxy: <%= ENV['X509_USER_PROXY'] or "./grid_proxy "%>

data/example/task.json

@@ -0,0 +1,7 @@
+{
+  "executable": "/usr/bin/sort",
+  "args": ["-o", "outfile", "infile"],
+  "inputs": [{"name": "infile"}],
+  "outputs": [{"name": "outfile"}],
+  "options": {"prefix": "/people/plgkfigiela/data/"}
+}

data/hyperflow-amqp-executor.gemspec

@@ -0,0 +1,94 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: hyperflow-amqp-executor 1.0.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "hyperflow-amqp-executor"
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["Kamil Figiela"]
+  s.date = "2015-04-30"
+  s.description = "AMQP job executor for Hyperflow workflow engine (http://github.com/dice-cyfronet/hyperflow)"
+  s.email = "kamil.figiela@gmail.com"
+  s.executables = ["hyperflow-amqp-executor", "hyperflow-amqp-metric-collector"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "bin/hyperflow-amqp-executor",
+    "bin/hyperflow-amqp-metric-collector",
+    "example/settings.yml",
+    "example/task.json",
+    "hyperflow-amqp-executor.gemspec",
+    "lib/hyperflow-amqp-executor.rb",
+    "lib/hyperflow-amqp-executor/cloud_storage.rb",
+    "lib/hyperflow-amqp-executor/gridftp_storage.rb",
+    "lib/hyperflow-amqp-executor/helpers.rb",
+    "lib/hyperflow-amqp-executor/job.rb",
+    "lib/hyperflow-amqp-executor/local_storage.rb",
+    "lib/hyperflow-amqp-executor/nfs_storage.rb",
+    "lib/hyperflow-amqp-executor/plgdata_storage.rb",
+    "lib/hyperflow-amqp-executor/settings.rb",
+    "test/helper.rb",
+    "test/test_hyperflow-amqp-executor.rb"
+  ]
+  s.homepage = "http://github.com/kfigiela/hyperflow-amqp-executor"
+  s.licenses = ["MIT"]
+  s.rubygems_version = "2.2.2"
+  s.summary = "AMQP job executor for Hyperflow workflow engine"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<fog>, ["~> 1.18"])
+      s.add_runtime_dependency(%q<unf>, ["~> 0.1"])
+      s.add_runtime_dependency(%q<recursive-open-struct>, ["~> 0.4"])
+      s.add_runtime_dependency(%q<amqp>, ["~> 1.1"])
+      s.add_runtime_dependency(%q<deep_merge>, ["~> 1.0"])
+      s.add_runtime_dependency(%q<httpclient>, ["~> 2.6"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8"])
+      s.add_development_dependency(%q<pry>, [">= 0"])
+    else
+      s.add_dependency(%q<fog>, ["~> 1.18"])
+      s.add_dependency(%q<unf>, ["~> 0.1"])
+      s.add_dependency(%q<recursive-open-struct>, ["~> 0.4"])
+      s.add_dependency(%q<amqp>, ["~> 1.1"])
+      s.add_dependency(%q<deep_merge>, ["~> 1.0"])
+      s.add_dependency(%q<httpclient>, ["~> 2.6"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8"])
+      s.add_dependency(%q<pry>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<fog>, ["~> 1.18"])
+    s.add_dependency(%q<unf>, ["~> 0.1"])
+    s.add_dependency(%q<recursive-open-struct>, ["~> 0.4"])
+    s.add_dependency(%q<amqp>, ["~> 1.1"])
+    s.add_dependency(%q<deep_merge>, ["~> 1.0"])
+    s.add_dependency(%q<httpclient>, ["~> 2.6"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8"])
+    s.add_dependency(%q<pry>, [">= 0"])
+  end
+end
+

data/lib/hyperflow-amqp-executor.rb

@@ -0,0 +1,49 @@
+require 'amqp'
+require 'json'
+require 'recursive-open-struct'
+require 'open3'
+require 'tmpdir'
+require 'logger'
+
+require_relative 'hyperflow-amqp-executor/helpers'
+require_relative 'hyperflow-amqp-executor/job'
+require_relative 'hyperflow-amqp-executor/local_storage'
+require_relative 'hyperflow-amqp-executor/cloud_storage'
+require_relative 'hyperflow-amqp-executor/nfs_storage'
+require_relative 'hyperflow-amqp-executor/plgdata_storage'
+require_relative 'hyperflow-amqp-executor/gridftp_storage'
+require_relative 'hyperflow-amqp-executor/settings'
+
+module Executor
+  class << self
+    attr_accessor :events_exchange, :id, :settings
+
+    def logger
+      @logger ||= Logger.new($stdout)
+    end
+
+    def cpu_count
+      unless ENV['THREADS'].nil?
+        ENV['THREADS']
+      else
+        begin
+          `nproc`
+        rescue
+          1
+        end
+      end.to_i
+    end
+
+    def publish_event(type, routing_key, payload = {})
+      data = payload
+      data['timestamp'] = Time.now.utc.to_f
+      data['type']      = type
+      data['executor']  = @id
+      EM.next_tick do
+        logger.debug "Publishing event #{type}"
+        @events_exchange.publish(JSON.dump(data), content_type: 'application/json', routing_key: routing_key)
+      end
+      data['timestamp']
+    end
+  end
+end

data/lib/hyperflow-amqp-executor/cloud_storage.rb

@@ -0,0 +1,34 @@
+require 'fog'
+
+module Executor
+  module CloudStorage
+    def storage_init
+      @provider = Fog::Storage.new(@job.options.cloud_storage || Executor::settings.cloud_storage.to_h)
+    end
+
+    def stage_in
+      @bucket = @provider.directories.get(@job.options.bucket)
+
+      @job.inputs.each do |file|
+        Executor::logger.debug "[#{@id}] Downloading #{file.name}"
+        File.open(@workdir+"/"+file.name, File::RDWR|File::CREAT) do |local_file|
+          @bucket.files.get(@job.options.prefix+file.name) do |chunk, remaining_bytes, total_bytes|
+            local_file.write(chunk)
+            # print "\rDownloading #{file.name}: #{100*(total_bytes-remaining_bytes)/total_bytes}%"
+          end
+        end
+      end
+    end
+
+    def stage_out
+      @job.outputs.each do |file|
+        Executor::logger.debug "[#{@id}] Uploading #{file.name}"
+        @bucket.files.create(key: @job.options.prefix+file.name, body: File.open(@workdir+"/"+file.name))
+      end
+    end
+
+    def workdir(&block)
+      Dir::mktmpdir(&block)
+    end
+  end
+end

data/lib/hyperflow-amqp-executor/gridftp_storage.rb

@@ -0,0 +1,39 @@
+require 'pry'
+module Executor
+  module GridFTPStorage
+    def storage_init
+      raise Exception, "Unable to locate user proxy certificate" if Executor::settings.plgdata.proxy.nil? or !File.exists?(Executor::settings.plgdata.proxy)
+      @proxy_file = Executor::settings.gridftp.proxy
+    end
+
+    def stage_in
+      @job.inputs.each do |file|
+          local_file_name = @workdir + "/" + file.name
+          url = @job.options.prefix + "/" + file.name
+          
+          Executor::logger.debug "[#{@id}] Downloading #{url} to #{local_file_name}"
+          stdout, stderr, status = Open3.capture3({'X509_USER_PROXY' => @proxy_file}, 'globus-url-copy', url, local_file_name, chdir: @workdir)
+          unless status == 0
+            raise Exception, "Failed downloading input from GridFTP, status: #{status}\nstdout:\n#{stdout}\n\n stderr:\n#{stderr}"
+          end
+      end
+    end
+
+    def stage_out
+      @job.outputs.each do |file|
+        local_file_name = @workdir + "/" + file.name
+        url = @job.options.prefix + "/" + file.name
+
+        Executor::logger.debug "[#{@id}] Uploading #{file.name} to #{url}"
+        stdout, stderr, status = Open3.capture3({'X509_USER_PROXY' => @proxy_file}, 'globus-url-copy', local_file_name, url, chdir: @workdir)
+        unless status == 0
+          raise Exception, "Failed uploading input from GridFTP, status: #{status}\nstdout:\n#{stdout}\n\n stderr:\n#{stderr}"
+        end
+      end
+    end
+
+    def workdir(&block)
+      Dir::mktmpdir(&block)
+    end
+  end
+end

data/lib/hyperflow-amqp-executor/helpers.rb

@@ -0,0 +1,6 @@
+def time
+  start = Time.now
+  result = yield
+  t = Time.now - start
+  [result, t]
+end

data/lib/hyperflow-amqp-executor/job.rb

@@ -0,0 +1,110 @@
+module Executor
+  class Job
+    attr_reader :metrics
+    
+    def initialize(id, job)
+      @job = job
+      @id = id
+      @metrics = {
+              timestamps: { },
+              executor: Executor::id
+            }
+        
+      storage_module = case (@job.options.storage or Executor::settings.storage)
+      when 's3', 'cloud'
+        CloudStorage
+      when 'local'
+        LocalStorage
+      when 'nfs'
+        NFSStorage
+      when 'plgdata'
+        PLGDataStorage
+      when 'gridftp'
+        GridFTPStorage
+      else
+        raise "Unknown storage #{@job.storage}"
+      end
+      self.extend(storage_module)
+    end
+
+    def run
+      @metrics[:timestamps]["job.started"] = Executor::publish_event 'job.started', "job.#{@id}.started", job: @id, thread: Thread.current.__id__
+      @metrics[:thread] = Thread.current.__id__
+
+      results = {}
+      
+      workdir do |tmpdir|
+        @workdir = tmpdir
+        raise "Couldn't get workdir" unless @workdir
+
+        storage_init if self.respond_to? :storage_init
+
+        if self.respond_to? :stage_in
+          publish_events "stage_in" do
+            _ , @metrics[:stage_in]     = time { stage_in }
+            @metrics[:input_size]       = input_size
+            {bytes: @metrics[:input_size], time: @metrics[:stage_in]}
+          end
+        else
+          @metrics[:input_size] = input_size
+        end
+
+        publish_events "execution" do
+          results, @metrics[:execution] = time { execute }
+          { executable: @job.executable, exit_status: results[:exit_status], time: @metrics[:execution] }
+        end
+
+        if self.respond_to? :stage_out
+          publish_events "stage_out" do
+            _, @metrics[:stage_out]     = time { stage_out }
+            @metrics[:output_size]      = output_size
+            { bytes: @metrics[:output_size], time: @metrics[:stage_out] }
+          end
+        else
+          @metrics[:output_size] = output_size
+        end
+
+      end
+      @metrics[:timestamps]["job.finished"] = Executor::publish_event 'job.finished', "job.#{@id}.finished", job: @id, executable: @job.executable, exit_status: results[:exit_status], metrics: @metrics, thread: Thread.current.__id__
+
+      results[:metrics] = @metrics
+      results
+    end
+
+    def publish_events(name)
+      @metrics[:timestamps]["#{name}.started"]  = Executor::publish_event "job.#{name}.started", "job.#{@id}.#{name}.started", job: @id, thread: Thread.current.__id__
+      results = yield
+      @metrics[:timestamps]["#{name}.finished"] = Executor::publish_event "job.#{name}.finished", "job.#{@id}.#{name}.finished", {job: @id, thread: Thread.current.__id__}.merge(results || {})
+      results
+    end
+
+    def cmdline
+       if @job.args.is_a? Array
+         ([@job.executable] + @job.args).map { |e| e.to_s }
+      else
+        "#{@job.executable} #{@job.args}"
+      end
+    end
+
+    def execute
+      begin
+        Executor::logger.debug "[#{@id}] Executing #{cmdline}"
+        stdout, stderr, status = Open3.capture3(*cmdline, chdir: @workdir)
+
+        {exit_status: status, stderr: stderr, stdout: stdout}
+      rescue Exception => e
+        Executor::logger.error "[#{@id}] Error executing job: #{e}"
+        Executor::logger.debug "[#{@id}] Backtrace\n#{e.backtrace.join("\n")}"
+        {exit_status: -1, exceptions: [e]}
+      end
+    end
+
+    def input_size
+      @job.inputs.map{ |file| begin File.size(@workdir+"/"+file.name) rescue 0 end }.reduce(:+) or 0
+    end
+
+    def output_size
+      @job.outputs.map{ |file| begin File.size(@workdir+"/"+file.name) rescue 0 end }.reduce(:+) or 0
+    end
+  end
+end

data/lib/hyperflow-amqp-executor/local_storage.rb

@@ -0,0 +1,8 @@
+module Executor
+  module LocalStorage
+    def workdir
+      yield @job.options.workdir
+    end
+  end
+end
+

data/lib/hyperflow-amqp-executor/nfs_storage.rb

@@ -0,0 +1,23 @@
+require 'fog'
+
+module Executor
+  module NFSStorage
+    def stage_in
+      @job.inputs.each do |file|
+        Executor::logger.debug "[#{@id}] Copying #{file.name} to tmpdir"
+        FileUtils.copy(@job.options.workdir + file.name, @workdir + "/" + file.name)
+      end
+    end
+
+    def stage_out
+      @job.outputs.each do |file|
+        Executor::logger.debug "[#{@id}] Copying #{file.name} from tmpdir"
+        FileUtils.copy(@workdir + "/" + file.name, @job.options.workdir + file.name)
+      end
+    end
+
+    def workdir(&block)
+      Dir::mktmpdir(&block)
+    end
+  end
+end

data/lib/hyperflow-amqp-executor/plgdata_storage.rb

@@ -0,0 +1,47 @@
+require 'httpclient'
+
+module Executor
+  module PLGDataStorage
+    PLGDATA_ENDPOINT = (ENV['PLGDATA_ENDPOINT'] or 'https://data.plgrid.pl')
+    def storage_init
+      @http_client = HTTPClient.new()
+      
+      raise Exception, "Unable to load proxy certificate" unless File.exists?(Executor::settings.plgdata.proxy)
+      @proxy_string = File.read(Executor::settings.plgdata.proxy)
+    end
+
+    def stage_in
+      @job.inputs.each do |file|
+          url = PLGDATA_ENDPOINT+'/download/' + @job.options.prefix + "/" + file.name
+          local_file_name = @workdir + "/" + file.name
+
+          Executor::logger.debug "[#{@id}] Downloading #{url} to #{local_file_name}"
+          File.open(local_file_name, File::RDWR|File::CREAT) do |local_file|
+            payload = {proxy: @proxy_string}
+            response = @http_client.get(url, payload) do |chunk|
+              local_file.write(chunk)
+            end
+            raise Exception, "Failed downloading input file" unless response.ok?
+          end
+      end
+    end
+
+    def stage_out
+      @job.outputs.each do |file|
+        url = PLGDATA_ENDPOINT+'/upload/' + @job.options.prefix + "/" + File.dirname(file.name)
+        local_file_name = @workdir+"/"+file.name
+
+        Executor::logger.debug "[#{@id}] Uploading #{file.name} to #{url}"
+        File.open(local_file_name) do |local_file|
+          payload = {proxy: @proxy_string, file: local_file}
+          response = @http_client.post(url, payload)
+          raise Exception, "Failed uploading output file: #{response.content}" unless response.ok?
+        end
+      end
+    end
+
+    def workdir(&block)
+      Dir::mktmpdir(&block)
+    end
+  end
+end

data/lib/hyperflow-amqp-executor/settings.rb

@@ -0,0 +1,34 @@
+require 'deep_merge'
+module Executor
+  class Settings
+    class << self
+      def load(file=nil)
+        settings = defaults
+        unless file.nil?
+          file_settings = YAML.load(ERB.new(File.read(file)).result)
+          settings.deep_merge! file_settings
+        end
+        RecursiveOpenStruct.new(settings)
+      end
+    
+      def defaults
+        {
+          amqp_url: ENV['AMQP_URL'],
+          storage: 'cloud',
+          threads: Executor::cpu_count,
+          cloud_storage: {
+            provider: "AWS",
+            aws_access_key_id:      ENV['AWS_ACCESS_KEY_ID'],
+            aws_secret_access_key:  ENV['AWS_SECRET_ACCESS_KEY']
+          },
+          plgdata: {
+            proxy: ENV['X509_USER_PROXY']
+          },
+          gridftp: {
+            proxy: ENV['X509_USER_PROXY']
+          }
+        }
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'hyperflow-amqp-executor'
+
+class Test::Unit::TestCase
+end

data/test/test_hyperflow-amqp-executor.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestHyperflowAmqpExecutor < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,224 @@
+--- !ruby/object:Gem::Specification
+name: hyperflow-amqp-executor
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Kamil Figiela
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+- !ruby/object:Gem::Dependency
+  name: unf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: recursive-open-struct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: AMQP job executor for Hyperflow workflow engine (http://github.com/dice-cyfronet/hyperflow)
+email: kamil.figiela@gmail.com
+executables:
+- hyperflow-amqp-executor
+- hyperflow-amqp-metric-collector
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- ".document"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- bin/hyperflow-amqp-executor
+- bin/hyperflow-amqp-metric-collector
+- example/settings.yml
+- example/task.json
+- hyperflow-amqp-executor.gemspec
+- lib/hyperflow-amqp-executor.rb
+- lib/hyperflow-amqp-executor/cloud_storage.rb
+- lib/hyperflow-amqp-executor/gridftp_storage.rb
+- lib/hyperflow-amqp-executor/helpers.rb
+- lib/hyperflow-amqp-executor/job.rb
+- lib/hyperflow-amqp-executor/local_storage.rb
+- lib/hyperflow-amqp-executor/nfs_storage.rb
+- lib/hyperflow-amqp-executor/plgdata_storage.rb
+- lib/hyperflow-amqp-executor/settings.rb
+- test/helper.rb
+- test/test_hyperflow-amqp-executor.rb
+homepage: http://github.com/kfigiela/hyperflow-amqp-executor
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: AMQP job executor for Hyperflow workflow engine
+test_files: []