rest-ftp-daemon 0.2.0

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+
+gem "sinatra"
+gem "json"
+
+group :development do
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0"
+  gem "jeweler", "~> 2.0.1"
+  gem "simplecov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,89 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (4.1.4)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    addressable (2.3.6)
+    builder (3.2.2)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    docile (1.1.5)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    git (1.2.7)
+    github_api (0.12.0)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.2)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.3)
+      oauth2
+    hashie (3.2.0)
+    highline (1.6.21)
+    i18n (0.6.11)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.1)
+    jwt (1.0.0)
+    mini_portile (0.6.0)
+    minitest (5.4.0)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rack-protection (1.5.3)
+      rack
+    rake (10.3.2)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.2.1)
+    shoulda-matchers (2.6.2)
+      activesupport (>= 3.0.0)
+    simplecov (0.9.0)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    sinatra (1.4.5)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    thread_safe (0.3.4)
+    tilt (1.4.1)
+    tzinfo (1.2.1)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  jeweler (~> 2.0.1)
+  json
+  rdoc (~> 3.12)
+  shoulda
+  simplecov
+  sinatra

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2014 Bruno Medici Consultant
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,88 @@
+## rest-ftp-daemon ##
+
+This is a fairly basic FTP client daemon, driven by REST-based webservice calls.
+As of today, its main features are :
+
+* Start a job through a basic structure posted with PUT /jobs
+* Spawn a dedicated thread to handle this job in its own contexte
+* Report transfer progress, error and activity for each job
+* Gather jobs status into the main process to get a global view
+* Report JSON status of workers on GET /jobs/ for automated monitoring
+
+
+## Quick setup ##
+
+This project requires ruby >= 1.9 and rubygems installed.
+
+Furthermore, gems are required to run it:
+
+ ``` gem install sinatra ```
+
+Starting the daemon:
+
+``` thin start ```
+
+
+## Basic usage ##
+
+Starting a job transferring file named "file.ova"
+
+  curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
+  '{"source":"~/file.ova","target":"ftp://anonymous@localhost/incoming/dest2.ova"}' "http://localhost:3000/jobs"
+
+
+Starting a job transferring file named "dmg"
+
+  curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
+  '{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg"}' "http://localhost:3000/jobs"
+
+
+Delete a specific job
+
+  curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
+
+
+
+## Getting status ##
+
+  GET /jobs
+
+Would return:
+
+  [
+    {
+      "id": "bob-49126-8",
+      "process": "sleep",
+      "status": "transferring",
+      "context": {
+        "source": "~\/file.ova",
+        "target": "ftp:\/\/anonymous@localhost\/incoming\/dest2.ova",
+        "code": -1,
+        "errmsg": "running",
+        "source_size": 1849036800,
+        "progress": 1.9,
+        "transferred": 34800000
+      }
+    },
+    {
+      "id": "bob-49126-9",
+      "process": "sleep",
+      "status": "transferring",
+      "context": {
+        "source": "~\/file.dmg",
+        "target": "ftp:\/\/anonymous@localhost\/incoming\/dest4.dmg",
+        "code": -1,
+        "errmsg": "running",
+        "source_size": 37109074,
+        "progress": 32.9,
+        "transferred": 12200000
+      }
+    }
+  ]
+
+
+### About ###
+
+Bruno MEDICI
+
+http://bmconseil.com/

data/Rakefile

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "rest-ftp-daemon"
+  gem.homepage = "http://github.com/bmedici/rest-ftp-daemon"
+  gem.license = "MIT"
+  gem.summary = "RESTful FTP client daemon"
+  gem.description = "A fairly basic FTP client daemon, driven by RESTful webservice calls"
+  gem.email = "rest-ftp-daemon@bmconseil.com"
+  gem.authors = ["Bruno"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['test'].execute
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "rest-ftp-daemon #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/bin/rest-ftp-daemon

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+# Libs and init
+require 'thin'
+APP_ROOT = File.dirname(__FILE__) + '/../'
+DEFAULT_PORT = 3000
+
+# Prepare thin
+rackup_file = File.expand_path "#{APP_ROOT}/lib/rest-ftp-daemon/config.ru"
+argv = ARGV
+argv << ["-R", rackup_file] unless ARGV.include?("-R")
+argv << ["-p", DEFAULT_PORT.to_s] unless ARGV.include?("-p")
+argv << ["-e", "production"] unless ARGV.include?("-e")
+
+# Start thin
+Thin::Runner.new(argv.flatten).run!

data/lib/config.rb

@@ -0,0 +1,2 @@
+TRANSFER_CHUNK_SIZE = 100000
+THREAD_SLEEP_BEFORE_DIE = 120

data/lib/errors.rb

@@ -0,0 +1,16 @@
+# Error codes
+# O: all ok
+# 1x: request body errors
+# 2x: transfer errors
+
+ERR_OK                            = 0
+ERR_BUSY                          = -1
+
+ERR_REQ_SOURCE_MISSING            = 11
+ERR_REQ_TARGET_MISSING            = 12
+ERR_REQ_TARGET_SCHEME             = 13
+
+ERR_JOB_SOURCE_NOTFOUND            = 21
+ERR_JOB_TARGET_UNPARSEABLE         = 22
+ERR_JOB_PERMISSION                 = 24
+

data/lib/rest-ftp-daemon.rb

@@ -0,0 +1,242 @@
+class RestFtpDaemon < Sinatra::Base
+  helpers Sinatra::JSON
+
+  # General config
+  configure :development, :production do
+
+    # Create new thread group
+    @@threads = ThreadGroup.new
+    #set :dummy, true
+    # set :sessions, false
+    # set :logging, true
+    # set :root, APP_ROOT + '/lib/'
+  end
+
+  # Server initialization
+  def initialize
+    # Setup logger
+    @logger = Logger.new(APP_ROOT + '/main.log','daily')
+    @logger.level = Logger::INFO
+
+    # Other stuff
+    @@last_worker_id = 0
+    @@hostname = `hostname`.chomp
+
+    super
+  end
+
+  # Server global status
+  get "/" do
+    content_type :json
+    json get_status
+  end
+
+  # List jobs
+  get "/jobs" do
+    # Build response
+    content_type :json
+    json get_jobs
+    #@@threads.count
+  end
+
+  # List jobs
+  delete "/jobs/:name" do
+    # Kill this job
+    ret = delete_job params[:name]
+
+    # Fail if no process has been killed
+    error 404 if ret<1
+
+    # Build response
+    content_type :json
+    json nil
+  end
+
+  # Spawn a new thread for this new job
+  post '/jobs' do
+    request.body.rewind
+    payload = JSON.parse request.body.read
+    info "POST / with #{payload.to_json}"
+
+    # Spawn a thread
+    #config = {}
+    result = new_job payload
+
+    # Build response
+    content_type :json
+    json result
+  end
+
+  protected
+
+  def process_job
+    # Init
+    info "process_job: starting"
+    context = Thread.current[:context]
+    transferred = 0
+
+    # Check source
+    job_source = File.expand_path(context["source"])
+    if !(File.exists? job_source)
+      job_status ERR_JOB_SOURCE_NOTFOUND, :ERR_JOB_SOURCE_NOTFOUND
+      return
+    end
+    info "process_job: job_source: #{job_source}"
+    source_size = File.size job_source
+
+    # Check target
+    job_target = context["target"]
+    target = URI(job_target) rescue nil
+    if job_target.nil? || target.nil?
+      job_status ERR_JOB_TARGET_UNPARSEABLE, :ERR_JOB_TARGET_UNPARSEABLE
+      return
+    end
+    info "process_job: job_target: #{job_target}"
+
+    # Split URI
+    target_path = File.dirname target.path
+    target_name = File.basename target.path
+    info "ftp_transfer: job_target.host [#{target.host}]"
+    info "ftp_transfer: target_path [#{target_path}]"
+    info "ftp_transfer: target_name [#{target_name}]"
+
+    # Prepare FTP transfer
+    ftp = Net::FTP.new(target.host)
+    ftp.passive = true
+    ftp.login
+    ftp.chdir(target_path)
+
+     # Do transfer
+    info "source: starting stransfer"
+    Thread.current[:status] = :transferring
+    job_status ERR_BUSY, :running
+    job_set :source_size, source_size
+
+    begin
+      ftp.putbinaryfile(job_source, target_name, TRANSFER_CHUNK_SIZE) do |block|
+        # Update thread info
+        percent = (100.0 * transferred / source_size).round(1)
+        info "transferring [#{percent} %] of [#{target_name}]"
+        job_set :progress, percent
+        job_set :transferred, transferred
+
+        # Update counters
+        transferred += TRANSFER_CHUNK_SIZE
+      end
+
+    rescue Net::FTPPermError
+      Thread.current[:status] = :failed
+      job_status ERR_JOB_PERMISSION, :ERR_JOB_PERMISSION
+      info "source: FAILED: PERMISSIONS ERROR"
+
+    else
+      Thread.current[:status] = :finished
+      job_status ERR_OK, :finished
+      info "source: finished stransfer"
+    end
+
+    ftp.close
+  end
+
+  def get_status
+    {
+    greeting: "REST FTP daemon: up and running",
+    jobs_count: @@threads.list.count,
+    }
+  end
+
+  def get_jobs
+    output = []
+    @@threads.list.each do |thread|
+      output << {
+      :id => thread[:name],
+      :process => thread.status,
+      :status =>  thread[:status],
+      :context => thread[:context],
+      }
+    end
+    output
+  end
+
+
+  def delete_job name
+    count = 0
+    @@threads.list.collect do |thread|
+      next unless thread[:name] == name
+      Thread.kill(thread)
+      count += 1
+    end
+    count
+  end
+
+  def new_job context = {}
+    info "new_job: creating thread"
+
+    # Generate name
+    @@last_worker_id +=1
+    host = @@hostname.split('.')[0]
+    name = "#{host}-#{Process.pid.to_s}-#{@@last_worker_id}"
+    info "new_job: creating thread [#{name}]"
+
+    # Parse parameters
+    job_source = context["source"]
+    job_target = context["target"]
+    return { code: ERR_REQ_SOURCE_MISSING, errmsg: :ERR_REQ_SOURCE_MISSING} if job_source.nil?
+    return { code: ERR_REQ_TARGET_MISSING, errmsg: :ERR_REQ_TARGET_MISSING} if job_target.nil?
+    #return { code: ERR_TRX_SOURCE_FILE_NOT_FOUND, errmsg: "ERR_TRX_SOURCE_FILE_NOT_FOUND [#{job_source}]"} unless File.exists? job_source
+
+    # Parse dest URI
+    target = URI(job_target)
+    info target.scheme
+    return { code: ERR_REQ_TARGET_SCHEME, errmsg: :ERR_REQ_TARGET_SCHEME} unless target.scheme == "ftp"
+
+    # Create thread
+    job = Thread.new(name, job) do
+      # Initialize context
+      Thread.current[:name] = name
+      Thread.current[:created] = Time.now.to_f;
+      Thread.current[:status] = :thread_initializing
+
+      # Store job info
+      Thread.current[:context] = context
+      job_status ERR_BUSY, :thread_initializing
+      Thread.abort_on_exception = true
+
+      # Do the job
+      info "new_job: thread running"
+      process_job
+
+      # Sleep a few seconds before dying
+      Thread.current[:status] = :thread_ending
+      sleep THREAD_SLEEP_BEFORE_DIE
+      info "new_job: thread finished"
+    end
+
+    # Keep thread in thread group
+    info "new_job: attaching thread [#{name}] to group"
+    @@threads.add job
+
+    return { code: 0, errmsg: 'success', name: name, context: context }
+  end
+
+  def log level, msg=""
+    @logger.send(level.to_s, msg)
+  end
+
+  def info msg=""
+    log :info, msg
+  end
+
+  def job_status code, errmsg
+    Thread.current[:context] ||= {}
+    Thread.current[:context][:code] = code
+    Thread.current[:context][:errmsg] = errmsg
+  end
+
+  def job_set attribute, value
+    Thread.current[:context] ||= {}
+    Thread.current[:context][attribute] = value
+  end
+
+
+end

data/lib/rest-ftp-daemon/config.ru

@@ -0,0 +1,13 @@
+# Main libs
+require 'sinatra'
+require 'sinatra/base'
+require "sinatra/json"
+require "sinatra/config_file"
+require 'net/ftp'
+require 'json'
+
+# My local libs
+Dir[APP_ROOT+"/lib/*.rb"].each {|file| require File.expand_path file }
+
+# Start application
+run RestFtpDaemon

data/test/helper.rb

@@ -0,0 +1,34 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rest-ftp-daemon'
+
+class Test::Unit::TestCase
+end

data/test/test_rest-ftp-daemon.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestRestFtpDaemon < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,176 @@
+--- !ruby/object:Gem::Specification
+name: rest-ftp-daemon
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- Bruno
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fairly basic FTP client daemon, driven by RESTful webservice calls
+email: rest-ftp-daemon@bmconseil.com
+executables:
+- rest-ftp-daemon
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- bin/rest-ftp-daemon
+- lib/config.rb
+- lib/errors.rb
+- lib/rest-ftp-daemon.rb
+- lib/rest-ftp-daemon/config.ru
+- test/helper.rb
+- test/test_rest-ftp-daemon.rb
+homepage: http://github.com/bmedici/rest-ftp-daemon
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3830976928581563014
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: RESTful FTP client daemon
+test_files: []