rest-ftp-daemon 0.6 → 0.9.0

data/lib/rest-ftp-daemon.rb

@@ -1,19 +1,305 @@
-# Global libs
-require 'rubygems'
-require 'json'
-require 'securerandom'
-# require 'celluloid/autostart'
-
-# My libs
-require 'rest-ftp-daemon/config'
-require 'rest-ftp-daemon/exceptions'
-require 'rest-ftp-daemon/common'
-require 'rest-ftp-daemon/job_queue'
-require 'rest-ftp-daemon/worker_pool'
-require 'rest-ftp-daemon/logger'
-require 'rest-ftp-daemon/job'
-require 'rest-ftp-daemon/notification'
-require 'rest-ftp-daemon/api/defaults'
-require 'rest-ftp-daemon/api/jobs'
-require 'rest-ftp-daemon/api/root'
+class RestFtpDaemon < Sinatra::Base
 
+  # General config
+  configure :development, :production do
+
+    # Create new thread group
+    @@workers = ThreadGroup.new
+
+    # Logging configuration
+    #use Rack::CommonLogger, logger
+
+    # Some other configuration
+    disable :sessions
+    disable :logging
+  end
+
+  # Server initialization
+  def initialize
+    # Setup logger
+    @logger = Logger.new(APP_LOGTO, 'daily')
+    #@logger = Logger.new
+    #@logger.level = Logger::INFO
+
+    # Other stuff
+    @@last_worker_id = 0
+    @@hostname = `hostname`.chomp
+
+    super
+  end
+
+  # Server global status
+  get "/" do
+    # Debug query
+    info "GET /"
+
+    # Build response
+    content_type :json
+    JSON.pretty_generate get_status
+  end
+
+  # List jobs
+  get "/jobs" do
+    # Debug query
+    info "GET /jobs"
+
+    # Build response
+    content_type :json
+    JSON.pretty_generate get_jobs
+  end
+
+  # Get job info
+  get "/jobs/:id" do
+    # Debug query
+    info "GET /jobs/#{params[:id]}"
+
+    # Find this process by name
+    found = find_job params[:id]
+
+    # Build response
+    error 404 and return if found.nil?
+    content_type :json
+    JSON.pretty_generate found
+  end
+
+  # Delete jobs
+  delete "/jobs/:id" do
+    # Debug query
+    info "DELETE /jobs/#{params[:name]}"
+
+    # Find and kill this job
+    found = delete_job params[:id]
+
+    # Build response
+    error 404 and return if found.nil?
+    content_type :json
+    JSON.pretty_generate found
+  end
+
+  # Spawn a new thread for this new job
+  post '/jobs' do
+    # Extract payload
+    request.body.rewind
+    payload = JSON.parse request.body.read
+
+    # Debug query
+    info "POST /jobs: #{payload.to_json}"
+
+    # Spawn a thread for this job
+    result = new_job payload
+
+    # Build response
+    content_type :json
+    JSON.pretty_generate result
+  end
+
+  protected
+
+  def process_job
+    # Init
+    info "process_job: starting"
+    job = Thread.current.job
+    job_status :started
+    transferred = 0
+
+    # Check source
+    job_source = File.expand_path(job["source"])
+    if !(File.exists? job_source)
+      job_error ERR_JOB_SOURCE_NOTFOUND, :ERR_JOB_SOURCE_NOTFOUND
+      return
+    end
+    info "process_job: job_source: #{job_source}"
+    source_size = File.size job_source
+    job_set :source_size, source_size
+
+    # Check target
+    job_target = job["target"]
+    target = URI(job_target) rescue nil
+    if job_target.nil? || target.nil?
+      job_error ERR_JOB_TARGET_UNPARSEABLE, :ERR_JOB_TARGET_UNPARSEABLE
+      return
+    end
+    info "process_job: job_target: #{job_target}"
+
+    # Split URI
+    target_path = File.dirname target.path
+    target_name = File.basename target.path
+    info "ftp_transfer: job_target.host [#{target.host}]"
+    info "ftp_transfer: target_path [#{target_path}]"
+    info "ftp_transfer: target_name [#{target_name}]"
+
+    # Prepare FTP transfer
+    ftp = Net::FTP.new(target.host)
+    ftp.passive = true
+    ftp.login
+    ftp.chdir(target_path)
+
+
+    # Check if target file is found
+    info "source: checking target file"
+    job_status :checking_target
+    job_error ERR_BUSY, :checking_target
+
+    results = ftp.list(target_name)
+    info "ftp.list: #{results}"
+    unless results.count.zero?
+      job_error ERR_JOB_TARGET_PRESENT, :ERR_JOB_TARGET_PRESENT
+      info "target: existing: ERR_JOB_TARGET_PRESENT"
+      ftp.close
+      return
+    end
+
+
+     # Do transfer
+    info "source: starting stransfer"
+    #Thread.current[:status] = :transferring
+    job_status :uploading
+    job_error ERR_BUSY, :uploading
+
+    begin
+      ftp.putbinaryfile(job_source, target_name, TRANSFER_CHUNK_SIZE) do |block|
+        # Update thread info
+        percent = (100.0 * transferred / source_size).round(1)
+        job_set :progress, percent
+        job_set :transferred, transferred
+        info "transferring [#{percent} %] of [#{target_name}]"
+
+        # Update counters
+        transferred += TRANSFER_CHUNK_SIZE
+      end
+
+    rescue Net::FTPPermError
+      #job_status :failed
+      job_error ERR_JOB_PERMISSION, :ERR_JOB_PERMISSION
+      info "source: FAILED: PERMISSIONS ERROR"
+
+    else
+      #job_status :finished
+      job_error ERR_OK, :finished
+      info "source: finished stransfer"
+    end
+
+    # Close FTP connexion
+    ftp.close
+  end
+
+  def get_status
+    info "> get_status"
+    {
+    app_name: APP_NAME,
+    hostname: @@hostname,
+    version: APP_VER,
+    started: APP_STARTED,
+    uptime: (Time.now - APP_STARTED).round(1),
+    jobs_count: @@workers.list.count,
+    }
+  end
+
+  def get_jobs
+    info "> get_jobs"
+
+    # Collect info's
+    @@workers.list.map { |thread| thread.job }
+  end
+
+  def delete_job id
+    info "> delete_job(#{id})"
+
+    # Find jobs with this id
+    jobs = jobs_with_id id
+
+    # Kill them
+    jobs.each{ |thread| Thread.kill(thread) }
+
+    # Return the first one
+    return nil if jobs.empty?
+    jobs.first.job
+  end
+
+  def find_job id
+    info "> find_job(#{id})"
+
+    # Find jobs with this id
+    jobs = jobs_with_id id
+
+    # Return the first one
+    return nil if jobs.empty?
+    jobs.first.job
+  end
+
+  def jobs_with_id id
+    info "> find_jobs_by_id(#{id})"
+    @@workers.list.select{ |thread| thread[:id].to_s == id.to_s }
+  end
+
+  def new_job context = {}
+    info "new_job"
+
+    # Generate name
+    @@last_worker_id +=1
+    host = @@hostname.split('.')[0]
+    worker_id = @@last_worker_id
+    worker_name = "#{host}-#{Process.pid.to_s}-#{worker_id}"
+    info "new_job: creating thread [#{worker_name}]"
+
+    # Parse parameters
+    job_source = context["source"]
+    job_target = context["target"]
+    return { code: ERR_REQ_SOURCE_MISSING, errmsg: :ERR_REQ_SOURCE_MISSING} if job_source.nil?
+    return { code: ERR_REQ_TARGET_MISSING, errmsg: :ERR_REQ_TARGET_MISSING} if job_target.nil?
+
+    # Parse dest URI
+    target = URI(job_target)
+    info target.scheme
+    return { code: ERR_REQ_TARGET_SCHEME, errmsg: :ERR_REQ_TARGET_SCHEME} unless target.scheme == "ftp"
+
+    # Create thread
+    job = Thread.new(worker_id, worker_name, job) do
+      # Tnitialize thread
+      Thread.abort_on_exception = true
+      job_status :initializing
+      job_error ERR_OK
+
+      # Initialize job info
+      Thread.current[:job] = {}
+      Thread.current[:job].merge! context if context.is_a? Enumerable
+      Thread.current[:id] = worker_id
+      job_set :worker_name, worker_name
+      job_set :created, Time.now
+
+      # Do the job
+      info "new_job: thread running"
+      process_job
+
+      # Sleep a few seconds before dying
+      job_status :graceful_ending
+      sleep THREAD_SLEEP_BEFORE_DIE
+      job_status :ended
+      info "new_job: thread finished"
+    end
+
+    # Keep thread in thread group
+    info "new_job: attaching thread [#{worker_name}] to group"
+    @@workers.add job
+
+    return { code: 0, errmsg: 'success', worker_id: worker_id, context: context }
+  end
+
+  def info msg=""
+    @logger.info msg
+  end
+
+  def job_error error, errmsg = nil
+    job_set :error, error
+    job_set :errmsg, errmsg
+  end
+  def job_status status
+    job_set :status, status
+  end
+
+  def job_set attribute, value, thread = Thread.current
+    thread[:job][attribute] = value if thread[:job].is_a? Enumerable
+  end
+
+
+end

data/rest-ftp-daemon.gemspec

@@ -1,36 +1,67 @@
-# coding: utf-8
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
 
-# Libs
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'rest-ftp-daemon/config'
+Gem::Specification.new do |s|
+  s.name = "rest-ftp-daemon"
+  s.version = "0.7.0"
 
-Gem::Specification.new do |spec|
-  spec.name = Settings[:name]
-  spec.date = Time.now.strftime("%Y-%m-%d")
-  spec.authors = ["Bruno MEDICI"]
-  spec.email = "rest-ftp-daemon@bmconseil.com"
-  spec.description = "This is a pretty simple FTP client daemon, controlled through a RESTfull API"
-  spec.summary = "RESTful FTP client daemon"
-  spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
-  spec.licenses = ["MIT"]
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Bruno"]
+  s.date = "2014-08-01"
+  s.description = "This is a pretty simple FTP client daemon, controlled through a RESTfull API"
+  s.email = "rest-ftp-daemon@bmconseil.com"
+  s.executables = ["rest-ftp-daemon"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "bin/rest-ftp-daemon",
+    "lib/config.rb",
+    "lib/config.ru",
+    "lib/errors.rb",
+    "lib/extend_threads.rb",
+    "lib/rest-ftp-daemon.rb",
+    "rest-ftp-daemon.gemspec",
+    "test/helper.rb",
+    "test/test_rest-ftp-daemon.rb"
+  ]
+  s.homepage = "http://github.com/bmedici/rest-ftp-daemon"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.23"
+  s.summary = "RESTful FTP client daemon"
 
-  spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-  spec.version       = Settings[:version]
-
-  spec.required_ruby_version = '>= 1.9.3'
-
-  spec.add_development_dependency "bundler", "~> 1.6"
-  spec.add_development_dependency "rake"
-
-  spec.add_runtime_dependency "thin", "~> 1.6"
-  spec.add_runtime_dependency "grape"
-  spec.add_runtime_dependency "settingslogic"
-  spec.add_runtime_dependency "haml"
-  spec.add_runtime_dependency "json"
-  spec.add_runtime_dependency "facter"
-  spec.add_runtime_dependency "sys-cpu"
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
 
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sinatra>, [">= 0"])
+      s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
+    else
+      s.add_dependency(%q<sinatra>, [">= 0"])
+      s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+    end
+  else
+    s.add_dependency(%q<sinatra>, [">= 0"])
+    s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+  end
 end
+

metadata

@@ -1,60 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: '0.6'
+  version: 0.9.0
+  prerelease: 
 platform: ruby
 authors:
-- Bruno MEDICI
+- Bruno
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-23 00:00:00.000000000 Z
+date: 2014-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.6'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.6'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: thin
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.6'
-- !ruby/object:Gem::Dependency
-  name: grape
+  name: sinatra
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,27 +22,15 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: settingslogic
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: haml
+  name: json
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -90,112 +38,110 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: json
+  name: shoulda
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: facter
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '1.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: sys-cpu
+  name: jeweler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: 2.0.1
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.1
 description: This is a pretty simple FTP client daemon, controlled through a RESTfull
   API
 email: rest-ftp-daemon@bmconseil.com
 executables:
 - rest-ftp-daemon
 extensions: []
-extra_rdoc_files: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
 files:
-- .gitignore
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
+- VERSION
 - bin/rest-ftp-daemon
-- config.ru
+- lib/config.rb
+- lib/config.ru
+- lib/errors.rb
+- lib/extend_threads.rb
 - lib/rest-ftp-daemon.rb
-- lib/rest-ftp-daemon/api/defaults.rb
-- lib/rest-ftp-daemon/api/jobs.rb
-- lib/rest-ftp-daemon/api/root.rb
-- lib/rest-ftp-daemon/api/workers.rb
-- lib/rest-ftp-daemon/common.rb
-- lib/rest-ftp-daemon/config.rb
-- lib/rest-ftp-daemon/exceptions.rb
-- lib/rest-ftp-daemon/job.rb
-- lib/rest-ftp-daemon/job_queue.rb
-- lib/rest-ftp-daemon/logger.rb
-- lib/rest-ftp-daemon/notification.rb
-- lib/rest-ftp-daemon/static/css/bootstrap.min.css
-- lib/rest-ftp-daemon/views/dashboard.haml
-- lib/rest-ftp-daemon/views/dashboard_jobs.haml
-- lib/rest-ftp-daemon/views/index.haml
-- lib/rest-ftp-daemon/worker_pool.rb
 - rest-ftp-daemon.gemspec
-- rest-ftp-daemon.yml.sample
 - test/helper.rb
 - test/test_rest-ftp-daemon.rb
 homepage: http://github.com/bmedici/rest-ftp-daemon
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: '0'
+      segments:
+      - 0
+      hash: 3650939871823663291
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: RESTful FTP client daemon
 test_files: []
-has_rdoc: 

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YWY3YjIzYTk0MzIyODlmMmE1MGE3ZTgzZWZkMjllY2JlYThkZjQ1YQ==
-  data.tar.gz: !binary |-
-    YWM0ZmUwNmU3YjY3NjBjZTAyYWEwYjZlYjJjMmNjZjJiZjc3MTJhMA==
-SHA512:
-  metadata.gz: !binary |-
-    ZDUzMTU5NThlMWQ1Njk1YWY1YTgyOTg1ZDVjMWMxZjJkMmJhNzRhNmQ0NmYw
-    NjhkZmFhOWFhZDIxZjY3NzNkMGRjMTA3Y2RmNDJiZDViZGQzYmJlNjYyMGM2
-    ZWE4ZjI2ODgzZDk4MWRjOTU2ZjhhMmQzZjBkYTE0MjlkYjk5ZTQ=
-  data.tar.gz: !binary |-
-    ZjEzNjg4YmQ3Yjg5N2JjNTZhZTY0OGQwOWRjMDZmNTIyZjdlODFmNGI5OTQ3
-    ZjBiNjcyYTZlMWZlMWRhOTFiMjI4MTIxMDEzM2QzNzM5YTA3MDk2ODc2NGIx
-    YWZiZTk2NzNhZGYyNDQ2Y2YwN2I3OTI5ZjE0ODQyNTNjZmRkNWU=

data/.gitignore

@@ -1,6 +0,0 @@
-pkg
-.bundle
-.DS_Store
-*.log
-tmp/
-work/

data/config.ru

@@ -1,14 +0,0 @@
-# Load gem files
-$LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
-require 'rest-ftp-daemon'
-
-# Some extra constants
-APP_STARTED = Time.now
-
-# Create worker pool
-$queue = RestFtpDaemon::JobQueue.new
-$pool = RestFtpDaemon::WorkerPool.new(Settings.workers.to_i)
-
-# Start REST FTP Daemon
-use Rack::Static, :urls => ["/css", "/images"], :root => "lib/rest-ftp-daemon/static/"
-run Rack::Cascade.new [RestFtpDaemon::API::Root]