rest-ftp-daemon 0.222.0 → 0.230.0

data/lib/rest-ftp-daemon/remote_sftp.rb

@@ -0,0 +1,160 @@
+module RestFtpDaemon
+  class RemoteSFTP < Remote
+    attr_reader :sftp
+
+    def initialize url, log_context, options = {}
+      # Call super
+      super
+
+      # Use debug ?
+      @debug = (Settings.at :debug, :sftp) == true
+
+      # Announce object
+      log_info "RemoteSFTP.initialize"
+    end
+
+    def connect
+      # Connect init
+      super
+      log_info "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
+
+      # Debug level
+      verbosity = @debug ? Logger::INFO : false
+
+      # Connect remote server
+      @sftp = Net::SFTP.start(@url.host, @url.user, password: @url.password, verbose: verbosity)
+    end
+
+    def present? target
+      log_info "RemoteSFTP.present? [#{target.name}]"
+      stat = @sftp.stat! target.full
+      size = "?"
+
+      rescue Net::SFTP::StatusException
+        return false
+      else
+        return stat.size
+    end
+
+    # def remove target
+    #   log_info "RemoteSFTP.remove [#{target.name}]"
+    #   @sftp.remove target.full
+    # end
+
+    def remove! target
+      log_info "RemoteSFTP.remove! [#{target.name}]"
+      @sftp.remove target.full
+
+      rescue Net::SFTP::StatusException
+        log_info "#{LOG_INDENT}[#{target.name}] file not found"
+      else
+        log_info "#{LOG_INDENT}[#{target.name}] removed"
+    end
+
+    def mkdir directory
+      log_info "RemoteSFTP.mkdir [#{directory}]"
+      @sftp.mkdir! directory
+
+      rescue
+        raise JobTargetPermissionError
+    end
+
+    def chdir_or_create directory, mkdir = false
+      # Init, extract my parent name and my own name
+      log_info "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+      parent, current = Helpers.extract_parent(directory)
+
+      # Access this directory
+      begin
+        # log_info "   chdir [/#{directory}]"
+        handle = @sftp.opendir! "./#{directory}"
+
+      rescue Net::SFTP::StatusException => e
+        # If not allowed to create path, that's over, we're stuck
+        return false unless mkdir
+
+        # Recurse upward
+        #log_info "#{LOG_INDENT}upward [#{parent}]"
+        chdir_or_create parent, mkdir
+
+        # Now I was able to chdir into my parent, create the current directory
+        #log_info "#{LOG_INDENT}mkdir [#{directory}]"
+        mkdir directory
+
+        # Finally retry the chdir
+        retry
+      else
+        return true
+      end
+
+      # We should never get here
+      raise JobTargetShouldBeDirectory
+    end
+
+    # def dir_contents directory
+    #   # Access this directory
+    #   handle = @sftp.opendir! directory
+    #   @sftp.readdir! handle
+    # end
+
+    def push source, target, tempname = nil, &callback
+      # Push init
+      raise RestFtpDaemon::JobAssertionFailed, "push/1" if @sftp.nil?
+
+      # Temp file if provided
+      destination = target.clone
+      destination.name = tempname if tempname
+
+      # Do the transfer
+      log_info "RemoteSFTP.push [#{destination.full}]"
+      @sftp.upload! source.full, destination.full do |event, uploader, *args|
+        case event
+        when :open then
+          # args[0] : file metadata
+        when :put then
+          # args[0] : file metadata
+          # args[1] : byte offset in remote file
+          # args[2] : data being written (as string)
+          # puts "writing #{args[2].length} bytes to #{args[0].remote} starting at #{args[1]}"
+
+          # Update the worker activity marker
+          #FIXME worker_is_still_active
+
+          # Update job status after this block transfer
+          yield args[2].length, destination.name
+
+        when :close then
+          # args[0] : file metadata
+        when :mkdir
+          # args[0] : remote path name
+        when :finish
+        end
+
+      end
+
+      # flags = 0x0001 + 0x0002
+      flags = 0x00000001
+
+      # Rename if needed
+      if tempname
+        log_info "RemoteSFTP.push rename to\t[#{target.name}]"
+        @sftp.rename! destination.full, target.full, flags
+      end
+
+      # progress:
+      # Net::SFTP::StatusException
+    end
+
+    def close
+      # Close init
+      super
+
+      # @sftp.close
+    end
+
+    def connected?
+      @sftp && !@sftp.closed?
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/uri.rb

@@ -1,12 +1,19 @@
 module URI
+
   class FTPS < Generic
     DEFAULT_PORT = 21
   end
-  @@schemes["FTPS"] = FTPS
-end
-module URI
+
   class FTPES < Generic
-    DEFAULT_PORT = 990
+    # DEFAULT_PORT = 990
+    DEFAULT_PORT = 21
+  end
+
+  class SFTP < Generic
+    DEFAULT_PORT = 22
   end
+
+  @@schemes["FTPS"] = FTPS
   @@schemes["FTPES"] = FTPES
+  @@schemes["SFTP"] = SFTP
 end

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -6,7 +6,7 @@
   - presented = present job, with: RestFtpDaemon::API::Entities::JobPresenter, hide_params: true
   - bitrate = job.get :transfer_bitrate
 
-  - trclass = JOB_STYLES[job.status]
+  - trclass = DASHBOARD_JOB_STYLES[job.status]
 
   - unless job.error.nil?
     - trclass = "warning"

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -12,7 +12,7 @@
     - wid = vars[:wid]
     - status = vars[:status]
     - alive = $pool.worker_alive? wid
-    - trclass = WORKER_STYLES[status]
+    - trclass = DASHBOARD_WORKER_STYLES[status]
 
     - unless alive
       - trclass = "danger"

data/lib/rest-ftp-daemon/worker.rb

@@ -10,6 +10,7 @@ module RestFtpDaemon
     def initialize wid
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
+      @log_worker_status_changes = true
 
       # Worker name
       @wid = wid
@@ -17,7 +18,7 @@ module RestFtpDaemon
       # Set thread context
       Thread.current.thread_variable_set :wid, wid
       Thread.current.thread_variable_set :started_at, Time.now
-      worker_status :starting
+      worker_status WORKER_STATUS_STARTING
     end
 
   protected
@@ -40,9 +41,16 @@ module RestFtpDaemon
       end
     end
 
-    def worker_status status
+    def worker_status status, extra = ""
+      # Update thread variables
       Thread.current.thread_variable_set :status, status
       Thread.current.thread_variable_set :updted_at, Time.now
+
+      # Nothin' to log if "silent"
+      return unless @log_worker_status_changes
+
+      # Log this status change
+      log_info "worker: #{status} #{extra}"
     end
 
     def worker_jid jid

data/lib/rest-ftp-daemon/worker_conchita.rb

@@ -5,6 +5,10 @@ module RestFtpDaemon
       # Generic worker initialize
       super
 
+      # Use debug ?
+      @debug = (Settings.at :debug, :conchita) == true
+      @log_worker_status_changes = @debug
+
       # Conchita configuration
       @conchita = Settings.conchita
       if !@conchita.is_a? Hash
@@ -21,23 +25,25 @@ module RestFtpDaemon
   protected
 
     def work
-      worker_status :cleaning
+      # Announce we are working
+      worker_status WORKER_STATUS_CLEANING
 
       # Cleanup queues according to configured max-age
-      $queue.expire JOB_STATUS_FINISHED,  maxage(JOB_STATUS_FINISHED)
-      $queue.expire JOB_STATUS_FAILED,    maxage(JOB_STATUS_FAILED)
-      $queue.expire JOB_STATUS_QUEUED,    maxage(JOB_STATUS_QUEUED)
+      $queue.expire JOB_STATUS_FINISHED,  maxage(JOB_STATUS_FINISHED),  @debug
+      $queue.expire JOB_STATUS_FAILED,    maxage(JOB_STATUS_FAILED),    @debug
+      $queue.expire JOB_STATUS_QUEUED,    maxage(JOB_STATUS_QUEUED),    @debug
 
       # Force garbage collector
-      worker_status :collecting
       GC.start if @conchita["garbage_collector"]
 
     rescue StandardError => e
       log_error "EXCEPTION: #{e.inspect}"
       sleep 1
     else
+      # Restore previous status
+      worker_status WORKER_STATUS_WAITING
+
       # Sleep for a few seconds
-      worker_status :sleeping
       sleep @conchita[:timer]
     end
 

data/lib/rest-ftp-daemon/worker_job.rb

@@ -17,14 +17,13 @@ module RestFtpDaemon
 
     def work
       # Wait for a job to come into the queue
-      worker_status :waiting
-      log_info "waiting for a job"
+      worker_status WORKER_STATUS_WAITING
+      #log_info "waiting"
       job = $queue.pop
 
       # Prepare the job for processing
-      worker_status :working
+      worker_status WORKER_STATUS_RUNNING, "job [#{job.id}]"
       worker_jid job.id
-      log_info "working with job [#{job.id}]"
       job.wid = Thread.current.thread_variable_get :wid
 
       # Processs this job protected by a timeout
@@ -33,8 +32,7 @@ module RestFtpDaemon
       end
 
       # Processing done
-      worker_status :finished
-      log_info "finished with job [#{job.id}]"
+      worker_status WORKER_STATUS_FINISHED, "job [#{job.id}]"
       worker_jid nil
       job.wid = nil
 
@@ -42,8 +40,8 @@ module RestFtpDaemon
       $queue.counter_inc :jobs_processed
 
     rescue RestFtpDaemon::JobTimeout => ex
-      log_error "JOB TIMED OUT", lines: ex.backtrace
-      worker_status :timeout
+      log_error "JOB TIMED OUT", ex.backtrace
+      worker_status WORKER_STATUS_TIMEOUT
       worker_jid nil
       job.wid = nil
 
@@ -51,14 +49,13 @@ module RestFtpDaemon
       sleep 1
 
     rescue StandardError => ex
-      log_error "JOB UNHDNALED EXCEPTION: #{ex.message}", lines: ex.backtrace
-      worker_status :crashed
+      log_error "JOB UNHDNALED EXCEPTION: #{ex.message}", ex.backtrace
+      worker_status WORKER_STATUS_CRASHED
       job.oops_after_crash ex unless job.nil?
       sleep 1
 
     else
       # Clean job status
-      worker_status :free
       job.wid = nil
 
     end

data/rest-ftp-daemon.gemspec

@@ -16,7 +16,9 @@ Gem::Specification.new do |spec|
   spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
   spec.licenses = ["MIT"]
 
-  spec.files         = `git ls-files -z`.split("\x0")
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f == 'dashboard.png'
+  end
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
   spec.version       = APP_VER
@@ -28,6 +30,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 3.1"
   spec.add_development_dependency "http", "~> 0.8"
+  spec.add_development_dependency "rubocop", "~> 0.32.0"
+  spec.add_development_dependency "pry"
 
   spec.add_runtime_dependency "thin", "~> 1.6"
   spec.add_runtime_dependency "grape"
@@ -35,6 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "settingslogic"
   spec.add_runtime_dependency "haml"
   spec.add_runtime_dependency "json"
+  spec.add_runtime_dependency "net-sftp"
   spec.add_runtime_dependency "double-bag-ftps"
   spec.add_runtime_dependency "facter"
   spec.add_runtime_dependency "sys-cpu"

data/rest-ftp-daemon.yml.sample

@@ -4,10 +4,10 @@ defaults: &defaults
   workers: 2
   user: rftpd
   group: rftpd
-  host: "myhost"
+  #host: "myhost"
+  #appname: "replay"          # appname prefix used for NewRelic reporting
 
   transfer:
-    update_every_kb: 1024     # block size to transfer between counter updates
     notify_after_sec: 5       # wait at least X seconds between HTTP notifications
     mkdir: true               # build directory tree if missing
     tempfile: true            # transfer to temporary file, rename after sucessful transfer
@@ -23,6 +23,8 @@ defaults: &defaults
 
   debug:
     ftp: false
+    sftp: false
+    conchita: false
     # newrelic: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
   logs:

data/spec/rest-ftp-daemon/features/dashboard_spec.rb

@@ -5,19 +5,23 @@ describe 'Dashboard', feature: true do
   describe "GET /" do
     context 'without a password' do
       it 'is forbidden' do
-        expect(HTTP.accept(:json).get("http://localhost:5678").status).to eq 401
+        expect(HTTP.accept(:json).get("http://localhost:#{RequestHelpers::PORT}").status).to eq 401
       end
     end
 
     context 'with a password' do
       it 'can be accessed' do
         expect(
-          HTTP.accept(:json).
-            basic_auth(user: 'admin', pass: 'admin').
-            get("http://localhost:5678").status
+            get("/").status
         ).to eq 200
       end
     end
+
+    it "has an HTML representation" do
+      expect(
+        get("/", accept: 'html').status
+      ).to eq 200
+    end
   end # GET /
 
 end

data/spec/rest-ftp-daemon/features/jobs_spec.rb

@@ -0,0 +1,68 @@
+require "spec_helper"
+
+describe "Jobs", feature: true do
+
+  describe "GET /jobs" do
+    let!(:response) { get "/jobs" }
+
+    it "responds successfully" do
+      expect(response.status).to eq 200
+    end
+
+    it "exposes an array" do
+      expect(JSON.parse(response.body)).to be_an_instance_of(Array)
+    end
+  end # GET /jobs
+
+  describe "POST /jobs" do
+    def jobs_list
+      JSON.parse get("/jobs").body
+    end
+
+    context "when params are valid" do
+      let(:params) do
+        {
+          source: "/tmp/foo",
+          target: "/tmp/bar"
+        }
+      end
+
+      it "issues a 201 response" do
+        expect(
+          post("/jobs", json: params).status
+        ).to eq 201
+      end
+
+      it "exposes the new job id" do
+        response = JSON.parse post("/jobs", json: params)
+        expect(response['id']).not_to be_nil
+      end
+
+      it "assigns a status" do
+        response = JSON.parse post("/jobs", json: params)
+        expect(response['status']).to match(/^(queued|failed)$/)
+      end
+
+      it "creates a new job" do
+        expect {
+          post("/jobs", json: params)
+        }.to change { jobs_list.size }.by(1)
+      end
+    end
+  end # POST /jobs
+
+  describe "GET /jobs/:id" do
+    let(:creation_response) do
+      JSON.parse post("/jobs", json: {source: "/tmp/foo", target: "/tmp/bar"}).body
+    end
+
+    let(:job_id) { creation_response.fetch('id') }
+
+    it "is properly exposed" do
+      expect(
+        get("/jobs/#{job_id}").status
+      ).to eq 200
+    end
+  end # GET /jobs/:id
+
+end