sp-job 0.2.3 → 0.3.22

data/lib/sp/job/unique_file.rb

@@ -63,10 +63,46 @@ module SP
           return nil if r != 0
           return nil if ptr.null?
 
-          rv = ptr.read_string
+          rv = ptr.read_string.force_encoding('UTF-8')
 
           return rv
         end
+
+        #
+        # Creates a uniquely named file inside the specified folder. The created file is empty
+        #
+        # @param folder Folder where the file will be created
+        # @param name
+        # @param extension
+        # @return Absolute file path
+        #
+        def self.create_n (folder:, name:, extension:)
+          FileUtils.mkdir_p(folder) if !Dir.exist?(folder)
+          if nil != name
+            fd = ::SP::Job::Unique::File.mkstemps("#{folder}/#{name}.XXXXXX.#{extension}", extension.length + 1)
+          else
+            fd = ::SP::Job::Unique::File.mkstemps("#{folder}/XXXXXX.#{extension}", extension.length + 1)
+          end
+          return nil if fd < 0
+
+          ptr = FFI::MemoryPointer.new(:char, 8192)     # Assumes max path is less that this
+          if OS.mac?
+            r = ::SP::Job::Unique::File.fcntl(fd, 50, ptr) # 50 is F_GETPATH in OSX
+          else
+            r = ::SP::Job::Unique::File.readlink("/proc/self/fd/#{fd}", ptr, 8192)
+            if r > 0 && r < 8192
+              r = 0
+            end
+          end
+          ::SP::Job::Unique::File.close(fd)
+          return nil if r != 0
+          return nil if ptr.null?
+
+          rv = ptr.read_string.force_encoding('UTF-8')
+
+          return rv
+        end
+
       end
     end
   end

data/lib/sp/job/uploaded_image_converter.rb

@@ -23,24 +23,23 @@
 #
 # How to use this to implement a customized image conversion
 #
-# require 'rmagick'
 # require 'sp-job'
 # require 'sp/job/back_burner'
 # require 'sp/job/uploaded_image_converter'
-# 
+#
 # class CLASSNAME < ::SP::Job::UploadedImageConverter
-# 
+#
 #   def self.perform(job)
-# 
+#
 #     ... Your code before the image conversion ...
 #
 #     SP::Job::UploadedImageConverter.perform(job)
-# 
+#
 #     ... your code after the conversion ...
 #
 #   end
 # end
-# 
+#
 # Backburner.work
 #
 
@@ -51,24 +50,34 @@ module SP
 
       def self.perform (job)
 
-        raise_error(message: 'i18n_entity_id_must_be_defined') if job[:entity_id].nil? || job[:entity_id].to_i == 0
+        raise_error(message: 'i18n_entity_id_must_be_defined') if job[:to_entity_id].nil? || job[:to_entity_id].to_i == 0
 
         step        = 100 / (job[:copies].size + 1)
         progress    = step
-        original    = File.join(config[:paths][:temporary_uploads], job[:original])
-        destination = File.join(config[:paths][:uploads_storage], job[:entity], id_to_path(job[:entity_id]), job[:folder])
+        original    = File.join(config[:scp_config][:temp_uploads], job[:original])
+        destination = File.join(config[:scp_config][:path], job[:entity], id_to_path(job[:to_entity_id]), job[:folder])
+
+        if config[:scp_config][:local]
+          ssh = ''
+          FileUtils::mkdir_p destination
+        else
+          ssh = "ssh #{config[:scp_config][:server]} "
+          %x[#{ssh}mkdir -p #{destination}]
+          unless $?.success?
+            raise_error(message: 'i18n_internal_error', info: "unable to create remote directory")
+          end
+        end
 
         #
         # Check the original image, check format and limits
-        # 
-        FileUtils::mkdir_p destination
-        update_progress(progress: progress, message: 'i18n_reading_original_$image', image: job[:original])
-        img_info = %x[identify #{original}]
+        #
+        update_progress(progress: progress, message: 'i18n_reading_original_$image', image: job[:original_file_path] || job[:original])
+        img_info = %x[#{ssh}identify #{original}]
         m = %r[.*\.ul\s(\w+)\s(\d+)x(\d+)\s.*].match img_info
         if $?.success? == false
           return report_error(message: 'i18n_invalid_image', info: "Image #{original} can't be identified '#{img_info}'")
         end
-        if m.nil? || m.size != 4 
+        if m.nil? || m.size != 4
           return report_error(message: 'i18n_invalid_image', info: "Image #{original} can't be identified '#{img_info}'")
         end
         unless config[:options][:formats].include? m[1]
@@ -87,7 +96,7 @@ module SP
         # Iterate the copies array
         #
         job[:copies].each do |copy|
-          %x[convert #{original} -geometry #{copy[:geometry]} #{File.join(destination, copy[:name])}]
+          %x[#{ssh}convert #{original} -geometry #{copy[:geometry]} #{File.join(destination, copy[:name])}]
           unless $?.success?
             raise_error(message: 'i18n_internal_error', info: "convert failed to scale #{original} to #{copy[:geometry]}")
           end
@@ -97,11 +106,10 @@ module SP
           barrier = false
         end
 
+        #
         # Closing arguments, all done
-        send_response(message: 'i18n_image_conversion_complete', link: File.join('/',job[:entity], id_to_path(job[:entity_id]), job[:folder], 'logo_template.png'))
-
-        # Remove original file
-        FileUtils::rm_f(original) if config[:options][:delete_originals] 
+        #
+        return { hostname: config[:urls][:upload_public], path: File.join('/',job[:entity], id_to_path(job[:to_entity_id]), job[:folder])}
 
       end
 

data/lib/sp/job/worker.rb

@@ -26,7 +26,15 @@ module SP
       def start
         prepare
         loop do 
-          work_one_job
+            begin
+              work_one_job(connection)
+            rescue Beaneater::NotFoundError => bnfe
+              # Do nothing if try to delete the task and it´s not found
+            rescue Beaneater::DeadlineSoonError => dse 
+              # By default there is nothing we can do to speed up
+            rescue Backburner::Job::JobTimeout => jte
+              # What to do?
+            end
           unless connection.connected?
             log_error "Connection to beanstalk closed, exiting now"
             Kernel.exit
@@ -34,6 +42,48 @@ module SP
         end
       end
 
+      # Performs a job by reserving a job from beanstalk and processing it
+      #
+      # @example
+      #   @worker.work_one_job
+      # @raise [Beaneater::NotConnected] If beanstalk fails to connect multiple times.
+      def work_one_job(conn = connection)
+        begin
+          job = reserve_job(conn)
+        rescue Beaneater::TimedOutError => e
+          return
+        end
+  
+        self.log_job_begin(job.name, job.args)
+        job.process
+        self.log_job_end(job.name)
+  
+      rescue Backburner::Job::JobFormatInvalid => e
+        self.log_error self.exception_message(e)
+      rescue => e # Error occurred processing job
+        self.log_error self.exception_message(e)
+  
+        unless job
+          self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
+          return
+        end
+  
+        # NB: There's a slight chance here that the connection to beanstalkd has
+        # gone down between the time we reserved / processed the job and here.
+        num_retries = job.stats.nil? ? 0 : job.stats.releases
+        retry_status = "failed: attempt #{num_retries+1} of #{queue_config.max_job_retries+1}"
+        if num_retries < queue_config.max_job_retries # retry again
+          delay = queue_config.retry_delay_proc.call(queue_config.retry_delay, num_retries) rescue queue_config.retry_delay
+          job.retry(num_retries + 1, delay)
+          self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
+        else # retries failed, bury
+          job.bury
+          self.log_job_end(job.name, "#{retry_status}, burying") if job_started_at
+        end
+  
+        handle_error(e, job.name, job.args, job)
+      end
+
     end # Worker
   end # Module Job
 end # Module SP

data/lib/sp/job/worker_thread.rb

@@ -33,25 +33,40 @@ module SP
       #
       def prepare
         log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
-        $config[:options][:threads].times do 
+        $config[:options][:threads].times do
           connection = new_connection.tap{ |conn| conn.tubes.watch!(*tube_names) }
           connection.on_reconnect = lambda { |conn| conn.tubes.watch!(*tube_names) }
 
           $threads << Thread.new {
             $thread_data[Thread.current] = ::SP::Job::ThreadData.new
-            logger.debug "Thread for #{tube_names.join(',')} #{Thread.current}"
-            loop do 
-              work_one_job(connection)
+            logger.info "Thread for #{tube_names.join(',')} #{Thread.current}"
+            loop do
+              begin
+                work_one_job(connection)
+              rescue Beaneater::NotFoundError => bnfe
+                # Do nothing if try to delete the task and it´s not found
+              rescue Beaneater::DeadlineSoonError => dse 
+                # By default there is nothing we can do to speed up
+              rescue Backburner::Job::JobTimeout => jte
+                # What to do?
+                logger.info "Thread #{Thread.current} job timeout".yellow
+                Rollbar.warning(jte)
+              rescue => e
+                Rollbar.error(e)
+              end
+
               unless connection.connected?
                 log_error "Connection to beanstalk closed, exiting now"
                 Kernel.exit
               end
-            end          
+            end
+            logger.info "Thread #{Thread.current} exiting".yellow
+            $threads.delete(Thread.current)
+            $thread_data.delete Thread.current
           }
         end
-        #end
       end
 
     end # Worker
   end # Module Job
-end # Module SP
+end # Module SP

data/lib/sp/jsonapi.rb

@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2011-2017 Cloudware S.A. All rights reserved.
+#
+# This file is part of sp-job.
+#
+# sp-job is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# sp-job is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with sp-job.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+module SP
+  module JSONAPI
+  	MODULE_PATH = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+  end
+end

data/lib/sp/jsonapi/adapters/base.rb

@@ -0,0 +1,177 @@
+module SP
+  module JSONAPI
+    module Adapters
+
+      class Base
+
+        def service ; @service ; end
+
+        def initialize(service)
+          @service = service
+        end
+
+        def get(path, params = {})
+          request('GET', path, params)
+        end
+        def post(path, params = {})
+          request('POST', path, params)
+        end
+        def patch(path, params = {})
+          request('PATCH', path, params)
+        end
+        def delete(path)
+          request('DELETE', path, nil)
+        end
+
+        def get!(path, params = {})
+          request!('GET', path, params)
+        end
+        def post!(path, params = {})
+          request!('POST', path, params)
+        end
+        def patch!(path, params = {})
+          request!('PATCH', path, params)
+        end
+        def delete!(path)
+          request!('DELETE', path, nil)
+        end
+
+        def get_explicit!(exp_subentity_schema, exp_subentity_prefix, path, params = {})
+          explicit_request!(exp_subentity_schema, exp_subentity_prefix, 'GET', path, params)
+        end
+        def post_explicit!(exp_subentity_schema, exp_subentity_prefix, path, params = {})
+          explicit_request!(exp_subentity_schema, exp_subentity_prefix, 'POST', path, params)
+        end
+        def patch_explicit!(exp_subentity_schema, exp_subentity_prefix, path, params = {})
+          explicit_request!(exp_subentity_schema, exp_subentity_prefix, 'PATCH', path, params)
+        end
+        def delete_explicit!(exp_subentity_schema, exp_subentity_prefix, path)
+          explicit_request!(exp_subentity_schema, exp_subentity_prefix, 'DELETE', path, nil)
+        end
+
+        def get_specific_service!(path, params, service_params)
+          specific_service_do_request!('GET', path, params, service_params)
+        end
+
+        alias_method :put, :patch
+        alias_method :put!, :patch!
+        alias_method :put_explicit!, :patch_explicit!
+
+        def unwrap_request
+          unwrap_response(yield)
+        end
+
+        # do_request MUST be implemented by each specialized adapter, and returns a tuple: the request status and a JSONAPI string or hash with the result
+        def do_request(method, path, params) ; ; end
+        def explicit_do_request(exp_subentity_schema, exp_subentity_prefix, method, path, params) ; ; end
+
+        def request(method, path, params)
+          # As it is now, this method is EXACTLY the same as request!()
+          # And it cannot be reverted without affecting lots of changes already made in the app's controllers.
+          # TODO: end it, or end the !() version
+          # begin
+            unwrap_request do
+              do_request(method, path, params)
+            end
+          # THIS CAN'T BE DONE, because the same method cannot return both a single result (in case there is NOT an error) and a pair (in case there IS an error)
+          # rescue SP::JSONAPI::Exceptions::GenericModelError => e
+          #   [
+          #     e.status,
+          #     e.result
+          #   ]
+          # rescue Exception => e
+          #   [
+          #     SP::JSONAPI::Status::ERROR,
+          #     get_error_response(path, e)
+          #   ]
+          # end
+        end
+
+        def request!(method, path, params)
+          unwrap_request do
+            do_request(method, path, params)
+          end
+        end
+
+        def explicit_request!(exp_subentity_schema, exp_subentity_prefix, method, path, params)
+          unwrap_request do
+            explicit_do_request(exp_subentity_schema, exp_subentity_prefix, method, path, params)
+          end
+        end
+
+        def specific_service_do_request!(method, path, params, service_params)
+          unwrap_request do
+            specific_service_do_request(method, path, params, service_params)
+          end
+        end
+
+        protected
+
+          def url(path) ; File.join(service.url, path) ; end
+
+          def url_with_params_for_query(path, params)
+            query = params_for_query(params)
+            query_url = url(path)
+            query.blank? ? query_url : query_url + (query_url.include?('?') ? '&' : '?') + query
+          end
+
+          def params_for_query(params)
+            query = ""
+            if !params.blank?
+              case
+                when params.is_a?(Array)
+                  # query = params.join('&')
+                  query = params.map{ |v| URI.encode(URI.encode(v).gsub("'","''"), "&") }.join('&')
+                when params.is_a?(Hash)
+                  query = params.map do |k,v|
+                    if v.is_a?(String)
+                      "#{k}=\"#{URI.encode(URI.encode(v).gsub("'","''"), "&")}\""
+                    else
+                      "#{k}=#{v}"
+                    end
+                  end.join('&')
+                else
+                  query = params.to_s
+              end
+            end
+            query
+          end
+
+          def params_for_body(params)
+            params.blank? ?  '' : params.to_json.gsub("'","''")
+          end
+
+          # unwrap_response SHOULD be implemented by each specialized adapter, and returns the request result as a JSONAPI string or hash and raises an exception if there was an error
+          def unwrap_response(response)
+            # As the method request() is EXACTLY the same as request!(), and it cannot be reverted without affecting lots of changes already made in the app's controllers...
+            # Allow for response being both a [ status, result ] pair (as of old) OR a single result (as of now)
+            if response.is_a?(Array)
+              status = response[0].to_i
+              result = response[1]
+              result
+            else
+              response
+            end
+          end
+
+          def error_response(path, error)
+            {
+              errors: [
+                {
+                  status: "#{SP::JSONAPI::Status::ERROR}",
+                  code: error.message
+                }
+              ],
+              links: { self: url(path) },
+              jsonapi: { version: SP::JSONAPI::VERSION }
+            }
+          end
+
+          # get_error_response MUST be implemented by each specialized adapter, and returns a JSONAPI error result as a string or hash
+          def get_error_response(path, error) ; ; end
+
+      end
+
+    end
+  end
+end