sp-job 0.1.17 → 0.2.2

data/lib/sp/job/worker.rb

@@ -18,29 +18,22 @@
 #
 # encoding: utf-8
 #
-
 module SP
   module Job
 
     class Worker < Backburner::Workers::Simple
 
-      def initialize (tube_names=nil)
-        super(tube_names)
-      end
-
       def start
         prepare
         loop do 
           work_one_job
           unless connection.connected?
             log_error "Connection to beanstalk closed, exiting now"
-            exit
+            Kernel.exit
           end
         end
       end
 
-      private
-
     end # Worker
   end # Module Job
 end # Module SP

data/lib/sp/job/worker_thread.rb

@@ -0,0 +1,57 @@
+#
+#
+# Copyright (c) 2011-2017 Cloudware S.A. All rights reserved.
+#
+# This file is part of sp-job.
+#
+# sp-job is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# sp-job is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with sp-job.  If not, see <http://www.gnu.org/licenses/>.
+#
+# encoding: utf-8
+#
+module SP
+  module Job
+
+    class WorkerThread < Backburner::Workers::Simple
+
+      # Used to prepare job queues before processing jobs.
+      # Setup beanstalk tube_names and watch all specified tubes for jobs.
+      #
+      # @raise [Beaneater::NotConnected] If beanstalk fails to connect.
+      # @example
+      #   @worker.prepare
+      #
+      def prepare
+        log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
+        $config[:options][:threads].times do 
+          connection = new_connection.tap{ |conn| conn.tubes.watch!(*tube_names) }
+          connection.on_reconnect = lambda { |conn| conn.tubes.watch!(*tube_names) }
+
+          $threads << Thread.new {
+            $thread_data[Thread.current] = ::SP::Job::ThreadData.new
+            logger.debug "Thread for #{tube_names.join(',')} #{Thread.current}"
+            loop do 
+              work_one_job(connection)
+              unless connection.connected?
+                log_error "Connection to beanstalk closed, exiting now"
+                Kernel.exit
+              end
+            end          
+          }
+        end
+        #end
+      end
+
+    end # Worker
+  end # Module Job
+end # Module SP

data/lib/tasks/configure.rake

@@ -5,8 +5,38 @@ require 'ostruct'
 require 'awesome_print'
 require 'os'
 require 'fileutils'
+require 'tempfile'
 require 'etc'
 
+class SpDataStruct < OpenStruct
+
+  def self.to_hash_sp (object)
+    hash = {}
+    object.each_pair do |key, value|
+      if value.is_a?(SpDataStruct)
+        hash[key] = SpDataStruct::to_hash_sp(value)
+      elsif value.is_a?(Array)
+        hash[key] = []
+        value.each do |member|
+          if member.is_a?(SpDataStruct)
+            hash[key] << SpDataStruct::to_hash_sp(member)
+          else
+            hash[key] << member
+          end
+        end
+      else
+        hash[key] = value
+      end
+    end
+    hash
+  end
+  
+  def to_json
+    SpDataStruct::to_hash_sp(self).to_json
+  end
+
+end
+
 def safesudo(cmd)
   unless true == system(cmd)
     system("sudo #{cmd}")
@@ -15,7 +45,7 @@ end
 
 def create_directory (path)
 
-  if ! Dir.exists?(path)
+  if ! Dir.exist?(path)
     if OS.mac?
       if path.match("^/usr/local/")
         info = Etc.getpwnam(Etc.getlogin)
@@ -113,8 +143,26 @@ def diff_and_write (contents:, path:, diff: true, dry_run: false)
     FileUtils.rm(tmp_file)
 end
 
-def get_config
+def pg_conn_string (db)
+  "host=#{db.host} port=#{db.port} dbname=#{db.dbname} user=#{db.user}#{db.password.size != 0 ? ' password='+ db.password : '' }"
+end
 
+def expand_template (template, pretty_json: false)
+  begin
+    contents = ERB.new(File.read(template), nil, '-').result()
+    if pretty_json
+      JSON.pretty_generate(JSON.parse(contents))
+    else
+      contents
+    end
+  rescue Exception => e
+    puts "Expansion of #{template} failed".yellow
+    puts e.message.red
+    exit
+  end
+end
+
+def get_config
   hostname = %x[hostname -s].strip
   @project = Dir.pwd
   @user_home = File.expand_path('~')
@@ -165,13 +213,16 @@ def get_config
   end
 
   #
-  # Pre-cook the connection string
+  # Pre-cook the connection string # TODO remove this after Hydra goes live
   #
-  dbname = conf['db']['dbname']
-  dbuser = conf['db']['user']
-  dbhost = conf['db']['host']
-  dbpass = conf['db']['password'] || ''
-  conf['db']['connection_string'] = "host=#{dbhost} dbname=#{dbname} user=#{dbuser}#{dbpass.size != 0 ? ' password='+ dbpass : '' }"
+  if conf['db']
+    dbname = conf['db']['dbname']
+    dbuser = conf['db']['user']
+    dbhost = conf['db']['host']
+    dbport = conf['db']['port'] || 5432
+    dbpass = conf['db']['password'] || ''
+    conf['db']['connection_string'] = "host=#{dbhost} port=#{dbport} dbname=#{dbname} user=#{dbuser}#{dbpass.size != 0 ? ' password='+ dbpass : '' }"
+  end
 
   #
   # Resolve project and user relative paths
@@ -187,7 +238,7 @@ def get_config
   conf.clean_keys!
 
   ap conf
-  return JSON.parse(conf.to_json, object_class: OpenStruct), conf
+  return JSON.parse(conf.to_json, object_class: SpDataStruct), conf
 end
 
 desc 'Update project configuration: action=overwrite => update system,user,project; action => hotfix update project only; other no change (dryrun)'
@@ -338,7 +389,7 @@ task :configure, [ :action ] do |task, args|
       if m && m.size == 3
         key_l1 = m[1].gsub('-', '_')
         if conf[key_l1].nil? or conf[key_l1][m[2]].nil?
-          puts "Filtered #{m[1]} - #{m[2]} - #{dst_file}".yellow
+          puts "Filtered #{m[1]} - #{m[2]} - #{dst_file}"
           next
         end
       end
@@ -347,7 +398,7 @@ task :configure, [ :action ] do |task, args|
       if m && m.size == 3
         key_l1 = m[1].gsub('-', '_')
         if conf[key_l1].nil?
-          puts "Filtered #{m[1]} - #{m[2]} - #{dst_file}".yellow
+          puts "Filtered #{m[1]} - #{m[2]} - #{dst_file}"
           next
         end
       end
@@ -361,9 +412,8 @@ task :configure, [ :action ] do |task, args|
         end
       end
 
-      # puts "Expanding #{template}".red
       # Now expand the template
-      file_contents = ERB.new(File.read(template), nil, '-').result()
+      file_contents = expand_template(template)
 
       if /.*(nginx-broker|nginx-epaper)\/conf\.d\/(.*)\.conf$/.match(dst_file)
         includes = file_contents.scan(/^\s*include\s+conf\.d\/(.*)\.location\;/)
@@ -393,7 +443,7 @@ task :configure, [ :action ] do |task, args|
         if used_locations.include? m[1]
           # Write text expanded configuration file
           create_directory(File.dirname dst_file)
-          diff_and_write(contents: ERB.new(File.read(template), nil, '-').result(),
+          diff_and_write(contents: expand_template(template),
                          path: dst_file,
                          diff: diff_before_copy,
                          dry_run: dry_run
@@ -427,7 +477,7 @@ task :configure, [ :action ] do |task, args|
       end
       create_directory "#{@config.prefix}/etc/#{@job_name}"
       create_directory "#{@config.prefix}/var/log/#{@job_name}"
-      diff_and_write(contents: JSON.pretty_generate(JSON.parse(ERB.new(File.read(template), nil, '-').result())),
+      diff_and_write(contents: expand_template(template, pretty_json: true),
                      path: "#{@config.prefix}/etc/#{@job_name}/conf.json",
                      diff: diff_before_copy,
                      dry_run: dry_run
@@ -442,7 +492,7 @@ task :configure, [ :action ] do |task, args|
         throw "Missing service file for #{@job_name}"
       end
 
-      diff_and_write(contents: ERB.new(File.read(template)).result(),
+      diff_and_write(contents: expand_template(template),
                      path: "#{@config.prefix}/lib/systemd/system/#{@job_name}@.service",
                      diff: diff_before_copy,
                      dry_run: dry_run

data/sp-job.gemspec

@@ -26,24 +26,26 @@ Gem::Specification.new do |spec|
     f.match(%r{^(test|spec|features)/})
   end
   spec.bindir        = 'bin'
-  spec.executables   = 'queue-job'
+  spec.executables   = ['queue-job', 'unique-file']
   spec.require_paths = ['lib']
-  spec.add_dependency 'concurrent-ruby'
   spec.add_dependency 'os'
-  spec.add_dependency 'sp-duh'
+  spec.add_dependency 'ffi'
   spec.add_dependency 'redis'
   spec.add_dependency 'backburner'
-  spec.add_dependency 'pg'
-  spec.add_dependency 'oauth2'
-  spec.add_dependency 'oauth2-client'
-  spec.add_dependency 'curb'
+  spec.add_dependency 'pg'            unless RUBY_ENGINE == 'jruby'
+  spec.add_dependency 'pg_jruby'      if     RUBY_ENGINE == 'jruby'
+  spec.add_dependency 'curb'          unless RUBY_ENGINE == 'jruby'
+  spec.add_dependency 'oauth2'        unless RUBY_ENGINE == 'jruby'
+  spec.add_dependency 'oauth2-client' unless RUBY_ENGINE == 'jruby'
   spec.add_dependency 'mail'
-  spec.add_dependency 'rails'
   spec.add_dependency 'json'
+  spec.add_dependency 'jwt'
   spec.add_dependency 'awesome_print'
   spec.add_dependency 'rollbar'
   spec.add_dependency 'roadie'
 
+  spec.add_development_dependency 'ruby-debug' if     RUBY_ENGINE == 'jruby'
+  spec.add_development_dependency 'byebug'     unless RUBY_ENGINE == 'jruby'
   spec.add_development_dependency 'bundler', '~> 1.14'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: sp-job
 version: !ruby/object:Gem::Version
-  version: 0.1.17
+  version: 0.2.2
 platform: ruby
 authors:
 - Eurico Inocencio, Vitor Pinho
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-06 00:00:00.000000000 Z
+date: 2018-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
+  name: os
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: os
+  name: ffi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sp-duh
+  name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redis
+  name: backburner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: backburner
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: curb
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: curb
+  name: mail
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mail
+  name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -151,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: jwt
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -165,7 +165,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: json
+  name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -179,7 +179,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: rollbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -193,7 +193,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rollbar
+  name: roadie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -207,13 +207,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: roadie
+  name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -267,6 +267,7 @@ email:
 - eurico.inocencio@gmail.com, vitor.pinho@servicepartner.pt
 executables:
 - queue-job
+- unique-file
 extensions: []
 extra_rdoc_files: []
 files:
@@ -281,6 +282,7 @@ files:
 - bin/console
 - bin/queue-job
 - bin/setup
+- bin/unique-file
 - lib/sp-job.rb
 - lib/sp/job.rb
 - lib/sp/job/back_burner.rb
@@ -288,12 +290,16 @@ files:
 - lib/sp/job/broker_http_client.rb
 - lib/sp/job/broker_oauth2_client.rb
 - lib/sp/job/common.rb
-- lib/sp/job/engine.rb
+- lib/sp/job/job_db_adapter.rb
 - lib/sp/job/jsonapi_error.rb
+- lib/sp/job/jwt.rb
+- lib/sp/job/mail_queue.rb
 - lib/sp/job/pg_connection.rb
+- lib/sp/job/unique_file.rb
 - lib/sp/job/uploaded_image_converter.rb
 - lib/sp/job/version.rb
 - lib/sp/job/worker.rb
+- lib/sp/job/worker_thread.rb
 - lib/tasks/configure.rake
 - sp-job.gemspec
 homepage: https://github.com/vpfpinho/sp-job