spiderfw 0.5.13 → 0.5.14

data/apps/messenger/backends/email/smtp.rb

@@ -0,0 +1,76 @@
+require 'apps/messenger/lib/email_backend'
+require 'net/smtp'
+
+module Spider; module Messenger; module Backends; module Email
+    
+    module SMTP
+        include Messenger::EmailBackend
+        
+        def self.send_message(msg)
+            Spider.logger.debug("Sending e-mail #{msg.ticket}")
+            res = false
+            Net::SMTP.start(
+              Spider.conf.get('messenger.smtp.address'),
+              Spider.conf.get('messenger.smpt.port'),
+              Spider.conf.get('messenger.smtp.domain'),
+              Spider.conf.get('messenger.smtp.username'),
+              Spider.conf.get('messenger.smtp.password'),
+              Spider.conf.get('messenger.smtp.auth_scheme')
+            ) do |smtp|
+                msg_str = msg.headers+"\r\n\r\n"+msg.body
+                res = smtp.send_message msg_str, msg.from, msg.to
+            end
+            return res.string
+        end
+        
+        def self.update_statuses
+            log_path = nil
+            return unless log_path = Spider.conf.get('messenger.smtp.log_path')
+            previous_position  = nil
+            memory_file = File.join(Spider.paths[:var], 'messenger_smtp_log_position')
+            begin; previous_position = IO.read(memory_file).to_i; rescue; end
+            current_position = `wc -c #{log_path}`.split(' ')[0].to_i
+            if previous_position and current_position < previous_position
+                # log file rotated - set position to zero
+                previous_position = 0
+            elsif previous_position.nil?
+                # first run
+                previous_position = 0
+            end
+            File.open(memory_file, 'w'){ |f| f << current_position }
+            found = {}
+            File.open(log_path, 'rb') do |f|
+                f.seek(previous_position, IO::SEEK_SET)
+                f.each_line do |line|
+                    if line =~ /\: ([A-F\d]+)\: to(.+)status=(\w+)/
+                        found[$1] = line
+                    end
+                end
+            end
+            Messenger::Email.where{ |e| e.status == :backend }.each do |msg|
+                if msg.backend_id || (msg.backend_response && msg.backend_response =~ /queued as ([A-F\d]+)/)
+                    msg_id = msg.backend_id || msg.backend_response
+                    msg.backend_id ||= msg_id
+                    next unless msg_id
+                    if line = found[msg_id]
+                        msg.backend_response = line
+                        if line =~ /\: ([A-F\d]+)\: to(.+)status=(\w+)/
+                            status = $2
+                        end
+                        if status == 'bounced'
+                            msg.status = :failed
+                        elsif status == 'sent'
+                            msg.status = :sent
+                        end
+                        msg.save
+                    end
+                end
+            end
+            
+            
+        end
+        
+        
+    end
+    
+end; end; end; end

data/apps/messenger/backends/sms/smstools.rb

@@ -0,0 +1,46 @@
+require 'apps/messenger/lib/sms_backend'
+
+module Spider; module Messenger; module Backends; module SMS
+    
+    module SmsTools
+        include Messenger::SMSBackend
+        
+        def self.send_message(msg)
+            Spider.logger.debug("Sending SMS #{msg.ticket}")
+            file_path = File.join(Spider.conf.get('messenger.smstools.path_outgoing'), msg.ticket)
+            File.open(file_path, 'w') do |f|
+                f << "To: +39#{msg.to}\n\n"
+                f << msg.text
+            end
+            File.chmod(0666, file_path)
+            return true
+        end
+        
+        
+        def self.update_statuses
+            Messenger::SMS.where{ |sms| sms.status == :backend }.each do |msg|
+                path_sent = nil
+                path_failed = nil
+                if p_sent = Spider.conf.get('messenger.smstools.path_sent')
+                    path_sent = File.join(p_sent, msg.ticket)
+                end
+                if p_failed = Spider.conf.get('messenger.smstools.path_failed')
+                    path_failed = File.join(p_failed, msg.ticket)
+                end
+                if path_sent && File.exist?(path_sent)
+                    msg.status = :sent
+                    msg.save
+                    File.rm(path_failed) if Spider.conf.get('messenger.smstools.remove_sent')
+                elsif path_failed && File.exist?(path_failed)
+                    msg.add_failure
+                    msg.save
+                    File.rm(path_failed) if Spider.conf.get('messenger.smstools.remove_failed')
+                end
+            end
+        end
+        
+        
+        
+    end
+    
+end; end; end; end

data/apps/messenger/config/options.rb

@@ -7,11 +7,37 @@ module Spider
     config_option 'messenger.smtp.password', _("SMTP authentication password"), :default => nil
     config_option 'messenger.smtp.auth_scheme', _("SMTP authentication scheme"), :default => nil, :type => Symbol,
         :choices => [nil, :plain, :login, :cram_md5]
+    config_option 'messenger.smtp.log_path', _("Smtp logfile (e.g. /var/log/mail.log)"), :default => nil
         
-    config_option 'messenger.email.retries', :type => Fixnum, :default => 5
+    config_option 'messenger.email.retries', _("How many times to retry sending an e-mail"), :type => Fixnum, :default => 5
     config_option 'messenger.email.retry_time', _("Seconds to wait until retry (will be incremented at each retry)"), 
         :type => Fixnum, :default => 10
     config_option 'messenger.queue.run_every', _("Time in seconds between queue runs"), :type => Fixnum, :default => 60
     
+    config_option 'messenger.email.backends', _("The backends to use for sending mail (if more than one)"), :type => Array, :default => ['smtp']
+    config_option 'messenger.email.backend', _("The backend to use for sending sms"), :type => String, :do => lambda{ |val|
+        Spider.conf.set('messenger.email.backends', [val])
+    }
     
-end
+    config_option 'messenger.sms.backends', _("The backends to use for sending sms (if more than one)"), :type => Array, :default => []
+    config_option 'messenger.sms.backend', _("The backend to use for sending sms"), :type => String, :do => lambda{ |val|
+        Spider.conf.set('messenger.sms.backends', [val])
+    }
+    config_option 'messenger.sms.retries', _("How many times to retry sending an sms"), :type => Fixnum, :default => 5
+    config_option 'messenger.sms.retry_time', _("Seconds to wait until retry (will be incremented at each retry)"), 
+        :type => Fixnum, :default => 10
+    config_option 'messenger.smstools.path_spool', _("The path to the smstools 'spool' folder"), :default => '/var/spool/sms'
+    config_option 'messenger.smstools.path_outgoing', _("The path to the smstools 'outgoing' folder"), 
+        :default => lambda{ File.join(Spider.conf.get('messenger.smstools.path_spool'), 'outgoing')}
+    config_option 'messenger.smstools.path_failed', _("The path to the smstools 'failed' folder"), 
+        :default => lambda{ File.join(Spider.conf.get('messenger.smstools.path_spool'), 'failed')}    
+    config_option 'messenger.smstools.path_sent', _("The path to the smstools 'sent' folder"), 
+        :default => lambda{ File.join(Spider.conf.get('messenger.smstools.path_sent'), 'sent')}    
+    config_option 'messenger.smstools.remove_failed', _("Whether to remove failed messages from the smstools failed folder"),
+        :type => Spider::Bool, :default => true
+    config_option 'messenger.smstools.remove_sent', _("Whether to remove failed messages from the smstools sent folder"),
+        :type => Spider::Bool, :default => true
+    config_option 'messenger.smstools.log_path', _("Smsd logfile"), :default => '/var/log/smsd.log'
+
+    
+end

data/apps/messenger/controllers/mixins/messenger_helper.rb

@@ -14,14 +14,17 @@ module Spider; module Messenger
         # attachments must be an array of hashes like 
         # {:file => '/full/file/path', :type => 'mime type', :file_name => 'optional email file name', 
         # :headers => 'optional string or array of additional headers'}
-        def email(template, scene, from, to, headers={}, attachments=[], params={})
-            Spider::Messenger::MessengerHelper.email(self.class, template, scene, from, to, headers, attachments, params)
+        def send_email(template, scene, from, to, headers={}, attachments=[], params={})
+            klass = self.class if self.class.respond_to?(:find_resouce_path)
+            klass ||= self.class.app if self.class.respond_to?(:app)
+            klass ||= Spider.home
+            Spider::Messenger::MessengerHelper.send_email(klass, template, scene, from, to, headers, attachments, params)
         end
         
-        def self.email(klass, template, scene, from, to, headers={}, attachments=[], params={})
+        def self.send_email(klass, template, scene, from, to, headers={}, attachments=[], params={})
             path_txt = klass.find_resource_path(:email, template+'.txt')
             path_txt = nil unless File.exist?(path_txt)
-            path_html = klass.find_resource_path(:email, template+'.txt')
+            path_html = klass.find_resource_path(:email, template+'.html')
             path_html = nil unless File.exist?(path_html)
             scene_binding = scene.instance_eval{ binding }
             if (path_txt || path_html)
@@ -48,7 +51,8 @@ module Spider; module Messenger
                     end
                 end
             end
-            mail_headers, mail_body = mail.to_s.split("\r\n\r\n")
+            mail_headers, mail_body = mail.to_s.split("\r\n\r\n", 2)
+            mail_headers += "\r\n"
             Messenger.email(from, to, mail_headers, mail_body, params)
         end
     end

data/apps/messenger/lib/email_backend.rb

@@ -0,0 +1,14 @@
+require 'apps/messenger/models/email'
+
+module Spider; module Messenger
+   
+   module EmailBackend 
+      
+      def self.included(mod) 
+          Messenger.add_backend(:email, mod)
+      end
+     
+       
+   end
+    
+end; end

data/apps/messenger/lib/sms_backend.rb

@@ -0,0 +1,14 @@
+require 'apps/messenger/models/sms'
+
+module Spider; module Messenger
+   
+   module SMSBackend 
+      
+      def self.included(mod) 
+          Messenger.add_backend(:sms, mod)
+      end
+      
+       
+   end
+    
+end; end

data/apps/messenger/messenger.rb

@@ -1,19 +1,24 @@
-require 'apps/messenger/models/email'
-
-
 module Spider
     
     module Messenger
         
         def self.queues
             {
-                :email => {
-                    :label => _("Email"),
-                    :model => Email
-                }
+                :email => { :label => _("Email"), :model => :Email },
+                :sms => { :label => _("SMS"), :model => :SMS }
             }
         end
         
+        def self.backends
+            @backends ||= {}
+        end
+        
+        def self.add_backend(queue, mod)
+            @backends ||= {}
+            @backends[queue] ||= []
+            @backends[queue] << mod
+        end
+        
         def self.process_queues
             self.queues.each_key do |queue|
                 self.process_queue(queue)
@@ -25,36 +30,30 @@ module Spider
             @mutexes ||= {}
             mutex = @mutexes[queue] ||= Mutex.new
             return if mutex.locked?
-            model = self.queues[queue][:model]
+            model = Spider::Messenger.const_get(self.queues[queue][:model])
             mutex.synchronize do
                 now = DateTime.now
                 list = model.where{ (sent == nil) & (next_try <= now) }
                 list.each do |msg|
                     res = false
                     exc = nil
-                    begin
-                        res = self.send(:"send_#{queue}", msg)
-                    rescue => exc
+                    self.backends[queue].each do |backend|
+                        begin
+                            res = backend.send_message(msg)
+                        rescue => exc
+                            Spider.logger.error(exc)
+                        end
+                        break if res
                     end
                     if (res)
                         msg.sent = now
+                        msg.status = :backend
                         msg.next_try = nil
                         msg.backend_response = res
                         msg.save
                     else
-                        msg.last_try = now
-                        msg.attempts ||= 0
-                        msg.attempts += 1
-                        if (exc)
-                            msg.backend_response = exc.to_s
-                        else
-                            msg.backend_response = res
-                        end
-                        if (msg.attempts >= Spider.conf.get("messenger.#{queue}.retries"))
-                            msg.next_try = nil
-                        else
-                            msg.next_try = msg.last_try.to_local_time + (msg.attempts * Spider.conf.get("messenger.#{queue}.retry_time") * 60)
-                        end
+                        backend_response = exc ? exc.to_s : res
+                        msg.add_failure(backend_response)
                         msg.save
                     end
                             
@@ -78,24 +77,16 @@ module Spider
             return msg
         end
         
-        # Actual SMTP send.
-        def self.send_email(msg)
-            require 'net/smtp'
-            res = false
-            Net::SMTP.start(
-              Spider.conf.get('messenger.smtp.address'),
-              Spider.conf.get('messenger.smpt.port'),
-              Spider.conf.get('messenger.smtp.domain'),
-              Spider.conf.get('messenger.smtp.username'),
-              Spider.conf.get('messenger.smtp.password'),
-              Spider.conf.get('messenger.smtp.auth_scheme')
-            ) do |smtp|
-                msg_str = msg.headers+"\r\n"+msg.body
-                res = smtp.send_message msg_str, msg.from, msg.to
-            end
-            return res.string
+        def self.sms(to, text, params={})
+            msg = SMS.new(
+                :to => to, :text => text
+            )
+            msg.next_try = params[:send_from] || DateTime.now
+            msg.save
+            return msg
         end
         
+        
     end
     
-end
+end

data/apps/messenger/models/email.rb

@@ -10,6 +10,8 @@ module Spider; module Messenger
         element :subject, String, :label => _("Subject"), :computed_from => [:headers]
         element :body, Text, :label => _("Body")
         
+        queue :email
+        
         def subject
             if (self.headers =~ /^Subject: (.+)$/)
                 return $1

data/apps/messenger/models/message.rb

@@ -7,18 +7,18 @@ module Spider; module Messenger
         element :last_try, DateTime, :label => _("Last try")
         element :next_try, DateTime, :label => _("Next try")
         element :attempts, Fixnum, :label => _("Attempts")
+        element :backend, String, :label => _("Backend")
+        element :backend_id, String, :label => _("Backend id")
         element :backend_response, String, :label => _("Last status")
         element :sent, DateTime, :label => _("Sent at")
-        
-        def status
-            if self.sent
-                :sent
-            elsif (self.next_try)
-                :queue
-            else
-                :failed
-            end
-        end
+        element :status, {
+            :retry => _('Retry'),
+            :queued => _('Queued'),
+            :failed => _('Failed'),
+            :backend => _('Handed to backend'),
+            :sent => _('Sent'),
+            :delivered => _('Delivered')
+        }
         
         def self.sent_messages
             self.where{ q.sent .not nil }
@@ -32,13 +32,35 @@ module Spider; module Messenger
             self.where{ (q.sent == nil) & (q.next_try .not nil) }
         end
         
-        with_mapper do
-            def before_save(obj, mode)
-                obj.ticket = UUIDTools::UUID.random_create.to_s if mode == :insert
-                super
+        def self.queue(val=nil)
+            @queue = val if val
+            @queue
+        end
+            
+        
+        def add_failure(backend_response=nil)
+            msg = self
+            msg.last_try = DateTime.now
+            msg.attempts ||= 0
+            msg.attempts += 1
+            queue = self.class.queue
+            if (msg.attempts >= Spider.conf.get("messenger.#{queue}.retries"))
+                msg.next_try = nil
+                msg.status = :failed
+            else
+	        msg.last_try ||= DateTime.now
+                msg.next_try = msg.last_try + (msg.attempts * Spider.conf.get("messenger.#{queue}.retry_time") * 60)
+                msg.status = :retry
             end
         end
         
+        # with_mapper do
+        #     def before_save(obj, mode)
+        #         obj.ticket = UUIDTools::UUID.random_create.to_s if mode == :insert
+        #         super
+        #     end
+        # end
+        
     end
     
-end; end
+end; end

data/apps/messenger/models/sms.rb

@@ -0,0 +1,15 @@
+require 'apps/messenger/models/message'
+
+module Spider; module Messenger
+    
+    class SMS < Message
+        class_table_inheritance :add_polymorphic => true
+        element :to, String, :label => _("To")
+        element :text, Text, :label => _("Text")
+        
+        queue :sms
+        
+                
+    end
+    
+end; end

data/apps/servant/bin/spider-servant.rb

@@ -44,4 +44,5 @@ class Cmd
 end
 
 cmd = Cmd.new
-cmd.parse
+cmd.parse
+exit

data/apps/servant/cmd.rb

@@ -0,0 +1,62 @@
+require 'cmdparse'
+
+module Spider; module Servant
+
+    class Cmd < ::CmdParse::Command
+        
+        def initialize
+            super('servant', true)
+            
+            status = CmdParse::Command.new('status', false )
+            status.short_desc = _("Check worker process status")
+            status.set_execution_block do
+                pid = Worker.running?
+                if (pid)
+                    puts "Worker running (#{pid})"
+                else
+                    puts "Worker not running"
+                end
+            end
+            self.add_command(status)
+            
+            start = CmdParse::Command.new('start', false)
+            start.short_desc = _("Start worker")
+            start.options = CmdParse::OptionParserWrapper.new do |opt|
+                opt.on("--daemonize", _("Daemonize worker"), "-d"){ |d|
+                    @daemonize = true
+                }
+            end
+            start.set_execution_block do
+                if (@daemonize)
+                    Worker.options[:fork] = true
+                    Worker.options[:detach] = true
+                else
+                    Worker.options[:fork] = false
+                    Worker.options[:detach] = false
+                end
+                Worker.app_startup
+                if (@daemonize)
+                    STDIN.reopen "/dev/null"       # Free file descriptors and
+                    STDOUT.reopen "/dev/null", "a" # point them somewhere sensible
+                    STDERR.reopen STDOUT           # STDOUT/STDERR should go to a logfile
+                else
+                    # trap('TERM') { Worker.app_shutdown }
+                    #                     trap('INT') { Worker.app_shutdown }
+                    Worker.join
+                end
+            end
+            self.add_command(start)
+            
+            stop = CmdParse::Command.new('stop', false)
+            stop.short_desc = _("Stop worker")
+            stop.set_execution_block do
+                Worker.app_shutdown
+            end
+            self.add_command(stop)
+            
+            
+        end
+        
+    end
+    
+end; end

data/apps/servant/controllers/servant_controller.rb

@@ -0,0 +1,12 @@
+module Spider; module Servant
+    
+    
+    class ServantController
+        
+        def command
+        end
+        
+    end
+    
+    
+end; end

data/apps/servant/lib/resource.rb

@@ -6,6 +6,9 @@ module Spider; module Servant
             {}
         end
         
+        def self.each_ip
+        end
+        
     end
     
 end; end

data/apps/servant/lib/resources/db/mysql.rb

@@ -26,6 +26,10 @@ module Spider; module Servant; module Resources
             @storage.execute(sql)
         end
         
+        def self.discovery(ip)
+            
+        end
+        
     end
     
 end; end; end

data/apps/servant/lib/resources/db.rb

@@ -30,6 +30,7 @@ module Spider; module Servant; module Resources
             @storage = Spider::Model::Storage.get_storage('db', url)
         end
         
+        # __.command :name => _('Create Db'), :params => [:name => String]
         def create_db(name)
             raise "Unimplemented"
         end

data/apps/servant/lib/servant.rb

@@ -17,7 +17,7 @@ module Spider
         
             def initialize(config_file=nil)
                 @config = {
-                    :server_url => 'http://localhost:8989/spider/master/ping'
+                    :server_url => 'http://localhost:8989/spider/master'
                 }
                 config_file ||= Servant::DEFAULTS[:config_file] if File.exists?(Servant::DEFAULTS[:config_file])
                 if config_file
@@ -34,10 +34,16 @@ module Spider
         
             def ping_server(url=nil)
                 url ||= @config[:server_url]
-
                 @system_status = @ohai.to_json
                 clnt = HTTPClient.new
-                clnt.post(url, Spider::HTTP.params_to_hash(self.description))
+                res = clnt.post("#{url}/ping", Spider::HTTP.params_to_hash(self.description))
+                JSON.parse(res)
+                if res[:commands]
+                    res[:commands].each do |command|
+                        command_result = execute_command(command[:name], command[:arguments])
+                        clnt.post()
+                    end
+                end
             end
         
             def create_resources

data/apps/worker/cmd.rb

@@ -34,7 +34,7 @@ module Spider; module Worker
                     Worker.options[:fork] = false
                     Worker.options[:detach] = false
                 end
-                Worker.app_startup
+                Worker.start
                 if (@daemonize)
                     STDIN.reopen "/dev/null"       # Free file descriptors and
                     STDOUT.reopen "/dev/null", "a" # point them somewhere sensible

data/apps/worker/models/job.rb

@@ -10,9 +10,12 @@ module Spider; module Worker
         }
         
         def run
+            Spider.logger.debug("Running job #{self.uuid}")
             t = self.task.untaint
-            $SAFE = 4
-            eval(t)
+            Thread.start{
+                $SAFE = 3
+                eval(t)
+            }.join
         end
         
     end

data/apps/worker/worker.rb

@@ -23,14 +23,13 @@ module Spider
             @options
         end
         
-        
-        def self.app_startup
-            
+        def self.start
+            @runner = start_runner unless running?
         end
         
         def self.app_startup
             return unless Spider.conf.get('worker.enable')
-            @runner = start_runner unless running?
+            self.start
         end
         
         def self.app_shutdown
@@ -49,7 +48,7 @@ module Spider
                         Spider::Logger.info("Sending TERM signal to worker #{pid}")
                         Process::kill 'TERM', pid
                     end
-                rescue Errno::ENOENT
+                rescue Errno::ENOENT, Errno::ESRCH
                 end
             end
             begin
@@ -80,6 +79,9 @@ module Spider
                             load @scripts_dir+'/'+script
                         end
                     end
+                    Spider::Worker.every("#{Spider.conf.get('worker.jobs_interval')}s") do
+                        Spider::Worker.run_jobs
+                    end
                     Spider.apps.each do |name, mod|
                         if File.directory?(mod.path+'/config/worker')
                             Dir.glob(mod.path+'/config/worker/*.rb').each do |path|

data/blueprints/install/config.ru

@@ -1,4 +1,3 @@
-ENV['LANG'] = 'it_IT.UTF-8'
 require 'spiderfw'
 require 'spiderfw/http/adapters/rack'
 PhusionPassenger.on_event(:starting_worker_process) do