spiderfw 0.6.5 → 0.6.6

data/apps/master/views/installation.shtml

@@ -1,9 +1,9 @@
 <div>
-    <div class="actions">
+    <div class="actions" sp:if="@customer">
         <a href="{ Master.url }/customers/{ @customer.id }">_(Back to customer)</a>
     </div>
     <div sp:if="!@installation">
-        <h3>_(New installation for %s) % @customer.name</h3>
+        <h2>_(New installation for %s) % @customer.name</h2>
         <form action="" method="POST">
             <div class="row">
                 <label for="installation_name">_(Name):</label>
@@ -12,24 +12,36 @@
             </div>
         </form>
     </div>
-    <div sp:if="@installation">
-        <h3 sp:if="@installation">_(Installation "%s" for %s) % [@installation.name, @customer.name] </h3>
-        <core:tabs id="install_tabs">
-            <tab id="edit" label="_(Edit installation)">
+    <tpl:pass sp:if="@edit">
+           <forms:form id="installation_form" model="Spider::Master::Installation" pk="@pk" auto_redirect="true"/>
+       </tpl:pass>
+    <div sp:if="@installation && !@edit">
+        <div class="page-title title-block">
+            <h2>_(Installation "%s") % [@installation]</h2>
+            <a href="?edit" class="manage edit">_(Edit)</a>
+        </div>
+        <core:tabs id="tabs">
+            <tab label="_(Apps)" id="apps">
                 <div class="apps">
                     <h4>_(Apps)</h4>
-                    <form action="" method="POST">
-                        <ul>
-                            <li sp:each="@apps |app|">
-                                <input type="checkbox" name="apps[{ app.app_id }]" value="true" sp:attr-if="@install_apps.include?(app.app_id),checked">
-                                { app.name }
-                            </li>
-                        </ul>
-                        <input type="submit" name="save_apps" value="_(Save)">
-                    </form>
+                    <ul>
+                        <li sp:each="@install_apps |id, details|">
+                            { @apps[id].name } { details["version"] }
+                        </li>
+                    </ul>
                 </div>
             </tab>
-            <tab id="targets" label="_(Targets)">
+            <tab label="_(Configuration)" id="config">
+                <div class="config">
+                    <h4>_(Configuration)</h4>
+                    <textarea>{ @installation.configuration }</textarea>
+                </div>
+            </tab>
+            <tab label="_(Logs)" id="logs">
+                <div class="logs">
+                    <h4>_(Logs)</h4>
+                    <core:table id="logs_table" queryset="@logs" sort="time,desc" />
+                </div>
             </tab>
         </core:tabs>
     </div>

data/apps/master/views/master.layout.shtml

@@ -18,7 +18,8 @@
                 <ul id="top-nav">
                     <li><a href="{ Master.url }">_(Home)</a></li>
                     <li><a href="{ Master.url }/customers" sp:if="@user.global?">_(Customers)</a></li>
-                    <li><a href="{ Master.url }/servants">_(Servants)</a></li>
+                    <li><a href="{ Master.url }/servers">_(Servers)</a></li>
+                    <li><a href="{ Master.url }/installs">_(Installs)</a></li>
                 </ul>
                 <ul id="navigation">
                     <li sp:each_with_index="@navigation |nav, i|">

data/apps/master/views/plugin_data.shtml

@@ -12,7 +12,7 @@
         <div id="chart_setup">
             <h5>_(Compare):</h5>
             <select id="chart_add_column" name="add_column">
-                <tpl:pass sp:each="@servant.scout_plugins |plugin|">
+                <tpl:pass sp:each="@server.scout_plugins |plugin|">
                     <option value="" disabled="disabled">{ plugin.name }</option>
                     <option sp:each="plugin.plugin.metadata |k, d|" 
                         value="{ plugin.id }|{ k }">--- { (d["label"].blank? ? k : d["label"]) }</option>

data/apps/master/views/{servant.shtml → server.shtml}

@@ -1,36 +1,36 @@
 <div>
-    <tpl:pass sp:if="@servant">
+    <tpl:pass sp:if="@server">
         <div class="page-title title-block">
-            <h2>{ @servant.name }</h2>
+            <h2>{ @server.name }</h2>
             <tpl:pass sp:if="!@edit">
                 <a href="?edit" class="manage edit">_(Edit)</a>
             </tpl:pass>
         </div>
         <div class="info-block main">
             <div class="row">
-                <label>Uuid:</label> { @servant.uuid }
+                <label>Uuid:</label> { @server.uuid }
             </div>
-            <div class="row" sp:if="@servant.customer">
+            <div class="row" sp:if="@server.customer">
                 <label>_(Customer):</label>
-                <a href="{ Master.url }/customers/{ @servant.customer.id }">
-                    { @servant.customer }
+                <a href="{ Master.url }/customers/{ @server.customer.id }">
+                    { @server.customer }
                 </a>
             </div>
         </div>
     </tpl:pass>
     <div class="page-title title-block">
-        <h2 sp:if="!@servant">
-            _(New servant) <tpl:pass sp:if="@customer">_(for customer %s) % @customer.name</tpl:pass>
+        <h2 sp:if="!@server">
+            _(New server) <tpl:pass sp:if="@customer">_(for customer %s) % @customer.name</tpl:pass>
         </h2>
     </div>
     <tpl:pass sp:if="@edit">
-        <forms:form id="servant_form" model="Spider::Master::Servant" pk="@pk" auto_redirect="true"/>
+        <forms:form id="server_form" model="Spider::Master::Server" pk="@pk" auto_redirect="true"/>
     </tpl:pass>
     <tpl:pass sp:if="!@edit">
         <div class="list-title title-block">
             <h3>_(Sites)</h3>
-            <a href="{ Master.url_for_servant(@servant) }/sites/create" class="add manage">_(New site)</a>
-            <a href="{ Master.url_for_servant(@servant) }/sites/create" class="add manage">_(Add existing site)</a>
+            <a href="{ Master.url_for_server(@server) }/sites/create" class="add manage">_(New site)</a>
+            <a href="{ Master.url_for_server(@server) }/sites/create" class="add manage">_(Add existing site)</a>
         </div>
         <table class="list-table">
             <!-- <tr sp:each="@servant.scout_plugins |plugin|">
@@ -56,10 +56,10 @@
             </div>
         </div>
         <table class="list-table">
-            <tr sp:each="@servant.scout_plugins |plugin|">
+            <tr sp:each="@server.scout_plugins |plugin|">
                 <td>
-                    <a href="{ Master.url }/servants/{ @servant.id }/plugins/{ plugin.id }">{ plugin.name }</a>
-                    <a href="{ @servant.id }?remove_plugin={ plugin.id }" class="manage remove">_(Remove)</a>
+                    <a href="{ Master.url }/server/{ @server.id }/plugins/{ plugin.id }">{ plugin.name }</a>
+                    <a href="{ @server.id }?remove_plugin={ plugin.id }" class="manage remove">_(Remove)</a>
                 </td>
             </tr>
         </table>

data/apps/master/views/servers.shtml

@@ -0,0 +1,11 @@
+<div>
+    <h3>Servers</h3>
+    <div class="actions">
+        <a href="{ Master.url }/servers/new">_(New server)</a>
+    </div>
+    <ul>
+        <li sp:each="@servers |server|">
+            <a href="{ Master.url }/servers/{ servant.id }">{ server.name }</a>
+        </li>
+    </ul>
+</div>

data/apps/master/views/site_edit.shtml

@@ -1,7 +1,7 @@
 <div class="site-edit">
     <div class="page-title title-block">
         <h2 sp:if="@site && @site.id">_(Edit Site %s) % @site.name</h2>
-        <h2 sp:if="!@site || !@site.id">_(Create new Site for %s) % @servant.name</h2>
+        <h2 sp:if="!@site || !@site.id">_(Create new Site for %s) % @server.name</h2>
     </div>
     <div sp:if="!@site_type">
         _(Choose site type):

data/apps/master/views/trigger_edit.shtml

@@ -4,13 +4,13 @@
         <h2 sp:if="!@trigger.id">_(New Trigger for plugin %s) % @instance.name</h2>
     </div>
     <form method="post" class="spider forms form">
-        <div class="row" sp:if="@servant.customer">
+        <div class="row" sp:if="@server.customer">
             <label>_(Customer):</label>
-            <span class="data">{ @servant.customer.name }</span>
+            <span class="data">{ @server.customer.name }</span>
         </div>
         <div class="row">
-            <label>_(Servant):</label>
-            <span class="data">{ @servant.name }</span>
+            <label>_(Server):</label>
+            <span class="data">{ @server.name }</span>
         </div>
          <div class="row">
             <label>_(Plugin):</label>

data/apps/messenger/controllers/mixins/messenger_helper.rb

@@ -29,9 +29,9 @@ module Spider; module Messenger
         
         def self.send_email(klass, template, scene, from, to, headers={}, attachments=[], params={})
             path_txt = klass.find_resource_path(:email, template+'.txt')
-            path_txt = nil unless File.exist?(path_txt)
+            path_txt = nil unless path_txt && File.exist?(path_txt)
             path_html = klass.find_resource_path(:email, template+'.html')
-            path_html = nil unless File.exist?(path_html)
+            path_html = nil unless path_html && File.exist?(path_html)
             scene_binding = scene.instance_eval{ binding }
             if (path_txt || path_html)
                 text = ERB.new(IO.read(path_txt)).result(scene_binding) if path_txt
@@ -83,4 +83,4 @@ module Spider; module Messenger
         
     end
     
-end; end
+end; end

data/apps/servant/Gemfile

@@ -0,0 +1 @@
+gem "httpclient"

data/apps/servant/_init.rb

@@ -1,5 +1,9 @@
 module Spider
     module Servant
+        @short_name = 'servant'
         include Spider::App
+        @controller = :ServantController
     end
-end
+end
+
+require 'apps/servant/controllers/servant_controller'

data/apps/servant/cmd.rb

@@ -1,4 +1,5 @@
 require 'cmdparse'
+require 'apps/servant/lib/client'
 
 module Spider; module Servant
 
@@ -7,53 +8,21 @@ module Spider; module Servant
         def initialize
             super('servant', true)
             
-            status = CmdParse::Command.new('status', false )
-            status.short_desc = _("Check worker process status")
-            status.set_execution_block do
-                pid = Worker.running?
-                if (pid)
-                    puts "Worker running (#{pid})"
-                else
-                    puts "Worker not running"
+            ping = CmdParse::Command.new('ping', false )
+            ping.short_desc = _("Ping server")
+            ping.set_execution_block do |args|
+                require 'spiderfw/spider'
+                Spider.init_base
+                url = args.first || Spider.config.get('spider.master.url')
+                unless url
+                    puts _("No url provided, exiting.")
+                    exit
                 end
+                servant = Spider::Servant::Client.new(url)
+                servant.ping_server
             end
-            self.add_command(status)
-            
-            start = CmdParse::Command.new('start', false)
-            start.short_desc = _("Start worker")
-            start.options = CmdParse::OptionParserWrapper.new do |opt|
-                opt.on("--daemonize", _("Daemonize worker"), "-d"){ |d|
-                    @daemonize = true
-                }
-            end
-            start.set_execution_block do
-                if (@daemonize)
-                    Worker.options[:fork] = true
-                    Worker.options[:detach] = true
-                else
-                    Worker.options[:fork] = false
-                    Worker.options[:detach] = false
-                end
-                Worker.app_startup
-                if (@daemonize)
-                    STDIN.reopen "/dev/null"       # Free file descriptors and
-                    STDOUT.reopen "/dev/null", "a" # point them somewhere sensible
-                    STDERR.reopen STDOUT           # STDOUT/STDERR should go to a logfile
-                else
-                    # trap('TERM') { Worker.app_shutdown }
-                    #                     trap('INT') { Worker.app_shutdown }
-                    Worker.join
-                end
-            end
-            self.add_command(start)
-            
-            stop = CmdParse::Command.new('stop', false)
-            stop.short_desc = _("Stop worker")
-            stop.set_execution_block do
-                Worker.app_shutdown
-            end
-            self.add_command(stop)
-            
+            self.add_command(ping)
+    
             
         end
         

data/apps/servant/config/options.rb

@@ -0,0 +1 @@
+Spider.config_option 'spider.master.url', _('Url of spider master')

data/apps/servant/controllers/servant_controller.rb

@@ -1,11 +1,17 @@
 module Spider; module Servant
     
     
-    class ServantController
+    class ServantController < Spider::Controller
         
         def command
         end
         
+        __.action
+        def broken
+            a = 0
+            a / 0
+        end
+        
     end
     
     

data/apps/servant/lib/client.rb

@@ -0,0 +1,55 @@
+require 'rubygems'
+require 'httpclient'
+
+module Spider
+    
+    module Servant
+        
+        class Client
+        
+            def initialize(url)
+                @url = url
+            end
+        
+            def ping_server(url=@url)
+                clnt = HTTPClient.new
+                status = Servant.status
+                status[:apps] = status[:apps].to_json
+                last_check_file = File.join(Spider.paths[:var], 'memory', 'servant_last_check')
+                if File.exists?(last_check_file)
+                    last_check = Time.parse(File.read(last_check_file))
+                    status[:interval] = Time.now - last_check
+                end
+                status[:log] = Servant.gather_log.to_json
+                status[:configuration] = compress_string(Servant.gather_configuration)
+                res = clnt.post("#{url}/ping", Spider::HTTP.params_to_hash(status))
+                unless res.status == 200
+                    puts _("The server responded with: %s - %s") % [res.status, res.reason]
+                    exit
+                end
+                File.open(last_check_file, 'w'){ |f| f << Time.now.to_s }
+                res = JSON.parse(res.content)
+                if res[:commands]
+                    res[:commands].each do |command|
+                        command_result = execute_command(command[:name], command[:arguments])
+                        clnt.post()
+                    end
+                end
+            end
+            
+            private
+            
+            def compress_string(str)
+                compr = ""
+                gz = Zlib::GzipWriter.new(StringIO.new(compr))
+                gz.write str
+                gz.close
+                compr
+            end
+        
+            
+        end
+        
+    end
+    
+end

data/apps/servant/servant.appspec

@@ -1,4 +1,4 @@
 name "Servant"
 version "0.1"
 author "Ivan Pirlik"
-gems "ohai >0.3", "httpclient"
+gems "httpclient"

data/apps/servant/servant.rb

@@ -0,0 +1,90 @@
+require 'uuidtools'
+
+module Spider
+
+    module Servant
+        
+        def self.install_id
+            uuid_file = File.join(Spider.paths[:var], 'install_id')
+            return File.read(uuid_file) if File.exists?(uuid_file)
+            uuid = UUIDTools::UUID.random_create.to_s
+            File.open(uuid_file, 'w'){ |f| f << uuid }
+            uuid
+        end
+        
+        def self.status
+            Spider.init_base
+            apps = self.apps
+            status = {
+                :install_id => self.install_id,
+                :apps => apps
+            }
+        end
+        
+        def self.apps
+            active = Spider.config.get('apps')
+            apps = Spider.home.apps
+            h = {}
+            apps.each do |k, v|
+                h[k] = {}
+                h[k][:version] = v[:spec].version
+                h[k][:active] = true if active.include?(k)
+            end
+            h
+        end
+        
+        def self.gather_log(level=:WARN)
+            level = level.to_s
+            levels = ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL']
+            lev = levels.index(level)
+            
+            log_file = File.join(Spider.paths[:log], Spider.conf.get('log.file_name'))
+            return [] unless File.exists?(log_file)
+            log_prev_file = File.join(Spider.paths[:var], 'memory', 'servant_log_prev')
+            previous_position = 0
+            if File.exists?(log_prev_file)
+                previous_position = File.read(log_prev_file).to_i
+            end
+            current_position = open(log_file){ |fd| fd.stat.size }
+            if current_position < previous_position
+                previous_position = 0
+            end
+            
+            log_line = /\w, \[(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}) \#(\d+):(-?\d+)\] ([\s\w]{5}) -- : (.+)/
+            lines = []
+
+            File.open(log_file, 'rb') do |f| 
+                # seek to the last position in the log file
+                f.seek(previous_position, IO::SEEK_SET)
+                cnt = 0
+                f.each_line do |line|
+                    prev_cnt = cnt
+                    cnt += line.length
+                    parts = log_line.match(line)
+                    next unless parts
+                    m, time, pid, thread, severity, details = *parts
+                    sev = levels.index(severity.strip)
+                    next unless sev
+                    next unless sev >= lev
+                    lines << [time, severity.strip, details]
+                end
+
+            end # File.open
+            File.open(log_prev_file, 'w'){ |f| f << current_position}
+            lines
+        end
+        
+        def self.gather_configuration
+            res = ""
+            first = true
+            Dir.glob(File.join(Spider.paths[:config], '*.yml')).each do |f|
+                res << "\n\n" unless first
+                res << File.read(f)
+                first = false
+            end
+            res
+        end
+    
+    end
+    
+end

data/apps/worker/cmd.rb

@@ -34,15 +34,20 @@ module Spider; module Worker
                     Worker.options[:fork] = false
                     Worker.options[:detach] = false
                 end
-                Worker.start
+                Spider.conf.set('worker.enable', true)
+                #Worker.start
+                Spider.main_process_startup
                 if (@daemonize)
+                    Worker.start
                     STDIN.reopen "/dev/null"       # Free file descriptors and
                     STDOUT.reopen "/dev/null", "a" # point them somewhere sensible
                     STDERR.reopen STDOUT           # STDOUT/STDERR should go to a logfile
                 else
-                    # trap('TERM') { Worker.app_shutdown }
-                    #                     trap('INT') { Worker.app_shutdown }
+                    Spider.startup
                     Worker.join
+                    # trap('TERM') { Spider.shutdown; exit }
+                    # trap('INT') { Spider.shutdown; exit }
+                    # Worker.join
                 end
             end
             self.add_command(start)
@@ -50,7 +55,7 @@ module Spider; module Worker
             stop = CmdParse::Command.new('stop', false)
             stop.short_desc = _("Stop worker")
             stop.set_execution_block do
-                Worker.app_shutdown
+                Spider.shutdown
             end
             self.add_command(stop)
             

data/apps/worker/worker.rb

@@ -33,7 +33,7 @@ module Spider
         end
         
         def self.app_shutdown
-            return unless Spider.conf.get('worker.enable')
+            return unless @runner || Spider.conf.get('worker.enable')
             @mutex.try_lock || return
             Spider::Logger.info("Shutting down worker in #{Process.pid}")
             if @runner
@@ -41,7 +41,7 @@ module Spider
                     @runner.stop
                     @runner = nil
                 end
-            elsif(File.exist?(@pid_file))
+            elsif File.exist?(@pid_file)
                 begin
                     pid = IO.read(@pid_file).to_i
                     unless pid == Process.pid
@@ -99,14 +99,19 @@ module Spider
                 end
             end
             if (options[:fork])
-                forked = Spider.fork do
+                Spider.logger.debug("Forking worker in #{Process.pid}")
+                @forked = Spider.fork do
                     $0 = 'spider-worker'
-                    trap('TERM') { app_shutdown }
-                    trap('INT') { app_shutdown }
+                    $SPIDER_NO_RESPAWN = true if $SPIDER_SPAWNED
+                    Spider.main_process_startup
+                    Spider.on_main_process_shutdown do
+                        Worker.app_shutdown
+                    end
                     start.call
+                    Spider.logger.debug("Forked worker started")
                     @runner.join if @runner
                 end
-                Process.detach(forked) if (options[:detach])
+                Process.detach(@forked) if (options[:detach])
                 @runner = nil
             else
                 start.call

data/blueprints/bin/spider

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'spiderfw/env'
+require 'spiderfw/cmd/cmd'
+cmd = Spider::CommandLine::Cmd.new
+cmd.parse()

data/blueprints/home/config/config.yml

@@ -1,6 +1,5 @@
 runmode: 'devel'
-session.store: 'file'
-client.text_editor: 'textmate'
+apps: ['core']
 
 storages:
     default: 
@@ -9,4 +8,4 @@ storages:
 set devel:
     template.cache.disable: true
     debugger.start: true
-    debug.console.level: DEBUG
+    log.level: DEBUG

data/blueprints/home/init.rb

@@ -1,3 +1,3 @@
-Spider.load_apps 'core'
+Spider.load_apps
 
 Spider.route_apps