knjappserver 0.0.6

data/lib/include/class_knjappserver.rb

@@ -0,0 +1,455 @@
+require "#{File.dirname(__FILE__)}/class_knjappserver_errors"
+require "#{File.dirname(__FILE__)}/class_knjappserver_logging"
+require "#{File.dirname(__FILE__)}/class_knjappserver_mailing"
+require "#{File.dirname(__FILE__)}/class_knjappserver_threadding"
+require "#{File.dirname(__FILE__)}/class_knjappserver_web"
+require "#{File.dirname(__FILE__)}/class_knjappserver_cleaner"
+
+require "timeout"
+require "digest"
+
+class Knjappserver
+  attr_reader :config, :httpserv, :db, :db_handler, :ob, :translations, :paused, :should_restart, :events, :mod_event, :paused, :db_handler, :gettext, :sessions, :logs_access_pending, :threadpool, :vars, :magic_vars, :types, :eruby_cache
+  attr_accessor :served, :should_restart, :should_restart_done
+  
+  autoload :ERBHandler, "#{File.dirname(__FILE__)}/class_erbhandler"
+  
+  def initialize(config)
+    @config = {
+      :timeout => 30,
+      :default_page => "index.rhtml",
+      :default_filetype => "text/html",
+      :max_requests_working => 20
+    }.merge(config)
+    
+    @config[:timeout] = 30 if !@config.has_key?(:timeout)
+    @config[:engine_knjengine] = true if !@config[:engine_knjengine] and !@config[:engine_webrick] and !@config[:engine_mongrel]
+    raise "No ':doc_root' was given in arguments." if !@config.has_key?(:doc_root)
+    
+    if !@config.has_key?(:handlers)
+      @erbhandler = Knjappserver::ERBHandler.new
+      @config[:handlers] = [
+        {
+          :file_ext => "rhtml",
+          :callback => @erbhandler.method(:erb_handler)
+        },{
+          :path => "/fckeditor",
+          :mount => "/usr/share/fckeditor"
+        }
+      ]
+    end
+    
+    @paused = 0
+    @paused_mutex = Mutex.new
+    @should_restart = false
+    @mod_events = {}
+    @served = 0
+    @mod_files = {}
+    @sessions = {}
+    @eruby_cache = {}
+    
+    @path_knjappserver = File.dirname(__FILE__)
+    if @config[:knjrbfw_path]
+      @path_knjrbfw = @config[:knjrbfw_path]
+    elsif $knjappserver_config and $knjappserver_config["knjrbfw"]
+      @path_knjrbfw = $knjappserver_config["knjrbfw"]
+    else
+      @path_knjrbfw = ""
+    end
+    
+    
+    #If auto-restarting is enabled - start the modified events-module.
+    if @config[:autorestart]
+      paths = [
+        "#{@path_knjappserver}/../knjappserver.rb",
+        "#{@path_knjappserver}/class_knjappserver.rb",
+        "#{@path_knjappserver}/class_customio.rb"
+      ]
+      
+      print "Auto restarting.\n" if @config[:debug]
+      @mod_event = Knj::Event_filemod.new(:wait => 2, :paths => paths) do |event, path|
+        print "File changed - restart server: #{path}\n"
+        @should_restart = true
+        @mod_event.destroy if @mod_event
+      end
+    end
+    
+    @types = {
+      :ico => "image/x-icon",
+      :jpeg => "image/jpeg",
+      :jpg => "image/jpeg",
+      :gif => "image/gif",
+      :png => "image/png",
+      :html => "text/html",
+      :htm => "text/html",
+      :rhtml => "text/html",
+      :css => "text/css",
+      :xml => "text/xml",
+      :js => "text/javascript"
+    }
+    @types = @types.merge(@config[:filetypes]) if @config[:filetypes]
+    
+    
+    files = [
+      "#{@path_knjappserver}/class_httpresp.rb",
+      "#{@path_knjappserver}/class_httpserver.rb",
+      "#{@path_knjappserver}/class_httpsession.rb",
+      "#{@path_knjappserver}/class_session.rb",
+      "#{@path_knjappserver}/class_log.rb",
+      "#{@path_knjappserver}/class_log_access.rb",
+      "#{@path_knjappserver}/class_log_data_value.rb",
+      "#{@path_knjrbfw}knjrbfw.rb",
+      "#{@path_knjrbfw}knj/objects.rb",
+      "#{@path_knjrbfw}knj/web.rb",
+      "#{@path_knjrbfw}knj/datet.rb",
+      "#{@path_knjrbfw}knj/thread.rb",
+      "#{@path_knjrbfw}knj/threadhandler.rb",
+      "#{@path_knjrbfw}knj/knjdb/libknjdb.rb"
+    ]
+    files.each do |file|
+      STDOUT.print "Loading: '#{file}'.\n" if @config[:debug]
+      
+      if @config[:autorestart]
+        self.loadfile(file)
+      else
+        require file
+      end
+    end
+    
+    
+    print "Setting up database.\n" if @config[:debug]
+    if @config[:db].is_a?(Knj::Db)
+      @db = @config[:db]
+    elsif @config[:db].is_a?(Hash)
+      @db = Knj::Db.new(@config[:db])
+    else
+      raise "Unknown object given as db: '#{@config[:db].class.name}'."
+    end
+    
+    
+    print "Starting objects.\n" if @config[:debug]
+    @ob = Knj::Objects.new(
+      :db => db,
+      :class_path => @path_knjappserver,
+      :module => Knjappserver,
+      :datarow => true,
+      :knjappserver => self
+    )
+    @ob.events.connect(:no_date) do |event, classname|
+      "[no date]"
+    end
+    
+    
+    if @config[:httpsession_db_args]
+      @db_handler = Knj::Db.new(@config[:httpsession_db_args])
+    else
+      @db_handler = @db
+    end
+    
+    
+    #Start the Knj::Gettext_threadded- and Knj::Translations modules for translations.
+    print "Loading Gettext and translations.\n" if @config[:debug]
+    @translations = Knj::Translations.new(:db => @db)
+    if @config[:locales_root]
+      @gettext = Knj::Gettext_threadded.new("dir" => config[:locales_root])
+    end
+    
+    if @config[:locales_gettext_funcs]
+      require "#{@path_knjappserver}/gettext_funcs"
+    end
+    
+    if @config[:magic_methods] or !@config.has_key?(:magic_methods)
+      print "Loading magic-methods.\n" if @config[:debug]
+      require "#{@path_knjappserver}/magic_methods"
+    end
+    
+    if @config[:customio] or !@config.has_key?(:customio)
+      print "Loading custom-io.\n" if @config[:debug]
+      require "#{@path_knjappserver}/class_customio.rb"
+      cio = Knjappserver::CustomIO.new
+      $stdout = cio
+    end
+    
+    
+    #Save the PID to the run-file.
+    print "Setting run-file.\n" if @config[:debug]
+    require "tmpdir"
+    tmpdir = "#{Dir.tmpdir}/knjappserver"
+    tmppath = "#{tmpdir}/run_#{@config[:title]}"
+    
+    Dir.mkdir(tmpdir) if !File.exists?(tmpdir)
+    File.open(tmppath, "w") do |fp|
+      fp.write(Process.pid)
+    end
+    
+    
+    #Set up various events for the appserver.
+    print "Loading events.\n" if @config[:debug]
+    @events = Knj::Event_handler.new
+    @events.add_event(
+      :name => :check_page_access,
+      :connections_max => 1
+    )
+    @events.add_event(
+      :name => :ob,
+      :connections_max => 1
+    )
+    
+    
+    #Set up the 'vars'-variable that can be used to set custom global variables for web-requests.
+    @vars = Knj::Hash_methods.new
+    @magic_vars = {}
+    
+    
+    #Initialize the various feature-modules.
+    print "Init threadding.\n" if @config[:debug]
+    initialize_threadding
+    
+    print "Init mailing.\n" if @config[:debug]
+    initialize_mailing
+    
+    print "Init errors.\n" if @config[:debug]
+    initialize_errors
+    
+    print "Init logging.\n" if @config[:debug]
+    initialize_logging
+    
+    print "Init cleaner.\n" if @config[:debug]
+    initialize_cleaner
+    
+    
+    #Start the appserver.
+    print "Spawning appserver.\n" if @config[:debug]
+    @httpserv = Knjappserver::Httpserver.new(self)
+    
+    
+    #Clear memory at exit.
+    at_exit do
+      self.stop
+    end
+    
+    
+    print "Appserver spawned.\n" if @config[:debug]
+  end
+  
+  def loadfile(fpath)
+    if !@config[:autorestart]
+      require fpath
+      return nil
+    end
+    
+    rpath = Knj::Php.realpath(fpath)
+    raise "No such filepath: #{fpath}" if !rpath or !File.exists?(rpath)
+    
+    return true if @mod_files[rpath]
+    
+    @mod_event.args[:paths] << rpath
+    @mod_files = rpath
+    
+    require rpath
+    return false
+  end
+  
+  def start
+    print "Starting appserver.\n" if @config[:debug]
+    Thread.current[:knjappserver] = {:kas => self} if !Thread.current[:knjappserver]
+    
+    if @config[:autoload]
+      print "Autoloading #{@config[:autoload]}\n" if @config[:debug]
+      require @config[:autoload]
+    end
+    
+    begin
+      @threadpool.start if @threadpool
+      print "Threadpool startet.\n" if @config[:debug]
+      @httpserv.start
+      print "Appserver startet.\n" if @config[:debug]
+    rescue Interrupt
+      print "Got interrupt - stopping appserver.\n" if @config[:debug]
+      stop
+    end
+  end
+  
+  def stop
+    proc_stop = proc{
+      print "Stopping appserver for real.\n" if @config[:debug]
+      @httpserv.stop if @httpserv and @httpserv.respond_to?(:stop)
+      
+      print "Stopping threadpool.\n" if @config[:debug]
+      @threadpool.stop if @threadpool
+      
+      print "Cleaning out loaded sessions.\n" if @config[:debug]
+      if @sessions
+        @sessions.each do |ip, ip_sessions|
+          ip_sessions.each do |session_hash, session_data|
+            session_data[:dbobj].flush
+            @ob.unset(session_data[:dbobj])
+            session_data[:hash].clear
+            ip_sessions.delete(session_hash)
+            session_data.clear
+          end
+        end
+        @sessions.clear
+      end
+      
+      print "Stopping databases.\n" if @config[:debug]
+      @db.destroy if @db.is_a?(Knj::Threadhandler)
+      @db.close if @db.is_a?(Knj::Db)
+      
+      @db_handler.destroy if @db.is_a?(Knj::Threadhandler)
+      @db_handler.close if @db_handler.is_a?(Knj::Db)
+    }
+    
+    #If we cant get a paused-execution in 10 secs - we just force the stop.
+    begin
+      Timeout.timeout(10) do
+        self.paused_exec do
+          proc_stop.call
+        end
+      end
+    rescue Timeout::Error
+      proc_stop.call
+    end
+  end
+  
+  # Stop running any more http requests - make them wait.
+  def pause
+    @paused += 1
+  end
+  
+  def unpause
+    @paused -= 1
+  end
+  
+  def paused?
+    return true if @paused > 0
+    return false
+  end
+  
+  def paused_exec
+    self.pause
+    
+    begin
+      loop do
+        if @httpserv.working_count > 0
+          sleep 0.2
+          next
+        end
+        
+        @paused_mutex.synchronize do
+          yield
+        end
+        
+        break
+      end
+    ensure
+      self.unpause
+    end
+  end
+  
+  def working?
+    return true if @httpserv and @httpserv.working_count > 0
+    return false
+  end
+  
+  def self.data
+    raise "Could not register current thread." if !Thread.current[:knjappserver]
+    return Thread.current[:knjappserver]
+  end
+  
+  def session_fromid(args)
+    ip = args[:ip].to_s
+    idhash = args[:idhash].to_s
+    ip = "bot" if idhash == "bot"
+    
+    @sessions[ip] = {} if !@sessions.has_key?(ip)
+    
+    if !@sessions[ip].has_key?(idhash)
+      session = @ob.get_by(:Session, {"idhash" => args[:idhash]})
+      if !session
+        session = @ob.add(:Session, {
+          :idhash => idhash,
+          :ip => ip
+        })
+      end
+      
+      @sessions[ip][idhash] = {
+        :dbobj => session,
+        :hash => {}
+      }
+    end
+    
+    @sessions[ip][idhash][:time_lastused] = Time.now
+    return @sessions[ip][idhash]
+  end
+  
+  def trans(obj, key)
+    args = {}
+    args[:locale] = _session[:locale] if _session[:locale] and !args[:locale]
+    args[:locale] = _httpsession.data[:locale] if _httpsession.data[:locale] and !args[:locale]
+    @translations.get(obj, key, args)
+  end
+  
+  def trans_set(obj, values)
+    args = {}
+    args[:locale] = _session[:locale] if _session[:locale] and !args[:locale]
+    args[:locale] = _httpsession.data[:locale] if _httpsession.data[:locale] and !args[:locale]
+    @translations.set(obj, values, args)
+  end
+  
+  def trans_del(obj)
+    @translations.delete(obj)
+  end
+  
+  def import(filepath)
+    _httpsession.eruby.import(filepath)
+  end
+  
+  def update_db
+    require "rubygems"
+    require "#{@config[:knjdbrevision_path]}knjdbrevision"
+    
+    dbschemapath = "#{File.dirname(__FILE__)}/../files/database_schema.rb"
+    raise "'#{dbschemapath}' did not exist." if !File.exists?(dbschemapath)
+    require dbschemapath
+    raise "No schema-variable was spawned." if !$tables
+    
+    dbpath = "#{File.dirname(__FILE__)}/../files/database.sqlite3"
+    dbrev = Knjdbrevision.new
+    dbrev.init_db($tables, @db)
+  end
+  
+  def join
+    raise "No http-server or http-server not running." if !@httpserv or !@httpserv.thread_accept
+    
+    begin
+      @httpserv.thread_accept.join
+      @httpserv.thread_restart.join
+    rescue Interrupt
+      stop
+    end
+    
+    if @should_restart
+      loop do
+        if @should_restart_done
+          STDOUT.print "Ending join because the restart is done.\n"
+          break
+        end
+        
+        sleep 1
+      end
+    end
+  end
+  
+  def define_magic_var(method_name, var)
+    @magic_vars[method_name] = var
+    
+    if !Object.respond_to?(method_name)
+      Object.send(:define_method, method_name) do
+        return Thread.current[:knjappserver][:kas].magic_vars[method_name] if Thread.current[:knjappserver] and Thread.current[:knjappserver][:kas]
+        return $knjappserver[:knjappserver].magic_vars[method_name] if $knjappserver and $knjappserver[:knjappserver]
+        raise "Could not figure out the object: '#{method_name}'."
+      end
+    end
+  end
+end

data/lib/include/class_knjappserver_cleaner.rb

@@ -0,0 +1,109 @@
+class Knjappserver
+	def initialize_cleaner
+    if @config[:autorestart]
+      time = 1
+    else
+      time = 15
+    end
+    
+    #This should not be runned via _kas.timeout because timeout wont run when @should_restart is true! - knj
+    Knj::Thread.new do
+      loop do
+        sleep time
+        
+        if @config.has_key?(:restart_when_used_memory) and !@should_restart
+          mbs_used = (Knj::Php.memory_get_usage / 1024) / 1024
+          #STDOUT.print "Used: #{mbs_used}\n"
+          
+          if mbs_used.to_i >= @config[:restart_when_used_memory].to_i
+            STDOUT.print "Memory is over #{@config[:restart_when_used_memory]} - restarting.\n"
+            @should_restart = true
+          end
+        end
+        
+        if @should_restart and !@should_restart_done and !@should_restart_runnning
+          begin
+            @should_restart_runnning = true
+            
+            #When we begin to restart it should go as fast as possible - so start by flushing out any emails waiting...
+            STDOUT.print "Flushing mails.\n"
+            self.mail_flush
+            
+            #Lets try to find a time where no thread is working within the next 30 seconds. If we cant - we interrupt after 10 seconds and restart the server.
+            begin
+              Timeout.timeout(30) do
+                loop do
+                  working_count = self.httpserv.working_count
+                  working = false
+                  
+                  if working_count > 0
+                    working = true
+                    STDOUT.print "Someone is working - wait two sec and try to restart again!\n"
+                  end
+                  
+                  if !working
+                    STDOUT.print "Found window where no sessions were active - restarting!\n"
+                    break
+                  else
+                    sleep 0.2
+                  end
+                  
+                  STDOUT.print "Trying to find window with no active sessions to restart...\n"
+                end
+              end
+            rescue Timeout::Error
+              STDOUT.print "Could not find a timing window for restarting... Forcing restart!\n"
+            end
+            
+            #Flush emails again if any are pending (while we tried to find a window to restart)...
+            STDOUT.print "Flushing mails.\n"
+            self.mail_flush
+            
+            STDOUT.print "Stopping appserver.\n"
+            self.stop
+            
+            STDOUT.print "Figuring out restart-command.\n"
+            mycmd = @config[:restart_cmd]
+            
+            if !mycmd or mycmd.to_s.strip.length <= 0
+              fpath = Knj::Php.realpath(File.dirname(__FILE__) + "/../knjappserver.rb")
+              mycmd = Knj::Os.executed_cmd
+              
+              STDOUT.print "Previous cmd: #{mycmd}\n"
+              mycmd = mycmd.gsub(/\s+knjappserver.rb/, " #{Knj::Strings.unixsafe(fpath)}")
+            end
+            
+            STDOUT.print "Restarting knjAppServer with command: #{mycmd}\n"
+            @should_restart_done = true
+            print exec(mycmd)
+            exit
+          rescue Exception => e
+            STDOUT.puts e.inspect
+            STDOUT.puts e.backtrace
+          end
+        end
+      end
+    end
+    
+    self.timeout(:time => 300) do
+      STDOUT.print "Cleaning sessions on appserver.\n" if @config[:debug]
+      
+      self.paused_exec do
+        time_check = Time.now.to_i - 300
+        @sessions.each do |ip, ip_sessions|
+          ip_sessions.each do |session_hash, session_data|
+            if session_data[:time_lastused].to_i <= time_check
+              session_data[:dbobj].flush
+              @ob.unset(session_data[:dbobj])
+              session_data[:hash].clear
+              ip_sessions.delete(session_hash)
+              session_data.clear
+            end
+          end
+          
+          @sessions.delete(ip) if ip_sessions.empty?
+        end
+      end
+    end
+	end
+end

data/lib/include/class_knjappserver_errors.rb

@@ -0,0 +1,117 @@
+class Knjappserver
+  attr_reader :error_emails_pending
+  
+	def initialize_errors
+		@error_emails_pending = {}
+		@error_emails_pending_mutex = Mutex.new
+		
+		if @config[:error_emails_time]
+      @error_emails_time = @config[:error_emails_time]
+    elsif @config[:debug]
+      @error_emails_time = 5
+    else
+      @error_emails_time = 180
+    end
+		
+		self.timeout(:time => @error_emails_time) do
+			self.flush_error_emails
+		end
+	end
+	
+	def flush_error_emails
+		@error_emails_pending_mutex.synchronize do
+			send_time_older_than = Time.new.to_i - @error_emails_time
+			
+			@error_emails_pending.each do |backtrace_hash, error_email|
+				if send_time_older_than < error_email[:last_time].to_i and error_email[:messages].length < 1000
+					next
+				end
+				
+				@config[:error_report_emails].each do |email|
+          next if !email
+          
+					if error_email[:messages].length == 1
+						html = error_email[:messages].first
+					else
+						html = "<b>First time:</b> #{Knj::Datet.in(error_email[:first_time]).out}<br />"
+						html += "<b>Last time:</b> #{Knj::Datet.in(error_email[:last_time]).out}<br />"
+						html += "<b>Number of errors:</b> #{error_email[:messages].length}<br />"
+						count = 0
+						
+						error_email[:messages].each do |error_msg|
+							count += 1
+							
+							if count > 10
+								html += "<br /><br /><b><i>Limiting to showing 10 out of #{error_email[:messages].length} messages.</i></b>"
+								break
+							end
+							
+							html += "<br /><br />"
+							html += "<b>Message #{count}</b><br />"
+							html += error_msg
+						end
+					end
+					
+					self.mail(
+						:to => email,
+						:subject => error_email[:subject],
+						:html => html,
+						:from => @config[:error_report_from]
+					)
+				end
+				
+				@error_emails_pending.delete(backtrace_hash)
+			end
+		end
+	end
+	
+	def handle_error(e, args = {})
+		@error_emails_pending_mutex.synchronize do
+			if !Thread.current[:knjappserver] or !Thread.current[:knjappserver][:httpsession]
+				STDOUT.print "Error: "
+				STDOUT.puts e.inspect
+				STDOUT.print "\n"
+				STDOUT.puts e.backtrace
+				STDOUT.print "\n\n"
+			end
+			
+			if @config.has_key?(:smtp_args) and @config[:error_report_emails] and (!args.has_key?(:email) or args[:email])
+				backtrace_hash = Knj::ArrayExt.array_hash(e.backtrace)
+				
+				if !@error_emails_pending.has_key?(backtrace_hash)
+					@error_emails_pending[backtrace_hash] = {
+						:first_time => Time.new,
+						:messages => [],
+						:subject => sprintf("Error @ %s", @config[:title]) + " (#{e.message})"
+					}
+				end
+				
+				html = "An error occurred.<br /><br />"
+				html += "<b>#{e.class.name.html}: #{e.message.html}</b><br /><br />"
+				
+				e.backtrace.each do |line|
+					html += line.html + "<br />"
+				end
+				
+				html += "<br /><b>Post:</b><br /><pre>#{Knj::Php.print_r(_post, true)}</pre>" if _post
+				html += "<br /><b>Get:</b><br /><pre>#{Knj::Php.print_r(_get, true)}</pre>" if _get
+				html += "<br /><b>Server:</b><br /><pre>#{Knj::Php.print_r(_server, true).html}</pre>" if _server
+				html += "<br /><b>Cookie:</b><br /><pre>#{Knj::Php.print_r(_cookie, true).html}</pre>" if _meta
+				html += "<br /><b>Session:</b><br /><pre>#{Knj::Php.print_r(_session, true).html}</pre>" if _session
+				html += "<br /><b>Session hash:</b><br /><pre>#{Knj::Php.print_r(_session_hash, true).html}</pre>" if _session_hash
+				
+				error_hash = @error_emails_pending[backtrace_hash]
+				error_hash[:last_time] = Time.new
+				error_hash[:messages] << html
+			end
+		end
+	end
+	
+	def on_error_go_back(&block)
+    begin
+      block.call
+    rescue => e
+      self.alert(e.message).back
+    end
+	end
+end