knjappserver 0.0.16 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/VERSION +1 -1
  2. data/bin/knjappserver_start.rb +17 -25
  3. data/knjappserver.gemspec +17 -6
  4. data/lib/conf/conf_example.rb +3 -3
  5. data/lib/files/database_schema.rb +1 -2
  6. data/lib/include/class_customio.rb +8 -25
  7. data/lib/include/class_erbhandler.rb +7 -1
  8. data/lib/include/class_httpserver.rb +50 -34
  9. data/lib/include/class_httpsession.rb +138 -101
  10. data/lib/include/class_httpsession_contentgroup.rb +52 -24
  11. data/lib/include/class_httpsession_http_request.rb +203 -0
  12. data/lib/include/{class_httpresp.rb → class_httpsession_http_response.rb} +27 -28
  13. data/lib/include/class_httpsession_post_multipart.rb +117 -0
  14. data/lib/include/class_knjappserver.rb +146 -96
  15. data/lib/include/class_knjappserver_cleaner.rb +74 -68
  16. data/lib/include/class_knjappserver_cmdline.rb +23 -17
  17. data/lib/include/class_knjappserver_errors.rb +121 -104
  18. data/lib/include/class_knjappserver_leakproxy_client.rb +6 -0
  19. data/lib/include/class_knjappserver_leakproxy_server.rb +56 -0
  20. data/lib/include/class_knjappserver_logging.rb +25 -25
  21. data/lib/include/class_knjappserver_mailing.rb +84 -56
  22. data/lib/include/class_knjappserver_sessions.rb +15 -22
  23. data/lib/include/class_knjappserver_threadding.rb +70 -43
  24. data/lib/include/class_knjappserver_threadding_timeout.rb +20 -4
  25. data/lib/include/class_knjappserver_translations.rb +6 -4
  26. data/lib/include/class_knjappserver_web.rb +87 -35
  27. data/lib/include/class_log.rb +9 -9
  28. data/lib/include/class_log_link.rb +4 -4
  29. data/lib/include/class_session.rb +8 -4
  30. data/lib/include/gettext_funcs.rb +8 -6
  31. data/lib/include/magic_methods.rb +4 -0
  32. data/lib/pages/debug_database_connections.rhtml +46 -0
  33. data/lib/pages/debug_http_sessions.rhtml +40 -0
  34. data/lib/pages/error_notfound.rhtml +12 -0
  35. data/lib/pages/spec.rhtml +1 -1
  36. data/lib/pages/spec_post.rhtml +3 -0
  37. data/lib/pages/spec_thread_joins.rhtml +21 -0
  38. data/lib/pages/spec_threadded_content.rhtml +2 -0
  39. data/lib/pages/tests.rhtml +14 -0
  40. data/lib/scripts/benchmark.rb +25 -8
  41. data/lib/scripts/knjappserver_cgi.rb +60 -0
  42. data/lib/scripts/knjappserver_fcgi.rb +135 -0
  43. data/lib/scripts/leakproxy.rb +27 -0
  44. data/spec/knjappserver_spec.rb +16 -5
  45. data/spec/leakproxy_spec.rb +56 -0
  46. metadata +38 -27
  47. data/lib/include/class_httpsession_knjengine.rb +0 -154
  48. data/lib/include/class_httpsession_mongrel.rb +0 -75
  49. data/lib/include/class_httpsession_webrick.rb +0 -75
@@ -0,0 +1,6 @@
1
+ #This is the part of Leakproxy that actually starts the copy of the Knjappserver.
2
+ class Knjappserver::Leakproxy_client
3
+ def initialize(args = {})
4
+
5
+ end
6
+ end
@@ -0,0 +1,56 @@
1
+ #This class starts a Knjappserver in another process. This process can be used for scripts that leak memory. The memoy-usage is
2
+ #looked over and the process restarted when it reaches a certain point. Doing the restart all waiting requests will wait gracefully.
3
+ class Knjappserver::Leakproxy_server
4
+ def initialize(args)
5
+ require "#{$knjpath}/process"
6
+
7
+ leakproxy_path = "#{File.dirname(__FILE__)}/../scripts/leakproxy.rb"
8
+ executable = Knj::Os.executed_executable
9
+
10
+ @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(executable, leakproxy_path, "r+")
11
+ @kas = args[:kas]
12
+ @config = @kas.config
13
+
14
+ Thread.new do
15
+ STDOUT.print "Doing loop:\n"
16
+ @stderr.each_line do |str|
17
+ STDOUT.print "Test: #{str}"
18
+ end
19
+ end
20
+
21
+ pass_conf = {}
22
+ pass_conf_keys = [:knjrbfw_path]
23
+ pass_conf_keys.each do |key, val|
24
+ pass_conf[key] = val if @config.key?(key)
25
+ end
26
+
27
+ args_pass = {
28
+ :config => pass_conf
29
+ }
30
+
31
+ @stdin.write("#{Marshal.dump(args_pass)}\n")
32
+
33
+ @process = Knj::Process.new(
34
+ :out => @stdin,
35
+ :in => @stdout,
36
+ :err => @stderr,
37
+ :listen => true,
38
+ :debug => true,
39
+ :on_rec => proc{|d|
40
+ obj = d.obj
41
+
42
+ if obj.is_a?(Hash)
43
+ if obj["type"] == "print"
44
+ STDOUT.print obj["str"]
45
+ end
46
+ else
47
+ STDOUT.print Knj::Php.print_r(obj, true)
48
+ end
49
+ }
50
+ )
51
+ end
52
+
53
+ def spawn
54
+
55
+ end
56
+ end
@@ -259,17 +259,17 @@ class Knjappserver
259
259
  links = @ob.list(:Log_link, {"object_class" => obj.class.name, "object_id" => obj.id, "limit" => 500, "orderby" => [["id", "desc"]]})
260
260
 
261
261
  html = "<table class=\"list knjappserver_log_table\">"
262
- html += "<thead>"
263
- html += "<tr>"
264
- html += "<th>ID</th>"
265
- html += "<th>Message</th>"
266
- html += "<th style=\"width: 130px;\">Date &amp; time</th>"
267
- html += "<th>Tag</th>"
268
- html += "<th>Objects</th>" if args[:ob_use]
269
- html += "<th>IP</th>" if args[:show_ip]
270
- html += "</tr>"
271
- html += "</thead>"
272
- html += "<tbody>"
262
+ html << "<thead>"
263
+ html << "<tr>"
264
+ html << "<th>ID</th>"
265
+ html << "<th>Message</th>"
266
+ html << "<th style=\"width: 130px;\">Date &amp; time</th>"
267
+ html << "<th>Tag</th>"
268
+ html << "<th>Objects</th>" if args[:ob_use]
269
+ html << "<th>IP</th>" if args[:show_ip]
270
+ html << "</tr>"
271
+ html << "</thead>"
272
+ html << "<tbody>"
273
273
 
274
274
  links.each do |link|
275
275
  log = link.log
@@ -280,32 +280,32 @@ class Knjappserver
280
280
  classes = ["knjappserver_log", "knjappserver_log_#{log.id}"]
281
281
  classes << "knjappserver_log_multiple_lines" if msg_lines.length > 1
282
282
 
283
- html += "<tr class=\"#{classes.join(" ")}\">"
284
- html += "<td>#{log.id}</td>"
285
- html += "<td>#{first_line.html}</td>"
286
- html += "<td>#{log.date_saved_str}</td>"
287
- html += "<td>#{log.tag.html}</td>"
283
+ html << "<tr class=\"#{classes.join(" ")}\">"
284
+ html << "<td>#{log.id}</td>"
285
+ html << "<td>#{first_line.html}</td>"
286
+ html << "<td>#{log.date_saved_str}</td>"
287
+ html << "<td>#{log.tag.html}</td>"
288
288
 
289
289
  if args[:ob_use]
290
290
  begin
291
- html += "<td>#{log.objects_html(args[:ob_use])}</td>"
291
+ html << "<td>#{log.objects_html(args[:ob_use])}</td>"
292
292
  rescue => e
293
- html += "<td>#{e.message.html}</td>"
293
+ html << "<td>#{e.message.html}</td>"
294
294
  end
295
295
  end
296
296
 
297
- html += "<td>#{log.ip}</td>" if args[:show_ip]
298
- html += "</tr>"
297
+ html << "<td>#{log.ip}</td>" if args[:show_ip]
298
+ html << "</tr>"
299
299
  end
300
300
 
301
301
  if links.empty?
302
- html += "<tr>"
303
- html += "<td colspan=\"2\" class=\"error\">No logs were found for that object.</td>"
304
- html += "</tr>"
302
+ html << "<tr>"
303
+ html << "<td colspan=\"2\" class=\"error\">No logs were found for that object.</td>"
304
+ html << "</tr>"
305
305
  end
306
306
 
307
- html += "</tbody>"
308
- html += "</table>"
307
+ html << "</tbody>"
308
+ html << "</table>"
309
309
 
310
310
  return html
311
311
  end
@@ -2,8 +2,7 @@ class Knjappserver
2
2
  attr_reader :mails_waiting
3
3
 
4
4
  def initialize_mailing
5
- STDOUT.print "Loading mail.\n" if @config[:debug]
6
- require "mail" if !@config.has_key?(:mail_require) or @config[:mail_require]
5
+ require "knj/autoload/ping"
7
6
 
8
7
  @mails_waiting = []
9
8
  @mails_mutex = Mutex.new
@@ -15,6 +14,8 @@ class Knjappserver
15
14
 
16
15
  #Queue a mail for sending. Possible keys are: :subject, :from, :to, :text and :html.
17
16
  def mail(mail_args)
17
+ raise "'smtp_args' has not been given for the Knjappserver." if !@config[:smtp_args]
18
+
18
19
  @mails_queue_mutex.synchronize do
19
20
  count_wait = 0
20
21
  while @mails_waiting.length > 100
@@ -36,28 +37,44 @@ class Knjappserver
36
37
  #Sends all queued mails to the respective servers, if we are online.
37
38
  def mail_flush
38
39
  @mails_mutex.synchronize do
39
- return false if @mails_waiting.length <= 0
40
-
41
- status = Ping.pingecho("google.dk", 10, 80)
42
- if !status
43
- STDOUT.print "We are not online - skipping mail flush.\n"
44
- return false #Dont run if we dont have a connection to the internet and then properly dont have a connection to the SMTP as well.
45
- end
46
-
47
- @mails_waiting.each do |mail|
48
- begin
49
- if mail.send
50
- @mails_waiting.delete(mail)
51
- end
52
- rescue Timeout::Error
53
- #ignore -
54
- rescue => e
55
- @mails_waiting.delete(mail)
56
- self.handle_error(e, {:email => false})
57
- end
58
-
59
- sleep 1 #sleep so we dont take up too much bandwidth.
60
- end
40
+ return false if @mails_waiting.length <= 0
41
+
42
+ status = Ping.pingecho("google.dk", 10, 80)
43
+ if !status
44
+ STDOUT.print "We are not online - skipping mail flush.\n"
45
+ return false #Dont run if we dont have a connection to the internet and then properly dont have a connection to the SMTP as well.
46
+ end
47
+
48
+ begin
49
+ #Use subprocessing to avoid the mail-framework (activesupport and so on, also possible memory leaks in those large frameworks).
50
+ require "knj/process_meta"
51
+ subproc = Knj::Process_meta.new("debug" => @debug, "debug_err" => true)
52
+ subproc.static("Object", "require", "rubygems")
53
+ subproc.static("Object", "require", "mail")
54
+ subproc.static("Object", "require", "#{@config[:knjrbfw_path]}knjrbfw")
55
+ subproc.static("Object", "require", "knj/autoload")
56
+
57
+ STDOUT.print "Flushing emails." if @debug
58
+ @mails_waiting.each do |mail|
59
+ begin
60
+ STDOUT.print "Sending email: #{mail.__id__}\n" if @debug
61
+ if mail.send("proc" => subproc)
62
+ STDOUT.print "Email sent: #{mail.__id__}\n" if @debug
63
+ @mails_waiting.delete(mail)
64
+ end
65
+ rescue Timeout::Error
66
+ #ignore -
67
+ rescue => e
68
+ @mails_waiting.delete(mail)
69
+ self.handle_error(e, {:email => false})
70
+ end
71
+
72
+ sleep 1 #sleep so we dont take up too much bandwidth.
73
+ end
74
+ ensure
75
+ subproc.destroy if subproc
76
+ subproc = nil
77
+ end
61
78
  end
62
79
  end
63
80
 
@@ -77,43 +94,54 @@ class Knjappserver
77
94
  end
78
95
 
79
96
  #Sends the email to the receiver.
80
- def send
97
+ def send(args = {})
81
98
  STDOUT.print "Sending mail '#{__id__}'.\n" if @args[:kas].debug
82
99
 
83
- begin
100
+ if @args[:from]
101
+ from = @args[:from]
102
+ elsif @args[:kas].config[:error_report_from]
103
+ from = @args[:kas].config[:error_report_from]
104
+ else
105
+ raise "Dont know where to take the 'from'-paramter from - none given in appserver config or mail-method-arguments?"
106
+ end
107
+
108
+ if args["proc"]
109
+ args["proc"].static("Object", "require", "knj/mailobj")
110
+ mail = args["proc"].new("Knj::Mailobj", @args[:kas].config[:smtp_args])
111
+ mail._pm_send_noret("to=", @args[:to])
112
+ mail._pm_send_noret("subject=", @args[:subject]) if @args[:subject]
113
+ mail._pm_send_noret("html=", Knj::Strings.email_str_safe(@args[:html])) if @args[:html]
114
+ mail._pm_send_noret("text=", Knj::Strings.email_str_safe(@args[:text])) if @args[:text]
115
+ mail._pm_send_noret("from=", from)
116
+ mail._pm_send_noret("send")
117
+ else
118
+ require "knj/mailobj"
84
119
  mail = Knj::Mailobj.new(@args[:kas].config[:smtp_args])
85
120
  mail.to = @args[:to]
86
121
  mail.subject = @args[:subject] if @args[:subject]
87
- mail.html = @args[:html] if @args[:html]
88
- mail.text = @args[:text] if @args[:text]
89
-
90
- if @args[:from]
91
- mail.from = @args[:from]
92
- elsif @args[:kas].config[:error_report_from]
93
- mail.from = @args[:kas].config[:error_report_from]
94
- else
95
- raise "Dont know where to take the 'from'-paramter from - none given in appserver config or mail-method-arguments?"
96
- end
97
-
98
- mail.send
99
- @args[:status] = :sent
100
- STDOUT.print "Sent email #{self.__id__}\n" if @args[:kas].debug
101
- return true
102
- rescue Exception => e
103
- if @args[:kas].debug
104
- STDOUT.print "Could not send email.\n"
105
- STDOUT.puts e.inspect
106
- STDOUT.puts e.backtrace
107
- end
108
-
109
- @args[:errors][e.class.name] = {:count => 0} if !@args[:errors].has_key?(e.class.name)
110
- @args[:errors][e.class.name][:count] += 1
111
- raise e if @args[:errors][e.class.name][:count] >= 5
112
- @args[:status] = :error
113
- @args[:error] = e
114
-
115
- return false
116
- end
122
+ mail.html = Knj::Strings.email_str_safe(@args[:html]) if @args[:html]
123
+ mail.text = Knj::Strings.email_str_safe(@args[:text]) if @args[:text]
124
+ mail.from = from
125
+ mail.send
126
+ end
127
+
128
+ @args[:status] = :sent
129
+ STDOUT.print "Sent email #{self.__id__}\n" if @args[:kas].debug
130
+ return true
131
+ rescue Exception => e
132
+ if @args[:kas].debug
133
+ STDOUT.print "Could not send email.\n"
134
+ STDOUT.puts e.inspect
135
+ STDOUT.puts e.backtrace
136
+ end
137
+
138
+ @args[:errors][e.class.name] = {:count => 0} if !@args[:errors].has_key?(e.class.name)
139
+ @args[:errors][e.class.name][:count] += 1
140
+ raise e if @args[:errors][e.class.name][:count] >= 5
141
+ @args[:status] = :error
142
+ @args[:error] = e
143
+
144
+ return false
117
145
  end
118
146
  end
119
147
  end
@@ -4,9 +4,7 @@ class Knjappserver
4
4
  end
5
5
 
6
6
  #Returns or adds session based on idhash and meta-data.
7
- def session_fromid(args)
8
- ip = args[:ip].to_s
9
- idhash = args[:idhash].to_s
7
+ def session_fromid(ip, idhash, meta)
10
8
  ip = "bot" if idhash == "bot"
11
9
 
12
10
  if !@sessions.key?(idhash)
@@ -14,42 +12,31 @@ class Knjappserver
14
12
  if !session
15
13
  session = @ob.add(:Session, {
16
14
  :idhash => idhash,
17
- :user_agent => args[:meta]["HTTP_USER_AGENT"],
15
+ :user_agent => meta["HTTP_USER_AGENT"],
18
16
  :ip => ip
19
17
  })
20
18
  end
21
19
 
20
+ hash = {}
22
21
  @sessions[idhash] = {
23
22
  :dbobj => session,
24
- :hash => {}
23
+ :hash => hash
25
24
  }
26
25
  else
27
26
  session = @sessions[idhash][:dbobj]
27
+ hash = @sessions[idhash][:hash]
28
28
  end
29
29
 
30
- =begin
31
- if ip != "bot"
32
- if session[:user_agent] != args[:meta]["HTTP_USER_AGENT"]
33
- STDOUT.print "Invalid user-agent!\n"
34
- STDOUT.print Knj::Php.print_r(session, true)
35
-
36
- raise Knj::Errors::InvalidData, "Invalid user-agent."
37
- elsif !session.remember? and ip.to_s != session[:ip].to_s
38
- STDOUT.print "Invalid IP!\n"
39
- STDOUT.print Knj::Php.print_r(session, true)
40
-
41
- raise Knj::Errors::InvalidData, "Invalid IP."
42
- end
30
+ if ip != "bot" and !session.remember? and ip.to_s != session[:ip].to_s
31
+ raise Knj::Errors::InvalidData, "Invalid IP."
43
32
  end
44
- =end
45
33
 
46
34
  @sessions[idhash][:time_lastused] = Time.now
47
- return @sessions[idhash]
35
+ return [session, hash]
48
36
  end
49
37
 
50
38
  #Generates a new session-ID by the meta data.
51
- def session_generate_id(args)
52
- meta = args[:meta]
39
+ def session_generate_id(meta)
53
40
  return Digest::MD5.hexdigest("#{Time.now.to_f}_#{meta["HTTP_HOST"]}_#{meta["REMOTE_HOST"]}_#{meta["HTTP_X_FORWARDED_SERVER"]}_#{meta["HTTP_X_FORWARDED_FOR"]}_#{meta["HTTP_X_FORWARDED_HOST"]}_#{meta["REMOTE_ADDR"]}_#{meta["HTTP_USER_AGENT"]}")
54
41
  end
55
42
 
@@ -75,4 +62,10 @@ class Knjappserver
75
62
  end
76
63
  end
77
64
  end
65
+
66
+ #Writes all session-data and resets the hash.
67
+ def sessions_reset
68
+ self.sessions_flush
69
+ @sessions = {}
70
+ end
78
71
  end
@@ -1,27 +1,33 @@
1
1
  class Knjappserver
2
- def initialize_threadding
2
+ def initialize_threadding
3
3
  @config[:threadding] = {} if !@config.has_key?(:threadding)
4
- @config[:threadding][:max_running] = 25 if !@config[:threadding].has_key?(:max_running)
4
+ @config[:threadding][:max_running] = 1 if !@config[:threadding].has_key?(:max_running)
5
5
 
6
- @threadpool = Knj::Threadpool.new(:threads => @config[:threadding][:max_running])
7
- @threadpool.events.connect(:on_error) do |event, error|
8
- STDOUT.print "Error!\n"
9
- self.handle_error(error)
10
- end
11
- end
12
-
13
- #Inits the thread so it has access to the appserver and various magic methods can be used.
14
- def thread_init(thread)
6
+ @threadpool = Knj::Threadpool.new(:threads => @config[:threadding][:max_running], :sleep => 0.1)
7
+ @threadpool.events.connect(:on_error) do |event, error|
8
+ self.handle_error(error)
9
+ end
10
+ end
11
+
12
+ #Inits the thread so it has access to the appserver and various magic methods can be used.
13
+ def thread_init(thread = nil)
14
+ thread = Thread.current if thread == nil
15
15
  thread[:knjappserver] = {} if !thread[:knjappserver]
16
16
  thread[:knjappserver][:kas] = self
17
- end
18
-
19
- #Spawns a new thread with access to magic methods, _db-method and various other stuff in the appserver.
20
- def thread(args = {})
21
- raise "No block given." if !block_given?
22
- args[:args] = [] if !args[:args]
23
-
24
- @threadpool.run_async do
17
+ end
18
+
19
+ #Spawns a new thread with access to magic methods, _db-method and various other stuff in the appserver.
20
+ def thread(args = {})
21
+ raise "No block given." if !block_given?
22
+ args[:args] = [] if !args[:args]
23
+
24
+ thread_obj = Knjappserver::Thread_instance.new(
25
+ :running => false,
26
+ :error => false,
27
+ :done => false
28
+ )
29
+
30
+ @threadpool.run_async do
25
31
  @ob.db.get_and_register_thread if @ob.db.opts[:threadsafe]
26
32
  @db_handler.get_and_register_thread if @db_handler.opts[:threadsafe]
27
33
 
@@ -30,31 +36,52 @@ class Knjappserver
30
36
  :db => @db_handler
31
37
  }
32
38
 
33
- begin
34
- yield(*args[:args])
35
- rescue Exception => e
36
- handle_error(e)
37
- ensure
38
- @ob.db.free_thread if @ob.db.opts[:threadsafe]
39
- @db_handler.free_thread if @db_handler.opts[:threadsafe]
40
- end
41
- end
42
- end
43
-
44
- #Runs a proc every number of seconds.
45
- def timeout(args = {}, &block)
46
- to = Knjappserver::Threadding_timeout.new(
39
+ begin
40
+ thread_obj.args[:running] = true
41
+ yield(*args[:args])
42
+ rescue Exception => e
43
+ handle_error(e)
44
+ thread_obj.args[:error] = true
45
+ thread_obj.args[:error_obj] = e
46
+ ensure
47
+ @ob.db.free_thread if @ob.db.opts[:threadsafe]
48
+ @db_handler.free_thread if @db_handler.opts[:threadsafe]
49
+ thread_obj.args[:running] = false
50
+ thread_obj.args[:done] = true
51
+ end
52
+ end
53
+
54
+ return thread_obj
55
+ end
56
+
57
+ #Runs a proc every number of seconds.
58
+ def timeout(args = {}, &block)
59
+ return Knjappserver::Threadding_timeout.new({
47
60
  :kas => self,
48
61
  :block => block,
49
- :args => args
50
- )
51
- to.start
52
-
53
- return to
54
- end
55
-
56
- #Spawns a thread to run the given proc and add the output of that block in the correct order to the HTML.
57
- def threadded_content(&block)
62
+ :args => []
63
+ }.merge(args)).start
64
+ end
65
+
66
+ #Spawns a thread to run the given proc and add the output of that block in the correct order to the HTML.
67
+ def threadded_content(&block)
58
68
  _httpsession.threadded_content(block)
59
- end
69
+ end
70
+ end
71
+
72
+ class Knjappserver::Thread_instance
73
+ attr_reader :args
74
+
75
+ def initialize(args)
76
+ @args = args
77
+ end
78
+
79
+ def join
80
+ sleep 0.1 while !@args[:done]
81
+ end
82
+
83
+ def join_error
84
+ self.join
85
+ raise @args[:error_obj] if @args[:error_obj]
86
+ end
60
87
  end