tcell_agent 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/bin/tcell_agent +97 -32
  3. data/lib/tcell_agent.rb +4 -4
  4. data/lib/tcell_agent/agent.rb +68 -13
  5. data/lib/tcell_agent/agent/event_processor.rb +256 -79
  6. data/lib/tcell_agent/agent/fork_pipe_manager.rb +114 -0
  7. data/lib/tcell_agent/agent/policy_manager.rb +28 -16
  8. data/lib/tcell_agent/agent/policy_types.rb +3 -4
  9. data/lib/tcell_agent/agent/route_manager.rb +45 -0
  10. data/lib/tcell_agent/agent/static_agent.rb +48 -10
  11. data/lib/tcell_agent/api.rb +0 -2
  12. data/lib/tcell_agent/appsensor/path_traversal.rb +1 -1
  13. data/lib/tcell_agent/configuration.rb +19 -6
  14. data/lib/tcell_agent/instrumentation.rb +123 -0
  15. data/lib/tcell_agent/logger.rb +4 -1
  16. data/lib/tcell_agent/policies/content_security_policy.rb +44 -8
  17. data/lib/tcell_agent/policies/dataloss_policy.rb +122 -65
  18. data/lib/tcell_agent/rails.rb +19 -10
  19. data/{config/initializers/authlogic_auth.rb → lib/tcell_agent/rails/auth/authlogic.rb} +1 -0
  20. data/{config/initializers/devise_auth.rb → lib/tcell_agent/rails/auth/devise.rb} +2 -81
  21. data/lib/tcell_agent/rails/dlp.rb +40 -36
  22. data/lib/tcell_agent/rails/middleware/body_filter_middleware.rb +2 -2
  23. data/lib/tcell_agent/rails/middleware/headers_middleware.rb +8 -2
  24. data/lib/tcell_agent/rails/routes.rb +15 -19
  25. data/lib/tcell_agent/routes/table.rb +35 -0
  26. data/lib/tcell_agent/sensor_events/app_sensor.rb +22 -33
  27. data/lib/tcell_agent/sensor_events/discovery.rb +30 -0
  28. data/lib/tcell_agent/sensor_events/dlp.rb +9 -4
  29. data/lib/tcell_agent/sensor_events/sensor.rb +3 -0
  30. data/lib/tcell_agent/sensor_events/server_agent.rb +30 -6
  31. data/lib/tcell_agent/sensor_events/util/utils.rb +5 -1
  32. data/lib/tcell_agent/start_background_thread.rb +27 -22
  33. data/lib/tcell_agent/utils/queue_with_timeout.rb +65 -1
  34. data/lib/tcell_agent/version.rb +1 -1
  35. data/spec/lib/tcell_agent/instrumentation_spec.rb +198 -0
  36. data/spec/lib/tcell_agent/policies/content_security_policy_spec.rb +37 -2
  37. data/spec/lib/tcell_agent/policies/dataloss_policy_spec.rb +81 -8
  38. data/spec/lib/tcell_agent/rails/middleware/global_middleware_spec.rb +3 -3
  39. data/spec/spec_helper.rb +16 -0
  40. metadata +11 -11
  41. data/config/initializers/init.rb +0 -8
  42. data/lib/tcell_agent/dataloss.rb +0 -0
  43. data/lib/tcell_agent/policies/add_script_tag_policy.rb +0 -47
  44. data/lib/tcell_agent/rails/devise.rb +0 -0
  45. data/spec/lib/tcell_agent/policies/add_script_tag_policy_spec.rb +0 -37
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 89400e737468ee73780a481cccf4e21c684f0749
4
- data.tar.gz: 61f6bdb301b0b0586f955183401b7004aaaa5c84
3
+ metadata.gz: 6a8919fddc03b6bf97fc9a8c6bd48d9703d870b9
4
+ data.tar.gz: df20c9fadd9142dda76a34e9d37b90a351ef781d
5
5
  SHA512:
6
- metadata.gz: 22bc24714bf4f233b7b338b9f48e9abad27e78982d6730f75f8774f49bb9a16dbbd54266f21e7762309e878de6f6093845992d1ca3e76f6725d633062b826b03
7
- data.tar.gz: 4becf0b297b2dc0bda7199525d0973cc737009b8fac7ebd7e18dd6277e445b6dd11e42f17a0f73ca6de73eed46c074de4fc96267708b50a199beac9e49b07495
6
+ metadata.gz: 3fe7bbdebfdb3e6c240e365b8b13a1ebc159e07f8481e9431fc7eb8bda7c61b1054e3d84a282b8c36731aec9bd5ebd8d1bbd369e89a52329ba0f46c884f9e340
7
+ data.tar.gz: b1c209b6a3ad6dc3e2ceaffcd59e56de1c15fecf0d06a11c75250ae955ca38b62ed7655afedb349eb267c3b88451019c2cd029f66cc883f4470725d804ab7659
data/bin/tcell_agent CHANGED
@@ -4,6 +4,20 @@
4
4
 
5
5
  require 'fileutils'
6
6
  require 'json'
7
+ require 'optparse'
8
+
9
+ options = {}
10
+
11
+
12
+ subtext = <<HELP
13
+ Commonly used command are:
14
+ setup : Setup new config file
15
+ test : Run diagnostics classes and config
16
+ loglevel : Set the loglevel of the tcell agent
17
+
18
+ See 'tcell_agent COMMAND --help' for more information on a specific command.
19
+
20
+ HELP
7
21
 
8
22
  def yesno(default=true)
9
23
  begin
@@ -24,16 +38,55 @@ end
24
38
  CONFIG_DIR = 'config'
25
39
  CONFIG_FILE = 'config/tcell_agent.config'
26
40
 
27
- if (ARGV.length == 0)
28
- puts "Usage: tcell_agent command [options]"
29
- puts " setup Setup new config file"
30
- puts " test Test classes and config"
31
- puts " preload (filename|rm) Set the preload file, 'rm' removes it"
41
+ global = OptionParser.new do |opts|
42
+ opts.banner = "Usage: tcell_agent [options] [subcommand [options]]"
43
+ opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
44
+ options[:verbose] = v
45
+ end
46
+ opts.separator ""
47
+ opts.separator subtext
48
+ end
32
49
 
33
- Kernel.exit(1)
50
+ subcommands = {
51
+ 'setup' => OptionParser.new do |opts|
52
+ opts.banner = "Usage: setup"
53
+ end,
54
+ 'loglevel' => OptionParser.new do |opts|
55
+ opts.banner = "Usage: loglevel [options] error|warn|info|debug"
56
+ opts.on("-o", "--off", "turn logging off ") do |v|
57
+ options[:off] = v
58
+ end
59
+ end,
60
+ 'preload' => OptionParser.new do |opts|
61
+ opts.banner = "Usage: loglevel [options] [preload_filename]"
62
+ opts.on("-o", "--off", "turn preloading filename off ") do |v|
63
+ options[:off] = v
64
+ end
65
+ end,
66
+ 'demomode' => OptionParser.new do |opts|
67
+ opts.banner = "Usage: loglevel [options]"
68
+ opts.on("-o", "--off", "turn preloading filename off ") do |v|
69
+ options[:off] = v
70
+ end
71
+ end,
72
+ 'test' => OptionParser.new do |opts|
73
+ opts.banner = "Usage: test"
74
+ #opts.on("-q", "--[no-]quiet", "quietly run ") do |v|
75
+ # options[:quiet] = v
76
+ #end
77
+ end
78
+ }
79
+
80
+ global.order!
81
+ command = ARGV.shift
82
+ if command == nil
83
+ puts global
84
+ Kernel.exit(1)
34
85
  end
86
+ subcommands[command].order!
87
+
35
88
 
36
- if (ARGV[0] == 'setup')
89
+ if (command == 'setup')
37
90
  if !File.directory?(CONFIG_DIR)
38
91
  print "Directory 'config' not found, create? [Y/n]"
39
92
  answer = yesno()
@@ -69,49 +122,61 @@ if (ARGV[0] == 'setup')
69
122
  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
70
123
  puts "done."
71
124
 
72
- elsif (ARGV[0] == 'loglevel')
73
- if (ARGV.length != 2)
74
- puts "Usage: tcell_agent loglevel ERROR|INFO|DEBUG|OFF"
75
- Kernel.exit(1)
76
- end
125
+ elsif (command == 'loglevel')
77
126
  file = File.read(CONFIG_FILE)
78
127
  config_hash = JSON.parse(file)
79
- loglevel = ARGV[1].upcase
80
128
  logging_options = config_hash["applications"][0].fetch("logging_options",{})
81
- if loglevel == "OFF"
129
+
130
+ if options[:off] == true
82
131
  logging_options["enabled"] = false
83
- elsif loglevel == "ERROR" || loglevel == "INFO" || loglevel == "DEBUG"
84
- logging_options["enabled"] = true
85
- logging_options["level"] = loglevel
86
132
  else
87
- puts "Usage: tcell_agent loglevel ERROR|INFO|DEBUG|OFF"
88
- Kernel.exit(1)
133
+ loglevel = ARGV.pop
134
+ if (loglevel == nil)
135
+ puts subcommands[command]
136
+ Kernel.exit(1)
137
+ end
138
+ loglevel = loglevel.upcase
139
+ if ["ERROR","WARN","INFO","DEBUG"].include?(loglevel)
140
+ logging_options["enabled"] = true
141
+ logging_options["level"] = loglevel
142
+ else
143
+ puts subcommands[command]
144
+ Kernel.exit(1)
145
+ end
89
146
  end
90
147
  config_hash["applications"][0]["logging_options"] = logging_options
91
148
  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
92
149
  puts "done."
93
150
 
94
- elsif (ARGV[0] == 'preload')
95
- if (ARGV.length != 2)
96
- puts "Usage tcell_agent preload <filename>|rm"
97
- Kernel.exit(1)
98
- end
99
- if !File.exists?(CONFIG_FILE)
100
- puts "Config file not found, run 'tcell_agent setup' first"
101
- Kernel.exit(1)
102
- end
151
+ elsif (command == 'preload')
103
152
  file = File.read(CONFIG_FILE)
104
153
  config_hash = JSON.parse(file)
105
- preload_policy_filename = ARGV[1]
106
- if preload_policy_filename == "rm"
154
+
155
+ if options[:off] == true
107
156
  config_hash["applications"][0].delete("preload_policy_filename")
108
157
  else
109
- config_hash["applications"][0]["preload_policy_filename"] = ARGV[1]
158
+ preload_policy_filename = ARGV.pop
159
+ if (preload_policy_filename == nil)
160
+ puts subcommands[command]
161
+ Kernel.exit(1)
162
+ end
163
+ config_hash["applications"][0]["preload_policy_filename"] = preload_policy_filename
164
+ end
165
+ File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
166
+ puts "done."
167
+
168
+ elsif (command == 'demomode')
169
+ file = File.read(CONFIG_FILE)
170
+ config_hash = JSON.parse(file)
171
+ if options[:off] == true
172
+ config_hash["applications"][0].delete("demomode")
173
+ else
174
+ config_hash["applications"][0]["demomode"] = true
110
175
  end
111
176
  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
112
177
  puts "done."
113
178
 
114
- elsif (ARGV[0] == 'test')
179
+ elsif (command == 'test')
115
180
  puts
116
181
  printf "%-50s", "Config file exists... "
117
182
  if !File.exists?(CONFIG_FILE)
data/lib/tcell_agent.rb CHANGED
@@ -5,10 +5,10 @@ require 'tcell_agent/configuration'
5
5
 
6
6
  module TCellAgent
7
7
  TCellAgent.logger.debug("Instrumenting")
8
- #if (TCellAgent.configuration.enabled && TCellAgent.configuration.instrument_for_events)
8
+ if (TCellAgent.configuration.enabled && TCellAgent.configuration.instrument_for_events)
9
9
  require 'tcell_agent/rails' if defined?(Rails)
10
10
  require 'tcell_agent/sinatra' if defined?(Sinatra)
11
- #end
11
+ end
12
12
  end
13
13
 
14
14
  require 'tcell_agent/agent'
@@ -20,7 +20,6 @@ require 'tcell_agent/policies/secure_headers_policy'
20
20
  require 'tcell_agent/policies/honeytokens_policy'
21
21
  require 'tcell_agent/policies/clickjacking_policy'
22
22
  require 'tcell_agent/policies/appsensor_policy'
23
- require 'tcell_agent/policies/add_script_tag_policy'
24
23
  require 'tcell_agent/policies/login_fraud_policy'
25
24
  require 'tcell_agent/policies/dataloss_policy'
26
25
 
@@ -30,4 +29,5 @@ require 'tcell_agent/appsensor'
30
29
  require 'tcell_agent/sensor_events/util/sanitizer_utilities'
31
30
  require 'tcell_agent/sensor_events/util/redirect_utils'
32
31
 
33
- require 'tcell_agent/dataloss'
32
+ require 'tcell_agent/instrumentation'
33
+ require 'tcell_agent/start_background_thread'
@@ -12,33 +12,63 @@ require "tcell_agent/agent/event_processor"
12
12
  require "tcell_agent/agent/policy_manager"
13
13
  require "tcell_agent/agent/static_agent"
14
14
  require "tcell_agent/agent/policy_types"
15
+ require "tcell_agent/agent/route_manager"
16
+ require "tcell_agent/agent/fork_pipe_manager"
17
+
18
+ require "tcell_agent/routes/table"
15
19
 
16
20
  require 'net/http'
17
21
  require 'thread'
18
22
  require 'logger'
19
23
  require 'json'
24
+ require 'monitor'
20
25
 
21
26
  module TCellAgent
22
27
  class Agent
23
28
 
24
29
  attr_accessor :start_pid
25
30
  attr_accessor :eventQueue
31
+
32
+ attr_accessor :fork_event_queue
33
+ attr_accessor :fork_event_thread
34
+ attr_accessor :fork_event_thread_mutex
35
+
36
+ attr_accessor :metrics_event_queue
37
+ attr_accessor :metrics_event_thread
38
+ attr_accessor :metrics_event_thread_mutex
39
+
26
40
  attr_accessor :policies
27
41
  attr_accessor :eventProcessorThread
28
- attr_accessor :route_counter_table
42
+ attr_accessor :response_time_table
43
+ attr_accessor :route_table
29
44
 
30
- def initialize(start_pid)
31
- @start_pid = start_pid
32
- @lock = Monitor.new
33
- @metricsLock = Monitor.new
34
- @policies = {}
35
- @route_counter_table = {}
45
+ attr_accessor :event_processor_thread
46
+ attr_accessor :event_processor
47
+ attr_accessor :worker_mutex
36
48
 
37
- @dispatchEventsTimeout = TCellAgent.configuration.event_time_limit_seconds || 30
49
+ attr_accessor :policy_polling_thread
50
+ attr_accessor :policy_polling_worker_mutex
51
+
52
+
53
+ attr_accessor :event_queue_monitor
54
+ attr_accessor :event_dispatch_monitor
55
+
56
+ attr_accessor :stop_agent
57
+
58
+
59
+ def initialize(start_pid=Process.pid)
60
+ @start_pid = start_pid
61
+ @dispatchEventsTimeout = TCellAgent.configuration.event_time_limit_seconds || 55
38
62
  @dispatchEventsLimit = TCellAgent.configuration.event_batch_size_limit || 20
39
- @dispatchEvents = []
63
+ @worker_mutex = Mutex.new
64
+ @policy_polling_worker_mutex = Mutex.new
65
+ @@policy_tapi = TCellApi.new
40
66
 
41
- @eventQueue = QueueWithTimeout.new
67
+ # Agent request thread
68
+ @policies = {}
69
+ @lock = Monitor.new
70
+
71
+ self.initialize_processor_variables
42
72
 
43
73
  if TCellAgent.configuration.preload_policy_filename != nil
44
74
  TCellAgent.logger.info("Preloading a policy file");
@@ -59,6 +89,30 @@ module TCellAgent
59
89
  end
60
90
  end
61
91
 
92
+ def initialize_processor_variables
93
+ @metricsLock = Monitor.new
94
+ @stop_agent = false
95
+ @route_table = TCellAgent::Routes::RouteTable.new
96
+
97
+ @event_queue_monitor = Monitor.new
98
+ @event_dispatch_monitor = Monitor.new
99
+ @mutex = Monitor.new
100
+
101
+ @response_time_table = {}
102
+ @dispatchEvents = []
103
+ @eventQueue = BoundedQueue.new(200)
104
+
105
+ @fork_event_queue=Queue.new
106
+ @fork_event_thread_mutex=Monitor.new
107
+
108
+ @metrics_event_queue=Queue.new
109
+ @metrics_event_thread_mutex=Monitor.new
110
+ end
111
+
112
+ def is_parent_process?
113
+ @start_pid == Process.pid
114
+ end
115
+
62
116
  def start
63
117
  if (TCellAgent.configuration.api_key == nil ||
64
118
  TCellAgent.configuration.app_id == nil)
@@ -70,9 +124,10 @@ module TCellAgent
70
124
  puts " ********* ********* ********* *********"
71
125
  return
72
126
  end
73
- self.start_event_processor()
74
- self.start_policy_polling()
75
-
127
+ #puts "Starting"
128
+ self.ensure_policy_polling_running
129
+ self.ensure_event_processor_running
130
+ #puts "Ending"
76
131
  end
77
132
 
78
133
  end
@@ -1,3 +1,5 @@
1
+ # encoding: utf-8
2
+
1
3
  # See the file "LICENSE" for the full license governing this code.
2
4
 
3
5
  require "tcell_agent/logger"
@@ -12,13 +14,15 @@ require "tcell_agent/policies/http_redirect_policy"
12
14
  require "tcell_agent/policies/secure_headers_policy"
13
15
  require "tcell_agent/policies/honeytokens_policy"
14
16
  require "tcell_agent/policies/appsensor_policy"
15
- require "tcell_agent/policies/add_script_tag_policy"
16
17
 
17
18
  require "tcell_agent/sensor_events/server_agent"
18
19
  require "tcell_agent/sensor_events/metrics"
19
20
 
20
21
  require "tcell_agent/utils/queue_with_timeout"
21
22
 
23
+ require "tcell_agent/agent/fork_pipe_manager"
24
+ require "tcell_agent/agent/static_agent"
25
+
22
26
  require 'net/http'
23
27
  require 'thread'
24
28
  require 'logger'
@@ -28,106 +32,279 @@ require 'json'
28
32
  module TCellAgent
29
33
  class Agent
30
34
 
31
- def reset_dispatch
32
- @dispatchEvents = []
35
+ def ensure_event_processor_running
36
+ return if event_processor_running?
37
+ @worker_mutex.synchronize do
38
+ return if event_processor_running?
39
+ if self.is_parent_process? == false
40
+ self.initialize_processor_variables
41
+ end
42
+ start_event_processor
43
+ end
33
44
  end
34
- def start_event_processor(send_empties=true)
35
- @events_send_empties = send_empties
36
45
 
37
- @eventProcessorThread = Thread.new do
38
- TCellAgent.logger.debug("starting background event listener")
39
- tapi = TCellApi.new
40
-
41
- def dispatch(tapi, events)
42
- if (@events_send_empties || events.length > 0)
43
- metrics_event = TCellAgent::SensorEvents::MetricsEvent.new
44
- metrics_event.set_route_count_table(@eventQueue.get_response_time_table)
45
- events.push( metrics_event )
46
- @eventQueue.reset_response_time_table
47
- return tapi.sendEventSet(events)
48
- end
49
- return true
50
- end
46
+ def event_processor_running?
47
+ if TCellAgent::Agent.is_parent_process? == false
48
+ return true
49
+ end
50
+ @event_processor_thread && @event_processor_thread.alive?
51
+ end
51
52
 
53
+ def start_event_processor(send_empties=true)
54
+ if TCellAgent::Agent.is_parent_process? == false
55
+ return
56
+ end
57
+ @events_send_empties = send_empties
58
+ Thread.abort_on_exception = true
59
+ @event_processor_thread = Thread.new do
52
60
  begin
53
- event = TCellAgent::SensorEvents::ServerAgentDetailsSensorEvent.new
54
- TCellAgent.send_event(event)
55
-
56
- event = TCellAgent::SensorEvents::ServerAgentPackagesSensorEvent.new
57
- TCellAgent.send_event(event)
58
- rescue Exception => te
59
- TCellAgent.logger.error("Exception sending initial events: #{te.message}")
60
- TCellAgent.logger.debug(te.backtrace)
61
- end
62
- timeout_flag = false
63
- last_run_time = Time.now
64
- # orig = (Time.now.to_f * 1000).to_i
65
- # response = @app.call(env)
66
- # response_time = (Time.now.to_f * 1000).to_i - orig
61
+ tapi = TCellApi.new
62
+ begin
63
+ event = TCellAgent::SensorEvents::ServerAgentDetailsSensorEvent.new
64
+ TCellAgent.send_event(event)
65
+ rescue Exception => te
66
+ TCellAgent.logger.error("Exception sending server agent details: #{te.message}")
67
+ TCellAgent.logger.debug(te.backtrace)
68
+ end
67
69
 
68
- loop do
69
70
  begin
71
+ event = TCellAgent::SensorEvents::ServerAgentPackagesSensorEvent.new
72
+ TCellAgent.send_event(event)
73
+ rescue Exception => te
74
+ TCellAgent.logger.error("Exception sending server agent packages: #{te.message}")
75
+ TCellAgent.logger.debug(te.backtrace)
76
+ end
77
+ timeout_flag = false
78
+ last_run_time = Time.now
79
+ loop do
70
80
  begin
71
- now = Time.now
72
- wait_for = @dispatchEventsTimeout - (now - last_run_time).to_i.abs
73
- #puts "---- ---- "
74
- #puts wait_for
75
-
76
- event = @eventQueue.pop_with_timeout([wait_for, 0].max)
77
- # puts "Hey, found event"
78
- # puts event
79
- if event["event_type"] == "RequestRouteTimer"
80
- # puts "Request"
81
- # puts event
82
- route_id = event.route_id
83
- # puts "Route"
84
- # puts route_id
85
- response_time = event.response_time
86
- # puts "ROUTE"
87
- #puts response_time
88
- @eventQueue.add_response_time(route_id, response_time)
89
- #puts @eventQueue.get_response_time_table
90
- # puts "xxxx"
91
- end
92
- event.post_process
93
- if event.send == true
94
- @dispatchEvents.push(event)
95
- end
96
- if (event.flush or @dispatchEvents.length >= @dispatchEventsLimit or wait_for < 0)
97
- last_run_time = Time.now
98
- if ( dispatch(tapi, @dispatchEvents) )
99
- self.reset_dispatch
100
- else
101
- @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
81
+ begin
82
+ now = Time.now
83
+ wait_for = @dispatchEventsTimeout - (now - last_run_time).to_i.abs
84
+ event = @eventQueue.pop([wait_for, 1].max)
85
+ if event == nil
86
+ last_run_time = Time.now
87
+ # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
88
+ if (@events_send_empties || @dispatchEvents.length > 0)
89
+ if (@response_time_table.size > 0)
90
+ metrics_event = TCellAgent::SensorEvents::MetricsEvent.new
91
+ metrics_event.set_route_count_table(@response_time_table)
92
+ @event_dispatch_monitor.synchronize {
93
+ @dispatchEvents.push( metrics_event )
94
+ }
95
+ @mutex.synchronize do
96
+ @response_time_table = {}
97
+ end
98
+ end
99
+ @event_dispatch_monitor.synchronize {
100
+ events_to_send = @dispatchEvents
101
+ result = tapi.sendEventSet(events_to_send)
102
+ if ( result )
103
+ @event_dispatch_monitor.synchronize {
104
+ @dispatchEvents = []
105
+ }
106
+ else
107
+ @event_dispatch_monitor.synchronize {
108
+ @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
109
+ }
110
+ end
111
+ }
112
+ end
113
+ # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
114
+ else
115
+ event.post_process
116
+ if event.send == true
117
+ @event_dispatch_monitor.synchronize {
118
+ @dispatchEvents.push(event)
119
+ }
120
+ end
121
+ if (event.flush or @dispatchEvents.length >= @dispatchEventsLimit or wait_for < 0)
122
+ last_run_time = Time.now
123
+ # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
124
+ if (@events_send_empties || @dispatchEvents.length > 0)
125
+ if (@response_time_table.size > 0)
126
+ metrics_event = TCellAgent::SensorEvents::MetricsEvent.new
127
+ metrics_event.set_route_count_table(@response_time_table)
128
+ @event_dispatch_monitor.synchronize {
129
+ @dispatchEvents.push( metrics_event )
130
+ }
131
+ @mutex.synchronize do
132
+ @response_time_table = {}
133
+ end
134
+ end
135
+ @event_dispatch_monitor.synchronize {
136
+ events_to_send = @dispatchEvents
137
+ result = tapi.sendEventSet(events_to_send)
138
+ if ( result )
139
+ @dispatchEvents = []
140
+ else
141
+ @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
142
+ end
143
+ }
144
+ end
145
+ # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
146
+ end
102
147
  end
148
+ rescue ThreadError => te
149
+ last_run_time = Time.now
150
+ @event_dispatch_monitor.synchronize {
151
+ @dispatchEvents = []
152
+ }
103
153
  end
104
- rescue ThreadError => te
154
+ rescue Exception => e
105
155
  last_run_time = Time.now
106
- if ( dispatch(tapi, @dispatchEvents) )
107
- self.reset_dispatch
156
+ TCellAgent.logger.error("Exception while processing events: #{e.message}")
157
+ TCellAgent.logger.debug(e.backtrace)
158
+ @event_dispatch_monitor.synchronize {
159
+ @dispatchEvents = []
160
+ }
161
+ end
162
+ end
163
+ rescue Exception => xyz
164
+ TCellAgent.logger.error("Exception while starting processor: #{xyz.message}")
165
+ TCellAgent.logger.debug(xyz.backtrace)
166
+ end
167
+ end
168
+ end
169
+
170
+ def ensure_fork_event_thread_running
171
+ return if event_fork_thread_running?
172
+ @fork_event_thread_mutex.synchronize do
173
+ return if event_fork_thread_running?
174
+ start_fork_event_thread
175
+ end
176
+ end
177
+
178
+ def event_fork_thread_running?
179
+ if TCellAgent::Agent.is_parent_process?
180
+ return true
181
+ end
182
+ @fork_event_thread && @fork_event_thread.alive?
183
+ end
184
+
185
+ def start_fork_event_thread
186
+ @fork_event_thread = Thread.new do
187
+ begin
188
+ loop do
189
+ begin
190
+ event = @fork_event_queue.pop
191
+ if event.send == false
192
+ event.post_process
108
193
  else
109
- @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
194
+ TCellAgent::Agent.send_to_event_pipe(event)
110
195
  end
196
+ rescue Exception => e
197
+ TCellAgent.logger.error("Exception while processing (forked) events: #{e.message}")
198
+ TCellAgent.logger.debug(e.backtrace)
199
+ end
200
+ end
201
+ rescue Exception => e
202
+ TCellAgent.logger.error("Exception while running (forked) events: #{e.message}")
203
+ TCellAgent.logger.debug(e.backtrace)
204
+ end
205
+ end
206
+ end
207
+
208
+ def queue_forked_event(event)
209
+ begin
210
+ self.ensure_fork_event_thread_running
211
+ if (@fork_event_queue.length() > 100)
212
+ TCellAgent.logger.debug("Dropping (forked) event because queue full")
213
+ return
214
+ end
215
+ @fork_event_queue.push(event)
216
+ rescue Exception => queue_exception
217
+ TCellAgent.logger.debug("Could not add (forked) event #{queue_exception.message}")
218
+ end
219
+ end
220
+
221
+ def ensure_metrics_event_thread_running
222
+ return if event_metrics_thread_running?
223
+ @metrics_event_thread_mutex.synchronize do
224
+ return if event_metrics_thread_running?
225
+ start_metrics_event_thread
226
+ end
227
+ end
228
+
229
+ def event_metrics_thread_running?
230
+ if TCellAgent::Agent.is_parent_process?
231
+ return true
232
+ end
233
+ @start_metrics_event_thread && @start_metrics_event_thread.alive?
234
+ end
235
+
236
+ def start_metrics_event_thread
237
+ @metrics_event_thread = Thread.new do
238
+ begin
239
+ loop do
240
+ begin
241
+ event = @metrics_event_queue.pop
242
+ TCellAgent::Agent.send_to_metrics_pipe(event)
243
+ rescue Exception => e
244
+ TCellAgent.logger.error("Exception while processing (forked) metrics: #{e.message}")
245
+ TCellAgent.logger.debug(e.backtrace)
111
246
  end
112
- #puts "AFTER "
113
- #puts Thread.current
114
- rescue Exception => e
115
- last_run_time = Time.now
116
- TCellAgent.logger.error("Exception while processing events: #{e.message}")
117
- TCellAgent.logger.debug(e.backtrace)
118
- @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
119
247
  end
248
+ rescue Exception => e
249
+ TCellAgent.logger.error("Exception while running (forked) metrics: #{e.message}")
250
+ TCellAgent.logger.debug(e.backtrace)
251
+ end
252
+ end
253
+ end
254
+
255
+ def _queue_metric(event)
256
+ begin
257
+ self.ensure_metrics_event_thread_running
258
+ if (@metrics_event_queue.length() > 100)
259
+ TCellAgent.logger.debug("Dropping (forked) metric because queue full")
260
+ return
120
261
  end
262
+ @metrics_event_queue.push(event)
263
+ rescue Exception => queue_exception
264
+ TCellAgent.logger.debug("Could not add (forked) metric #{queue_exception.message}")
121
265
  end
122
266
  end
123
267
 
124
268
  def queueSensorEvent(event)
125
- if @eventProcessorThread != nil
126
- if @eventQueue.length < 200
127
- @eventQueue << event
269
+ begin
270
+ if TCellAgent::Agent.is_parent_process? == false
271
+ self.queue_forked_event(event)
272
+ return
273
+ end
274
+ begin
275
+ self.ensure_event_processor_running
276
+ @event_queue_monitor.synchronize {
277
+ @eventQueue.push(event)
278
+ }
279
+ rescue RuntimeError => te
280
+ TCellAgent.logger.debug("Dropping event because queue full")
281
+ end
282
+ rescue Exception => queue_exception
283
+ TCellAgent.logger.debug("Could not add event #{queue_exception.message}")
284
+ end
285
+ end
286
+
287
+ def increment_route(route_id, response_time)
288
+ begin
289
+ if TCellAgent::Agent.is_parent_process? == false
290
+ TCellAgent.queue_metric({"_type"=>"increment_route", "route_id"=>route_id, "response_time"=>response_time})
291
+ return
292
+ end
293
+ @mutex.synchronize do
294
+ if (route_id == nil || route_id == "")
295
+ route_id = "?"
296
+ end
297
+ @response_time_table[route_id] = @response_time_table.fetch(route_id,{})
298
+ @response_time_table[route_id]["c"] = @response_time_table[route_id].fetch("c",0) + 1
299
+ @response_time_table[route_id]["mx"] = [@response_time_table[route_id].fetch("mx",0), response_time].max
300
+ @response_time_table[route_id]["mn"] = [@response_time_table[route_id].fetch("mn",response_time), response_time].min
301
+ @response_time_table[route_id]["t"] = ((@response_time_table[route_id].fetch("t",0)*(@response_time_table[route_id]["c"]-1)) + response_time) / @response_time_table[route_id]["c"]
128
302
  end
303
+ rescue Exception => inc_exception
304
+ TCellAgent.logger.debug("Could not add incrememnt exception #{inc_exception.message}")
129
305
  end
130
306
  end
131
307
 
308
+
132
309
  end
133
310
  end