tcell_agent 0.4.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +9 -22
- data/bin/tcell_agent +127 -132
- data/lib/tcell_agent/agent/event_processor.rb +23 -22
- data/lib/tcell_agent/agent/fork_pipe_manager.rb +7 -7
- data/lib/tcell_agent/agent/policy_manager.rb +20 -15
- data/lib/tcell_agent/agent/policy_types.rb +5 -11
- data/lib/tcell_agent/agent/static_agent.rb +5 -1
- data/lib/tcell_agent/agent.rb +6 -4
- data/lib/tcell_agent/api.rb +7 -9
- data/lib/tcell_agent/appsensor/meta_data.rb +11 -4
- data/lib/tcell_agent/authlogic.rb +3 -3
- data/lib/tcell_agent/cmdi.rb +6 -4
- data/lib/tcell_agent/config/unknown_options.rb +3 -1
- data/lib/tcell_agent/configuration.rb +47 -49
- data/lib/tcell_agent/devise.rb +2 -2
- data/lib/tcell_agent/hooks/login_fraud.rb +58 -29
- data/lib/tcell_agent/instrumentation.rb +11 -10
- data/lib/tcell_agent/logger.rb +2 -2
- data/lib/tcell_agent/patches/meta_data.rb +9 -13
- data/lib/tcell_agent/patches.rb +7 -10
- data/lib/tcell_agent/policies/clickjacking_policy.rb +4 -5
- data/lib/tcell_agent/policies/content_security_policy.rb +6 -12
- data/lib/tcell_agent/policies/dataloss_policy.rb +2 -2
- data/lib/tcell_agent/policies/http_redirect_policy.rb +2 -2
- data/lib/tcell_agent/policies/policy.rb +0 -2
- data/lib/tcell_agent/policies/rust_policies.rb +90 -0
- data/lib/tcell_agent/policies/secure_headers_policy.rb +2 -2
- data/lib/tcell_agent/rails/auth/authlogic.rb +42 -24
- data/lib/tcell_agent/rails/auth/devise.rb +44 -23
- data/lib/tcell_agent/rails/auth/doorkeeper.rb +33 -15
- data/lib/tcell_agent/rails/better_ip.rb +1 -1
- data/lib/tcell_agent/rails/csrf_exception.rb +2 -2
- data/lib/tcell_agent/rails/dlp/process_request.rb +1 -1
- data/lib/tcell_agent/rails/dlp.rb +6 -6
- data/lib/tcell_agent/rails/dlp_handler.rb +1 -1
- data/lib/tcell_agent/rails/js_agent_insert.rb +1 -1
- data/lib/tcell_agent/rails/middleware/body_filter_middleware.rb +1 -1
- data/lib/tcell_agent/rails/middleware/context_middleware.rb +3 -2
- data/lib/tcell_agent/rails/middleware/headers_middleware.rb +10 -9
- data/lib/tcell_agent/rails/routes/grape.rb +6 -6
- data/lib/tcell_agent/rails/routes.rb +8 -11
- data/lib/tcell_agent/rust/libtcellagent-0.11.1.dylib +0 -0
- data/lib/tcell_agent/rust/{libtcellagent-0.6.1.so → libtcellagent-0.11.1.so} +0 -0
- data/lib/tcell_agent/rust/models.rb +16 -0
- data/lib/tcell_agent/rust/tcellagent-0.11.1.dll +0 -0
- data/lib/tcell_agent/rust/whisperer.rb +119 -48
- data/lib/tcell_agent/sensor_events/appsensor_meta_event.rb +17 -20
- data/lib/tcell_agent/sensor_events/command_injection.rb +50 -5
- data/lib/tcell_agent/sensor_events/login_fraud.rb +34 -18
- data/lib/tcell_agent/sensor_events/patches.rb +21 -0
- data/lib/tcell_agent/sensor_events/server_agent.rb +3 -3
- data/lib/tcell_agent/sensor_events/util/utils.rb +4 -3
- data/lib/tcell_agent/servers/puma.rb +2 -2
- data/lib/tcell_agent/servers/unicorn.rb +1 -1
- data/lib/tcell_agent/utils/passwords.rb +28 -0
- data/lib/tcell_agent/version.rb +1 -1
- data/lib/tcell_agent.rb +1 -5
- data/spec/apps/rails-3.2/config/tcell_agent.config +15 -0
- data/spec/apps/rails-3.2/log/development.log +0 -0
- data/spec/apps/rails-3.2/log/test.log +12 -0
- data/spec/apps/rails-4.1/log/test.log +0 -0
- data/spec/lib/tcell_agent/agent/fork_pipe_manager_spec.rb +46 -45
- data/spec/lib/tcell_agent/agent/policy_manager_spec.rb +276 -164
- data/spec/lib/tcell_agent/agent/static_agent_spec.rb +44 -47
- data/spec/lib/tcell_agent/api/api_spec.rb +16 -16
- data/spec/lib/tcell_agent/appsensor/injections_reporter_spec.rb +131 -116
- data/spec/lib/tcell_agent/appsensor/meta_data_spec.rb +55 -51
- data/spec/lib/tcell_agent/cmdi_spec.rb +413 -436
- data/spec/lib/tcell_agent/config/unknown_options_spec.rb +145 -128
- data/spec/lib/tcell_agent/configuration_spec.rb +165 -169
- data/spec/lib/tcell_agent/hooks/login_fraud_spec.rb +144 -153
- data/spec/lib/tcell_agent/instrumentation_spec.rb +84 -85
- data/spec/lib/tcell_agent/patches_spec.rb +70 -111
- data/spec/lib/tcell_agent/policies/appsensor_policy_spec.rb +313 -244
- data/spec/lib/tcell_agent/policies/clickjacking_policy_spec.rb +28 -28
- data/spec/lib/tcell_agent/policies/command_injection_policy_spec.rb +643 -513
- data/spec/lib/tcell_agent/policies/content_security_policy_spec.rb +55 -102
- data/spec/lib/tcell_agent/policies/dataloss_policy_spec.rb +111 -134
- data/spec/lib/tcell_agent/policies/http_redirect_policy_spec.rb +141 -146
- data/spec/lib/tcell_agent/policies/http_tx_policy_spec.rb +8 -8
- data/spec/lib/tcell_agent/policies/login_policy_spec.rb +15 -17
- data/spec/lib/tcell_agent/policies/patches_policy_spec.rb +231 -559
- data/spec/lib/tcell_agent/policies/secure_headers_policy_spec.rb +27 -27
- data/spec/lib/tcell_agent/rails/better_ip_spec.rb +30 -34
- data/spec/lib/tcell_agent/rails/logger_spec.rb +50 -49
- data/spec/lib/tcell_agent/rails/middleware/appsensor_middleware_spec.rb +182 -199
- data/spec/lib/tcell_agent/rails/middleware/dlp_middleware_spec.rb +110 -84
- data/spec/lib/tcell_agent/rails/middleware/global_middleware_spec.rb +107 -85
- data/spec/lib/tcell_agent/rails/middleware/redirect_middleware_spec.rb +68 -40
- data/spec/lib/tcell_agent/rails/middleware/tcell_body_proxy_spec.rb +81 -67
- data/spec/lib/tcell_agent/rails/responses_spec.rb +33 -37
- data/spec/lib/tcell_agent/rails/routes/grape_spec.rb +116 -121
- data/spec/lib/tcell_agent/rails/routes/route_id_spec.rb +25 -28
- data/spec/lib/tcell_agent/rails/routes/routes_spec.rb +87 -85
- data/spec/lib/tcell_agent/rails_spec.rb +1 -6
- data/spec/lib/tcell_agent/rust/models_spec.rb +112 -0
- data/spec/lib/tcell_agent/rust/whisperer_spec.rb +502 -179
- data/spec/lib/tcell_agent/sensor_events/appsensor_meta_event_spec.rb +44 -33
- data/spec/lib/tcell_agent/sensor_events/dlp_spec.rb +4 -4
- data/spec/lib/tcell_agent/sensor_events/sessions_metric_spec.rb +183 -169
- data/spec/lib/tcell_agent/sensor_events/util/sanitizer_utilities_spec.rb +25 -25
- data/spec/lib/tcell_agent/utils/bounded_queue_spec.rb +17 -20
- data/spec/lib/tcell_agent/utils/params_spec.rb +28 -28
- data/spec/lib/tcell_agent/utils/passwords_spec.rb +143 -0
- data/spec/lib/tcell_agent/utils/strings_spec.rb +35 -35
- data/spec/lib/tcell_agent_spec.rb +8 -8
- data/spec/spec_helper.rb +4 -4
- data/spec/support/middleware_helper.rb +10 -10
- data/spec/support/static_agent_overrides.rb +16 -12
- data/tcell_agent.gemspec +17 -33
- metadata +43 -198
- data/LICENSE_libinjection +0 -32
- data/Readme.txt +0 -7
- data/ext/libinjection/extconf.rb +0 -3
- data/ext/libinjection/libinjection.h +0 -65
- data/ext/libinjection/libinjection_html5.c +0 -847
- data/ext/libinjection/libinjection_html5.h +0 -54
- data/ext/libinjection/libinjection_sqli.c +0 -2317
- data/ext/libinjection/libinjection_sqli.h +0 -295
- data/ext/libinjection/libinjection_sqli_data.h +0 -9004
- data/ext/libinjection/libinjection_wrap.c +0 -3525
- data/ext/libinjection/libinjection_xss.c +0 -531
- data/ext/libinjection/libinjection_xss.h +0 -21
- data/lib/tcell_agent/appsensor/injections_matcher.rb +0 -155
- data/lib/tcell_agent/appsensor/rules/appsensor_rule_manager.rb +0 -49
- data/lib/tcell_agent/appsensor/rules/appsensor_rule_set.rb +0 -67
- data/lib/tcell_agent/appsensor/rules/baserules.json +0 -467
- data/lib/tcell_agent/patches/block_rule.rb +0 -93
- data/lib/tcell_agent/patches/sensors_matcher.rb +0 -31
- data/lib/tcell_agent/policies/appsensor/cmdi_sensor.rb +0 -23
- data/lib/tcell_agent/policies/appsensor/fpt_sensor.rb +0 -23
- data/lib/tcell_agent/policies/appsensor/injection_sensor.rb +0 -117
- data/lib/tcell_agent/policies/appsensor/nullbyte_sensor.rb +0 -26
- data/lib/tcell_agent/policies/appsensor/retr_sensor.rb +0 -22
- data/lib/tcell_agent/policies/appsensor/sqli_sensor.rb +0 -34
- data/lib/tcell_agent/policies/appsensor/xss_sensor.rb +0 -34
- data/lib/tcell_agent/policies/appsensor_policy.rb +0 -49
- data/lib/tcell_agent/policies/command_injection_policy.rb +0 -196
- data/lib/tcell_agent/policies/honeytokens_policy.rb +0 -69
- data/lib/tcell_agent/policies/patches_policy.rb +0 -84
- data/lib/tcell_agent/rust/libtcellagent-0.6.1.dylib +0 -0
- data/lib/tcell_agent/rust/tcellagent-0.6.1.dll +0 -0
- data/spec/apps/rails-3.2/Gemfile +0 -25
- data/spec/apps/rails-3.2/Gemfile.lock +0 -126
- data/spec/apps/rails-3.2/Rakefile +0 -7
- data/spec/apps/rails-3.2/app/assets/images/rails.png +0 -0
- data/spec/apps/rails-3.2/app/assets/javascripts/application.js +0 -15
- data/spec/apps/rails-3.2/app/assets/stylesheets/application.css +0 -13
- data/spec/apps/rails-3.2/app/controllers/application_controller.rb +0 -3
- data/spec/apps/rails-3.2/app/controllers/t_cell_app_controller.rb +0 -5
- data/spec/apps/rails-3.2/app/helpers/application_helper.rb +0 -2
- data/spec/apps/rails-3.2/app/views/layouts/application.html.erb +0 -14
- data/spec/apps/rails-3.2/app/views/t_cell_app/index.html.erb +0 -1
- data/spec/apps/rails-3.2/config/application.rb +0 -63
- data/spec/apps/rails-3.2/config/boot.rb +0 -6
- data/spec/apps/rails-3.2/config/environment.rb +0 -5
- data/spec/apps/rails-3.2/config/environments/test.rb +0 -37
- data/spec/apps/rails-3.2/config/routes.rb +0 -11
- data/spec/apps/rails-3.2/config.ru +0 -4
- data/spec/apps/rails-4.1/Gemfile +0 -7
- data/spec/apps/rails-4.1/Gemfile.lock +0 -114
- data/spec/apps/rails-4.1/Rakefile +0 -6
- data/spec/apps/rails-4.1/app/assets/javascripts/application.js +0 -16
- data/spec/apps/rails-4.1/app/assets/stylesheets/application.css +0 -15
- data/spec/apps/rails-4.1/app/controllers/application_controller.rb +0 -5
- data/spec/apps/rails-4.1/app/controllers/t_cell_app_controller.rb +0 -5
- data/spec/apps/rails-4.1/app/helpers/application_helper.rb +0 -2
- data/spec/apps/rails-4.1/app/views/layouts/application.html.erb +0 -14
- data/spec/apps/rails-4.1/app/views/t_cell_app/index.html.erb +0 -1
- data/spec/apps/rails-4.1/config/application.rb +0 -24
- data/spec/apps/rails-4.1/config/boot.rb +0 -4
- data/spec/apps/rails-4.1/config/environment.rb +0 -5
- data/spec/apps/rails-4.1/config/environments/test.rb +0 -41
- data/spec/apps/rails-4.1/config/initializers/assets.rb +0 -8
- data/spec/apps/rails-4.1/config/initializers/backtrace_silencers.rb +0 -7
- data/spec/apps/rails-4.1/config/initializers/cookies_serializer.rb +0 -3
- data/spec/apps/rails-4.1/config/initializers/filter_parameter_logging.rb +0 -4
- data/spec/apps/rails-4.1/config/initializers/inflections.rb +0 -16
- data/spec/apps/rails-4.1/config/initializers/mime_types.rb +0 -4
- data/spec/apps/rails-4.1/config/initializers/session_store.rb +0 -3
- data/spec/apps/rails-4.1/config/initializers/wrap_parameters.rb +0 -14
- data/spec/apps/rails-4.1/config/locales/en.yml +0 -23
- data/spec/apps/rails-4.1/config/routes.rb +0 -12
- data/spec/apps/rails-4.1/config/secrets.yml +0 -22
- data/spec/apps/rails-4.1/config.ru +0 -4
- data/spec/controllers/application_controller.rb +0 -12
- data/spec/lib/tcell_agent/appsensor/injections_matcher_spec.rb +0 -522
- data/spec/lib/tcell_agent/appsensor/rules/appsensor_rule_manager_spec.rb +0 -23
- data/spec/lib/tcell_agent/appsensor/rules/appsensor_rule_set_spec.rb +0 -159
- data/spec/lib/tcell_agent/patches/block_rule_spec.rb +0 -458
- data/spec/lib/tcell_agent/patches/sensors_matcher_spec.rb +0 -35
- data/spec/lib/tcell_agent/policies/appsensor/cmdi_sensor_spec.rb +0 -139
- data/spec/lib/tcell_agent/policies/appsensor/fpt_sensor_spec.rb +0 -139
- data/spec/lib/tcell_agent/policies/appsensor/nullbyte_sensor_spec.rb +0 -167
- data/spec/lib/tcell_agent/policies/appsensor/retr_sensor_spec.rb +0 -139
- data/spec/lib/tcell_agent/policies/appsensor/sqli_sensor_spec.rb +0 -246
- data/spec/lib/tcell_agent/policies/appsensor/xss_sensor_spec.rb +0 -882
- data/spec/lib/tcell_agent/policies/honeytokens_policy_spec.rb +0 -22
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e3bfdacf0f493764e58e984f770dc53d04787a98
|
4
|
+
data.tar.gz: 27c3464c78adda7a3224e5ea080e2615ff857975
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ef098d1aeb0feb1398c0ce002d706d0cef7b1a82ce1f9dd600eda39230e2bd718e60800ee3c9d7bb23ed345e8728f8fefb4b6bd5bda59a379868aea5f63ae0c7
|
7
|
+
data.tar.gz: d320becd35d19b4e124dfa04d703264b538191ccaf27c699fbdfde939769c4b8e97ff9ca6ac0cca16a47fe4d71ccdba83feb28c29eae7ff4a218aadfd29a7994
|
data/Rakefile
CHANGED
@@ -1,31 +1,18 @@
|
|
1
|
-
require 'rake/clean'
|
2
|
-
require 'rake/extensiontask'
|
3
1
|
require 'rspec/core/rake_task'
|
4
2
|
|
5
|
-
CLEAN.include('ext/**/*{.o,.log,.so,.bundle}')
|
6
|
-
CLEAN.include('ext/**/Makefile')
|
7
|
-
CLOBBER.include('lib/*{.so,.bundle}')
|
8
|
-
|
9
|
-
Rake::ExtensionTask.new("libinjection") do |ext|
|
10
|
-
ext.lib_dir = File.join("lib", "libinjection")
|
11
|
-
end
|
12
|
-
|
13
|
-
task :default => [:compile, :spec]
|
14
|
-
|
15
3
|
RSpec::Core::RakeTask.new(:spec)
|
16
4
|
|
17
|
-
desc
|
5
|
+
desc 'Run tests'
|
6
|
+
task :default => [:spec]
|
18
7
|
task :test => :spec
|
19
8
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
system(
|
24
|
-
system("docker-compose run railsintegration224 bundle exec rake db:create db:setup")
|
25
|
-
system("docker-compose stop")
|
9
|
+
task 'init-integration-tests' do
|
10
|
+
system('docker-compose run railsintegration224 bundle install')
|
11
|
+
system('docker-compose run railsintegration224 bundle exec rake db:create db:setup')
|
12
|
+
system('docker-compose stop')
|
26
13
|
end
|
27
14
|
|
28
|
-
task
|
29
|
-
system(
|
30
|
-
system(
|
15
|
+
task 'integration-test' do
|
16
|
+
system('docker-compose up railsintegration224')
|
17
|
+
system('docker-compose stop')
|
31
18
|
end
|
data/bin/tcell_agent
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
#
|
3
|
+
# TODO: so a small bit becames something, larger, rewrite as a real cmdline script
|
4
4
|
|
5
5
|
require 'fileutils'
|
6
6
|
require 'json'
|
@@ -8,7 +8,6 @@ require 'optparse'
|
|
8
8
|
|
9
9
|
options = {}
|
10
10
|
|
11
|
-
|
12
11
|
subtext = <<HELP
|
13
12
|
Commonly used command are:
|
14
13
|
setup : Setup new config file
|
@@ -21,264 +20,260 @@ See 'tcell_agent COMMAND --help' for more information on a specific command.
|
|
21
20
|
|
22
21
|
HELP
|
23
22
|
|
24
|
-
def yesno(default=true)
|
23
|
+
def yesno(default = true)
|
25
24
|
begin
|
26
|
-
system(
|
25
|
+
system('stty raw -echo')
|
27
26
|
str = STDIN.getc
|
28
27
|
ensure
|
29
|
-
system(
|
30
|
-
end
|
31
|
-
if str == "Y" || str == "y"
|
32
|
-
return true
|
33
|
-
elsif str == "N" || str == "n"
|
34
|
-
return false
|
35
|
-
else
|
36
|
-
return default
|
28
|
+
system('stty -raw echo')
|
37
29
|
end
|
30
|
+
|
31
|
+
return true if %w[Y y].include?(str)
|
32
|
+
return false if %w[N n].include?(str)
|
33
|
+
|
34
|
+
default
|
38
35
|
end
|
39
36
|
|
40
|
-
CONFIG_DIR = 'config'
|
41
|
-
CONFIG_FILE = 'config/tcell_agent.config'
|
37
|
+
CONFIG_DIR = 'config'.freeze
|
38
|
+
CONFIG_FILE = 'config/tcell_agent.config'.freeze
|
42
39
|
|
43
40
|
global = OptionParser.new do |opts|
|
44
|
-
opts.banner =
|
45
|
-
opts.on(
|
41
|
+
opts.banner = 'Usage: tcell_agent [options] [subcommand [options]]'
|
42
|
+
opts.on('--version', 'Print version') do |_v|
|
46
43
|
require 'tcell_agent/version'
|
47
44
|
puts "TCell.io Ruby Agent (Version #{TCellAgent::VERSION})"
|
48
45
|
Kernel.exit(1)
|
49
46
|
end
|
50
|
-
opts.on(
|
47
|
+
opts.on('-v', '--[no-]verbose', 'Run verbosely') do |v|
|
51
48
|
options[:verbose] = v
|
52
49
|
end
|
53
|
-
opts.separator
|
50
|
+
opts.separator ''
|
54
51
|
opts.separator subtext
|
55
52
|
end
|
56
53
|
|
57
54
|
subcommands = {
|
58
55
|
'setup' => OptionParser.new do |opts|
|
59
|
-
|
56
|
+
opts.banner = 'Usage: setup'
|
60
57
|
end,
|
61
58
|
'loglevel' => OptionParser.new do |opts|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
59
|
+
opts.banner = 'Usage: loglevel [options] error|warn|info|debug'
|
60
|
+
opts.on('-o', '--off', 'turn logging off ') do |v|
|
61
|
+
options[:off] = v
|
62
|
+
end
|
66
63
|
end,
|
67
64
|
'preload' => OptionParser.new do |opts|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
65
|
+
opts.banner = 'Usage: loglevel [options] [preload_filename]'
|
66
|
+
opts.on('-o', '--off', 'turn preloading filename off ') do |v|
|
67
|
+
options[:off] = v
|
68
|
+
end
|
72
69
|
end,
|
73
70
|
'demomode' => OptionParser.new do |opts|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
71
|
+
opts.banner = 'Usage: loglevel [options]'
|
72
|
+
opts.on('-o', '--off', 'turn preloading filename off ') do |v|
|
73
|
+
options[:off] = v
|
74
|
+
end
|
78
75
|
end,
|
79
76
|
'enable' => OptionParser.new do |opts|
|
80
|
-
|
77
|
+
opts.banner = 'Usage: enable'
|
81
78
|
end,
|
82
79
|
'disable' => OptionParser.new do |opts|
|
83
|
-
|
80
|
+
opts.banner = 'Usage: disable'
|
84
81
|
end,
|
85
82
|
'test' => OptionParser.new do |opts|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
83
|
+
opts.banner = 'Usage: test'
|
84
|
+
# opts.on("-q", "--[no-]quiet", "quietly run ") do |v|
|
85
|
+
# options[:quiet] = v
|
86
|
+
# end
|
87
|
+
end
|
88
|
+
}
|
92
89
|
|
93
90
|
global.order!
|
94
91
|
command = ARGV.shift
|
95
|
-
if command
|
96
|
-
|
97
|
-
|
92
|
+
if command.nil?
|
93
|
+
puts global
|
94
|
+
Kernel.exit(1)
|
98
95
|
end
|
99
96
|
subcommands[command].order!
|
100
97
|
|
101
|
-
|
102
|
-
|
103
|
-
if !File.directory?(CONFIG_DIR)
|
98
|
+
if command == 'setup'
|
99
|
+
unless File.directory?(CONFIG_DIR)
|
104
100
|
print "Directory 'config' not found, create? [Y/n]"
|
105
|
-
answer = yesno
|
101
|
+
answer = yesno
|
106
102
|
print "\n"
|
107
|
-
|
108
|
-
puts
|
103
|
+
unless answer
|
104
|
+
puts 'ERROR: Could not create config'
|
109
105
|
Kernel.exit(1)
|
110
106
|
end
|
111
|
-
FileUtils
|
107
|
+
FileUtils.mkdir_p CONFIG_DIR
|
112
108
|
end
|
113
|
-
if File.
|
114
|
-
print
|
109
|
+
if File.exist?(CONFIG_FILE)
|
110
|
+
print 'Config file already exists, overwrite? [y/N]'
|
115
111
|
answer = yesno(false)
|
116
112
|
print "\n"
|
117
|
-
|
118
|
-
puts
|
113
|
+
unless answer
|
114
|
+
puts 'Keeping existing config'
|
119
115
|
Kernel.exit(1)
|
120
116
|
end
|
121
117
|
end
|
122
|
-
print
|
118
|
+
print 'Enter your API Key (ie gAABAAAA...): '
|
123
119
|
api_key = STDIN.gets.chomp
|
124
|
-
print
|
120
|
+
print 'Enter your App ID (ie MyApp-Fdk4j): '
|
125
121
|
app_id = STDIN.gets.chomp
|
126
122
|
config_hash = {
|
127
|
-
|
128
|
-
|
123
|
+
'version' => 1,
|
124
|
+
'applications' => [
|
129
125
|
{
|
130
|
-
|
131
|
-
|
126
|
+
'app_id' => app_id,
|
127
|
+
'api_key' => api_key
|
132
128
|
}
|
133
129
|
]
|
134
130
|
}
|
135
|
-
File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
|
136
|
-
puts
|
131
|
+
File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
|
132
|
+
puts 'done.'
|
137
133
|
|
138
|
-
elsif
|
134
|
+
elsif command == 'loglevel'
|
139
135
|
file = File.read(CONFIG_FILE)
|
140
136
|
config_hash = JSON.parse(file)
|
141
|
-
logging_options = config_hash[
|
137
|
+
logging_options = config_hash['applications'][0].fetch('logging_options', {})
|
142
138
|
|
143
139
|
if options[:off] == true
|
144
|
-
logging_options[
|
140
|
+
logging_options['enabled'] = false
|
145
141
|
else
|
146
142
|
loglevel = ARGV.pop
|
147
|
-
if
|
143
|
+
if loglevel.nil?
|
148
144
|
puts subcommands[command]
|
149
145
|
Kernel.exit(1)
|
150
146
|
end
|
151
147
|
loglevel = loglevel.upcase
|
152
|
-
if [
|
153
|
-
logging_options[
|
154
|
-
logging_options[
|
148
|
+
if %w[ERROR WARN INFO DEBUG].include?(loglevel)
|
149
|
+
logging_options['enabled'] = true
|
150
|
+
logging_options['level'] = loglevel
|
155
151
|
else
|
156
152
|
puts subcommands[command]
|
157
153
|
Kernel.exit(1)
|
158
154
|
end
|
159
155
|
end
|
160
|
-
config_hash[
|
161
|
-
File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
|
162
|
-
puts
|
156
|
+
config_hash['applications'][0]['logging_options'] = logging_options
|
157
|
+
File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
|
158
|
+
puts 'done.'
|
163
159
|
|
164
|
-
elsif
|
160
|
+
elsif command == 'preload'
|
165
161
|
file = File.read(CONFIG_FILE)
|
166
162
|
config_hash = JSON.parse(file)
|
167
163
|
|
168
164
|
if options[:off] == true
|
169
|
-
config_hash[
|
165
|
+
config_hash['applications'][0].delete('preload_policy_filename')
|
170
166
|
else
|
171
167
|
preload_policy_filename = ARGV.pop
|
172
|
-
if
|
168
|
+
if preload_policy_filename.nil?
|
173
169
|
puts subcommands[command]
|
174
170
|
Kernel.exit(1)
|
175
171
|
end
|
176
|
-
config_hash[
|
172
|
+
config_hash['applications'][0]['preload_policy_filename'] = preload_policy_filename
|
177
173
|
end
|
178
|
-
File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
|
179
|
-
puts
|
174
|
+
File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
|
175
|
+
puts 'done.'
|
180
176
|
|
181
|
-
elsif
|
177
|
+
elsif command == 'enable'
|
182
178
|
file = File.read(CONFIG_FILE)
|
183
179
|
config_hash = JSON.parse(file)
|
184
|
-
config_hash[
|
185
|
-
File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
|
186
|
-
puts
|
180
|
+
config_hash['applications'][0].delete('enabled')
|
181
|
+
File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
|
182
|
+
puts 'Enabled, you will need to restart the server.'
|
187
183
|
|
188
|
-
elsif
|
184
|
+
elsif command == 'disable'
|
189
185
|
file = File.read(CONFIG_FILE)
|
190
186
|
config_hash = JSON.parse(file)
|
191
|
-
config_hash[
|
192
|
-
File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
|
193
|
-
puts
|
187
|
+
config_hash['applications'][0]['enabled'] = false
|
188
|
+
File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
|
189
|
+
puts 'Disabled, you will need to restart the server.'
|
194
190
|
|
195
|
-
elsif
|
191
|
+
elsif command == 'demomode'
|
196
192
|
file = File.read(CONFIG_FILE)
|
197
193
|
config_hash = JSON.parse(file)
|
198
194
|
if options[:off] == true
|
199
|
-
config_hash[
|
195
|
+
config_hash['applications'][0].delete('demomode')
|
200
196
|
else
|
201
|
-
config_hash[
|
197
|
+
config_hash['applications'][0]['demomode'] = true
|
202
198
|
end
|
203
|
-
File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
|
204
|
-
puts
|
199
|
+
File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
|
200
|
+
puts 'done.'
|
205
201
|
|
206
|
-
elsif
|
202
|
+
elsif command == 'test'
|
207
203
|
puts
|
208
|
-
printf
|
209
|
-
|
210
|
-
puts
|
204
|
+
printf '%-50s', 'Config file exists... '
|
205
|
+
unless File.exist?(CONFIG_FILE)
|
206
|
+
puts 'failed'
|
211
207
|
Kernel.exit(1)
|
212
208
|
end
|
213
|
-
puts
|
209
|
+
puts 'passed'
|
214
210
|
|
215
|
-
printf
|
211
|
+
printf '%-50s', 'Config valid json... '
|
216
212
|
file = File.read(CONFIG_FILE)
|
217
213
|
config_hash = JSON.parse(file)
|
218
|
-
puts
|
214
|
+
puts 'passed'
|
219
215
|
|
220
|
-
printf
|
221
|
-
if config_hash.fetch(
|
222
|
-
puts
|
216
|
+
printf '%-50s', 'Config file has valid version... '
|
217
|
+
if config_hash.fetch('version') != 1
|
218
|
+
puts 'failed'
|
223
219
|
Kernel.exit(1)
|
224
220
|
end
|
225
|
-
puts
|
221
|
+
puts 'passed'
|
226
222
|
|
227
|
-
printf
|
228
|
-
if config_hash.fetch(
|
229
|
-
puts
|
223
|
+
printf '%-50s', 'Config file has application...'
|
224
|
+
if config_hash.fetch('applications').empty?
|
225
|
+
puts 'failed'
|
230
226
|
Kernel.exit(1)
|
231
227
|
end
|
232
|
-
puts
|
228
|
+
puts 'passed'
|
233
229
|
|
234
|
-
printf
|
235
|
-
tcell_application = config_hash.fetch(
|
236
|
-
if !tcell_application.key?(
|
237
|
-
puts
|
230
|
+
printf '%-50s', 'Application has api_key and app_id... '
|
231
|
+
tcell_application = config_hash.fetch('applications')[0]
|
232
|
+
if !tcell_application.key?('app_id') || !tcell_application.key?('api_key')
|
233
|
+
puts 'failed'
|
238
234
|
Kernel.exit(1)
|
239
235
|
end
|
240
|
-
puts
|
236
|
+
puts 'passed'
|
241
237
|
|
242
|
-
printf
|
243
|
-
require
|
238
|
+
printf '%-50s', 'Check for unknown settings... '
|
239
|
+
require 'tcell_agent/config/unknown_options'
|
244
240
|
messages = TCellAgent::Config::Validate.get_unknown_options(config_hash)
|
245
|
-
|
246
|
-
puts
|
241
|
+
unless messages.empty?
|
242
|
+
puts 'failed'
|
247
243
|
messages.each do |message|
|
248
|
-
|
244
|
+
puts message
|
249
245
|
end
|
250
246
|
Kernel.exit(1)
|
251
247
|
end
|
252
|
-
puts
|
248
|
+
puts 'passed'
|
253
249
|
|
254
|
-
printf
|
250
|
+
printf '%-50s', 'Requiring configuration library... '
|
255
251
|
require 'tcell_agent/configuration'
|
256
252
|
require 'tcell_agent/api'
|
257
|
-
puts
|
253
|
+
puts 'passed'
|
258
254
|
|
259
|
-
printf
|
255
|
+
printf '%-50s', 'Make test API call for policies... '
|
260
256
|
api = TCellAgent::TCellApi.new
|
261
|
-
api.poll_api
|
262
|
-
puts
|
257
|
+
api.poll_api
|
258
|
+
puts 'passed'
|
263
259
|
|
264
|
-
printf
|
260
|
+
printf '%-50s', 'Sending a Test event... '
|
265
261
|
send_succeeded = api.send_event_set([])
|
266
|
-
|
267
|
-
puts
|
262
|
+
unless send_succeeded
|
263
|
+
puts 'failed'
|
268
264
|
Kernel.exit(1)
|
269
265
|
end
|
270
|
-
puts
|
266
|
+
puts 'passed'
|
271
267
|
|
272
|
-
printf
|
268
|
+
printf '%-50s', 'Loading native library... '
|
273
269
|
require 'tcell_agent/rust/whisperer'
|
274
|
-
|
275
|
-
puts
|
270
|
+
unless TCellAgent::Rust::Wrapper.common_lib_available?
|
271
|
+
puts 'failed'
|
276
272
|
Kernel.exit(1)
|
277
273
|
end
|
278
|
-
puts
|
274
|
+
puts 'passed'
|
279
275
|
|
280
276
|
puts
|
281
|
-
puts
|
282
|
-
puts
|
277
|
+
puts 'all tests passed, looks good.'
|
278
|
+
puts 'done.'
|
283
279
|
end
|
284
|
-
|
@@ -12,8 +12,6 @@ require "tcell_agent/policies/clickjacking_policy"
|
|
12
12
|
require "tcell_agent/policies/http_tx_policy"
|
13
13
|
require "tcell_agent/policies/http_redirect_policy"
|
14
14
|
require "tcell_agent/policies/secure_headers_policy"
|
15
|
-
require "tcell_agent/policies/honeytokens_policy"
|
16
|
-
require "tcell_agent/policies/appsensor_policy"
|
17
15
|
|
18
16
|
require "tcell_agent/sensor_events/server_agent"
|
19
17
|
require "tcell_agent/sensor_events/metrics"
|
@@ -32,6 +30,14 @@ require 'json'
|
|
32
30
|
module TCellAgent
|
33
31
|
class Agent
|
34
32
|
|
33
|
+
# cmdi events are special because they can be triggered very easily by running any shell command.
|
34
|
+
# Startup scripts are likely to run shell commands. It's not a good idea to startup the event
|
35
|
+
# processor before worker processses are forked, so the safest thing to do is let a different
|
36
|
+
# event start the event processor to avoid deadlocking worker processes.
|
37
|
+
def is_it_safe_to_send_cmdi_events?()
|
38
|
+
event_processor_running?
|
39
|
+
end
|
40
|
+
|
35
41
|
def ensure_event_processor_running
|
36
42
|
return if event_processor_running?
|
37
43
|
return if TCellAgent.configuration.should_start_event_manager? == false
|
@@ -124,7 +130,7 @@ module TCellAgent
|
|
124
130
|
@dispatchEvents.push(event)
|
125
131
|
}
|
126
132
|
end
|
127
|
-
if (event.flush
|
133
|
+
if (event.flush || @dispatchEvents.length >= @dispatchEventsLimit || wait_for < 0)
|
128
134
|
last_run_time = Time.now
|
129
135
|
self.send_dispatch_events(tapi)
|
130
136
|
end
|
@@ -137,7 +143,7 @@ module TCellAgent
|
|
137
143
|
end
|
138
144
|
rescue TCellAgent::ConfigurationException
|
139
145
|
Thread.exit
|
140
|
-
rescue
|
146
|
+
rescue StandardError => e
|
141
147
|
last_run_time = Time.now
|
142
148
|
TCellAgent.logger.error("Exception while processing events: #{e.message}")
|
143
149
|
TCellAgent.logger.debug(e.backtrace)
|
@@ -146,7 +152,7 @@ module TCellAgent
|
|
146
152
|
}
|
147
153
|
end
|
148
154
|
end
|
149
|
-
rescue
|
155
|
+
rescue StandardError => xyz
|
150
156
|
TCellAgent.logger.error("Exception while starting processor: #{xyz.message}")
|
151
157
|
TCellAgent.logger.debug(xyz.backtrace)
|
152
158
|
end
|
@@ -181,12 +187,12 @@ module TCellAgent
|
|
181
187
|
else
|
182
188
|
TCellAgent::Agent.send_to_event_pipe(event)
|
183
189
|
end
|
184
|
-
rescue
|
190
|
+
rescue StandardError => e
|
185
191
|
TCellAgent.logger.error("Exception while processing (forked) events: #{e.message}")
|
186
192
|
TCellAgent.logger.debug(e.backtrace)
|
187
193
|
end
|
188
194
|
end
|
189
|
-
rescue
|
195
|
+
rescue StandardError => e
|
190
196
|
TCellAgent.logger.error("Exception while running (forked) events: #{e.message}")
|
191
197
|
TCellAgent.logger.debug(e.backtrace)
|
192
198
|
end
|
@@ -201,7 +207,7 @@ module TCellAgent
|
|
201
207
|
return
|
202
208
|
end
|
203
209
|
@fork_event_queue.push(event)
|
204
|
-
rescue
|
210
|
+
rescue StandardError => queue_exception
|
205
211
|
TCellAgent.logger.debug("Could not add (forked) event #{queue_exception.message}")
|
206
212
|
end
|
207
213
|
end
|
@@ -231,19 +237,14 @@ module TCellAgent
|
|
231
237
|
def start_metrics_event_thread
|
232
238
|
return if TCellAgent.configuration.should_consume_event? == false
|
233
239
|
@metrics_event_thread = Thread.new do
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
TCellAgent.logger.debug(e.backtrace)
|
242
|
-
end
|
240
|
+
loop do
|
241
|
+
begin
|
242
|
+
event = @metrics_event_queue.pop
|
243
|
+
TCellAgent::Agent.send_to_metrics_pipe(event)
|
244
|
+
rescue StandardError => e
|
245
|
+
TCellAgent.logger.error("Exception while processing (forked) metrics: #{e.message}")
|
246
|
+
TCellAgent.logger.debug(e.backtrace)
|
243
247
|
end
|
244
|
-
rescue Exception => e
|
245
|
-
TCellAgent.logger.error("Exception while running (forked) metrics: #{e.message}")
|
246
|
-
TCellAgent.logger.debug(e.backtrace)
|
247
248
|
end
|
248
249
|
end
|
249
250
|
end
|
@@ -257,7 +258,7 @@ module TCellAgent
|
|
257
258
|
return
|
258
259
|
end
|
259
260
|
@metrics_event_queue.push(event)
|
260
|
-
rescue
|
261
|
+
rescue StandardError => queue_exception
|
261
262
|
TCellAgent.logger.debug("Could not add (forked) metric #{queue_exception.message}")
|
262
263
|
end
|
263
264
|
end
|
@@ -280,7 +281,7 @@ module TCellAgent
|
|
280
281
|
TCellAgent.logger.debug("Dropping event because queue full")
|
281
282
|
end
|
282
283
|
|
283
|
-
rescue
|
284
|
+
rescue StandardError => queue_exception
|
284
285
|
TCellAgent.logger.debug("Could not add event #{queue_exception.message}")
|
285
286
|
end
|
286
287
|
end
|
@@ -12,14 +12,14 @@ module TCellAgent
|
|
12
12
|
@@parent_id = Process.pid
|
13
13
|
def initialize(&block)
|
14
14
|
begin
|
15
|
-
@readp, @writep = IO.pipe('ASCII-8BIT', 'ASCII-8BIT', binmode
|
15
|
+
@readp, @writep = IO.pipe('ASCII-8BIT', 'ASCII-8BIT', :binmode => true)
|
16
16
|
if defined?(::Encoding::ASCII_8BIT)
|
17
17
|
@writep.set_encoding(::Encoding::ASCII_8BIT)
|
18
18
|
end
|
19
19
|
if is_parent?
|
20
20
|
self.start_listener(&block)
|
21
21
|
end
|
22
|
-
rescue
|
22
|
+
rescue StandardError => init_exception
|
23
23
|
TCellAgent.logger.error("Could not start listener for pipe to forks")
|
24
24
|
TCellAgent.logger.error(init_exception.message)
|
25
25
|
TCellAgent.logger.debug(init_exception.backtrace)
|
@@ -30,7 +30,7 @@ module TCellAgent
|
|
30
30
|
end
|
31
31
|
def start_listener(&block)
|
32
32
|
Thread.new {
|
33
|
-
|
33
|
+
loop do
|
34
34
|
begin
|
35
35
|
packed_bytes = @readp.read(4)
|
36
36
|
event_length = packed_bytes.unpack("L>").first
|
@@ -39,7 +39,7 @@ module TCellAgent
|
|
39
39
|
if block
|
40
40
|
block.call(event)
|
41
41
|
end
|
42
|
-
rescue
|
42
|
+
rescue StandardError => block_exception
|
43
43
|
TCellAgent.logger.error("Could not decode block")
|
44
44
|
TCellAgent.logger.error(block_exception.message)
|
45
45
|
TCellAgent.logger.debug(block_exception.backtrace)
|
@@ -57,7 +57,7 @@ module TCellAgent
|
|
57
57
|
packed_event = Marshal.dump(event)
|
58
58
|
packed_bytes = [packed_event.bytesize].pack("L>")
|
59
59
|
@writep.write(packed_bytes+packed_event)
|
60
|
-
rescue
|
60
|
+
rescue StandardError => block_exception
|
61
61
|
TCellAgent.logger.error("Could not write to pipe")
|
62
62
|
TCellAgent.logger.error(block_exception.message)
|
63
63
|
TCellAgent.logger.debug(block_exception.backtrace)
|
@@ -69,7 +69,7 @@ module TCellAgent
|
|
69
69
|
@@event_pipe_manager = ForkPipeManager.new { |event|
|
70
70
|
begin
|
71
71
|
TCellAgent.send_event(event)
|
72
|
-
rescue
|
72
|
+
rescue StandardError => block_exception
|
73
73
|
TCellAgent.logger.error("Could handle send_event_block")
|
74
74
|
TCellAgent.logger.error(block_exception.message)
|
75
75
|
TCellAgent.logger.debug(block_exception.backtrace)
|
@@ -100,7 +100,7 @@ module TCellAgent
|
|
100
100
|
val.fetch("user_agent", nil)
|
101
101
|
)
|
102
102
|
else
|
103
|
-
raise
|
103
|
+
raise StandardError.new("Metrics Pipe Manager received unknown metric: #{val.fetch("_type","")}")
|
104
104
|
end
|
105
105
|
end
|
106
106
|
}
|