threatstack-agent-ruby 0.2.1 → 0.2.2

data/lib/jobs/delayed_job.rb

@@ -6,8 +6,8 @@ module Threatstack
   module Jobs
 
     class DelayedJob
-      def initialize(name_or_logger, initial_delay = 0, args = nil, &block)
-        @job = RecurrentJob.new(name_or_logger,1, initial_delay, 1, args, &block)
+      def initialize(name_or_logger, initial_delay = 0, *args, &block)
+        @job = RecurrentJob.new(name_or_logger, 1, initial_delay, 1, *args, &block)
       end
 
       def stop

data/lib/jobs/event_submitter.rb

@@ -46,40 +46,34 @@ module Threatstack
       end
 
       def submit
-        @logger.debug 'Starting event submission'
-
         # fetch events from the accumulator
-        events = fetch_events
-        @logger.debug "Found a total of #{events.length} events"
-
+        events = @accumulator.remove_events EVENTS_PER_REQ
         if events.empty?
           @logger.debug 'No events to report'
           return
         end
+        @logger.debug "Found a total of #{events.length} events"
 
-        # split events array into groups of EVENTS_PER_REQ
-        events.each_slice(EVENTS_PER_REQ) do |group|
-          # convert payload to JSON
-          json_payload = "[#{group.collect(&:to_json_string).join(', ')}]"
-          # submit http request
-          uri = URI.parse(APPSEC_BASE_URL + APPSEC_EVENTS_URL)
-          headers = {
-            'Content-Type' => 'application/json',
-            'bluefyre-agent-id' => AGENT_ID,
-            'bluefyre-agent-instance-id' => AGENT_INSTANCE_ID
-          }
-          http = Net::HTTP.new(uri.host, uri.port)
-          http.use_ssl = true
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          req = Net::HTTP::Post.new(uri.request_uri, headers)
-          req.body = json_payload
-          @logger.debug "Sending #{group.length} events with payload: #{json_payload}"
-          begin
-            resp = http.request(req)
-            @logger.debug "Sent #{group.length} events, HTTP code: #{resp.code}"
-          rescue StandardError => e
-            @logger.error "StandardError: #{e.inspect}"
-          end
+        # convert payload to JSON
+        json_payload = "[#{events.collect(&:to_json_string).join(', ')}]"
+        # submit http request
+        uri = URI.parse(APPSEC_BASE_URL + APPSEC_EVENTS_URL)
+        headers = {
+          'Content-Type' => 'application/json',
+          'bluefyre-agent-id' => AGENT_ID,
+          'bluefyre-agent-instance-id' => AGENT_INSTANCE_ID
+        }
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        req = Net::HTTP::Post.new(uri.request_uri, headers)
+        req.body = json_payload
+        @logger.debug "Sending #{events.length} events with payload: #{json_payload}"
+        begin
+          resp = http.request(req)
+          @logger.debug "Sent #{events.length} events, HTTP code: #{resp.code}"
+        rescue StandardError => e
+          @logger.error "StandardError: #{e.inspect}"
         end
       end
 
@@ -87,15 +81,6 @@ module Threatstack
         start
         @accumulator.add_event event
       end
-
-      def fetch_events
-        # check total number of events first
-        total_events = @accumulator.events.length
-        return [] unless total_events > 0
-
-        # retrieve events from the accumulator in a thread-safe manner
-        @accumulator.remove_events total_events
-      end
     end
   end
 end

data/lib/jobs/job_queue.rb

@@ -7,22 +7,31 @@ module Threatstack
       attr_reader :job_queue
       attr_reader :job_thread
       attr_reader :stopped
+      attr_reader :working
 
       def initialize
         @stopped = false
+        @working = false
         @job_queue = Queue.new
         @job_thread = Thread.new do
           # Fetch an item from the worker queue, or wait until one is available
           while (job = @job_queue.pop)
-            Thread.stop if @stopped
-            job[:action].call(job[:args])
+            if @stopped
+              # clear the queue
+              @job_queue.clear
+              break
+            end
+            @working = true
+            job[:action].call(*job[:args])
+            @working = false if @job_queue.empty?
           end
+          @working = false
           @stopped = true
           Thread.stop
         end
       end
 
-      def add_job(args = nil, &block)
+      def add_job(*args, &block)
         raise 'Invalid proc provided' unless block_given?
 
         @job_queue.push(:action => block, :args => args)
@@ -30,8 +39,8 @@ module Threatstack
 
       def stop
         @stopped = true
-        # push nil to stop the while loop
-        add_job nil
+        # push empty job to stop the while loop
+        add_job(nil) {}
       end
     end
   end

data/lib/jobs/recurrent_job.rb

@@ -12,7 +12,7 @@ module Threatstack
       attr_reader :stopped
       attr_reader :logger
 
-      def initialize(name_or_logger, repeat_every_sec = 10, initial_delay_sec = 0, total_times = nil, args = nil, &block)
+      def initialize(name_or_logger, repeat_every_sec = 10, initial_delay_sec = 0, total_times = nil, *args, &block)
         raise 'Block not specified' unless block_given?
 
         # create logger or use passed one if available
@@ -43,7 +43,7 @@ module Threatstack
             # perform main action
             begin
               @logger.debug 'Calling job action'
-              block.call args
+              block.call *args
               @logger.debug 'Job action called'
             rescue StandardError => e
               @logger.error "StandardError in job action: #{e.inspect}"

data/lib/utils/capped_queue.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'thread'
+
+module Threatstack
+  module Utils
+    class CappedQueue
+
+      def initialize(max)
+        @queue = []
+        @max = max
+        @mutex = Mutex.new
+      end
+
+      def length
+        @queue.length
+      end
+
+      def push(*args)
+        @mutex.synchronize do
+          @queue.push(*args)
+          @queue.shift(@queue.length - @max) if @queue.length > @max
+        end
+      end
+
+      def shift(*args)
+        @mutex.synchronize do
+          @queue.shift(*args)
+        end
+      end
+
+      def clear
+        @mutex.synchronize do
+          @queue.clear
+        end
+      end
+    end
+  end
+end

data/lib/utils/logger.rb

@@ -8,15 +8,17 @@ require_relative './formatter'
 module Threatstack
   module Utils
     module TSLogger
-      def self.create(name)
+      COLORS = ['blue', 'gray', 'green', 'yellow', 'cyan', 'magenta'].freeze
+
+      def self.create(name, opts = {})
         formatter = TSFormatter.new
         logger = Logger.new STDOUT
         logger.progname = "TS#{name}"
         logger.datetime_format = '%FT%T%:z'
         logger.level = Threatstack::Constants::LOG_LEVEL
         colors = Threatstack::Constants::LOG_COLORS
-        random_color = ['blue', 'gray', 'green', 'yellow', 'cyan', 'magenta'].sample
-        progname = colors ? formatter.send(random_color, '%16.16s' % logger.progname) : '%16.16s' % logger.progname
+        log_color = opts[:color] || COLORS.sample
+        progname = colors ? formatter.send(log_color, '%16.16s' % logger.progname) : '%16.16s' % logger.progname
         logger.formatter = proc do |severity, datetime, _progname, msg|
           pid = Process.pid
           sev_padded = '%5.5s' % severity

data/threatstack-agent-ruby.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'threatstack-agent-ruby'
-  spec.version       = '0.2.1'
+  spec.version       = '0.2.2'
   spec.authors       = ['Threat Stack Inc']
   spec.email         = ['support@threatstack.com']
   spec.summary       = 'Ruby version of the ThreatStack agent which helps identify security vulnerabilities at runtime'
@@ -13,7 +13,8 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://www.threatstack.com'
   spec.extra_rdoc_files = [
     "LICENSE"
-  ]  
+  ]
+  spec.required_ruby_version = '>= 1.8.7'
 
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^((test|spec|features|)/|Gemfile_release|Rakefile|README.md|.gitlab-ci.yml|.rubocop.yml|Gemfile.lock|.gitignore)}) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: threatstack-agent-ruby
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Threat Stack Inc
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-20 00:00:00.000000000 Z
+date: 2021-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: network_interface
@@ -161,11 +161,15 @@ extra_rdoc_files:
 files:
 - Gemfile
 - LICENSE
+- ci/requirements.txt
+- ci/trigger.py
 - ext/libinjection/extconf.rb
 - ext/libinjection/libinjection.h
 - ext/libinjection/libinjection.i
 - ext/libinjection/libinjection_html5.c
 - ext/libinjection/libinjection_html5.h
+- ext/libinjection/libinjection_pathtraversal.c
+- ext/libinjection/libinjection_pathtraversal_data.h
 - ext/libinjection/libinjection_sqli.c
 - ext/libinjection/libinjection_sqli.h
 - ext/libinjection/libinjection_sqli_data.h
@@ -182,15 +186,17 @@ files:
 - lib/events/models/instrumentation_event.rb
 - lib/exceptions/request_blocked_error.rb
 - lib/instrumentation/common.rb
+- lib/instrumentation/frameworks/kernel.rb
+- lib/instrumentation/frameworks/rails.rb
+- lib/instrumentation/frameworks/random.rb
 - lib/instrumentation/instrumenter.rb
-- lib/instrumentation/kernel.rb
-- lib/instrumentation/rails.rb
 - lib/jobs/delayed_job.rb
 - lib/jobs/event_submitter.rb
 - lib/jobs/job_queue.rb
 - lib/jobs/recurrent_job.rb
 - lib/threatstack-agent-ruby.rb
 - lib/utils/aws_utils.rb
+- lib/utils/capped_queue.rb
 - lib/utils/formatter.rb
 - lib/utils/logger.rb
 - threatstack-agent-ruby.gemspec
@@ -206,14 +212,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Ruby version of the ThreatStack agent which helps identify security vulnerabilities

data/lib/instrumentation/kernel.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require_relative './common.rb'
-require_relative './instrumenter.rb'
-require_relative '../utils/logger'
-
-module Threatstack
-  module Instrumentation
-    module TSKernel
-      @@logger = Threatstack::Utils::TSLogger.create 'KernelInstrumentation'
-
-      # methods to wrap
-      METHOD_NAMES = ['exec', 'system', '`'].freeze
-
-      def self.wrap_methods
-        # executed every time a wrapped method is called
-        on_method_call = Proc.new do |params|
-          module_name = params[:target_class].name.downcase
-          method_name = params[:method_name].downcase
-          called_by = params[:caller_loc] ? params[:caller_loc].first : nil
-          file_path = called_by ? called_by.absolute_path : nil
-          # special case for ` method emulation
-          if method_name == '`' && !file_path.nil? && file_path =~ /.*\/kernel\/agnostics\.rb$/
-            called_by = params[:caller_loc][1]
-            file_path = called_by ? called_by.absolute_path : nil
-          end
-          line_num = called_by ? called_by.lineno : nil
-
-          arg = params[:args] ? params[:args].first : nil
-          args = arg ? [arg] : []
-
-          # create and queue the event
-          Threatstack::Instrumentation.create_instrumentation_event(module_name, method_name, file_path, line_num, args)
-        end
-        @@logger.info "Instrumenting Kernel methods: #{METHOD_NAMES}"
-        instrumenter =  Threatstack::Instrumentation::Instrumenter.instance
-        METHOD_NAMES.each do |method_name|
-          instrumenter.wrap_class_method(Kernel, method_name, &on_method_call)
-          instrumenter.wrap_instance_method(Kernel, method_name, &on_method_call)
-        end
-      end
-
-    end
-  end
-end

data/lib/instrumentation/rails.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-require_relative '../constants'
-require_relative '../exceptions/request_blocked_error'
-require_relative './common'
-require_relative '../utils/logger'
-
-module Threatstack
-  module Instrumentation
-    module TSRails
-      @@logger = Threatstack::Utils::TSLogger.create 'RailsInstrumentation'
-
-      def self.patch_action_controller
-        @@logger.info 'Looking for Rails gem'
-        return unless defined?(::Rails) && defined?(::Rails::VERSION)
-        return unless defined?(ActionController) && defined?(ActionController::Base)
-
-        @@logger.info "Rails #{Rails::VERSION::MAJOR.to_s} gem found, instrumenting ActionController"
-        ActionController::Base.class_eval do
-          include Threatstack::Instrumentation::TSRails::TSActionController
-        end
-        @@logger.info 'Rails instrumentation done'
-      end
-
-      module TSActionController
-        include Threatstack::Constants
-        @@logger = Threatstack::Utils::TSLogger.create 'RailsInstrumentation'
-
-        def process_action(*args)
-          # we need the headers hack below because Rails adds a lot of internal headers
-          headers = request.headers.each_with_object({}) do |(k, v), obj|
-            obj[k] = v if k.in?(CGI_VARIABLES) || k =~ /^HTTP_/
-          end
-          @@logger.debug("Incoming request: #{{ :headers => headers, :path => request.path_parameters,
-                                                :query => Threatstack::Instrumentation.drop_sensitive_fields(request.query_parameters),
-                                                :body => Threatstack::Instrumentation.drop_sensitive_fields(request.request_parameters) }}")
-          backtrace = caller.join("\n")
-
-          # check path/query/body parameters
-          path_res = Threatstack::Instrumentation.check_parameters(request.path_parameters, 'path', request, headers, backtrace)
-          query_res = Threatstack::Instrumentation.check_parameters(request.query_parameters, 'query', request, headers, backtrace)
-          body_res = Threatstack::Instrumentation.check_parameters(request.request_parameters, 'body', request, headers, backtrace)
-
-          @@logger.debug "RequestStats -- Path: #{path_res}, Query: #{query_res}, Body: #{body_res}"
-          sqli_found = (path_res[:sqli] || query_res[:sqli] || body_res[:sqli])
-          xss_found = (path_res[:xss] || query_res[:xss] || body_res[:xss])
-          # raise an exception if any attack payloads were detected and blocking is enabled
-          if (BLOCK_SQLI && sqli_found) || (BLOCK_XSS && xss_found)
-            raise Threatstack::Exceptions::RequestBlockedError, REQUEST_BLOCKED
-          end
-
-          # continue processing the request normally if no issues were found
-          super
-        end
-
-      end
-    end
-  end
-end