corn 0.5.6 → 0.5.7.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2128d61b559f2364bb060b87e258069ed0573622
-  data.tar.gz: 84dadacc4a71ad7b6c33209ef3cf1c5829e91936
+  metadata.gz: 8cd2ed0bc9b102eca18ed7e80ea7a9313702fc02
+  data.tar.gz: 6ce6d074da698b05112110f83635b497f8453735
 SHA512:
-  metadata.gz: fa94880e60a3bf2e53e8901fc13e62d74424708c67d1f0c9848bbc63bcf825fc5c286298ca3ea0da847a315cc516fbb075ead74f4f1fd881ce17d4d3ee5a1ac0
-  data.tar.gz: 7791a5595a58ac89cf35ec041112ee78999e14f17665f8adf168f8f32b3bb78f28b75d2e3dc7f46ac58f7a4ecad7a612f88a4bc5ebe105a57c3c3857d46a9713
+  metadata.gz: 07ad76f726b438be79b29f51144c913fd1f413abbd221c2c0beff3c9453db7a439441de4080bc4ebb111c330ef5c6c7f727709f8605b0aeca26925f3920eb22e
+  data.tar.gz: 5d216ec62d5c8d3ccc111d7aee67b3740a3b1c4899023479ed725b27fe3855fd6c27afbaa8322d673d321c28d94471aa2590a65592532d190ad96ce84e6311d6

data/lib/corn.rb

@@ -33,9 +33,12 @@ module Corn
            :rack_middleware => Rack::SlowRequestProfiler,
            :rack_slow_request_profiler => Rack::SlowRequestProfiler,
            :slow_request_threshold => 5,
+           :fast_request_threshold => lambda { [Corn.sampling_interval * 5, Corn.slow_request_threshold.to_f / 5].max },
            :profiling => true,
            :sampling_interval => 0.1,
-           :post_interval => 2
+           :post_interval => 2,
+           :post_fast_request_interval => 60, #seconds
+           :fast_request_sampling_limit => 1024 * 1024 * 2 #2MB
          })
 
   module_function

data/lib/corn/post.rb

@@ -5,9 +5,12 @@ require 'time'
 
 module Corn
   class Post
-    def initialize(interval)
+    def initialize(interval, sampling_limit, sampling_time)
       @queue = Queue.new
       @thread = start_post_thread(interval)
+      @sampling_limit = sampling_limit
+      @sampling_time = sampling_time
+      reset_sampling
     end
 
     def terminate
@@ -24,8 +27,21 @@ module Corn
       Thread.start do
         begin
           loop do
-            http_post(@queue.pop)
-            sleep interval
+            d = @queue.pop
+            case d[:action]
+            when :post
+              http_post([d])
+              sleep interval
+            when :sampling
+              @sampling << d
+              if Time.now - @sampling_start > @sampling_time
+                http_post(@sampling.items, :sampling)
+                reset_sampling
+                sleep interval
+              end
+            else
+              Corn.logger.info("Corn: Ignore unknown action: #{d[:action]}")
+            end
           end
         rescue => e
           Corn.logger.error("Corn post thread stopped by error #{e.message}\n#{e.backtrace.join("\n")}")
@@ -34,14 +50,25 @@ module Corn
     end
 
     def enqueue(data)
-      return if data.nil?
+      return if data.nil? || data.empty?
       @queue << data
     end
 
-    def http_post(data)
+    def http_post(reports, type=nil)
       uri = URI.parse(submit_url)
       req = Net::HTTP::Post.new(uri.path)
-      req.set_form_data(data.merge('client_id' => Corn.client_id))
+      form_data = [['client_id', Corn.client_id]]
+      if type
+        form_data << ['type', type]
+      end
+      reports.each_with_index do |rep, i|
+        [:name, :start_at, :end_at, :data].each do |k|
+          if v = rep[k]
+            form_data << ["reports[][#{k}]", v]
+          end
+        end
+      end
+      req.set_form_data(form_data)
 
       http = Net::HTTP.new(uri.host, uri.port)
       if uri.scheme == 'https'
@@ -55,7 +82,7 @@ module Corn
         end
       end
       res = http.request(req)
-      Corn.logger.info("Corn report submitted to #{submit_url}")
+      Corn.logger.info("Corn reports(#{reports.size}) submitted to #{submit_url}")
       unless res.is_a?(Net::HTTPSuccess)
         Corn.logger.error("Post failed: #{res.message}(#{res.code}), response body: \n#{res.body}")
       end
@@ -66,5 +93,11 @@ module Corn
     def submit_url
       Corn.submit_url
     end
+
+    private
+    def reset_sampling
+      @sampling = ReservoirSampling.new(@sampling_limit)
+      @sampling_start = Time.now
+    end
   end
 end

data/lib/corn/profiler.rb

@@ -3,8 +3,9 @@ require 'sampling_prof'
 
 module Corn
   class Profiler
-    def initialize(post_interval, sampling_interval)
-      @post = Post.new(post_interval)
+    def initialize(post_interval, post_sampling_limit, post_sampling_time,
+                   sampling_interval)
+      @post = Post.new(post_interval, post_sampling_limit, post_sampling_time)
       @prof = SamplingProf.new(sampling_interval)
       at_exit { terminate }
     end

data/lib/corn/rack/request_env.rb

@@ -31,9 +31,9 @@ module Corn
           name = "#{name}?#{@env[:query_string]}"
         end
         {
-          'report[name]' => name,
-          'report[start_at]' => @env[:start_time].iso8601,
-          'report[end_at]' => end_at.iso8601
+          :name => name,
+          :start_at => @env[:start_time].iso8601,
+          :end_at => end_at.iso8601
         }
       end
     end

data/lib/corn/rack/slow_request_profiler.rb

@@ -1,4 +1,6 @@
+require 'thread'
 require 'corn/profiler'
+require 'corn/reservoir_sampling'
 require 'corn/rack/request_env'
 
 module Corn
@@ -7,8 +9,11 @@ module Corn
       class ProfilingApp
         def initialize(app)
           @@prof ||= Profiler.new(Corn.post_interval,
+                                  Corn.fast_request_sampling_limit,
+                                  Corn.post_fast_request_interval,
                                   Corn.sampling_interval)
           @app = app
+
           Corn.logger.info("Corn sampling interval: #{Corn.sampling_interval}")
           Corn.logger.info("Corn slow request threshold: #{Corn.slow_request_threshold}")
         end
@@ -29,11 +34,20 @@ module Corn
         def output_handler(env)
           request_env = RequestEnv.new(env)
           lambda do |data|
-            if request_env.time > Corn.slow_request_threshold
-              request_env.to_report.merge("data" => data)
+            t = request_env.time
+            if t < fast_request_threshold || t > slow_request_threshold
+              action = t > Corn.slow_request_threshold ? :post : :sampling
+              request_env.to_report.merge(:data => data, :action => action)
             end
           end
         end
+
+        def fast_request_threshold
+          @frt ||= Corn.fast_request_threshold
+        end
+        def slow_request_threshold
+          @srt ||= Corn.slow_request_threshold
+        end
       end
 
       def initialize(app)
@@ -44,6 +58,7 @@ module Corn
         @app.call(env)
       end
 
+      # for test
       def terminate
         @app.terminate if @app.respond_to?(:terminate)
       end

data/lib/corn/reservoir_sampling.rb

@@ -0,0 +1,29 @@
+module Corn
+  class ReservoirSampling
+    attr_reader :items
+    def initialize(limit)
+      @limit = limit
+      @items = []
+      @size = -1
+      @count = 0
+    end
+
+    def <<(item)
+      @count += 1
+      if @size < 0 && bytesize < @limit
+        items << item
+      else
+        @size = items.size
+        j = rand(@count).to_i
+        if j < @size
+          items[j] = item
+        end
+      end
+    end
+
+    private
+    def bytesize
+      items.map{|d|d[:data].bytesize}.reduce(:+) || 0
+    end
+  end
+end

data/lib/generators/corn/config/config_generator.rb

@@ -56,7 +56,7 @@ Corn.config({
               # called. So you can also use it to turn on profiling by a request parameter.
               # For example: Corn.config(:profiling => lambda {|env| env["QUERY_STRING"] =~ /corn_profiling=true/ })
               # This configuration will be checked for every request, so don't do anything
-              # expense here.
+              # expensive here.
               # :profiling => true,
             })
 # Install corn rack middleware for profiling slow requests

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: corn
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.5.7.beta1
 platform: ruby
 authors:
 - Xiao Li
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-07-10 00:00:00.000000000 Z
+date: 2014-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sampling_prof
@@ -38,6 +38,7 @@ files:
 - lib/corn/post.rb
 - lib/corn/profiler.rb
 - lib/corn/rack.rb
+- lib/corn/reservoir_sampling.rb
 - lib/corn/rack/request_env.rb
 - lib/corn/rack/slow_request_profiler.rb
 - lib/generators/corn/config/config_generator.rb
@@ -56,9 +57,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9