thinkingdata-ruby 2.0.0 → 2.0.2

data/lib/thinkingdata-ruby/td_batch_consumer.rb

@@ -1,142 +1,171 @@
-require 'json'
-require 'net/http'
-require 'stringio'
-
-module ThinkingData
-  ##
-  # Upload data by http
-  class TDBatchConsumer
-
-    # buffer count
-    DEFAULT_LENGTH = 20
-    MAX_LENGTH = 2000
-
-    ##
-    # Init batch consumer
-    def initialize(server_url, app_id, max_buffer_length = DEFAULT_LENGTH)
-      @server_uri = URI.parse(server_url)
-      @server_uri.path = '/sync_server'
-      @app_id = app_id
-      @compress = true
-      @max_length = [max_buffer_length, MAX_LENGTH].min
-      @buffers = []
-      TDLog.info("TDBatchConsumer init success. ServerUrl: #{server_url}, appId: #{app_id}")
-    end
-
-    ##
-    # http request compress
-    # @param compress [Boolean] compress or not
-    # @deprecated please use: set_compress
-    def _set_compress(compress)
-      @compress = compress
-    end
-
-    ##
-    # http request compress
-    # @param compress [Boolean] compress or not
-    def set_compress(compress)
-      @compress = compress
-    end
-
-    def add(message)
-      TDLog.info("Enqueue data to buffer. buffer size: #{@buffers.length}, data: #{message}")
-      @buffers << message
-      flush if @buffers.length >= @max_length
-    end
-
-    def close
-      flush
-      TDLog.info("TDBatchConsumer close.")
-    end
-
-    def flush
-      TDLog.info("TDBatchConsumer flush data.")
-      begin
-        @buffers.each_slice(@max_length) do |chunk|
-          if @compress
-            wio = StringIO.new("w")
-            gzip_io = Zlib::GzipWriter.new(wio)
-            gzip_io.write(chunk.to_json)
-            gzip_io.close
-            data = wio.string
-          else
-            data = chunk.to_json
-          end
-          compress_type = @compress ? 'gzip' : 'none'
-          headers = {'Content-Type' => 'application/plaintext',
-                     'appid' => @app_id,
-                     'compress' => compress_type,
-                     'TE-Integration-Type'=>'Ruby',
-                     'TE-Integration-Version'=>ThinkingData::VERSION,
-                     'TE-Integration-Count'=>@buffers.count,
-                     'TA_Integration-Extra'=>'batch'}
-          request = CaseSensitivePost.new(@server_uri.request_uri, headers)
-          request.body = data
-
-          TDLog.info("Send data, request: #{data}")
-          begin
-            response_code, response_body = _request(@server_uri, request)
-            TDLog.info("Send data, response: #{response_body}")
-          rescue => e
-            raise ConnectionError.new("Could not connect to TE server, with error \"#{e.message}\".")
-          end
-
-          result = {}
-          if response_code.to_i == 200
-            begin
-              result = JSON.parse(response_body.to_s)
-            rescue JSON::JSONError
-              raise ServerError.new("Could not interpret TE server response: '#{response_body}'")
-            end
-          end
-
-          if result['code'] != 0
-            raise ServerError.new("Could not write to TE, server responded with #{response_code} returning: '#{response_body}'")
-          end
-        end
-      rescue
-        raise
-      end
-      @buffers = []
-    end
-
-    private
-    def _request(uri, request)
-      client = Net::HTTP.new(uri.host, uri.port)
-      client.use_ssl = uri.scheme === 'https' ? true : false
-      client.open_timeout = 10
-      client.continue_timeout = 10
-      client.read_timeout = 10
-      client.ssl_timeout = 10
-
-      response = client.request(request)
-      [response.code, response.body]
-    end
-  end
-
-  ##
-  # Private class. Send data tools
-  class CaseSensitivePost < Net::HTTP::Post
-    def initialize_http_header(headers)
-      @header = {}
-      headers.each{|k,v| @header[k.to_s] = [v] }
-    end
-
-    def [](name)
-      @header[name.to_s]
-    end
-
-    def []=(name, val)
-      if val
-        @header[name.to_s] = [val]
-      else
-        @header.delete(name.to_s)
-      end
-    end
-
-    def capitalize(name)
-      name
-    end
-  end
-
-end
+require 'json'
+require 'net/http'
+require 'stringio'
+
+module ThinkingData
+  ##
+  # Upload data by http
+  class TDBatchConsumer
+
+    # buffer count
+    DEFAULT_LENGTH = 20
+    MAX_LENGTH = 2000
+
+    ##
+    # Init batch consumer
+    def initialize(server_url, app_id, max_buffer_length = DEFAULT_LENGTH)
+      @server_uri = URI.parse(server_url)
+      @server_uri.path = '/sync_server'
+      @app_id = app_id
+      @compress = true
+      @max_length = [max_buffer_length, MAX_LENGTH].min
+      @buffers = []
+      @mutex = Mutex.new
+      @owner_pid = Process.pid
+      TDLog.info("TDBatchConsumer init success. ServerUrl: #{server_url}, appId: #{app_id}")
+    end
+
+    ##
+    # http request compress
+    # @param compress [Boolean] compress or not
+    # @deprecated please use: set_compress
+    def _set_compress(compress)
+      @compress = compress
+    end
+
+    ##
+    # http request compress
+    # @param compress [Boolean] compress or not
+    def set_compress(compress)
+      @compress = compress
+    end
+
+    def add(message)
+      TDLog.info("Enqueue data to buffer. buffer size: #{@buffers.length}, data: #{message}")
+      need_flush = false
+      _reset_after_fork_if_needed
+      @mutex.synchronize do
+        @buffers << message
+        need_flush = @buffers.length >= @max_length
+      end
+      flush if need_flush
+    end
+
+    def close
+      _reset_after_fork_if_needed
+      flush
+      TDLog.info("TDBatchConsumer close.")
+    end
+
+    def flush
+      _reset_after_fork_if_needed
+      TDLog.info("TDBatchConsumer flush data.")
+      data_to_send = nil
+      @mutex.synchronize do
+        data_to_send = @buffers.dup
+        @buffers = []
+      end
+
+      return if data_to_send.empty?
+
+      chunks = data_to_send.each_slice(@max_length).to_a
+      chunks.each_with_index do |chunk, idx|
+        begin
+          if @compress
+            wio = StringIO.new("w")
+            gzip_io = Zlib::GzipWriter.new(wio)
+            gzip_io.write(chunk.to_json)
+            gzip_io.close
+            data = wio.string
+          else
+            data = chunk.to_json
+          end
+          compress_type = @compress ? 'gzip' : 'none'
+          headers = {'Content-Type' => 'application/plaintext',
+                     'appid' => @app_id,
+                     'compress' => compress_type,
+                     'TE-Integration-Type'=>'Ruby',
+                     'TE-Integration-Version'=>ThinkingData::VERSION,
+                     'TE-Integration-Count'=>chunk.count,
+                     'TA_Integration-Extra'=>'batch'}
+          request = CaseSensitivePost.new(@server_uri.request_uri, headers)
+          request.body = data
+
+          TDLog.info("Send data, request: #{data}")
+          begin
+            response_code, response_body = _request(@server_uri, request)
+            TDLog.info("Send data, response: #{response_body}")
+          rescue => e
+            raise ConnectionError.new("Could not connect to TE server, with error \"#{e.message}\".")
+          end
+
+          result = {}
+          if response_code.to_i == 200
+            begin
+              result = JSON.parse(response_body.to_s)
+            rescue JSON::JSONError
+              raise ServerError.new("Could not interpret TE server response: '#{response_body}'")
+            end
+          end
+
+          if result['code'] != 0
+            raise ServerError.new("Could not write to TE, server responded with #{response_code} returning: '#{response_body}'")
+          end
+        rescue => e
+          # Re-enqueue the failed chunk AND every not-yet-sent chunk to the back,
+          # so that raising here never drops data still pending in this flush.
+          # New data already in @buffers stays ahead, failed data retries next flush.
+          remaining = chunks[idx..-1].flatten(1)
+          @mutex.synchronize { @buffers = @buffers + remaining }
+          raise e
+        end
+      end
+    end
+
+    private
+    def _reset_after_fork_if_needed
+      return if @owner_pid == Process.pid
+      @owner_pid = Process.pid
+      @mutex = Mutex.new
+      @buffers = []
+    end
+
+    def _request(uri, request)
+      client = Net::HTTP.new(uri.host, uri.port)
+      client.use_ssl = uri.scheme === 'https' ? true : false
+      client.open_timeout = 10
+      client.continue_timeout = 10
+      client.read_timeout = 10
+      client.ssl_timeout = 10
+
+      response = client.request(request)
+      [response.code, response.body]
+    end
+  end
+
+  ##
+  # Private class. Send data tools
+  class CaseSensitivePost < Net::HTTP::Post
+    def initialize_http_header(headers)
+      @header = {}
+      headers.each{|k,v| @header[k.to_s] = [v] }
+    end
+
+    def [](name)
+      @header[name.to_s]
+    end
+
+    def []=(name, val)
+      if val
+        @header[name.to_s] = [val]
+      else
+        @header.delete(name.to_s)
+      end
+    end
+
+    def capitalize(name)
+      name
+    end
+  end
+
+end

data/lib/thinkingdata-ruby/td_debug_consumer.rb

@@ -1,73 +1,73 @@
-require 'json'
-require 'net/http'
-
-module ThinkingData
-  ##
-  # The data is reported one by one, and when an error occurs, the log will be printed on the console.
-  class TDDebugConsumer
-
-    ##
-    # Init debug consumer
-    #   @param server_url: server url
-    #   @param app_id: app id
-    #   @param write_data: is write data to TE
-    #   @param device_id: device id
-    def initialize(server_url, app_id, write_data = true, device_id: nil)
-      @server_uri = URI.parse(server_url)
-      @server_uri.path = '/data_debug'
-      @app_id = app_id
-      @write_data = write_data
-      @device_id = device_id
-      TDLog.info("TDDebugConsumer init success. ServerUrl: #{server_url}, appId: #{app_id}, deviceId: #{device_id}")
-    end
-
-    def add(message)
-      msg_json_str = message.to_json
-      TDLog.info("Send data, request: #{msg_json_str}")
-      headers =  {
-          'TE-Integration-Type'=>'Ruby',
-          'TE-Integration-Version'=>ThinkingData::VERSION,
-          'TE-Integration-Count'=>'1',
-          'TA_Integration-Extra'=>'debug'
-      }
-      form_data = {"data" => msg_json_str, "appid" => @app_id, "dryRun" => @write_data ? "0" : "1", "source" => "server"}
-      @device_id.is_a?(String) ? form_data["deviceId"] = @device_id : nil
-
-      begin
-        response_code, response_body = request(@server_uri, form_data,headers)
-        TDLog.info("Send data, response: #{response_body}")
-      rescue => e
-        raise ConnectionError.new("Could not connect to TE server, with error \"#{e.message}\".")
-      end
-
-      result = {}
-      if response_code.to_i == 200
-        begin
-          result = JSON.parse(response_body.to_s)
-        rescue JSON::JSONError
-          raise ServerError.new("Could not interpret TE server response: '#{response_body}'")
-        end
-      end
-
-      if result['errorLevel'] != 0
-        raise ServerError.new("Could not write to TE, server responded with #{response_code} returning: '#{response_body}'")
-      end
-    end
-
-    def request(uri, form_data,headers)
-      request = Net::HTTP::Post.new(uri.request_uri, headers)
-      request.set_form_data(form_data)
-
-      client = Net::HTTP.new(uri.host, uri.port)
-      client.use_ssl = uri.scheme === 'https' ? true : false
-      client.open_timeout = 10
-      client.continue_timeout = 10
-      client.read_timeout = 10
-      client.ssl_timeout = 10
-
-      response = client.request(request)
-      [response.code, response.body]
-    end
-  end
-
-end
+require 'json'
+require 'net/http'
+
+module ThinkingData
+  ##
+  # The data is reported one by one, and when an error occurs, the log will be printed on the console.
+  class TDDebugConsumer
+
+    ##
+    # Init debug consumer
+    #   @param server_url: server url
+    #   @param app_id: app id
+    #   @param write_data: is write data to TE
+    #   @param device_id: device id
+    def initialize(server_url, app_id, write_data = true, device_id: nil)
+      @server_uri = URI.parse(server_url)
+      @server_uri.path = '/data_debug'
+      @app_id = app_id
+      @write_data = write_data
+      @device_id = device_id
+      TDLog.info("TDDebugConsumer init success. ServerUrl: #{server_url}, appId: #{app_id}, deviceId: #{device_id}")
+    end
+
+    def add(message)
+      msg_json_str = message.to_json
+      TDLog.info("Send data, request: #{msg_json_str}")
+      headers =  {
+          'TE-Integration-Type'=>'Ruby',
+          'TE-Integration-Version'=>ThinkingData::VERSION,
+          'TE-Integration-Count'=>'1',
+          'TA_Integration-Extra'=>'debug'
+      }
+      form_data = {"data" => msg_json_str, "appid" => @app_id, "dryRun" => @write_data ? "0" : "1", "source" => "server"}
+      @device_id.is_a?(String) ? form_data["deviceId"] = @device_id : nil
+
+      begin
+        response_code, response_body = request(@server_uri, form_data,headers)
+        TDLog.info("Send data, response: #{response_body}")
+      rescue => e
+        raise ConnectionError.new("Could not connect to TE server, with error \"#{e.message}\".")
+      end
+
+      result = {}
+      if response_code.to_i == 200
+        begin
+          result = JSON.parse(response_body.to_s)
+        rescue JSON::JSONError
+          raise ServerError.new("Could not interpret TE server response: '#{response_body}'")
+        end
+      end
+
+      if result['errorLevel'] != 0
+        raise ServerError.new("Could not write to TE, server responded with #{response_code} returning: '#{response_body}'")
+      end
+    end
+
+    def request(uri, form_data,headers)
+      request = Net::HTTP::Post.new(uri.request_uri, headers)
+      request.set_form_data(form_data)
+
+      client = Net::HTTP.new(uri.host, uri.port)
+      client.use_ssl = uri.scheme === 'https' ? true : false
+      client.open_timeout = 10
+      client.continue_timeout = 10
+      client.read_timeout = 10
+      client.ssl_timeout = 10
+
+      response = client.request(request)
+      [response.code, response.body]
+    end
+  end
+
+end

data/lib/thinkingdata-ruby/td_errors.rb

@@ -1,38 +1,38 @@
-module ThinkingData
-  ##
-  # SDK error
-  TDAnalyticsError = Class.new(StandardError)
-
-  ##
-  # SDK error: illegal parameter
-  IllegalParameterError = Class.new(TDAnalyticsError)
-
-  ##
-  # SDK error: connection error
-  ConnectionError = Class.new(TDAnalyticsError)
-
-  ##
-  # SDK error: server error
-  ServerError = Class.new(TDAnalyticsError)
-
-  ##
-  # Error handler
-  #
-  # e.g.
-  #    class MyErrorHandler < ThinkingData::ErrorHandler
-  #      def handle(error)
-  #          puts error
-  #          raise error
-  #      end
-  #    end
-  #
-  #    my_error_handler = MyErrorHandler.new
-  #    tracker = ThinkingData::TDAnalytics.new(consumer, my_error_handler)
-  class TDErrorHandler
-    ##
-    # Override #handle to customize error handling
-    def handle(error)
-      false
-    end
-  end
+module ThinkingData
+  ##
+  # SDK error
+  TDAnalyticsError = Class.new(StandardError)
+
+  ##
+  # SDK error: illegal parameter
+  IllegalParameterError = Class.new(TDAnalyticsError)
+
+  ##
+  # SDK error: connection error
+  ConnectionError = Class.new(TDAnalyticsError)
+
+  ##
+  # SDK error: server error
+  ServerError = Class.new(TDAnalyticsError)
+
+  ##
+  # Error handler
+  #
+  # e.g.
+  #    class MyErrorHandler < ThinkingData::ErrorHandler
+  #      def handle(error)
+  #          puts error
+  #          raise error
+  #      end
+  #    end
+  #
+  #    my_error_handler = MyErrorHandler.new
+  #    tracker = ThinkingData::TDAnalytics.new(consumer, my_error_handler)
+  class TDErrorHandler
+    ##
+    # Override #handle to customize error handling
+    def handle(error)
+      false
+    end
+  end
 end