trd-rails-collector 0.1.0 → 0.1.1

data/lib/trd-rails-collector.rb

@@ -38,7 +38,7 @@ module TreasureData
       require 'yaml'
       sample = <<EOF
   defaults: &defaults
-    apikey: "78fbbc1ff83088ab72e2b68c980c4c75786bee6e"
+    apikey: "YOUR_API_KEY"
     database: myapp
     table: access
 
@@ -76,9 +76,10 @@ EOF
       config.middleware.use TreasureData::Rack
     end
 
-    SIZE_LIMIT = 1024*1024  # 1MB
-    TIME_LIMIT = 5         # 1min TODO
-    SLEEP_TIME = 10         # 10sec TODO
+    SIZE_LIMIT = 1024*1024 # bytes
+    TIME_LIMIT = 60        # sec
+    SLEEP_TIME = 10        # sec
+    RETRY_LIMIT = 8
 
     API_HOST = 'api.treasure-data.com'
     API_PORT = 80
@@ -138,6 +139,7 @@ EOF
 
     def upload_main
       tuple = nil
+      error_count = 0
 
       until @finish
         @map.synchronize do
@@ -160,9 +162,20 @@ EOF
 
         begin
           code, body = upload(*tuple)
+          if code[0] != ?2
+            raise "import failed: #{body}"
+          end
         rescue
-          # TODO retry
-          $stderr.puts $!
+          if error_count < RETRY_LIMIT
+            # TODO wait before retry
+            sleep (2 ** error_count)
+            tuple = nil
+            error_count += 1
+            RAILS_DEFAULT_LOGGER.error "Failed to upload log to Treasure Data, retrying: #{$!}"
+          else
+            RAILS_DEFAULT_LOGGER.error "Failed to upload log to Treasure Data, trashed: #{$!}"
+            $stderr.puts "Failed to upload log to Treasure Data, trashed: #{$!}"
+          end
         end
       end
 
@@ -189,9 +202,9 @@ EOF
       header['Date'] = Time.now.rfc2822
       header['Content-Length'] = data.size.to_s
 
-      url = "/v2/import/#{e @db}/#{e @table}/msgpack.gz"
+      url = "/v3/table/import/#{e @db}/#{e @table}/msgpack.gz"
 
-      RAILS_DEFAULT_LOGGER.debug "Uploading compressed logs #{data.size} bytes to #{url}"
+      RAILS_DEFAULT_LOGGER.info "Uploading compressed logs (#{data.size} bytes) to Treasure Data"
       request = Net::HTTP::Put.new(url, header)
       request.body = data
 
@@ -225,6 +238,9 @@ EOF
       # Rails 3
       'action_dispatch.request.path_parameters',
 
+      # Rails 2
+      'action_controller.request.path_parameters',
+
       # Rack default
       'rack.routing_args',
     ]
@@ -262,6 +278,7 @@ EOF
       # time parameter is required
       m['time'] = Time.now.to_i
 
+      #puts m.to_json
       TreasureData.emit(m)
 
       #File.open('out.log', "a") {|f|
@@ -269,6 +286,7 @@ EOF
       #}
 
       #File.open('envlog.txt', "a") {|f|
+      #  require 'pp'
       #  f.write env.pretty_inspect
       #}
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: trd-rails-collector
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 25
   prerelease: false
   segments: 
   - 0
   - 1
-  - 0
-  version: 0.1.0
+  - 1
+  version: 0.1.1
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-25 00:00:00 +09:00
+date: 2011-06-27 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency