td 0.10.21 → 0.10.22

data/data/sample_apache_gen.rb

@@ -0,0 +1,193 @@
+# -*- encoding: utf-8 -*-
+require 'json'
+require 'msgpack'
+require 'digest/md5'
+require 'cgi'
+require 'uuidtools'
+require 'fileutils'
+require 'zlib'
+require 'parallel'
+
+RECORDS = 5000
+HOSTS = RECORDS/4
+PAGES = RECORDS/4
+
+AGENT_LIST_STRING = <<END
+Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3
+Mozilla/5.0 (iPad; CPU OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3
+Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)
+Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)
+Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)
+Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)
+Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)
+Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
+Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
+Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
+Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
+Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.46 Safari/535.11
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1
+Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; YTB730; GTB7.2; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E; Media Center PC 6.0)
+Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; YTB730; GTB7.2; EasyBits GO v1.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C)
+Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; GTB7.2; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C)
+Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; YTB730; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C)
+Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; Trident/4.0; GTB6; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618; .NET4.0C)
+Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; YTB720; GTB7.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)
+Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; BTRS122159; GTB7.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; BRI/2)
+END
+AGENT_LIST = AGENT_LIST_STRING.split("\n")
+
+PAGE_CATEGORIES = %w[
+books
+books
+books
+electronics
+electronics
+electronics
+electronics
+electronics
+electronics
+software
+software
+software
+software
+games
+games
+games
+office
+office
+cameras
+computers
+finance
+giftcards
+garden
+health
+music
+sports
+toys
+networking
+jewelry
+]
+
+RANDOM = Random.new
+def grand(n)
+  RANDOM.rand(n)
+end
+
+class Host
+  def initialize
+    @ip = "#{(grand(210)+20)/4*4}.#{(grand(210)+20)/3*3}.#{grand(210)+20}.#{grand(210)+20}"
+    @agents = []
+  end
+
+  attr_reader :ip
+
+  def agent
+    if @agents.size == 4
+      @agents[grand(4)]
+    else
+      agent = AGENT_LIST[grand(AGENT_LIST.size)]
+      @agents << agent
+      agent
+    end
+  end
+end
+
+class Page
+  def initialize
+    cate = PAGE_CATEGORIES[grand(PAGE_CATEGORIES.size)]
+    item = grand(RECORDS)
+
+    if grand(2) == 0
+      w = [cate, PAGE_CATEGORIES[grand(PAGE_CATEGORIES.size)]]
+    else
+      w = [cate]
+    end
+    q = w.map {|k| k[0].upcase + k[1..-1] }.join('+')
+    search_path = "/search/?c=#{q}"
+    google_ref = "http://www.google.com/search?ie=UTF-8&q=google&sclient=psy-ab&q=#{q}&oq=#{q}&aq=f&aqi=g-vL1&aql=&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.,cf.osb&biw=#{grand(5000)}&bih=#{grand(600)}"
+
+    case grand(12)
+    when 0,1,2,3,4,5
+      @path = "/category/#{cate}"
+      @referers = [nil, nil, nil, nil, nil, nil, nil, google_ref]
+      @method = 'GET'
+      @code = 200
+
+    when 6
+      @path = "/category/#{cate}?from=#{grand(3)*10}"
+      @referers = [search_path, "/category/#{cate}"]
+      @method = 'GET'
+      @code = 200
+
+    when 7,8,9,10
+      @path = "/item/#{cate}/#{item}"
+      @referers = [search_path, search_path, google_ref, "/category/#{cate}"]
+      @method = 'GET'
+      if grand(100) == 0
+        @code = 404
+      else
+        @code = 200
+      end
+
+    when 11
+      @path = search_path
+      @referers = [nil]
+      @method = 'POST'
+      @code = 200
+    end
+
+    @size = grand(100) + 40
+  end
+
+  attr_reader :path, :size, :method, :code
+
+  def referer
+    if grand(2) == 0
+      @referers[grand(@referers.size)]
+    end
+  end
+end
+
+@pages = []
+PAGES.times do
+  @pages << Page.new
+end
+
+@hosts = []
+HOSTS.times do
+  @hosts << Host.new
+end
+
+now = Time.now.to_i
+
+RECORDS.times do
+  now += grand(5)
+  page = @pages[grand(@pages.size)]
+  host = @hosts[grand(@hosts.size)]
+  record = {
+    'host' => host.ip,
+    'user' => '-',
+    'method' => page.method,
+    'path' => page.path,
+    'code' => grand(10000) == 0 ? 500 : page.code,
+    'referer' => (grand(2) == 0 ? @pages[grand(@pages.size)].path : page.referer) || '-',
+    'size' => page.size,
+    'agent' => host.agent,
+  }
+  puts record.to_json
+  #puts %[#{record['host']} - #{record['user']} [#{Time.at(now).strftime('%d/%b/%Y:%H:%M:%S %z')}] "#{record['method']} #{record['path']} HTTP/1.1" #{record['code']} #{record['size']} "#{record['referer']}" "#{record['agent']}"]
+end
+

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.21'
+VERSION = '0.10.22'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.21
+  version: 0.10.22
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-04-04 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70131070404920 !ruby/object:Gem::Requirement
+  requirement: &70323068907380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70131070404920
+  version_requirements: *70323068907380
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70131070404060 !ruby/object:Gem::Requirement
+  requirement: &70323068906820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70131070404060
+  version_requirements: *70323068906820
 - !ruby/object:Gem::Dependency
   name: hirb
-  requirement: &70131070401640 !ruby/object:Gem::Requirement
+  requirement: &70323068906340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 0.4.5
   type: :runtime
   prerelease: false
-  version_requirements: *70131070401640
+  version_requirements: *70323068906340
 - !ruby/object:Gem::Dependency
   name: td-client
-  requirement: &70131070399480 !ruby/object:Gem::Requirement
+  requirement: &70323068905860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.8.14
   type: :runtime
   prerelease: false
-  version_requirements: *70131070399480
+  version_requirements: *70323068905860
 - !ruby/object:Gem::Dependency
   name: td-logger
-  requirement: &70131070397240 !ruby/object:Gem::Requirement
+  requirement: &70323068905360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 0.3.8
   type: :runtime
   prerelease: false
-  version_requirements: *70131070397240
+  version_requirements: *70323068905360
 description: 
 email: 
 executables:
@@ -75,6 +75,8 @@ extra_rdoc_files:
 - ChangeLog
 - README.rdoc
 files:
+- data/sample_apache.json
+- data/sample_apache_gen.rb
 - lib/td.rb
 - lib/td/command/account.rb
 - lib/td/command/aggr.rb