td 0.10.21 → 0.10.22

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. data/ChangeLog +5 -0
  2. data/data/sample_apache.json +5000 -0
  3. data/data/sample_apache_gen.rb +193 -0
  4. data/lib/td/version.rb +1 -1
  5. metadata +13 -11
data/data/sample_apache_gen.rb ADDED
@@ -0,0 +1,193 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require 'json'
3
+ require 'msgpack'
4
+ require 'digest/md5'
5
+ require 'cgi'
6
+ require 'uuidtools'
7
+ require 'fileutils'
8
+ require 'zlib'
9
+ require 'parallel'
10
+
11
+ RECORDS = 5000
12
+ HOSTS = RECORDS/4
13
+ PAGES = RECORDS/4
14
+
15
+ AGENT_LIST_STRING = <<END
16
+ Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3
17
+ Mozilla/5.0 (iPad; CPU OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3
18
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)
19
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)
20
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)
21
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)
22
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)
23
+ Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
24
+ Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
25
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
26
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
27
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7
28
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7
29
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
30
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
31
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
32
+ Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
33
+ Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
34
+ Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
35
+ Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1
36
+ Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
37
+ Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11
38
+ Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.46 Safari/535.11
39
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1
40
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1
41
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1
42
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; YTB730; GTB7.2; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E; Media Center PC 6.0)
43
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; YTB730; GTB7.2; EasyBits GO v1.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C)
44
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; GTB7.2; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C)
45
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; YTB730; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C)
46
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; Trident/4.0; GTB6; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618; .NET4.0C)
47
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; YTB720; GTB7.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)
48
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; BTRS122159; GTB7.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; BRI/2)
49
+ END
50
+ AGENT_LIST = AGENT_LIST_STRING.split("\n")
51
+
52
+ PAGE_CATEGORIES = %w[
53
+ books
54
+ books
55
+ books
56
+ electronics
57
+ electronics
58
+ electronics
59
+ electronics
60
+ electronics
61
+ electronics
62
+ software
63
+ software
64
+ software
65
+ software
66
+ games
67
+ games
68
+ games
69
+ office
70
+ office
71
+ cameras
72
+ computers
73
+ finance
74
+ giftcards
75
+ garden
76
+ health
77
+ music
78
+ sports
79
+ toys
80
+ networking
81
+ jewelry
82
+ ]
83
+
84
+ RANDOM = Random.new
85
+ def grand(n)
86
+ RANDOM.rand(n)
87
+ end
88
+
89
+ class Host
90
+ def initialize
91
+ @ip = "#{(grand(210)+20)/4*4}.#{(grand(210)+20)/3*3}.#{grand(210)+20}.#{grand(210)+20}"
92
+ @agents = []
93
+ end
94
+
95
+ attr_reader :ip
96
+
97
+ def agent
98
+ if @agents.size == 4
99
+ @agents[grand(4)]
100
+ else
101
+ agent = AGENT_LIST[grand(AGENT_LIST.size)]
102
+ @agents << agent
103
+ agent
104
+ end
105
+ end
106
+ end
107
+
108
+ class Page
109
+ def initialize
110
+ cate = PAGE_CATEGORIES[grand(PAGE_CATEGORIES.size)]
111
+ item = grand(RECORDS)
112
+
113
+ if grand(2) == 0
114
+ w = [cate, PAGE_CATEGORIES[grand(PAGE_CATEGORIES.size)]]
115
+ else
116
+ w = [cate]
117
+ end
118
+ q = w.map {|k| k[0].upcase + k[1..-1] }.join('+')
119
+ search_path = "/search/?c=#{q}"
120
+ google_ref = "http://www.google.com/search?ie=UTF-8&q=google&sclient=psy-ab&q=#{q}&oq=#{q}&aq=f&aqi=g-vL1&aql=&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.,cf.osb&biw=#{grand(5000)}&bih=#{grand(600)}"
121
+
122
+ case grand(12)
123
+ when 0,1,2,3,4,5
124
+ @path = "/category/#{cate}"
125
+ @referers = [nil, nil, nil, nil, nil, nil, nil, google_ref]
126
+ @method = 'GET'
127
+ @code = 200
128
+
129
+ when 6
130
+ @path = "/category/#{cate}?from=#{grand(3)*10}"
131
+ @referers = [search_path, "/category/#{cate}"]
132
+ @method = 'GET'
133
+ @code = 200
134
+
135
+ when 7,8,9,10
136
+ @path = "/item/#{cate}/#{item}"
137
+ @referers = [search_path, search_path, google_ref, "/category/#{cate}"]
138
+ @method = 'GET'
139
+ if grand(100) == 0
140
+ @code = 404
141
+ else
142
+ @code = 200
143
+ end
144
+
145
+ when 11
146
+ @path = search_path
147
+ @referers = [nil]
148
+ @method = 'POST'
149
+ @code = 200
150
+ end
151
+
152
+ @size = grand(100) + 40
153
+ end
154
+
155
+ attr_reader :path, :size, :method, :code
156
+
157
+ def referer
158
+ if grand(2) == 0
159
+ @referers[grand(@referers.size)]
160
+ end
161
+ end
162
+ end
163
+
164
+ @pages = []
165
+ PAGES.times do
166
+ @pages << Page.new
167
+ end
168
+
169
+ @hosts = []
170
+ HOSTS.times do
171
+ @hosts << Host.new
172
+ end
173
+
174
+ now = Time.now.to_i
175
+
176
+ RECORDS.times do
177
+ now += grand(5)
178
+ page = @pages[grand(@pages.size)]
179
+ host = @hosts[grand(@hosts.size)]
180
+ record = {
181
+ 'host' => host.ip,
182
+ 'user' => '-',
183
+ 'method' => page.method,
184
+ 'path' => page.path,
185
+ 'code' => grand(10000) == 0 ? 500 : page.code,
186
+ 'referer' => (grand(2) == 0 ? @pages[grand(@pages.size)].path : page.referer) || '-',
187
+ 'size' => page.size,
188
+ 'agent' => host.agent,
189
+ }
190
+ puts record.to_json
191
+ #puts %[#{record['host']} - #{record['user']} [#{Time.at(now).strftime('%d/%b/%Y:%H:%M:%S %z')}] "#{record['method']} #{record['path']} HTTP/1.1" #{record['code']} #{record['size']} "#{record['referer']}" "#{record['agent']}"]
192
+ end
193
+
data/lib/td/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  module TreasureData
2
2
 
3
- VERSION = '0.10.21'
3
+ VERSION = '0.10.22'
4
4
 
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: td
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.21
4
+ version: 0.10.22
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -13,7 +13,7 @@ date: 2012-04-04 00:00:00.000000000Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: msgpack
16
- requirement: &70131070404920 !ruby/object:Gem::Requirement
16
+ requirement: &70323068907380 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ~>
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: 0.4.4
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70131070404920
24
+ version_requirements: *70323068907380
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: json
27
- requirement: &70131070404060 !ruby/object:Gem::Requirement
27
+ requirement: &70323068906820 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
32
32
  version: 1.4.3
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *70131070404060
35
+ version_requirements: *70323068906820
36
36
  - !ruby/object:Gem::Dependency
37
37
  name: hirb
38
- requirement: &70131070401640 !ruby/object:Gem::Requirement
38
+ requirement: &70323068906340 !ruby/object:Gem::Requirement
39
39
  none: false
40
40
  requirements:
41
41
  - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
43
43
  version: 0.4.5
44
44
  type: :runtime
45
45
  prerelease: false
46
- version_requirements: *70131070401640
46
+ version_requirements: *70323068906340
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: td-client
49
- requirement: &70131070399480 !ruby/object:Gem::Requirement
49
+ requirement: &70323068905860 !ruby/object:Gem::Requirement
50
50
  none: false
51
51
  requirements:
52
52
  - - ~>
@@ -54,10 +54,10 @@ dependencies:
54
54
  version: 0.8.14
55
55
  type: :runtime
56
56
  prerelease: false
57
- version_requirements: *70131070399480
57
+ version_requirements: *70323068905860
58
58
  - !ruby/object:Gem::Dependency
59
59
  name: td-logger
60
- requirement: &70131070397240 !ruby/object:Gem::Requirement
60
+ requirement: &70323068905360 !ruby/object:Gem::Requirement
61
61
  none: false
62
62
  requirements:
63
63
  - - ~>
@@ -65,7 +65,7 @@ dependencies:
65
65
  version: 0.3.8
66
66
  type: :runtime
67
67
  prerelease: false
68
- version_requirements: *70131070397240
68
+ version_requirements: *70323068905360
69
69
  description:
70
70
  email:
71
71
  executables:
@@ -75,6 +75,8 @@ extra_rdoc_files:
75
75
  - ChangeLog
76
76
  - README.rdoc
77
77
  files:
78
+ - data/sample_apache.json
79
+ - data/sample_apache_gen.rb
78
80
  - lib/td.rb
79
81
  - lib/td/command/account.rb
80
82
  - lib/td/command/aggr.rb