rubberband_flamethrower 0.4.3 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/bin/flamethrower +4 -0
- data/lib/rubberband_flamethrower/data_generator.rb +15 -1
- data/lib/rubberband_flamethrower/flamethrower.rb +18 -1
- data/lib/rubberband_flamethrower.rb +20 -0
- data/rubberband_flamethrower.gemspec +2 -2
- metadata +13 -13
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.5.0
|
data/bin/flamethrower
CHANGED
@@ -12,6 +12,10 @@ command = args.shift.strip rescue 'auto'
|
|
12
12
|
case command
|
13
13
|
when "fire"
|
14
14
|
RubberbandFlamethrower.fire(*args)
|
15
|
+
when "generate_dataset"
|
16
|
+
RubberbandFlamethrower.generate_dataset(*args)
|
17
|
+
when "fire_dataset"
|
18
|
+
RubberbandFlamethrower.fire_dataset(*args)
|
15
19
|
when "help"
|
16
20
|
RubberbandFlamethrower.help
|
17
21
|
else
|
@@ -2,6 +2,10 @@
|
|
2
2
|
# It creates a JSON object that approaxiamates data like you would fine in a tweet
|
3
3
|
# with fields for a message (max 140 characters), username, and post_date.
|
4
4
|
# The post_date format is parsable as a date object by default by Elastic Search
|
5
|
+
|
6
|
+
require "active_support/core_ext"
|
7
|
+
|
8
|
+
|
5
9
|
module RubberbandFlamethrower
|
6
10
|
class DataGenerator
|
7
11
|
attr_accessor :word_list
|
@@ -58,6 +62,16 @@ module RubberbandFlamethrower
|
|
58
62
|
def generate_random_insert_data
|
59
63
|
{message: "#{random_tweet}", username: "#{random_username}", post_date: "#{current_timestamp}"}.to_json
|
60
64
|
end
|
61
|
-
|
65
|
+
|
66
|
+
def generate_dataset(batch_size)
|
67
|
+
File.open("dataset", 'w') do |file|
|
68
|
+
batch_size.to_i.times do |i|
|
69
|
+
file.write(generate_random_insert_data+"\n")
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
|
75
|
+
|
62
76
|
end
|
63
77
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# This class is designed to insert objects created with the DataGenerator class into Elastic Search
|
2
2
|
|
3
|
-
require "active_support/core_ext"
|
4
3
|
require 'httparty'
|
5
4
|
require "benchmark"
|
6
5
|
require File.dirname(__FILE__)+"/data_generator.rb"
|
@@ -48,5 +47,23 @@ module RubberbandFlamethrower
|
|
48
47
|
id = id + 1
|
49
48
|
end
|
50
49
|
end
|
50
|
+
|
51
|
+
def fire_dataset(filename, starting_id, server_url, index, type, printing_level=0)
|
52
|
+
# this will constantly flush the printing to the display so the progress dots appear for each insert
|
53
|
+
($stdout.sync = true) if printing_level == 1
|
54
|
+
# a unique ID must be provided for each document stored in Elastic Search
|
55
|
+
id = starting_id.to_i
|
56
|
+
# loop through lines in the file and insert each
|
57
|
+
IO.foreach(filename) do |line|
|
58
|
+
(puts insert_data) if printing_level == 2
|
59
|
+
response = HTTParty.put("#{server_url}/#{index}/#{type}/#{id}", body: line)
|
60
|
+
(puts response.body) if printing_level == 2
|
61
|
+
(print ".") if printing_level == 1
|
62
|
+
id = id + 1
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
|
67
|
+
|
51
68
|
end
|
52
69
|
end
|
@@ -42,6 +42,26 @@ module RubberbandFlamethrower
|
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
45
|
+
def self.generate_dataset(batch_size=500)
|
46
|
+
require File.dirname(__FILE__)+"/rubberband_flamethrower/data_generator.rb"
|
47
|
+
data = DataGenerator.new
|
48
|
+
data.generate_dataset(batch_size)
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.fire_dataset(filename="dataset", starting_id=1, server_url="http://localhost:9200", index="twitter", type="tweet")
|
52
|
+
require File.dirname(__FILE__)+"/rubberband_flamethrower/flamethrower.rb"
|
53
|
+
flamethrower = Flamethrower.new
|
54
|
+
time = Benchmark.measure do
|
55
|
+
flamethrower.fire_dataset(filename, starting_id, server_url, index, type, 1)
|
56
|
+
end
|
57
|
+
puts "\nFinished Inserting Dataset (#{filename}) into Elastic Search."
|
58
|
+
puts " user system total real"
|
59
|
+
puts time
|
60
|
+
|
61
|
+
end
|
62
|
+
|
63
|
+
|
64
|
+
|
45
65
|
# Displays help menu of the available help menu commands
|
46
66
|
# this is adapted from the file README.md
|
47
67
|
# things updated here should be updated there
|
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "rubberband_flamethrower"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.5.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Michael Orr"]
|
12
|
-
s.date = "2013-04-
|
12
|
+
s.date = "2013-04-17"
|
13
13
|
s.description = "Use to quickly fill up some indicies in Elastic Search and to retrieve statistics about insertion rates"
|
14
14
|
s.email = "michael@cloudspace.com"
|
15
15
|
s.executables = ["flamethrower"]
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rubberband_flamethrower
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-04-
|
12
|
+
date: 2013-04-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: httparty
|
16
|
-
requirement: &
|
16
|
+
requirement: &9560560 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 0.10.2
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *9560560
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: activesupport
|
27
|
-
requirement: &
|
27
|
+
requirement: &9559180 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ~>
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 3.2.13
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *9559180
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: rdoc
|
38
|
-
requirement: &
|
38
|
+
requirement: &9557580 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ~>
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '3.12'
|
44
44
|
type: :development
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *9557580
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: bundler
|
49
|
-
requirement: &
|
49
|
+
requirement: &9573900 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ~>
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: 1.3.0
|
55
55
|
type: :development
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *9573900
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: jeweler
|
60
|
-
requirement: &
|
60
|
+
requirement: &9573020 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ~>
|
@@ -65,7 +65,7 @@ dependencies:
|
|
65
65
|
version: 1.8.4
|
66
66
|
type: :development
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *9573020
|
69
69
|
description: Use to quickly fill up some indicies in Elastic Search and to retrieve
|
70
70
|
statistics about insertion rates
|
71
71
|
email: michael@cloudspace.com
|
@@ -115,7 +115,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
115
115
|
version: '0'
|
116
116
|
segments:
|
117
117
|
- 0
|
118
|
-
hash:
|
118
|
+
hash: -31644688981302429
|
119
119
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
120
|
none: false
|
121
121
|
requirements:
|