mines 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +7 -0
- data/.yardopts +6 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +37 -0
- data/README.md +25 -0
- data/Rakefile +8 -0
- data/bin/mines +8 -8
- data/config/application.rb +11 -0
- data/config/help.rb +13 -0
- data/examples/README.md +4 -0
- data/examples/hashtags/Gemfile +24 -0
- data/examples/hashtags/Gemfile.lock +81 -0
- data/examples/hashtags/README.md +22 -0
- data/examples/hashtags/config/application.rb +30 -0
- data/examples/hashtags/miners/metrics.rb +157 -0
- data/examples/hashtags/miners/process.rb +46 -0
- data/examples/hashtags/miners/publicstream.rb +108 -0
- data/examples/hashtags/spec/metrics_spec.rb +23 -0
- data/examples/hashtags/spec/process_spec.rb +23 -0
- data/examples/hashtags/spec/publicstream_spec.rb +23 -0
- data/lib/generators/README.md +13 -0
- data/lib/generators/application.rb +3 -2
- data/lib/generators/miner.rb +10 -2
- data/lib/generators/templates/Gemfile.erb +24 -0
- data/lib/generators/templates/README.md +24 -0
- data/lib/generators/templates/application_config.erb +11 -0
- data/lib/generators/templates/metrics_miner.erb +28 -0
- data/lib/generators/templates/miner_spec.erb +23 -0
- data/lib/generators/templates/network_miner.erb +32 -0
- data/lib/generators/templates/process_miner.erb +26 -1
- data/lib/generators/templates/twitter_miner.erb +111 -0
- data/lib/logging.rb +10 -3
- data/mines.gemspec +33 -0
- data/spec/logging_spec.rb +19 -0
- data/spec/mines_spec.rb +19 -0
- data/spec/redis_store_spec.rb +64 -0
- data/spec/utilities_spec.rb +26 -0
- metadata +45 -11
@@ -1,7 +1,32 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# coding: utf-8
|
3
3
|
|
4
|
-
|
4
|
+
## Gem dependencies are managed by Bundler inside project's Gemfile
|
5
|
+
require 'bundler'
|
6
|
+
Bundler.require :process
|
5
7
|
|
6
8
|
log_name :<%= name %>
|
7
9
|
|
10
|
+
@log.info "Process miner pid: #{Process.pid}"
|
11
|
+
|
12
|
+
## Connect to an Object Queue
|
13
|
+
## elementes are pushed to this queue by the network miner
|
14
|
+
queue = ObjectQueue.new "<%= name %>"
|
15
|
+
|
16
|
+
## Create a timeseries metrics store to
|
17
|
+
## save the occurence rate for something
|
18
|
+
ts = TimeSeries.new("<%= name %>")
|
19
|
+
|
20
|
+
## Start the process loop
|
21
|
+
loop {
|
22
|
+
## Get en element from the queue
|
23
|
+
element = queue.pop
|
24
|
+
|
25
|
+
##
|
26
|
+
##... Process element ...
|
27
|
+
##
|
28
|
+
|
29
|
+
## Push some metric to the time-series
|
30
|
+
ts.push ("...")
|
31
|
+
|
32
|
+
} # end loop
|
@@ -0,0 +1,111 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# coding: utf-8
|
3
|
+
|
4
|
+
## Gem dependencies are managed by Bundler inside project's Gemfile
|
5
|
+
require 'bundler'
|
6
|
+
Bundler.require :network
|
7
|
+
|
8
|
+
log_name :<%= name %>
|
9
|
+
|
10
|
+
## Create an Object Queue
|
11
|
+
queue = ObjectQueue.new "<%= name %>"
|
12
|
+
|
13
|
+
## Provide your Twitter Streaming API keys and credentials
|
14
|
+
##
|
15
|
+
## example:
|
16
|
+
# TweetStream.configure do |config|
|
17
|
+
# config.consumer_key = "mZ0gsfwgA"
|
18
|
+
# config.consumer_secret = "nqwQqELrvdb6C"
|
19
|
+
# config.oauth_token = "OfIUtfTmP6NQuwWKlH"
|
20
|
+
# config.oauth_token_secret = 'u4bFMpQxfgwZl'
|
21
|
+
# config.auth_method = :oauth
|
22
|
+
# end
|
23
|
+
|
24
|
+
## It is recommended to provide your credentials
|
25
|
+
## in a different file outside version control system
|
26
|
+
## to avoid publishing them.
|
27
|
+
## example:
|
28
|
+
require_relative '../../twitter_credentials'
|
29
|
+
|
30
|
+
## Initialize TweetStream Client
|
31
|
+
client = TweetStream::Client.new
|
32
|
+
|
33
|
+
## Print custom messages when client receives special event messages
|
34
|
+
## See also: https://dev.twitter.com/docs/streaming-apis/messages
|
35
|
+
client.on_error do |message|
|
36
|
+
msg = "\nError message: #{message}\n"+
|
37
|
+
"An HTTP error is encountered in the processing of the stream. "+
|
38
|
+
"Note that TweetStream will automatically try to reconnect, "+
|
39
|
+
"this is for reference only. Don't panic!"
|
40
|
+
puts red msg
|
41
|
+
@lod.error msg
|
42
|
+
end
|
43
|
+
client.on_unauthorized do
|
44
|
+
msg = "\nUnauthorized.\n"+
|
45
|
+
"An HTTP status 401 is encountered while connecting to Twitter. "+
|
46
|
+
"This could happen when system clock drift has occured."
|
47
|
+
puts red msg
|
48
|
+
@log.error msg
|
49
|
+
end
|
50
|
+
client.on_inited do
|
51
|
+
msg = "Connection Established"
|
52
|
+
puts green msg
|
53
|
+
@log.info msg
|
54
|
+
end
|
55
|
+
client.on_no_data_received do
|
56
|
+
msg = "\nNo data was received from the server and a stall occurred. "+
|
57
|
+
"Twitter defines this to be 90 seconds."
|
58
|
+
puts red msg
|
59
|
+
@log.info msg
|
60
|
+
end
|
61
|
+
client.on_delete do |status_id, user_id|
|
62
|
+
msg = "\nStatus deletion notice: user: #{user_id} status: #{status_id}"
|
63
|
+
puts yellow msg
|
64
|
+
@log.info msg
|
65
|
+
end
|
66
|
+
client.on_reconnect do |timeout, retries|
|
67
|
+
msg = "Reconnect Timeout: #{timeout} retries: #{retries}"
|
68
|
+
puts red msg
|
69
|
+
@log.info msg
|
70
|
+
end
|
71
|
+
client.on_limit do |skip_count|
|
72
|
+
msg = "\nA rate limit notice is received from the Twitter stream.\n"+
|
73
|
+
"Discarded Count: #{skip_count}"
|
74
|
+
puts yellow msg
|
75
|
+
@log.info msg
|
76
|
+
end
|
77
|
+
client.on_enhance_your_calm do
|
78
|
+
msg = "\nEnhance your calm"
|
79
|
+
puts yellow msg
|
80
|
+
@log.info msg
|
81
|
+
end
|
82
|
+
|
83
|
+
## Make a call to the statuses/filter method of the Streaming API,
|
84
|
+
## you may provide :follow, :track or :locations.
|
85
|
+
##
|
86
|
+
## :track Phrases of keywords to track
|
87
|
+
## :follow A list of user IDs, indicating the
|
88
|
+
## users to return statuses for in the stream.
|
89
|
+
## :locations Specifies a set of bounding boxes to track
|
90
|
+
## e.g. :locations => [-122.75,36.8,-121.75,37.8,-74,40,-73,41]
|
91
|
+
##
|
92
|
+
## example: tracking common greek words
|
93
|
+
client.filter(:track => %w( μια το να και το του της τα τι
|
94
|
+
για δεν στο στον στην με από απο
|
95
|
+
θα τις οι μου) ) do |status|
|
96
|
+
|
97
|
+
## status is an object of class Tweet
|
98
|
+
|
99
|
+
## Print the user's screen name
|
100
|
+
# puts blue status.user.screen_name
|
101
|
+
|
102
|
+
## Print the tweet's text
|
103
|
+
# puts green status.text
|
104
|
+
|
105
|
+
## Print a blue dot to indicate a tweet has been received
|
106
|
+
print blue "."
|
107
|
+
|
108
|
+
## Push the status to the object queue
|
109
|
+
## the object queue will marshal the object so
|
110
|
+
queue.push(status)
|
111
|
+
end
|
data/lib/logging.rb
CHANGED
@@ -14,10 +14,17 @@ require_relative 'configuration'
|
|
14
14
|
# require_relative '../config/application'
|
15
15
|
#end
|
16
16
|
|
17
|
+
|
18
|
+
# Check if log directory exists
|
19
|
+
log_dir = "log"
|
20
|
+
unless File.exists?(log_dir) && File.directory?(log_dir)
|
21
|
+
log_dir="."
|
22
|
+
end
|
23
|
+
|
17
24
|
unless defined?(Application.logname).nil?
|
18
|
-
@log ||= Logger.new "
|
25
|
+
@log ||= Logger.new "#{log_dir}/#{Application.logname}"
|
19
26
|
else
|
20
|
-
@log ||= Logger.new '
|
27
|
+
@log ||= Logger.new '#{log_dir}/mines.log'
|
21
28
|
end
|
22
29
|
|
23
30
|
@log.datetime_format= "%H:%M:%S"
|
@@ -26,7 +33,7 @@ end
|
|
26
33
|
end
|
27
34
|
|
28
35
|
def log_name name
|
29
|
-
@log ||= Logger.new '
|
36
|
+
@log ||= Logger.new '#{log_dir}/#{name}.log'
|
30
37
|
@log.datetime_format= "%H:%M:%S"
|
31
38
|
@log.formatter = proc do |severity, datetime, progname, msg|
|
32
39
|
"#{severity} #{datetime} #{msg}\n"
|
data/mines.gemspec
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.platform = Gem::Platform::RUBY
|
8
|
+
spec.name = 'mines'
|
9
|
+
spec.version = '0.0.2'
|
10
|
+
spec.summary = 'Data mining application framework.'
|
11
|
+
spec.description = 'Ruby in Mines is a framework for creating data mining application prototypes that focus on processing near real-time human generated content.'
|
12
|
+
|
13
|
+
spec.license = 'MIT'
|
14
|
+
|
15
|
+
spec.author = 'Panayiotis Vlantis'
|
16
|
+
spec.email = 'p.vlantis@di.uoa.gr'
|
17
|
+
spec.homepage = 'https://github.com/panayiotis/mines'
|
18
|
+
|
19
|
+
spec.files = `git ls-files`.split($/)
|
20
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
25
|
+
spec.add_development_dependency "rake"
|
26
|
+
spec.add_development_dependency "minitest"
|
27
|
+
|
28
|
+
spec.add_dependency 'thor'
|
29
|
+
spec.add_dependency 'colored'
|
30
|
+
spec.add_dependency 'redis'
|
31
|
+
spec.add_dependency 'hiredis'
|
32
|
+
spec.add_dependency 'twitter'
|
33
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# http://mattsears.com/articles/2011/12/10/minitest-quick-reference
|
3
|
+
require_relative "../lib/logging.rb"
|
4
|
+
require "minitest/autorun"
|
5
|
+
|
6
|
+
describe "Logging" do
|
7
|
+
include Mines::Logging
|
8
|
+
before do
|
9
|
+
|
10
|
+
end
|
11
|
+
|
12
|
+
after do
|
13
|
+
|
14
|
+
end
|
15
|
+
|
16
|
+
it "instantiates a logger" do
|
17
|
+
log.must_be_instance_of Logger
|
18
|
+
end
|
19
|
+
end
|
data/spec/mines_spec.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# http://mattsears.com/articles/2011/12/10/minitest-quick-reference
|
3
|
+
|
4
|
+
require "minitest/autorun"
|
5
|
+
|
6
|
+
describe "Mines executable" do
|
7
|
+
|
8
|
+
before do
|
9
|
+
|
10
|
+
end
|
11
|
+
|
12
|
+
after do
|
13
|
+
|
14
|
+
end
|
15
|
+
|
16
|
+
it "prints help" do
|
17
|
+
#proc { exec 'bin/mines' }.must_output "2!"
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
#!/bin/ruby
|
2
|
+
# coding: utf-8
|
3
|
+
|
4
|
+
# http://mattsears.com/articles/2011/12/10/minitest-quick-reference
|
5
|
+
require_relative "../lib/redis_store"
|
6
|
+
require 'redis'
|
7
|
+
require "minitest/autorun"
|
8
|
+
require 'minitest/pride'
|
9
|
+
|
10
|
+
describe "MessageQueue" do
|
11
|
+
|
12
|
+
before do
|
13
|
+
@redis ||= Redis.new(:path => "/tmp/redis.sock")
|
14
|
+
@redis.del "test_queue"
|
15
|
+
@redis.rpush "test_queue", ["one", "two", "three"]
|
16
|
+
@q = MessageQueue.new("test_queue")
|
17
|
+
end
|
18
|
+
|
19
|
+
it "gets created" do
|
20
|
+
@q.key.must_equal "test_queue"
|
21
|
+
end
|
22
|
+
|
23
|
+
it "has size" do
|
24
|
+
@q.size.must_be_instance_of Fixnum
|
25
|
+
end
|
26
|
+
|
27
|
+
it "can push" do
|
28
|
+
s = @q.size
|
29
|
+
@q.push "four"
|
30
|
+
@q.size.must_equal s+=1
|
31
|
+
end
|
32
|
+
|
33
|
+
it "can pop" do
|
34
|
+
s = @q.size
|
35
|
+
@q.pop.must_equal "one"
|
36
|
+
@q.pop.must_equal "two"
|
37
|
+
@q.pop.must_equal "three"
|
38
|
+
@q.size.must_equal s-=3
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
describe "ObjectQueue" do
|
44
|
+
|
45
|
+
before do
|
46
|
+
@redis ||= Redis.new(:path => "/tmp/redis.sock")
|
47
|
+
@redis.del "test_queue"
|
48
|
+
@q = ObjectQueue.new("test_queue")
|
49
|
+
end
|
50
|
+
|
51
|
+
it "can push" do
|
52
|
+
s = @q.size
|
53
|
+
@q.push({ :a=>"a", :b => "b" })
|
54
|
+
@q.size.must_equal s+=1
|
55
|
+
end
|
56
|
+
|
57
|
+
it "can pop" do
|
58
|
+
@q.push({ :a=>"a", :b => "b" })
|
59
|
+
s = @q.size
|
60
|
+
@q.pop.must_equal({ :a=>"a", :b => "b" })
|
61
|
+
@q.size.must_equal s-=1
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# http://mattsears.com/articles/2011/12/10/minitest-quick-reference
|
3
|
+
|
4
|
+
require "minitest/autorun"
|
5
|
+
|
6
|
+
require_relative '../lib/utilities'
|
7
|
+
|
8
|
+
|
9
|
+
describe "Mines::gem_exists? method" do
|
10
|
+
|
11
|
+
before do
|
12
|
+
|
13
|
+
end
|
14
|
+
|
15
|
+
after do
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
it "returns true for gem mines" do
|
20
|
+
Mines::gem_exists?('mines').must_equal true
|
21
|
+
end
|
22
|
+
|
23
|
+
it "returns false for gem vgbo145o" do
|
24
|
+
Mines::gem_exists?('vgbo145o').must_equal false
|
25
|
+
end
|
26
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mines
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-09-
|
12
|
+
date: 2013-09-09 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -147,18 +147,49 @@ executables:
|
|
147
147
|
extensions: []
|
148
148
|
extra_rdoc_files: []
|
149
149
|
files:
|
150
|
-
-
|
151
|
-
-
|
152
|
-
-
|
153
|
-
-
|
154
|
-
-
|
150
|
+
- .gitignore
|
151
|
+
- .yardopts
|
152
|
+
- Gemfile
|
153
|
+
- Gemfile.lock
|
154
|
+
- README.md
|
155
|
+
- Rakefile
|
156
|
+
- bin/mines
|
157
|
+
- config/application.rb
|
158
|
+
- config/help.rb
|
159
|
+
- examples/README.md
|
160
|
+
- examples/hashtags/Gemfile
|
161
|
+
- examples/hashtags/Gemfile.lock
|
162
|
+
- examples/hashtags/README.md
|
163
|
+
- examples/hashtags/config/application.rb
|
164
|
+
- examples/hashtags/miners/metrics.rb
|
165
|
+
- examples/hashtags/miners/process.rb
|
166
|
+
- examples/hashtags/miners/publicstream.rb
|
167
|
+
- examples/hashtags/spec/metrics_spec.rb
|
168
|
+
- examples/hashtags/spec/process_spec.rb
|
169
|
+
- examples/hashtags/spec/publicstream_spec.rb
|
155
170
|
- lib/application.rb
|
156
171
|
- lib/configuration.rb
|
172
|
+
- lib/generators/README.md
|
157
173
|
- lib/generators/application.rb
|
158
174
|
- lib/generators/miner.rb
|
159
|
-
- lib/generators/templates/
|
175
|
+
- lib/generators/templates/Gemfile.erb
|
176
|
+
- lib/generators/templates/README.md
|
160
177
|
- lib/generators/templates/application_config.erb
|
161
|
-
-
|
178
|
+
- lib/generators/templates/metrics_miner.erb
|
179
|
+
- lib/generators/templates/miner_spec.erb
|
180
|
+
- lib/generators/templates/network_miner.erb
|
181
|
+
- lib/generators/templates/process_miner.erb
|
182
|
+
- lib/generators/templates/twitter_miner.erb
|
183
|
+
- lib/help.rb
|
184
|
+
- lib/logging.rb
|
185
|
+
- lib/mines.rb
|
186
|
+
- lib/redis_store.rb
|
187
|
+
- lib/utilities.rb
|
188
|
+
- mines.gemspec
|
189
|
+
- spec/logging_spec.rb
|
190
|
+
- spec/mines_spec.rb
|
191
|
+
- spec/redis_store_spec.rb
|
192
|
+
- spec/utilities_spec.rb
|
162
193
|
homepage: https://github.com/panayiotis/mines
|
163
194
|
licenses:
|
164
195
|
- MIT
|
@@ -166,7 +197,6 @@ post_install_message:
|
|
166
197
|
rdoc_options: []
|
167
198
|
require_paths:
|
168
199
|
- lib
|
169
|
-
- lib
|
170
200
|
required_ruby_version: !ruby/object:Gem::Requirement
|
171
201
|
none: false
|
172
202
|
requirements:
|
@@ -185,5 +215,9 @@ rubygems_version: 1.8.25
|
|
185
215
|
signing_key:
|
186
216
|
specification_version: 3
|
187
217
|
summary: Data mining application framework.
|
188
|
-
test_files:
|
218
|
+
test_files:
|
219
|
+
- spec/logging_spec.rb
|
220
|
+
- spec/mines_spec.rb
|
221
|
+
- spec/redis_store_spec.rb
|
222
|
+
- spec/utilities_spec.rb
|
189
223
|
has_rdoc:
|