mines 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. data/.gitignore +7 -0
  2. data/.yardopts +6 -0
  3. data/Gemfile +5 -0
  4. data/Gemfile.lock +37 -0
  5. data/README.md +25 -0
  6. data/Rakefile +8 -0
  7. data/bin/mines +8 -8
  8. data/config/application.rb +11 -0
  9. data/config/help.rb +13 -0
  10. data/examples/README.md +4 -0
  11. data/examples/hashtags/Gemfile +24 -0
  12. data/examples/hashtags/Gemfile.lock +81 -0
  13. data/examples/hashtags/README.md +22 -0
  14. data/examples/hashtags/config/application.rb +30 -0
  15. data/examples/hashtags/miners/metrics.rb +157 -0
  16. data/examples/hashtags/miners/process.rb +46 -0
  17. data/examples/hashtags/miners/publicstream.rb +108 -0
  18. data/examples/hashtags/spec/metrics_spec.rb +23 -0
  19. data/examples/hashtags/spec/process_spec.rb +23 -0
  20. data/examples/hashtags/spec/publicstream_spec.rb +23 -0
  21. data/lib/generators/README.md +13 -0
  22. data/lib/generators/application.rb +3 -2
  23. data/lib/generators/miner.rb +10 -2
  24. data/lib/generators/templates/Gemfile.erb +24 -0
  25. data/lib/generators/templates/README.md +24 -0
  26. data/lib/generators/templates/application_config.erb +11 -0
  27. data/lib/generators/templates/metrics_miner.erb +28 -0
  28. data/lib/generators/templates/miner_spec.erb +23 -0
  29. data/lib/generators/templates/network_miner.erb +32 -0
  30. data/lib/generators/templates/process_miner.erb +26 -1
  31. data/lib/generators/templates/twitter_miner.erb +111 -0
  32. data/lib/logging.rb +10 -3
  33. data/mines.gemspec +33 -0
  34. data/spec/logging_spec.rb +19 -0
  35. data/spec/mines_spec.rb +19 -0
  36. data/spec/redis_store_spec.rb +64 -0
  37. data/spec/utilities_spec.rb +26 -0
  38. metadata +45 -11
@@ -1,7 +1,32 @@
1
1
  #!/usr/bin/env ruby
2
2
  # coding: utf-8
3
3
 
4
- require 'mines'
4
+ ## Gem dependencies are managed by Bundler inside project's Gemfile
5
+ require 'bundler'
6
+ Bundler.require :process
5
7
 
6
8
  log_name :<%= name %>
7
9
 
10
+ @log.info "Process miner pid: #{Process.pid}"
11
+
12
+ ## Connect to an Object Queue
13
+ ## elementes are pushed to this queue by the network miner
14
+ queue = ObjectQueue.new "<%= name %>"
15
+
16
+ ## Create a timeseries metrics store to
17
+ ## save the occurence rate for something
18
+ ts = TimeSeries.new("<%= name %>")
19
+
20
+ ## Start the process loop
21
+ loop {
22
+ ## Get en element from the queue
23
+ element = queue.pop
24
+
25
+ ##
26
+ ##... Process element ...
27
+ ##
28
+
29
+ ## Push some metric to the time-series
30
+ ts.push ("...")
31
+
32
+ } # end loop
@@ -0,0 +1,111 @@
1
+ #!/usr/bin/env ruby
2
+ # coding: utf-8
3
+
4
+ ## Gem dependencies are managed by Bundler inside project's Gemfile
5
+ require 'bundler'
6
+ Bundler.require :network
7
+
8
+ log_name :<%= name %>
9
+
10
+ ## Create an Object Queue
11
+ queue = ObjectQueue.new "<%= name %>"
12
+
13
+ ## Provide your Twitter Streaming API keys and credentials
14
+ ##
15
+ ## example:
16
+ # TweetStream.configure do |config|
17
+ # config.consumer_key = "mZ0gsfwgA"
18
+ # config.consumer_secret = "nqwQqELrvdb6C"
19
+ # config.oauth_token = "OfIUtfTmP6NQuwWKlH"
20
+ # config.oauth_token_secret = 'u4bFMpQxfgwZl'
21
+ # config.auth_method = :oauth
22
+ # end
23
+
24
+ ## It is recommended to provide your credentials
25
+ ## in a different file outside version control system
26
+ ## to avoid publishing them.
27
+ ## example:
28
+ require_relative '../../twitter_credentials'
29
+
30
+ ## Initialize TweetStream Client
31
+ client = TweetStream::Client.new
32
+
33
+ ## Print custom messages when client receives special event messages
34
+ ## See also: https://dev.twitter.com/docs/streaming-apis/messages
35
+ client.on_error do |message|
36
+ msg = "\nError message: #{message}\n"+
37
+ "An HTTP error is encountered in the processing of the stream. "+
38
+ "Note that TweetStream will automatically try to reconnect, "+
39
+ "this is for reference only. Don't panic!"
40
+ puts red msg
41
+ @lod.error msg
42
+ end
43
+ client.on_unauthorized do
44
+ msg = "\nUnauthorized.\n"+
45
+ "An HTTP status 401 is encountered while connecting to Twitter. "+
46
+ "This could happen when system clock drift has occured."
47
+ puts red msg
48
+ @log.error msg
49
+ end
50
+ client.on_inited do
51
+ msg = "Connection Established"
52
+ puts green msg
53
+ @log.info msg
54
+ end
55
+ client.on_no_data_received do
56
+ msg = "\nNo data was received from the server and a stall occurred. "+
57
+ "Twitter defines this to be 90 seconds."
58
+ puts red msg
59
+ @log.info msg
60
+ end
61
+ client.on_delete do |status_id, user_id|
62
+ msg = "\nStatus deletion notice: user: #{user_id} status: #{status_id}"
63
+ puts yellow msg
64
+ @log.info msg
65
+ end
66
+ client.on_reconnect do |timeout, retries|
67
+ msg = "Reconnect Timeout: #{timeout} retries: #{retries}"
68
+ puts red msg
69
+ @log.info msg
70
+ end
71
+ client.on_limit do |skip_count|
72
+ msg = "\nA rate limit notice is received from the Twitter stream.\n"+
73
+ "Discarded Count: #{skip_count}"
74
+ puts yellow msg
75
+ @log.info msg
76
+ end
77
+ client.on_enhance_your_calm do
78
+ msg = "\nEnhance your calm"
79
+ puts yellow msg
80
+ @log.info msg
81
+ end
82
+
83
+ ## Make a call to the statuses/filter method of the Streaming API,
84
+ ## you may provide :follow, :track or :locations.
85
+ ##
86
+ ## :track Phrases of keywords to track
87
+ ## :follow A list of user IDs, indicating the
88
+ ## users to return statuses for in the stream.
89
+ ## :locations Specifies a set of bounding boxes to track
90
+ ## e.g. :locations => [-122.75,36.8,-121.75,37.8,-74,40,-73,41]
91
+ ##
92
+ ## example: tracking common greek words
93
+ client.filter(:track => %w( μια το να και το του της τα τι
94
+ για δεν στο στον στην με από απο
95
+ θα τις οι μου) ) do |status|
96
+
97
+ ## status is an object of class Tweet
98
+
99
+ ## Print the user's screen name
100
+ # puts blue status.user.screen_name
101
+
102
+ ## Print the tweet's text
103
+ # puts green status.text
104
+
105
+ ## Print a blue dot to indicate a tweet has been received
106
+ print blue "."
107
+
108
+ ## Push the status to the object queue
109
+ ## the object queue will marshal the object so
110
+ queue.push(status)
111
+ end
@@ -14,10 +14,17 @@ require_relative 'configuration'
14
14
  # require_relative '../config/application'
15
15
  #end
16
16
 
17
+
18
+ # Check if log directory exists
19
+ log_dir = "log"
20
+ unless File.exists?(log_dir) && File.directory?(log_dir)
21
+ log_dir="."
22
+ end
23
+
17
24
  unless defined?(Application.logname).nil?
18
- @log ||= Logger.new "log/#{Application.logname}"
25
+ @log ||= Logger.new "#{log_dir}/#{Application.logname}"
19
26
  else
20
- @log ||= Logger.new 'log/mines.log'
27
+ @log ||= Logger.new '#{log_dir}/mines.log'
21
28
  end
22
29
 
23
30
  @log.datetime_format= "%H:%M:%S"
@@ -26,7 +33,7 @@ end
26
33
  end
27
34
 
28
35
  def log_name name
29
- @log ||= Logger.new 'log/#{name}.log'
36
+ @log ||= Logger.new '#{log_dir}/#{name}.log'
30
37
  @log.datetime_format= "%H:%M:%S"
31
38
  @log.formatter = proc do |severity, datetime, progname, msg|
32
39
  "#{severity} #{datetime} #{msg}\n"
@@ -0,0 +1,33 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.platform = Gem::Platform::RUBY
8
+ spec.name = 'mines'
9
+ spec.version = '0.0.2'
10
+ spec.summary = 'Data mining application framework.'
11
+ spec.description = 'Ruby in Mines is a framework for creating data mining application prototypes that focus on processing near real-time human generated content.'
12
+
13
+ spec.license = 'MIT'
14
+
15
+ spec.author = 'Panayiotis Vlantis'
16
+ spec.email = 'p.vlantis@di.uoa.gr'
17
+ spec.homepage = 'https://github.com/panayiotis/mines'
18
+
19
+ spec.files = `git ls-files`.split($/)
20
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
21
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
22
+ spec.require_paths = ["lib"]
23
+
24
+ spec.add_development_dependency "bundler", "~> 1.3"
25
+ spec.add_development_dependency "rake"
26
+ spec.add_development_dependency "minitest"
27
+
28
+ spec.add_dependency 'thor'
29
+ spec.add_dependency 'colored'
30
+ spec.add_dependency 'redis'
31
+ spec.add_dependency 'hiredis'
32
+ spec.add_dependency 'twitter'
33
+ end
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/ruby
2
+ # http://mattsears.com/articles/2011/12/10/minitest-quick-reference
3
+ require_relative "../lib/logging.rb"
4
+ require "minitest/autorun"
5
+
6
+ describe "Logging" do
7
+ include Mines::Logging
8
+ before do
9
+
10
+ end
11
+
12
+ after do
13
+
14
+ end
15
+
16
+ it "instantiates a logger" do
17
+ log.must_be_instance_of Logger
18
+ end
19
+ end
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/ruby
2
+ # http://mattsears.com/articles/2011/12/10/minitest-quick-reference
3
+
4
+ require "minitest/autorun"
5
+
6
+ describe "Mines executable" do
7
+
8
+ before do
9
+
10
+ end
11
+
12
+ after do
13
+
14
+ end
15
+
16
+ it "prints help" do
17
+ #proc { exec 'bin/mines' }.must_output "2!"
18
+ end
19
+ end
@@ -0,0 +1,64 @@
1
+ #!/bin/ruby
2
+ # coding: utf-8
3
+
4
+ # http://mattsears.com/articles/2011/12/10/minitest-quick-reference
5
+ require_relative "../lib/redis_store"
6
+ require 'redis'
7
+ require "minitest/autorun"
8
+ require 'minitest/pride'
9
+
10
+ describe "MessageQueue" do
11
+
12
+ before do
13
+ @redis ||= Redis.new(:path => "/tmp/redis.sock")
14
+ @redis.del "test_queue"
15
+ @redis.rpush "test_queue", ["one", "two", "three"]
16
+ @q = MessageQueue.new("test_queue")
17
+ end
18
+
19
+ it "gets created" do
20
+ @q.key.must_equal "test_queue"
21
+ end
22
+
23
+ it "has size" do
24
+ @q.size.must_be_instance_of Fixnum
25
+ end
26
+
27
+ it "can push" do
28
+ s = @q.size
29
+ @q.push "four"
30
+ @q.size.must_equal s+=1
31
+ end
32
+
33
+ it "can pop" do
34
+ s = @q.size
35
+ @q.pop.must_equal "one"
36
+ @q.pop.must_equal "two"
37
+ @q.pop.must_equal "three"
38
+ @q.size.must_equal s-=3
39
+ end
40
+
41
+ end
42
+
43
+ describe "ObjectQueue" do
44
+
45
+ before do
46
+ @redis ||= Redis.new(:path => "/tmp/redis.sock")
47
+ @redis.del "test_queue"
48
+ @q = ObjectQueue.new("test_queue")
49
+ end
50
+
51
+ it "can push" do
52
+ s = @q.size
53
+ @q.push({ :a=>"a", :b => "b" })
54
+ @q.size.must_equal s+=1
55
+ end
56
+
57
+ it "can pop" do
58
+ @q.push({ :a=>"a", :b => "b" })
59
+ s = @q.size
60
+ @q.pop.must_equal({ :a=>"a", :b => "b" })
61
+ @q.size.must_equal s-=1
62
+ end
63
+
64
+ end
@@ -0,0 +1,26 @@
1
+ #!/usr/bin/ruby
2
+ # http://mattsears.com/articles/2011/12/10/minitest-quick-reference
3
+
4
+ require "minitest/autorun"
5
+
6
+ require_relative '../lib/utilities'
7
+
8
+
9
+ describe "Mines::gem_exists? method" do
10
+
11
+ before do
12
+
13
+ end
14
+
15
+ after do
16
+
17
+ end
18
+
19
+ it "returns true for gem mines" do
20
+ Mines::gem_exists?('mines').must_equal true
21
+ end
22
+
23
+ it "returns false for gem vgbo145o" do
24
+ Mines::gem_exists?('vgbo145o').must_equal false
25
+ end
26
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mines
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-09-08 00:00:00.000000000 Z
12
+ date: 2013-09-09 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -147,18 +147,49 @@ executables:
147
147
  extensions: []
148
148
  extra_rdoc_files: []
149
149
  files:
150
- - lib/logging.rb
151
- - lib/utilities.rb
152
- - lib/mines.rb
153
- - lib/redis_store.rb
154
- - lib/help.rb
150
+ - .gitignore
151
+ - .yardopts
152
+ - Gemfile
153
+ - Gemfile.lock
154
+ - README.md
155
+ - Rakefile
156
+ - bin/mines
157
+ - config/application.rb
158
+ - config/help.rb
159
+ - examples/README.md
160
+ - examples/hashtags/Gemfile
161
+ - examples/hashtags/Gemfile.lock
162
+ - examples/hashtags/README.md
163
+ - examples/hashtags/config/application.rb
164
+ - examples/hashtags/miners/metrics.rb
165
+ - examples/hashtags/miners/process.rb
166
+ - examples/hashtags/miners/publicstream.rb
167
+ - examples/hashtags/spec/metrics_spec.rb
168
+ - examples/hashtags/spec/process_spec.rb
169
+ - examples/hashtags/spec/publicstream_spec.rb
155
170
  - lib/application.rb
156
171
  - lib/configuration.rb
172
+ - lib/generators/README.md
157
173
  - lib/generators/application.rb
158
174
  - lib/generators/miner.rb
159
- - lib/generators/templates/process_miner.erb
175
+ - lib/generators/templates/Gemfile.erb
176
+ - lib/generators/templates/README.md
160
177
  - lib/generators/templates/application_config.erb
161
- - bin/mines
178
+ - lib/generators/templates/metrics_miner.erb
179
+ - lib/generators/templates/miner_spec.erb
180
+ - lib/generators/templates/network_miner.erb
181
+ - lib/generators/templates/process_miner.erb
182
+ - lib/generators/templates/twitter_miner.erb
183
+ - lib/help.rb
184
+ - lib/logging.rb
185
+ - lib/mines.rb
186
+ - lib/redis_store.rb
187
+ - lib/utilities.rb
188
+ - mines.gemspec
189
+ - spec/logging_spec.rb
190
+ - spec/mines_spec.rb
191
+ - spec/redis_store_spec.rb
192
+ - spec/utilities_spec.rb
162
193
  homepage: https://github.com/panayiotis/mines
163
194
  licenses:
164
195
  - MIT
@@ -166,7 +197,6 @@ post_install_message:
166
197
  rdoc_options: []
167
198
  require_paths:
168
199
  - lib
169
- - lib
170
200
  required_ruby_version: !ruby/object:Gem::Requirement
171
201
  none: false
172
202
  requirements:
@@ -185,5 +215,9 @@ rubygems_version: 1.8.25
185
215
  signing_key:
186
216
  specification_version: 3
187
217
  summary: Data mining application framework.
188
- test_files: []
218
+ test_files:
219
+ - spec/logging_spec.rb
220
+ - spec/mines_spec.rb
221
+ - spec/redis_store_spec.rb
222
+ - spec/utilities_spec.rb
189
223
  has_rdoc: