mines 0.0.1 → 0.0.2

data/lib/generators/templates/process_miner.erb

@@ -1,7 +1,32 @@
 #!/usr/bin/env ruby
 # coding: utf-8
 
-require 'mines'
+## Gem dependencies are managed by Bundler inside project's Gemfile
+require 'bundler'
+Bundler.require :process
 
 log_name :<%= name %>  
 
+@log.info "Process miner pid: #{Process.pid}"
+
+## Connect to an Object Queue
+## elementes are pushed to this queue by the network miner
+queue = ObjectQueue.new "<%= name %>"
+
+## Create a timeseries metrics store to
+## save the occurence rate for something
+ts = TimeSeries.new("<%= name %>")
+
+## Start the process loop
+loop {
+  ## Get en element from the queue
+  element = queue.pop
+  
+  ##
+  ##... Process element ...
+  ##
+
+  ## Push some metric to the time-series
+  ts.push ("...")
+  
+} # end loop

data/lib/generators/templates/twitter_miner.erb

@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+## Gem dependencies are managed by Bundler inside project's Gemfile
+require 'bundler'
+Bundler.require :network
+
+log_name :<%= name %>  
+
+## Create an Object Queue
+queue = ObjectQueue.new "<%= name %>"
+
+## Provide your Twitter Streaming API keys and credentials
+##
+## example:
+#  TweetStream.configure do |config|
+#    config.consumer_key       = "mZ0gsfwgA"
+#    config.consumer_secret    = "nqwQqELrvdb6C"
+#    config.oauth_token        = "OfIUtfTmP6NQuwWKlH"
+#    config.oauth_token_secret = 'u4bFMpQxfgwZl'
+#    config.auth_method = :oauth
+#  end
+
+## It is recommended to provide your credentials
+## in a different file outside version control system
+## to avoid publishing them.
+## example:
+require_relative '../../twitter_credentials'
+
+## Initialize TweetStream Client
+client = TweetStream::Client.new
+
+## Print custom messages when client receives special event messages
+## See also: https://dev.twitter.com/docs/streaming-apis/messages
+client.on_error do |message|
+  msg = "\nError message: #{message}\n"+
+        "An HTTP error is encountered in the processing of the stream. "+
+        "Note that TweetStream will automatically try to reconnect, "+
+        "this is for reference only. Don't panic!"
+  puts red msg
+  @lod.error msg
+end
+client.on_unauthorized do
+  msg = "\nUnauthorized.\n"+
+        "An HTTP status 401 is encountered while connecting to Twitter. "+ 
+        "This could happen when system clock drift has occured."
+  puts red msg
+  @log.error msg
+end
+client.on_inited do
+  msg = "Connection Established"
+  puts green msg
+  @log.info msg
+end
+client.on_no_data_received do
+  msg = "\nNo data was received from the server and a stall occurred. "+
+        "Twitter defines this to be 90 seconds."
+  puts red msg
+  @log.info msg
+end
+client.on_delete do |status_id, user_id|
+  msg = "\nStatus deletion notice: user: #{user_id} status: #{status_id}"
+  puts yellow msg
+  @log.info msg
+end
+client.on_reconnect do |timeout, retries|
+  msg = "Reconnect Timeout: #{timeout} retries: #{retries}"
+  puts red msg
+  @log.info msg
+end
+client.on_limit do |skip_count|
+  msg = "\nA rate limit notice is received from the Twitter stream.\n"+
+        "Discarded Count: #{skip_count}"
+  puts yellow msg
+  @log.info msg
+end
+client.on_enhance_your_calm do
+  msg = "\nEnhance your calm"
+  puts yellow msg
+  @log.info msg
+end
+
+## Make a call to the statuses/filter method of the Streaming API,
+## you may provide :follow, :track or :locations.
+##
+##   :track      Phrases of keywords to track
+##   :follow     A list of user IDs, indicating the
+##               users to return statuses for in the stream.
+##   :locations  Specifies a set of bounding boxes to track
+##               e.g. :locations => [-122.75,36.8,-121.75,37.8,-74,40,-73,41]                
+##
+##   example:    tracking common greek words
+client.filter(:track => %w( μια το να και το του της τα τι
+                            για δεν στο στον στην με από απο
+                            θα τις οι μου) ) do |status|
+
+  ## status is an object of class Tweet
+  
+  ## Print the user's screen name
+  # puts blue status.user.screen_name
+  
+  ## Print the tweet's text
+  # puts green status.text
+  
+  ## Print a blue dot to indicate a tweet has been received
+  print blue "." 
+  
+  ## Push the status to the object queue
+  ## the object queue will marshal the object so
+  queue.push(status)
+end

data/lib/logging.rb

@@ -14,10 +14,17 @@ require_relative 'configuration'
 #  require_relative '../config/application'
 #end
 
+
+# Check if log directory exists
+log_dir = "log"
+unless File.exists?(log_dir) && File.directory?(log_dir)
+  log_dir="."
+end
+
 unless defined?(Application.logname).nil?
-  @log ||= Logger.new "log/#{Application.logname}"
+  @log ||= Logger.new "#{log_dir}/#{Application.logname}"
 else
-  @log ||= Logger.new 'log/mines.log'
+  @log ||= Logger.new '#{log_dir}/mines.log'
 end
 
 @log.datetime_format= "%H:%M:%S"
@@ -26,7 +33,7 @@ end
 end
 
 def log_name name
-  @log ||=  Logger.new 'log/#{name}.log'
+  @log ||=  Logger.new '#{log_dir}/#{name}.log'
   @log.datetime_format= "%H:%M:%S"
   @log.formatter = proc do |severity, datetime, progname, msg|
     "#{severity} #{datetime} #{msg}\n"

data/mines.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+
+Gem::Specification.new do |spec|
+  spec.platform = Gem::Platform::RUBY
+  spec.name = 'mines'
+  spec.version = '0.0.2'
+  spec.summary = 'Data mining application framework.'
+  spec.description = 'Ruby in Mines is a framework for creating data mining application prototypes that focus on processing near real-time human generated content.'
+
+  spec.license = 'MIT'
+
+  spec.author = 'Panayiotis Vlantis'
+  spec.email = 'p.vlantis@di.uoa.gr'
+  spec.homepage = 'https://github.com/panayiotis/mines'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+  
+  spec.add_dependency 'thor'
+  spec.add_dependency 'colored'
+  spec.add_dependency 'redis'
+  spec.add_dependency 'hiredis'
+  spec.add_dependency 'twitter'      
+end

data/spec/logging_spec.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/ruby
+# http://mattsears.com/articles/2011/12/10/minitest-quick-reference
+require_relative "../lib/logging.rb"
+require "minitest/autorun"
+
+describe "Logging" do
+include Mines::Logging
+  before do
+    
+  end
+
+  after do
+  
+  end 
+
+  it "instantiates a logger" do
+   log.must_be_instance_of Logger
+  end
+end

data/spec/mines_spec.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/ruby
+# http://mattsears.com/articles/2011/12/10/minitest-quick-reference
+
+require "minitest/autorun"
+
+describe "Mines executable" do
+
+  before do
+    
+  end
+
+  after do
+  
+  end 
+
+  it "prints help" do
+    #proc { exec 'bin/mines' }.must_output "2!"
+  end
+end

data/spec/redis_store_spec.rb

@@ -0,0 +1,64 @@
+#!/bin/ruby
+# coding: utf-8
+
+# http://mattsears.com/articles/2011/12/10/minitest-quick-reference
+require_relative "../lib/redis_store"
+require 'redis'
+require "minitest/autorun"
+require 'minitest/pride'
+
+describe "MessageQueue" do
+
+  before do
+    @redis ||= Redis.new(:path => "/tmp/redis.sock")
+    @redis.del "test_queue"
+    @redis.rpush "test_queue", ["one", "two", "three"]
+    @q = MessageQueue.new("test_queue")
+  end
+  
+  it "gets created" do
+    @q.key.must_equal "test_queue"
+  end
+
+  it "has size" do
+    @q.size.must_be_instance_of Fixnum
+  end
+  
+  it "can push" do
+   s = @q.size
+   @q.push "four"
+   @q.size.must_equal s+=1
+  end
+  
+  it "can pop" do
+   s = @q.size
+   @q.pop.must_equal "one"
+   @q.pop.must_equal "two"
+   @q.pop.must_equal "three"
+   @q.size.must_equal s-=3
+  end  
+
+end
+
+describe "ObjectQueue" do
+
+  before do
+    @redis ||= Redis.new(:path => "/tmp/redis.sock")
+    @redis.del "test_queue"
+    @q = ObjectQueue.new("test_queue")
+  end
+  
+  it "can push" do
+   s = @q.size
+   @q.push({ :a=>"a", :b => "b" })
+   @q.size.must_equal s+=1
+  end
+  
+  it "can pop" do
+   @q.push({ :a=>"a", :b => "b" })
+   s = @q.size
+   @q.pop.must_equal({ :a=>"a", :b => "b" })
+   @q.size.must_equal s-=1
+  end  
+
+end

data/spec/utilities_spec.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/ruby
+# http://mattsears.com/articles/2011/12/10/minitest-quick-reference
+
+require "minitest/autorun"
+
+require_relative '../lib/utilities'
+
+
+describe "Mines::gem_exists? method" do
+
+  before do
+    
+  end
+
+  after do
+  
+  end 
+
+  it "returns true for gem mines" do
+    Mines::gem_exists?('mines').must_equal true
+  end
+
+  it "returns false for gem vgbo145o" do
+    Mines::gem_exists?('vgbo145o').must_equal false
+  end  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mines
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-08 00:00:00.000000000 Z
+date: 2013-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -147,18 +147,49 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/logging.rb
-- lib/utilities.rb
-- lib/mines.rb
-- lib/redis_store.rb
-- lib/help.rb
+- .gitignore
+- .yardopts
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/mines
+- config/application.rb
+- config/help.rb
+- examples/README.md
+- examples/hashtags/Gemfile
+- examples/hashtags/Gemfile.lock
+- examples/hashtags/README.md
+- examples/hashtags/config/application.rb
+- examples/hashtags/miners/metrics.rb
+- examples/hashtags/miners/process.rb
+- examples/hashtags/miners/publicstream.rb
+- examples/hashtags/spec/metrics_spec.rb
+- examples/hashtags/spec/process_spec.rb
+- examples/hashtags/spec/publicstream_spec.rb
 - lib/application.rb
 - lib/configuration.rb
+- lib/generators/README.md
 - lib/generators/application.rb
 - lib/generators/miner.rb
-- lib/generators/templates/process_miner.erb
+- lib/generators/templates/Gemfile.erb
+- lib/generators/templates/README.md
 - lib/generators/templates/application_config.erb
-- bin/mines
+- lib/generators/templates/metrics_miner.erb
+- lib/generators/templates/miner_spec.erb
+- lib/generators/templates/network_miner.erb
+- lib/generators/templates/process_miner.erb
+- lib/generators/templates/twitter_miner.erb
+- lib/help.rb
+- lib/logging.rb
+- lib/mines.rb
+- lib/redis_store.rb
+- lib/utilities.rb
+- mines.gemspec
+- spec/logging_spec.rb
+- spec/mines_spec.rb
+- spec/redis_store_spec.rb
+- spec/utilities_spec.rb
 homepage: https://github.com/panayiotis/mines
 licenses:
 - MIT
@@ -166,7 +197,6 @@ post_install_message:
 rdoc_options: []
 require_paths:
 - lib
-- lib
 required_ruby_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -185,5 +215,9 @@ rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Data mining application framework.
-test_files: []
+test_files:
+- spec/logging_spec.rb
+- spec/mines_spec.rb
+- spec/redis_store_spec.rb
+- spec/utilities_spec.rb
 has_rdoc: