mines 0.0.1 → 0.0.2

data/examples/hashtags/miners/process.rb

@@ -0,0 +1,46 @@
+#!/bin/ruby
+# coding: utf-8
+
+## Gem dependencies are managed by Bundler inside project's Gemfile
+require 'bundler'
+Bundler.require :process
+
+@log.info "Process miner pid: #{Process.pid}"
+
+## Connect to an Object Queue
+## elementes are pushed to this queue by the network miner
+queue = ObjectQueue.new "publicstream"
+
+counter = Counter.new "original"
+
+channel = MessageChannel.new("web") # example channel
+
+## Create a timeseries metrics store to
+## save the occurence rate for each tag for each minute
+ts = TimeSeries.new("hashtags")
+
+## Create a timeseries metrics store to
+## save the total number of tweets each minute
+tweets_rate = TimeSeries.new("tweets_rate")
+
+## Start the process loop
+loop {
+  ## Get en element from the queue
+  status = queue.pop
+  tweets_rate.push "all"
+  
+  ## Filter out Retweets and Replies
+  unless status.text =~ /^RT/ or
+         status.retweet_count > 0 or
+         status.text =~ /^@/
+         
+    print_tweet(status.user.screen_name, status.text)
+    
+    status.hashtags.each do |hashtag|
+      ts.push (hashtag.text)
+    end
+    counter.incr
+    channel.publish({:user => status.user.screen_name, :text => status.text}.to_json)
+    #web.publish({:user => status.user.screen_name, :text => status.text.gsub(/(https?:\/\/\S+)/, '<a href="\1" target="_blank">\1</a> ').to_s}.to_json)
+  end
+} # end loop

data/examples/hashtags/miners/publicstream.rb

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+## Gem dependencies are managed by Bundler inside project's Gemfile
+require 'bundler'
+Bundler.require :network
+
+## Create an Object Queue
+queue = ObjectQueue.new "publicstream"
+
+## Provide your Twitter Streaming API keys and credentials
+##
+## example:
+#  TweetStream.configure do |config|
+#    config.consumer_key       = "mZ0gsfwgA"
+#    config.consumer_secret    = "nqwQqELrvdb6C"
+#    config.oauth_token        = "OfIUtfTmP6NQuwWKlH"
+#    config.oauth_token_secret = 'u4bFMpQxfgwZl'
+#    config.auth_method = :oauth
+#  end
+
+## It is recommended to provide your credentials
+## in a different file outside version control system
+## to avoid publishing them.
+## example:
+require_relative '../../twitter_credentials'
+
+## Initialize TweetStream Client
+client = TweetStream::Client.new
+
+## Print custom messages when client receives special event messages
+## See also: https://dev.twitter.com/docs/streaming-apis/messages
+client.on_error do |message|
+  msg = "\nError message: #{message}\n"+
+        "An HTTP error is encountered in the processing of the stream. "+
+        "Note that TweetStream will automatically try to reconnect, "+
+        "this is for reference only. Don't panic!"
+  puts red msg
+  @lod.error msg
+end
+client.on_unauthorized do
+  msg = "\nUnauthorized.\n"+
+        "An HTTP status 401 is encountered while connecting to Twitter. "+ 
+        "This could happen when system clock drift has occured."
+  puts red msg
+  @log.error msg
+end
+client.on_inited do
+  msg = "Connection Established"
+  puts green msg
+  @log.info msg
+end
+client.on_no_data_received do
+  msg = "\nNo data was received from the server and a stall occurred. "+
+        "Twitter defines this to be 90 seconds."
+  puts red msg
+  @log.info msg
+end
+client.on_delete do |status_id, user_id|
+  msg = "\nStatus deletion notice: user: #{user_id} status: #{status_id}"
+  puts yellow msg
+  @log.info msg
+end
+client.on_reconnect do |timeout, retries|
+  msg = "Reconnect Timeout: #{timeout} retries: #{retries}"
+  puts red msg
+  @log.info msg
+end
+client.on_limit do |skip_count|
+  msg = "\nA rate limit notice is received from the Twitter stream.\n"+
+        "Discarded Count: #{skip_count}"
+  puts yellow msg
+  @log.info msg
+end
+client.on_enhance_your_calm do
+  msg = "\nEnhance your calm"
+  puts yellow msg
+  @log.info msg
+end
+
+## Make a call to the statuses/filter method of the Streaming API,
+## you may provide :follow, :track or :locations.
+##
+##   :track      Phrases of keywords to track
+##   :follow     A list of user IDs, indicating the
+##               users to return statuses for in the stream.
+##   :locations  Specifies a set of bounding boxes to track
+##               e.g. :locations => [-122.75,36.8,-121.75,37.8,-74,40,-73,41]                
+##
+client.filter(:track => %w( μια το να και το του της τα τι
+                            για δεν στο στον στην με από απο
+                            θα τις οι μου) ) do |status|
+#client.filter(:track => %w( hi ) ) do |status|  
+  ## status is an object of class Tweet
+  
+  ## Print the user's screen name
+  # puts blue status.user.screen_name
+  
+  ## Print the tweet's text
+  # puts green status.text
+  
+  ## Print a blue dot to indicate a tweet has been received
+  print blue "." 
+  
+  ## Push the status to the object queue
+  ## the object queue will marshal the object so
+  queue.push(status)
+end

data/examples/hashtags/spec/metrics_spec.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require "minitest/autorun"
+require_relative '../miners/metrics.rb'
+
+describe "metrics Miner" do
+
+  before do
+  end
+
+   after do
+  end 
+
+  it "metrics does this" do
+    true.must_equal true
+  end
+
+  it "metrics does that" do
+    true.must_equal true
+  end  
+
+end

data/examples/hashtags/spec/process_spec.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require "minitest/autorun"
+require_relative '../miners/process.rb'
+
+describe "process Miner" do
+
+  before do
+  end
+
+   after do
+  end 
+
+  it "process does this" do
+    true.must_equal true
+  end
+
+  it "process does that" do
+    true.must_equal true
+  end  
+
+end

data/examples/hashtags/spec/publicstream_spec.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require "minitest/autorun"
+require_relative '../miners/publicstream.rb'
+
+describe "publicstream Miner" do
+
+  before do
+  end
+
+   after do
+  end 
+
+  it "publicstream does this" do
+    true.must_equal true
+  end
+
+  it "publicstream does that" do
+    true.must_equal true
+  end  
+
+end

data/lib/generators/README.md

@@ -0,0 +1,13 @@
+### Generators
+
+Generators are used to create directories, files with boilerplate code etc
+
+
+#### Generators
+
+* **Application**  
+  creates the default file structure and prints appropriate messages
+
+* **Miner**  
+      creates spesific miners based on templates
+

data/lib/generators/application.rb

@@ -14,7 +14,7 @@ module Mines::Generator
   # Accepts as an argument the name of the new application. A new directory with this name will be created
   class Application < Thor::Group
     include Thor::Actions
-    include Mines::Logging
+    #include Mines::Logging
 
     # Define arguments and options
     argument :name
@@ -29,13 +29,14 @@ module Mines::Generator
     # - Files: TODO
     # also prints appropriate messages
     def create_directory_structure
-      log.info "Creating directory structure in dir: " + name
+      #@log.info "Creating directory structure in dir: " + name
       empty_directory name
       empty_directory name + "/miners"
       empty_directory name + "/lib"
       empty_directory name + "/log"
       empty_directory name + "/config"
       template "templates/application_config.erb", "#{name}/config/application.rb"
+      template "templates/Gemfile.erb", "#{name}/Gemfile"      
       puts "Application created successfully! ".green
       print "Type "
       print "'cd #{name}' ".yellow

data/lib/generators/miner.rb

@@ -13,7 +13,7 @@ module Mines::Generator
   # - Metrics
   class Miner < Thor::Group
     include Thor::Actions
-    include Mines::Logging
+    #include Mines::Logging
 
     # Define arguments and options
     argument :type, :desc => "The type of the miner, [Network,Process,Metrics]", :required => true
@@ -32,7 +32,15 @@ module Mines::Generator
       #puts options[:evented]
       template "templates/#{type}_miner.erb", "miners/#{name}.rb"
     end
-    
+
+    # Use the appropriate template file according to type
+    # and put the generated file in miners directory
+    def create_miner_spec
+      puts "Copy miner spec template"
+      #puts options[:evented]
+      template "templates/miner_spec.erb", "spec/#{name}_spec.rb"
+    end
+        
   end # class
 
 end # module

data/lib/generators/templates/Gemfile.erb

@@ -0,0 +1,24 @@
+source 'https://rubygems.org'
+
+gem 'minitest'
+
+group :network do
+  gem 'mines' , :git => 'git://github.com/panayiotis/mines.git'
+  gem 'tweetstream'     # Twitter Streaming API library
+end
+
+group :process do
+  gem 'mines' , :git => 'git://github.com/panayiotis/mines.git'
+  gem 'json'
+  gem 'time_series' , :git => 'git://github.com/panayiotis/time_series.git'
+  gem 'twitter'
+  gem 'awesome_print'
+end
+
+group :metrics do
+  gem 'mines' , :git => 'git://github.com/panayiotis/mines.git'
+  gem 'time_series' , :git => 'git://github.com/panayiotis/time_series.git'
+  gem 'json'
+  gem 'eventmachine'
+  gem 'awesome_print'  
+end

data/lib/generators/templates/README.md

@@ -0,0 +1,24 @@
+### Templates
+
+Files with boilerplate code are created based on these templates
+
+
+#### Templates
+
+* **network miner**  
+  Boilerplate code for the network miner. Will be placed in miners directory.
+  
+* **process miner**  
+  Boilerplate code for the process miner. Will be placed in miners directory.
+
+* **twitter miner**  
+  Boilerplate code for the twitter miner. Will be placed in miners directory.
+
+* **metrics miner**  
+  Boilerplate code for the metrics miner. Will be placed in miners directory.
+
+* **application config**  
+  Application configuration. Will be placed in config directory.
+
+* **minner spec**  
+  Miner spec. Will be created for each miner and placed in spec directory.

data/lib/generators/templates/application_config.erb

@@ -6,6 +6,8 @@ Application.config do
   parameter :var1
   parameter :var2
   parameter :var3
+  parameter :logname
+  parameter :app_prefix  
 end
 
 ## Change their values
@@ -13,7 +15,16 @@ Application.config do
   var1  "value"
   var2  ['one','two','three']
   var3  { :one=>1, :two=>2 }
+  logname  "<%= name %>.log"
+  app_prefix "<%= name %>"
 end
 
+
+## Initializer
+## e.g. initialize a global variable
+## $app_prefix variable needs to be global
+$app_prefix = Application.app_prefix
+
+
 ## Usage example
 # puts Application.var1

data/lib/generators/templates/metrics_miner.erb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+## Gem dependencies are managed by Bundler inside project's Gemfile
+require 'bundler'
+Bundler.require :network
+
+log_name :<%= name %>  
+
+## Connect to a TimeSeries metrics store
+## Data is been pushed constantly to the timeseries by
+## the process miner
+@ts = TimeSeries.new("<%= name %>")
+
+
+## Start Event Machine
+EM.run do
+  
+  ## Periodic execution
+  EM::PeriodicTimer.new(13){
+    print blue '.'
+  }
+  
+  ## Periodic execution  
+  EM::PeriodicTimer.new(1){
+    print red '.'
+  }
+end # end EM

data/lib/generators/templates/miner_spec.erb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require "minitest/autorun"
+require_relative '../miners/<%= name %>.rb'
+
+describe "<%= name %> Miner" do
+
+  before do
+  end
+
+   after do
+  end 
+
+  it "<%= name %> does this" do
+    true.must_equal true
+  end
+
+  it "<%= name %> does that" do
+    true.must_equal true
+  end  
+
+end

data/lib/generators/templates/network_miner.erb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+## Gem dependencies are managed by Bundler inside project's Gemfile
+require 'bundler'
+Bundler.require :network
+
+#log_name :<%= name %>  
+
+## Use event Machine for non-blocking IO
+## or Threads for blocking IO
+
+## Create Thread example
+Thread.new { loop { print green '.'; `sleep 1` } }
+
+# Start Event Machine
+EM.run do
+  
+  ## Periodic execution
+  EM::PeriodicTimer.new(1){
+    print blue '.'
+    ## Avoid blocking commands inside the Reactor.
+    ## This 'sleep' is blocking the other
+    ## timer too. Use Threads instead.
+    `sleep 1` 
+  }
+  
+  ## Periodic execution  
+  EM::PeriodicTimer.new(0.5){
+    print red '.'
+  }
+end # end EM