mines 0.0.1 → 0.0.2

data/.gitignore

@@ -0,0 +1,7 @@
+app/
+tmp/
+mines.log
+.yardoc/
+doc/
+reinstall
+*.gem

data/.yardopts

@@ -0,0 +1,6 @@
+--title "Ruby in Mines Documentation"
+bin/mines
+lib/**/*.rb
+spec/*.rb
+ -
+README.md

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in redis_timeseries.gemspec
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,37 @@
+PATH
+  remote: .
+  specs:
+    mines (0.0.1)
+      colored
+      hiredis
+      redis
+      thor
+      twitter
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    colored (1.2)
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    hiredis (0.4.5)
+    minitest (5.0.6)
+    multi_json (1.7.9)
+    multipart-post (1.2.0)
+    rake (10.0.4)
+    redis (3.0.4)
+    simple_oauth (0.2.0)
+    thor (0.18.1)
+    twitter (4.8.1)
+      faraday (~> 0.8, < 0.10)
+      multi_json (~> 1.0)
+      simple_oauth (~> 0.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  mines!
+  minitest
+  rake

data/README.md

@@ -0,0 +1,25 @@
+# Ruby in Mines
+
+Ruby in Mines is a ruby framework for creating data-mining application prototypes that focus on processing near real-time human generated content. 
+
+Install the gem and run with:  
+<tt>gem install mines </tt>
+  
+**Note:**  
+This project is under development. It has limited capabilities yet.
+
+## Commands
+
+* <tt>mines help </tt>  
+      print help
+
+* <tt>mines new AppName </tt>  
+      create a new app in the current directory
+
+* <tt>mines generate MinerName </tt>   
+      generate a new miner
+
+* <tt>mines start </tt>  
+      start the miners, and restart them if source code changes 
+
+

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs.push "lib"
+  t.test_files = FileList['spec/*_spec.rb']
+  t.verbose = true
+end
+

data/bin/mines

@@ -3,7 +3,7 @@
 
 require 'thor'
 require_relative '../lib/utilities'
-require_relative '../lib/logging'
+#require_relative '../lib/logging'
 require_relative '../lib/mines'
 
 # require all generators in lib
@@ -16,7 +16,7 @@ module Mines
   # invocates generators according to parameters
   class Cli < Thor
     
-    include Logging
+    #include Logging
     
     def initialize *args
       super
@@ -28,17 +28,17 @@ module Mines
     # @param name The name of the new application
     def new(name)
       ARGV.shift
-      log.debug "command 'new' with arguments: #{ARGV.join(', ')}"
+      #@log.debug "command 'new' with arguments: #{ARGV.join(', ')}"
       Generator::Application.start ARGV
     end
     
     # help
     def help *args
-       if args.empty?
-         log.debug "command 'help' with no arguments"
-       else
-         log.debug "command 'help' with arguments: #{args.join(', ').to_s}"
-       end
+      if args.empty?
+        #@log.debug "command 'help' with no arguments"
+      else
+        #@log.debug "command 'help' with arguments: #{args.join(', ').to_s}"
+      end
        puts "Summary:"
        super
        puts "Now print my help :)\nYeah!"

data/config/application.rb

@@ -0,0 +1,11 @@
+# This file provides some default config options
+
+Application.config do
+  parameter :logname
+  parameter :var
+end
+
+Application.config do
+  logname  "mines.log"
+  var  'default var'
+end

data/config/help.rb

@@ -0,0 +1,13 @@
+# coding: utf-8
+
+Mines::Help.configure do |config|
+  config.new = { sum: "new APP_NAME" , 
+                desc: "Create Application APP_NAME" , 
+                help: ""
+               }
+  config.generate = { sum: "generate [twitter|network|process|metrics] NAME" , 
+                     desc: "Create a twitter or network or process or metrics miner" , 
+                     help: ""
+                    }
+end
+

data/examples/README.md

@@ -0,0 +1,4 @@
+# Examples
+
+* hashtags  
+a twitter trending hashtags app

data/examples/hashtags/Gemfile

@@ -0,0 +1,24 @@
+source 'https://rubygems.org'
+
+gem 'minitest'
+
+group :network do
+  gem 'mines' , :git => 'git://github.com/panayiotis/mines.git'
+  gem 'tweetstream'     # Twitter Streaming API library
+end
+
+group :process do
+  gem 'mines' , :git => 'git://github.com/panayiotis/mines.git'
+  gem 'json'
+  gem 'time_series' , :git => 'git://github.com/panayiotis/time_series.git'
+  gem 'twitter'
+  gem 'awesome_print'
+end
+
+group :metrics do
+  gem 'mines' , :git => 'git://github.com/panayiotis/mines.git'
+  gem 'time_series' , :git => 'git://github.com/panayiotis/time_series.git'
+  gem 'json'
+  gem 'eventmachine'
+  gem 'awesome_print'  
+end

data/examples/hashtags/Gemfile.lock

@@ -0,0 +1,81 @@
+GIT
+  remote: git://github.com/panayiotis/mines.git
+  revision: c7fb4978b77d8aae68019be020e118557ecb86ea
+  specs:
+    mines (0.0.1)
+      colored
+      hiredis
+      redis
+      thor
+      twitter
+
+GIT
+  remote: git://github.com/panayiotis/time_series.git
+  revision: 7c0896db0c88152bbe5394e149fcc1c746eb6b83
+  specs:
+    time_series (0.0.2)
+      activesupport
+      awesome_print
+      hiredis
+      redis
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (3.2.14)
+      i18n (~> 0.6, >= 0.6.4)
+      multi_json (~> 1.0)
+    addressable (2.3.5)
+    awesome_print (1.1.0)
+    colored (1.2)
+    cookiejar (0.3.0)
+    daemons (1.1.9)
+    em-http-request (1.0.3)
+      addressable (>= 2.2.3)
+      cookiejar
+      em-socksify
+      eventmachine (>= 1.0.0.beta.4)
+      http_parser.rb (>= 0.5.3)
+    em-socksify (0.3.0)
+      eventmachine (>= 1.0.0.beta.4)
+    em-twitter (0.2.2)
+      eventmachine (~> 1.0)
+      http_parser.rb (~> 0.5)
+      simple_oauth (~> 0.1)
+    eventmachine (1.0.3)
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    hiredis (0.4.5)
+    http_parser.rb (0.5.3)
+    i18n (0.6.5)
+    json (1.8.0)
+    minitest (5.0.7)
+    multi_json (1.8.0)
+    multipart-post (1.2.0)
+    redis (3.0.4)
+    simple_oauth (0.2.0)
+    thor (0.18.1)
+    tweetstream (2.5.0)
+      daemons (~> 1.1)
+      em-http-request (~> 1.0.2)
+      em-twitter (~> 0.2)
+      twitter (~> 4.5)
+      yajl-ruby (~> 1.1)
+    twitter (4.8.1)
+      faraday (~> 0.8, < 0.10)
+      multi_json (~> 1.0)
+      simple_oauth (~> 0.2)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  awesome_print
+  eventmachine
+  json
+  mines!
+  minitest
+  time_series!
+  tweetstream
+  twitter

data/examples/hashtags/README.md

@@ -0,0 +1,22 @@
+# Example Hashtags app
+
+To create the basic directory structure and miners use the following commands
+
+Create the app:  
+<tt>mines new hashtags</tt>
+
+Create the twitter miner:  
+<tt>mines generate twitter publicstream</tt>
+
+Create the process miner:  
+<tt>mines generate process process</tt>
+
+Create the metrics miner:  
+<tt>mines generate metrics metrics</tt>
+
+Resolve project's dependencies with bundler:  
+<tt>bundle install</tt>
+
+Each miner is independent from the others,
+they use Message Queues to communicate and a [TimeSeries metrics store](https://github.com/panayiotis/time_series)
+to share data.

data/examples/hashtags/config/application.rb

@@ -0,0 +1,30 @@
+# coding: utf-8
+## Configuration file for your app
+
+## Declare the configuration parameters you want to use
+Application.config do
+  parameter :var1
+  parameter :var2
+  parameter :var3
+  parameter :logname
+  parameter :app_prefix  
+end
+
+## Change their values
+Application.config do
+  var1  "value"
+  var2  ['one','two','three']
+  var3  { :one=>1, :two=>2 }
+  logname  "hashtags.log"
+  app_prefix "app"
+end
+
+
+## Initializer
+## e.g. initialize a global variable
+## $app_prefix variable needs to be global
+$app_prefix = Application.app_prefix
+
+
+## Usage example
+# puts Application.var1

data/examples/hashtags/miners/metrics.rb

@@ -0,0 +1,157 @@
+#!/bin/ruby
+# coding: utf-8
+
+## Gem dependencies are managed by Bundler inside project's Gemfile
+require 'bundler'
+Bundler.require :metrics
+#require 'active_support/core_ext/date/calculations.rb'
+
+## Initialize TimeSeries
+## Data is been pushed constantly to the timeseries by
+## the process miner
+@ts = TimeSeries.new("hashtags")
+@tweets_rate = TimeSeries.new("tweets_rate")
+
+
+## Print strings in columns in the terminal
+## the terminal needs to have the 'tput' command  
+def columnize *args
+ columns = args.size
+ width = Integer(`tput cols`) / columns
+  args.each{|a| print a.to_s.slice(0,width).ljust(width)}
+  puts ""
+end
+
+## Print strings in columns in the terminal
+## the terminal needs to have the 'tput' command  
+def columnize_arrays *args
+ columns = args.size
+ width = Integer(`tput cols`) / columns
+ lines = 0
+ args.each { |a| lines = a.size - 1 if (a.size - 1 > lines)}
+ line = 0
+ lines.times do
+   args.each do |a|
+     unless a[line].nil?
+       print a[line][0].to_s.slice(0,width).ljust(width)
+     else
+       print '-'.ljust(width)
+     end
+   end
+   puts ""
+   line+=1
+ end
+ 
+end
+
+## Print info and results to the terminal
+def refresh_terminal
+  `reset`
+  
+  ## General info
+  puts Time.now.to_s
+  info = @ts.redis.info
+  print "Redis: "
+  puts info["db0"]
+  print "Redis Memory: "
+  puts info["used_memory_human"]
+  
+  ## Timeseries info
+  print "TimeSeries Keys: "
+  puts @ts.keys.size
+  print "TimeSeries resolution: "
+  puts @ts.resolution  
+  puts ""
+  print "Tweets per minute: "
+  puts @tweets_rate.last["all"]
+  
+  
+  ## Print a table about hashtags popularity during the
+  ## last minute
+  width = Integer(`tput cols`)
+  puts "Trending hashtags during the last minute".center(width)
+  puts '-'*width
+  columnize "Hashtag","Last Minute", "Prev Minute","Last Hour", "Prev Hour", "Today", "Yesterday"
+  puts '-'*width
+  last = @ts.last_key.get 
+  previous = @ts.previous_key.get
+  last_hour = @ts.hour(Time.now)
+  previous_hour = @ts.hour(Time.now - 1.hour)
+  today = @ts.day(Date.today)
+  yesterday = @ts.day(Date.yesterday)  
+
+  if @ts.last_key.persistant?
+    last.each do |hashtag,value|
+      if previous.has_key? hashtag
+        previous_value = '%.2f' % previous[hashtag]
+      else
+        previous_value = "-"
+      end
+  
+      if last_hour.has_key? hashtag
+        last_hour_value = '%.2f' % last_hour[hashtag]
+      else
+        last_hour_value = "-"
+      end
+    
+      if previous_hour.has_key? hashtag
+        previous_hour_value = '%.2f' % previous_hour[hashtag]
+      else
+        previous_hour_value = "-"
+      end   
+      
+      if today.has_key? hashtag
+        today_value = '%.2f' % today[hashtag]
+      else
+        today_value = "-"
+      end
+      
+      if yesterday.has_key? hashtag
+        yesterday_value = '%.2f' % yesterday[hashtag]
+      else
+        yesterday_value = "-"
+      end    
+      columnize( hashtag, '%.2f' % value,
+                 previous_value,
+                 last_hour_value,
+                 previous_hour_value,
+                 today_value,
+                 yesterday_value
+               )
+    end
+  else
+    puts "(no hashtags during the last minute)".center(width)
+  end
+  
+  
+  
+  ## Print a table about hashtags popularity during the
+  ## various time periods
+  puts "\n\n"
+  puts "Popular Hashtags".center(width)
+  puts '-'*width
+  columnize "Last Minute", "Prev Minute","Last Hour",
+            "Prev Hour", "Today", "Yesterday"
+  puts '-'*width
+  last = last.sort_by { |k, v| v }.reverse.slice(0,15)
+  previous = previous.sort_by { |k, v| v }.reverse.slice(0,15)
+  last_hour = last_hour.sort_by { |k, v| v }.reverse.slice(0,15)
+  previous_hour = previous_hour.sort_by { |k, v| v }.reverse.slice(0,15)
+  today = today.sort_by { |k, v| v }.reverse.slice(0,15)
+  yesterday = yesterday.sort_by { |k, v| v }.reverse.slice(0,15)
+  
+  columnize_arrays last, previous, last_hour, 
+                   previous_hour, today, yesterday
+end
+
+
+refresh_terminal
+
+## Start Event Machine
+EM.run do
+  
+  ## Periodic execution
+  EM::PeriodicTimer.new(10){
+    refresh_terminal
+  }
+end # end EM