mines 0.0.1.alpha → 0.0.1

data/bin/mines

@@ -2,16 +2,69 @@
 # coding: utf-8
 
 require 'thor'
-require_relative '../lib/generators/application'
+require_relative '../lib/utilities'
+require_relative '../lib/logging'
+require_relative '../lib/mines'
+
+# require all generators in lib
+mines_root = File.expand_path(File.dirname(__FILE__)+'/..')
+Dir[mines_root+'/lib/generators/*.rb'].each {|file| require file }
+
 module Mines
+  
+  # The mines executable. It subclasses Thor and
+  # invocates generators according to parameters
   class Cli < Thor
+    
+    include Logging
+    
+    def initialize *args
+      super
+    end
+    
+    
     desc "new APP_NAME", "Create Application APP_NAME"
+    # Define the 'new' action. Invocate the new generator
+    # @param name The name of the new application
     def new(name)
-      puts "Create App #{name}"
       ARGV.shift
+      log.debug "command 'new' with arguments: #{ARGV.join(', ')}"
       Generator::Application.start ARGV
     end
+    
+    # help
+    def help *args
+       if args.empty?
+         log.debug "command 'help' with no arguments"
+       else
+         log.debug "command 'help' with arguments: #{args.join(', ').to_s}"
+       end
+       puts "Summary:"
+       super
+       puts "Now print my help :)\nYeah!"
+    end
+    
+    # generate
+    desc "generate [twitter|network|process|metrics] NAME", "Create a twitter or network or process or metrics miner"
+    #method_option :evented, :type => :boolean, :default => false
+    # generate a miner
+    def generate(type,name)
+      Mines.mines_app_check #TODO Test
+      types = %w{twitter network process metrics}
+      unless types.include? type
+        puts "The type of the miner could be either twitter or network or process or metrics"
+        puts "you typed: #{type}"
+        exit 0
+      end
+      puts "Generate a #{type} miner with name: #{name}"
+      
+      Generator::Miner.start [type, name]
+    end
   end
 end
 
+# Check if 'mines' is invoked in the root directory of a Mines App
+#Mines.mines_app_check #TODO This code can not be here
+
+# Start client
 Mines::Cli.start ARGV

data/lib/application.rb

@@ -0,0 +1,40 @@
+# coding: utf-8
+
+# This module is providing the configuration DSL
+module Application
+  # we don't want to instantiate this class - it's a singleton,
+  # so just keep it as a self-extended module
+  extend self
+
+  # Appdata provides a basic single-method DSL with .parameter method
+  # being used to define a set of available settings.
+  # This method takes one or more symbols, with each one being
+  # a name of the configuration option.
+  def parameter(*names)
+    names.each do |name|
+      attr_accessor name
+
+      # For each given symbol we generate accessor method that sets option's
+      # value being called with an argument, or returns option's current value
+      # when called without arguments
+      define_method name do |*values|
+        value = values.first
+        value ? self.send("#{name}=", value) : instance_variable_get("@#{name}")
+      end
+    end
+  end
+
+  # And we define a wrapper for the configuration block, that we'll use to set up
+  # our set of options
+  def config(&block)
+    instance_eval &block
+  end
+end
+
+require_relative '../config/application'
+
+begin
+  require './config/application'
+rescue
+  puts red "Warning: Configuration file ./config/application.rb is missing"
+end

data/lib/configuration.rb

@@ -0,0 +1,43 @@
+# coding: utf-8
+
+# This module is providing the configuration DSL
+module Application
+  # we don't want to instantiate this class - it's a singleton,
+  # so just keep it as a self-extended module
+  extend self
+
+  # Appdata provides a basic single-method DSL with .parameter method
+  # being used to define a set of available settings.
+  # This method takes one or more symbols, with each one being
+  # a name of the configuration option.
+  def parameter(*names)
+    names.each do |name|
+      attr_accessor name
+
+      # For each given symbol we generate accessor method that sets option's
+      # value being called with an argument, or returns option's current value
+      # when called without arguments
+      define_method name do |*values|
+        value = values.first
+        value ? self.send("#{name}=", value) : instance_variable_get("@#{name}")
+      end
+    end
+  end
+
+  # And we define a wrapper for the configuration block, that we'll use to set up
+  # our set of options
+  def config(&block)
+    instance_eval &block
+  end
+end
+
+
+# Load default configuration
+require_relative '../config/application'
+
+# Load user's configuration
+begin
+  require './config/application'
+rescue Exception
+  puts red "Warning: Configuration file ./config/application.rb is missing"
+end

data/lib/generators/application.rb

@@ -1,25 +1,49 @@
 # coding: utf-8
 
 require 'thor'
-module Generator
+require 'colored'
+require_relative '../logging'
+
+# The Generator module is the namespace of all generators
+# Namely:
+# - Application
+# - Miner
+module Mines::Generator
+
+  # Application Generator creates the default file structure.
+  # Accepts as an argument the name of the new application. A new directory with this name will be created
   class Application < Thor::Group
     include Thor::Actions
+    include Mines::Logging
 
-    desc "new APP_NAME"
-    
     # Define arguments and options
     argument :name
-
+    
+    # This is needed by thor. Returns the source root directory.
     def self.source_root
       File.dirname(__FILE__)
-      puts File.dirname(__FILE__)
     end
 
+    # Create the default directory structure
+    # - Directories: miners, lib, log, config
+    # - Files: TODO
+    # also prints appropriate messages
     def create_directory_structure
+      log.info "Creating directory structure in dir: " + name
       empty_directory name
       empty_directory name + "/miners"
       empty_directory name + "/lib"
       empty_directory name + "/log"
+      empty_directory name + "/config"
+      template "templates/application_config.erb", "#{name}/config/application.rb"
+      puts "Application created successfully! ".green
+      print "Type "
+      print "'cd #{name}' ".yellow
+      print "to go to the applications root directory, and then "
+      print "'mines help' ".yellow
+      puts "to see all available commands and options."
     end
-  end
-end
+    
+  end # class
+
+end # module

data/lib/generators/miner.rb

@@ -0,0 +1,38 @@
+# coding: utf-8
+
+require 'thor'
+require_relative '../logging'
+
+module Mines::Generator
+  
+  # Miner Generator generates files with boilerblate code 
+  # in the miners directory. According to arguments it generates the
+  # following types of miners:
+  # - Network
+  # - Process
+  # - Metrics
+  class Miner < Thor::Group
+    include Thor::Actions
+    include Mines::Logging
+
+    # Define arguments and options
+    argument :type, :desc => "The type of the miner, [Network,Process,Metrics]", :required => true
+    argument :name, :desc => "The name of the miner", :required => true
+    #class_option :evented, :type => :boolean, :default => false, :lazy_default => true
+    
+    # Define the source template root
+    def self.source_root
+      File.dirname(__FILE__)
+    end
+    
+    # Use the appropriate template file according to type
+    # and put the generated file in miners directory
+    def create_miner_file
+      puts "Copy miner template"
+      #puts options[:evented]
+      template "templates/#{type}_miner.erb", "miners/#{name}.rb"
+    end
+    
+  end # class
+
+end # module

data/lib/generators/templates/application_config.erb

@@ -0,0 +1,19 @@
+# coding: utf-8
+## Configuration file for your app
+
+## Declare the configuration parameters you want to use
+Application.config do
+  parameter :var1
+  parameter :var2
+  parameter :var3
+end
+
+## Change their values
+Application.config do
+  var1  "value"
+  var2  ['one','two','three']
+  var3  { :one=>1, :two=>2 }
+end
+
+## Usage example
+# puts Application.var1

data/lib/generators/templates/process_miner.erb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require 'mines'
+
+log_name :<%= name %>  
+

data/lib/help.rb

@@ -0,0 +1,10 @@
+# coding: utf-8
+
+module Mines::Help
+  class << self
+    attr_accessor :new, :generate
+    def configure
+      yield self
+    end
+  end
+end

data/lib/logging.rb

@@ -0,0 +1,69 @@
+# coding: utf-8
+
+require 'logger'
+
+# Configuration
+require_relative 'configuration'
+
+#require_relative 'application' # needed by config/application
+
+# ./config/application may not have been created yet
+#begin
+#  require './config/application'
+#rescue Exception => e  
+#  require_relative '../config/application'
+#end
+
+unless defined?(Application.logname).nil?
+  @log ||= Logger.new "log/#{Application.logname}"
+else
+  @log ||= Logger.new 'log/mines.log'
+end
+
+@log.datetime_format= "%H:%M:%S"
+@log.formatter = proc do |severity, datetime, progname, msg|
+  "#{severity} #{datetime} #{msg}\n"
+end
+
+def log_name name
+  @log ||=  Logger.new 'log/#{name}.log'
+  @log.datetime_format= "%H:%M:%S"
+  @log.formatter = proc do |severity, datetime, progname, msg|
+    "#{severity} #{datetime} #{msg}\n"
+  end  
+end
+
+=begin
+
+module Mines
+
+  # Use this module by including it in a class,
+  # it defines the log method that instantiates a logger if it does not already exist
+  # Reads the global configuration for options
+  module Logging
+
+    # instantiates a logger if it does not already exist or
+    # it returns the existing logger
+    def log
+      if defined?(@log).nil?
+        log_device = ""
+        if not defined?(Application.log).nil?
+          log_device = Application.log
+        elsif Dir.exists?('log')
+          log_device ='log/mines.log'
+        else
+          log_device = STDOUT
+        end
+        @log = Logger.new log_device
+        @log.datetime_format= "%H:%M:%S"
+        @log.formatter = proc do |severity, datetime, progname, msg|
+          "#{msg}\n".yellow
+        end
+      end
+      @log
+    end # end def
+  end # end module
+end # end module
+
+=end
+

data/lib/mines.rb

@@ -0,0 +1,8 @@
+# coding: utf-8
+
+#require 'bundler/setup'
+require_relative 'redis_store'   # Redis Queues, etc
+require_relative 'utilities'     # Utilities
+require_relative 'configuration' # Configuration
+require_relative 'logging'       # Logging
+

data/lib/redis_store.rb

@@ -0,0 +1,135 @@
+# coding: utf-8
+
+require 'redis'
+require 'hiredis'
+
+def keygen
+  Time.now.strftime("ts:%Y:%m:%d:%H:%M")
+end
+  
+class RedisConnection
+  attr_reader :redis
+  def initialize *args
+    #@key= key
+    $redis ||= Redis.new(:path => "/tmp/redis.sock",:driver => :hiredis)
+    @redis = $redis
+    # puts "Connected to Redis"
+  end # end initialize
+  
+  def counters
+    @redis.hgetall "counters"
+  end
+end
+
+class MessageQueue < RedisConnection
+  attr_reader :key
+  
+  def initialize key
+    @key = key
+    super
+  end
+  
+  def pop()
+    @redis.blpop(@key)[1]
+  end # end pop
+  
+  def push(item)
+    redis.pipelined do
+      @redis.hincrby "counters", @key, 1
+      #@redis.hincrby "ts:counters", keygen+"_count", 1      
+      @redis.rpush(@key, item )
+    end
+  end # end push
+
+  def del
+    @redis.del @key
+  end
+  
+  def size
+    @redis.llen @key
+  end
+
+  def info
+    out = "items in list:".ljust(25) + size.to_s + "\n"
+    out += "number of keys:".ljust(25) + @redis.dbsize.to_s + "\n"
+    %w{ used_memory_human used_memory_peak_human total_commands_processed}.each do |w|
+      out += "#{w.gsub("_", " ")}:".ljust(25)
+      out += @redis.info[w]
+      out += "\n"
+    end
+    return out
+  end
+        
+  def to_s
+    @key
+  end
+    
+end
+
+
+class ObjectQueue < MessageQueue
+  
+  def pop()
+    Marshal.load(@redis.blpop(@key)[1])
+  end # end pop
+  
+  def push(item)
+    redis.pipelined do
+      @redis.hincrby "counters", @key, 1
+      #@redis.incr(@key+"_count")
+      #@redis.incr(keygen+"_count")
+      @redis.rpush(@key, Marshal.dump(item) )
+    end
+  end # end push
+end
+
+
+class Counter < RedisConnection
+  attr_reader :key
+  
+  def initialize key
+    @key = key
+    super  
+  end
+  
+  def incr
+    @redis.hincrby "counters", @key, 1
+  end
+  
+  def del
+    @redis.hdel "counters", @key
+  end
+  
+  def to_s
+    @redis.hget "counters", @key
+  end  
+end
+
+class MessageChannel < RedisConnection
+  attr_reader :channel
+  
+  def initialize channel
+    @channel = channel
+    super
+  end
+  
+  def subscribe &proc
+    @redis.subscribe @channel do |on|
+    on.message &proc
+    end
+  end
+  
+  def publish message
+    redis.pipelined do
+      @redis.hincrby "counters", @channel, 1
+      #@redis.hincrby "ts:counters", keygen+"_count", 1      
+      @redis.publish(@channel, message )
+    end
+  end
+       
+  def to_s
+    @channel
+  end
+    
+end
+

data/lib/utilities.rb

@@ -0,0 +1,111 @@
+# coding: utf-8
+
+require 'colored'
+
+def green(text); "\033[32m#{text}\033[0m" end
+def blue(text); "\033[34m#{text}\033[0m" end
+def red(text); "\033[31m#{text}\033[0m" end
+def yellow(text); "\033[33m#{text}\033[0m" end
+
+module Mines
+  extend self
+
+  # Check if mines is executed inside a mines application directory
+  def in_app_root?
+    # simply check if basic dirs exist
+    %w{ miners lib log config config/application.rb}.each do |file|
+      return false if not File.exists?(file)
+    end
+    return true
+  end
+  
+  # Exit if mines is invokated outside application root
+  def mines_app_check
+    if not in_app_root?
+      print "It seems your current directory: "
+      puts File.expand_path('.').yellow
+      puts "does not contains a Mines application."
+      puts "You can create one by invoking: " + "mines new APPNAME".blue.bold
+      exit 0
+    end
+  end
+end
+
+
+module Mines
+  extend self
+  
+  # Examines if a gem axists
+  #
+  # @param gem [String] the gem's name
+  # @return [Boolean] If gem exists or not
+  def gem_exists? gem
+    ! Gem::Specification.find_all_by_name(gem).empty?
+  end
+
+end
+
+def print_tweet(user, text)
+  width = Integer(`tput cols`) - 18
+  sum = 0
+  print user.center(16).blue.bold
+  text.split.each do |word|
+    if (sum + word.size >= width)
+      puts ""
+      print "".ljust 16
+      sum=0
+    end   
+    sum+=word.size+1
+    if word.include?("http")
+      print word.cyan.underline
+    elsif word.start_with? "@"
+      print word.blue.bold
+    elsif word.start_with? "#"
+      print yellow word
+    else
+      print word
+    end
+    print " "  
+  end
+  puts ""
+end
+
+# TODO Is this still needed?
+def ago_time(time)
+  ago = Time.now - time
+  if  ago < 3600
+    return "#{ago.div(1)} minutes ago"
+  elsif ago < 3600 * 12
+    return "#{ago.div(3600)} hours ago"
+  elsif ago < 3600 * 24
+    return "Yesterday"
+  elsif ago < 3600 * 24 * 7
+    return "#{time.strftime('%A')}"
+  else 
+    return "#{time.strftime('%A %-d %B %Y')}"
+  end       
+end
+
+module Mines
+  module Utilities
+    extend self
+
+    def print_command(command, text)
+      width = Integer(`tput cols`) - 22
+      sum = 0
+      print "  "+command.ljust(20).blue.bold
+      text.split.each do |word|
+        if (sum + word.size >= width)
+          puts ""
+          print "".ljust 22
+          sum=0
+        end   
+        sum+=word.size+1
+        print " "
+        print word  
+      end # end text.split
+      puts ""
+    end # end def
+
+  end # end module
+end # end module

metadata

@@ -1,16 +1,64 @@
 --- !ruby/object:Gem::Specification
 name: mines
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha
-  prerelease: 6
+  version: 0.0.1
+  prerelease: 
 platform: ruby
 authors:
 - Panayiotis Vlantis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-27 00:00:00.000000000 Z
+date: 2013-09-08 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -27,6 +75,70 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: colored
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hiredis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ruby in Mines is a framework for creating data mining application prototypes
   that focus on processing near real-time human generated content.
 email: p.vlantis@di.uoa.gr
@@ -35,9 +147,19 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/logging.rb
+- lib/utilities.rb
+- lib/mines.rb
+- lib/redis_store.rb
+- lib/help.rb
+- lib/application.rb
+- lib/configuration.rb
 - lib/generators/application.rb
+- lib/generators/miner.rb
+- lib/generators/templates/process_miner.erb
+- lib/generators/templates/application_config.erb
 - bin/mines
-homepage: 
+homepage: https://github.com/panayiotis/mines
 licenses:
 - MIT
 post_install_message: 
@@ -54,9 +176,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>'
+  - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25
@@ -64,3 +186,4 @@ signing_key:
 specification_version: 3
 summary: Data mining application framework.
 test_files: []
+has_rdoc: