flumify 0.0.1

data/.bundle/config

@@ -0,0 +1,2 @@
+--- 
+BUNDLE_DISABLE_SHARED_GEMS: "1"

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm use  ruby-1.8.7-p302@flumify
+

data/Gemfile

@@ -0,0 +1,13 @@
+source :gemcutter
+source "http://gems.trafficbroker.co.uk"
+
+# Specify your gem's dependencies in ace.gemspec
+gemspec
+
+group :test do
+  gem 'rspec', '2.5.0'
+end
+
+group :development do
+  gem "fwd", "0.2.0"
+end

data/Gemfile.lock

@@ -0,0 +1,29 @@
+PATH
+  remote: .
+  specs:
+    flumify (0.0.1)
+      net-ssh (~> 2.1.4)
+
+GEM
+  remote: http://rubygems.org/
+  remote: http://gems.trafficbroker.co.uk/
+  specs:
+    diff-lcs (1.1.2)
+    fwd (0.2.0)
+    net-ssh (2.1.4)
+    rspec (2.5.0)
+      rspec-core (~> 2.5.0)
+      rspec-expectations (~> 2.5.0)
+      rspec-mocks (~> 2.5.0)
+    rspec-core (2.5.2)
+    rspec-expectations (2.5.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  flumify!
+  fwd (= 0.2.0)
+  rspec (= 2.5.0)

data/example/decommission.rb

@@ -0,0 +1,10 @@
+require "rubygems"
+require File.join(File.dirname(__FILE__), %w[.. lib flume_master])
+
+logical_nodes = ["a","b","c","d", "e","f"]
+logical_nodes = logical_nodes.map {|node| "#{node}.my.machine.ip"}
+
+master = FlumeMaster.new(:host => "flume.master", :username => "user", :password => "password")
+
+
+master.decommission(logical_nodes)

data/example/example.rb

@@ -0,0 +1,19 @@
+require "rubygems"
+require File.join(File.dirname(__FILE__), %w[.. lib flume_master])
+require File.join(File.dirname(__FILE__), %w[.. lib flumify])
+
+
+config = Flumify.new("machine-to-configure") do 
+  internal_collector :host => "internal-collector", :port => 35853
+  external_collector :host => "external-collector", :port => 35853
+
+  logical_node "access-logs", {:tail => "/mnt/logs/*.log", :bucket => "logs"}
+  logical_node "advert-clicks", {:tail => "/mnt/logs/clicks.log", :bucket => "clicks"}
+
+end
+  
+master = FlumeMaster.new(:host => "flume.master", :username => "user", :password => "password")
+master.apply(config)
+
+
+

data/flumify.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "flumify/version"
+
+Gem::Specification.new do |s|
+  s.name        = "flumify"
+  s.version     = Flumify::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ryan Greenhall", "Tom Hall"]
+  s.email       = ["ryan.greenhall@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Simple flume configuration}
+  s.description = %q{}
+
+  s.rubyforge_project = "flumify"
+  
+  s.add_dependency("net-ssh", ["~> 2.1.4"])
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/flume_master.rb

@@ -0,0 +1,41 @@
+require "rubygems"
+require 'net/ssh'
+require File.join(File.dirname(__FILE__), "ssh_connection")
+
+class FlumeMaster
+  def initialize(opts)
+    @connection = SSHConnection.new({ :host => opts[:host], 
+                                       :username => opts[:username], 
+                                       :password => opts[:password]})
+    @connection.debug = true if opts[:debug] == true
+  end
+  
+  def apply(config)
+    config.logical_nodes.each do |logical_node_command|
+      execute logical_node_command
+    end
+    
+    config.node_configurations.each do |config|
+      config = config.gsub("\"", "\\\"")
+      execute "flume shell -c localhost -e \"exec multiconfig '#{config}'\""
+    end
+  end
+  
+  def decommission(nodes)
+    nodes.each do |node|
+      execute "flume shell -c localhost -e 'exec decommission #{node}'"
+      execute "flume shell -c localhost -e 'exec purge #{node}'"
+    end
+  end
+  
+  private
+  
+  def execute(command)
+    3.times do
+      response = @connection.send_command command
+      puts response
+      break if response.include? "Command succeeded"
+      sleep 3
+    end
+  end
+end

data/lib/flumify/version.rb

@@ -0,0 +1,3 @@
+module Flumify
+  VERSION = "0.0.1"
+end

data/lib/flumify.rb

@@ -0,0 +1,35 @@
+require File.join(File.dirname(__FILE__), "ssh_connection")
+require File.join(File.dirname(__FILE__), "flume_master")
+
+class Flumify 
+  
+  attr_accessor :the_internal_collector, :the_external_collector, :logical_nodes, :node_configurations
+  
+  def initialize(server, &blk)
+    @server = server
+    @logical_nodes = []
+    @node_configurations = []
+    instance_eval(&blk) if blk
+  end
+  
+  def internal_collector(collector)
+    @the_internal_collector = collector
+  end
+  
+  def external_collector(collector)
+    @the_external_collector = collector
+  end
+  
+  def logical_node(name, opts)
+    @logical_nodes << "flume shell -c localhost -e 'exec spawn #{@server} #{name}.#{@server}'"
+
+    if opts[:tail].include? "*"
+      split = opts[:tail].split("*")
+      directory = split[0]
+      glob_pattern = ".*" + split[1]
+      @node_configurations << "#{name}.#{@server} : tailDir( \"#{directory}\", \"#{glob_pattern}\" ) | { value(\"bucket\", \"#{opts[:bucket]}\") => agentBEChain(\"#{@the_internal_collector[:host]}:#{@the_internal_collector[:port]}\",\"#{@the_external_collector[:host]}:#{@the_external_collector[:port]}\") };"   
+    else
+      @node_configurations << "#{name}.#{@server} : tail( \"#{opts[:tail]}\", true ) | { value(\"bucket\", \"#{opts[:bucket]}\") => agentBEChain(\"#{@the_internal_collector[:host]}:#{@the_internal_collector[:port]}\",\"#{@the_external_collector[:host]}:#{@the_external_collector[:port]}\") };"
+    end
+  end
+end

data/lib/ssh_connection.rb

@@ -0,0 +1,15 @@
+class SSHConnection
+  attr_accessor :debug
+  
+  def initialize(config)    
+    @connection = Net::SSH.start(config[:host], config[:username], :password => config[:password])
+  end
+  
+  def send_command(commandstr)
+    if !@debug
+      @connection.exec! commandstr
+    else
+      puts "command: #{commandstr}"
+    end
+  end
+end

data/rakefile

@@ -0,0 +1,12 @@
+require 'rspec/core/rake_task'
+require 'bundler'
+
+Bundler::GemHelper.install_tasks
+
+desc 'Default: run specs.'
+task :default => :spec
+
+desc "Run specs"
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = "./spec/**/*_spec.rb" 
+end

data/spec/flumify_spec.rb

@@ -0,0 +1,68 @@
+require File.join(File.dirname(__FILE__), "../lib/flumify")
+
+describe "Flumify" do
+  
+  it "allows collectors to be specified" do
+    config = Flumify.new("ec2.us-west-1a") do 
+      internal_collector "ec2-foo"
+      external_collector "ec2-bar"
+
+      logical_node "query", {:tail => "/my/file", :bucket => "ask-query"}
+      logical_node "adclick", {:tail => "/my/file", :bucket => "ask-query"}
+      logical_node "redirect", {:tail => "/my/file", :bucket => "ask-query"}
+      logical_node "query", {:tail => "/my/file", :bucket => "ask-query"}
+    end
+    
+    config.the_internal_collector.should == "ec2-foo"
+    config.the_external_collector.should == "ec2-bar"
+  end
+  
+  it "allows logical nodes to be specified" do
+    config = Flumify.new("ec2.us-west-1a") do 
+      internal_collector "ec2-foo"
+      external_collector "ec2-bar"
+
+      logical_node "query", {:tail => "/my/file", :bucket => "ask-query"}
+    end
+    
+    config.logical_nodes.should include "flume shell -c localhost -e 'exec spawn ec2.us-west-1a query.ec2.us-west-1a'"
+  end
+  
+  it "creates node configuration" do
+    config = Flumify.new("ec2.us-west-1a") do 
+      internal_collector :host => "ec2-foo", :port => 35853
+      external_collector :host => "ec2-bar", :port => 35853
+
+      logical_node "query", {:tail => "/path/to/query.log", :bucket => "ask-query"}
+    end
+    
+    config.node_configurations.should include "query.ec2.us-west-1a : tail( \"/path/to/query.log\", true ) | { value(\"bucket\", \"ask-query\") => agentBEChain(\"ec2-foo:35853\",\"ec2-bar:35853\") };"
+    config.node_configurations.length.should == 1
+  end
+  
+  it "can create multiple node configurations" do
+    config = Flumify.new("ec2.us-west-1a") do 
+      internal_collector :host => "ec2-foo", :port => 35853
+      external_collector :host => "ec2-bar", :port => 35853
+
+      logical_node "query", {:tail => "/path/to/query.log", :bucket => "ask-query"}
+      logical_node "click", {:tail => "/path/to/click.log", :bucket => "ask-click"}
+    end
+
+    config.node_configurations.should include "query.ec2.us-west-1a : tail( \"/path/to/query.log\", true ) | { value(\"bucket\", \"ask-query\") => agentBEChain(\"ec2-foo:35853\",\"ec2-bar:35853\") };"
+    config.node_configurations.should include "click.ec2.us-west-1a : tail( \"/path/to/click.log\", true ) | { value(\"bucket\", \"ask-click\") => agentBEChain(\"ec2-foo:35853\",\"ec2-bar:35853\") };"
+    config.node_configurations.length.should == 2
+  end
+  
+  it "allows multiple log files to be specified" do
+    config = Flumify.new("ec2.us-west-1a") do 
+      internal_collector :host => "ec2-foo", :port => 35853
+      external_collector :host => "ec2-bar", :port => 35853
+
+      logical_node "query", {:tail => "/path/to/*.log", :bucket => "ask-query"}
+    end
+
+     config.node_configurations.should include "query.ec2.us-west-1a : tailDir( \"/path/to/\", \".*.log\" ) | { value(\"bucket\", \"ask-query\") => agentBEChain(\"ec2-foo:35853\",\"ec2-bar:35853\") };"
+     config.node_configurations.length.should == 1
+  end
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification 
+name: flumify
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Ryan Greenhall
+- Tom Hall
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-26 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: net-ssh
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 2
+        - 1
+        - 4
+        version: 2.1.4
+  type: :runtime
+  version_requirements: *id001
+description: ""
+email: 
+- ryan.greenhall@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .bundle/config
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- README
+- config/config.yaml
+- example/decommission.rb
+- example/example.rb
+- flumify.gemspec
+- lib/flume_master.rb
+- lib/flumify.rb
+- lib/flumify/version.rb
+- lib/ssh_connection.rb
+- rakefile
+- spec/flumify_spec.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: flumify
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Simple flume configuration
+test_files: 
+- spec/flumify_spec.rb