floom 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+# gem files
+pkg

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/README.md

@@ -0,0 +1,15 @@
+# Floom
+
+A small set of Ruby tools to make using Apache Flume a little less painful.
+
+## Usage
+
+Floom ships with several command line utilities that shorten lengthy, often-used Flume commands.
+
+`floom oneshot <flume config string>`
+
+Executes a no watchdog, no heartbeat, exit on failure Flume node with the supplied config.
+
+`floom debug <decorator file>`
+
+Executes a oneshot node with a console source, and a console sink decorated with the definition supplied.

data/Rakefile

@@ -0,0 +1,3 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+

data/bin/debug

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+decorator = Pathname(ARGV[0]).realpath rescue nil
+
+help_string = <<HELP
+Usage: floom debug <decorator file>
+
+Launches 'decorator-debug: console | wukongDecorator("/path/to/decorator") console;'
+
+HELP
+
+if decorator.nil?
+  puts help_string
+else
+  flow_name = decorator.basename('.rb').to_s
+  flow      = "#{flow_name}-debug: console | wukongDecorator(\"#{decorator.to_s}\") console;"
+  system('flume', 'node_nowatch', '-1', '-s', '-n', flow_name, '-c', flow)
+end

data/bin/floom

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+floom_command      = Pathname(__FILE__).realpath
+command_dir        = floom_command.dirname
+available_commands = Pathname.glob(command_dir.join('*')).reject{ |cmd| cmd.fnmatch? floom_command.to_s }.map(&:basename).map(&:to_s)
+printable_commands = available_commands.map{ |cmd| "\t" + cmd.to_s }.join("\n")
+command_name       = ARGV.shift
+help_string        = <<HELP
+Usage: floom <command> [options]
+
+Available Commands:
+    #{printable_commands}
+
+Use floom <command> --help for specific options
+HELP
+
+if available_commands.include? command_name
+  exec command_dir.join(command_name).to_s, *ARGV
+else
+  puts help_string
+end

data/bin/oneshot

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+conf        = ARGV.shift
+validations = [ !conf.nil?,
+                conf.match(/:/), 
+                conf.match(/\|/),
+                conf.match(/;/),
+                conf.split(/:/).size > 1, 
+                !conf.match(/-help/),
+                ARGV.size == 0 ]
+
+help_string = <<HELP
+Usage: floom oneshot '<flume config string>'
+
+Tips:
+
+* Make sure you use single quotes around the config string (')
+
+* Make sure your config begins with a logical node name followed by a colon (:)
+
+* If anything inside the config needs to be quoted, use double quotes (")
+
+* Make sure to end the config with a semicolon (;)
+
+* Flume config strings have the following format:
+    'name: source | decorator decorator ... sink;'
+
+Examples
+  * 'terminal: console | console;'
+  * 'writer: console | text("/tmp/flume_output.txt", raw);'
+  * 'fake_data: asciisynth(1000, 100) | stubbornAppend rpcSink("localhost", 33333);'
+
+HELP
+
+if validations.all?
+  name = conf.split(':').first
+  cmd  = "flume node_nowatch -1 -s -n #{name} -c '#{conf}'"
+  system cmd
+else
+  puts help_string
+end

data/bin/source

@@ -0,0 +1,74 @@
+#!/usr/bin/env jruby --1.9
+require 'java'
+require 'socket'
+require 'configliere' ; Settings.use(:commandline)
+
+Settings.define :flume_home, :default     => ENV['FLUME_HOME'],  
+                             :required    => true, 
+                             :description => 'Set to the top-level directory of your Flume install'
+Settings.define :host,       :default     => Socket.gethostname, 
+                             :required    => true, 
+                             :flag        => 'c',
+                             :description => 'The hostname where a Flume rpcSource is running'
+Settings.define :port,       :default     => 33333,              
+                             :required    => true, 
+                             :flag        => 'p',
+                             :description => 'The port where a Flume rpcSource is listening'
+Settings.resolve!
+
+Dir[File.join(Settings.flume_home, '**/*.jar')].each{ |jar| require jar }
+
+java_import 'java.net.URL'
+java_import 'java.net.ConnectException'
+java_import 'org.apache.avro.ipc.HttpTransceiver'
+java_import 'org.apache.avro.ipc.AccountingTransceiver'
+java_import 'org.apache.avro.ipc.specific.SpecificRequestor'
+java_import 'com.cloudera.flume.core.EventImpl'
+java_import 'com.cloudera.flume.handlers.avro.FlumeEventAvroServer'
+java_import 'com.cloudera.flume.handlers.avro.AvroEventConvertUtil'
+java_import 'org.slf4j.LoggerFactory'
+
+class RubyRpcClient
+
+  attr_reader :client, :transport
+  
+  def initialize(config = {})
+    @transport = avro_transport(config[:host], config[:port]) 
+    @client    = create_rpc_connection(transport)
+  end
+
+  def log
+    @log ||= Java::OrgSlf4j::LoggerFactory.getLogger(self.class.to_s)
+  end
+
+  def avro_transport(host, port)
+    url  = Java::JavaNet::URL.new("http://#{host}:#{port}")
+    http = Java::OrgApacheAvroIpc::HttpTransceiver.new(url)
+    Java::OrgApacheAvroIpc::AccountingTransceiver.new(http)
+  end
+
+  def create_rpc_connection(transport)
+    client_klass = Java::ComClouderaFlumeHandlersAvro::FlumeEventAvroServer.java_class
+    Java::OrgApacheAvroIpcSpecific::SpecificRequestor.getClient(client_klass, transport)
+  end
+
+  def avro_event(str)
+    event = Java::ComClouderaFlumeCore::EventImpl.new(str.to_java_bytes)
+    Java::ComClouderaFlumeHandlersAvro::AvroEventConvertUtil.toAvroEvent(event)
+  end
+
+  def run!
+    while line = $stdin.readline.chomp rescue nil
+      begin
+        client.append avro_event(line)
+      rescue Exception => e
+        log.error("Error while appending event: #{line}")
+        log.error(e.message)
+        exit(1)
+      end
+    end
+  end
+
+end
+
+RubyRpcClient.new(Settings).run!

data/floom.gemspec

@@ -0,0 +1,23 @@
+$:.unshift File.expand_path('../lib', __FILE__)
+require 'floom/version'
+
+Gem::Specification.new do |s|
+
+  s.name          = 'floom'
+  s.version       = Floom::VERSION
+  s.authors       = ['Travis Dempsey']
+  s.email         = 'travis@infochimps.org'
+  s.homepage      = 'https://github.com/kornypoet/floom.git'
+  s.summary       = 'Simple Thrift class extensions for Flume'  
+  s.description   = <<DESC
+
+Flume extra fun: Floom
+
+DESC
+
+  s.files         = `git ls-files`.split("\n")
+  s.executables   = 'floom'
+  s.require_paths = ['lib']
+  s.add_dependency('thrift', '>= 0.8.0')
+  
+end

data/lib/floom.rb

@@ -0,0 +1,18 @@
+$:.unshift File.expand_path('../thrift', __FILE__)
+
+require 'thrift/flume_master_admin_server'
+require 'thrift/flume_constants'
+require 'thrift/flumeconfig_constants'
+require 'thrift/flumereportserver_constants'
+require 'thrift/mastercontrol_constants'
+require 'thrift/thrift_flume_client_server'
+require 'thrift/thrift_flume_event_server'
+require 'thrift/thrift_flume_report_server'
+
+require 'floom/models/configuration'
+require 'floom/models/status'
+require 'floom/models/report'
+require 'floom/models/request'
+require 'floom/client'
+require 'floom/client/master'
+require 'floom/client/reporter'

data/lib/floom/client.rb

@@ -0,0 +1,33 @@
+module Floom
+  class Client
+
+    attr_reader :connection
+
+    def initialize(options = {})
+      @host = options[:host]
+      @port = options[:port]
+      establish_connection!
+    end
+    
+    def to_s
+      "#<#{self.class}:#{object_id} host:#{@host} port:#{@port}>"
+    end
+
+    def reset_connection!
+      @socket = @transport = @protocol = @connection = nil
+      establish_connection!
+    end
+
+  private
+    
+    def establish_connection!
+      @socket     ||= Thrift::Socket.new(@host, @port)
+      @transport  ||= Thrift::BufferedTransport.new(@socket)
+      @protocol   ||= Thrift::BinaryProtocol.new(@transport)      
+      @connection ||= self.class.thrift_class.new(@protocol)
+      @transport.open
+      self
+    end
+
+  end
+end

data/lib/floom/client/master.rb

@@ -0,0 +1,106 @@
+module Floom
+  class Master < Floom::Client
+    
+    def self.thrift_class() FlumeMasterAdminServer::Client end
+    
+    def configurations
+      connection.getConfigs.inject({}){ |hsh, (node, conf)| hsh[node] = Floom::Configuration.parse(conf) ; hsh }
+    end
+
+    def mappings(physical_node = nil)
+      connection.getMappings(physical_node)
+    end
+    
+    def statuses
+      connection.getNodeStatuses.inject({}){ |hsh, (node, stat)| hsh[node] = Floom::Status.parse(stat) ; hsh }
+    end
+
+    def logical_nodes
+      (configurations.keys + mappings.values + statuses.keys).flatten.uniq
+    end
+
+    def physical_nodes
+      mappings.keys
+    end
+
+    def mapped? logical_node
+      mappings.values.include? logical_node
+    end
+
+    def configured? logical_node
+      configurations.keys.include? logical_node
+    end
+
+    def has_status? logical_node
+      statuses.keys.include? logical_node
+    end
+
+    def perform_request(*params)
+      req = Floom::Request.new(connection, *params).fetch
+      req.parse
+    end    
+
+    def map(physical_node, logical_node)
+      perform_request(:map, physical_node, logical_node)
+    end
+    
+    def unmap(*logical_nodes)
+      logical_nodes.map do |logical_node|
+        physical_node = mappings.detect(->{ mappings.keys }){ |name, nodes| nodes.include? logical_node }.first
+        perform_request(:unmap, physical_node, logical_node)
+      end
+    end
+
+    def unmap_all
+      unmap(*logical_nodes.select{ |node| mapped? node })
+    end
+
+    def decommission(*logical_nodes)
+      logical_nodes.map do |logical_node|
+        perform_request(:decommission, logical_node)
+      end      
+    end
+
+    def decommission_all
+      decommission(*logical_nodes.select{ |node| mapped?(node) or configured?(node) })
+    end
+
+    def purge(*logical_nodes)
+      logical_nodes.map do |logical_node|
+        perform_request(:purge, logical_node)
+      end
+    end
+
+    def purge_all
+      purge(*logical_nodes.select{ |node| has_status? node })
+    end
+
+    def refresh(*logical_nodes)
+      logical_nodes.map do |logical_node|
+        perform_request(:refresh, logical_node)
+      end      
+    end
+    
+    def refresh_all
+      refresh(*logical_nodes)
+    end
+
+    def configure(spec = {})
+      spec.map do |logical_node, conf|
+        params = case conf
+                 when Floom::Configuration  then conf.to_params
+                 when ThriftFlumeConfigData then Floom::Configuration.parse(conf).to_params
+                 when Hash                  then Floom::Configuration.create(conf).to_params
+                 when Array then conf
+                 end          
+        perform_request(:config, logical_node.to_s, *params)
+      end
+    end
+    
+    def unconfigure(*logical_nodes)
+      logical_nodes.map do |logical_node|
+        perform_request(:unconfig, logical_node)
+      end            
+    end        
+  end
+end

data/lib/floom/client/reporter.rb

@@ -0,0 +1,64 @@
+module Floom
+  class Reporter < Client
+
+    def self.thrift_class() ThriftFlumeReportServer::Client ; end
+
+    def reports
+      connection.getAllReports.inject({}){ |hsh, (name, report)| hsh[name] = Floom::Report.parse(report) ; hsh }
+    end
+
+    def extract_master(metrics)
+      master_key = metrics.keys.detect{ |key| key =~ /^flume-master-\d+$/ }
+      master     = metrics.delete(master_key) || {}
+      master.delete('name')
+      { master: master }
+    end
+
+    def extract_system_info(metrics, namespace = 'null')
+      system_info_key = metrics.keys.detect{ |key| key =~ /^#{namespace}\.system-info$/ }
+      system_info     = metrics.delete(system_info_key) || {}
+      system_info.delete('name')
+      { system_info: system_info }
+    end
+
+    def extract_jvm_info(metrics, namespace = 'null')
+      jvm_info_key = metrics.keys.detect{ |key| key =~ /^#{namespace}\.jvm-Info$/ }
+      jvm_info     = metrics.delete(jvm_info_key) || {} 
+      jvm_info.delete('name')
+      { jvm_info: jvm_info }
+    end
+
+    def extract_logical_nodes(metrics, physical_node)
+      logical_node_keys = metrics.keys.map{ |key| key.match(/^#{physical_node}\.(?<node_name>[\w-]+)\.(?<metric>[\w-.]+)$/) }.compact
+      logical_nodes  = logical_node_keys.inject({}) do |hsh, node_key|
+        logical_node = node_key[:node_name]
+        metric_key   = node_key[:metric]
+        hsh[logical_node] = {} unless hsh[logical_node]
+        hsh[logical_node].merge!(metric_key => metrics.delete(node_key.to_s))
+        hsh
+      end
+      { logical_nodes: logical_nodes }
+    end
+
+    def extract_physical_nodes(metrics)
+      physical_node_keys = metrics.keys.map{ |key| key.match(/^pn-(?<node_name>[\w-]+)$/) }.compact
+      physical_nodes   = physical_node_keys.inject({}) do |hsh, node_key|
+        node_name      = node_key[:node_name]
+        hsh[node_name] = {} unless hsh[node_name]
+        hsh[node_name].merge!(metrics.delete(node_key.to_s)).
+          merge!(extract_system_info(metrics, node_key.to_s)).
+          merge!(extract_jvm_info(metrics, node_key.to_s)).
+          merge!(extract_logical_nodes(metrics, node_name))
+        hsh
+      end
+      { physical_nodes: physical_nodes }
+    end
+
+    def rehash(metrics = reports)
+      {}.merge!(extract_master metrics).
+        merge!(extract_jvm_info metrics).
+        merge!(extract_system_info metrics).
+        merge!(extract_physical_nodes metrics)
+    end  
+  end
+end

data/lib/floom/models/configuration.rb

@@ -0,0 +1,71 @@
+module Floom
+  class Configuration
+    
+    class << self
+
+      def parse(conf)
+        new(source:         conf.sourceConfig,
+            sink:           conf.sinkConfig, 
+            flow:           conf.flowID,            
+            source_version: conf.sourceVersion,
+            sink_version:   conf.sinkVersion, 
+            timestamp:      conf.timestamp).to_hash
+      end
+
+      def create(options = {}, &blk)
+        config = new(options)
+        config.instance_eval(&blk) if block_given?
+        config
+      end
+
+    end
+
+    def initialize(options = {}) 
+      @source         = options[:source]
+      @sink           = options[:sink]
+      @flow           = options[:flow]
+      @source_version = options[:source_version]
+      @sink_version   = options[:sink_version]
+      @timestamp      = options[:timestamp]
+    end 
+    
+    def timestamp(val = nil)
+      @timestamp = val if val
+      @timestamp
+    end
+
+    def source(val = nil)
+      @source = val if val
+      @source
+    end
+
+    def sink(val = nil)
+      @sink = val if val
+      @sink
+    end
+
+    def flow(val = nil)
+      @flow = val if val
+      @flow
+    end
+
+    def source_version(val = nil)
+      @source_version = val if val
+      @source_version
+    end
+
+    def sink_version(val = nil)
+      @sink_version = val if val
+      @sink_version
+    end    
+
+    def to_hash
+      self.instance_variables.map{ |var| var.to_s.slice(1..-1).to_sym }.inject({}){ |hsh, var| hsh[var] = self.send(var) ; hsh }
+    end
+
+    def to_params
+      [ flow, source, sink ].compact
+    end
+    
+  end
+end