fluent-plugin-mongo 0.7.16 → 0.8.0.rc1

data/test/plugin/out_mongo_tag_mapped.rb

@@ -1,69 +0,0 @@
-require 'test_helper'
-require 'fluent/plugin/out_mongo'
-
-class MongoTagCollectionTest < Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  CONFIG = %[
-    type mongo
-    database fluent
-    tag_mapped
-    remove_tag_prefix should.remove.
-  ]
-
-  def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput) {
-      def start
-        super
-      end
-
-      def shutdown
-        super
-      end
-
-      def operate(collection, records)
-        [format_collection_name(collection), records]
-      end
-
-      def get_or_create_collection(collection_name)
-        collection_name
-      end
-
-      def mongod_version
-        "2.0.0"
-      end
-    }.configure(conf)
-  end
-
-  def test_configure
-    d = create_driver(CONFIG)
-    assert_equal(/^should\.remove\./, d.instance.instance_variable_get(:@remove_tag_prefix))
-  end
-
-  def emit_documents(d)
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({'a' => 1}, time)
-    d.emit({'a' => 2}, time)
-    time
-  end
-
-  def test_write
-    omit('BufferedOutputTestDriver should support emit arguments(chain and key)')
-
-    d = create_driver(CONFIG)
-    d.tag = 'mytag'
-    t = emit_documents(d)
-    mock(d.instance).operate('mytag', [{'a' => 1, d.instance.time_key => Time.at(t)},
-                                       {'a' => 2, d.instance.time_key => Time.at(t)}])
-    d.run
-  end
-
-  def test_remove_prefix_collection
-    d = create_driver(CONFIG)
-    assert_equal('prefix', d.instance.__send__(:format_collection_name, 'should.remove.prefix'))
-    assert_equal('test', d.instance.__send__(:format_collection_name, '..test..'))
-    assert_equal('test.foo', d.instance.__send__(:format_collection_name, '..test.foo.'))
-  end
-end

data/test/test_helper.rb

@@ -1,59 +0,0 @@
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-
-require 'rr'
-require 'test/unit'
-
-if ENV['SIMPLE_COV']
-  require 'simplecov'
-  SimpleCov.start do 
-    add_filter 'test/'
-    add_filter 'pkg/'
-    add_filter 'vendor/'
-  end
-end
-
-require 'test/unit'
-require 'fluent/test'
-
-# for testing
-
-def unused_port
-  s = TCPServer.open(0)
-  port = s.addr[1]
-  s.close
-  port
-end
-
-# for MongoDB
-
-require 'mongo'
-
-MONGO_DB_DB = 'fluent_test'
-MONGO_DB_PATH = File.join(File.dirname(__FILE__), 'plugin', 'data')
-
-module MongoTestHelper
-  def self.cleanup_mongod_env
-    Process.kill "TERM", @@pid
-    Process.waitpid @@pid
-    system("rm -rf #{MONGO_DB_PATH}")
-  end
-
-  def self.setup_mongod
-    system("rm -rf #{MONGO_DB_PATH}")
-    system("mkdir -p #{MONGO_DB_PATH}")
-
-    @@mongod_port = unused_port
-    @@pid = spawn(ENV['mongod'], "--port=#{@@mongod_port}", "--dbpath=#{MONGO_DB_PATH}")
-    sleep 3
-  end
-
-  def self.teardown_mongod
-    Mongo::Connection.new('localhost', @@mongod_port).drop_database(MONGO_DB_DB)
-    cleanup_mongod_env
-  end
-
-  def self.mongod_port
-    @@mongod_port
-  end
-end

data/test/tools/auth_repl_set_manager.rb

@@ -1,14 +0,0 @@
-require File.join((File.expand_path(File.dirname(__FILE__))), 'repl_set_manager')
-
-class AuthReplSetManager < ReplSetManager
-  def initialize(opts={})
-    super(opts)
-
-    @key_path = opts[:key_path] || File.join(File.expand_path(File.dirname(__FILE__)), "keyfile.txt")
-    system("chmod 600 #{@key_path}")
-  end
-
-  def start_cmd(n)
-    super + " --keyFile #{@key_path}"
-  end
-end

data/test/tools/repl_set_manager.rb

@@ -1,420 +0,0 @@
-require 'thread'
-
-STDOUT.sync = true
-
-require 'mongo'
-#unless defined? Mongo
-#  require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
-#end
-
-class ReplSetManager
-
-  attr_accessor :host, :start_port, :ports, :name, :mongods, :tags, :version
-
-  def initialize(opts={})
-    @mongod     = ENV['mongod'] || 'mongod'
-    @start_port = opts[:start_port] || 30000
-    @ports      = []
-    @name       = opts[:name] || 'replica-set-foo'
-    @host       = opts[:host]  || 'localhost'
-    @retries    = opts[:retries] || 30
-    @config     = {"_id" => @name, "members" => []}
-    @durable    = opts.fetch(:durable, false)
-    @smallfiles = opts.fetch(:smallfiles, true)
-    @prealloc   = opts.fetch(:prealloc, false)
-    @path       = File.join(File.expand_path(File.dirname(__FILE__)), "data")
-    @oplog_size = opts.fetch(:oplog_size, 16)
-    @tags = [{"dc" => "ny", "rack" => "a", "db" => "main"},
-             {"dc" => "ny", "rack" => "b", "db" => "main"},
-             {"dc" => "sf", "rack" => "a", "db" => "main"}]
-
-    @arbiter_count   = opts[:arbiter_count]   || 0
-    @secondary_count = opts[:secondary_count] || 2
-    @passive_count   = opts[:passive_count] || 0
-    @primary_count   = 1
-
-    @count = @primary_count + @passive_count + @arbiter_count + @secondary_count
-    if @count > 7
-      raise StandardError, "Cannot create a replica set with #{node_count} nodes. 7 is the max."
-    end
-
-    @mongods   = {}
-    version_string = `#{@mongod} --version`
-    version_string =~ /(\d\.\d\.\d)/
-    @version = $1.split(".").map {|d| d.to_i }
-  end
-
-  def start_set
-    @mongods.keys.each do |node|
-      kill(node)
-    end
-    should_start = true
-    puts "** Starting a replica set with #{@count} nodes"
-
-    n = 0
-    (@primary_count + @secondary_count).times do
-      init_node(n, should_start) do |attrs|
-        if @version[0] >= 2
-          attrs['tags'] = @tags[n % @tags.size]
-        end
-      end
-      n += 1
-    end
-
-    @passive_count.times do
-      init_node(n, should_start) do |attrs|
-        attrs['priority'] = 0
-      end
-      n += 1
-    end
-
-    @arbiter_count.times do
-      init_node(n, should_start) do |attrs|
-        attrs['arbiterOnly'] = true
-      end
-      n += 1
-    end
-
-    initiate
-    ensure_up
-  end
-
-  def cleanup_set
-    @mongods.keys.each do |node|
-      kill(node)
-    end
-    @count.times do |n|
-      system("rm -rf #{@mongods[n]['db_path']}")
-    end
-  end
-
-  def init_node(n, should_start=true)
-    @mongods[n] ||= {}
-    port = @start_port + n
-    @ports << port
-    @mongods[n]['port'] = port
-    @mongods[n]['db_path'] = get_path("rs-#{port}")
-    @mongods[n]['log_path'] = get_path("log-#{port}")
-    @mongods[n]['start'] = start_cmd(n)
-
-    if should_start
-      system("rm -rf #{@mongods[n]['db_path']}")
-      system("mkdir -p #{@mongods[n]['db_path']}")
-      start(n)
-    end
-
-    member = {'_id' => n, 'host' => "#{@host}:#{@mongods[n]['port']}"}
-
-    if block_given?
-      custom_attrs = {}
-      yield custom_attrs
-      member.merge!(custom_attrs)
-      @mongods[n].merge!(custom_attrs)
-    end
-
-    @config['members'] << member
-  end
-
-  def journal_switch
-    if @version[0] >= 2
-      if @durable
-        "--journal"
-      else
-        "--nojournal"
-      end
-    elsif @durable
-      "--journal"
-    end
-  end
-
-  def start_cmd(n)
-    @mongods[n]['start'] = "#{@mongod} --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
-     "--oplogSize #{@oplog_size} #{journal_switch} --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
-    @mongods[n]['start'] += " --dur" if @durable
-    @mongods[n]['start'] += " --smallfiles" if @smallfiles
-    @mongods[n]['start'] += " --noprealloc" unless @prealloc
-    @mongods[n]['start']
-  end
-
-  def remove_secondary_node
-    primary = get_node_with_state(1)
-    con = get_connection(primary)
-    config = con['local']['system.replset'].find_one
-    secondary = get_node_with_state(2)
-    host_port = "#{@host}:#{@mongods[secondary]['port']}"
-    kill(secondary)
-    @mongods.delete(secondary)
-    @config['members'].reject! {|m| m['host'] == host_port}
-    @config['version'] = config['version'] + 1
-
-    begin
-      con['admin'].command({'replSetReconfig' => @config})
-    rescue Mongo::ConnectionFailure
-    end
-
-    con.close
-
-    return secondary
-  end
-
-  def add_node(n=nil, &block)
-    primary = get_node_with_state(1)
-    con = get_connection(primary)
-
-    init_node(n || @mongods.length, &block)
-    config = con['local']['system.replset'].find_one
-    @config['version'] = config['version'] + 1
-
-    # We expect a connection failure on reconfigure here.
-    begin
-      con['admin'].command({'replSetReconfig' => @config})
-    rescue Mongo::ConnectionFailure
-    end
-
-    con.close
-    ensure_up
-  end
-  
-  def add_arbiter
-    add_node do |attrs|
-      attrs['arbiterOnly'] = true
-    end
-  end
-  
-  def wait_for_death(pid)
-    @retries.times do
-      if `ps a | grep mongod`.include?("#{pid}")
-        puts "waiting for mongod @ pid #{pid} to die..."
-        sleep(1)
-      else
-        puts "mongod @ pid #{pid} was killed successfully"
-        return true
-      end
-    end
-    puts "mongod never died"
-    return false
-  end
-
-  def kill(node, signal=2)
-    return unless @mongods[node]['up']
-
-    pid = @mongods[node]['pid']
-    puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
-    Process.kill signal, pid
-    dead = wait_for_death(pid)
-    @mongods[node]['up'] = false if dead
-  end
-
-  def kill_primary(signal=2)
-    node = get_node_with_state(1)
-    kill(node, signal)
-    return node
-  end
-
-  # Note that we have to rescue a connection failure
-  # when we run the StepDown command because that
-  # command will close the connection.
-  def step_down_primary
-    primary = get_node_with_state(1)
-    con = get_connection(primary)
-    begin
-      con['admin'].command({'replSetStepDown' => 90})
-    rescue Mongo::ConnectionFailure
-    end
-    con.close
-  end
-
-  def kill_secondary
-    node = get_node_with_state(2)
-    kill(node)
-    return node
-  end
-
-  def kill_all_secondaries
-    nodes = get_all_nodes_with_state(2)
-    if nodes
-      nodes.each do |n|
-        kill(n)
-      end
-    end
-  end
-
-  def restart_killed_nodes
-    nodes = @mongods.keys.select do |key|
-      @mongods[key]['up'] == false
-    end
-
-    nodes.each do |node|
-      start(node)
-    end
-
-    ensure_up
-  end
-
-  def get_node_from_port(port)
-    @mongods.keys.detect { |key| @mongods[key]['port'] == port }
-  end
-
-  def start(node)
-    system(@mongods[node]['start'])
-    @mongods[node]['up'] = true
-    sleep(0.5)
-    @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip.to_i
-  end
-  alias :restart :start
-
-  def ensure_up(n=nil, connection=nil)
-    print "** Ensuring members are up..."
-
-    attempt(n) do
-      print "."
-      con = connection || get_connection
-      begin
-        status = con['admin'].command({:replSetGetStatus => 1})
-      rescue Mongo::OperationFailure => ex
-        con.close
-        raise ex
-      end
-      if status['members'].all? { |m| m['health'] == 1 &&
-        [1, 2, 7].include?(m['state']) } &&
-        status['members'].any? { |m| m['state'] == 1 }
-
-        connections = []
-        states      = []
-        status['members'].each do |member|
-        begin
-          host, port = member['name'].split(':')
-          port = port.to_i
-          conn = Mongo::Connection.new(host, port, :slave_ok => true)
-          connections << conn
-          state = conn['admin'].command({:ismaster => 1})
-          states << state
-        rescue Mongo::ConnectionFailure
-          connections.each {|c| c.close }
-          con.close
-          raise Mongo::OperationFailure
-        end
-      end
-
-      if states.any? {|s| s['ismaster']}
-        print "all members up!\n\n"
-        connections.each {|c| c.close }
-        con.close
-        return status
-      else
-        con.close
-        raise Mongo::OperationFailure
-      end
-      else
-        con.close
-        raise Mongo::OperationFailure
-      end
-    end
-    return false
-  end
-
-  def primary
-    nodes = get_all_host_pairs_with_state(1)
-    nodes.empty? ? nil : nodes[0]
-  end
-
-  def secondaries
-    get_all_host_pairs_with_state(2)
-  end
-
-  def arbiters
-    get_all_host_pairs_with_state(7)
-  end
-
-  # String used for adding a shard via mongos
-  # using the addshard command.
-  def shard_string
-    str = "#{@name}/"
-    str << @mongods.map do |k, mongod|
-      "#{@host}:#{mongod['port']}"
-    end.join(',')
-    str
-  end
-
-  private
-
-  def initiate
-    puts "Initiating replica set..."
-    con = get_connection
-
-    attempt do
-      con.object_id
-      con['admin'].command({'replSetInitiate' => @config})
-    end
-
-    con.close
-  end
-
-  def get_all_nodes_with_state(state)
-    status = ensure_up
-    nodes = status['members'].select {|m| m['state'] == state}
-    nodes = nodes.map do |node|
-      host_port = node['name'].split(':')
-      port = host_port[1] ? host_port[1].to_i : 27017
-      @mongods.keys.detect {|key| @mongods[key]['port'] == port}
-    end
-
-    nodes == [] ? false : nodes
-  end
-
-  def get_node_with_state(state)
-    status = ensure_up
-    node = status['members'].detect {|m| m['state'] == state}
-    if node
-      host_port = node['name'].split(':')
-      port = host_port[1] ? host_port[1].to_i : 27017
-      key = @mongods.keys.detect {|n| @mongods[n]['port'] == port}
-      return key
-    else
-      return false
-    end
-  end
-
-  def get_all_host_pairs_with_state(state)
-    status = ensure_up
-    nodes = status['members'].select {|m| m['state'] == state}
-    nodes.map do |node|
-      host_port = node['name'].split(':')
-      port = host_port[1] ? host_port[1].to_i : 27017
-      [host, port]
-    end
-  end
-
-  def get_connection(node=nil)
-    con = attempt do
-      if !node
-        node = @mongods.keys.detect {|key| !@mongods[key]['arbiterOnly'] && @mongods[key]['up'] }
-      end
-      con = Mongo::Connection.new(@host, @mongods[node]['port'], :slave_ok => true)
-    end
-
-    return con
-  end
-
-  def get_path(name)
-    File.join(@path, name)
-  end
-
-  def attempt(retries=nil)
-    raise "No block given!" unless block_given?
-    count = 0
-
-    while count < (retries || @retries) do
-      begin
-        return yield
-        rescue Mongo::OperationFailure, Mongo::ConnectionFailure => ex
-          sleep(2)
-          count += 1
-      end
-    end
-
-    puts "NO MORE ATTEMPTS"
-    raise ex
-  end
-
-end