pager-mogilefs-client 1.2.1.20080519

data/lib/mogilefs/nfsfile.rb

@@ -0,0 +1,81 @@
+require 'mogilefs/backend'
+
+##
+# NFSFile wraps up the new file operations for storing files onto an NFS
+# storage node.
+#
+# You really don't want to create an NFSFile by hand.  Instead you want to
+# create a new file using MogileFS::MogileFS.new_file.
+
+class MogileFS::NFSFile < File
+
+  ##
+  # The path of this file not including the local mount point.
+
+  attr_reader :path
+
+  ##
+  # The key for this file.  This key won't represent a real file until you've
+  # called #close.
+
+  attr_reader :key
+
+  ##
+  # The class of this file.
+
+  attr_reader :class
+
+  class << self
+
+    ##
+    # Wraps up File.new with MogileFS-specific data.  Use
+    # MogileFS::MogileFS#new_file instead of this method.
+
+    def new(mg, fid, path, devid, klass, key)
+      fp = super join(mg.root, path), 'w+'
+      fp.send :setup, mg, fid, path, devid, klass, key
+      return fp
+    end
+
+    ##
+    # Wraps up File.open with MogileFS-specific data.  Use
+    # MogileFS::MogileFS#new_file instead of this method.
+
+    def open(mg, fid, path, devid, klass, key, &block)
+      fp = new mg, fid, path, devid, klass, key
+
+      return fp if block.nil?
+
+      begin
+        yield fp
+      ensure
+        fp.close
+      end
+    end
+
+  end
+
+  ##
+  # Closes the file handle and marks it as closed in MogileFS.
+
+  def close
+    super
+    @mg.backend.create_close(:fid => @fid, :devid => @devid,
+                             :domain => @mg.domain, :key => @key,
+                             :path => @path)
+    return nil
+  end
+
+  private
+
+  def setup(mg, fid, path, devid, klass, key)
+    @mg = mg
+    @fid = fid
+    @path = path
+    @devid = devid
+    @klass = klass
+    @key = key
+  end
+
+end
+

data/lib/mogilefs/pool.rb

@@ -0,0 +1,50 @@
+require 'thread'
+require 'mogilefs'
+
+class MogileFS::Pool
+
+  class BadObjectError < RuntimeError; end
+
+  def initialize(klass, *args)
+    @args = args
+    @klass = klass
+    @queue = Queue.new
+    @objects = []
+  end
+
+  def get
+    begin
+      object = @queue.pop true
+    rescue ThreadError
+      object = @klass.new(*@args)
+      @objects << object
+    end
+    return object
+  end
+
+  def put(o)
+    raise BadObjectError unless @objects.include? o
+    @queue.push o
+    purge
+  end
+
+  def use
+    object = get
+    yield object
+  ensure
+    put object
+  end
+
+  def purge
+    return if @queue.length < 5
+    begin
+      until @queue.length <= 2 do
+        obj = @queue.pop true
+        @objects.delete obj
+      end
+    rescue ThreadError
+    end
+  end
+
+end
+

data/test/setup.rb

@@ -0,0 +1,54 @@
+require 'test/unit'
+
+$TESTING = true
+
+require 'mogilefs'
+
+class FakeBackend
+
+  attr_reader :lasterr, :lasterrstr
+
+  def initialize
+    @responses = Hash.new { |h,k| h[k] = [] }
+    @lasterr = nil
+    @lasterrstr = nil
+  end
+
+  def method_missing(meth, *args)
+    meth = meth.to_s
+    if meth =~ /(.*)=$/ then
+      @responses[$1] << args.first
+    else
+      response = @responses[meth].shift
+      case response
+      when Array then
+        @lasterr = response.first
+        @lasterrstr = response.last
+        return nil
+      end
+      return response
+    end
+  end
+
+end
+
+class MogileFS::Client
+  attr_writer :readonly
+end
+
+class TestMogileFS < Test::Unit::TestCase
+
+  def setup
+    return if self.class == TestMogileFS
+    @root = '/mogilefs/test'
+    @client = @klass.new :hosts => ['kaa:6001'], :domain => 'test',
+                                  :root => @root
+    @backend = FakeBackend.new
+    @client.instance_variable_set '@backend', @backend
+  end
+
+  def test_nothing
+  end
+
+end
+

data/test/test_admin.rb

@@ -0,0 +1,174 @@
+require 'test/setup'
+
+class TestMogileFS__Admin < TestMogileFS
+
+  def setup
+    @klass = MogileFS::Admin
+    super
+  end
+
+  def test_clean
+    res = {"host1_remoteroot"=>"/mnt/mogilefs/rur-1",
+           "host1_hostname"=>"rur-1",
+           "host1_hostid"=>"1",
+           "host1_http_get_port"=>"",
+           "host1_altip"=>"",
+           "hosts"=>"1",
+           "host1_hostip"=>"",
+           "host1_http_port"=>"",
+           "host1_status"=>"alive",
+           "host1_altmask"=>""}
+    actual = @client.clean 'hosts', 'host', res
+  
+    expected = [{"status"=>"alive",
+                 "http_get_port"=>"",
+                 "http_port"=>"",
+                 "hostid"=>"1",
+                 "hostip"=>"",
+                 "hostname"=>"rur-1",
+                 "remoteroot"=>"/mnt/mogilefs/rur-1",
+                 "altip"=>"",
+                 "altmask"=>""}]
+
+    assert_equal expected, actual
+  end
+
+  def test_each_fid
+    @backend.stats = {
+      'fidmax' => '182',
+      'fidcount' => '2',
+    }
+
+    @backend.list_fids = {
+      'fid_count' => '1',
+      'fid_1_fid' => '99',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'file_key',
+      'fid_1_length' => '4',
+    }
+
+    @backend.list_fids = {
+      'fid_count' => '1',
+      'fid_1_fid' => '182',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'new_new_key',
+      'fid_1_length' => '9',
+    }
+
+    fids = []
+    @client.each_fid { |fid| fids << fid }
+
+    expected = [
+      { "fid"      => "99",
+        "class"    => "normal",
+        "domain"   => "test",
+        "devcount" => "2",
+        "length"   => "4",
+        "key"      => "file_key" },
+      { "fid"      => "182",
+        "class"    => "normal",
+        "devcount" => "2",
+        "domain"   => "test",
+        "length"   => "9",
+        "key"      => "new_new_key" },
+    ]
+
+    assert_equal expected, fids
+  end
+
+  def test_get_domains
+    @backend.get_domains = {
+      'domains' => 2,
+      'domain1' => 'test',
+      'domain2' => 'images',
+      'domain1classes' => '1',
+      'domain2classes' => '2',
+      'domain1class1name' => 'default',
+      'domain1class1mindevcount' => '2',
+      'domain2class1name' => 'default',
+      'domain2class1mindevcount' => '2',
+      'domain2class2name' => 'resize',
+      'domain2class2mindevcount' => '1',
+    }
+
+    expected = {
+      'test'   => { 'default' => 2, },
+      'images' => { 'default' => 2, 'resize' => 1 },
+    }
+
+    assert_equal expected, @client.get_domains
+  end
+
+  def disabled_test_get_stats
+    @backend.stats = {}
+
+    expected = {
+      'fids' => { 'max' => '99', 'count' => '2' },
+      'device' => [
+        { 'status' => 'alive', 'files' => '2', 'id' => '1', 'host' => 'rur-1' },
+        { 'status' => 'alive', 'files' => '2', 'id' => '2', 'host' => 'rur-2' }
+      ],
+      'replication' => [
+        { 'files' => '2', 'class' => 'normal', 'devcount' => '2',
+          'domain' => 'test' }
+      ],
+      'file' => [{ 'files' => '2', 'class' => 'normal', 'domain' => 'test' }]
+    }
+
+    assert_equal 
+  end
+
+  def test_get_stats_fids
+    @backend.stats = {
+      'fidmax' => 99,
+      'fidcount' => 2,
+    }
+
+    expected = {
+      'fids' => { 'max' => 99, 'count' => 2 },
+    }
+
+    assert_equal expected, @client.get_stats('all')
+  end
+
+  def test_list_fids
+    @backend.list_fids = {
+      'fid_count' => '2',
+      'fid_1_fid' => '99',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'file_key',
+      'fid_1_length' => '4',
+      'fid_2_fid' => '82',
+      'fid_2_class' => 'normal',
+      'fid_2_devcount' => '2',
+      'fid_2_domain' => 'test',
+      'fid_2_key' => 'new_new_key',
+      'fid_2_length' => '9',
+    }
+
+    expected = [
+      { "fid"      => "99",
+        "class"    => "normal",
+        "domain"   => "test",
+        "devcount" => "2",
+        "length"   => "4",
+        "key"      => "file_key" },
+      { "fid"      => "82",
+        "class"    => "normal",
+        "devcount" => "2",
+        "domain"   => "test",
+        "length"   => "9",
+        "key"      => "new_new_key" },
+    ]
+
+    assert_equal expected, @client.list_fids(0, 100)
+  end
+
+end
+

data/test/test_backend.rb

@@ -0,0 +1,220 @@
+require 'test/unit'
+
+$TESTING = true
+
+require 'mogilefs/backend'
+
+class MogileFS::Backend
+
+  attr_accessor :hosts
+  attr_reader :timeout, :dead
+  attr_writer :lasterr, :lasterrstr, :socket
+
+end
+
+class FakeSocket
+
+  def initialize
+    @closed = false
+  end
+
+  def closed?
+    @closed
+  end
+
+  def close
+    @closed = true
+    return nil
+  end
+
+  def peeraddr
+    ['AF_INET', 6001, 'localhost', '127.0.0.1']
+  end
+
+end
+
+class TestBackend < Test::Unit::TestCase
+
+  def setup
+    @backend = MogileFS::Backend.new :hosts => ['localhost:1']
+  end
+
+  def test_initialize
+    assert_raises ArgumentError do MogileFS::Backend.new end
+    assert_raises ArgumentError do MogileFS::Backend.new :hosts => [] end
+    assert_raises ArgumentError do MogileFS::Backend.new :hosts => [''] end
+
+    assert_equal ['localhost:1'], @backend.hosts
+    assert_equal 3, @backend.timeout
+    assert_equal nil, @backend.lasterr
+    assert_equal nil, @backend.lasterrstr
+    assert_equal({}, @backend.dead)
+
+    @backend = MogileFS::Backend.new :hosts => ['localhost:6001'], :timeout => 1
+    assert_equal 1, @backend.timeout
+  end
+
+  def test_do_request
+    socket_request = ''
+    socket = Object.new
+    def socket.closed?() false end
+    def socket.send(request, flags) return request.length end
+    def @backend.select(*args) return [true] end
+    def socket.gets() return 'OK 1 you=win' end
+
+    @backend.instance_variable_set '@socket', socket
+
+    assert_equal({'you' => 'win'},
+                 @backend.do_request('go!', { 'fight' => 'team fight!' }))
+  end
+
+  def test_do_request_send_error
+    socket_request = ''
+    socket = Object.new
+    def socket.closed?() false end
+    def socket.send(request, flags) raise SystemCallError, 'dummy' end
+
+    @backend.instance_variable_set '@socket', socket
+
+    assert_raises RuntimeError do
+      @backend.do_request 'go!', { 'fight' => 'team fight!' }
+    end
+
+    assert_equal nil, @backend.instance_variable_get('@socket')
+  end
+
+  def test_do_request_truncated
+    socket_request = ''
+    socket = Object.new
+    def socket.closed?() false end
+    def socket.send(request, flags) return request.length - 1 end
+
+    @backend.instance_variable_set '@socket', socket
+
+    assert_raises RuntimeError do
+      @backend.do_request 'go!', { 'fight' => 'team fight!' }
+    end
+  end
+
+  def test_make_request
+    assert_equal "go! fight=team+fight%21\r\n",
+                 @backend.make_request('go!', { 'fight' => 'team fight!' })
+  end
+
+  def test_parse_response
+    assert_equal({'foo' => 'bar', 'baz' => 'hoge'},
+                 @backend.parse_response('OK 1 foo=bar&baz=hoge'))
+    assert_equal nil, @backend.parse_response('ERR you totally suck')
+    assert_equal 'you', @backend.lasterr
+    assert_equal 'totally suck', @backend.lasterrstr
+
+    assert_raises RuntimeError do
+      @backend.parse_response 'garbage'
+    end
+  end
+
+  def test_readable_eh_readable
+    socket = Object.new
+    def socket.closed?() false end
+    def @backend.select(*args) return [true] end
+    @backend.instance_variable_set '@socket', socket
+
+    assert_equal true, @backend.readable?
+  end
+
+  def test_readable_eh_not_readable
+    socket = FakeSocket.new
+    def socket.closed?() false end
+    def @backend.select(*args) return [] end
+    @backend.instance_variable_set '@socket', socket
+
+    begin
+      @backend.readable?
+    rescue MogileFS::UnreadableSocketError => e
+      assert_equal '127.0.0.1:6001 never became readable', e.message
+    rescue Exception
+      flunk "MogileFS::UnreadableSocketError not raised"
+    else
+      flunk "MogileFS::UnreadableSocketError not raised"
+    end
+  end
+
+  def test_socket
+    assert_equal({}, @backend.dead)
+    assert_raises RuntimeError do @backend.socket end
+    assert_equal(['localhost:1'], @backend.dead.keys)
+  end
+
+  def test_socket_robust
+    @backend.hosts = ['localhost:6001', 'localhost:6002']
+    def @backend.connect_to(host, port)
+      @first = (defined? @first) ? false : true
+      raise Errno::ECONNREFUSED if @first
+    end
+
+    assert_equal({}, @backend.dead)
+    @backend.socket
+    assert_equal false, @backend.dead.keys.empty?
+  end
+
+  def test_shutdown
+    fake_socket = FakeSocket.new
+    @backend.socket = fake_socket
+    assert_equal fake_socket, @backend.socket
+    @backend.shutdown
+    assert_equal nil, @backend.instance_variable_get(:@socket)
+  end
+
+  def test_url_decode
+    assert_equal({"\272z" => "\360opy", "f\000" => "\272r"},
+                 @backend.url_decode("%baz=%f0opy&f%00=%bar"))
+  end
+
+  def test_url_encode
+    params = [["f\000", "\272r"], ["\272z", "\360opy"]]
+    assert_equal "f%00=%bar&%baz=%f0opy", @backend.url_encode(params)
+  end
+
+  def test_url_escape # \n for unit_diff
+    actual = (0..255).map { |c| @backend.url_escape c.chr }.join "\n"
+
+    expected = []
+    expected.push(*(0..0x1f).map { |c| "%%%0.2x" % c })
+    expected << '+'
+    expected.push(*(0x21..0x2b).map { |c| "%%%0.2x" % c })
+    expected.push(*%w[, - . /])
+    expected.push(*('0'..'9'))
+    expected.push(*%w[: %3b %3c %3d %3e %3f %40])
+    expected.push(*('A'..'Z'))
+    expected.push(*%w[%5b \\ %5d %5e _ %60])
+    expected.push(*('a'..'z'))
+    expected.push(*(0x7b..0xff).map { |c| "%%%0.2x" % c })
+
+    expected = expected.join "\n"
+
+    assert_equal expected, actual
+  end
+
+  def test_url_unescape
+    input = []
+    input.push(*(0..0x1f).map { |c| "%%%0.2x" % c })
+    input << '+'
+    input.push(*(0x21..0x2b).map { |c| "%%%0.2x" % c })
+    input.push(*%w[, - . /])
+    input.push(*('0'..'9'))
+    input.push(*%w[: %3b %3c %3d %3e %3f %40])
+    input.push(*('A'..'Z'))
+    input.push(*%w[%5b \\ %5d %5e _ %60])
+    input.push(*('a'..'z'))
+    input.push(*(0x7b..0xff).map { |c| "%%%0.2x" % c })
+
+    actual = input.map { |c| @backend.url_unescape c }.join "\n"
+
+    expected = (0..255).map { |c| c.chr }.join "\n"
+    expected.sub! '+', ' '
+
+    assert_equal expected, actual
+  end
+
+end
+