mogilefs-client 1.0.1 → 1.2.1

data/History.txt

@@ -0,0 +1,11 @@
+= 1.2.1
+
+* Switched to Hoe.
+* Moved to p4.
+* Fixed bug #7273 in HTTP mode of client where data would not get
+  returned.  Submitted by Matthew Willson.
+
+= 1.2.0
+
+* Changes lost to time.
+

data/{LICENSE → LICENSE.txt}

@@ -1,7 +1,4 @@
-Portions copyright 2004 David Heinemeier Hansson.
-
-All original code copyright 2005 Eric Hodel, The Robot Co-op.  All rights
-reserved.
+Copyright 2005 Eric Hodel, The Robot Co-op.  All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
@@ -15,8 +12,6 @@ are met:
 3. Neither the names of the authors nor the names of their contributors
    may be used to endorse or promote products derived from this software
    without specific prior written permission.
-4. Redistribution in Rails or any sub-projects of Rails is not allowed
-   until Rails runs without warnings with the ``-W2'' flag enabled.
 
 THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
 OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

data/Manifest.txt

@@ -1,6 +1,7 @@
-LICENSE
+History.txt
+LICENSE.txt
 Manifest.txt
-README
+README.txt
 Rakefile
 lib/mogilefs.rb
 lib/mogilefs/admin.rb

data/README.txt

@@ -0,0 +1,66 @@
+= mogilefs-client
+
+A Ruby MogileFS client
+
+Rubyforge Project:
+
+http://rubyforge.org/projects/seattlerb/
+
+Documentation:
+
+http://seattlerb.org/mogilefs-client
+
+File bugs:
+
+http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
+
+== About
+
+A Ruby MogileFS client.  MogileFS is a distributed filesystem written
+by Danga Interactive.  This client supports NFS and HTTP modes.
+
+For information on MogileFS see:
+
+http://danga.com/mogilefs/
+
+== Installing mogilefs-client
+
+First you need a MogileFS setup.  You can find information on how to do that at the above URL.
+
+Then install the gem:
+
+  $ sudo gem install mogilefs-client
+
+== Using mogilefs-client
+
+  # Create a new instance that will communicate with these trackers:
+  hosts = %w[192.168.1.69:6001 192.168.1.70:6001]
+  mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts
+                              :root => '/mnt/mogilefs')
+  
+  # Stores "A bunch of text to store" into 'some_key' with a class of 'text'.
+  mg.store_content 'some_key', 'text', "A bunch of text to store"
+  
+  # Retrieve data from 'some_key'
+  data = mg.get_file_data 'some_key'
+  
+  # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
+  # 'image'.
+  mg.store_file 'my_image', 'image', 'image.jpeg'
+  
+  # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
+  # 'image' using an open IO.
+  File.open 'image.jpeg' do |fp|
+    mg.store_file 'my_image', 'image', fp
+  end
+  
+  # Remove the key 'my_image' and 'some_key'.
+  mg.delete 'my_image'
+  mg.delete 'some_key'
+
+== WARNING!
+
+This client is only known to work in NFS mode.  HTTP mode is implemented but
+only lightly tested in production environments.  If you find a bug,
+please report it on the Rubyforge project.
+

data/Rakefile

@@ -1,56 +1,18 @@
 require 'rubygems'
-require 'rake'
-require 'rake/testtask'
-require 'rake/rdoctask'
-require 'rake/gempackagetask'
-
-$VERBOSE = nil
-
-spec = Gem::Specification.new do |s|
-  s.name = 'mogilefs-client'
-  s.version = '1.0.1'
-  s.summary = 'A Ruby MogileFS client'
-  s.description = 'A Ruby MogileFS client.  MogileFS is a distributed filesystem written by Danga Interactive.  This client supports NFS mode and has untested support for HTTP mode.'
-  s.author = 'Eric Hodel'
-  s.email = 'eric@robotcoop.com'
-
-  s.has_rdoc = true
-  s.files = File.read('Manifest.txt').split($/)
-  s.require_path = 'lib'
-end
-
-desc 'Run tests'
-task :default => [ :test ]
-
-Rake::TestTask.new('test') do |t|
-  t.libs << 'test'
-  t.pattern = 'test/test_*.rb'
-  t.verbose = true
-end
-
-desc 'Update Manifest.txt'
-task :update_manifest do
-  sh "find . -type f | sed -e 's%./%%' | egrep -v 'svn|swp|~' | egrep -v '^(doc|pkg)/' | sort > Manifest.txt"
-end
-
-desc 'Generate RDoc'
-Rake::RDocTask.new :rdoc do |rd|
-  rd.rdoc_dir = 'doc'
-  rd.rdoc_files.add 'lib', 'README', 'LICENSE'
-  rd.main = 'README'
-  rd.options << '-d' if `which dot` =~ /\/dot/
+require 'hoe'
+
+$:.unshift 'lib'
+require 'mogilefs'
+
+Hoe.new 'mogilefs-client', MogileFS::VERSION do |p|
+  p.rubyforge_name = 'seattlerb'
+  p.author = 'Eric Hodel'
+  p.email = 'drbrain@segment7.net'
+  p.summary = p.paragraphs_of('README.txt', 1).first
+  p.description = p.paragraphs_of('README.txt', 9).first
+  p.url = p.paragraphs_of('README.txt', 5).first
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
 end
 
-desc 'Build Gem'
-Rake::GemPackageTask.new spec do |pkg|
-  pkg.need_tar = true
-end
-
-desc 'Clean up'
-task :clean => [ :clobber_rdoc, :clobber_package ]
-
-desc 'Clean up'
-task :clobber => [ :clean ]
-
 # vim: syntax=Ruby
 

data/lib/mogilefs.rb

@@ -6,6 +6,8 @@
 
 module MogileFS
 
+  VERSION = '1.2.1'
+
   ##
   # Raised when a socket remains unreadable for too long.
 

data/lib/mogilefs/admin.rb

@@ -5,6 +5,22 @@ require 'mogilefs/client'
 
 class MogileFS::Admin < MogileFS::Client
 
+  ##
+  # Enumerates fids using #list_fids.
+
+  def each_fid
+    low = 0
+    high = nil
+
+    max = get_stats('fids')['fids']['max']
+
+    0.step max, 100 do |high|
+      fids = list_fids low, high
+      fids.each { |fid| yield fid }
+      low = high + 1
+    end
+  end
+
   ##
   # Returns an Array of host status Hashes.  If +hostid+ is given only that
   # host is returned.
@@ -95,14 +111,21 @@ class MogileFS::Admin < MogileFS::Client
     res = @backend.stats type => 1
     stats = {}
 
-    stats['replication'] = clean 'replicationcount', 'replication', res, false
-    stats['file'] = clean 'filescount', 'files', res, false
     stats['device'] = clean 'devicescount', 'devices', res, false
+    stats['file'] = clean 'filescount', 'files', res, false
+    stats['replication'] = clean 'replicationcount', 'replication', res, false
 
     if res['fidmax'] or res['fidcount'] then
-      stats['fids'] = { 'max' => res['fidmax'], 'count' => res['fidcount'] }
+      stats['fids'] = {
+        'max' => res['fidmax'].to_i,
+        'count' => res['fidcount'].to_i
+      }
     end
 
+    stats.delete 'device' if stats['device'].empty?
+    stats.delete 'file' if stats['file'].empty?
+    stats.delete 'replication' if stats['replication'].empty?
+
     return stats
   end
 
@@ -113,7 +136,7 @@ class MogileFS::Admin < MogileFS::Client
   #
   # Returns:
   #
-  #   {"test"=>{"normal"=>"2", "default"=>"2"}}
+  #   {"test"=>{"normal"=>3, "default"=>2}}
 
   def get_domains
     res = @backend.get_domains
@@ -121,7 +144,8 @@ class MogileFS::Admin < MogileFS::Client
     domains = {}
     (1..res['domains'].to_i).each do |i|
       domain = clean "domain#{i}classes", "domain#{i}class", res, false
-      domains[res["domain#{i}"]] = Hash[*domain.map { |d| d.values}.flatten]
+      domain = domain.map { |d| [d.values.first, d.values.last.to_i] }
+      domains[res["domain#{i}"]] = Hash[*domain.flatten]
     end
 
     return domains

data/lib/mogilefs/backend.rb

@@ -153,7 +153,10 @@ class MogileFS::Backend
 
   def readable?
     found = select [socket], nil, nil, @timeout
-    raise MogileFS::UnreadableSocketError if found.nil? or found.empty?
+    if found.nil? or found.empty? then
+      peer = (@socket ? "#{@socket.peeraddr[3]}:#{@socket.peeraddr[1]} " : nil)
+      raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
+    end
     return true
   end
 

data/lib/mogilefs/client.rb

@@ -24,6 +24,7 @@ class MogileFS::Client
   def initialize(args)
     @hosts = args[:hosts]
     @readonly = args[:readonly] ? true : false
+    @timeout = args[:timeout]
 
     reload
   end
@@ -32,7 +33,7 @@ class MogileFS::Client
   # Creates a new MogileFS::Backend.
 
   def reload
-    @backend = MogileFS::Backend.new :hosts => @hosts
+    @backend = MogileFS::Backend.new :hosts => @hosts, :timeout => @timeout
   end
 
   ##

data/lib/mogilefs/mogilefs.rb

@@ -38,19 +38,38 @@ class MogileFS::MogileFS < MogileFS::Client
     super
   end
 
+  ##
+  # Enumerates keys starting with +key+.
+
+  def each_key(prefix)
+    after = nil
+
+    keys, after = list_keys prefix
+
+    until keys.empty? do
+      keys.each { |k| yield k }
+      keys, after = list_keys prefix, after
+    end
+
+    return nil
+  end
+
   ##
   # Retrieves the contents of +key+.
 
   def get_file_data(key)
-    get_paths(key).each do |path|
+    paths = get_paths key
+
+    return nil unless paths
+
+    paths.each do |path|
       next unless path
       case path
       when /^http:\/\// then
         begin
-          puts "path: #{path}"
           path = URI.parse path
           data = timeout(5, MogileFS::Timeout) { path.read }
-          puts "data: #{data}"
+          return data
         rescue MogileFS::Timeout
           next
         end

data/test/setup.rb

@@ -9,25 +9,24 @@ class FakeBackend
   attr_reader :lasterr, :lasterrstr
 
   def initialize
-    @responses = {}
+    @responses = Hash.new { |h,k| h[k] = [] }
     @lasterr = nil
     @lasterrstr = nil
   end
 
-  def method_missing(sym, *args)
-    sym = sym.to_s
-    if sym =~ /(.*)=$/ then
-      @responses[$1] = args.first
+  def method_missing(meth, *args)
+    meth = meth.to_s
+    if meth =~ /(.*)=$/ then
+      @responses[$1] << args.first
     else
-      response = @responses[sym]
+      response = @responses[meth].shift
       case response
-      when Hash then
-        return response
       when Array then
         @lasterr = response.first
         @lasterrstr = response.last
         return nil
       end
+      return response
     end
   end
 
@@ -42,7 +41,7 @@ class TestMogileFS < Test::Unit::TestCase
   def setup
     return if self.class == TestMogileFS
     @root = '/mogilefs/test'
-    @client = @class.new :hosts => ['kaa:6001'], :domain => 'test',
+    @client = @klass.new :hosts => ['kaa:6001'], :domain => 'test',
                                   :root => @root
     @backend = FakeBackend.new
     @client.instance_variable_set '@backend', @backend

data/test/test_admin.rb

@@ -3,7 +3,7 @@ require 'test/setup'
 class TestMogileFS__Admin < TestMogileFS
 
   def setup
-    @class = MogileFS::Admin
+    @klass = MogileFS::Admin
     super
   end
 
@@ -33,5 +33,142 @@ class TestMogileFS__Admin < TestMogileFS
     assert_equal expected, actual
   end
 
+  def test_each_fid
+    @backend.stats = {
+      'fidmax' => '182',
+      'fidcount' => '2',
+    }
+
+    @backend.list_fids = {
+      'fid_count' => '1',
+      'fid_1_fid' => '99',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'file_key',
+      'fid_1_length' => '4',
+    }
+
+    @backend.list_fids = {
+      'fid_count' => '1',
+      'fid_1_fid' => '182',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'new_new_key',
+      'fid_1_length' => '9',
+    }
+
+    fids = []
+    @client.each_fid { |fid| fids << fid }
+
+    expected = [
+      { "fid"      => "99",
+        "class"    => "normal",
+        "domain"   => "test",
+        "devcount" => "2",
+        "length"   => "4",
+        "key"      => "file_key" },
+      { "fid"      => "182",
+        "class"    => "normal",
+        "devcount" => "2",
+        "domain"   => "test",
+        "length"   => "9",
+        "key"      => "new_new_key" },
+    ]
+
+    assert_equal expected, fids
+  end
+
+  def test_get_domains
+    @backend.get_domains = {
+      'domains' => 2,
+      'domain1' => 'test',
+      'domain2' => 'images',
+      'domain1classes' => '1',
+      'domain2classes' => '2',
+      'domain1class1name' => 'default',
+      'domain1class1mindevcount' => '2',
+      'domain2class1name' => 'default',
+      'domain2class1mindevcount' => '2',
+      'domain2class2name' => 'resize',
+      'domain2class2mindevcount' => '1',
+    }
+
+    expected = {
+      'test'   => { 'default' => 2, },
+      'images' => { 'default' => 2, 'resize' => 1 },
+    }
+
+    assert_equal expected, @client.get_domains
+  end
+
+  def disabled_test_get_stats
+    @backend.stats = {}
+
+    expected = {
+      'fids' => { 'max' => '99', 'count' => '2' },
+      'device' => [
+        { 'status' => 'alive', 'files' => '2', 'id' => '1', 'host' => 'rur-1' },
+        { 'status' => 'alive', 'files' => '2', 'id' => '2', 'host' => 'rur-2' }
+      ],
+      'replication' => [
+        { 'files' => '2', 'class' => 'normal', 'devcount' => '2',
+          'domain' => 'test' }
+      ],
+      'file' => [{ 'files' => '2', 'class' => 'normal', 'domain' => 'test' }]
+    }
+
+    assert_equal 
+  end
+
+  def test_get_stats_fids
+    @backend.stats = {
+      'fidmax' => 99,
+      'fidcount' => 2,
+    }
+
+    expected = {
+      'fids' => { 'max' => 99, 'count' => 2 },
+    }
+
+    assert_equal expected, @client.get_stats('all')
+  end
+
+  def test_list_fids
+    @backend.list_fids = {
+      'fid_count' => '2',
+      'fid_1_fid' => '99',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'file_key',
+      'fid_1_length' => '4',
+      'fid_2_fid' => '82',
+      'fid_2_class' => 'normal',
+      'fid_2_devcount' => '2',
+      'fid_2_domain' => 'test',
+      'fid_2_key' => 'new_new_key',
+      'fid_2_length' => '9',
+    }
+
+    expected = [
+      { "fid"      => "99",
+        "class"    => "normal",
+        "domain"   => "test",
+        "devcount" => "2",
+        "length"   => "4",
+        "key"      => "file_key" },
+      { "fid"      => "82",
+        "class"    => "normal",
+        "devcount" => "2",
+        "domain"   => "test",
+        "length"   => "9",
+        "key"      => "new_new_key" },
+    ]
+
+    assert_equal expected, @client.list_fids(0, 100)
+  end
+
 end
 

data/test/test_backend.rb

@@ -27,12 +27,16 @@ class FakeSocket
     return nil
   end
 
+  def peeraddr
+    ['AF_INET', 6001, 'localhost', '127.0.0.1']
+  end
+
 end
 
 class TestBackend < Test::Unit::TestCase
 
   def setup
-    @backend = MogileFS::Backend.new :hosts => ['localhost:6001']
+    @backend = MogileFS::Backend.new :hosts => ['localhost:1']
   end
 
   def test_initialize
@@ -40,7 +44,7 @@ class TestBackend < Test::Unit::TestCase
     assert_raises ArgumentError do MogileFS::Backend.new :hosts => [] end
     assert_raises ArgumentError do MogileFS::Backend.new :hosts => [''] end
 
-    assert_equal ['localhost:6001'], @backend.hosts
+    assert_equal ['localhost:1'], @backend.hosts
     assert_equal 3, @backend.timeout
     assert_equal nil, @backend.lasterr
     assert_equal nil, @backend.lasterrstr
@@ -119,18 +123,26 @@ class TestBackend < Test::Unit::TestCase
   end
 
   def test_readable_eh_not_readable
-    socket = Object.new
+    socket = FakeSocket.new
     def socket.closed?() false end
     def @backend.select(*args) return [] end
     @backend.instance_variable_set '@socket', socket
 
-    assert_raises MogileFS::UnreadableSocketError do @backend.readable? end
+    begin
+      @backend.readable?
+    rescue MogileFS::UnreadableSocketError => e
+      assert_equal '127.0.0.1:6001 never became readable', e.message
+    rescue Exception
+      flunk "MogileFS::UnreadableSocketError not raised"
+    else
+      flunk "MogileFS::UnreadableSocketError not raised"
+    end
   end
 
   def test_socket
     assert_equal({}, @backend.dead)
     assert_raises RuntimeError do @backend.socket end
-    assert_equal(['localhost:6001'], @backend.dead.keys)
+    assert_equal(['localhost:1'], @backend.dead.keys)
   end
 
   def test_socket_robust

data/test/test_client.rb

@@ -11,9 +11,13 @@ class TestClient < Test::Unit::TestCase
   end
 
   def test_initialize
-    assert_not_nil @client
-    assert_instance_of MogileFS::Backend, @client.backend
-    assert_equal ['kaa:6001'], @client.hosts
+    client = MogileFS::Client.new :hosts => ['kaa:6001']
+    assert_not_nil client
+    assert_instance_of MogileFS::Backend, client.backend
+    assert_equal ['kaa:6001'], client.hosts
+
+    client = MogileFS::Client.new :hosts => ['kaa:6001'], :timeout => 5
+    assert_equal 5, client.backend.timeout
   end
 
   def test_err

data/test/test_mogilefs.rb

@@ -1,9 +1,21 @@
 require 'test/setup'
 
+class URI::HTTP
+
+  class << self
+    attr_accessor :read_data
+  end
+
+  def read
+    self.class.read_data.shift
+  end
+
+end
+
 class TestMogileFS__MogileFS < TestMogileFS
 
   def setup
-    @class = MogileFS::MogileFS
+    @klass = MogileFS::MogileFS
     super
   end
 
@@ -16,6 +28,17 @@ class TestMogileFS__MogileFS < TestMogileFS
     end
   end
 
+  def test_get_file_data_http
+    URI::HTTP.read_data = %w[data!]
+
+    path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
+    path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
+
+    @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
+
+    assert_equal 'data!', @client.get_file_data('key')
+  end
+
   def test_get_paths
     path1 = 'rur-1/dev1/0/000/000/0000000062.fid'
     path2 = 'rur-2/dev2/0/000/000/0000000062.fid'
@@ -54,6 +77,20 @@ class TestMogileFS__MogileFS < TestMogileFS
     end
   end
 
+  def test_each_key
+    @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_2',
+                           'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
+    @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_4',
+                           'key_1' => 'new_key_3', 'key_2' => 'new_key_4' }
+    @backend.list_keys = { 'key_count' => 0, 'next_after' => 'new_key_4' }
+    keys = []
+    @client.each_key 'new' do |key|
+      keys << key
+    end
+
+    assert_equal %w[new_key_1 new_key_2 new_key_3 new_key_4], keys
+  end
+
   def test_list_keys
     @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_2',
                            'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }

metadata

@@ -1,22 +1,28 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.8.11.6
+rubygems_version: 0.9.4.3
 specification_version: 1
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  version: 1.0.1
-date: 2006-03-21 00:00:00 -08:00
+  version: 1.2.1
+date: 2007-07-31 00:00:00 -07:00
 summary: A Ruby MogileFS client
 require_paths: 
 - lib
-email: eric@robotcoop.com
-homepage: 
-rubyforge_project: 
-description: A Ruby MogileFS client.  MogileFS is a distributed filesystem written by Danga Interactive.  This client supports NFS mode and has untested support for HTTP mode.
+email: drbrain@segment7.net
+homepage: http://seattlerb.org/mogilefs-client
+rubyforge_project: seattlerb
+description: A Ruby MogileFS client.  MogileFS is a distributed filesystem written by Danga Interactive.  This client supports NFS and HTTP modes.
 autorequire: 
 default_executable: 
 bindir: bin
 has_rdoc: true
-required_ruby_version: !ruby/object:Gem::Version::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">"
     - !ruby/object:Gem::Version 
@@ -29,9 +35,10 @@ post_install_message:
 authors: 
 - Eric Hodel
 files: 
-- LICENSE
+- History.txt
+- LICENSE.txt
 - Manifest.txt
-- README
+- README.txt
 - Rakefile
 - lib/mogilefs.rb
 - lib/mogilefs/admin.rb
@@ -47,17 +54,33 @@ files:
 - test/test_client.rb
 - test/test_mogilefs.rb
 - test/test_pool.rb
-test_files: []
-
-rdoc_options: []
-
-extra_rdoc_files: []
-
+test_files: 
+- test/test_admin.rb
+- test/test_backend.rb
+- test/test_client.rb
+- test/test_mogilefs.rb
+- test/test_pool.rb
+rdoc_options: 
+- --main
+- README.txt
+extra_rdoc_files: 
+- History.txt
+- LICENSE.txt
+- Manifest.txt
+- README.txt
 executables: []
 
 extensions: []
 
 requirements: []
 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.2
+    version: 

data/README

@@ -1,9 +0,0 @@
-This is a client for Danga's MogileFS file store.  For information on MogileFS
-see:
-
-http://danga.com/mogilefs/
-
-This client is only known to work in NFS mode.  HTTP mode is implemented but completely untested.  If you find a bug, please report it in an email to:
-
-eric@robotcoop.com.
-