mogilefs-client 1.3.1 → 2.0.0

data/lib/mogilefs/mogilefs.rb

@@ -1,27 +1,13 @@
-require 'open-uri'
-require 'net/http'
-require 'timeout'
-
 require 'mogilefs/client'
-require 'mogilefs/nfsfile'
 require 'mogilefs/util'
 
-##
-# Timeout error class.
-
-class MogileFS::Timeout < Timeout::Error; end
-
 ##
 # MogileFS File manipulation client.
 
 class MogileFS::MogileFS < MogileFS::Client
 
   include MogileFS::Util
-
-  ##
-  # The path to the local MogileFS mount point if you are using NFS mode.
-
-  attr_reader :root
+  include MogileFS::Bigfile
 
   ##
   # The domain of keys for this MogileFS client.
@@ -33,20 +19,26 @@ class MogileFS::MogileFS < MogileFS::Client
 
   attr_accessor :get_file_data_timeout
 
+  ##
+  # internal Regexp for matching an "HTTP 200 OK" head response
+  HTTP_200_OK = %r{\AHTTP/\d+\.\d+\s+200\s+}.freeze
+
   ##
   # Creates a new MogileFS::MogileFS instance.  +args+ must include a key
-  # :domain specifying the domain of this client.  A key :root will be used to
-  # specify the root of the NFS file system.
+  # :domain specifying the domain of this client.
 
   def initialize(args = {})
     @domain = args[:domain]
-    @root = args[:root]
 
     @get_file_data_timeout = 5
 
     raise ArgumentError, "you must specify a domain" unless @domain
 
-    super
+    if @backend = args[:db_backend]
+      @readonly = true
+    else
+      super
+    end
   end
 
   ##
@@ -62,7 +54,7 @@ class MogileFS::MogileFS < MogileFS::Client
       keys, after = list_keys prefix, after
     end
 
-    return nil
+    nil
   end
 
   ##
@@ -78,23 +70,10 @@ class MogileFS::MogileFS < MogileFS::Client
       case path
       when /^http:\/\// then
         begin
-          path = URI.parse path
-
-          if block_given?
-            sock = nil
-            timeout @get_file_data_timeout, MogileFS::Timeout do
-              sock = TCPSocket.new(path.host, path.port)
-              sock.sync = true
-              sock.syswrite("GET #{path.request_uri} HTTP/1.0\r\n\r\n")
-              buf = sock.recv(4096, Socket::MSG_PEEK)
-              head, body = buf.split(/\r\n\r\n/, 2)
-              head = sock.recv(head.size + 4)
-            end
-            return yield(sock)
-          else
-            return path.read
-          end
-        rescue MogileFS::Timeout
+          sock = http_get_sock(URI.parse(path))
+          return block_given? ? yield(sock) : sock.read
+        rescue MogileFS::Timeout, Errno::ECONNREFUSED,
+               EOFError, SystemCallError, MogileFS::InvalidResponseError
           next
         end
       else
@@ -103,22 +82,18 @@ class MogileFS::MogileFS < MogileFS::Client
       end
     end
 
-    return nil
+    nil
   end
 
   ##
   # Get the paths for +key+.
 
   def get_paths(key, noverify = true, zone = nil)
-    noverify = noverify ? 1 : 0
-    res = @backend.get_paths(:domain => @domain, :key => key,
-                             :noverify => noverify, :zone => zone)
-    
-    return nil if res.nil? and @backend.lasterr == 'unknown_key'
-    paths = (1..res['paths'].to_i).map { |i| res["path#{i}"] }
-    return paths if paths.empty?
-    return paths if paths.first =~ /^http:\/\//
-    return paths.map { |path| File.join @root, path }
+    opts = { :domain => @domain, :key => key,
+             :noverify => noverify ? 1 : 0, :zone => zone }
+    @backend.respond_to?(:_get_paths) and return @backend._get_paths(opts)
+    res = @backend.get_paths(opts)
+    (1..res['paths'].to_i).map { |i| res["path#{i}"] }
   end
 
   ##
@@ -126,41 +101,35 @@ class MogileFS::MogileFS < MogileFS::Client
   #
   # The +block+ operates like File.open.
 
-  def new_file(key, klass, bytes = 0, &block) # :yields: file
-    raise 'readonly mogilefs' if readonly?
-
-    res = @backend.create_open(:domain => @domain, :class => klass,
-                               :key => key, :multi_dest => 1)
+  def new_file(key, klass = nil, bytes = 0, &block) # :yields: file
+    raise MogileFS::ReadOnlyError if readonly?
+    opts = { :domain => @domain, :key => key, :multi_dest => 1 }
+    opts[:class] = klass if klass
+    res = @backend.create_open(opts)
 
-    raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? || res == {} # HACK
-
-    dests = nil
-
-    if res.include? 'dev_count' then # HACK HUH?
-      dests = (1..res['dev_count'].to_i).map do |i|
+    dests = if dev_count = res['dev_count'] # multi_dest succeeded
+      (1..dev_count.to_i).map do |i|
         [res["devid_#{i}"], res["path_#{i}"]]
       end
-    else
+    else # single destination returned
       # 0x0040:  d0e4 4f4b 2064 6576 6964 3d31 2666 6964  ..OK.devid=1&fid
       # 0x0050:  3d33 2670 6174 683d 6874 7470 3a2f 2f31  =3&path=http://1
       # 0x0060:  3932 2e31 3638 2e31 2e37 323a 3735 3030  92.168.1.72:7500
       # 0x0070:  2f64 6576 312f 302f 3030 302f 3030 302f  /dev1/0/000/000/
       # 0x0080:  3030 3030 3030 3030 3033 2e66 6964 0d0a  0000000003.fid..
 
-      dests = [[res['devid'], res['path']]]
+      [[res['devid'], res['path']]]
     end
 
-    dest = dests.first
-    devid, path = dest
-
-    case path
+    case (dests[0][1] rescue nil)
     when nil, '' then
-      raise 'Empty path for mogile upload'
+      raise MogileFS::EmptyPathError
     when /^http:\/\// then
-      MogileFS::HTTPFile.open(self, res['fid'], path, devid, klass, key,
+      MogileFS::HTTPFile.open(self, res['fid'], klass, key,
                               dests, bytes, &block)
     else
-      MogileFS::NFSFile.open(self, res['fid'], path, devid, klass, key, &block)
+      raise MogileFS::UnsupportedPathError,
+            "paths '#{dests.inspect}' returned by backend is not supported"
     end
   end
 
@@ -169,15 +138,15 @@ class MogileFS::MogileFS < MogileFS::Client
   # either a file name or an object that responds to #read.
 
   def store_file(key, klass, file)
-    raise 'readonly mogilefs' if readonly?
+    raise MogileFS::ReadOnlyError if readonly?
 
     new_file key, klass do |mfp|
       if file.respond_to? :sysread then
         return sysrwloop(file, mfp)
       else
 	if File.size(file) > 0x10000 # Bigass file, handle differently
-	  mfp.bigfile = file
-	  return mfp.close
+	  mfp.big_io = file
+	  return
 	else
           return File.open(file) { |fp| sysrwloop(fp, mfp) }
         end
@@ -189,26 +158,22 @@ class MogileFS::MogileFS < MogileFS::Client
   # Stores +content+ into +key+ in class +klass+.
 
   def store_content(key, klass, content)
-    raise 'readonly mogilefs' if readonly?
+    raise MogileFS::ReadOnlyError if readonly?
 
     new_file key, klass do |mfp|
       mfp << content
     end
 
-    return content.length
+    content.length
   end
 
   ##
   # Removes +key+.
 
   def delete(key)
-    raise 'readonly mogilefs' if readonly?
+    raise MogileFS::ReadOnlyError if readonly?
 
-    res = @backend.delete :domain => @domain, :key => key
-
-    if res.nil? and @backend.lasterr != 'unknown_key' then
-      raise "unable to delete #{key}: #{@backend.lasterr}"
-    end
+    @backend.delete :domain => @domain, :key => key
   end
 
   ##
@@ -222,40 +187,43 @@ class MogileFS::MogileFS < MogileFS::Client
   # Renames a key +from+ to key +to+.
 
   def rename(from, to)
-    raise 'readonly mogilefs' if readonly?
-
-    res = @backend.rename :domain => @domain, :from_key => from, :to_key => to
+    raise MogileFS::ReadOnlyError if readonly?
 
-    if res.nil? and @backend.lasterr != 'unknown_key' then
-      raise "unable to rename #{from} to #{to}: #{@backend.lasterr}"
-    end
+    @backend.rename :domain => @domain, :from_key => from, :to_key => to
+    nil
   end
 
   ##
   # Returns the size of +key+.
   def size(key)
-    paths = get_paths key
-
-    return nil unless paths
+    @backend.respond_to?(:_size) and return @backend._size(domain, key)
+    paths = get_paths(key) or return nil
+    paths_size(paths)
+  end
 
+  def paths_size(paths)
     paths.each do |path|
       next unless path
       case path
       when /^http:\/\// then
         begin
           url = URI.parse path
-
-          req = Net::HTTP::Head.new url.request_uri
-
-          res = timeout @get_file_data_timeout, MogileFS::Timeout do
-            Net::HTTP.start url.host, url.port do |http|
-              http.request req
+          s = Socket.mogilefs_new_request(url.host, url.port,
+                                   "HEAD #{url.request_uri} HTTP/1.0\r\n\r\n",
+                                   @get_file_data_timeout)
+          res = s.recv(4096, 0)
+          if res =~ HTTP_200_OK
+            head, body = res.split(/\r\n\r\n/, 2)
+            if head =~ /^Content-Length:\s*(\d+)/i
+              return $1.to_i
             end
           end
-
-          return res['Content-Length'].to_i
-        rescue MogileFS::Timeout
           next
+        rescue MogileFS::Timeout, Errno::ECONNREFUSED,
+               EOFError, SystemCallError
+          next
+        ensure
+          s.close rescue nil
         end
       else
         next unless File.exist? path
@@ -263,23 +231,55 @@ class MogileFS::MogileFS < MogileFS::Client
       end
     end
 
-    return nil
+    nil
   end
 
   ##
   # Lists keys starting with +prefix+ follwing +after+ up to +limit+.  If
   # +after+ is nil the list starts at the beginning.
 
-  def list_keys(prefix, after = nil, limit = 1000)
-    res = @backend.list_keys(:domain => domain, :prefix => prefix,
-                             :after => after, :limit => limit)
+  def list_keys(prefix, after = nil, limit = 1000, &block)
+    if @backend.respond_to?(:_list_keys)
+      return @backend._list_keys(domain, prefix, after, limit, &block)
+    end
 
-    return nil if res.nil?
+    res = begin
+      @backend.list_keys(:domain => domain, :prefix => prefix,
+                         :after => after, :limit => limit)
+    rescue MogileFS::Backend::NoneMatchError
+      return nil
+    end
 
     keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
+    if block_given?
+      # emulate the MogileFS::Mysql interface, slowly...
+      keys.each do |key|
+        paths = get_paths(key) or next
+        length = paths_size(paths) or next
+        yield key, length, paths.size
+      end
+    end
 
-    return keys, res['next_after']
+    [ keys, res['next_after'] ]
   end
 
+  protected
+
+    # given a URI, this returns a readable socket with ready data from the
+    # body of the response.
+    def http_get_sock(uri)
+      sock = Socket.mogilefs_new_request(uri.host, uri.port,
+                                    "GET #{uri.request_uri} HTTP/1.0\r\n\r\n",
+                                    @get_file_data_timeout)
+      buf = sock.recv(4096, Socket::MSG_PEEK)
+      head, body = buf.split(/\r\n\r\n/, 2)
+      if head =~ HTTP_200_OK
+        sock.recv(head.size + 4, 0)
+        return sock
+      end
+      raise MogileFS::InvalidResponseError,
+            "GET on #{uri} returned: #{head.inspect}"
+    end # def http_get_sock
+
 end
 

data/lib/mogilefs/mysql.rb

@@ -0,0 +1,166 @@
+require 'mogilefs'
+require 'mogilefs/backend' # for the exceptions
+
+# read-only interface that can be a backend for MogileFS::MogileFS
+#
+# This provides direct, read-only access to any slave MySQL database to
+# provide better performance, scalability and eliminate mogilefsd as a
+# point of failure
+class MogileFS::Mysql
+
+  attr_reader :my
+  attr_reader :query_method
+
+  ##
+  # Creates a new MogileFS::Mysql instance.  +args+ must include a key
+  # :domain specifying the domain of this client and :mysql, specifying
+  # an already-initialized Mysql object.
+  #
+  # The Mysql object can be either the standard Mysql driver or the
+  # Mysqlplus one supporting c_async_query.
+  def initialize(args = {})
+    @my = args[:mysql]
+    @query_method = @my.respond_to?(:c_async_query) ? :c_async_query : :query
+    @last_update_device = @last_update_domain = Time.at(0)
+    @cache_domain = @cache_device = nil
+  end
+
+  ##
+  # Lists keys starting with +prefix+ follwing +after+ up to +limit+.  If
+  # +after+ is nil the list starts at the beginning.
+  def _list_keys(domain, prefix = '', after = '', limit = 1000, &block)
+    # this code is based on server/lib/MogileFS/Worker/Query.pm
+    dmid = get_dmid(domain)
+
+    # don't modify passed arguments
+    limit ||= 1000
+    limit = limit.to_i
+    limit = 1000 if limit > 1000 || limit <= 0
+    after, prefix = "#{after}", "#{prefix}"
+
+    if after.length > 0 && /^#{Regexp.quote(prefix)}/ !~ after
+      raise MogileFS::Backend::AfterMismatchError
+    end
+
+    raise MogileFS::Backend::InvalidCharsError if /[%\\]/ =~ prefix
+    prefix.gsub!(/_/, '\_') # not sure why MogileFS::Worker::Query does this...
+
+    sql = <<-EOS
+    SELECT dkey,length,devcount FROM file
+    WHERE dmid = #{dmid}
+      AND dkey LIKE '#{@my.quote(prefix)}%'
+      AND dkey > '#{@my.quote(after)}'
+    ORDER BY dkey LIMIT #{limit}
+    EOS
+
+    keys = []
+    query(sql).each do |dkey,length,devcount|
+      yield(dkey, length, devcount) if block_given?
+      keys << dkey
+    end
+
+    keys.empty? ? nil : [ keys, (keys.last || '') ]
+  end
+
+  ##
+  # Returns the size of +key+.
+  def _size(domain, key)
+    dmid = get_dmid(domain)
+
+    sql = <<-EOS
+    SELECT length FROM file
+    WHERE dmid = #{dmid} AND dkey = '#{@my.quote(key)}'
+    LIMIT 1
+    EOS
+
+    res = query(sql).fetch_row
+    return res[0].to_i if res && res[0]
+    raise MogileFS::Backend::UnknownKeyError
+  end
+
+  ##
+  # Get the paths for +key+.
+  def _get_paths(params = {})
+    zone = params[:zone]
+    noverify = (params[:noverify] == 1) # TODO this is unused atm
+    dmid = get_dmid(params[:domain])
+    devices = refresh_device or raise MogileFS::Backend::NoDevicesError
+    urls = []
+    sql = <<-EOS
+    SELECT fid FROM file
+    WHERE dmid = #{dmid} AND dkey = '#{@my.quote(params[:key])}'
+    LIMIT 1
+    EOS
+
+    res = query(sql).fetch_row
+    res && res[0] or raise MogileFS::Backend::UnknownKeyError
+    fid = res[0]
+    sql = "SELECT devid FROM file_on WHERE fid = '#{@my.quote(fid)}'"
+    query(sql).each do |devid,|
+      unless devinfo = devices[devid.to_i]
+        devices = refresh_device(true)
+        devinfo = devices[devid.to_i] or next
+      end
+
+      port = devinfo[:http_get_port]
+      host = zone && zone == 'alt' ? devinfo[:altip] : devinfo[:hostip]
+      nfid = '%010u' % fid
+      b, mmm, ttt = /(\d)(\d{3})(\d{3})(?:\d{3})/.match(nfid)[1..3]
+      uri = "/dev#{devid}/#{b}/#{mmm}/#{ttt}/#{nfid}.fid"
+      urls << "http://#{host}:#{port}#{uri}"
+    end
+    urls
+  end
+
+  def sleep(params); Kernel.sleep(params[:duration] || 10); {}; end
+
+  private
+
+    unless defined? GET_DEVICES
+      GET_DOMAINS = 'SELECT dmid,namespace FROM domain'.freeze
+
+      GET_DEVICES = <<-EOS
+        SELECT d.devid, h.hostip, h.altip, h.http_port, h.http_get_port
+        FROM device d
+          LEFT JOIN host h ON d.hostid = h.hostid
+        WHERE d.status IN ('alive','readonly','drain');
+      EOS
+      GET_DEVICES.freeze
+    end
+
+    def query(sql)
+      @my.send(@query_method, sql)
+    end
+
+    def refresh_device(force = false)
+      return @cache_device if ! force && ((Time.now - @last_update_device) < 60)
+      tmp = {}
+      res = query(GET_DEVICES)
+      res.each do |devid, hostip, altip, http_port, http_get_port|
+        http_port = http_port ? http_port.to_i : 80
+        tmp[devid.to_i] = {
+          :hostip => hostip.freeze,
+          :altip => (altip || hostip).freeze,
+          :http_port => http_port,
+          :http_get_port => http_get_port ?  http_get_port.to_i : http_port,
+        }.freeze
+      end
+      @last_update_device = Time.now
+      @cache_device = tmp.freeze
+    end
+
+    def refresh_domain(force = false)
+      return @cache_domain if ! force && ((Time.now - @last_update_domain) < 5)
+      tmp = {}
+      res = query(GET_DOMAINS)
+      res.each { |dmid,namespace| tmp[namespace] = dmid.to_i }
+      @last_update_domain = Time.now
+      @cache_domain = tmp.freeze
+    end
+
+    def get_dmid(domain)
+      refresh_domain[domain] || refresh_domain(true)[domain] or \
+        raise MogileFS::Backend::DomainNotFoundError, domain
+    end
+
+end