omgf 0.0.0.GIT

data/lib/omgf.rb

@@ -0,0 +1,6 @@
+# :stopdoc:
+# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+# :startdoc:
+module OMGF
+end

data/lib/omgf/hysterical_raisins.rb

@@ -0,0 +1,336 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+# :startdoc:
+require "omgf/pool"
+require "time"
+require "json"
+require "mogilefs"
+require "rack"
+require "rack/utils"
+require "rack/mime"
+
+# Basic REST interface.  This is bug-for-bug compatible with an old app
+# that's been deployed for years in a private LAN.
+#
+# This started out as a WORM (write-once, read-many) system, but
+# eventually gained the ability to handle deletes.
+#
+# This was meant to behave like the MogileFS protocol somewhat (but is
+# pure-HTTP), so it redirects to the storage nodes and bypasses Ruby
+# for bulk I/O when retrieving files.  PUTs (writes) still go through
+# Ruby, however.
+class OMGF::HystericalRaisins
+  include OMGF::Pool
+
+  def initialize(opts)
+    @default_class_cb = opts[:default_class_cb] || {}
+    mg_opts = {
+      domain: "any",
+      hosts: opts[:hosts],
+      fail_timeout: opts[:fail_timeout] || 0.5,
+
+      # high defaults because of slow seeks on storage nodes (for size verify)
+      timeout: opts[:timeout] || 30,
+      get_file_data_timeout: opts[:get_file_data_timeout] || 30,
+    }
+    @reproxy_header = opts[:reproxy_header] || "HTTP_X_OMGF_REPROXY"
+    @reproxy_path = opts[:reproxy_path]
+    @get_paths_opts = {
+      noverify: opts[:noverify],
+      pathcount: opts[:pathcount] || 0x7fffffff,
+    }
+    @put_overwrite_header = opts[:put_overwrite_header] || "HTTP_X_OMGF_FORCE"
+    pool_init(mg_opts)
+  end
+
+  # The entry point for Rack
+  def call(env)
+    case env["REQUEST_METHOD"]
+    when "GET"
+      get(env)
+    when "HEAD"
+      head(env)
+    when "PUT"
+      put(env)
+    when "DELETE"
+      delete(env)
+    else
+      r(405)
+    end
+  rescue MogileFS::Backend::UnknownKeyError,
+         MogileFS::Backend::DomainNotFoundError
+    r(404, "")
+  rescue => e
+    logger = env["rack.logger"]
+    logger.error "#{e.message} (#{e.class})"
+    e.backtrace.each { |line| logger.error(line) }
+    r(500, "")
+  end
+
+  # GET /$DOMAIN?prefix=foo - list keys
+  # GET /$DOMAIN/$KEY - redirects to FIDs on storage nodes
+  def get(env)
+    case env["PATH_INFO"].squeeze("/")
+    when %r{\A/([^/]+)/(.+)\z} # /$DOMAIN/$KEY
+      redirect_key(env, $1, $2)
+    when %r{\A/([^/]+)/?\z}    # /$DOMAIN
+      get_keys(env, $1)
+    when "/"
+      r(200, "")
+    else
+      r(404, "")
+    end
+  end
+
+  # returns metadata for a given domain/key
+  def head(env)
+    case env["PATH_INFO"].squeeze("/")
+    when %r{\A/([^/]+)/(.+)\z} # HEAD /$DOMAIN/$KEY
+      stat_key(env, $1, $2)
+    else
+      # pass on headers from listing results
+      status, headers, _ = get(env)
+      [ status, headers, [] ]
+    end
+  end
+
+  # HEAD /$DOMAIN/$KEY
+  def stat_key(env, domain, key)
+    size, paths = mg_size_and_paths(domain, key, @get_paths_opts)
+    return r(503, "") unless paths && location = paths[0]
+    h = { "Content-Length" => size.to_s }
+    fn = filename(h, query(env)) || key
+    h["Content-Type"] = key_mime_type(fn)
+    unless reproxy?(env, key, h, location)
+      paths.each_with_index { |path,i| h["X-URL-#{i}"] = path }
+    end
+    [ 200, h, [] ]
+  end
+
+  # GET /$DOMAIN/$KEY
+  def redirect_key(env, domain, key)
+    uris = mg_get_uris(domain, key, @get_paths_opts)
+
+    return r(503, "") unless uris && dest = uris.shift
+
+    location = dest.to_s
+    h = {
+      'Content-Length' => '0',
+      'Location' => location,
+      'Content-Type' => 'text/html'
+    }
+
+    unless reproxy?(env, key, h, location)
+      uris -= [ dest ]
+      uris.each_with_index { |uri,i| h["X-Alt-Location-#{i}"] = uri.to_s }
+    end
+    [ 302, h, [] ]
+  end
+
+  # GET /$DOMAIN/
+  def get_keys(env, domain) # :nodoc:
+    params = query(env)
+    prefix = params["prefix"] || ""
+    after = params["after"]
+    limit = (params["limit"] || 1000).to_i
+
+    case env["HTTP_ACCEPT"]
+    when "application/json"
+      tmp = []
+      h = { "Content-Type" => "application/json" }
+      mg_list_keys(domain, prefix, after, limit) { |*x| tmp << x }
+      tmp = tmp.to_json
+    else
+      tmp = ""
+      h = { "Content-Type" => "text/plain" }
+      mg_list_keys(domain, prefix, after, limit) do |dkey,length,devcount|
+        tmp << "#{dkey}|#{length}|#{devcount}\n"
+      end
+    end
+    h["Content-Length"] = tmp.size.to_s
+
+    [ 200, h, [ tmp ] ]
+  rescue MogileFS::Backend::UnregDomainError,
+         MogileFS::Backend::DomainNotFoundError
+    r(404, "Domain not found")
+  end
+
+  # returns a plain-text HTTP response
+  def r(code, msg = nil, env = nil) # :nodoc:
+    if env && logger = env["rack.logger"]
+      logger.warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \
+                  "#{code} #{msg.inspect}")
+    end
+
+    if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code)
+      [ code, {}, [] ]
+    else
+      msg ||= Rack::Utils::HTTP_STATUS_CODES[code] || ""
+
+      if msg.size > 0
+        # using += to not modify original string (owned by Rack)
+        msg += "\n"
+      end
+
+      [ code,
+        { 'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s },
+        [ msg ] ]
+    end
+  end
+
+  # returns the mime type based on key name
+  def key_mime_type(key) # :nodoc:
+    /\.([^.]+)\z/ =~ key
+    Rack::Mime.mime_type($1) # nil => 'application/octet-stream'
+  end
+
+  # sets Content-Disposition response header if requested by the user
+  # in a query string (via "inline" or "attachment" param)
+  # Returns requested filename, or nil if invalid
+  def filename(h, params) # :nodoc:
+    if fn = params['inline'] and Rack::Utils.escape(fn) == fn
+      h['Content-Disposition'] = "inline; filename=#{fn}"
+      return fn
+    elsif fn = (params['attachment'] || params['filename']) and
+          Rack::Utils.escape(fn) == fn
+      h['Content-Disposition'] = "attachment; filename=#{fn}"
+      return fn
+    end
+  end
+
+  # This relies on nginx to reproxy paths for us
+  def reproxy?(env, key, h, location) # :nodoc:
+    return false unless @reproxy_path && env[@reproxy_header].to_i != 0
+    %r{/0*(\d+)\.fid\z} =~ location
+    fid = $1
+    h['ETag'] = %("#{fid}")
+    fn = filename(h, query(env)) || key
+
+    case env["REQUEST_METHOD"]
+    when "GET"
+      # Fake a Last-Modified time (enough to bust caches)
+      h['X-Redirect-Last-Modified'] = Time.at(fid.to_i).httpdate
+
+      # yes we violate Rack::Lint here, no Content-Type or Content-Length
+      h.delete("Content-Length")
+      h.delete("Content-Type")
+      h['X-Redirect-Content-Type'] = key_mime_type(fn)
+      h['X-Accel-Redirect'] = @reproxy_path
+    when "HEAD"
+      # Fake a Last-Modified time (enough to bust caches)
+      h['Last-Modified'] = Time.at(fid.to_i).httpdate
+    else
+      raise "BUG: bad request method #{env["REQUEST_METHOD"]}"
+    end
+
+    true
+  end
+
+  # PUT /domain/key
+  def put(env)
+    case env["PATH_INFO"]
+    when %r{\A/([^/]+)/(.+)\z} # /$DOMAIN/$KEY
+      put_key(env, $1, $2)
+    else
+      r(404, "")
+    end
+  end
+
+  # we're very picky about keys, we only allow keys which won't cause
+  # ambiguity when passed through URI escapers/normalizers.
+  def bad_key?(key) # :nodoc:
+    %r{\A[\w./-]+\z} !~ key ||
+      %r{//} =~ key ||
+      %r{[./]} =~ "#{key[0]}#{key[-1]}"
+  end
+
+  # PUT /$DOMAIN/$KEY
+  def put_key(env, domain, key)
+    return r(403, "") if env[@reproxy_header].to_i != 0
+    return r(406, "key `#{key}' is not URI-friendly") if bad_key?(key)
+    return r(406, "key is too long") if key.size > 128
+
+    # this was written before MogileFS supported empty files,
+    # but empty files waste DB space so we don't support them
+    # Not bothering with Transfer-Encoding: chunked, though...
+    return r(403, "empty files forbidden") if env["CONTENT_LENGTH"] == "0"
+
+    params = query(env)
+    input = env["rack.input"]
+    paths = nil
+    retried = false
+
+    begin
+      pool_use(domain) do |mg|
+        begin
+          # TOCTOU issue, but probably not worth worrying about
+          # Nothing we can do about it without explicit MogileFS support
+          # or a 3rd-party locking daemon
+          paths = mg.get_paths(key)
+          if paths && paths[0]
+
+            # overwriting existing files is not permitted by default
+            if env[@put_overwrite_header] != "true"
+              # show the existing paths in response
+              return r(403, paths.join("\n"))
+            end
+          end
+        rescue MogileFS::Backend::UnknownKeyError
+          # good, not clobbering anything
+        end
+
+        # the original deployment of this created a class for every
+        # domain with the same class having the same name as the domain
+        klass = params['class'] || @default_class_cb[domain] || "default"
+        mg.store_file(key, klass, input)
+      end # pool_use
+
+      # should always return 201 if ! found, but we keep 200 for legacy
+      # compat if they're not logged in (via REMOTE_USER)
+      status = paths ? 204 : (env["REMOTE_USER"] ? 201 : 200)
+      r(status, "")
+    rescue MogileFS::Backend::UnregDomainError,
+           MogileFS::Backend::DomainNotFoundError
+      r(406, "Invalid domain: #{domain}")
+    rescue => e
+      if retried == false && input.respond_to?(:rewind)
+        begin
+          retried = true
+          input.rewind # may raise on future perfectly compliant Rack servers
+          env["rack.logger"].warn("#{e.message} (#{e.class})")
+          retry
+        rescue
+        end
+      end
+
+      raise
+    end
+  end
+
+  # DELETE /domain/key
+  def delete(env)
+    case env["PATH_INFO"]
+    when %r{\A/([^/]+)/(.+)\z} # /$DOMAIN/$KEY
+      delete_key(env, $1, $2)
+    else
+      r(404, "")
+    end
+  end
+
+  # DELETE /domain/key
+  def delete_key(env, domain, key)
+    pool_use(domain) { |mg| mg.delete(key) }
+    r(204, "")
+  rescue MogileFS::Backend::UnregDomainError,
+         MogileFS::Backend::DomainNotFoundError
+    r(406, "Invalid domain: #{domain}")
+  rescue MogileFS::Backend::UnknownKeyError
+    r(404, "")
+  end
+
+  def query(env)
+    Rack::Utils.parse_query(env["QUERY_STRING"])
+  end
+end

data/lib/omgf/pool.rb

@@ -0,0 +1,48 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "thread"
+require "omgf"
+
+module OMGF::Pool
+  # TODO: backport these improvements to MogileFS::Pool (but break compat?)
+  def pool_init(mg_opts)
+    @mg_opts = mg_opts
+    @pool = []
+    @lock = Mutex.new
+  end
+
+  def pool_use(domain)
+    # Array#pop is faster than Queue#pop since Queue#pop shifts off
+    # the array internally, and taking the last element off the end is faster.
+    mg = @lock.synchronize { @pool.pop || MogileFS::MogileFS.new(@mg_opts) }
+    mg.domain = domain
+    rv = yield mg
+    @lock.synchronize do
+      if @pool.size < 3
+        @pool << mg
+        mg = nil # prevent shutdown
+      end
+    end
+    rv
+  ensure
+    # shutdown if we didn't return to the pool
+    if mg
+      mg.backend.shutdown rescue nil
+    end
+  end
+
+  def mg_list_keys(domain, prefix, after, limit, &block) # :nodoc:
+    pool_use(domain) { |mg| mg.list_keys(prefix, after, limit, &block) }
+  end
+
+  def mg_get_uris(domain, key, opts) # :nodoc:
+    pool_use(domain) { |mg| mg.get_uris(key, opts) }
+  end
+
+  def mg_size_and_paths(domain, key, opts) # :nodoc:
+    pool_use(domain) do |mg|
+      [ mg.size(key), mg.get_paths(key, opts) ]
+    end
+  end
+end

data/omgf.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: binary -*-
+ENV["VERSION"] or abort "VERSION= must be specified"
+manifest = File.readlines('.manifest').map! { |x| x.chomp! }
+require 'wrongdoc'
+extend Wrongdoc::Gemspec
+name, summary, title = readme_metadata
+
+Gem::Specification.new do |s|
+  s.name = %q{omgf}
+  s.version = ENV["VERSION"].dup
+
+  s.authors = ["OMGF hackers"]
+  s.date = Time.now.utc.strftime('%Y-%m-%d')
+  s.description = readme_description
+  s.email = %q{omgf@librelist.org}
+  s.extra_rdoc_files = extra_rdoc_files(manifest)
+  s.files = manifest
+  s.homepage = Wrongdoc.config[:rdoc_url]
+  s.summary = summary
+  s.rdoc_options = rdoc_options
+  s.test_files = Dir['test/test_*.rb']
+  s.add_dependency('rack', ['~> 1.3'])
+  s.add_dependency('mogilefs-client', ['~> 3.1'])
+
+  s.licenses = %w(AGPLv3+)
+end

data/pkg.mk

@@ -0,0 +1,175 @@
+RUBY = ruby
+RAKE = rake
+RSYNC = rsync
+WRONGDOC = wrongdoc
+
+GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
+	@./GIT-VERSION-GEN
+-include GIT-VERSION-FILE
+-include local.mk
+DLEXT := $(shell $(RUBY) -rrbconfig -e 'puts RbConfig::CONFIG["DLEXT"]')
+RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
+RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
+lib := lib
+
+ifeq ($(shell test -f script/isolate_for_tests && echo t),t)
+isolate_libs := tmp/isolate/$(RUBY_ENGINE)-$(RUBY_VERSION)/isolate.mk
+$(isolate_libs): script/isolate_for_tests
+	@$(RUBY) script/isolate_for_tests
+-include $(isolate_libs)
+lib := $(lib):$(ISOLATE_LIBS)
+endif
+
+ext := $(firstword $(wildcard ext/*))
+ifneq ($(ext),)
+ext_pfx := tmp/ext/$(RUBY_ENGINE)-$(RUBY_VERSION)
+ext_h := $(wildcard $(ext)/*/*.h $(ext)/*.h)
+ext_src := $(wildcard $(ext)/*.c $(ext_h))
+ext_pfx_src := $(addprefix $(ext_pfx)/,$(ext_src))
+ext_d := $(ext_pfx)/$(ext)/.d
+$(ext)/extconf.rb: $(wildcard $(ext)/*.h)
+	@>> $@
+$(ext_d):
+	@mkdir -p $(@D)
+	@> $@
+$(ext_pfx)/$(ext)/%: $(ext)/% $(ext_d)
+	install -m 644 $< $@
+$(ext_pfx)/$(ext)/Makefile: $(ext)/extconf.rb $(ext_d) $(ext_h)
+	$(RM) -f $(@D)/*.o
+	cd $(@D) && $(RUBY) $(CURDIR)/$(ext)/extconf.rb
+ext_sfx := _ext.$(DLEXT)
+ext_dl := $(ext_pfx)/$(ext)/$(notdir $(ext)_ext.$(DLEXT))
+$(ext_dl): $(ext_src) $(ext_pfx_src) $(ext_pfx)/$(ext)/Makefile
+	@echo $^ == $@
+	$(MAKE) -C $(@D)
+lib := $(lib):$(ext_pfx)/$(ext)
+build: $(ext_dl)
+else
+build:
+endif
+
+pkg_extra += GIT-VERSION-FILE NEWS ChangeLog LATEST
+ChangeLog: GIT-VERSION-FILE .wrongdoc.yml
+	$(WRONGDOC) prepare
+NEWS LATEST: ChangeLog
+
+manifest:
+	$(RM) .manifest
+	$(MAKE) .manifest
+
+.manifest: $(pkg_extra)
+	(git ls-files && for i in $@ $(pkg_extra); do echo $$i; done) | \
+		LC_ALL=C sort > $@+
+	cmp $@+ $@ || mv $@+ $@
+	$(RM) $@+
+
+doc:: .document .wrongdoc.yml $(pkg_extra)
+	-find lib -type f -name '*.rbc' -exec rm -f '{}' ';'
+	-find ext -type f -name '*.rbc' -exec rm -f '{}' ';'
+	$(RM) -r doc
+	$(WRONGDOC) all
+	install -m644 COPYING doc/COPYING
+	install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/
+
+ifneq ($(VERSION),)
+pkggem := pkg/$(rfpackage)-$(VERSION).gem
+pkgtgz := pkg/$(rfpackage)-$(VERSION).tgz
+release_notes := release_notes-$(VERSION)
+release_changes := release_changes-$(VERSION)
+
+release-notes: $(release_notes)
+release-changes: $(release_changes)
+$(release_changes):
+	$(WRONGDOC) release_changes > $@+
+	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
+$(release_notes):
+	$(WRONGDOC) release_notes > $@+
+	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
+
+# ensures we're actually on the tagged $(VERSION), only used for release
+verify:
+	test x"$(shell umask)" = x0022
+	git rev-parse --verify refs/tags/v$(VERSION)^{}
+	git diff-index --quiet HEAD^0
+	test $$(git rev-parse --verify HEAD^0) = \
+	     $$(git rev-parse --verify refs/tags/v$(VERSION)^{})
+
+fix-perms:
+	-git ls-tree -r HEAD | awk '/^100644 / {print $$NF}' | xargs chmod 644
+	-git ls-tree -r HEAD | awk '/^100755 / {print $$NF}' | xargs chmod 755
+
+gem: $(pkggem)
+
+install-gem: $(pkggem)
+	gem install $(CURDIR)/$<
+
+$(pkggem): manifest fix-perms
+	gem build $(rfpackage).gemspec
+	mkdir -p pkg
+	mv $(@F) $@
+
+$(pkgtgz): distdir = $(basename $@)
+$(pkgtgz): HEAD = v$(VERSION)
+$(pkgtgz): manifest fix-perms
+	@test -n "$(distdir)"
+	$(RM) -r $(distdir)
+	mkdir -p $(distdir)
+	tar cf - $$(cat .manifest) | (cd $(distdir) && tar xf -)
+	cd pkg && tar cf - $(basename $(@F)) | gzip -9 > $(@F)+
+	mv $@+ $@
+
+package: $(pkgtgz) $(pkggem)
+
+test-release:: verify package $(release_notes) $(release_changes)
+	# make tgz release on RubyForge
+	@echo rubyforge add_release -f \
+	  -n $(release_notes) -a $(release_changes) \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
+	@echo gem push $(pkggem)
+	@echo rubyforge add_file \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
+release:: verify package $(release_notes) $(release_changes)
+	# make tgz release on RubyForge
+	rubyforge add_release -f -n $(release_notes) -a $(release_changes) \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
+	# push gem to RubyGems.org
+	gem push $(pkggem)
+	# in case of gem downloads from RubyForge releases page
+	rubyforge add_file \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
+else
+gem install-gem: GIT-VERSION-FILE
+	$(MAKE) $@ VERSION=$(GIT_VERSION)
+endif
+
+all:: test
+test_units := $(wildcard test/test_*.rb)
+test: test-unit
+test-unit: $(test_units)
+$(test_units): build
+	$(RUBY) -I $(lib) $@ $(RUBY_TEST_OPTS)
+
+# this requires GNU coreutils variants
+ifneq ($(RSYNC_DEST),)
+publish_doc:
+	-git set-file-times
+	$(MAKE) doc
+	find doc/images -type f | \
+		TZ=UTC xargs touch -d '1970-01-01 00:00:06' doc/rdoc.css
+	$(MAKE) doc_gz
+	$(RSYNC) -av doc/ $(RSYNC_DEST)/
+	git ls-files | xargs touch
+endif
+
+# Create gzip variants of the same timestamp as the original so nginx
+# "gzip_static on" can serve the gzipped versions directly.
+doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$')
+doc_gz:
+	for i in $(docs); do \
+	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
+check-warnings:
+	@(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \
+	  do $(RUBY) -d -W2 -c $$i; done) | grep -v '^Syntax OK$$' || :
+
+.PHONY: all .FORCE-GIT-VERSION-FILE doc test $(test_units) manifest
+.PHONY: check-warnings