attache 1.0.0

data/lib/attache/download.rb

@@ -0,0 +1,86 @@
+require 'connection_pool'
+
+class Attache::Download < Attache::Base
+  REMOTE_GEOMETRY = ENV.fetch('REMOTE_GEOMETRY') { 'remote' }
+  OUTPUT_EXTENSIONS = %w[png jpg jpeg gif]
+  RESIZE_JOB_POOL = ConnectionPool.new(JSON.parse(ENV.fetch('RESIZE_POOL') { '{ "size": 2, "timeout": 60 }' }).symbolize_keys) { Attache::ResizeJob.new }
+
+  def initialize(app)
+    @app = app
+  end
+
+  def _call(env, config)
+    case env['PATH_INFO']
+    when %r{\A/view/}
+      parse_path_info(env['PATH_INFO']['/view/'.length..-1]) do |dirname, geometry, basename, relpath|
+        if geometry == REMOTE_GEOMETRY && config.storage && config.bucket
+          headers = config.download_headers.merge({
+                      'Location' => config.storage_url(relpath: relpath),
+                      'Cache-Control' => 'private, no-cache',
+                    })
+          return [302, headers, []]
+        end
+
+        file = begin
+          cachekey = File.join(request_hostname(env), relpath)
+          Attache.cache.fetch(cachekey) do
+            config.storage_get(relpath: relpath) if config.storage && config.bucket
+          end
+        rescue Exception # Errno::ECONNREFUSED, OpenURI::HTTPError, Excon::Errors, Fog::Errors::Error
+          Attache.logger.error $@
+          Attache.logger.error $!
+          Attache.logger.error "ERROR REFERER #{env['HTTP_REFERER'].inspect}"
+          nil
+        end
+
+        unless file
+          return [404, config.download_headers, []]
+        end
+
+        thumbnail = if geometry == 'original' || geometry == REMOTE_GEOMETRY
+          file
+        else
+          extension = basename.split(/\W+/).last
+          extension = OUTPUT_EXTENSIONS.first unless OUTPUT_EXTENSIONS.index(extension.to_s.downcase)
+          make_thumbnail_for(file.tap(&:close), geometry, extension)
+        end
+
+        headers = {
+          'Content-Type' => content_type_of(thumbnail.path),
+        }.merge(config.download_headers)
+
+        [200, headers, rack_response_body_for(thumbnail)].tap do
+          unless file == thumbnail # cleanup
+            File.unlink(thumbnail.path) rescue Errno::ENOENT
+          end
+        end
+      end
+    else
+      @app.call(env)
+    end
+  rescue Exception
+    Attache.logger.error $@
+    Attache.logger.error $!
+    Attache.logger.error "ERROR REFERER #{env['HTTP_REFERER'].inspect}"
+    [500, { 'X-Exception' => $!.to_s }, []]
+  end
+
+  private
+
+    def parse_path_info(geometrypath)
+      parts = geometrypath.split('/')
+      basename = CGI.unescape parts.pop
+      geometry = CGI.unescape parts.pop
+      dirname  = parts.join('/')
+      relpath  = File.join(dirname, basename)
+      yield dirname, geometry, basename, relpath
+    end
+
+    def make_thumbnail_for(file, geometry, extension)
+      Attache.logger.info "[POOL] new job"
+      RESIZE_JOB_POOL.with do |job|
+        job.perform(file, geometry, extension)
+      end
+    end
+
+end

data/lib/attache/file_response_body.rb

@@ -0,0 +1,20 @@
+class Attache::FileResponseBody
+  def initialize(file, range_start = nil, range_end = nil)
+    @file        = file
+    @range_start = range_start || 0
+    @range_end   = range_end || File.size(@file.path)
+  end
+
+  # adapted from rack/file.rb
+  def each
+    @file.seek(@range_start)
+    remaining_len = @range_end
+    while remaining_len > 0
+      part = @file.read([8192, remaining_len].min)
+      break unless part
+      remaining_len -= part.length
+
+      yield part
+    end
+  end
+end

data/lib/attache/job.rb

@@ -0,0 +1,32 @@
+class Attache::Job
+  RETRY_DURATION = ENV.fetch('CACHE_EVICTION_INTERVAL_SECONDS') { 60 }.to_i / 3
+
+  def perform(method, env, args)
+    config = Attache::VHost.new(env)
+    config.send(method, args.symbolize_keys)
+  rescue Exception
+    Attache.logger.error $@
+    Attache.logger.error $!
+    Attache.logger.error [method, args].inspect
+    self.class.perform_in(RETRY_DURATION, method, env, args)
+  end
+
+  # Background processing setup
+
+  if defined?(::SuckerPunch::Job)
+    include ::SuckerPunch::Job
+    def later(sec, *args)
+      after(sec) { perform(*args) }
+    end
+    def self.perform_async(*args)
+      self.new.async.perform(*args)
+    end
+    def self.perform_in(duration, *args)
+      self.new.async.later(duration, *args)
+    end
+  else
+    include Sidekiq::Worker
+    sidekiq_options :queue => :attache_vhost_jobs
+    sidekiq_retry_in {|count| RETRY_DURATION} # uncaught exception, retry after RETRY_DURATION
+  end
+end

data/lib/attache/resize_job.rb

@@ -0,0 +1,44 @@
+require 'digest/sha1'
+
+class Attache::ResizeJob
+  def perform(closed_file, target_geometry_string, extension)
+    t = Time.now
+    Attache.logger.info "[POOL] start"
+    thumbnail = thumbnail_for(closed_file: closed_file, target_geometry_string: target_geometry_string, extension: extension)
+    thumbnail.instance_variable_set('@basename', thumbnail.instance_variable_get('@basename').gsub(/[^\w\.]/, '_'))
+    thumbnail.make.tap do
+      Attache.logger.info "[POOL] done in #{Time.now - t}s"
+    end
+  rescue Paperclip::Errors::NotIdentifiedByImageMagickError
+    closed_file
+  end
+
+  private
+
+    def thumbnail_for(closed_file:, target_geometry_string:, extension:, max: 2048)
+      thumbnail = Paperclip::Thumbnail.new(closed_file, geometry: target_geometry_string, format: extension)
+      current_geometry = current_geometry_for(thumbnail)
+      target_geometry = Paperclip::GeometryParser.new(target_geometry_string).make
+      if target_geometry.larger <= max && current_geometry.larger > max
+        # optimization:
+        #  when users upload "super big files", we can speed things up
+        #  by working from a "reasonably large 2048x2048 thumbnail" (<2 seconds)
+        #  instead of operating on the original (>10 seconds)
+        #  we store this reusably in Attache.cache to persist reboot, but not uploaded to cloud
+        working_geometry = "#{max}x#{max}>"
+        working_file = Attache.cache.fetch(Digest::SHA1.hexdigest(working_geometry + closed_file.path)) do
+          Attache.logger.info "[POOL] generate working_file"
+          Paperclip::Thumbnail.new(closed_file, geometry: working_geometry, format: extension).make
+        end
+        Attache.logger.info "[POOL] use working_file #{working_file.path}"
+        thumbnail = Paperclip::Thumbnail.new(working_file.tap(&:close), geometry: target_geometry_string, format: extension)
+      end
+      thumbnail
+    end
+
+    # allow stub in spec
+    def current_geometry_for(thumbnail)
+      thumbnail.current_geometry.tap(&:auto_orient)
+    end
+
+end

data/lib/attache/tus.rb

@@ -0,0 +1,53 @@
+class Attache::Tus
+  LENGTH_KEYS   = %w[Upload-Length   Entity-Length]
+  OFFSET_KEYS   = %w[Upload-Offset   Offset]
+  METADATA_KEYS = %w[Upload-Metadata Metadata]
+
+  attr_accessor :env, :config
+
+  def initialize(env, config)
+    @env = env
+    @config = config
+  end
+
+  def header_value(keys)
+    value = nil
+    keys.find {|k| value = env["HTTP_#{k.gsub('-', '_').upcase}"]}
+    value
+  end
+
+  def upload_length
+    header_value LENGTH_KEYS
+  end
+
+  def upload_offset
+    header_value OFFSET_KEYS
+  end
+
+  def upload_metadata
+    value = header_value METADATA_KEYS
+    Hash[*value.split(/[, ]/)].inject({}) do |h, (k, v)|
+      h.merge(k => Base64.decode64(v))
+    end
+  end
+
+  def resumable_version
+    header_value ["Tus-Resumable"]
+  end
+
+  def headers_with_cors(headers = {}, offset: nil)
+    tus_headers = {
+      "Access-Control-Allow-Methods" => "PATCH",
+      "Access-Control-Allow-Headers" => "Tus-Resumable, #{LENGTH_KEYS.join(', ')}, #{METADATA_KEYS.join(', ')}, #{OFFSET_KEYS.join(', ')}",
+      "Access-Control-Expose-Headers" => "Location, #{OFFSET_KEYS.join(', ')}",
+    }
+    OFFSET_KEYS.each do |k|
+      tus_headers[k] = offset
+    end if offset
+
+    # append
+    tus_headers.inject(config.headers_with_cors.merge(headers)) do |sum, (k, v)|
+      sum.merge(k => [*sum[k], v].join(', '))
+    end
+  end
+end

data/lib/attache/tus/upload.rb

@@ -0,0 +1,102 @@
+class Attache::Tus::Upload < Attache::Base
+  def initialize(app)
+    @app = app
+  end
+
+  def _call(env, config)
+    case env['PATH_INFO']
+    when '/tus/files'
+      tus = ::Attache::Tus.new(env, config)
+      params = params_of(env) # avoid unnecessary `invalid byte sequence in UTF-8` on `request.params`
+      return config.unauthorized unless config.authorized?(params)
+
+      case env['REQUEST_METHOD']
+      when 'POST'
+        if positive_number?(tus.upload_length)
+          relpath = generate_relpath(Attache::Upload.sanitize(tus.upload_metadata['filename'] || params['file']))
+          cachekey = File.join(request_hostname(env), relpath)
+
+          bytes_wrote = Attache.cache.write(cachekey, StringIO.new)
+          uri = URI.parse(Rack::Request.new(env).url)
+          uri.query = (uri.query ? "#{uri.query}&" : '') + "relpath=#{CGI.escape relpath}"
+          [201, tus.headers_with_cors('Location' => uri.to_s), []]
+        else
+          [400, tus.headers_with_cors('X-Exception' => "Bad upload length"), []]
+        end
+
+      when 'PATCH'
+        relpath = params['relpath']
+        cachekey = File.join(request_hostname(env), relpath)
+        http_offset = tus.upload_offset
+        if positive_number?(env['CONTENT_LENGTH']) &&
+           positive_number?(http_offset) &&
+           (env['CONTENT_TYPE'] == 'application/offset+octet-stream') &&
+           tus.resumable_version.to_s == '1.0.0' &&
+           current_offset(cachekey, relpath, config) >= http_offset.to_i
+
+          append_to(cachekey, http_offset, env['rack.input'])
+          config.storage_create(relpath: relpath, cachekey: cachekey) if config.storage && config.bucket
+
+          [200,
+            tus.headers_with_cors({'Content-Type' => 'text/json'}, offset: current_offset(cachekey, relpath, config)),
+            [json_of(relpath, cachekey)],
+          ]
+        else
+          [400, tus.headers_with_cors('X-Exception' => 'Bad headers'), []]
+        end
+
+      when 'OPTIONS'
+        [201, tus.headers_with_cors, []]
+
+      when 'HEAD'
+        relpath = params['relpath']
+        cachekey = File.join(request_hostname(env), relpath)
+        [200,
+          tus.headers_with_cors({'Content-Type' => 'text/json'}, offset: current_offset(cachekey, relpath, config)),
+          [json_of(relpath, cachekey)],
+        ]
+
+      when 'GET'
+        relpath = params['relpath']
+        uri = URI.parse(Rack::Request.new(env).url)
+        uri.query = nil
+        uri.path = File.join('/view', File.dirname(relpath), 'original', CGI.escape(File.basename(relpath)))
+        [302, tus.headers_with_cors('Location' => uri.to_s), []]
+      end
+    else
+      @app.call(env)
+    end
+  rescue Exception
+    Attache.logger.error $@
+    Attache.logger.error $!
+    Attache.logger.error "ERROR REFERER #{env['HTTP_REFERER'].inspect}"
+    [500, { 'X-Exception' => $!.to_s }, []]
+  end
+
+  private
+
+    def current_offset(cachekey, relpath, config)
+      file = Attache.cache.fetch(cachekey) do
+        config.storage_get(relpath: relpath) if config.storage && config.bucket
+      end
+      file.size
+    rescue
+      Attache.cache.write(cachekey, StringIO.new)
+    ensure
+      file.tap(&:close)
+    end
+
+    def append_to(cachekey, offset, io)
+      f = File.open(path_of(cachekey), 'r+b')
+      f.sync = true
+      f.seek(offset.to_i)
+      f.write(io.read)
+    ensure
+      f.close
+    end
+
+    def positive_number?(value)
+      (value.to_s == "0" || value.to_i > 0)
+    end
+
+end

data/lib/attache/upload.rb

@@ -0,0 +1,44 @@
+class Attache::Upload < Attache::Base
+  def initialize(app)
+    @app = app
+  end
+
+  def _call(env, config)
+    case env['PATH_INFO']
+    when '/upload'
+      case env['REQUEST_METHOD']
+      when 'POST', 'PUT', 'PATCH'
+        request  = Rack::Request.new(env)
+        params   = request.params
+        return config.unauthorized unless config.authorized?(params)
+
+        relpath = generate_relpath(Attache::Upload.sanitize params['file'])
+        cachekey = File.join(request_hostname(env), relpath)
+
+        bytes_wrote = Attache.cache.write(cachekey, request.body)
+        if bytes_wrote == 0
+          return [500, config.headers_with_cors.merge('X-Exception' => 'Local file failed'), []]
+        end
+
+        config.storage_create(relpath: relpath, cachekey: cachekey) if config.storage && config.bucket
+
+        [200, config.headers_with_cors.merge('Content-Type' => 'text/json'), [json_of(relpath, cachekey)]]
+      when 'OPTIONS'
+        [200, config.headers_with_cors, []]
+      else
+        [400, config.headers_with_cors, []]
+      end
+    else
+      @app.call(env)
+    end
+  rescue Exception
+    Attache.logger.error $@
+    Attache.logger.error $!
+    Attache.logger.error "ERROR REFERER #{env['HTTP_REFERER'].inspect}"
+    [500, { 'X-Exception' => $!.to_s }, []]
+  end
+
+  def self.sanitize(filename)
+    filename.to_s.gsub(/\%/, '_')
+  end
+end

data/lib/attache/version.rb

@@ -0,0 +1,3 @@
+module Attache
+  VERSION = "1.0.0"
+end

data/lib/attache/vhost.rb

@@ -0,0 +1,89 @@
+class Attache::VHost
+  attr_accessor :remotedir,
+                :secret_key,
+                :bucket,
+                :storage,
+                :download_headers,
+                :headers_with_cors,
+                :env
+
+  def initialize(hash)
+    self.env = hash || {}
+    self.remotedir  = env['REMOTE_DIR'] # nil means no fixed top level remote directory, and that's fine.
+    self.secret_key = env['SECRET_KEY'] # nil means no auth check; anyone can upload a file
+    if env['FOG_CONFIG']
+      self.bucket       = env['FOG_CONFIG'].fetch('bucket')
+      self.storage      = Fog::Storage.new(env['FOG_CONFIG'].except('bucket').symbolize_keys)
+    end
+    self.download_headers = {
+      "Cache-Control" => "public, max-age=31536000"
+    }.merge(env['DOWNLOAD_HEADERS'] || {})
+    self.headers_with_cors = {
+      'Access-Control-Allow-Origin' => '*',
+      'Access-Control-Allow-Methods' => 'POST, PUT',
+      'Access-Control-Allow-Headers' => 'Content-Type',
+    }.merge(env['UPLOAD_HEADERS'] || {})
+  end
+
+  def hmac_for(content)
+    OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), secret_key, content)
+  end
+
+  def hmac_valid?(params)
+    params['uuid'] &&
+    params['hmac']  &&
+    params['expiration'] &&
+    Time.at(params['expiration'].to_i) > Time.now &&
+    Rack::Utils.secure_compare(params['hmac'], hmac_for("#{params['uuid']}#{params['expiration']}"))
+  end
+
+  def storage_url(args)
+    remote_api.new({
+      key: File.join(*remotedir, args[:relpath]),
+    }).url(Time.now + 60)
+  end
+
+  def storage_get(args)
+    open storage_url(args)
+  end
+
+  def storage_create(args)
+    Attache.logger.info "[JOB] uploading #{args[:cachekey].inspect}"
+    body = begin
+      Attache.cache.read(args[:cachekey])
+    rescue Errno::ENOENT
+      :no_entry # upload file no longer exist; likely deleted immediately after upload
+    end
+    unless body == :no_entry
+      remote_api.create({
+        key: File.join(*remotedir, args[:relpath]),
+        body: body,
+      })
+      Attache.logger.info "[JOB] uploaded #{args[:cachekey]}"
+    end
+  end
+
+  def storage_destroy(args)
+    Attache.logger.info "[JOB] deleting #{args[:relpath]}"
+    remote_api.new({
+      key: File.join(*remotedir, args[:relpath]),
+    }).destroy
+    Attache.logger.info "[JOB] deleted #{args[:relpath]}"
+  end
+
+  def remote_api
+    storage.directories.new(key: bucket).files
+  end
+
+  def async(method, args)
+    ::Attache::Job.perform_async(method, env, args)
+  end
+
+  def authorized?(params)
+    secret_key.blank? || hmac_valid?(params)
+  end
+
+  def unauthorized
+    [401, headers_with_cors.merge('X-Exception' => 'Authorization failed'), []]
+  end
+end