attache 1.0.5 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 712ee55417c8e98d23531489e2e094c5848fbe85
-  data.tar.gz: e7961934d82f5da524db5f0e238c8002bfafbc04
+  metadata.gz: 8a5ec5e8b4431d1505fb6519428f01091da7a658
+  data.tar.gz: 1bdfa46bee9d2015ff5afc0fd9caaff55bb3f5f2
 SHA512:
-  metadata.gz: 45e693e294ae935e974c003f7ffbcdf905cb68a73ce87dedccaf7f637a075f29e949ad42390106101b7924325beb67c42c74b3bdd5b9b3f636ac2d6ecc42047e
-  data.tar.gz: 3b20dede4e93a1b95eca4cf8e4b37fd0385630729dfd47176b999e1b8b14063637383b09dde95a59ed3ff0c7cf4c4c33f1d2de53089273ed06e9beda0f183700
+  metadata.gz: 00659d9c0716246aa7c501dd6a37f8e6fd6a0e2c4e9ec23f17418f7536362f013ee05c885b9cbd74b7159b68ed12d33cb83326b4c1ff6ef8eca2b31ab46cb0fa
+  data.tar.gz: 1b0cb02c0d32909d23f5b5cb120e5a463f0aaa323c9304367ed28fc6f4b9d884ff2ce085f2db99654cca892d01355e7a99e00bd7482a456f768ca63179cd70b2

data/README.md

@@ -173,6 +173,23 @@ Removing 1 or more files from the local cache and remote storage can be done via
 
 The `paths` value should be delimited by the newline character, aka `\n`. In the example above, 3 files will be requested for deletion: `image1.jpg`, `prefix2/image2.jpg`, and `image3.jpg`.
 
+#### Backup
+
+> ```
+> POST /backup
+> paths=image1.jpg%0Aprefix2%2Fimage2.jpg%0Aimage3.jpg
+> ```
+
+Copying 1 or more files from the default remote storage to the backup remote storage (backup) can be done via a http `POST` request to `/backup`, with a `paths` parameter in the request body.
+
+The `paths` value should be delimited by the newline character, aka `\n`. In the example above, 3 files will be requested for backup: `image1.jpg`, `prefix2/image2.jpg`, and `image3.jpg`.
+
+If backup remote storage is not configured, this API call will be a noop. If configured, the backup storage must be accessible by the same credentials as default cloud storage as the system. Please refer to the `BACKUP_CONFIG` configuration illustrated in `config/vhost.example.yml` file in this repository.
+
+By default, `backup` operation is performed synchronously. Set `BACKUP_ASYNC` environment variable to make it follow the same synchronicity as `delete`
+
+The main reason to configure a backup storage is to make the default cloud storage auto expire files; mitigating [abuse](https://github.com/choonkeat/attache/issues/13). You should consult the documentation of your cloud storage provider on how to setup auto expiry, e.g. [here](https://aws.amazon.com/blogs/aws/amazon-s3-object-expiration/) or [here](https://cloud.google.com/storage/docs/lifecycle)
+
 ## License
 
 MIT

data/config.ru

@@ -4,6 +4,7 @@ use Attache::Delete
 use Attache::Upload
 use Attache::Download
 use Attache::Tus::Upload
+use Attache::Backup
 use Rack::Static, urls: ["/"], root: Attache.publicdir, index: "index.html"
 
 run proc {|env| [200, {}, []] }

data/config/vhost.example.yml

@@ -20,6 +20,9 @@
     "aws_secret_access_key": CHANGEME
     "bucket": CHANGEME
     "region": us-west-1
+  "BACKUP_CONFIG":
+    "bucket": CHANGEME_BAK
+    # only supports 1 key: `bucket`
 
 # This section will only take effect if a request is made to `localhost:9292`
 "localhost:9292":

data/lib/attache.rb

@@ -52,6 +52,7 @@ require 'attache/base'
 require 'attache/vhost'
 require 'attache/upload'
 require 'attache/delete'
+require 'attache/backup'
 require 'attache/download'
 require 'attache/file_response_body'
 

data/lib/attache/backup.rb

@@ -0,0 +1,29 @@
+class Attache::Backup < Attache::Base
+  def initialize(app)
+    @app = app
+  end
+
+  def _call(env, config)
+    case env['PATH_INFO']
+    when '/backup'
+      request  = Rack::Request.new(env)
+      params   = request.params
+      return config.unauthorized unless config.authorized?(params)
+
+      if config.storage && config.bucket
+        sync_method = (ENV['BACKUP_ASYNC'] ? :async : :send)
+        params['paths'].to_s.split("\n").each do |relpath|
+          Attache.logger.info "BACKUP remote #{relpath}"
+          config.send(sync_method, :backup_file, relpath: relpath)
+        end
+      end
+      [200, config.headers_with_cors, []]
+    else
+      @app.call(env)
+    end
+  rescue Exception
+    Attache.logger.error $@
+    Attache.logger.error $!
+    [500, { 'X-Exception' => $!.to_s }, []]
+  end
+end

data/lib/attache/delete.rb

@@ -11,13 +11,28 @@ class Attache::Delete < Attache::Base
       return config.unauthorized unless config.authorized?(params)
 
       params['paths'].to_s.split("\n").each do |relpath|
-        Attache.logger.info "DELETING local #{relpath}"
-        cachekey = File.join(request_hostname(env), relpath)
-        Attache.cache.delete(cachekey)
+        threads = []
+
+        if Attache.cache
+          threads << Thread.new do
+            Attache.logger.info "DELETING local #{relpath}"
+            cachekey = File.join(request_hostname(env), relpath)
+            Attache.cache.delete(cachekey)
+          end
+        end
         if config.storage && config.bucket
-          Attache.logger.info "DELETING remote #{relpath}"
-          config.async(:storage_destroy, relpath: relpath)
+          threads << Thread.new do
+            Attache.logger.info "DELETING remote #{relpath}"
+            config.async(:storage_destroy, relpath: relpath)
+          end
+        end
+        if config.backup
+          threads << Thread.new do
+            Attache.logger.info "DELETING backup #{relpath}"
+            config.backup.async(:storage_destroy, relpath: relpath)
+          end
         end
+        threads.each(&:join)
       end
       [200, config.headers_with_cors, []]
     else

data/lib/attache/download.rb

@@ -1,7 +1,6 @@
 require 'connection_pool'
 
 class Attache::Download < Attache::Base
-  REMOTE_GEOMETRY = ENV.fetch('REMOTE_GEOMETRY') { 'remote' }
   OUTPUT_EXTENSIONS = %w[png jpg jpeg gif]
   RESIZE_JOB_POOL = ConnectionPool.new(JSON.parse(ENV.fetch('RESIZE_POOL') { '{ "size": 2, "timeout": 60 }' }).symbolize_keys) { Attache::ResizeJob.new }
 
@@ -12,10 +11,14 @@ class Attache::Download < Attache::Base
   def _call(env, config)
     case env['PATH_INFO']
     when %r{\A/view/}
+      vhosts = {}
+      vhosts[ENV.fetch('REMOTE_GEOMETRY') { 'remote' }] = config.storage && config.bucket && config
+      vhosts[ENV.fetch('BACKUP_GEOMETRY') { 'backup' }] = config.backup
+
       parse_path_info(env['PATH_INFO']['/view/'.length..-1]) do |dirname, geometry, basename, relpath|
-        if geometry == REMOTE_GEOMETRY && config.storage && config.bucket
-          headers = config.download_headers.merge({
-                      'Location' => config.storage_url(relpath: relpath),
+        if vhost = vhosts[geometry]
+          headers = vhost.download_headers.merge({
+                      'Location' => vhost.storage_url(relpath: relpath),
                       'Cache-Control' => 'private, no-cache',
                     })
           return [302, headers, []]
@@ -24,11 +27,11 @@ class Attache::Download < Attache::Base
         file = begin
           cachekey = File.join(request_hostname(env), relpath)
           Attache.cache.fetch(cachekey) do
-            config.storage_get(relpath: relpath) if config.storage && config.bucket
+            get_first_result_async(vhosts.inject({}) {|sum,(k,v)|
+              v ? sum.merge("#{k} #{relpath}" => lambda { v.storage_get(relpath: relpath) }) : sum
+            })
           end
         rescue Exception # Errno::ECONNREFUSED, OpenURI::HTTPError, Excon::Errors, Fog::Errors::Error
-          Attache.logger.error $@
-          Attache.logger.error $!
           Attache.logger.error "ERROR REFERER #{env['HTTP_REFERER'].inspect}"
           nil
         end
@@ -37,7 +40,8 @@ class Attache::Download < Attache::Base
           return [404, config.download_headers, []]
         end
 
-        thumbnail = if geometry == 'original' || geometry == REMOTE_GEOMETRY
+        thumbnail = case geometry
+        when 'original', *vhosts.keys
           file
         else
           extension = basename.split(/\W+/).last
@@ -83,4 +87,25 @@ class Attache::Download < Attache::Base
       end
     end
 
+    def get_first_result_async(name_code_pairs)
+      result = nil
+      threads = name_code_pairs.collect {|name, code|
+        Thread.new do
+          Thread.handle_interrupt(BasicObject => :on_blocking) { # if killed
+            begin
+              if current_result = code.call
+                result = current_result
+                (threads - [Thread.current]).each(&:kill)        # kill siblings
+                Attache.logger.info "[POOL] found #{name.inspect}"
+              end
+            rescue Exception
+              Attache.logger.info "[POOL] not found #{name.inspect}"
+            ensure
+            end
+          }
+        end
+      }
+      threads.each(&:join)
+      result
+    end
 end

data/lib/attache/version.rb

@@ -1,3 +1,3 @@
 module Attache
-  VERSION = "1.0.5"
+  VERSION = "1.1.0"
 end

data/lib/attache/vhost.rb

@@ -1,6 +1,7 @@
 class Attache::VHost
   attr_accessor :remotedir,
                 :secret_key,
+                :backup,
                 :bucket,
                 :storage,
                 :download_headers,
@@ -14,6 +15,11 @@ class Attache::VHost
     if env['FOG_CONFIG']
       self.bucket       = env['FOG_CONFIG'].fetch('bucket')
       self.storage      = Fog::Storage.new(env['FOG_CONFIG'].except('bucket').symbolize_keys)
+
+      if env['BACKUP_CONFIG']
+        backup_fog = env['FOG_CONFIG'].merge(env['BACKUP_CONFIG'])
+        self.backup = Attache::VHost.new(env.except('BACKUP_CONFIG').merge('FOG_CONFIG' => backup_fog))
+      end
     end
     self.download_headers = {
       "Cache-Control" => "public, max-age=31536000"
@@ -86,4 +92,11 @@ class Attache::VHost
   def unauthorized
     [401, headers_with_cors.merge('X-Exception' => 'Authorization failed'), []]
   end
+
+  def backup_file(args)
+    if backup
+      key = File.join(*remotedir, args[:relpath])
+      storage.copy_object(bucket, key, backup.bucket, key)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: attache
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.1.0
 platform: ruby
 authors:
 - choonkeat
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-16 00:00:00.000000000 Z
+date: 2015-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -249,6 +249,7 @@ files:
 - config/vhost.example.yml
 - exe/attache
 - lib/attache.rb
+- lib/attache/backup.rb
 - lib/attache/base.rb
 - lib/attache/delete.rb
 - lib/attache/deleteme.log