s3browser 0.1.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a2af7ec1f09ccd3a5a9108b4db4cd02fff8d55bc
-  data.tar.gz: 37bc0b82fc54b2716711388437e688d32db12e06
+  metadata.gz: 2f0b52f998b7d61530d1781fa9fde463d59aa1bc
+  data.tar.gz: ecafe26c558b8dfece67e41abd119855627667fe
 SHA512:
-  metadata.gz: bfe118747868fdb3e74470a4b9d9551c97a988fc2efbb1f3b038d3c8604053384fb1df67e701b013a64d674605370acec46c53049463566e7d1bf2edbda4d664
-  data.tar.gz: 0b807d66b7fa7dc2f97cf2ba658d1dcb9a643c9c1cf0c91a6d965314032652c7548f93210a2fcbbdae7b26cbad6d5dc0e4c0e12cee016773f939a9a5273521e4
+  metadata.gz: 83213e2b455ea473babb63dfd34e035563554dbeb8451671e5c7b69a3729722802aeec80db15bbdfe5b2ca4761c3e18abbde63a370630b940eeb9ba0f939007c
+  data.tar.gz: ecd3b4f3905cec378da87d8204d88695c0c2b3699960f1841c54f8435700de3e61d261697ad7d3036288cccbbb4013014a8f89a96efc485b06ce228fe6dca109

data/.ruby-version

@@ -1 +1 @@
-2.3.0
+2.3.1

data/README.md

@@ -10,6 +10,7 @@ This wrapper gives you a couple of killer functions:
 * [x] Automatically update file information
 * [x] Upload files
 * [x] Download files
+* [x] Delete files
 
 ## Installation
 
@@ -33,6 +34,7 @@ Here's an example config.ru for booting S3Browser::Server in your choice of Rack
 
 ```ruby
 # config.ru
+#\-o 0.0.0.0 -p 9292
 require 's3browser/server'
 run S3Browser::Server
 ```
@@ -147,6 +149,23 @@ After setting up the queue and the bucket, a policy **without** the following ac
 * sqs:GetQueueAttributes
 * sqs:SetQueueAttributes
 
+## S3Proxy
+
+You might not want to allow public access to your S3 bucket. To work around this, the gem provides a proxy with which the server will fetch the file from S3 and serve it to the client, caching it in between.
+
+It's a simple Sinatra app, so can be mounted on a Rack endpoint as well:
+
+```
+# config.ru
+#\-o 0.0.0.0 -p 9292
+require 's3browser/server'
+require 's3browser/s3proxy'
+
+run Rack::URLMap.new('/s3' => S3Browser::S3Proxy, '/' => S3Browser::Server)
+```
+
+The above example allows you to set the `S3BROWSER_THUMBNAIL_URL` and `S3BROWSER_OBJECT_URL` environmental variables to `/s3` so that you can download and view the files from the browser.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Vagrantfile

@@ -29,6 +29,7 @@ Vagrant.configure(2) do |config|
     sudo apt-get install -y elasticsearch
     sudo service elasticsearch stop
     echo "ES_HEAP_SIZE=1g" | sudo tee -a /etc/default/elasticsearch
+    echo "network.host: 0.0.0.0" | sudo tee -a /etc/elasticsearch/elasticsearch.yml
 
     # Dependencies / Utilities
     sudo apt-get install -y screen curl git build-essential

data/lib/s3browser/fetch.rb

@@ -4,6 +4,8 @@ require 's3browser/store'
 
 module S3Browser
   class Fetch
+    raise 'Unconfigured' unless ENV['AWS_REGION']
+
     class Store < S3Browser::Store
       plugin :es
       plugin :images

data/lib/s3browser/plugins/es.rb

@@ -21,27 +21,28 @@ module S3Browser
             # TODO Can be optimized to do a bulk index every X requests
             object[:bucket] = bucket
             object[:last_modified] = object[:last_modified].to_i
-            object[:url] = "http://#{bucket}.s3.amazonaws.com/#{object[:key]}"
+            object[:url] = "#{object_url}/#{bucket}/#{object[:key]}"
             client.index(index: index, type: 'objects', id: object[:key], body: object)
             super(bucket, object)
           end
 
+          def remove(bucket, key)
+            client.delete(index: index, type: 'objects', id: key, ignore: [404])
+            super(bucket, key)
+          end
+
           def objects(bucket, options)
-            body = get_body(bucket, options)
-            if options[:key]
-              raise 'Not implemented yet'
-            else
-              result = client.search(index: index, type: 'objects', body: body)
-              result['hits']['hits'].map {|val| val['_source'].inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} }
-            end
+            result = client.search(index: index, type: 'objects', body: search_body(bucket, options))
+            result['hits']['hits'].map {|val| val['_source'].inject({}){|memo,(k,v)| memo[k.downcase.to_sym] = v; memo} }
           end
 
-          def search(bucket, term, options = {})
-            get(bucket, {term: term}.merge(options))
+          def object(bucket, key)
+            result = client.get(index: index, type: 'objects', id: key)
+            result['_source'].inject({}){|memo,(k,v)| memo[k.downcase.to_sym] = v; memo}
           end
 
           def buckets
-            client.search(index: index, type: 'objects', body: {
+            buckets = client.search(index: index, type: 'objects', body: {
               query: { match_all: {} },
               size: 0,
               aggregations: {
@@ -54,6 +55,8 @@ module S3Browser
                 }
               }
             })['aggregations']['buckets']['buckets'].map {|val| val['key'] }
+            return buckets unless buckets.empty?
+            super
           end
 
           def indices
@@ -78,7 +81,7 @@ module S3Browser
           end
 
           private
-          def get_body(bucket, options = {})
+          def search_body(bucket, options = {})
             body = {
               query: {
                 bool: {

data/lib/s3browser/plugins/images.rb

@@ -3,26 +3,36 @@ module S3Browser
     module StorePlugins
       module Images
         module InstanceMethods
-          def add(bucket, object)
-            return super(bucket, object) unless handle?(object)
-            # TODO Create a thumbnail and make it available
-
-            super(bucket, object)
-          end
-
           def objects(bucket, options)
-            result = super(bucket, options)
-            result.map do |elm|
-              elm[:thumbnail] = {
-                url: "http://#{bucket}.s3.amazonaws.com/#{elm[:key]}",
+            super(bucket, options).map do |object|
+              object[:thumbnail] = {
+                url: "#{thumbnail_url}/#{bucket}/#{object[:key]}",
                 width: 200
-              } if handle?(elm)
-              elm
+              } if handle?(object)
+              object
             end
           end
 
+          def object(bucket, key)
+            object = super(bucket, key)
+            object[:thumbnail] = {
+              url: "#{thumbnail_url}/#{bucket}/#{object[:key]}",
+              width: 200
+            } if handle?(object)
+            object
+          end
+
           def handle?(object)
-            object[:content_type] =~ /^image\/.*/
+            return (object[:content_type] =~ /^image\/.*/) unless (object[:content_type].nil? || object[:content_type] == '')
+            return (object[:type] =~ /^image\/.*/) unless (object[:type].nil? || object[:type] == '')
+            object[:key] =~ /(jpg|jpeg|png|gif|bmp)$/
+          end
+
+          def thumbnail_url
+            @thumbnail_url ||= begin
+              return ENV['S3BROWSER_THUMBNAIL_URL'] if ENV['S3BROWSER_THUMBNAIL_URL']
+              "http://s3-#{ENV['AWS_REGION']}.amazonaws.com"
+            end
           end
         end
       end

data/lib/s3browser/plugins/manager.rb

@@ -0,0 +1,30 @@
+require 'aws-sdk'
+
+module S3Browser
+  class Store
+    module StorePlugins
+      module Manager
+        module InstanceMethods
+          def upload(bucket, file)
+            filename = file[:filename]
+            tempfile = file[:tempfile]
+
+            s3 = Aws::S3::Resource.new
+            s3.bucket(bucket).object(filename).upload_file(tempfile.path)
+
+            super(bucket, file)
+          end
+
+          def delete(bucket, file)
+            s3 = Aws::S3::Client.new
+            s3.delete_object(bucket: bucket, key: file)
+
+            super(bucket, file)
+          end
+        end
+      end
+
+      register_plugin(:manager, Manager)
+    end
+  end
+end

data/lib/s3browser/policy.json

@@ -2,7 +2,7 @@
   "Version": "2012-10-17",
   "Statement": [
     {
-      "Sid": "example-statement-ID",
+      "Sid": "S3BrowserGenerateSQS",
       "Effect": "Allow",
       "Principal": { "AWS": "*" },
       "Action": [
@@ -10,8 +10,8 @@
       ],
       "Resource": "SQS-queue-ARN",
       "Condition": {
-        "ArnLike": {          
-          "aws:SourceArn": "arn:aws:s3:*:*:bucket-name"    
+        "ArnLike": {
+          "aws:SourceArn": "arn:aws:s3:*:*:bucket-name"
         }
       }
     }

data/lib/s3browser/s3proxy.rb

@@ -0,0 +1,55 @@
+require 'sinatra/base'
+
+module S3Browser
+  class S3Proxy < Sinatra::Base
+    raise 'Unconfigured' unless ENV['AWS_REGION']
+
+    configure :development do
+      enable :logging
+    end
+
+    get %r{(.+)} do |filename|
+      read_file filename
+    end
+
+    helpers do
+      def read_file(file)
+        cached_file = cache_location(file)
+
+        matches = file.match(/([^\/]+)\/(.*)/)
+        options = {
+          bucket: matches[1],
+          key: matches[2],
+        }
+        options[:if_modified_since] = File.mtime(cached_file) if File.exists?(cached_file)
+
+        begin
+          s3object = s3.get_object(options, target: cached_file)
+        rescue Aws::S3::Errors::NotModified
+          options.delete(:if_modified_since)
+          s3object = s3.head_object(options)
+        rescue
+          halt(404)
+        end
+        send_file cached_file, file_name: File.basename(file), type: s3object.content_type
+      end
+
+      def cache_location(file)
+        matches = file.match(/([^\/]+)\/(.*)/)
+        "#{cache_folder}/#{matches[2]}"
+      end
+
+      def cache_folder
+        path = '/tmp/s3proxy'
+        Dir.mkdir(path) unless File.exists?(path)
+        path
+      end
+
+      def s3
+        @s3 ||= Aws::S3::Client.new
+      end
+    end
+
+    run! if app_file == $0
+  end
+end

data/lib/s3browser/server.rb

@@ -5,20 +5,24 @@ require 's3browser/store'
 
 module S3Browser
   class Server < Sinatra::Base
-    enable :sessions
+    raise 'Unconfigured' unless ENV['AWS_REGION']
+
     use Rack::Flash
 
     class Store < S3Browser::Store
       plugin :es
       plugin :images
-      plugin :upload
+      plugin :manager
+    end
+
+    configure do
+      enable :sessions
+      enable :method_override
+      set :store, Store.new('s3browser')
     end
 
     configure :development do
-      set :port, 9292
-      set :bind, '0.0.0.0'
       enable :logging
-      set :store, Store.new('s3browser')
     end
 
     get '/' do
@@ -26,19 +30,38 @@ module S3Browser
       haml :index, locals: { title: 'S3Browser', buckets: buckets }
     end
 
-    get '/:bucket' do |bucket|
+    get '/:bucket/?' do |bucket|
       params['q'] = nil if params['q'] == ''
       objects = settings.store.objects(bucket, term: params['q'], sort: params['s'], direction: params['d'])
       haml :bucket, locals: { title: bucket, bucket: bucket, objects: objects, q: params['q'] }
     end
 
-    post '/upload' do
+    get '/:bucket/:key/?' do |bucket, key|
+      begin
+        object = settings.store.object(bucket, key)
+      rescue
+        halt(404)
+      end
+      haml :object, locals: { title: key, bucket: bucket, key: key, object: object }
+    end
+
+    delete '/:bucket/:key/?' do |bucket, key|
+      begin
+        settings.store.delete(bucket, key)
+        flash[:success] = 'File deleted'
+      rescue StandardError => e
+        flash[:error] = 'Could not remove the file: ' + e.message
+      end
+      redirect back
+    end
+
+    post '/upload/:bucket/?' do |bucket|
       if params['upload']
         begin
-          settings.store.upload params['upload']
+          settings.store.upload bucket, params['upload']
           flash[:success] = 'File uploaded'
-        rescue
-          flash[:error] = 'Could not upload the file'
+        rescue StandardError => e
+          flash[:error] = 'Could not upload the file: ' + e.message
         end
       end
       redirect back

data/lib/s3browser/store.rb

@@ -71,25 +71,39 @@ module S3Browser
         end
 
         module InstanceMethods
-          def initialize(bucket)
+          def add(bucket, object)
+            nil
           end
 
-          def add(bucket, object)
+          def remove(bucket, key)
             nil
           end
 
+          def upload(bucket, file)
+          end
+
+          def delete(bucket, file)
+          end
+
           def objects(bucket, options)
             s3.list_objects(bucket: bucket).contents.map do |object|
-              object.to_h.merge(bucket: bucket, url: "http://#{bucket}.s3.amazonaws.com/#{object[:key]}")
+              object.to_h.merge(bucket: bucket, url: "#{object_url}/#{bucket}/#{object[:key]}")
             end
           end
 
-          def search(bucket, term, options = {})
-            []
+          def object(bucket, key)
+            s3.head_object(bucket: bucket, key: key)
           end
 
           def buckets
-            s3.list_buckets.buckets
+            s3.list_buckets.buckets.map {|val| val['name'] }
+          end
+
+          def object_url
+            @object_url ||= begin
+              return ENV['S3BROWSER_OBJECT_URL'] if ENV['S3BROWSER_OBJECT_URL']
+              "http://s3-#{ENV['AWS_REGION']}.amazonaws.com"
+            end
           end
 
           private

data/lib/s3browser/version.rb

@@ -1,3 +1,3 @@
 module S3Browser
-  VERSION = "0.1.0"
+  VERSION = "0.5.0"
 end

data/lib/s3browser/views/bucket.haml

@@ -5,7 +5,7 @@
         = bucket
       .panel-body
         .col-md-10
-          %form.form-horizontal{method: 'get', action: "/#{bucket}"}
+          %form.form-horizontal{method: 'get', action: url("/#{bucket}")}
             .form-group
               .input-group
                 %input.form-control{ type: 'text', name: 'q', placeholder: 'Search', value: q }
@@ -22,6 +22,7 @@
             = haml :column_header, locals: { field: :key }
             = haml :column_header, locals: { field: :size }
             = haml :column_header, locals: { field: :content_type, field_name: 'Type' }
+            %th Download
             = haml :column_header, locals: { field: :last_modified }
             %th
               Thumbnail
@@ -29,17 +30,18 @@
           -objects.each do |obj|
             %tr
               %td
-                %a{ target: '_blank', href: obj[:url] }
-                  = obj[:key]
+                %a{ href: url("/#{bucket}/#{obj[:key]}") }= obj[:key]
               %td= obj[:size]
               %td= obj[:content_type]
+              %td
+                %a.btn.btn-default{ target: '_blank', href: obj[:url] } Download
               %td= Time.at(obj[:last_modified]).strftime('%Y-%m-%d %H:%M:%S')
               %td
                 -if obj[:thumbnail]
-                  %img{ src: obj[:thumbnail][:url], width: obj[:thumbnail][:width]}
+                  %img.img-response.center-block.img-thumbnail{ src: obj[:thumbnail][:url], width: obj[:thumbnail][:width]}
 
 - if settings.store.respond_to? :upload
-  %form{ enctype: 'multipart/form-data', method: 'post', action: '/upload' }
+  %form{ enctype: 'multipart/form-data', method: 'post', action: '/upload/' + bucket }
     #upload-modal.modal.fade{ tabindex: -1, role: 'dialog' }
       .modal-dialog
         .modal-content

data/lib/s3browser/views/index.haml

@@ -9,4 +9,4 @@
       -buckets.each do |bucket|
         %tr
           %td
-            %a{ href: "/#{bucket}" }= bucket
+            %a{ href: url("/#{bucket}") }= bucket

data/lib/s3browser/views/layout.haml

@@ -3,7 +3,7 @@
   %head
     %meta{:charset => 'utf-8'}
     %title
-      S3Browser 
+      S3Browser
       - if defined? title
         = "- #{title}"
     %meta{:name => "description", :content => ""}
@@ -34,6 +34,5 @@
 
     / Placed at the end of the document so the pages load faster
     %script{ type: 'text/javascript', src: '//ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js' }
-    %script{ type: 'text/javascript', src: '//maps.googleapis.com/maps/api/js?libraries=places' }
     %script{ type: 'text/javascript', src: '//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js' }
 

data/lib/s3browser/views/object.haml

@@ -0,0 +1,39 @@
+.row
+  .col-md-12
+    .panel.panel-primary
+      .panel-heading
+        = "#{bucket} / #{key}"
+
+      %table.table.table-hover
+        %tr
+          %th Key
+          %td= object[:key]
+        %tr
+          %th Bucket
+          %td= object[:bucket]
+        %tr
+          %th eTag
+          %td= object[:etag]
+        %tr
+          %th Size
+          %td= object[:size]
+        %tr
+          %th Type
+          %td= object[:content_type]
+        %tr
+          %th Last Modified
+          %td= Time.at(object[:last_modified]).strftime('%Y-%m-%d %H:%M:%S')
+        %tr
+          %th Storage Class
+          %td= object[:storage_class]
+        %tr
+          %td.text-left
+            %a.btn.btn-default{ target: '_base', href: object[:url] } Download
+          %td.text-right
+            %form{ method: 'post', action: url("/#{bucket}/#{object[:key]}") }
+              %input{ type: 'hidden', name: '_method', value: 'delete'}
+              %button.btn.btn-warning{ type: 'submit', onclick: 'return confirm("This will completely remove the file.");' } Delete
+-if object[:thumbnail]
+  .row
+    .col-md-12
+      %img.img-response.img-thumbnail.center-block{ src: object[:url] }

data/lib/s3browser/worker.rb

@@ -5,6 +5,8 @@ module S3Browser
   class Worker
     include Shoryuken::Worker
 
+    raise 'Unconfigured' unless ENV['AWS_REGION']
+
     shoryuken_options queue: ENV['AWS_SQS_QUEUE'], body_parser: :json, auto_delete: true
 
     class Store < S3Browser::Store
@@ -12,22 +14,44 @@ module S3Browser
       plugin :images
     end
 
+    def initialize
+      Shoryuken.logger.level = ::Logger::DEBUG if ENV['RACK_ENV'] != 'production'
+    end
+
     def perform(sqs_msg, body)
+      Shoryuken.logger.debug body
+
       if body['Records']
         body['Records'].each do |record|
-          bucket = record['s3']['bucket']['name']
-          key    = record['s3']['object']['key']
-
-          info = s3.head_object({ bucket: bucket, key: key })
-          info = info.to_h.merge(record['s3']['object'])
-
-          store.add bucket, info
+          case record['eventName']
+          when 'ObjectRemoved:Delete'
+            remove record
+          when 'ObjectCreated:Put'
+            add record
+          end
         end
       else
         raise unless body['Event'] == 's3:TestEvent'
       end
     end
 
+    def add(record)
+      bucket = record['s3']['bucket']['name']
+      key    = record['s3']['object']['key']
+
+      info = s3.head_object({ bucket: bucket, key: key })
+      info = info.to_h.merge(record['s3']['object'].inject({}){|memo,(k,v)| memo[k.downcase.to_sym] = v; memo})
+
+      store.add bucket, info
+    end
+
+    def remove(record)
+      bucket = record['s3']['bucket']['name']
+      key    = record['s3']['object']['key']
+
+      store.remove bucket, key
+    end
+
     private
     def store
       @store ||= Store.new('s3browser')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3browser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Jurgens du Toit
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-20 00:00:00.000000000 Z
+date: 2016-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -173,8 +173,9 @@ files:
 - lib/s3browser/gem_tasks.rb
 - lib/s3browser/plugins/es.rb
 - lib/s3browser/plugins/images.rb
-- lib/s3browser/plugins/upload.rb
+- lib/s3browser/plugins/manager.rb
 - lib/s3browser/policy.json
+- lib/s3browser/s3proxy.rb
 - lib/s3browser/server.rb
 - lib/s3browser/store.rb
 - lib/s3browser/version.rb
@@ -183,6 +184,7 @@ files:
 - lib/s3browser/views/flash_messages.haml
 - lib/s3browser/views/index.haml
 - lib/s3browser/views/layout.haml
+- lib/s3browser/views/object.haml
 - lib/s3browser/worker.rb
 - s3browser.gemspec
 homepage: https://github.com/jrgns/s3browser

data/lib/s3browser/plugins/upload.rb

@@ -1,22 +0,0 @@
-require 'aws-sdk'
-
-module S3Browser
-  class Store
-    module StorePlugins
-      module Upload
-        module InstanceMethods
-          def upload(file)
-            filename = file[:filename]
-            file     = file[:tempfile]
-            bucket   = 's3browser'
-
-            s3 = Aws::S3::Resource.new
-            s3.bucket(bucket).object(filename).upload_file(file.path)
-          end
-        end
-      end
-
-      register_plugin(:upload, Upload)
-    end
-  end
-end