s3browser 0.1.0

data/lib/s3browser/plugins/es.rb

@@ -0,0 +1,206 @@
+require 'elasticsearch'
+require 'logger'
+
+module S3Browser
+  class Store
+    module StorePlugins
+      module ES
+        def self.configure(plugin)
+          # TODO Maybe setup and check index here?
+        end
+
+        module InstanceMethods
+          attr_reader :index
+
+          def initialize(index)
+            @index = index
+            check_index
+          end
+
+          def add(bucket, object)
+            # TODO Can be optimized to do a bulk index every X requests
+            object[:bucket] = bucket
+            object[:last_modified] = object[:last_modified].to_i
+            object[:url] = "http://#{bucket}.s3.amazonaws.com/#{object[:key]}"
+            client.index(index: index, type: 'objects', id: object[:key], body: object)
+            super(bucket, object)
+          end
+
+          def objects(bucket, options)
+            body = get_body(bucket, options)
+            if options[:key]
+              raise 'Not implemented yet'
+            else
+              result = client.search(index: index, type: 'objects', body: body)
+              result['hits']['hits'].map {|val| val['_source'].inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} }
+            end
+          end
+
+          def search(bucket, term, options = {})
+            get(bucket, {term: term}.merge(options))
+          end
+
+          def buckets
+            client.search(index: index, type: 'objects', body: {
+              query: { match_all: {} },
+              size: 0,
+              aggregations: {
+                buckets: {
+                  terms: {
+                    field: :bucket,
+                    size: 0,
+                    order: { '_term' => :asc }
+                  }
+                }
+              }
+            })['aggregations']['buckets']['buckets'].map {|val| val['key'] }
+          end
+
+          def indices
+            lines = client.cat.indices
+            lines.split("\n").map do |line|
+              line = Hash[*[
+                :health,
+                :state,
+                :index,
+                :primaries ,
+                :replicas,
+                :count,
+                :deleted,
+                :total_size,
+                :size
+              ].zip(line.split(' ')).flatten]
+
+              [:primaries, :replicas, :count, :deleted].each {|key| line[key] = line[key].to_i}
+
+              line
+            end
+          end
+
+          private
+          def get_body(bucket, options = {})
+            body = {
+              query: {
+                bool: {
+                  filter: {
+                    terms: {
+                      bucket: [ bucket ]
+                    }
+                  }
+                }
+              }
+            }
+
+            # Sort using the raw field
+            options[:sort] = 'key.raw' if options[:sort] == 'key'
+            body[:sort] = { options[:sort] => options[:direction] ? options[:direction] : 'asc'} if options[:sort]
+
+            if options[:term]
+              body[:query][:bool][:must] = {
+                simple_query_string: {
+                  fields: [ 'key', 'key.raw' ],
+                  default_operator: 'OR',
+                  query: options[:term]
+                }
+              }
+            end
+
+            body
+          end
+
+          private
+          def client
+            @client ||= ::Elasticsearch::Client.new(client_options)
+          end
+
+          private
+          def client_options
+            {
+              log: true,
+              logger: Logger.new(STDOUT),
+              host: ENV['ELASTICSEARCH_URL'] || 'http://localhost:9200'
+            }
+          end
+
+          private
+          def check_index
+            client.indices.create index: index, body: {
+              settings: {
+                index: {
+                  number_of_shards: 1,
+                  number_of_replicas: 0
+                },
+                analysis: {
+                  analyzer: {
+                    filename: {
+                      type: :custom,
+                      char_filter: [  ],
+                      tokenizer: :standard,
+                      filter: [ :word_delimiter, :standard, :lowercase, :stop ]
+                    }
+                  }
+                }
+              },
+              mappings: mappings
+            }
+          rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+          end
+
+          private
+          def mappings
+            {
+              objects: {
+                _timestamp: {
+                  enabled: false
+                },
+                properties: {
+                  accept_ranges: {
+                    type: :string,
+                    index: :not_analyzed
+                  },
+                  last_modified: {
+                    type: :date
+                  },
+                  content_length: {
+                    type: :integer
+                  },
+                  etag: {
+                    type: :string,
+                    index: :not_analyzed
+                  },
+                  content_type: {
+                    type: :string,
+                    index: :not_analyzed
+                  },
+                  metadata: {
+                    type: :nested
+                  },
+                  key: {
+                    type: :string,
+                    index: :analyzed,
+                    analyzer: :filename,
+                    fields: {
+                      raw: {
+                        type: :string,
+                        index: :not_analyzed
+                      }
+                    }
+                  },
+                  size: {
+                    type: :integer
+                  },
+                  storage_class: {
+                    type: :string,
+                    index: :not_analyzed
+                  }
+                }
+              }
+            }
+          end
+        end
+      end
+
+      register_plugin(:es, ES)
+    end
+  end
+end

data/lib/s3browser/plugins/images.rb

@@ -0,0 +1,33 @@
+module S3Browser
+  class Store
+    module StorePlugins
+      module Images
+        module InstanceMethods
+          def add(bucket, object)
+            return super(bucket, object) unless handle?(object)
+            # TODO Create a thumbnail and make it available
+
+            super(bucket, object)
+          end
+
+          def objects(bucket, options)
+            result = super(bucket, options)
+            result.map do |elm|
+              elm[:thumbnail] = {
+                url: "http://#{bucket}.s3.amazonaws.com/#{elm[:key]}",
+                width: 200
+              } if handle?(elm)
+              elm
+            end
+          end
+
+          def handle?(object)
+            object[:content_type] =~ /^image\/.*/
+          end
+        end
+      end
+
+      register_plugin(:images, Images)
+    end
+  end
+end

data/lib/s3browser/plugins/upload.rb

@@ -0,0 +1,22 @@
+require 'aws-sdk'
+
+module S3Browser
+  class Store
+    module StorePlugins
+      module Upload
+        module InstanceMethods
+          def upload(file)
+            filename = file[:filename]
+            file     = file[:tempfile]
+            bucket   = 's3browser'
+
+            s3 = Aws::S3::Resource.new
+            s3.bucket(bucket).object(filename).upload_file(file.path)
+          end
+        end
+      end
+
+      register_plugin(:upload, Upload)
+    end
+  end
+end

data/lib/s3browser/policy.json

@@ -0,0 +1,19 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "example-statement-ID",
+      "Effect": "Allow",
+      "Principal": { "AWS": "*" },
+      "Action": [
+        "SQS:SendMessage"
+      ],
+      "Resource": "SQS-queue-ARN",
+      "Condition": {
+        "ArnLike": {          
+          "aws:SourceArn": "arn:aws:s3:*:*:bucket-name"    
+        }
+      }
+    }
+  ]
+}

data/lib/s3browser/server.rb

@@ -0,0 +1,81 @@
+require 'sinatra/base'
+require 'rack-flash'
+require 'tilt/haml'
+require 's3browser/store'
+
+module S3Browser
+  class Server < Sinatra::Base
+    enable :sessions
+    use Rack::Flash
+
+    class Store < S3Browser::Store
+      plugin :es
+      plugin :images
+      plugin :upload
+    end
+
+    configure :development do
+      set :port, 9292
+      set :bind, '0.0.0.0'
+      enable :logging
+      set :store, Store.new('s3browser')
+    end
+
+    get '/' do
+      buckets = settings.store.buckets
+      haml :index, locals: { title: 'S3Browser', buckets: buckets }
+    end
+
+    get '/:bucket' do |bucket|
+      params['q'] = nil if params['q'] == ''
+      objects = settings.store.objects(bucket, term: params['q'], sort: params['s'], direction: params['d'])
+      haml :bucket, locals: { title: bucket, bucket: bucket, objects: objects, q: params['q'] }
+    end
+
+    post '/upload' do
+      if params['upload']
+        begin
+          settings.store.upload params['upload']
+          flash[:success] = 'File uploaded'
+        rescue
+          flash[:error] = 'Could not upload the file'
+        end
+      end
+      redirect back
+    end
+
+    helpers do
+      def bucket
+        ENV['AWS_S3_BUCKET']
+      end
+
+      def sort_url(field)
+        field = field.to_s
+        url = '?s=' + field
+        if params['s'] == field
+          if params['d'] == 'desc'
+            url = url + '&d=asc'
+          else
+            url = url + '&d=desc'
+          end
+        end
+        url
+      end
+
+      def sort_icon(field)
+        field = field.to_s
+        if params['s'] == field
+          if ['asc', 'desc'].include?(params['d'])
+            'fa-sort-' + params['d']
+          else
+            'fa-sort-asc'
+          end
+        else
+          'fa-sort'
+        end
+      end
+    end
+
+    run! if app_file == $0
+  end
+end

data/lib/s3browser/store.rb

@@ -0,0 +1,106 @@
+require 'aws-sdk'
+
+module S3Browser
+  class Store
+    class StoreError < StandardError; end
+
+    # A thread safe cache class, offering only #[] and #[]= methods,
+    # each protected by a mutex.
+    # Ripped off from Roda - https://github.com/jeremyevans/roda
+    class StoreCache
+      # Create a new thread safe cache.
+      def initialize
+        @mutex = Mutex.new
+        @hash = {}
+      end
+
+      # Make getting value from underlying hash thread safe.
+      def [](key)
+        @mutex.synchronize{@hash[key]}
+      end
+
+      # Make setting value in underlying hash thread safe.
+      def []=(key, value)
+        @mutex.synchronize{@hash[key] = value}
+      end
+    end
+
+    # Ripped off from Roda - https://github.com/jeremyevans/roda
+    module StorePlugins
+      # Stores registered plugins
+      @plugins = StoreCache.new
+
+      # If the registered plugin already exists, use it.  Otherwise,
+      # require it and return it.  This raises a LoadError if such a
+      # plugin doesn't exist, or a StoreError if it exists but it does
+      # not register itself correctly.
+      def self.load_plugin(name)
+        h = @plugins
+        unless plugin = h[name]
+          require "s3browser/plugins/#{name}"
+          raise StoreError, "Plugin #{name} did not register itself correctly in S3Browser::Store::StorePlugins" unless plugin = h[name]
+        end
+        plugin
+      end
+
+      # Register the given plugin with Store, so that it can be loaded using #plugin
+      # with a symbol.  Should be used by plugin files. Example:
+      #
+      #   S3Browser::Store::StorePlugins.register_plugin(:plugin_name, PluginModule)
+      def self.register_plugin(name, mod)
+        @plugins[name] = mod
+      end
+
+      module Base
+        module ClassMethods
+          # Load a new plugin into the current class.  A plugin can be a module
+          # which is used directly, or a symbol represented a registered plugin
+          # which will be required and then used. Returns nil.
+          #
+          #   Store.plugin PluginModule
+          #   Store.plugin :csrf
+          def plugin(plugin, *args, &block)
+            raise StoreError, "Cannot add a plugin to a frozen Store class" if frozen?
+            plugin = StorePlugins.load_plugin(plugin) if plugin.is_a?(Symbol)
+            plugin.load_dependencies(self, *args, &block) if plugin.respond_to?(:load_dependencies)
+            include(plugin::InstanceMethods) if defined?(plugin::InstanceMethods)
+            extend(plugin::ClassMethods) if defined?(plugin::ClassMethods)
+            plugin.configure(self, *args, &block) if plugin.respond_to?(:configure)
+            nil
+          end
+        end
+
+        module InstanceMethods
+          def initialize(bucket)
+          end
+
+          def add(bucket, object)
+            nil
+          end
+
+          def objects(bucket, options)
+            s3.list_objects(bucket: bucket).contents.map do |object|
+              object.to_h.merge(bucket: bucket, url: "http://#{bucket}.s3.amazonaws.com/#{object[:key]}")
+            end
+          end
+
+          def search(bucket, term, options = {})
+            []
+          end
+
+          def buckets
+            s3.list_buckets.buckets
+          end
+
+          private
+          def s3
+            @s3 ||= Aws::S3::Client.new
+          end
+        end
+      end
+    end
+
+    extend StorePlugins::Base::ClassMethods
+    plugin StorePlugins::Base
+  end
+end