defile 0.2.0

data/lib/defile/backend/file_system.rb

@@ -0,0 +1,70 @@
+module Defile
+  module Backend
+    class FileSystem
+      attr_reader :directory
+
+      def initialize(directory, max_size: nil, hasher: Defile::RandomHasher.new)
+        @hasher = hasher
+        @directory = directory
+        @max_size = max_size
+
+        FileUtils.mkdir_p(@directory)
+      end
+
+      def upload(uploadable)
+        Defile.verify_uploadable(uploadable, @max_size)
+
+        id = @hasher.hash(uploadable)
+
+        if uploadable.respond_to?(:path) and ::File.exist?(uploadable.path)
+          FileUtils.cp(uploadable.path, path(id))
+        else
+          ::File.open(path(id), "wb") do |file|
+            buffer = "" # reuse the same buffer
+            until uploadable.eof?
+              uploadable.read(Defile.read_chunk_size, buffer)
+              file.write(buffer)
+            end
+            uploadable.close
+          end
+        end
+
+        Defile::File.new(self, id)
+      end
+
+      def get(id)
+        Defile::File.new(self, id)
+      end
+
+      def delete(id)
+        FileUtils.rm(path(id)) if exists?(id)
+      end
+
+      def open(id)
+        ::File.open(path(id), "rb")
+      end
+
+      def read(id)
+        ::File.read(path(id)) if exists?(id)
+      end
+
+      def size(id)
+        ::File.size(path(id)) if exists?(id)
+      end
+
+      def exists?(id)
+        ::File.exists?(path(id))
+      end
+
+      def clear!(confirm = nil)
+        raise ArgumentError, "are you sure? this will remove all files in the backend, call as `clear!(:confirm)` if you're sure you want to do this" unless confirm == :confirm
+        FileUtils.rm_rf(@directory)
+        FileUtils.mkdir_p(@directory)
+      end
+
+      def path(id)
+        ::File.join(@directory, id)
+      end
+    end
+  end
+end

data/lib/defile/backend/s3.rb

@@ -0,0 +1,129 @@
+require "aws-sdk"
+
+module Defile
+  module Backend
+    class S3
+      # Emulates an IO-object like interface on top of S3Object#read. To avoid
+      # memory allocations and unnecessary complexity, this treats the `length`
+      # parameter to read as a boolean flag instead. If given, it will read the
+      # file in chunks of undetermined size, if not given it will read the
+      # entire file.
+      class Reader
+        def initialize(object)
+          @object = object
+          @closed = false
+        end
+
+        def read(length = nil, buffer = nil)
+          result = if length
+            raise "closed" if @closed
+
+            unless eof? # sets @peek
+              @peek
+            end
+          else
+            @object.read
+          end
+          buffer.replace(result) if buffer and result
+          result
+        ensure
+          @peek = nil
+        end
+
+        def eof?
+          @peek ||= enumerator.next
+          false
+        rescue StopIteration
+          true
+        end
+
+        def size
+          @object.content_length
+        end
+
+        def close
+          @closed = true
+        end
+
+      private
+
+        def enumerator
+          @enumerator ||= @object.to_enum(:read)
+        end
+      end
+
+      Signature = Struct.new(:as, :id, :url, :fields)
+
+      attr_reader :access_key_id
+
+      def initialize(access_key_id:, secret_access_key:, bucket:, max_size: nil, prefix: nil, hasher: Defile::RandomHasher.new)
+        @access_key_id = access_key_id
+        @secret_access_key = secret_access_key
+        @s3 = AWS::S3.new(access_key_id: access_key_id, secret_access_key: secret_access_key)
+        @bucket_name = bucket
+        @bucket = @s3.buckets[@bucket_name]
+        @hasher = hasher
+        @prefix = prefix
+        @max_size = max_size
+      end
+
+      def upload(uploadable)
+        Defile.verify_uploadable(uploadable, @max_size)
+
+        id = @hasher.hash(uploadable)
+
+        if uploadable.is_a?(Defile::File) and uploadable.backend.is_a?(S3) and uploadable.backend.access_key_id == access_key_id
+          uploadable.backend.object(uploadable.id).copy_to(object(id))
+        else
+          object(id).write(uploadable, content_length: uploadable.size)
+        end
+
+        Defile::File.new(self, id)
+      end
+
+      def get(id)
+        Defile::File.new(self, id)
+      end
+
+      def delete(id)
+        object(id).delete
+      end
+
+      def open(id)
+        Reader.new(object(id))
+      end
+
+      def read(id)
+        object(id).read
+      rescue AWS::S3::Errors::NoSuchKey
+        nil
+      end
+
+      def size(id)
+        object(id).content_length
+      rescue AWS::S3::Errors::NoSuchKey
+        nil
+      end
+
+      def exists?(id)
+        object(id).exists?
+      end
+
+      def clear!(confirm = nil)
+        raise ArgumentError, "are you sure? this will remove all files in the backend, call as `clear!(:confirm)` if you're sure you want to do this" unless confirm == :confirm
+        @bucket.objects.with_prefix(@prefix).delete_all
+      end
+
+      def presign
+        id = RandomHasher.new.hash
+        signature = @bucket.presigned_post(key: [*@prefix, id].join("/"))
+        signature.where(content_length: @max_size) if @max_size
+        Signature.new("file", id, signature.url.to_s, signature.fields)
+      end
+
+      def object(id)
+        @bucket.objects[[*@prefix, id].join("/")]
+      end
+    end
+  end
+end

data/lib/defile/file.rb

@@ -0,0 +1,65 @@
+module Defile
+  class File
+    attr_reader :backend, :id
+
+    def initialize(backend, id)
+      @backend = backend
+      @id = id
+    end
+
+    def read(*args)
+      io.read(*args)
+    end
+
+    def eof?
+      io.eof?
+    end
+
+    def close
+      io.close
+    end
+
+    def size
+      backend.size(id)
+    end
+
+    def delete
+      backend.delete(id)
+    end
+
+    def exists?
+      backend.exists?(id)
+    end
+
+    def to_io
+      io
+    end
+
+    def download
+      tempfile = Tempfile.new(id)
+      tempfile.binmode
+      each do |chunk|
+        tempfile.write(chunk)
+      end
+      close
+      tempfile.close
+      tempfile
+    end
+
+    def each
+      if block_given?
+        until eof?
+          yield(read(Defile.read_chunk_size))
+        end
+      else
+        to_enum
+      end
+    end
+
+  private
+
+    def io
+      @io ||= backend.open(id)
+    end
+  end
+end

data/lib/defile/image_processing.rb

@@ -0,0 +1,73 @@
+require "defile"
+require "mini_magick"
+
+module Defile
+  class ImageProcessor
+    def initialize(method)
+      @method = method
+    end
+
+    def convert(img, format)
+      img.format(format.to_s.downcase)
+    end
+
+    def limit(img, width, height)
+      img.resize "#{width}x#{height}>"
+    end
+
+    def fit(img, width, height)
+      img.resize "#{width}x#{height}"
+    end
+
+    def fill(img, width, height, gravity = 'Center')
+      width = width.to_i
+      height = height.to_i
+      cols, rows = img[:dimensions]
+      img.combine_options do |cmd|
+        if width != cols || height != rows
+          scale_x = width/cols.to_f
+          scale_y = height/rows.to_f
+          if scale_x >= scale_y
+            cols = (scale_x * (cols + 0.5)).round
+            rows = (scale_x * (rows + 0.5)).round
+            cmd.resize "#{cols}"
+          else
+            cols = (scale_y * (cols + 0.5)).round
+            rows = (scale_y * (rows + 0.5)).round
+            cmd.resize "x#{rows}"
+          end
+        end
+        cmd.gravity gravity
+        cmd.background "rgba(255,255,255,0.0)"
+        cmd.extent "#{width}x#{height}" if cols != width || rows != height
+      end
+    end
+
+    def pad(img, width, height, background = "transparent", gravity = "Center")
+      img.combine_options do |cmd|
+        cmd.thumbnail "#{width}x#{height}>"
+        if background == "transparent"
+          cmd.background "rgba(255, 255, 255, 0.0)"
+        else
+          cmd.background background
+        end
+        cmd.gravity gravity
+        cmd.extent "#{width}x#{height}"
+      end
+    end
+
+    def call(file, *args)
+      path = file.download.path
+      img = ::MiniMagick::Image.open(path)
+      send(@method, img, *args)
+
+      img.write(path)
+
+      ::File.open(path, "rb")
+    end
+  end
+end
+
+[:fill, :fit, :limit, :pad, :convert].each do |name|
+  Defile.processor(name, Defile::ImageProcessor.new(name))
+end

data/lib/defile/rails.rb

@@ -0,0 +1,36 @@
+require "defile"
+
+module Defile
+  module Controller
+    def show
+      file = Defile.backends.fetch(params[:backend_name]).get(params[:id])
+
+      options = { disposition: "inline" }
+      options[:type] = Mime::Type.lookup_by_extension(params[:format]).to_s if params[:format]
+
+      send_data file.read, options
+    end
+  end
+
+  module AttachmentFieldHelper
+    def attachment_field(method, options = {})
+      self.multipart = true
+      @template.attachment_field(@object_name, method, objectify_options(options))
+    end
+  end
+
+  class Engine < Rails::Engine
+    initializer "defile", before: :load_environment_config do
+      Defile.store ||= Defile::Backend::FileSystem.new(Rails.root.join("tmp/uploads/store").to_s)
+      Defile.cache ||= Defile::Backend::FileSystem.new(Rails.root.join("tmp/uploads/cache").to_s)
+
+      Defile.app = Defile::App.new(logger: Rails.logger)
+
+      ActiveSupport.on_load :active_record do
+        require "defile/attachment/active_record"
+      end
+
+      ActionView::Helpers::FormBuilder.send(:include, AttachmentFieldHelper)
+    end
+  end
+end

data/lib/defile/random_hasher.rb

@@ -0,0 +1,5 @@
+class Defile::RandomHasher
+  def hash(uploadable=nil)
+    SecureRandom.hex(30)
+  end
+end

data/lib/defile/version.rb

@@ -0,0 +1,3 @@
+module Defile
+  VERSION = "0.2.0"
+end

data/spec/defile/app_spec.rb

@@ -0,0 +1,151 @@
+require "rack/test"
+
+Defile.processor(:reverse) do |file|
+  StringIO.new(file.read.reverse)
+end
+
+Defile.processor(:upcase, proc { |file| StringIO.new(file.read.upcase) })
+
+Defile.processor(:concat) do |file, *words|
+  content = File.read(file.download.path)
+  tempfile = Tempfile.new("concat")
+  tempfile.write(content)
+  words.each do |word|
+    tempfile.write(word)
+  end
+  tempfile.close
+  File.open(tempfile.path, "r")
+end
+
+describe Defile::App do
+  include Rack::Test::Methods
+
+  def app
+    Defile::App.new
+  end
+
+  describe "GET /:backend/:id/:filename" do
+    it "returns a stored file" do
+      file = Defile.store.upload(StringIO.new("hello"))
+
+      get "/store/#{file.id}/hello"
+
+      expect(last_response.status).to eq(200)
+      expect(last_response.body).to eq("hello")
+    end
+
+    it "returns a 404 if the file doesn't exist" do
+      file = Defile.store.upload(StringIO.new("hello"))
+
+      get "/store/doesnotexist/hello"
+
+      expect(last_response.status).to eq(404)
+      expect(last_response.body).to eq("not found")
+    end
+
+    it "returns a 404 if the backend doesn't exist" do
+      file = Defile.store.upload(StringIO.new("hello"))
+
+      get "/doesnotexist/#{file.id}/hello"
+
+      expect(last_response.status).to eq(404)
+      expect(last_response.body).to eq("not found")
+    end
+
+    context "with allow origin" do
+      def app
+        Defile::App.new(allow_origin: "example.com")
+      end
+
+      it "sets CORS header" do
+        file = Defile.store.upload(StringIO.new("hello"))
+
+        get "/store/#{file.id}/hello"
+
+        expect(last_response.status).to eq(200)
+        expect(last_response.body).to eq("hello")
+        expect(last_response.headers["Access-Control-Allow-Origin"]).to eq("example.com")
+      end
+    end
+
+    it "returns a 404 for non get requests" do
+      file = Defile.store.upload(StringIO.new("hello"))
+
+      post "/store/#{file.id}/hello"
+
+      expect(last_response.status).to eq(404)
+      expect(last_response.body).to eq("not found")
+    end
+  end
+
+  describe "GET /:backend/:processor/:id/:filename" do
+    it "returns 404 if processor does not exist" do
+      file = Defile.store.upload(StringIO.new("hello"))
+
+      get "/store/doesnotexist/#{file.id}/hello"
+
+      expect(last_response.status).to eq(404)
+      expect(last_response.body).to eq("not found")
+    end
+
+    it "applies block processor to file" do
+      file = Defile.store.upload(StringIO.new("hello"))
+
+      get "/store/reverse/#{file.id}/hello"
+
+      expect(last_response.status).to eq(200)
+      expect(last_response.body).to eq("olleh")
+    end
+
+    it "applies object processor to file" do
+      file = Defile.store.upload(StringIO.new("hello"))
+
+      get "/store/upcase/#{file.id}/hello"
+
+      expect(last_response.status).to eq(200)
+      expect(last_response.body).to eq("HELLO")
+    end
+
+    it "applies processor with arguments" do
+      file = Defile.store.upload(StringIO.new("hello"))
+
+      get "/store/concat/foo/bar/baz/#{file.id}/hello"
+
+      expect(last_response.status).to eq(200)
+      expect(last_response.body).to eq("hellofoobarbaz")
+    end
+  end
+
+  describe "POST /:backend" do
+    it "returns 404 if backend is not marked as direct upload" do
+      file = Rack::Test::UploadedFile.new(path("hello.txt"))
+      post "/store", file: file
+
+      expect(last_response.status).to eq(404)
+      expect(last_response.body).to eq("not found")
+    end
+
+    it "uploads a file for direct upload backends" do
+      file = Rack::Test::UploadedFile.new(path("hello.txt"))
+      post "/cache", file: file
+
+      expect(last_response.status).to eq(200)
+      expect(JSON.parse(last_response.body)["id"]).not_to be_empty
+    end
+  end
+
+  it "returns a 404 if id not given" do
+    get "/store"
+
+    expect(last_response.status).to eq(404)
+    expect(last_response.body).to eq("not found")
+  end
+
+  it "returns a 404 for root" do
+    get "/"
+
+    expect(last_response.status).to eq(404)
+    expect(last_response.body).to eq("not found")
+  end
+end
+