activestorage 0.1

data/lib/active_storage/service/disk_service.rb

@@ -0,0 +1,70 @@
+require "fileutils"
+require "pathname"
+require "active_support/core_ext/numeric/bytes"
+
+class ActiveStorage::Service::DiskService < ActiveStorage::Service
+  attr_reader :root
+
+  def initialize(root:)
+    @root = root
+  end
+
+  def upload(key, io, checksum: nil)
+    File.open(make_path_for(key), "wb") do |file|
+      while chunk = io.read(64.kilobytes)
+        file.write(chunk)
+      end
+    end
+
+    ensure_integrity_of(key, checksum) if checksum
+  end
+
+  def download(key)
+    if block_given?
+      File.open(path_for(key)) do |file|
+        while data = file.read(64.kilobytes)
+          yield data
+        end
+      end
+    else
+      File.open path_for(key), &:read
+    end
+  end
+
+  def delete(key)
+    File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted
+  end
+
+  def exist?(key)
+    File.exist? path_for(key)
+  end
+
+  def url(key, expires_in:, disposition:, filename:)
+    verified_key_with_expiration = ActiveStorage::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in)
+
+    if defined?(Rails) && defined?(Rails.application)
+      Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition)
+    else
+      "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}"
+    end
+  end
+
+  private
+    def path_for(key)
+      File.join root, folder_for(key), key
+    end
+
+    def folder_for(key)
+      [ key[0..1], key[2..3] ].join("/")
+    end
+
+    def make_path_for(key)
+      path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) }
+    end
+
+    def ensure_integrity_of(key, checksum)
+      unless Digest::MD5.file(path_for(key)).base64digest == checksum
+        raise ActiveStorage::IntegrityError
+      end
+    end
+end

data/lib/active_storage/service/gcs_service.rb

@@ -0,0 +1,41 @@
+require "google/cloud/storage"
+require "active_support/core_ext/object/to_query"
+
+class ActiveStorage::Service::GCSService < ActiveStorage::Service
+  attr_reader :client, :bucket
+
+  def initialize(project:, keyfile:, bucket:)
+    @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile)
+    @bucket = @client.bucket(bucket)
+  end
+
+  def upload(key, io, checksum: nil)
+    # FIXME: Ensure integrity by sending the checksum for service side verification
+    bucket.create_file(io, key)
+  end
+
+  # FIXME: Add streaming when given a block
+  def download(key)
+    io = file_for(key).download
+    io.rewind
+    io.read
+  end
+
+  def delete(key)
+    file_for(key).try(:delete)
+  end
+
+  def exist?(key)
+    file_for(key).present?
+  end
+
+  def url(key, expires_in:, disposition:, filename:)
+    file_for(key).signed_url(expires: expires_in) + "&" +
+      { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query
+  end
+
+  private
+    def file_for(key)
+      bucket.file(key)
+    end
+end

data/lib/active_storage/service/mirror_service.rb

@@ -0,0 +1,34 @@
+require "active_support/core_ext/module/delegation"
+
+class ActiveStorage::Service::MirrorService < ActiveStorage::Service
+  attr_reader :services
+
+  delegate :download, :exist?, :url, :byte_size, :checksum, to: :primary_service
+
+  def initialize(services:)
+    @services = services
+  end
+
+  def upload(key, io, checksum: nil)
+    services.collect do |service|
+      service.upload key, io, checksum: checksum
+      io.rewind
+    end
+  end
+
+  def delete(key)
+    perform_across_services :delete, key
+  end
+
+  private
+    def primary_service
+      services.first
+    end
+
+    def perform_across_services(method, *args)
+      # FIXME: Convert to be threaded
+      services.collect do |service|
+        service.public_send method, *args
+      end
+    end
+end

data/lib/active_storage/service/s3_service.rb

@@ -0,0 +1,55 @@
+require "aws-sdk"
+require "active_support/core_ext/numeric/bytes"
+
+class ActiveStorage::Service::S3Service < ActiveStorage::Service
+  attr_reader :client, :bucket
+
+  def initialize(access_key_id:, secret_access_key:, region:, bucket:)
+    @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region)
+    @bucket = @client.bucket(bucket)
+  end
+
+  def upload(key, io, checksum: nil)
+    # FIXME: Ensure integrity by sending the checksum for service side verification
+    object_for(key).put(body: io)
+  end
+
+  def download(key)
+    if block_given?
+      stream(key, &block)
+    else
+      object_for(key).get.body.read
+    end
+  end
+
+  def delete(key)
+    object_for(key).delete
+  end
+
+  def exist?(key)
+    object_for(key).exists?
+  end
+
+  def url(key, expires_in:, disposition:, filename:)
+    object_for(key).presigned_url :get, expires_in: expires_in,
+      response_content_disposition: "#{disposition}; filename=\"#{filename}\""
+  end
+
+  private
+    def object_for(key)
+      bucket.object(key)
+    end
+
+    # Reads the object for the given key in chunks, yielding each to the block.
+    def stream(key, options = {}, &block)
+      object = object_for(key)
+
+      chunk_size = 5.megabytes
+      offset = 0
+
+      while offset < object.content_length
+        yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}"))
+        offset += chunk_size
+      end
+    end
+end

data/lib/active_storage/storage_services.yml

@@ -0,0 +1,27 @@
+test:
+  service: Disk
+  root: <%= Rails.root.join("tmp/storage") %>
+
+local:
+  service: Disk
+  root: <%= Rails.root.join("storage") %>
+
+# Use rails secrets:edit to set the AWS secrets (as shared:aws:access_key_id|secret_access_key)
+amazon:
+  service: S3
+  access_key_id: <%= Rails.application.secrets.aws[:access_key_id] %>
+  secret_access_key: <%= Rails.application.secrets.aws[:secret_access_key] %>
+  region: us-east-1
+  bucket: your_own_bucket
+
+# Remember not to checkin your GCS keyfile to a repository
+google:
+  service: GCS
+  project: your_project
+  keyfile: <%= Rails.root.join("path/to/gcs.keyfile") %>
+  bucket: your_own_bucket
+
+mirror:
+  service: Mirror
+  primary: local
+  secondaries: [ amazon, google ]

data/lib/active_storage/verified_key_with_expiration.rb

@@ -0,0 +1,24 @@
+class ActiveStorage::VerifiedKeyWithExpiration
+  class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveStorage') : nil
+
+  class << self
+    def encode(key, expires_in: nil)
+      verifier.generate([ key, expires_at(expires_in) ])
+    end
+
+    def decode(encoded_key)
+      key, expires_at = verifier.verified(encoded_key)
+
+      key if key && fresh?(expires_at)
+    end
+
+    private
+      def expires_at(expires_in)
+        expires_in ? Time.now.utc.advance(seconds: expires_in) : nil
+      end
+
+      def fresh?(expires_at)
+        expires_at.nil? || Time.now.utc < expires_at
+      end
+  end
+end

data/lib/tasks/activestorage.rake

@@ -0,0 +1,19 @@
+require "fileutils"
+
+namespace :activestorage do
+  desc "Copy over the migration needed to the application"
+  task :install do
+    FileUtils.mkdir_p Rails.root.join("storage")
+    FileUtils.mkdir_p Rails.root.join("tmp/storage")
+    puts "Made storage and tmp/storage directories for development and testing"
+
+    FileUtils.cp  File.expand_path("../../active_storage/storage_services.yml", __FILE__), Rails.root.join("config")
+    puts "Copied default configuration to config/storage_services.yml"
+
+    migration_file_path = "db/migrate/#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_active_storage_create_tables.rb"
+    FileUtils.cp File.expand_path("../../active_storage/migration.rb", __FILE__), Rails.root.join(migration_file_path)
+    puts "Copied migration to #{migration_file_path}"
+
+    puts "Now run rails db:migrate to create the tables for Active Storage"
+  end
+end

data/test/attachments_test.rb

@@ -0,0 +1,95 @@
+require "test_helper"
+require "database/setup"
+require "active_storage/blob"
+
+require "active_job"
+ActiveJob::Base.queue_adapter = :test
+ActiveJob::Base.logger = nil
+
+# ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class User < ActiveRecord::Base
+  has_one_attached  :avatar
+  has_many_attached :highlights
+end
+
+class ActiveStorage::AttachmentsTest < ActiveSupport::TestCase
+  include ActiveJob::TestHelper
+
+  setup { @user = User.create!(name: "DHH") }
+
+  teardown { ActiveStorage::Blob.all.each(&:purge) }
+
+  test "attach existing blob" do
+    @user.avatar.attach create_blob(filename: "funky.jpg")
+    assert_equal "funky.jpg", @user.avatar.filename.to_s
+  end
+
+  test "attach new blob" do
+    @user.avatar.attach io: StringIO.new("STUFF"), filename: "town.jpg", content_type: "image/jpg"
+    assert_equal "town.jpg", @user.avatar.filename.to_s
+  end
+
+  test "purge attached blob" do
+    @user.avatar.attach create_blob(filename: "funky.jpg")
+    avatar_key = @user.avatar.key
+
+    @user.avatar.purge
+    assert_not @user.avatar.attached?
+    assert_not ActiveStorage::Blob.service.exist?(avatar_key)
+  end
+
+  test "purge attached blob later when the record is destroyed" do
+    @user.avatar.attach create_blob(filename: "funky.jpg")
+    avatar_key = @user.avatar.key
+
+    perform_enqueued_jobs do
+      @user.destroy
+
+      assert_nil ActiveStorage::Blob.find_by(key: avatar_key)
+      assert_not ActiveStorage::Blob.service.exist?(avatar_key)
+    end
+  end
+
+
+  test "attach existing blobs" do
+    @user.highlights.attach create_blob(filename: "funky.jpg"), create_blob(filename: "wonky.jpg")
+
+    assert_equal "funky.jpg", @user.highlights.first.filename.to_s
+    assert_equal "wonky.jpg", @user.highlights.second.filename.to_s
+  end
+
+  test "attach new blobs" do
+    @user.highlights.attach(
+      { io: StringIO.new("STUFF"), filename: "town.jpg", content_type: "image/jpg" }, 
+      { io: StringIO.new("IT"), filename: "country.jpg", content_type: "image/jpg" })
+
+    assert_equal "town.jpg", @user.highlights.first.filename.to_s
+    assert_equal "country.jpg", @user.highlights.second.filename.to_s
+  end
+
+  test "purge attached blobs" do
+    @user.highlights.attach create_blob(filename: "funky.jpg"), create_blob(filename: "wonky.jpg")
+    highlight_keys = @user.highlights.collect(&:key)
+    
+    @user.highlights.purge
+    assert_not @user.highlights.attached?
+    assert_not ActiveStorage::Blob.service.exist?(highlight_keys.first)
+    assert_not ActiveStorage::Blob.service.exist?(highlight_keys.second)
+  end
+
+  test "purge attached blobs later when the record is destroyed" do
+    @user.highlights.attach create_blob(filename: "funky.jpg"), create_blob(filename: "wonky.jpg")
+    highlight_keys = @user.highlights.collect(&:key)
+
+    perform_enqueued_jobs do
+      @user.destroy
+
+      assert_nil ActiveStorage::Blob.find_by(key: highlight_keys.first)
+      assert_not ActiveStorage::Blob.service.exist?(highlight_keys.first)
+
+      assert_nil ActiveStorage::Blob.find_by(key: highlight_keys.second)
+      assert_not ActiveStorage::Blob.service.exist?(highlight_keys.second)
+    end
+  end
+end

data/test/blob_test.rb

@@ -0,0 +1,28 @@
+require "test_helper"
+require "database/setup"
+require "active_storage/blob"
+
+class ActiveStorage::BlobTest < ActiveSupport::TestCase
+  test "create after upload sets byte size and checksum" do
+    data = "Hello world!"
+    blob = create_blob data: data
+
+    assert_equal data, blob.download
+    assert_equal data.length, blob.byte_size
+    assert_equal Digest::MD5.base64digest(data), blob.checksum
+  end
+
+  test "urls expiring in 5 minutes" do
+    blob = create_blob
+
+    travel_to Time.now do
+      assert_equal expected_url_for(blob), blob.url
+      assert_equal expected_url_for(blob, disposition: :attachment), blob.url(disposition: :attachment)
+    end
+  end
+
+  private
+    def expected_url_for(blob, disposition: :inline)
+      "/rails/blobs/#{ActiveStorage::VerifiedKeyWithExpiration.encode(blob.key, expires_in: 5.minutes)}?disposition=#{disposition}"
+    end
+end

data/test/database/create_users_migration.rb

@@ -0,0 +1,7 @@
+class ActiveStorage::CreateUsers < ActiveRecord::Migration[5.1]
+  def change
+    create_table :users do |t|
+      t.string :name
+    end
+  end
+end

data/test/database/setup.rb

@@ -0,0 +1,6 @@
+require "active_storage/migration"
+require_relative "create_users_migration"
+
+ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:')
+ActiveStorage::CreateTables.migrate(:up)
+ActiveStorage::CreateUsers.migrate(:up)

data/test/disk_controller_test.rb

@@ -0,0 +1,34 @@
+require "test_helper"
+require "database/setup"
+
+require "action_controller"
+require "action_controller/test_case"
+
+require "active_storage/disk_controller"
+require "active_storage/verified_key_with_expiration"
+
+class ActiveStorage::DiskControllerTest < ActionController::TestCase
+  Routes = ActionDispatch::Routing::RouteSet.new.tap do |routes|
+    routes.draw do
+      get "/rails/blobs/:encoded_key" => "active_storage/disk#show", as: :rails_disk_blob
+    end
+  end
+
+  setup do
+    @blob = create_blob
+    @routes = Routes
+    @controller = ActiveStorage::DiskController.new
+  end
+
+  test "showing blob inline" do
+    get :show, params: { encoded_key: ActiveStorage::VerifiedKeyWithExpiration.encode(@blob.key, expires_in: 5.minutes) }
+    assert_equal "inline; filename=\"#{@blob.filename}\"", @response.headers["Content-Disposition"]
+    assert_equal "text/plain", @response.headers["Content-Type"]
+  end
+
+  test "sending blob as attachment" do
+    get :show, params: { encoded_key: ActiveStorage::VerifiedKeyWithExpiration.encode(@blob.key, expires_in: 5.minutes), disposition: :attachment }
+    assert_equal "attachment; filename=\"#{@blob.filename}\"", @response.headers["Content-Disposition"]
+    assert_equal "text/plain", @response.headers["Content-Type"]
+  end
+end