populate-me 0.1.8 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0da7af0c111b47e4027615b2db466b6b5564ee1b
-  data.tar.gz: 6323247fa15e8b90a5fc8944273d93405782ff93
+  metadata.gz: 10a0d904a55bc88de4e01af6c15816cc3c943a78
+  data.tar.gz: cc36c361bd647861681d153f5bf2e9d9046a7e73
 SHA512:
-  metadata.gz: f8fe0a34b4f19b495b3fa77bcd833ebdbe83ecfaf39f98acd07d3ed23c99aceea4631cd085a60f6da2e95eb0ce76d32ee7254f7aa679622e5b6733705a3cb4f4
-  data.tar.gz: a70e4a4838712eeb3210552c65419e39f242858178ed1dc77a3432655c79d9b13336566753db1f1297520c6f4e6cf1554ed489dfa66c9aad7eb40bc03c476dde
+  metadata.gz: 34f55a27c047c8e0204eaf10a9a8c4d62b3ab06e9672812f4522f6b474b48261bf6521a538383770a3be5d1106651d48c5d4502c74d3dcd282bb87bcba7c075b
+  data.tar.gz: 901ce3866a5c98053a4b92f5f53fac015d7b4424f0c32c89b0a4418104b904083c1901f0a5e44606aa8c7677ac24e45aa20921a470c6548774c8be0ca67d1b61

data/example/config.ru

@@ -1,23 +1,34 @@
+# We use this example for running real tests on top of unit tests.
+# This is not meant to be an example to learn how PopulateMe would work.
+# Real examples will come later and this file will be removed.
+
 $:.unshift File.expand_path('../../lib', __FILE__)
 
 # Models ##########
 
-require 'populate_me/mongo'
-require 'mongo'
+require 'populate_me/document'
 
-MONGO = Mongo::Client.new([ '127.0.0.1:27017' ], :database => 'blog-populate-me-test')
-DB    = MONGO.database
-PopulateMe::Mongo.set :db, DB
+# MONGO = Mongo::Client.new([ '127.0.0.1:27017' ], :database => 'blog-populate-me-test')
+# DB    = MONGO.database
+# PopulateMe::Mongo.set :db, DB
 
 # require 'populate_me/attachment'
 # PopulateMe::Document.set :default_attachment_class, PopulateMe::Attachment
 # require 'populate_me/file_system_attachment'
 # PopulateMe::Document.set :default_attachment_class, PopulateMe::FileSystemAttachment
-require 'populate_me/grid_fs_attachment'
-PopulateMe::Mongo.set :default_attachment_class, PopulateMe::GridFS
-PopulateMe::GridFS.set :db, DB
+#
+# require 'populate_me/grid_fs_attachment'
+# PopulateMe::Mongo.set :default_attachment_class, PopulateMe::GridFS
+# PopulateMe::GridFS.set :db, DB
+
+require 'populate_me/s3_attachment'
+s3_resource = Aws::S3::Resource.new
+s3_bucket = s3_resource.bucket(ENV['BUCKET'])
+PopulateMe::Document.set :default_attachment_class, PopulateMe::S3Attachment
+PopulateMe::S3Attachment.set :bucket, s3_bucket
+
 
-class BlogPost < PopulateMe::Mongo
+class BlogPost < PopulateMe::Document
   field :title, required: true
   field :thumbnail, type: :attachment, variations: [
     PopulateMe::Variation.new_image_magick_job(:populate_me_thumb, :jpg, "-resize '400x225^' -gravity center -extent 400x225")
@@ -30,14 +41,14 @@ class BlogPost < PopulateMe::Mongo
     error_on(:content,'Cannot be blank') if WebUtils.blank?(self.content)
   end
 end
-class BlogPost::Author < PopulateMe::Mongo
+class BlogPost::Author < PopulateMe::Document
   # nested
   field :name
   def validate
     error_on(:name, 'Cannot be shit') if self.name=='shit'
   end
 end
-class BlogPost::Comment < PopulateMe::Mongo
+class BlogPost::Comment < PopulateMe::Document
   # not nested
   field :author, default: 'Anonymous'
   field :content, type: :text
@@ -45,7 +56,7 @@ class BlogPost::Comment < PopulateMe::Mongo
   position_field scope: :blog_post_id
 end
 
-class Article < PopulateMe::Mongo
+class Article < PopulateMe::Document
   field :title
   field :content, type: :text
   position_field
@@ -63,6 +74,6 @@ end
 
 # use PopulateMe::Attachment::Middleware
 # use PopulateMe::FileSystemAttachment::Middleware
-use PopulateMe::GridFS::Middleware
+# use PopulateMe::GridFS::Middleware
 run Admin
 

data/lib/populate_me/attachment.rb

@@ -52,8 +52,12 @@ module PopulateMe
       self.document.__send__(field)
     end
 
+    def field_options
+      self.document.class.fields[self.field]
+    end
+
     def variations
-      self.document.class.fields[self.field][:variations] || []
+      self.field_options[:variations] || []
     end
 
     def field_filename variation_name=:original

data/lib/populate_me/document_mixins/callbacks.rb

@@ -1,3 +1,5 @@
+require 'securerandom'
+
 module PopulateMe
   module DocumentMixins
     module Callbacks
@@ -24,7 +26,7 @@ module PopulateMe
 
       def ensure_id
         if self.id.nil?
-          self.id = WebUtils::generate_random_id
+          self.id = SecureRandom.hex
         end
         self
       end

data/lib/populate_me/s3_attachment.rb

@@ -0,0 +1,120 @@
+require 'populate_me/attachment'
+require 'aws-sdk'
+
+module PopulateMe
+
+  class MissingBucketError < StandardError; end
+
+  class S3Attachment < Attachment
+
+    # For S3 this option behaves a bit differently.
+    # Because S3 file are served directly instead of using a middleware,
+    # the url_prefix is just used in the key name before the attachee_prefix.
+    # It helps saving keys under /public for example which is a common idiom
+    # to save file in a path public by default.
+    # This option can be overriden at the field level.
+    set :url_prefix, '/public'
+
+    # Attachee_prefix is moved on field_value for S3
+    # as well as url_prefix
+    def url variation_name=:original
+      return nil if WebUtils.blank?(self.field_filename(variation_name))
+      "#{settings.bucket.url}/#{self.field_filename(variation_name)}"
+    end
+    # Attachee_prefix is moved on field_value for S3
+    # as well as url_prefix
+    def location_root
+      File.join(
+        settings.root, 
+        settings.url_prefix
+      )
+    end
+
+    def local_url_prefix
+      (
+        self.field_options[:url_prefix] || 
+        self.document.class.settings.s3_url_prefix || 
+        settings.url_prefix
+      ).gsub(/^\/|\/$/,'')
+    end
+
+    def next_available_filename filename
+      ext = File.extname(filename)
+      base = File.join(
+        local_url_prefix, 
+        attachee_prefix, 
+        File.basename(filename,ext)
+      ).gsub(/^\//,'')
+      i = 0
+      loop do
+        suffix = i==0 ? '' : "-#{i}"
+        potential_filename = [base,suffix,ext].join
+        if settings.bucket.object(potential_filename).exists?
+          i += 1
+        else
+          filename = potential_filename
+          break
+        end
+      end
+      filename
+    end
+
+    def perform_create hash
+      if hash[:variation].nil?
+        fn = next_available_filename(hash[:filename])
+        file = hash[:tempfile]
+        type = hash[:type]
+      else
+        fn = WebUtils.filename_variation hash[:future_field_value], hash[:variation].name, hash[:variation].ext
+        file = File.open(hash[:variation_path])
+        type = Rack::Mime.mime_type ".#{hash[:variation].ext}"
+      end
+      settings.bucket.put_object({
+        acl: self.field_options[:acl] || 'public-read',
+        key: fn,
+        content_type: type,
+        body: file,
+        metadata: {
+          parent_collection: (self.document.class.respond_to?(:collection) ? self.document.class.collection.name : self.attachee_prefix),
+        }
+      })
+      file.close unless hash[:variation].nil?
+      fn
+    end
+
+    def deletable? variation_name=:original
+      !WebUtils.blank? self.field_filename(variation_name)
+      # Fine since deleting a non-existent file does not raise an error in S3
+    end
+
+    def perform_delete variation_name=:original
+      s3file = settings.bucket.object(self.field_filename(variation_name))
+      s3file.delete unless s3file.nil?
+    end
+
+    class << self
+
+      def ensure_bucket
+        raise MissingBucketError, "Attachment class #{self.name} does not have an S3 bucket." if settings.bucket.nil?
+      end
+
+      def middleware
+        nil
+      end
+
+      # def middleware_options
+      #   [
+      #     {
+      #       prefix: settings.url_prefix.dup.gsub(/^\/|\/$/,''), 
+      #       db: settings.db,
+      #       # lookup: :path
+      #     }
+      #   ]
+      # end
+
+    end
+
+  end
+
+end
+

data/lib/populate_me/version.rb

@@ -1,4 +1,4 @@
 module PopulateMe
-  VERSION = '0.1.8'
+  VERSION = '0.2.0'
 end
 

data/populate-me.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rack-cerberus', '~> 1.0'
   s.add_development_dependency 'mongo', '~> 2.0'
   s.add_development_dependency 'rack-grid-serve', '~> 0.0.8'
+  s.add_development_dependency 'aws-sdk', '~> 3'
   s.add_development_dependency 'racksh', '~> 1.0'
   s.add_development_dependency 'rake', '~> 10.1'
 end

data/test/test_attachment.rb

@@ -43,6 +43,12 @@ describe PopulateMe::Attachment do
     end
   end
 
+  describe "#field_options" do
+    it "Get options of the field" do
+      assert_equal :attachment, subject.field_options[:type]
+    end
+  end
+
   describe "#variations" do
     it "Gets variations for the field" do
       variations = subject.variations

data/test/test_grid_fs_attachment.rb

@@ -2,6 +2,8 @@ require 'helper'
 require 'populate_me/mongo'
 require 'populate_me/grid_fs_attachment'
 
+Mongo::Logger.logger.level = Logger::ERROR
+
 GRIDMONGO = Mongo::Client.new([ '127.0.0.1:27017' ], :database => 'populate-me-grid-test')
 GRIDDB = GRIDMONGO.database
 GRIDDB.drop
@@ -29,7 +31,7 @@ describe 'PopulateMe::GridFS' do
 
   # Utils
 
-  it 'Returns URL with attachee_prefix' do
+  it 'Returns URL with url_prefix' do
     book = GridBook.new cover: "candy.jpg"
     assert_equal '/attachment/candy.jpg', book.attachment(:cover).url
     assert_equal '/attachment/candy.thumb.gif', book.attachment(:cover).url(:thumb)

data/test/test_mongo.rb

@@ -2,6 +2,8 @@ require 'helper'
 require 'populate_me/mongo'
 # require 'mongo'
 
+Mongo::Logger.logger.level = Logger::ERROR
+
 MONGO = Mongo::Client.new([ '127.0.0.1:27017' ], :database => 'populate-me-test')
 DB = MONGO.database
 DB.drop

data/test/test_s3_attachment.rb

@@ -0,0 +1,273 @@
+require 'helper'
+require 'populate_me/document'
+require 'populate_me/s3_attachment'
+
+s3_resource = Aws::S3::Resource.new
+s3_bucket = s3_resource.bucket(ENV['BUCKET'])
+PopulateMe::Document.set :default_attachment_class, PopulateMe::S3Attachment
+PopulateMe::S3Attachment.set :bucket, s3_bucket
+
+describe 'PopulateMe::S3Attachment' do
+  # parallelize_me!
+
+  class S3Book < PopulateMe::Document
+    field :cover, type: :attachment, variations: [
+      PopulateMe::Variation.new_image_magick_job(:thumb, :gif, "-resize '300x'")
+    ]
+    field :content, type: :attachment, variations: [
+      PopulateMe::Variation.new(:upcase, :txt, lambda{ |src,dst|
+        Kernel.system "cat \"#{src}\" | tr 'a-z' 'A-Z' > \"#{dst}\""
+      })
+    ]
+    field :open_content, type: :attachment, url_prefix: 'open', variations: [
+      PopulateMe::Variation.new(:upcase, :txt, lambda{ |src,dst|
+        Kernel.system "cat \"#{src}\" | tr 'a-z' 'A-Z' > \"#{dst}\""
+      })
+    ]
+  end
+
+  class S3BookNoPrefix < PopulateMe::Document
+    set :s3_url_prefix, ''
+    field :content, type: :attachment, variations: [
+      PopulateMe::Variation.new(:upcase, :txt, lambda{ |src,dst|
+        Kernel.system "cat \"#{src}\" | tr 'a-z' 'A-Z' > \"#{dst}\""
+      })
+    ]
+  end
+
+  before do
+    s3_bucket.clear!
+  end
+
+  # Utils
+
+  it 'Returns URL with bucket url' do
+    book = S3Book.new cover: "candy.jpg"
+    assert_equal "#{s3_bucket.url}/candy.jpg", book.attachment(:cover).url
+    assert_equal "#{s3_bucket.url}/candy.thumb.gif", book.attachment(:cover).url(:thumb)
+  end
+
+  it 'Has nil URL when field is blank' do
+    book = S3Book.new
+    assert_nil book.attachment(:cover).url
+  end
+
+  it 'Has location root without attachee prefix' do
+    book = S3Book.new
+    refute_match book.attachment(:cover).attachee_prefix, book.attachment(:cover).location_root
+  end
+
+  # Create
+
+  it "Saves attachments on create with variations" do
+    book = S3Book.new
+
+    file = Tempfile.new('foo')
+    file.write('hello')
+    file.rewind
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+    assert_equal 'public/s-3-book/story.txt', field_value
+    assert s3_bucket.object('public/s-3-book/story.txt').exists? 
+    assert s3_bucket.object('public/s-3-book/story.upcase.txt').exists? 
+
+    s3file = s3_bucket.object('public/s-3-book/story.txt')
+    assert_equal 'text/plain', s3file.content_type
+    assert check_if_public_read(s3file)
+    assert_equal 's-3-book', s3file.metadata['parent_collection']
+    assert_equal 'hello', s3file.get.body.read
+
+    vars3file = s3_bucket.object('public/s-3-book/story.upcase.txt')
+    assert_equal 'text/plain', vars3file.content_type
+    # assert_equal 'public-read', vars3file.acl
+    assert_equal 's-3-book', vars3file.metadata['parent_collection']
+    assert_equal 'HELLO', vars3file.get.body.read
+
+    file.close
+    file.unlink
+  end
+
+  it "Can override the url_prefix at document class level" do
+    file = Tempfile.new('foo')
+    book = S3BookNoPrefix.new
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+
+    assert_equal 's-3-book-no-prefix/story.txt', field_value
+    assert s3_bucket.object('s-3-book-no-prefix/story.txt').exists? 
+    assert s3_bucket.object('s-3-book-no-prefix/story.upcase.txt').exists? 
+  end
+
+  it "Can override the url_prefix at field level" do
+    file = Tempfile.new('foo')
+    book = S3Book.new
+
+    field_value = book.attachment(:open_content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+
+    assert_equal 'open/s-3-book/story.txt', field_value
+    assert s3_bucket.object('open/s-3-book/story.txt').exists? 
+    assert s3_bucket.object('open/s-3-book/story.upcase.txt').exists? 
+  end
+
+  it 'Does not create 2 files with the same name' do
+    file = Tempfile.new('foo')
+
+    book = S3Book.new
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+
+    assert_equal 'public/s-3-book/story.txt', field_value
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+
+    assert_equal 'public/s-3-book/story-1.txt', field_value
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+
+    assert_equal 'public/s-3-book/story-2.txt', field_value
+
+    file.close
+    file.unlink
+  end
+
+  # Delete
+
+  it 'Is deletable when field is not blank' do
+    book = S3Book.new cover: "candy.jpg"
+    assert book.attachment(:cover).deletable?
+  end
+
+  it 'Is not deletable when field is blank' do
+    book = S3Book.new
+    refute book.attachment(:cover).deletable?
+  end
+
+  it 'Deletes all attachments' do
+    file = Tempfile.new('foo')
+
+    book = S3Book.new
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+    book.content = field_value
+
+    assert s3_bucket.object('public/s-3-book/story.txt').exists? 
+    assert s3_bucket.object('public/s-3-book/story.upcase.txt').exists? 
+
+    book.attachment(:content).delete_all
+
+    refute s3_bucket.object('public/s-3-book/story.txt').exists? 
+    refute s3_bucket.object('public/s-3-book/story.upcase.txt').exists? 
+
+    file.close
+    file.unlink
+  end
+
+  it 'Deletes one attachment at a time' do
+    file = Tempfile.new('foo')
+
+    book = S3Book.new
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+    book.content = field_value
+
+    assert s3_bucket.object('public/s-3-book/story.txt').exists? 
+    assert s3_bucket.object('public/s-3-book/story.upcase.txt').exists? 
+
+    book.attachment(:content).delete
+
+    refute s3_bucket.object('public/s-3-book/story.txt').exists? 
+    assert s3_bucket.object('public/s-3-book/story.upcase.txt').exists? 
+
+    book.attachment(:content).delete :upcase
+
+    refute s3_bucket.object('public/s-3-book/story.upcase.txt').exists? 
+
+    file.close
+    file.unlink
+  end
+
+  # Update
+
+  it 'Deletes previous attachment when saving a new one' do
+    file = Tempfile.new('foo')
+    file.write('hello')
+    file.rewind
+
+    book = S3Book.new
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'story.txt',
+      type: 'text/plain'
+    })
+    book.content = field_value
+
+    assert s3_bucket.object('public/s-3-book/story.txt').exists? 
+    assert s3_bucket.object('public/s-3-book/story.upcase.txt').exists? 
+
+    file.rewind
+    file.write('world')
+    file.rewind
+
+    field_value = book.attachment(:content).create({
+      tempfile: file,
+      filename: 'history.md',
+      type: 'text/markdown'
+    })
+    book.content = field_value
+
+    refute s3_bucket.object('public/s-3-book/story.txt').exists? 
+    refute s3_bucket.object('public/s-3-book/story.upcase.txt').exists? 
+    assert s3_bucket.object('public/s-3-book/history.md').exists? 
+    assert s3_bucket.object('public/s-3-book/history.upcase.txt').exists? 
+
+    s3file = s3_bucket.object('public/s-3-book/history.md')
+    assert_equal 'text/markdown', s3file.content_type
+    assert_equal 's-3-book', s3file.metadata['parent_collection']
+    assert_equal 'world', s3file.get.body.read
+
+    s3file = s3_bucket.object('public/s-3-book/history.upcase.txt')
+    assert_equal 'text/plain', s3file.content_type
+    assert_equal 's-3-book', s3file.metadata['parent_collection']
+    assert_equal 'WORLD', s3file.get.body.read
+
+    file.close
+    file.unlink
+  end
+
+end
+
+s3_bucket.clear!
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: populate-me
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - Mickael Riga
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-24 00:00:00.000000000 Z
+date: 2017-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: web-utils
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.8
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: racksh
   requirement: !ruby/object:Gem::Requirement
@@ -200,6 +214,7 @@ files:
 - lib/populate_me/file_system_attachment.rb
 - lib/populate_me/grid_fs_attachment.rb
 - lib/populate_me/mongo.rb
+- lib/populate_me/s3_attachment.rb
 - lib/populate_me/variation.rb
 - lib/populate_me/version.rb
 - populate-me.gemspec
@@ -215,6 +230,7 @@ files:
 - test/test_document_typecasting.rb
 - test/test_grid_fs_attachment.rb
 - test/test_mongo.rb
+- test/test_s3_attachment.rb
 - test/test_variation.rb
 - test/test_version.rb
 homepage: https://github.com/mig-hub/populate-me
@@ -237,7 +253,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: PopulateMe is an admin system for web applications.
@@ -254,5 +270,6 @@ test_files:
 - test/test_document_typecasting.rb
 - test/test_grid_fs_attachment.rb
 - test/test_mongo.rb
+- test/test_s3_attachment.rb
 - test/test_variation.rb
 - test/test_version.rb