neofiles 1.1.4 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4a39d67d56f59a4bb677f560fb8aad30cdf9beb8
-  data.tar.gz: c452ac1bb6de68fec36a1c3f69370768ee5ae8ac
+  metadata.gz: b3a3e40bfe2629b65d69615621cbcdc37ab95e20
+  data.tar.gz: 7f259aec163ff50fc5669735f4477e3c058cd942
 SHA512:
-  metadata.gz: c66cf00f7b15ea2199a152ce86f04145aea3b90b1563279f6fb9dce9ef72c0b44d98c1c49a417cb571cedc36bee5b93e498c853fdf027533cb3d7cbc73dbd35e
-  data.tar.gz: 1e3242347e8314812ce3749ff7c58c7948be423a57f2f1b9a2cf59cb298c9a1b80591f54f7ca9d8baa70c457b58aa6f96c03371a3eb9abc17345daae6b329374
+  metadata.gz: 3f6faca2b60cb0d6cb706ba969d729b2b25146a8eaeaad7dc937e72aa74a5fa2119ff424f27660b477d1eacca2a716b5e2c59bd9d0b36fc0c6c30fed7433cb47
+  data.tar.gz: 63bb20c3acf34a8e337bb9097454577c46d045de915f5948c810cbb9f38b45c225d08b027d945dbba73ec3dbde29ccff1216cf4769c604ac6e586e614570176d

data/app/models/concerns/neofiles/data_store/mongo/file_helper.rb

@@ -0,0 +1,22 @@
+module Neofiles::DataStore::Mongo::FileHelper
+  extend ActiveSupport::Concern
+
+  included do
+    has_many :chunks, dependent: :destroy, order: [:n, :asc], class_name: 'Neofiles::FileChunk'
+    field :chunk_size, type: Integer, default: Neofiles::DataStore::Mongo::DEFAULT_CHUNK_SIZE
+    validates :chunk_size, presence: true
+
+    def self.copy_from_mongo_to_amazon_s3(ids)
+      ids.each do |id|
+        begin
+          mongo_object = Neofiles::DataStore::Mongo.find id
+          amazon_object = Neofiles::DataStore::AmazonS3.find(id) rescue nil
+          Neofiles::DataStore::AmazonS3.new(id).write(mongo_object.data) unless amazon_object
+        rescue Neofiles::DataStore::NotFoundException
+          next
+        end
+      end
+    end
+
+  end
+end

data/app/models/neofiles/data_store.rb

@@ -0,0 +1,2 @@
+module Neofiles::DataStore
+end

data/app/models/neofiles/data_store/amazon_s3.rb

@@ -0,0 +1,95 @@
+# Module for storing and reading files from Amazon S3
+# If you want to work with amazon s3 you need set values for the following parameters in your config file
+# amazon_s3_region - the AWS region to connect to. The region is used to construct the client endpoint.
+# amazon_s3_api, amazon_s3_secret - used to set credentials statically
+# bucket_name - storage name in amazon_s3. Bucket must have a name that conforms to the naming requirements for non-US Standard regions.
+# http://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html
+# File will be named as id of the Neofiles::File object
+
+require 'aws-sdk'
+
+class Neofiles::DataStore::AmazonS3
+
+  def self.bucket_name
+    Rails.application.config.neofiles.amazon_s3_bucket
+  end
+
+  def self.find(id)
+    s3_object = new(id)
+    if s3_object.data
+      s3_object
+    else
+      raise Neofiles::DataStore::NotFoundException
+    end
+  end
+
+
+
+  attr_reader :id, :data, :length, :md5
+
+  def initialize(id)
+    @id = id
+  end
+
+  def data
+    @data ||= client.get_object(
+          bucket: bucket_name,
+          key: s3_key
+      ).body.read
+  rescue Aws::S3::Errors::ServiceError
+    nil
+  end
+
+  def length
+    @length ||= data.length
+  end
+
+  def md5
+    @md5 ||= begin
+      md5 = Digest::MD5.new
+      md5 << data
+      md5.hexdigest
+    end
+  end
+
+  def write(data)
+    if data.is_a? Tempfile
+      data.flush
+      data.rewind
+      data = data.read
+    end
+
+    client.put_object(
+        body: data,
+        bucket: bucket_name,
+        key: s3_key
+    )
+    @data = data
+  end
+
+
+
+  private
+
+  def s3_key
+    object_id = @id.to_s
+    object_id[0..1] + '/' + object_id[2..4] + '/' + object_id
+  end
+
+  def bucket_name
+    self.class.bucket_name
+  end
+
+  def client
+    @client ||= Aws::S3::Client.new(
+        region: Rails.application.config.neofiles.amazon_s3_region,
+        credentials: Aws::Credentials.new(
+            Rails.application.config.neofiles.amazon_s3_api,
+            Rails.application.config.neofiles.amazon_s3_secret
+        )
+    )
+  rescue Aws::S3::Errors::ServiceError
+    nil
+  end
+
+end

data/app/models/neofiles/data_store/mongo.rb

@@ -0,0 +1,132 @@
+class Neofiles::DataStore::Mongo
+
+  DEFAULT_CHUNK_SIZE = Rails.application.config.neofiles.mongo_default_chunk_size
+
+  def self.chunks(id)
+    Neofiles::FileChunk.where(file_id: id).order_by(n: :asc)
+  end
+
+  def self.find(id)
+    if chunks(id).any?
+      new(id)
+    else
+      raise Neofiles::DataStore::NotFoundException
+    end
+  end
+
+
+
+  attr_reader :id, :data, :length, :md5
+
+  def initialize(id)
+    @id = id
+  end
+
+  def data
+    @data ||= chunks.pluck(:data).map(&:data).join
+  end
+
+  def length
+    @length ||= data.length
+  end
+
+  def md5
+    @md5 ||= begin
+      md5 = Digest::MD5.new
+      md5 << data
+      md5.hexdigest
+    end
+  end
+
+  def write(data)
+    data_buf  = []
+    md5       = Digest::MD5.new
+    length, n = 0, 0
+
+    reading(data) do |io|
+      chunking(io, DEFAULT_CHUNK_SIZE) do |buf|
+        md5 << buf
+        data_buf << buf
+        length += buf.size
+        chunk = chunks.build file_id: id
+        chunk.data = binary_for buf
+        chunk.n = n
+        n += 1
+        chunk.save!
+      end
+    end
+
+    @data   = data_buf.join
+    @length = length
+    @md5    = md5.hexdigest
+  end
+
+
+
+  private
+
+  def chunks
+    self.class.chunks id
+  end
+
+  # Yield block with IO stream made from input arg, which can be file name or other IO readable object.
+  def reading(arg, &block)
+    if arg.respond_to?(:read)
+      rewind(arg) do |io|
+        block.call(io)
+      end
+    else
+      open(arg.to_s) do |io|
+        block.call(io)
+      end
+    end
+  end
+
+  # Split IO stream by chunks chunk_size bytes each and yield each chunk in block.
+  def chunking(io, chunk_size, &block)
+    if io.method(:read).arity == 0
+      data = io.read
+      i = 0
+      loop do
+        offset = i * chunk_size
+        length = i + chunk_size < data.size ? chunk_size : data.size - offset
+
+        break if offset >= data.size
+
+        buf = data[offset, length]
+        block.call(buf)
+        i += 1
+      end
+    else
+      while buf = io.read(chunk_size)
+        block.call(buf)
+      end
+    end
+  end
+
+  # Construct Mongoid binary object from string of bytes.
+  def binary_for(*buf)
+    BSON::Binary.new buf.join, :generic
+  end
+
+  # Yield IO-like argument to block rewinding it first, if possible.
+  def rewind(io, &block)
+    begin
+      pos = io.pos
+      io.flush
+      io.rewind
+    rescue
+      nil
+    end
+
+    begin
+      block.call(io)
+    ensure
+      begin
+        io.pos = pos
+      rescue
+        nil
+      end
+    end
+  end
+end

data/app/models/neofiles/data_store/not_found_exception.rb

@@ -0,0 +1,2 @@
+class Neofiles::DataStore::NotFoundException < StandardError
+end

data/app/models/neofiles/file.rb

@@ -44,74 +44,36 @@ class Neofiles::File
 
   include Mongoid::Document
   include Mongoid::Timestamps
+  include Neofiles::DataStore::Mongo::FileHelper
 
   store_in collection: Rails.application.config.neofiles.mongo_files_collection, client: Rails.application.config.neofiles.mongo_client
 
-  has_many :chunks, dependent: :destroy, order: [:n, :asc], class_name: 'Neofiles::FileChunk'
-
-  DEFAULT_CHUNK_SIZE = Rails.application.config.neofiles.mongo_default_chunk_size
-
   field :filename, type: String
   field :content_type, type: String
   field :length, type: Integer, default: 0
-  field :chunk_size, type: Integer, default: DEFAULT_CHUNK_SIZE
   field :md5, type: String, default: Digest::MD5.hexdigest('')
   field :description, type: String
   field :owner_type, type: String
   field :owner_id, type: String
   field :is_deleted, type: Mongoid::Boolean
 
-  validates :filename, :length, :chunk_size, :md5, presence: true
-
   before_save :save_file
   after_save :nullify_unpersisted_file
 
-
-
-  # Yield block for each chunk.
-  def each(&block)
-    chunks.all.order_by([:n, :asc]).each do |chunk|
-      block.call(chunk.to_s)
-    end
-  end
-
-  # Get a portion of chunks, either via Range of Fixnum (length).
-  def slice(*args)
-    case args.first
-      when Range
-        range = args.first
-        first_chunk = (range.min / chunk_size).floor
-        last_chunk = (range.max / chunk_size).ceil
-        offset = range.min % chunk_size
-        length = range.max - range.min + 1
-      when Fixnum
-        start = args.first
-        start = self.length + start if start < 0
-        length = args.size == 2 ? args.last : 1
-        first_chunk = (start / chunk_size).floor
-        last_chunk = ((start + length) / chunk_size).ceil
-        offset = start % chunk_size
-    end
-
-    data = ''
-
-    chunks.where(n: first_chunk..last_chunk).order_by(n: :asc).each do |chunk|
-      data << chunk
-    end
-
-    data[offset, length]
-  end
-
   # Chunks bytes concatenated, that is the whole file content.
   def data
-    data = ''
-    each { |chunk| data << chunk }
-    data
+    self.class.read_data_stores.each do |store|
+      begin
+        return store.find(id).data
+      rescue Neofiles::DataStore::NotFoundException
+        next
+      end
+    end
   end
 
   # Encode bytes in base64.
   def base64
-    Array(to_s).pack('m')
+    Array(data).pack('m')
   end
 
   # Encode bytes id data uri.
@@ -120,18 +82,6 @@ class Neofiles::File
     "data:#{content_type};base64,#{data}"
   end
 
-  # Bytes as chunks array, if block is given — yield it.
-  def bytes(&block)
-    if block
-      each { |data| block.call(data) }
-      length
-    else
-      bytes = []
-      each { |data| bytes.push(*data) }
-      bytes
-    end
-  end
-
 
 
   attr_reader :file
@@ -159,26 +109,17 @@ class Neofiles::File
   # File length and md5 hash are computed automatically.
   def save_file
     if @file
-      self.chunks.delete_all
-
-      md5 = Digest::MD5.new
-      length, n = 0, 0
-
-      self.class.reading(@file) do |io|
-        self.class.chunking(io, chunk_size) do |buf|
-          md5 << buf
-          length += buf.size
-          chunk = self.chunks.build
-          chunk.data = self.class.binary_for(buf)
-          chunk.n = n
-          n += 1
-          chunk.save!
-          self.chunks.push(chunk)
+      self.class.write_data_stores.each do |store|
+        begin
+          data_store_object = store.new id
+          data_store_object.write @file
+          self.length = data_store_object.length
+          self.md5    = data_store_object.md5
+        rescue => ex
+          notify_airbrake(ex) if defined? notify_airbrake
+          next
         end
       end
-
-      self.length = length
-      self.md5    = md5.hexdigest
     end
   end
 
@@ -193,46 +134,6 @@ class Neofiles::File
     template.neofiles_link self, nil, target: '_blank'
   end
 
-  # Yield block with IO stream made from input arg, which can be file name or other IO readable object.
-  def self.reading(arg, &block)
-    if arg.respond_to?(:read)
-      self.rewind(arg) do |io|
-        block.call(io)
-      end
-    else
-      open(arg.to_s) do |io|
-        block.call(io)
-      end
-    end
-  end
-
-  # Split IO stream by chunks chunk_size bytes each and yield each chunk in block.
-  def self.chunking(io, chunk_size, &block)
-    if io.method(:read).arity == 0
-      data = io.read
-      i = 0
-      loop do
-        offset = i * chunk_size
-        length = i + chunk_size < data.size ? chunk_size : data.size - offset
-
-        break if offset >= data.size
-
-        buf = data[offset, length]
-        block.call(buf)
-        i += 1
-      end
-    else
-      while buf = io.read(chunk_size)
-        block.call(buf)
-      end
-    end
-  end
-
-  # Construct Mongoid binary object from string of bytes.
-  def self.binary_for(*buf)
-    BSON::Binary.new(buf.join, :generic)
-  end
-
   # Try different methods to extract file name or path from argument object.
   def self.extract_basename(object)
     filename = nil
@@ -300,24 +201,21 @@ class Neofiles::File
     class_by_file_name(extract_basename(file_object))
   end
 
-  # Yield IO-like argument to block rewinding it first, if possible.
-  def self.rewind(io, &block)
-    begin
-      pos = io.pos
-      io.flush
-      io.rewind
-    rescue
-      nil
-    end
+  def self.read_data_stores
+    get_stores_class_name Rails.application.config.neofiles.read_data_stores
+  end
 
-    begin
-      block.call(io)
-    ensure
-      begin
-        io.pos = pos
-      rescue
-        nil
-      end
+  def self.write_data_stores
+    get_stores_class_name Rails.application.config.neofiles.write_data_stores
+  end
+
+  # return array with names for each store
+  def self.get_stores_class_name(stores)
+    if stores.is_a?(Array)
+      stores.map { |store| Neofiles::DataStore.const_get(store.camelize) }
+    else
+      get_stores_class_name [stores]
     end
   end
+
 end

data/lib/neofiles.rb

@@ -1,4 +1,5 @@
 require 'neofiles/engine'
+require 'aspect_ratio'
 
 module Neofiles
   # Attach Neofiles specific routes in your routes.rb file:
@@ -34,8 +35,6 @@ module Neofiles
   #   width, height   - max width and height after resize
   #   resize_options  - {crop: '1'/'0'}, @see Neofiles::ImagesController#show
   #
-  # Can call ImageMagick.
-  #
   def resized_image_dimensions(image_file, width, height, resize_options)
     # dimensions are equal to requested ones if cropping
     return width, height if crop_requested? resize_options
@@ -58,17 +57,7 @@ module Neofiles
     # image fits into requested dimensions, no resizing will occur
     return image_file_width, image_file_height if image_file_width <= width && image_file_height <= height
 
-    # ... construct request ...
-    command = MiniMagick::CommandBuilder.new(:convert)            # convert input file...
-    command.size([image_file_width, image_file_height].join 'x')  # with the given dimensions...
-    command.xc('white')                                           # and filled with whites...
-    command.resize([width, height].join 'x')                      # to fit in the given rectangle...
-    command.push('info:-')                                        # return info about the resulting file
-
-    # ... and send it to ImageMagick
-    # the result will be: xc:white XC 54x100 54x100+0+0 16-bit DirectClass 0.070u 0:00.119
-    # extract dimensions and return them as array of integers
-    MiniMagick::Image.new(nil, nil).run(command).match(/ (\d+)x(\d+) /).values_at(1, 2).map(&:to_i)
+    AspectRatio.resize(image_file_width, image_file_height, width, height).map(&:to_i)
 
   rescue
     nil

data/lib/neofiles/version.rb

@@ -1,3 +1,3 @@
 module Neofiles
-  VERSION = '1.1.4'
+  VERSION = '1.2.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neofiles
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.2.1
 platform: ruby
 authors:
 - Konanykhin Ilya
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-04 00:00:00.000000000 Z
+date: 2018-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -80,6 +80,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: aspect_ratio
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 'Library for managing files: creating & storing, linking to file owners,
   serving files from MongoDB'
 email:
@@ -106,6 +134,11 @@ files:
 - app/controllers/neofiles/files_controller.rb
 - app/controllers/neofiles/images_controller.rb
 - app/helpers/neofiles/neofiles_helper.rb
+- app/models/concerns/neofiles/data_store/mongo/file_helper.rb
+- app/models/neofiles/data_store.rb
+- app/models/neofiles/data_store/amazon_s3.rb
+- app/models/neofiles/data_store/mongo.rb
+- app/models/neofiles/data_store/not_found_exception.rb
 - app/models/neofiles/file.rb
 - app/models/neofiles/file_chunk.rb
 - app/models/neofiles/image.rb