tp_common 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13d2cbc9467bcaee1a9a7af73e0bfb00bf2e9894566bba54e3d16dbdf0f82d16
-  data.tar.gz: 696b538814ac4e9310620d4765707a7375af377906a780e33c86d267168b260a
+  metadata.gz: 6ffe086d077ec950e5eb7cab45373b4ad12efc85a827016a53356aab033e3182
+  data.tar.gz: be6fffbe4ae86f5162c7e77ec893c9d68ec71d0d17b172d7061a9dace8d09763
 SHA512:
-  metadata.gz: 8391124ec637bc99aecb83b052d0b5bdf03bf5a5ac467a513ca98f41e638d3746888af7eb9276d8535853a34d7fe1c4d4512d19b499cf5d09a0ad94cbee2fa53
-  data.tar.gz: aa537440ba5a50225284356c93a22612c96e33543b5e2ca62a1a0b5577e7c273500cb860c1a6d31701927c27de424a40d4e671106b1cbc56228603dd5618c061
+  metadata.gz: 97786a1e6a6ea34a257d966dbaee0b51c4fb2f96b7afdfdcc064abeb0f62bbd8453dbfaeb0cb4b051b477ed35f3ec743b7474feb038d87d06b454464b5389e36
+  data.tar.gz: 2bd58921e8e8cc436d3434367ec7b73743d40640a2bd8e6fb585419ecbdc3714db152a20a8a94a999ed4762e4a8232d56bb11f3ac2f65270f57df45721ddf370

data/README.md

@@ -5,6 +5,12 @@
 
 This gem contains common libraries which shared between projects used in TINYpulse.
 
+Currently, we have:
+
+1. Timezone helpers
+2. Asset loader
+3. File storage
+
 ## Rails versions
 
 From v0.2.0 `tp_common` support only rails > 4. Mostly on ActiveSupport DateTime functions and `Rails.config_for` method.

data/gemfiles/rails_4.gemfile.lock

@@ -1,8 +1,11 @@
 PATH
   remote: ..
   specs:
-    tp_common (0.3.0)
+    tp_common (0.4.0)
       activesupport (>= 4.2.0, <= 5.2)
+      aws-sdk-s3 (~> 1.19)
+      fog-aws (~> 2.0)
+      mime-types (~> 3.2)
       tzinfo (~> 1.2, >= 1.2.0)
 
 GEM
@@ -47,6 +50,21 @@ GEM
       rake
       thor (>= 0.14.0)
     arel (6.0.4)
+    aws-eventstream (1.0.1)
+    aws-partitions (1.111.0)
+    aws-sdk-core (3.38.0)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.11.0)
+      aws-sdk-core (~> 3, >= 3.26.0)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.23.1)
+      aws-sdk-core (~> 3, >= 3.26.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.3)
     builder (3.2.3)
     byebug (9.1.0)
     coderay (1.1.2)
@@ -54,19 +72,42 @@ GEM
     crass (1.0.3)
     diff-lcs (1.3)
     erubis (2.7.0)
+    excon (0.62.0)
+    fog-aws (2.0.1)
+      fog-core (~> 1.38)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+      ipaddress (~> 0.8)
+    fog-core (1.45.0)
+      builder
+      excon (~> 0.58)
+      formatador (~> 0.2)
+    fog-json (1.2.0)
+      fog-core
+      multi_json (~> 1.10)
+    fog-xml (0.1.3)
+      fog-core
+      nokogiri (>= 1.5.11, < 2.0.0)
+    formatador (0.2.5)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
     i18n (0.9.1)
       concurrent-ruby (~> 1.0)
+    ipaddress (0.8.3)
+    jmespath (1.4.0)
     loofah (2.1.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.0)
       mini_mime (>= 0.1.1)
     method_source (0.9.0)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
     mini_mime (1.0.0)
     mini_portile2 (2.3.0)
     minitest (5.11.1)
+    multi_json (1.13.1)
     nokogiri (1.8.1)
       mini_portile2 (~> 2.3.0)
     pry (0.11.3)

data/gemfiles/rails_5.gemfile.lock

@@ -1,8 +1,11 @@
 PATH
   remote: ..
   specs:
-    tp_common (0.3.0)
+    tp_common (0.4.0)
       activesupport (>= 4.2.0, <= 5.2)
+      aws-sdk-s3 (~> 1.19)
+      fog-aws (~> 2.0)
+      mime-types (~> 3.2)
       tzinfo (~> 1.2, >= 1.2.0)
 
 GEM
@@ -50,6 +53,21 @@ GEM
       rake
       thor (>= 0.14.0)
     arel (7.1.4)
+    aws-eventstream (1.0.1)
+    aws-partitions (1.111.0)
+    aws-sdk-core (3.38.0)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.11.0)
+      aws-sdk-core (~> 3, >= 3.26.0)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.23.1)
+      aws-sdk-core (~> 3, >= 3.26.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.3)
     builder (3.2.3)
     byebug (9.1.0)
     coderay (1.1.2)
@@ -57,19 +75,42 @@ GEM
     crass (1.0.3)
     diff-lcs (1.3)
     erubis (2.7.0)
+    excon (0.62.0)
+    fog-aws (2.0.1)
+      fog-core (~> 1.38)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+      ipaddress (~> 0.8)
+    fog-core (1.45.0)
+      builder
+      excon (~> 0.58)
+      formatador (~> 0.2)
+    fog-json (1.2.0)
+      fog-core
+      multi_json (~> 1.10)
+    fog-xml (0.1.3)
+      fog-core
+      nokogiri (>= 1.5.11, < 2.0.0)
+    formatador (0.2.5)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
     i18n (0.9.1)
       concurrent-ruby (~> 1.0)
+    ipaddress (0.8.3)
+    jmespath (1.4.0)
     loofah (2.1.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.0)
       mini_mime (>= 0.1.1)
     method_source (0.9.0)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
     mini_mime (1.0.0)
     mini_portile2 (2.3.0)
     minitest (5.11.1)
+    multi_json (1.13.1)
     nio4r (2.2.0)
     nokogiri (1.8.1)
       mini_portile2 (~> 2.3.0)

data/lib/tp_common.rb

@@ -17,3 +17,4 @@ end
 require "tp_common/timezones"
 require "tp_common/timezones/zone"
 require "tp_common/assets_loader"
+require "tp_common/file_storage"

data/lib/tp_common/file_storage.rb

@@ -0,0 +1,38 @@
+require 'aws-sdk-s3'
+
+module TpCommon
+  module FileStorage
+    Configuration = Struct.new(:aws_region,
+                               :aws_key_id,
+                               :aws_secret_key,
+                               :key_prefix,
+                               :default_bucket)
+
+    def self.configure
+      @configuration = Configuration.new('us-east-1', nil, nil, nil, nil)
+
+      yield(@configuration)
+
+      Aws.config.update({
+        region: @configuration.aws_region,
+        credentials: Aws::Credentials.new(
+          @configuration.aws_key_id,
+          @configuration.aws_secret_key)
+      })
+    end
+
+    def self.key_prefix
+      raise ::StandardError.new('FileStorage is not config yet.') unless defined?(@configuration)
+
+      @configuration.key_prefix
+    end
+
+    def self.configuration
+      raise ::StandardError.new('FileStorage is not config yet.') unless defined?(@configuration)
+
+      @configuration
+    end
+  end
+end
+
+require 'tp_common/file_storage/base'

data/lib/tp_common/file_storage/base.rb

@@ -0,0 +1,40 @@
+require 'fog/aws'
+require 'tp_common/file_storage/fog_connection'
+require 'tp_common/file_storage/key_protector'
+
+module TpCommon
+  module FileStorage
+    # Prepare connection, key protector, consts for working wilt s3 files
+    class Base
+      MAX_RETRIES = 3
+
+      include TpCommon::FileStorage::FogConnection
+      include TpCommon::FileStorage::KeyProtector
+
+      def initialize(directory_path = nil)
+        @directory_path = directory_path || TpCommon::FileStorage.configuration.default_bucket
+      end
+
+      private
+
+      # Connection to S3, config in initializer
+      def connection
+        @connection ||= create_connection
+      end
+
+      # Fog directory to bucket config in constructor
+      def directory
+        @directory ||= connection.directories.get(@directory_path)
+      end
+    end
+  end
+end
+
+require 'tp_common/file_storage/errors/failed_to_download'
+require 'tp_common/file_storage/errors/file_not_found'
+
+require 'tp_common/file_storage/cleaners/cleaner'
+require 'tp_common/file_storage/direct_uploaders/public'
+require 'tp_common/file_storage/downloaders/private'
+require 'tp_common/file_storage/uploaders/private'
+require 'tp_common/file_storage/uploaders/public'

data/lib/tp_common/file_storage/cleaners/cleaner.rb

@@ -0,0 +1,16 @@
+module TpCommon
+  module FileStorage
+    module Cleaners
+      # Delete files on file storage service.
+      # Please handle with care
+      #
+      class Cleaner < FileStorage::Base
+        # Expect an array of file key to delete.
+        #
+        def clean(files)
+          connection.delete_multiple_objects(@directory_path, files.map{|file_key| mask_key(file_key) })
+        end
+      end
+    end
+  end
+end

data/lib/tp_common/file_storage/direct_uploaders/public.rb

@@ -0,0 +1,49 @@
+module TpCommon
+  module FileStorage
+    module DirectUploaders
+      # Prepare a presigned url to upload diectly to storage service
+      # Use in case content to upload is outside of system/server,
+      #   we shoud use this to avoid delay because of content stream over our server
+      #
+      class Public < FileStorage::Base
+        def initialize(bucket_name = nil)
+          @bucket_name = bucket_name || TpCommon::FileStorage.configuration.default_bucket
+          @bucket = Aws::S3::Bucket.new(name: @bucket_name)
+        end
+
+        # Presigned POST url, use for form POST with data. Use when file name isn't decided yet
+        # Request body is form data
+        # Return a hash
+        #
+        def presigned_post(file_key)
+          s3_direct_post = @bucket.presigned_post(key: mask_key(file_key),
+                                                  success_action_status: '201',
+                                                  acl: 'public-read')
+          {
+            host: URI(s3_direct_post.url).host,
+            bucket: @bucket_name,
+            url: "#{s3_direct_post.url}/",
+            form_data: s3_direct_post.fields
+          }
+        end
+
+        # Presigned PUT url, use for PUT file to storage service.
+        # File name must be matched, and request body is file content
+        # Return a hash
+        #
+        def presigned_put(file_key, mime_type, expires_in = 900)
+          @bucket.object(mask_key(file_key)).presigned_url(:put,
+                                            content_type: mime_type,
+                                            acl: 'public-read',
+                                            expires_in: expires_in)
+        end
+
+        # Public URL from key for uploaded file
+        #
+        def url(file_key)
+          @bucket.object(mask_key(file_key))&.public_url(path_style: true)
+        end
+      end
+    end
+  end
+end

data/lib/tp_common/file_storage/downloaders/private.rb

@@ -0,0 +1,31 @@
+module TpCommon
+  module FileStorage
+    module Downloaders
+      # Pull and read a files from storage service.
+      # Included retry mechanism.
+      #
+      class Private < FileStorage::Base
+        # Pull a file with provided key.
+        # Return a Fog file object https://www.rubydoc.info/github/fog/fog-aws/Fog/Storage/AWS/File
+        #
+        def download(file_key)
+          retried_count = 0
+          begin
+            directory.files.get(mask_key(file_key)).tap do |file|
+              raise FileStorage::Errors::FileNotFound.new("Could not find file: #{file_key}") unless file
+            end
+          rescue ::Fog::Errors::Error => error
+            retried_count += 1
+            retry if retried_count < MAX_RETRIES
+            raise FileStorage::Errors::FailedToDownload.new("Failed to download file via fog: #{error.message}")
+          end
+        end
+
+        # Same as #download but return file content
+        def read(file_key)
+          download(mask_key(file_key)).body
+        end
+      end
+    end
+  end
+end

data/lib/tp_common/file_storage/errors/failed_to_download.rb

@@ -0,0 +1,8 @@
+module TpCommon
+  module FileStorage
+    module Errors
+      class FailedToDownload < ::StandardError
+      end
+    end
+  end
+end

data/lib/tp_common/file_storage/errors/file_not_found.rb

@@ -0,0 +1,8 @@
+module TpCommon
+  module FileStorage
+    module Errors
+      class FileNotFound < ::StandardError
+      end
+    end
+  end
+end

data/lib/tp_common/file_storage/fog_connection.rb

@@ -0,0 +1,18 @@
+require 'fog/aws'
+
+module TpCommon
+  module FileStorage
+    # Provide default connection to file storage service
+    #
+    module FogConnection
+
+      private
+
+      def create_connection
+        Fog::Storage.new(provider: 'AWS',
+          aws_access_key_id: FileStorage.configuration.aws_key_id,
+          aws_secret_access_key: FileStorage.configuration.aws_secret_key)
+      end
+    end
+  end
+end

data/lib/tp_common/file_storage/key_protector.rb

@@ -0,0 +1,25 @@
+module TpCommon
+  module FileStorage
+    # Provide mechanism to protect keys conflict using prefix
+    # This module supports using 1 bucket for multiple environment, just have different prefix key
+    # Although it's recommended in real usage, using in development env is recommended.
+    #   To avoid issue local uploading works but trouble when work with S3
+    module KeyProtector
+
+      private
+
+      # Mask key with s3 dev prefix. !!! Make sure it's idempotent
+      #
+      def mask_key(key)
+        return key if !FileStorage.key_prefix
+
+        # Trim and remove redundant slash
+        key = key.gsub(/^\/*/, '').gsub(/\/*$/, '').gsub(/\/+/, '/')
+        regex = Regexp.new("^#{FileStorage.key_prefix}/")
+        return key if regex.match(key)
+
+        "#{FileStorage.key_prefix}/#{key}"
+      end
+    end
+  end
+end

data/lib/tp_common/file_storage/uploaders/private.rb

@@ -0,0 +1,39 @@
+module TpCommon
+  module FileStorage
+    module Uploaders
+      # Upload a content to file storage used for private purpose like user import. export files
+      # Use in case content to upload is inside of system/server
+      #
+      class Private < FileStorage::Base
+        # Upload content to file_key
+        #
+        def upload(file_key, content, content_type)
+          retry_count = 0
+
+          begin
+            directory.files.create(
+              'key' => mask_key(file_key),
+              "body" =>  content,
+              'public' =>  false,
+              'Content-Type' => content_type,
+              "Content-Disposition" => "attachment;filename=\"#{mask_key(file_key)}\""
+            )
+          rescue StandardError => e
+            retry_count += 1
+            retry if retry_count < MAX_RETRIES
+            raise e
+          end
+
+          mask_key(file_key)
+        end
+        # Get url from key of file #upload above to provide to outside.
+        #   To get file content to use inside system, please use Downloaders::Private instead
+        # As private file, link has a ttl, default 1 week.
+        #
+        def url(file_key, link_ttl = 1.week.from_now)
+          directory.files.get_https_url(mask_key(file_key), link_ttl)
+        end
+      end
+    end
+  end
+end

data/lib/tp_common/file_storage/uploaders/public.rb

@@ -0,0 +1,36 @@
+module TpCommon
+  module FileStorage
+    module Uploaders
+      # Upload a content to file storage used to public like avatar, logo
+      # Use in case content to upload is inside of system/server
+      #
+      class Public < FileStorage::Base
+        # Upload content to file_key
+        # Currently, _content_type is ignore but kept for compatible. Will be removed in next release
+        #
+        def upload(file_key, content, _content_type = nil)
+          retry_count = 0
+
+          begin
+            directory.files.create(
+              key: mask_key(file_key),
+              body: content,
+              public: true)
+          rescue StandardError => e
+            retry_count += 1
+            retry if retry_count < MAX_RETRIES
+            raise e
+          end
+
+          mask_key(file_key)
+        end
+
+        # Get public url from key of file #upload above
+        #
+        def url(file_key)
+          directory.files.get(mask_key(file_key))&.public_url
+        end
+      end
+    end
+  end
+end

data/lib/tp_common/timezones/config.rb

@@ -8,11 +8,12 @@ module TpCommon
         if defined?(::Rails::Railtie)
           begin
             TpCommon::Timezones.const_set("LIST_ZONES", Rails.application.config_for(:timezones))
+            puts "[TpCommon::Timezones::Config] Message-3: Use custom timezones in project config/timezones.yml"
             return
           rescue NameError, NoMethodError
-            puts "Couldn't load Rails or config methods. Use default."
+            puts "[TpCommon::Timezones::Config] Message-1: Use default timezones in tp_common/timezones/config/timezones.yml"
           rescue StandardError
-            puts "Couldn't load file config/timezones.yml. Use default."
+            puts "[TpCommon::Timezones::Config] Message-2: Use default timezones in tp_common/timezones/config/timezones.yml"
           end
         end
 

data/lib/tp_common/version.rb

@@ -1,3 +1,3 @@
 module TpCommon
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/tp_common.gemspec

@@ -37,4 +37,9 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "tzinfo", '~> 1.2', '>= 1.2.0'
   spec.add_runtime_dependency "activesupport", '<= 5.2', '>= 4.2.0'
+  spec.add_runtime_dependency 'fog-aws', '~> 2.0'
+  # fog-aws need a mime-type tool
+  spec.add_runtime_dependency 'mime-types', '~> 3.2'
+
+  spec.add_runtime_dependency 'aws-sdk-s3', '~> 1.19'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tp_common
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - TINYpulse Devops
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-31 00:00:00.000000000 Z
+date: 2018-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,62 +84,104 @@ dependencies:
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: tzinfo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
     - - "<="
       - !ruby/object:Gem::Version
         version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.2.0
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: fog-aws
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
 description: Common libraries used for TINYpulse apps. i.e. Timezone, ...
 email:
 - devops@tinypulse.com
@@ -167,6 +209,17 @@ files:
 - gemfiles/rails_5.gemfile.lock
 - lib/tp_common.rb
 - lib/tp_common/assets_loader.rb
+- lib/tp_common/file_storage.rb
+- lib/tp_common/file_storage/base.rb
+- lib/tp_common/file_storage/cleaners/cleaner.rb
+- lib/tp_common/file_storage/direct_uploaders/public.rb
+- lib/tp_common/file_storage/downloaders/private.rb
+- lib/tp_common/file_storage/errors/failed_to_download.rb
+- lib/tp_common/file_storage/errors/file_not_found.rb
+- lib/tp_common/file_storage/fog_connection.rb
+- lib/tp_common/file_storage/key_protector.rb
+- lib/tp_common/file_storage/uploaders/private.rb
+- lib/tp_common/file_storage/uploaders/public.rb
 - lib/tp_common/railtie.rb
 - lib/tp_common/timezones.rb
 - lib/tp_common/timezones/config.rb
@@ -195,7 +248,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: Common libraries used for TINYpulse apps