eitje_s3 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b3fab11f169aa044513495cd47f13853ec305a31fe72e5075b681f042d9712c8
+  data.tar.gz: 464f079e85d4e7df5373d70732ba93852ff9489b3d543404eee769d6d5e86dec
+SHA512:
+  metadata.gz: a8c6436222d6708cbe69ae3d4d38e97c6a015db22e870639520ea05ee36ad721fadb7cae6eac3daeb53a5649035ead3562d654865f785c2f2e8e77468b2647cf
+  data.tar.gz: 47f1b3af7e66c4740d8a4fcba65d824096e4287a0b22d122d2e132fbc37b660e573ac18ff3c38b72e1a0539dbf8540c792fcab032f9e1ebe77efc0ad0c399319

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2021 Jurriaan Schrofer
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# S3 gem
+
+Never stops lovin AWS, yay!
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+
+gem 's3', git: 'https://github.com/eitje-app/s3_engine', branch: 'production'
+
+```
+
+Or, for development purposes, add instead:
+
+```ruby
+
+local_paths = %w$ /Users/jurriaanschrofer/Documents/s3 $
+if ENV['IS_LOCAL_ENGINE'] && ( path = local_paths.find {|path| Dir.glob(path).any?} )
+  gem 's3', path: path
+else
+  gem 's3', git: 'https://github.com/eitje-app/s3_engine', branch: 'production'
+end
+
+```
+
+Create and run the required migrations:
+
+```bash
+
+$ rails s3:install:migrations
+$ rails db:migrate
+
+```
+Add the following to your routes.rb:
+```ruby
+
+mount S3::Engine => "/s3"
+
+```

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test

data/app/controllers/s3/application_controller.rb

@@ -0,0 +1,5 @@
+module S3
+  class ApplicationController < ActionController::API
+
+  end
+end

data/app/controllers/s3/records_controller.rb

@@ -0,0 +1,23 @@
+class S3::RecordsController < API::Admin::BaseController #ApplicationController
+
+  # before_action :authorize_admin
+
+  # queries files split per date and environment
+  def deleted_records
+    data = S3::NewDeletedRecordsService.get_records(**slice_params(:db_table, :start_date, :end_date, :env_id))
+    render json: {deleted_records: data}
+  end  
+
+  # queries files split per date and table
+  def legacy_deleted_records
+    data = S3::OldDeletedRecordsService.get_records(**slice_params(:db_table, :start_date, :end_date, :env_id, :env_name))
+    render json: {deleted_records: data}
+  end
+
+  private
+
+  def slice_params(*args)
+    params.slice(*args).as_json.symbolize_keys
+  end
+
+end

data/app/services/s3/new_deleted_records_service.rb

@@ -0,0 +1,49 @@
+module S3::NewDeletedRecordsService
+  class << self
+
+   DB_TABLES = %w$ shifts teams users contracts infos posts $
+
+    def test(table)
+      S3::NewDeletedRecordsService.get_records(
+        db_table: table, start_date: '2021-03-28', end_date: '2021-04-3', env_id: 513
+      )
+    end
+
+    def get_records(db_table:, start_date:, end_date:, env_id:)    
+      @date_range = Date.parse(start_date)..Date.parse(end_date)
+      @s3         = Aws::S3::Client.new 
+      @db_table   = db_table
+      @env_id     = env_id
+
+      set_file_name
+      validate_args
+
+      query_records
+      filter_records
+      @records
+    end
+
+    def validate_args
+      throw :db_table_name_is_not_valid unless DB_TABLES.include?(@db_table)
+    end
+
+    def set_file_name
+      @file_name = "env_#{@env_id}_deleted_#{@db_table}.json"
+    end
+
+    def query_records    
+      file     = @s3.get_object(bucket: 'eitje-deleted-jurr', key: @file_name)
+      @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
+    end
+
+    def filter_records
+      return if no_deleted_at?
+      @records.select! { |record| @date_range.include?(Date.parse(record[:deleted_at])) }
+    end
+
+    def no_deleted_at?
+      %w[infos].include? @db_table # files without deleted_at field
+    end
+
+  end
+end

data/app/services/s3/old_deleted_records_service.rb

@@ -0,0 +1,95 @@
+module S3::OldDeletedRecordsService
+  class << self
+
+   DB_TABLES = %w$ shifts teams users contracts infos posts $
+
+   # S3::OldDeletedRecordsService.get_records(env_id: 307, env_name: 'Kua - Den Haag', db_table: 'verlofverzoeks', start_date: '2021-02-01', end_date: '2021-04-01')
+
+    def get_records(db_table:, start_date:, end_date:, env_id:, env_name:)
+
+      # validate_args(db_table)
+      
+      @start_date = start_date
+      @end_date   = end_date
+      @db_table   = db_table
+      @env_name   = env_name
+      @env_id     = env_id
+      @date_range = get_date_range
+
+      @file_names_filtered_by_table = request_object_names
+      @file_names_filtered_by_date  = filter_by_date
+
+      read_all_files
+
+    end
+
+    # validations
+
+    def validate_args(db_table)
+      throw :db_table_name_is_not_valid unless DB_TABLES.include?(db_table)
+    end
+
+    # base method
+
+    def read_all_files
+      @file_names_filtered_by_date.map do |file_name|       
+        
+        @file_name = file_name
+        @file      = request_object 
+        @db_table == 'users' ? filter_users_table_by_env : filter_file_by_env
+
+      end.flatten
+    end
+
+    # filter methods
+
+    def filter_file_by_env
+      has_env_id ? filter_by_env_id : filter_by_env_name
+    end
+
+    def filter_by_env_id
+      @file.select { |row| row[:environment_id] == @env_id }
+    end
+
+    def filter_by_env_name
+      @file.select { |row| row[:env] == @env_name }
+    end
+
+    def filter_users_table_by_env
+      @file.select { |row| row[:envs].include? @env_name }
+    end
+
+    def filter_by_date   
+      @file_names_filtered_by_table.select { |file_name| @date_range.include?(get_date(file_name)) }
+    end
+
+    def has_env_id
+      @file.first.has_key?(:env_id)
+    end
+
+    # date methods
+
+    def get_date_range
+      (Date.parse(@start_date)..Date.parse(@end_date)).to_a.map {|x| x.strftime("%Y-%m-%d") }
+    end
+
+    def get_date(file_name)
+      file_name.match(/\d{4}-\d{2}-\d{2}/).to_s
+    end
+
+    # AWS methods
+
+    def request_object_names
+      s3      = Aws::S3::Client.new
+      objects = s3.list_objects(bucket: 'eitje-backups', prefix: @db_table).contents
+      names   = objects.collect &:key
+    end
+
+    def request_object
+      s3     = Aws::S3::Client.new
+      object = s3.get_object(bucket: 'eitje-backups', key: @file_name)
+      json   = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
+    end
+
+  end
+end

data/app/services/s3/transform_deleted_files_service.rb

@@ -0,0 +1,85 @@
+require 'json'
+
+module S3::TransformDeletedFilesService
+  class << self
+
+    BUCKET = 'eitje-deleted-jurr-2'
+
+    def migrate_files(start_date: '2019-07-18')
+      @start_date = start_date
+
+      set_logger
+      set_bucket
+      set_tables
+      set_dates(start_date)
+
+      Environment.find_each do |env|
+        @env = env
+        
+        @tables.each do |table| 
+          @table = table          
+          compose_file
+        end
+      end
+    end
+
+    def set_logger
+      @logger = Logger.new "log/migrate_deleted_records_#{DateTime.now.strftime('%Y_%m_%d_%H:%M:%S')}.log"
+    end
+
+    def set_bucket
+      @s3 = Aws::S3::Client.new
+    end
+
+    def set_tables
+      # @tables = S3::OldDeletedRecordsService::singleton_class::DB_TABLES
+      @tables = ['verlof_verzoeken']
+    end
+
+    def set_dates(start_date)
+      format_start_date
+      @dates = {start_date: @start_date, end_date: Date.today.strftime("%Y-%m-%d")}
+    end
+
+    def format_start_date
+      @start_date = @start_date.class == String ? @start_date : @start_date.strftime("%Y-%m-%d")
+    end
+
+    def set_records
+      @records = S3::OldDeletedRecordsService.get_records(env_id: @env.id, env_name: @env.naam, db_table: 'verlofverzoeks', **@dates)
+    end
+
+    def set_json
+      @json = JSON.pretty_generate(@records)
+    end
+
+    def set_file_name
+      @file_name = "env_#{@env.id}_deleted_#{@table}.json"
+    end
+
+    def set_existing_records
+      object = @s3.get_object(bucket: BUCKET, key: @file_name)
+      @existing_records = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
+      
+      rescue Aws::S3::Errors::NoSuchKey => e
+        @existing_records = nil
+    end
+
+    def compose_file
+      set_records
+      set_file_name
+      set_existing_records
+
+      (@records += @existing_records) if @existing_records
+      set_json
+      upload_file  
+      rescue => e
+        @logger.error "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
+    end
+
+    def upload_file
+      @s3.put_object(bucket: BUCKET, key: @file_name, body: @json)
+    end
+
+  end
+end

data/config/routes.rb

@@ -0,0 +1,6 @@
+S3::Engine.routes.draw do
+
+  get '/deleted_records',        to: 'records#deleted_records'
+  get '/legacy_deleted_records', to: 'records#legacy_deleted_records'
+
+end

data/lib/s3.rb

@@ -0,0 +1,6 @@
+require "s3/version"
+require "s3/engine"
+
+module S3
+  # Your code goes here...
+end

data/lib/s3/engine.rb

@@ -0,0 +1,6 @@
+module S3
+  class Engine < ::Rails::Engine
+    isolate_namespace S3
+    config.generators.api_only = true
+  end
+end

data/lib/s3/version.rb

@@ -0,0 +1,3 @@
+module S3
+  VERSION = '1.0.0'
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: eitje_s3
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jurriaan Schrofer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-06-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.3
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: figaro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: S3
+email:
+- jschrofer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/controllers/s3/application_controller.rb
+- app/controllers/s3/records_controller.rb
+- app/services/s3/new_deleted_records_service.rb
+- app/services/s3/old_deleted_records_service.rb
+- app/services/s3/transform_deleted_files_service.rb
+- config/routes.rb
+- lib/s3.rb
+- lib/s3/engine.rb
+- lib/s3/version.rb
+homepage: https://github.com/eitje-app/s3_engine
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/eitje-app/s3_engine
+  source_code_uri: https://github.com/eitje-app/s3_engine
+  changelog_uri: https://github.com/eitje-app/s3_engine
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: S3
+test_files: []