eitje_s3 1.1.1 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71fa767da62f09f63fe6a2d3da4d3b05d88002dffb1c1163af0e4b6be9fabb82
-  data.tar.gz: 5e14301d327038d95a7439c53eb2e3d1393a9ff7d59f75dfaa87ac66895a0ddb
+  metadata.gz: 230fdba2a8fd9f8de892d021ab35594b8a84970ff0a7c518b16d01716a698df0
+  data.tar.gz: d70670f7249f28fc27dbd92b44670cd46cd9b768b836ea9321deb18743d2197e
 SHA512:
-  metadata.gz: fe8c94a7707711d185dfa43b1dc63647ec67f1573faea4739c86cb0cb3eaba6bed226f7c6ff4f661b1406e74ea4d192928eabb5c78fb803966355478d9995257
-  data.tar.gz: 0f7afb80b297bf439d0487e2847ac405fd711f0da7eb16db3de1ed85086986bf33305f1abd93f57c76cbe7ae220bc8fee5470e3c1bc213433d728181b06bedf1
+  metadata.gz: e697501276a0a4921e2fd48e5daa5e697b5bae4a1554414deb910dceb1e2866e9770fb8fc489036e9299e341251efea4bf1faea96b91e07abeb28b3564c073c7
+  data.tar.gz: 7c1d4880cd61c4fffbe6df19f47b89d79012d74ab48f974e9e46023436d40a8af99a34a25b9ee7fe332ed76a1a17e1b8dea98f90f79919e4ba7c771f323261cb

data/app/services/s3/new_deleted_records_service.rb

@@ -1,7 +1,7 @@
 module S3::NewDeletedRecordsService
   class << self
 
-   DB_TABLES = %w$ shifts teams users contracts infos posts topics $
+   DB_TABLES = %w$ shifts teams users contracts infos posts topics verlof_verzoeken$
 
     def get_records(db_table:, start_date:, end_date:, env_id:)    
       @date_range = Date.parse(start_date)..Date.parse(end_date)
@@ -32,8 +32,12 @@ module S3::NewDeletedRecordsService
       # the original returns waaaaay many records, so probably does not filter by date or
       # something. Change for now and investigate if shit goes BG.
 
-      file     = @s3.get_object(bucket: 'eitje-deleted-jurr-2', key: @file_name)
-      @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
+      begin
+        file = @s3.get_object(bucket: 'eitje-deleted-jurr-2', key: @file_name)
+        @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
+      rescue Aws::S3::Errors::NoSuchKey
+        @records = [] # files are only generated when an env has some deleted envs, through by uploading them old-style first
+      end
     end
 
     def filter_records
@@ -46,4 +50,15 @@ module S3::NewDeletedRecordsService
     end
 
   end
-end
+end
+
+__END__
+
+def show_env(env_id, db_table = "verlof_verzoeken")
+  start_date = (Date.today - 1.years).to_s
+  end_date = (Date.today).to_s
+  records = S3::NewDeletedRecordsService.get_records(db_table: db_table, start_date: start_date, end_date: end_date, env_id: env_id)
+end
+
+show_env(1176)
+show_env(1176, "verlof_verzoeken")

data/app/services/s3/old_deleted_records_service.rb

@@ -1,7 +1,7 @@
 module S3::OldDeletedRecordsService
   class << self
 
-   DB_TABLES = %w$ shifts teams users contracts infos posts topics $
+   DB_TABLES = %w$ shifts teams users contracts infos posts topics verlof_verzoeken$
 
     def get_records(db_table:, start_date:, end_date:, env_id:, env_name:)
 
@@ -34,6 +34,8 @@ module S3::OldDeletedRecordsService
         @file_name = file_name
         @file      = request_object 
 
+        next unless @file
+
         case @db_table
         when 'users'
           filter_users_table_by_env
@@ -43,7 +45,7 @@ module S3::OldDeletedRecordsService
           filter_file_by_env
         end
 
-      end.flatten
+      end.flatten.compact
     end
 
     # filter methods
@@ -89,16 +91,29 @@ module S3::OldDeletedRecordsService
     # AWS methods
 
     def request_object_names
+      folder = @db_table
+      folder = "verlofverzoeks" if @db_table == "verlof_verzoeken"
+
       s3      = Aws::S3::Client.new
-      objects = s3.list_objects(bucket: 'eitje-backups', prefix: @db_table).contents
-      names   = objects.collect &:key
+
+      objects = []
+
+      # Pagination: AWS returns 1000 items at max
+      s3.list_objects(bucket: 'eitje-backups', prefix: folder).each do |response|
+        objects += response.contents
+      end
+
+      names = objects.collect &:key
     end
 
     def request_object
       s3     = Aws::S3::Client.new
       object = s3.get_object(bucket: 'eitje-backups', key: @file_name)
-      json   = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
+      body   = object.body.read.as_json
+      json   = JSON.parse(body).map(&:symbolize_keys)
+      rescue JSON::ParserError => e
+        puts "Invalid file... skip this file"
     end
 
   end
-end
+end

data/app/services/s3/transform_deleted_files_service.rb

@@ -20,20 +20,32 @@ module S3::TransformDeletedFilesService
       envs_to_migrate = []
 
       set_tables.each do |table|  
-        object  = s3.get_object(bucket: 'eitje-backups', key: "#{table}/#{start_date.strftime("%Y-%m-%d")}.json")
+
+        folder = table
+        folder = "verlofverzoeks" if table == "verlof_verzoeken"
+
+        puts "requesting #{table} on #{start_date.strftime("%Y-%m-%d")}..."
+
+        object  = s3.get_object(bucket: 'eitje-backups', key: "#{folder}/#{start_date.strftime("%Y-%m-%d")}.json")
         json    = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
         
         if table == 'topics'
           env_ids = json.map {|row| row[:environment_ids]}.flatten.compact.uniq
         else
-          env_ids = json.map {|row| row[:env]}.uniq.map { |name| Environment.find_by(naam: name)&.id }
+          puts "new table logic...?"
+          env_ids = json.map { _1[:environment_id] }.compact.uniq
+
+          unless env_ids.present?
+            env_ids = json.map {|row| row[:env]}.uniq.map { |name| Environment.find_by(naam: name)&.id }
+          end
         end
 
         envs_to_migrate << env_ids
-      rescue => e
+      rescue Aws::S3::Errors::NoSuchKey => e
         # in case the file does not exist on S3, cause there are no deleted 
         # records, skip to next table
-        next
+        puts "no records found for #{table} on #{start_date.strftime("%Y-%m-%d")}!"
+        []
       end
 
       envs_to_migrate = envs_to_migrate.flatten.uniq.compact
@@ -48,6 +60,11 @@ module S3::TransformDeletedFilesService
         @table = table          
         compose_file
       end
+
+    rescue ActiveRecord::RecordNotFound => e
+
+    rescue => e
+      binding.pry
     end
 
     def migrate_files_multi_env(environment_ids, start_date: Date.yesterday)
@@ -98,6 +115,7 @@ module S3::TransformDeletedFilesService
       @existing_records = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
       
       rescue Aws::S3::Errors::NoSuchKey => e
+        binding.pry
         @existing_records = nil
     end
 
@@ -106,12 +124,16 @@ module S3::TransformDeletedFilesService
       set_file_name
       set_existing_records
 
-      (@records += @existing_records) if @existing_records
+      @records = (@records | @existing_records) if @existing_records
       set_json
+      puts "completed successfully! new total: #{@records&.count}, old total: #{@existing_records&.count}"
       upload_file
 
       rescue => e
-        @logger.error "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
+        message = "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
+        puts message
+        binding.pry
+        @logger.error message
     end
 
     def upload_file
@@ -119,4 +141,4 @@ module S3::TransformDeletedFilesService
     end
 
   end
-end
+end

data/lib/s3/version.rb

@@ -1,3 +1,3 @@
 module S3
-  VERSION = '1.1.1'
+  VERSION = '1.1.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eitje_s3
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.4
 platform: ruby
 authors:
 - Jurriaan Schrofer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-12 00:00:00.000000000 Z
+date: 2022-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: S3