eitje_s3 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 257974c1578ffd4a62940c666f61266b93373edc2f2ccf4a54741fd88a8b5c94
-  data.tar.gz: c57ffdb1cbea2c08ceb676c1a8e7973b0c08ea36c1fc798a31a8db083459bc1a
+  metadata.gz: 819fd4119ba036edf00465389ac4fda4be8bc4003d41997a6a5824d19964fcf8
+  data.tar.gz: bdee3c536e21ded2851636d15cab0f54a6ff9ef270d0aa6b465db856a4c40ed2
 SHA512:
-  metadata.gz: 9454e1def04a8507f383c2eca3ebf2bd72a2c92a7db7821d2ef38bfb4b199b1f17cbd5d40c940bfebb8603379dd89f2c816abcb9aab855d19fe19507116ca1e7
-  data.tar.gz: daef1d17dac7b5b7b4d4b2bb32ca8b67c0ca29a85e5ea2dc6f95cdb454275408f38c07e21f038de24f10c579de885490ae1aa1a8e5479a3a24944233dcac3bd3
+  metadata.gz: c51ee452a93e0715ad4c6b515cfa35e282952fb72056bd9ba7a8d7413a9e8ec4220067ed0dc80bcbbb7bd96b03583f6ec022d972e427b9bf319635455cedba48
+  data.tar.gz: 6b6b6e69c2959d319f92bf4fd5541b93efd09fb48b441551c433a0485a1ae72d1fe78d02021021fe4b78c52f216b81fc99bea421158ee48e8868d8c8e7e6b8b1

data/app/services/s3/new_deleted_records_service.rb

@@ -32,8 +32,12 @@ module S3::NewDeletedRecordsService
       # the original returns waaaaay many records, so probably does not filter by date or
       # something. Change for now and investigate if shit goes BG.
 
-      file     = @s3.get_object(bucket: 'eitje-deleted-jurr-2', key: @file_name)
-      @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
+      begin
+        file = @s3.get_object(bucket: 'eitje-deleted-jurr-2', key: @file_name)
+        @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
+      rescue Aws::S3::Errors::NoSuchKey
+        @records = [] # files are only generated when an env has some deleted envs, through by uploading them old-style first
+      end
     end
 
     def filter_records
@@ -50,9 +54,11 @@ end
 
 __END__
 
-env_id = 22
-db_table = "verlof_verzoeken"
-start_date = (Date.today - 5.days).to_s
-end_date = (Date.today).to_s
+def show_env(env_id, db_table = "verlof_verzoeken")
+  start_date = (Date.today - 1.years).to_s
+  end_date = (Date.today).to_s
+  records = S3::NewDeletedRecordsService.get_records(db_table: db_table, start_date: start_date, end_date: end_date, env_id: env_id)
+end
 
-records = S3::NewDeletedRecordsService.get_records(db_table: db_table, start_date: start_date, end_date: end_date, env_id: env_id)
+show_env(1176)
+show_env(1176, "verlof_verzoeken")

data/app/services/s3/old_deleted_records_service.rb

@@ -1,7 +1,7 @@
 module S3::OldDeletedRecordsService
   class << self
 
-   DB_TABLES = %w$ shifts teams users contracts infos posts topics $
+   DB_TABLES = %w$ shifts teams users contracts infos posts topics verlof_verzoeken$
 
     def get_records(db_table:, start_date:, end_date:, env_id:, env_name:)
 
@@ -89,8 +89,11 @@ module S3::OldDeletedRecordsService
     # AWS methods
 
     def request_object_names
+      folder = @db_table
+      folder = "verlofverzoeks" if @db_table == "verlof_verzoeken"
+
       s3      = Aws::S3::Client.new
-      objects = s3.list_objects(bucket: 'eitje-backups', prefix: @db_table).contents
+      objects = s3.list_objects(bucket: 'eitje-backups', prefix: folder).contents
       names   = objects.collect &:key
     end
 
@@ -101,4 +104,4 @@ module S3::OldDeletedRecordsService
     end
 
   end
-end
+end

data/app/services/s3/transform_deleted_files_service.rb

@@ -20,7 +20,13 @@ module S3::TransformDeletedFilesService
       envs_to_migrate = []
 
       set_tables.each do |table|  
-        object  = s3.get_object(bucket: 'eitje-backups', key: "#{table}/#{start_date.strftime("%Y-%m-%d")}.json")
+
+        folder = table
+        folder = "verlofverzoeks" if table == "verlof_verzoeken"
+
+        puts "requesting #{table} on #{start_date.strftime("%Y-%m-%d")}..."
+
+        object  = s3.get_object(bucket: 'eitje-backups', key: "#{folder}/#{start_date.strftime("%Y-%m-%d")}.json")
         json    = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
         
         if table == 'topics'
@@ -30,10 +36,11 @@ module S3::TransformDeletedFilesService
         end
 
         envs_to_migrate << env_ids
-      rescue => e
+      rescue Aws::S3::Errors::NoSuchKey => e
         # in case the file does not exist on S3, cause there are no deleted 
         # records, skip to next table
-        next
+        puts "no records found for #{table} on #{start_date.strftime("%Y-%m-%d")}!"
+        []
       end
 
       envs_to_migrate = envs_to_migrate.flatten.uniq.compact
@@ -106,12 +113,14 @@ module S3::TransformDeletedFilesService
       set_file_name
       set_existing_records
 
-      (@records += @existing_records) if @existing_records
+      @records = (@records | @existing_records) if @existing_records
       set_json
       upload_file
 
       rescue => e
-        @logger.error "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
+        message = "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
+        puts message
+        @logger.error message
     end
 
     def upload_file
@@ -119,4 +128,4 @@ module S3::TransformDeletedFilesService
     end
 
   end
-end
+end

data/lib/s3/version.rb

@@ -1,3 +1,3 @@
 module S3
-  VERSION = '1.1.2'
+  VERSION = '1.1.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eitje_s3
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Jurriaan Schrofer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-12 00:00:00.000000000 Z
+date: 2022-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: S3