eitje_s3 1.0.3 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '06915019d41e0d0790ff75cc9478cb3a94aa9c2a3d203861326e4408e6e883bc'
-  data.tar.gz: 423a60af37a589c1aa57c088a241818bea588938dd935a8133625c6ebc84a038
+  metadata.gz: 257974c1578ffd4a62940c666f61266b93373edc2f2ccf4a54741fd88a8b5c94
+  data.tar.gz: c57ffdb1cbea2c08ceb676c1a8e7973b0c08ea36c1fc798a31a8db083459bc1a
 SHA512:
-  metadata.gz: 50a91cdacc1e1b7d33c74920213930a13b3c61adb75e2567d047462336738d40aab549e48c04f4ab068efa13061f64cf7f3aac54b35ca48a8a0832edd6a6e7c5
-  data.tar.gz: e09bb1c05cf7b35c14cb678cdd897494cae2ed1a3a003dfb4654b19fe316ea266bddf7e6bd89107c16c9523f8ddc1f6a52e082d65750a829b3af155299601272
+  metadata.gz: 9454e1def04a8507f383c2eca3ebf2bd72a2c92a7db7821d2ef38bfb4b199b1f17cbd5d40c940bfebb8603379dd89f2c816abcb9aab855d19fe19507116ca1e7
+  data.tar.gz: daef1d17dac7b5b7b4d4b2bb32ca8b67c0ca29a85e5ea2dc6f95cdb454275408f38c07e21f038de24f10c579de885490ae1aa1a8e5479a3a24944233dcac3bd3

data/app/services/s3/new_deleted_records_service.rb

@@ -1,13 +1,7 @@
 module S3::NewDeletedRecordsService
   class << self
 
-   DB_TABLES = %w$ shifts teams users contracts infos posts $
-
-    def test(table)
-      S3::NewDeletedRecordsService.get_records(
-        db_table: table, start_date: '2021-03-28', end_date: '2021-04-3', env_id: 513
-      )
-    end
+   DB_TABLES = %w$ shifts teams users contracts infos posts topics verlof_verzoeken$
 
     def get_records(db_table:, start_date:, end_date:, env_id:)    
       @date_range = Date.parse(start_date)..Date.parse(end_date)
@@ -32,7 +26,13 @@ module S3::NewDeletedRecordsService
     end
 
     def query_records    
-      file     = @s3.get_object(bucket: 'eitje-deleted-jurr', key: @file_name)
+
+      # Previously (before adding 'topics') the request bucket was 'eitje-deleted-jurr'
+      # but somehow topics break if we dont request the bucket '-2'. Now for other tables
+      # the original returns waaaaay many records, so probably does not filter by date or
+      # something. Change for now and investigate if shit goes BG.
+
+      file     = @s3.get_object(bucket: 'eitje-deleted-jurr-2', key: @file_name)
       @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
     end
 
@@ -46,4 +46,13 @@ module S3::NewDeletedRecordsService
     end
 
   end
-end
+end
+
+__END__
+
+env_id = 22
+db_table = "verlof_verzoeken"
+start_date = (Date.today - 5.days).to_s
+end_date = (Date.today).to_s
+
+records = S3::NewDeletedRecordsService.get_records(db_table: db_table, start_date: start_date, end_date: end_date, env_id: env_id)

data/app/services/s3/old_deleted_records_service.rb

@@ -3,8 +3,6 @@ module S3::OldDeletedRecordsService
 
    DB_TABLES = %w$ shifts teams users contracts infos posts topics $
 
-   # S3::OldDeletedRecordsService.get_records(env_id: 307, env_name: 'Kua - Den Haag', db_table: 'verlofverzoeks', start_date: '2021-02-01', end_date: '2021-04-01')
-
     def get_records(db_table:, start_date:, end_date:, env_id:, env_name:)
 
       # validate_args(db_table)
@@ -20,7 +18,6 @@ module S3::OldDeletedRecordsService
       @file_names_filtered_by_date  = filter_by_date
 
       read_all_files
-
     end
 
     # validations
@@ -36,7 +33,15 @@ module S3::OldDeletedRecordsService
         
         @file_name = file_name
         @file      = request_object 
-        @db_table == 'users' ? filter_users_table_by_env : filter_file_by_env
+
+        case @db_table
+        when 'users'
+          filter_users_table_by_env
+        when 'topics'
+          filter_topics_table_by_env
+        else
+          filter_file_by_env
+        end
 
       end.flatten
     end
@@ -59,6 +64,10 @@ module S3::OldDeletedRecordsService
       @file.select { |row| row[:envs].include? @env_name }
     end
 
+    def filter_topics_table_by_env
+      @file.select { |row| row[:environment_ids]&.include?(@env_id) }
+    end
+
     def filter_by_date   
       @file_names_filtered_by_table.select { |file_name| @date_range.include?(get_date(file_name)) }
     end

data/app/services/s3/transform_deleted_files_service.rb

@@ -22,15 +22,23 @@ module S3::TransformDeletedFilesService
       set_tables.each do |table|  
         object  = s3.get_object(bucket: 'eitje-backups', key: "#{table}/#{start_date.strftime("%Y-%m-%d")}.json")
         json    = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
-        env_ids = json.map {|row| row[:env]}.uniq.map { |name| Environment.find_by(naam: name)&.id }
+        
+        if table == 'topics'
+          env_ids = json.map {|row| row[:environment_ids]}.flatten.compact.uniq
+        else
+          env_ids = json.map {|row| row[:env]}.uniq.map { |name| Environment.find_by(naam: name)&.id }
+        end
+
         envs_to_migrate << env_ids
-      rescue 
+      rescue => e
         # in case the file does not exist on S3, cause there are no deleted 
         # records, skip to next table
         next
       end
+
+      envs_to_migrate = envs_to_migrate.flatten.uniq.compact
       
-      envs_to_migrate.flatten.uniq.each { |env_id| migrate_files_single_env(env_id, start_date: start_date, skip_setters: true) }
+      envs_to_migrate.each { |env_id| migrate_files_single_env(env_id, start_date: start_date, skip_setters: true) }
     end
 
     def migrate_files_single_env(environment_id, start_date: Date.yesterday, skip_setters: false)
@@ -100,7 +108,8 @@ module S3::TransformDeletedFilesService
 
       (@records += @existing_records) if @existing_records
       set_json
-      upload_file  
+      upload_file
+
       rescue => e
         @logger.error "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
     end

data/lib/s3/version.rb

@@ -1,3 +1,3 @@
 module S3
-  VERSION = '1.0.3'
+  VERSION = '1.1.2'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: eitje_s3
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.2
 platform: ruby
 authors:
 - Jurriaan Schrofer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-22 00:00:00.000000000 Z
+date: 2021-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.3
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.3
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: S3