eitje_s3 1.0.0 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3fab11f169aa044513495cd47f13853ec305a31fe72e5075b681f042d9712c8
-  data.tar.gz: 464f079e85d4e7df5373d70732ba93852ff9489b3d543404eee769d6d5e86dec
+  metadata.gz: 2807a3b8cb433040b1e0e32add53c9ad8cd6bdb01912067fb8466b352500e3f5
+  data.tar.gz: 6cf07414f5ad6db0e99f95b06917d87d109a3fecb8e0c2b2b032b420ddd2f022
 SHA512:
-  metadata.gz: a8c6436222d6708cbe69ae3d4d38e97c6a015db22e870639520ea05ee36ad721fadb7cae6eac3daeb53a5649035ead3562d654865f785c2f2e8e77468b2647cf
-  data.tar.gz: 47f1b3af7e66c4740d8a4fcba65d824096e4287a0b22d122d2e132fbc37b660e573ac18ff3c38b72e1a0539dbf8540c792fcab032f9e1ebe77efc0ad0c399319
+  metadata.gz: 76182141a7f669de16ed8f9de9740c1714f10cb0d8a7deb0822018b81921a0d5e448cb5e95ef1671264ab5ec47232f3c111fe71237e5d56ef403713c51240f72
+  data.tar.gz: c5c9450d3d14c3e5ada2a1a3c940c6876f179bbc1525f50bbb669b9bd3245e645eb41645a94c0d64ed2d38c00ac9e4259d78981100f8275a3c1fdd0b06a1c80f

data/app/services/s3/new_deleted_records_service.rb

@@ -1,13 +1,7 @@
 module S3::NewDeletedRecordsService
   class << self
 
-   DB_TABLES = %w$ shifts teams users contracts infos posts $
-
-    def test(table)
-      S3::NewDeletedRecordsService.get_records(
-        db_table: table, start_date: '2021-03-28', end_date: '2021-04-3', env_id: 513
-      )
-    end
+   DB_TABLES = %w$ shifts teams users contracts infos posts topics $
 
     def get_records(db_table:, start_date:, end_date:, env_id:)    
       @date_range = Date.parse(start_date)..Date.parse(end_date)
@@ -32,7 +26,13 @@ module S3::NewDeletedRecordsService
     end
 
     def query_records    
-      file     = @s3.get_object(bucket: 'eitje-deleted-jurr', key: @file_name)
+
+      # Previously (before adding 'topics') the request bucket was 'eitje-deleted-jurr'
+      # but somehow topics break if we dont request the bucket '-2'. Now for other tables
+      # the original returns waaaaay many records, so probably does not filter by date or
+      # something. Change for now and investigate if shit goes BG.
+
+      file     = @s3.get_object(bucket: 'eitje-deleted-jurr-2', key: @file_name)
       @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
     end
 

data/app/services/s3/old_deleted_records_service.rb

@@ -1,9 +1,7 @@
 module S3::OldDeletedRecordsService
   class << self
 
-   DB_TABLES = %w$ shifts teams users contracts infos posts $
-
-   # S3::OldDeletedRecordsService.get_records(env_id: 307, env_name: 'Kua - Den Haag', db_table: 'verlofverzoeks', start_date: '2021-02-01', end_date: '2021-04-01')
+   DB_TABLES = %w$ shifts teams users contracts infos posts topics $
 
     def get_records(db_table:, start_date:, end_date:, env_id:, env_name:)
 
@@ -20,7 +18,6 @@ module S3::OldDeletedRecordsService
       @file_names_filtered_by_date  = filter_by_date
 
       read_all_files
-
     end
 
     # validations
@@ -36,7 +33,15 @@ module S3::OldDeletedRecordsService
         
         @file_name = file_name
         @file      = request_object 
-        @db_table == 'users' ? filter_users_table_by_env : filter_file_by_env
+
+        case @db_table
+        when 'users'
+          filter_users_table_by_env
+        when 'topics'
+          filter_topics_table_by_env
+        else
+          filter_file_by_env
+        end
 
       end.flatten
     end
@@ -59,6 +64,10 @@ module S3::OldDeletedRecordsService
       @file.select { |row| row[:envs].include? @env_name }
     end
 
+    def filter_topics_table_by_env
+      @file.select { |row| row[:environment_ids]&.include?(@env_id) }
+    end
+
     def filter_by_date   
       @file_names_filtered_by_table.select { |file_name| @date_range.include?(get_date(file_name)) }
     end

data/app/services/s3/transform_deleted_files_service.rb

@@ -5,24 +5,61 @@ module S3::TransformDeletedFilesService
 
     BUCKET = 'eitje-deleted-jurr-2'
 
-    def migrate_files(start_date: '2019-07-18')
+    def set_setters(start_date = Date.yesterday)
       @start_date = start_date
-
       set_logger
       set_bucket
       set_tables
       set_dates(start_date)
+    end
+
+    def migrate_files(start_date: Date.today)
+      set_setters(start_date)
+
+      s3 = Aws::S3::Client.new
+      envs_to_migrate = []
 
-      Environment.find_each do |env|
-        @env = env
+      set_tables.each do |table|  
+        object  = s3.get_object(bucket: 'eitje-backups', key: "#{table}/#{start_date.strftime("%Y-%m-%d")}.json")
+        json    = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
         
-        @tables.each do |table| 
-          @table = table          
-          compose_file
+        if table == 'topics'
+          env_ids = json.map {|row| row[:environment_ids]}.flatten.compact.uniq
+        else
+          env_ids = json.map {|row| row[:env]}.uniq.map { |name| Environment.find_by(naam: name)&.id }
         end
+
+        envs_to_migrate << env_ids
+      rescue => e
+        # in case the file does not exist on S3, cause there are no deleted 
+        # records, skip to next table
+        next
+      end
+
+      envs_to_migrate = envs_to_migrate.flatten.uniq.compact
+      
+      envs_to_migrate.each { |env_id| migrate_files_single_env(env_id, start_date: start_date, skip_setters: true) }
+    end
+
+    def migrate_files_single_env(environment_id, start_date: Date.yesterday, skip_setters: false)
+      set_setters(start_date) unless skip_setters
+      @env = Environment.find(environment_id)
+      @tables.each do |table| 
+        @table = table          
+        compose_file
       end
     end
 
+    def migrate_files_multi_env(environment_ids, start_date: Date.yesterday)
+      set_setters(start_date)
+      environment_ids.each { |id| migrate_files_single_env(id, start_date: start_date, skip_setters: true) }
+    end
+
+    def migrate_files_single_org(organisation_id, start_date: Date.yesterday)
+      env_ids = Organisation.find(organisation_id).environment_ids
+      migrate_files_multi_env(env_ids, start_date: start_date)
+    end
+
     def set_logger
       @logger = Logger.new "log/migrate_deleted_records_#{DateTime.now.strftime('%Y_%m_%d_%H:%M:%S')}.log"
     end
@@ -32,8 +69,7 @@ module S3::TransformDeletedFilesService
     end
 
     def set_tables
-      # @tables = S3::OldDeletedRecordsService::singleton_class::DB_TABLES
-      @tables = ['verlof_verzoeken']
+      @tables = S3::OldDeletedRecordsService::singleton_class::DB_TABLES
     end
 
     def set_dates(start_date)
@@ -46,7 +82,7 @@ module S3::TransformDeletedFilesService
     end
 
     def set_records
-      @records = S3::OldDeletedRecordsService.get_records(env_id: @env.id, env_name: @env.naam, db_table: 'verlofverzoeks', **@dates)
+      @records = S3::OldDeletedRecordsService.get_records(env_id: @env.id, env_name: @env.naam, db_table: @table, **@dates)
     end
 
     def set_json
@@ -72,7 +108,8 @@ module S3::TransformDeletedFilesService
 
       (@records += @existing_records) if @existing_records
       set_json
-      upload_file  
+      upload_file
+
       rescue => e
         @logger.error "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
     end

data/lib/s3/version.rb

@@ -1,3 +1,3 @@
 module S3
-  VERSION = '1.0.0'
+  VERSION = '1.0.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eitje_s3
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.4
 platform: ruby
 authors:
 - Jurriaan Schrofer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-28 00:00:00.000000000 Z
+date: 2021-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails