eitje_s3 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/services/s3/transform_deleted_files_service.rb +40 -12
- data/lib/s3/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 780e597f34264cf258ebe42c734869d0e5c7603184dd12cd6279eea60c9cb9a5
|
|
4
|
+
data.tar.gz: 35d61bacafe58fb85b3e0af7c24828d11c999fc46550c43f78a2d1700b7438b2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 725400574bf70d36594ce8b5f86f62dbaced343f0d2da6a0a74ca124e9be3a785c997b9e2f2ed53084203088d0131ca12ce06c05116b9c07bbc2faa32a5eb511
|
|
7
|
+
data.tar.gz: 2fffbdb653da51accf6c157cc71ecbf246209f78e292d0caad78689b3cab7e081b9dd9d5eb1f0cef85ba84a25780f482e96bdddab79c0a161cb2339a971aebef
|
|
@@ -5,24 +5,53 @@ module S3::TransformDeletedFilesService
|
|
|
5
5
|
|
|
6
6
|
BUCKET = 'eitje-deleted-jurr-2'
|
|
7
7
|
|
|
8
|
-
def
|
|
8
|
+
def set_setters(start_date = Date.yesterday)
|
|
9
9
|
@start_date = start_date
|
|
10
|
-
|
|
11
10
|
set_logger
|
|
12
11
|
set_bucket
|
|
13
12
|
set_tables
|
|
14
13
|
set_dates(start_date)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def migrate_files(start_date: Date.today)
|
|
17
|
+
set_setters(start_date)
|
|
18
|
+
|
|
19
|
+
s3 = Aws::S3::Client.new
|
|
20
|
+
envs_to_migrate = []
|
|
21
|
+
|
|
22
|
+
set_tables.each do |table|
|
|
23
|
+
object = s3.get_object(bucket: 'eitje-backups', key: "#{table}/#{start_date.strftime("%Y-%m-%d")}.json")
|
|
24
|
+
json = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
|
|
25
|
+
env_ids = json.map {|row| row[:env]}.uniq.map { |name| Environment.find_by(naam: name)&.id }
|
|
26
|
+
envs_to_migrate << env_ids
|
|
27
|
+
rescue
|
|
28
|
+
# in case the file does not exist on S3, cause there are no deleted
|
|
29
|
+
# records, skip to next table
|
|
30
|
+
next
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
envs_to_migrate.flatten.uniq.each { |env_id| migrate_files_single_env(env_id, start_date: start_date, skip_setters: true) }
|
|
34
|
+
end
|
|
15
35
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
end
|
|
36
|
+
def migrate_files_single_env(environment_id, start_date: Date.yesterday, skip_setters: false)
|
|
37
|
+
set_setters(start_date) unless skip_setters
|
|
38
|
+
@env = Environment.find(environment_id)
|
|
39
|
+
@tables.each do |table|
|
|
40
|
+
@table = table
|
|
41
|
+
compose_file
|
|
23
42
|
end
|
|
24
43
|
end
|
|
25
44
|
|
|
45
|
+
def migrate_files_multi_env(environment_ids, start_date: Date.yesterday)
|
|
46
|
+
set_setters(start_date)
|
|
47
|
+
environment_ids.each { |id| migrate_files_single_env(id, start_date: start_date, skip_setters: true) }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def migrate_files_single_org(organisation_id, start_date: Date.yesterday)
|
|
51
|
+
env_ids = Organisation.find(organisation_id).environment_ids
|
|
52
|
+
migrate_files_multi_env(env_ids, start_date: start_date)
|
|
53
|
+
end
|
|
54
|
+
|
|
26
55
|
def set_logger
|
|
27
56
|
@logger = Logger.new "log/migrate_deleted_records_#{DateTime.now.strftime('%Y_%m_%d_%H:%M:%S')}.log"
|
|
28
57
|
end
|
|
@@ -32,8 +61,7 @@ module S3::TransformDeletedFilesService
|
|
|
32
61
|
end
|
|
33
62
|
|
|
34
63
|
def set_tables
|
|
35
|
-
|
|
36
|
-
@tables = ['verlof_verzoeken']
|
|
64
|
+
@tables = S3::OldDeletedRecordsService::singleton_class::DB_TABLES
|
|
37
65
|
end
|
|
38
66
|
|
|
39
67
|
def set_dates(start_date)
|
|
@@ -46,7 +74,7 @@ module S3::TransformDeletedFilesService
|
|
|
46
74
|
end
|
|
47
75
|
|
|
48
76
|
def set_records
|
|
49
|
-
@records = S3::OldDeletedRecordsService.get_records(env_id: @env.id, env_name: @env.naam, db_table:
|
|
77
|
+
@records = S3::OldDeletedRecordsService.get_records(env_id: @env.id, env_name: @env.naam, db_table: @table, **@dates)
|
|
50
78
|
end
|
|
51
79
|
|
|
52
80
|
def set_json
|
data/lib/s3/version.rb
CHANGED