eitje_s3 1.1.2 → 1.1.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 257974c1578ffd4a62940c666f61266b93373edc2f2ccf4a54741fd88a8b5c94
4
- data.tar.gz: c57ffdb1cbea2c08ceb676c1a8e7973b0c08ea36c1fc798a31a8db083459bc1a
3
+ metadata.gz: 819fd4119ba036edf00465389ac4fda4be8bc4003d41997a6a5824d19964fcf8
4
+ data.tar.gz: bdee3c536e21ded2851636d15cab0f54a6ff9ef270d0aa6b465db856a4c40ed2
5
5
  SHA512:
6
- metadata.gz: 9454e1def04a8507f383c2eca3ebf2bd72a2c92a7db7821d2ef38bfb4b199b1f17cbd5d40c940bfebb8603379dd89f2c816abcb9aab855d19fe19507116ca1e7
7
- data.tar.gz: daef1d17dac7b5b7b4d4b2bb32ca8b67c0ca29a85e5ea2dc6f95cdb454275408f38c07e21f038de24f10c579de885490ae1aa1a8e5479a3a24944233dcac3bd3
6
+ metadata.gz: c51ee452a93e0715ad4c6b515cfa35e282952fb72056bd9ba7a8d7413a9e8ec4220067ed0dc80bcbbb7bd96b03583f6ec022d972e427b9bf319635455cedba48
7
+ data.tar.gz: 6b6b6e69c2959d319f92bf4fd5541b93efd09fb48b441551c433a0485a1ae72d1fe78d02021021fe4b78c52f216b81fc99bea421158ee48e8868d8c8e7e6b8b1
@@ -32,8 +32,12 @@ module S3::NewDeletedRecordsService
32
32
  # the original returns waaaaay many records, so probably does not filter by date or
33
33
  # something. Change for now and investigate if shit goes BG.
34
34
 
35
- file = @s3.get_object(bucket: 'eitje-deleted-jurr-2', key: @file_name)
36
- @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
35
+ begin
36
+ file = @s3.get_object(bucket: 'eitje-deleted-jurr-2', key: @file_name)
37
+ @records = JSON.parse(file.body.read.as_json).map(&:symbolize_keys)
38
+ rescue Aws::S3::Errors::NoSuchKey
39
+ @records = [] # files are only generated when an env has some deleted envs, through by uploading them old-style first
40
+ end
37
41
  end
38
42
 
39
43
  def filter_records
@@ -50,9 +54,11 @@ end
50
54
 
51
55
  __END__
52
56
 
53
- env_id = 22
54
- db_table = "verlof_verzoeken"
55
- start_date = (Date.today - 5.days).to_s
56
- end_date = (Date.today).to_s
57
+ def show_env(env_id, db_table = "verlof_verzoeken")
58
+ start_date = (Date.today - 1.years).to_s
59
+ end_date = (Date.today).to_s
60
+ records = S3::NewDeletedRecordsService.get_records(db_table: db_table, start_date: start_date, end_date: end_date, env_id: env_id)
61
+ end
57
62
 
58
- records = S3::NewDeletedRecordsService.get_records(db_table: db_table, start_date: start_date, end_date: end_date, env_id: env_id)
63
+ show_env(1176)
64
+ show_env(1176, "verlof_verzoeken")
@@ -1,7 +1,7 @@
1
1
  module S3::OldDeletedRecordsService
2
2
  class << self
3
3
 
4
- DB_TABLES = %w$ shifts teams users contracts infos posts topics $
4
+ DB_TABLES = %w$ shifts teams users contracts infos posts topics verlof_verzoeken$
5
5
 
6
6
  def get_records(db_table:, start_date:, end_date:, env_id:, env_name:)
7
7
 
@@ -89,8 +89,11 @@ module S3::OldDeletedRecordsService
89
89
  # AWS methods
90
90
 
91
91
  def request_object_names
92
+ folder = @db_table
93
+ folder = "verlofverzoeks" if @db_table == "verlof_verzoeken"
94
+
92
95
  s3 = Aws::S3::Client.new
93
- objects = s3.list_objects(bucket: 'eitje-backups', prefix: @db_table).contents
96
+ objects = s3.list_objects(bucket: 'eitje-backups', prefix: folder).contents
94
97
  names = objects.collect &:key
95
98
  end
96
99
 
@@ -101,4 +104,4 @@ module S3::OldDeletedRecordsService
101
104
  end
102
105
 
103
106
  end
104
- end
107
+ end
@@ -20,7 +20,13 @@ module S3::TransformDeletedFilesService
20
20
  envs_to_migrate = []
21
21
 
22
22
  set_tables.each do |table|
23
- object = s3.get_object(bucket: 'eitje-backups', key: "#{table}/#{start_date.strftime("%Y-%m-%d")}.json")
23
+
24
+ folder = table
25
+ folder = "verlofverzoeks" if table == "verlof_verzoeken"
26
+
27
+ puts "requesting #{table} on #{start_date.strftime("%Y-%m-%d")}..."
28
+
29
+ object = s3.get_object(bucket: 'eitje-backups', key: "#{folder}/#{start_date.strftime("%Y-%m-%d")}.json")
24
30
  json = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
25
31
 
26
32
  if table == 'topics'
@@ -30,10 +36,11 @@ module S3::TransformDeletedFilesService
30
36
  end
31
37
 
32
38
  envs_to_migrate << env_ids
33
- rescue => e
39
+ rescue Aws::S3::Errors::NoSuchKey => e
34
40
  # in case the file does not exist on S3, cause there are no deleted
35
41
  # records, skip to next table
36
- next
42
+ puts "no records found for #{table} on #{start_date.strftime("%Y-%m-%d")}!"
43
+ []
37
44
  end
38
45
 
39
46
  envs_to_migrate = envs_to_migrate.flatten.uniq.compact
@@ -106,12 +113,14 @@ module S3::TransformDeletedFilesService
106
113
  set_file_name
107
114
  set_existing_records
108
115
 
109
- (@records += @existing_records) if @existing_records
116
+ @records = (@records | @existing_records) if @existing_records
110
117
  set_json
111
118
  upload_file
112
119
 
113
120
  rescue => e
114
- @logger.error "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
121
+ message = "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
122
+ puts message
123
+ @logger.error message
115
124
  end
116
125
 
117
126
  def upload_file
@@ -119,4 +128,4 @@ module S3::TransformDeletedFilesService
119
128
  end
120
129
 
121
130
  end
122
- end
131
+ end
data/lib/s3/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module S3
2
- VERSION = '1.1.2'
2
+ VERSION = '1.1.3'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: eitje_s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jurriaan Schrofer
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-11-12 00:00:00.000000000 Z
11
+ date: 2022-07-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rails
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
93
93
  - !ruby/object:Gem::Version
94
94
  version: '0'
95
95
  requirements: []
96
- rubygems_version: 3.0.9
96
+ rubygems_version: 3.2.22
97
97
  signing_key:
98
98
  specification_version: 4
99
99
  summary: S3