gi_job 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +6 -0
  5. data/CODE_OF_CONDUCT.md +74 -0
  6. data/Gemfile +15 -0
  7. data/Gemfile.lock +34 -0
  8. data/LICENSE.txt +21 -0
  9. data/README.md +69 -0
  10. data/Rakefile +6 -0
  11. data/bin/console +14 -0
  12. data/bin/setup +8 -0
  13. data/gi_job.gemspec +29 -0
  14. data/lib/generators/gi_job/install/install_generator.rb +49 -0
  15. data/lib/generators/gi_job/install/templates/models/gi_job_file.rb +28 -0
  16. data/lib/generators/gi_job/install/templates/models/gi_job_log.rb +29 -0
  17. data/lib/generators/gi_job/install/templates/models/gi_job_transaction.rb +39 -0
  18. data/lib/generators/gi_job/install/templates/schemas/gi_job_files.schema +11 -0
  19. data/lib/generators/gi_job/install/templates/schemas/gi_job_logs.schema +10 -0
  20. data/lib/generators/gi_job/install/templates/schemas/gi_job_transactions.schema +18 -0
  21. data/lib/generators/gi_job/install/templates/uploaders/gi_log_file_carrier_wave_uploader.rb +25 -0
  22. data/lib/gi_job.rb +32 -0
  23. data/lib/gi_job/jobs/concerns/job_constructor.rb +185 -0
  24. data/lib/gi_job/jobs/concerns/job_runner.rb +549 -0
  25. data/lib/gi_job/jobs/concerns/jobable.rb +28 -0
  26. data/lib/gi_job/jobs/job_command_suspended.rb +11 -0
  27. data/lib/gi_job/jobs/job_utils.rb +22 -0
  28. data/lib/gi_job/models/concerns/gi_job_fileable.rb +79 -0
  29. data/lib/gi_job/models/concerns/gi_job_loggable.rb +26 -0
  30. data/lib/gi_job/models/concerns/gi_job_ownerble.rb +20 -0
  31. data/lib/gi_job/models/concerns/gi_job_transactionable.rb +220 -0
  32. data/lib/gi_job/tasks/gi_job_task.rake +176 -0
  33. data/lib/gi_job/version.rb +3 -0
  34. metadata +78 -0
@@ -0,0 +1,11 @@
1
+ create_table "gi_job_files", id: :bigint, unsigned: true, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" do |t|
2
+ t.references "gi_job_transaction"
3
+ t.string "division"
4
+ t.string "name", null: false
5
+ t.json "file"
6
+ t.unsigned_integer "rows"
7
+ t.unsigned_bigint "size"
8
+ t.timestamps
9
+ end
10
+
11
+ add_index "gi_job_files", ["gi_job_transaction_id"], name: "gi_job_files_ix1", using: :btree
@@ -0,0 +1,10 @@
1
+ create_table "gi_job_logs", id: :bigint, unsigned: true, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" do |t|
2
+ t.references "gi_job_transaction"
3
+ t.integer "level", default: 100, null: false
4
+ t.integer "position"
5
+ t.string "division"
6
+ t.text "description"
7
+ t.timestamps
8
+ end
9
+
10
+ add_index "gi_job_logs", ["gi_job_transaction_id", "level", "division"], name: "gi_job_logs_ix1", using: :btree
@@ -0,0 +1,18 @@
1
+ create_table "gi_job_transactions", id: :bigint, unsigned: true, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" do |t|
2
+ t.references "owner", polymorphic: true
3
+ t.string "name", default: "-", null: false
4
+ t.integer "status", default: 100, null: false
5
+ t.integer "counts_all"
6
+ t.integer "counts_progress"
7
+ t.integer "counts_completed"
8
+ t.integer "counts_errord"
9
+ t.integer "command", default: 100, null: false
10
+ t.text "parameter"
11
+ t.string "process_id"
12
+ t.integer "process_status", default: 100, null: false
13
+ t.text "process_progress"
14
+ t.timestamps
15
+ end
16
+
17
+ add_index "gi_job_transactions", ["owner_id", "owner_type", "name", "status"], name: "gi_job_transactions_ix1", using: :btree
18
+ add_index "gi_job_transactions", ["updated_at"], name: "gi_job_transactions_ix2", using: :btree
@@ -0,0 +1,25 @@
1
+ class GiLogFileCarrierWaveUploader < CarrierWave::Uploader::Base
2
+
3
+ def store_dir
4
+ "#{Rails.env}/uploads/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
5
+ end
6
+
7
+ # def extension_white_list
8
+ # %w(txt csv)
9
+ # end
10
+
11
+ def filename
12
+ if original_filename.present?
13
+ "#{original_filename}"
14
+ end
15
+ end
16
+
17
+ def fog_public
18
+ false
19
+ end
20
+
21
+ def fog_authenticated_url_expiration
22
+ 5.minutes
23
+ end
24
+
25
+ end
@@ -0,0 +1,32 @@
1
+ require "gi_job/version"
2
+ require "gi_job/jobs/job_utils"
3
+ require "gi_job/jobs/job_command_suspended"
4
+ require "gi_job/jobs/concerns/jobable"
5
+ require "gi_job/models/concerns/gi_job_transactionable"
6
+ require "gi_job/models/concerns/gi_job_loggable"
7
+ require "gi_job/models/concerns/gi_job_fileable"
8
+ require "gi_job/models/concerns/gi_job_ownerble"
9
+
10
+ module GiJob
11
+ class Error < StandardError; end
12
+ # Your code goes here...
13
+
14
+ def self.logger
15
+ @logger ||= Rails::Logger.new($stdout, level: Rails::Logger::INFO)
16
+ end
17
+ def self.logger=(logger)
18
+ @logger = logger
19
+ end
20
+
21
+ def self.run
22
+ puts "GiJob call run!!"
23
+ puts "aaa!"
24
+ puts "abc!"
25
+ end
26
+
27
+ class GiJobRailtie < Rails::Railtie
28
+ rake_tasks do
29
+ load 'gi_job/tasks/gi_job_task.rake'
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,185 @@
1
+
2
+ module GiJob
3
+ module Jobs
4
+ module Concerns
5
+ module JobConstructor
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ included do # begin included
10
+ #### job作成
11
+ def self.create_job(**args)
12
+ self.create_job_with(args)
13
+ end
14
+ end # end included
15
+
16
+
17
+ class_methods do # begin class_methods
18
+
19
+ def create_job_with(
20
+ owner: nil,
21
+ parameter: {},
22
+ gi_job_transaction_params: {},
23
+ option: {
24
+ create_only: false,
25
+ uploaded_file: nil,
26
+ }
27
+ )
28
+
29
+ # transactionレコード作成
30
+ gi_job_transaction = self.create_gi_job_transaction(
31
+ owner: owner,
32
+ parameter: parameter,
33
+ gi_job_transaction_params: gi_job_transaction_params,
34
+ option: option
35
+ )
36
+
37
+ if option[:create_only].present?
38
+ # 作成のみの場合はstartしない
39
+ return {gi_job_transaction: gi_job_transaction}
40
+ else
41
+ self.start_job_with(gi_job_transaction: gi_job_transaction)
42
+ end
43
+
44
+ end
45
+
46
+ def start_job_with(gi_job_transaction:)
47
+ # 前処理
48
+ # GiJob.logger.ap({message: "pre_process"})
49
+ result = job_process_call(process_name: "前処理", gi_job_transaction: gi_job_transaction) do |**args|
50
+ self.job_pre_process_impl(args)
51
+ end
52
+
53
+ if result.present? && result[:error].present?
54
+ # 前処理エラー
55
+ return {gi_job_transaction: gi_job_transaction}.merge(result)
56
+ end
57
+
58
+ # 本処理
59
+ # GiJob.logger.ap({message: "call_process"})
60
+ result = job_process_call(process_name: "呼出処理", gi_job_transaction: gi_job_transaction) do |**args|
61
+ self.job_call_process_impl(args)
62
+ end
63
+
64
+ return {gi_job_transaction: gi_job_transaction}.merge(result)
65
+ end
66
+
67
+ def create_gi_job_transaction(
68
+ owner: nil,
69
+ parameter: {},
70
+ gi_job_transaction_params: {},
71
+ option: {}
72
+ )
73
+ # GiJob.logger.ap({create_job_transaction: {owner: owner, parameter: parameter, gi_job_transaction_params: gi_job_transaction_params, option: option,}})
74
+
75
+ tmp_parameter = parameter.deep_dup
76
+
77
+ # jobレコードを作成
78
+ merged_params = ({
79
+ owner: owner,
80
+ name: "#{self.name.underscore}",
81
+ parameter: parameter.present? ? parameter.deep_symbolize_keys! : {},
82
+ }).merge(gi_job_transaction_params)
83
+ gi_job_transaction = GiJobTransaction.new(merged_params)
84
+ gi_job_transaction.save!
85
+ # GiJob.logger.ap(created: {gi_job_transaction: gi_job_transaction})
86
+
87
+ if option.has_key?(:uploaded_file) && option[:uploaded_file].present?
88
+ # uploaded_fileを保存
89
+ uploaded_file = option.delete(:uploaded_file)
90
+ gi_job_transaction.create_file!(uploaded_file: uploaded_file)
91
+ end
92
+
93
+ # # その他のadditional_infoを保存
94
+ # gi_job_transaction.update!(additional_info: tmp_additional_info)
95
+ #
96
+ # # master_jobを紐づけ
97
+ # self.relation_master_schedule_job!(gi_job_transaction, master_schedule_job)
98
+
99
+ gi_job_transaction
100
+
101
+ end
102
+
103
+ def job_process_call(process_name: nil, **args, &func)
104
+ # GiJob.logger.ap(process_name: process_name, args: args, func: func)
105
+ return_object = {error: nil, messages: [], result: nil}
106
+ begin
107
+ return_object = func.call(args)
108
+ rescue => e
109
+ GiJob.logger.fatal(e.message)
110
+ GiJob.logger.fatal(e.backtrace.join("\n"))
111
+ message = "#{process_name.present? ? "#{process_name}で" : ""}エラーが発生しました。エラーコード: #{e}"
112
+ return_object = {error: e, messages: [message], result: nil}
113
+ end
114
+
115
+ if return_object.present? && return_object[:error].present?
116
+ gi_job_transaction = args[:gi_job_transaction]
117
+ gi_job_transaction.status_error!
118
+ gi_job_transaction.append_log_error(
119
+ division: "#{self.name}",
120
+ description: "#{return_object[:messages].present? ? [return_object[:messages]].flatten.join(" ") : ""}"
121
+ )
122
+ end
123
+
124
+ return_object
125
+ end
126
+
127
+ def job_pre_process_impl(**args)
128
+ # 前処理が必要であれば派生クラスで拡張する事
129
+ # この処理は、同期/非同期に関わらず、同期処理で行われる。
130
+ # また、ジョブ再開時には再度呼び出しは行われない。
131
+ # エラーの場合は {error: true} とする事
132
+ {error: nil, messages: [], result: nil}
133
+ end
134
+
135
+ def job_call_process_impl(**args)
136
+ # 呼び出し処理の変更必要であれば派生クラスで拡張する事
137
+ # エラーの場合は {error: true} とする事
138
+ return self.job_call_process_default(args[:gi_job_transaction])
139
+ end
140
+
141
+ def job_call_process_default(gi_job_transaction)
142
+ # GiJob.logger.ap(job_call_process_default: {gi_job_transaction: gi_job_transaction})
143
+
144
+ # TODO 再開時にsuspendをnoneに変更する
145
+ JobUtils.check_job_suspended(gi_job_transaction)
146
+
147
+ application_job = nil
148
+ if gi_job_transaction.parameter[:_perform_now].present?
149
+ # 即時実行指定
150
+ application_job = perform_now({gi_job_transaction: gi_job_transaction})
151
+ else
152
+ application_job = perform_later({gi_job_transaction: gi_job_transaction})
153
+ end
154
+
155
+ {error: nil, messages: [], result: {application_job: application_job}}
156
+ end
157
+
158
+ def job_post_process_impl(**args)
159
+ # 後処理必要であれば派生クラスで拡張する事
160
+ # この処理は、同期/非同期にかかわらず、ジョブ終了後に呼び出される
161
+ # エラーの場合は {error: true} とする事
162
+ return self.job_post_process_default(args[:gi_job_transaction])
163
+ end
164
+
165
+ def job_post_process_default(gi_job_transaction)
166
+ if gi_job_transaction.parameter[:_one_shot].present?
167
+ # 単発ジョブ指定の場合は過去のjobを削除する
168
+ GiJobTransaction.delete_old_for_one_shot(gi_job_transaction)
169
+ end
170
+
171
+ {error: nil, messages: [], result: nil}
172
+ end
173
+
174
+
175
+
176
+ def aaa()
177
+ end
178
+
179
+ end # end class_methods
180
+
181
+
182
+ end # end GiJobable
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,549 @@
1
+
2
+ module GiJob
3
+ module Jobs
4
+ module Concerns
5
+ module JobRunner
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ included do # begin included
10
+ end # end included
11
+
12
+ class_methods do # begin class_methods
13
+ end # end class_methods
14
+
15
+ ##################### job実行系
16
+
17
+ def perform(args)
18
+ set_signal_trap
19
+ perform_impl(args)
20
+ end
21
+
22
+ def perform_impl(**args)
23
+ _perform_default(args)
24
+ end
25
+
26
+ def _perform_default(**args)
27
+
28
+ @_gi_job_transaction = args[:gi_job_transaction]
29
+ # GiJob.logger.ap({_perform_default: {@_gi_job_transaction: @_gi_job_transaction}})
30
+
31
+ @_process_info = {
32
+ error: @_gi_job_transaction.counts_errord != nil && 0 < @_gi_job_transaction.counts_errord,
33
+ counts: {
34
+ all: @_gi_job_transaction.counts_all || 0,
35
+ progress: @_gi_job_transaction.counts_progress || 0,
36
+ completed: @_gi_job_transaction.counts_completed || 0,
37
+ errord: @_gi_job_transaction.counts_errord || 0,
38
+ }
39
+ }
40
+ # GiJob.logger.ap({_perform_default: {@_process_info: @_process_info}})
41
+
42
+
43
+ begin
44
+ run_job do
45
+
46
+ _suspendable = self.class.respond_to?(:GI_JOB_SUSPENDABLE) && self.class::GI_JOB_SUSPENDABLE
47
+ process_status_transaction(suspendable: _suspendable) do
48
+ run_job_impl(
49
+ gi_job_transaction: @_gi_job_transaction,
50
+ parameter: @_gi_job_transaction.parameter
51
+ )
52
+ end
53
+
54
+ end
55
+
56
+ rescue => e
57
+ JobUtils.logging_fatal(e)
58
+ @_gi_job_transaction.status_error!
59
+ @_gi_job_transaction.append_log(
60
+ level: :error,
61
+ division: "#{e.class.name}",
62
+ description: "処理中に内部エラーが発生しました。"
63
+ )
64
+ end
65
+
66
+ end
67
+
68
+ def run_job
69
+ # GiJob.logger.ap({run_job: {args: args}})
70
+
71
+ time_start = Time.zone.now
72
+ result = {}
73
+ begin
74
+ @_gi_job_transaction.status_running!
75
+ result = yield(@_gi_job_transaction)
76
+
77
+ rescue => e
78
+ GiJob.logger.fatal(e.message)
79
+ GiJob.logger.fatal(e.backtrace.join("\n"))
80
+ @_process_info[:error] = true
81
+ @_gi_job_transaction.status_error!
82
+ counts_progress = @_process_info[:counts][:progress]
83
+ message = "#{counts_progress}件目の処理で内部エラーが発生しました。"
84
+ @_gi_job_transaction.append_log(
85
+ level: :error,
86
+ division: "#{e.class.name}",
87
+ description: message,
88
+ )
89
+
90
+ end
91
+
92
+ GiJob.logger.ap({JOB: "done #{self.class.name}"})
93
+ if @_gi_job_transaction.process_status_suspended? || @_gi_job_transaction.status_ended?
94
+ # 以下の場合はステータスを変更しない
95
+ # - 途中でsuspendedが発生している場合
96
+ # - 途中で終了ステータスが設定されている場合
97
+
98
+ else
99
+ if @_process_info[:error]
100
+ @_gi_job_transaction.status_error!
101
+ else
102
+ @_gi_job_transaction.status_completed!
103
+ end
104
+ end
105
+
106
+ syms = @_process_info[:counts].keys
107
+ update_counts_by_process_info(syms)
108
+
109
+ _slack_notifire = self.class.respond_to?(:GI_JOB_SLACK_NOTIFIER) && self.class::GI_JOB_SLACK_NOTIFIER
110
+ if _slack_notifire && @_gi_job_transaction.parameter[:_dont_notify].blank?
111
+ # 通知
112
+ notifier_slack(gi_job_transaction: @_gi_job_transaction, time_start: time_start, result: result)
113
+ end
114
+
115
+ # 後処理
116
+ post_process_result = self.class.job_process_call(process_name: "後処理", gi_job_transaction: @_gi_job_transaction) do |**args|
117
+ self.class.job_post_process_impl(args)
118
+ end
119
+
120
+ end
121
+
122
+ def run_job_impl(gi_job_transaction:, parameter:)
123
+ # 派生クラスで拡張すること
124
+ GiJob.logger.warning("gi_job_transaction.id: #{gi_job_transaction.id}, ...not implemented run_job_imple in #{self.class.name}")
125
+ end
126
+
127
+
128
+ ################### counts系
129
+ def increment_counts(sym = :progress, threshold: 1)
130
+ @_process_info[:counts][sym] += 1
131
+
132
+ record_sym = "counts_#{sym.to_s}".to_sym
133
+ # GiJob.logger.ap({method: "increment_counts", threshold: threshold, record_sym: record_sym, record_value: @_gi_job_transaction[record_sym]})
134
+ if ((@_gi_job_transaction[record_sym] || 0) + (threshold || 1)) <= @_process_info[:counts][sym]
135
+ # thresholdが1000の場合は、1000件ごとにdbを更新する
136
+ # 更新するときは折角なので全部更新する
137
+ syms = @_process_info[:counts].keys
138
+ update_counts_by_process_info(syms)
139
+ end
140
+ end
141
+
142
+ def update_counts(**counts_hash)
143
+ # GiJob.logger.ap({method: "update_counts", counts_hash: counts_hash})
144
+ syms = []
145
+ counts_hash.each do |sym, value|
146
+ @_process_info[:counts][sym] = value
147
+ syms << sym
148
+ end
149
+ # GiJob.logger.ap({method: "update_counts", syms: syms})
150
+ update_counts_by_process_info(syms)
151
+ end
152
+
153
+ def update_counts_by_process_info(syms)
154
+ param = {}
155
+ syms.each do |sym|
156
+ value = @_process_info[:counts][sym]
157
+ record_sym = "counts_#{sym.to_s}".to_sym
158
+ param[record_sym] = value
159
+ # GiJob.logger.ap({method: "update_counts_by_process_info", sym: sym, record_sym: record_sym, value: value, param: param})
160
+ end
161
+ # GiJob.logger.ap({method: "update_counts_by_process_info", param: param})
162
+ @_gi_job_transaction.update!(param)
163
+ end
164
+
165
+ #
166
+ # def self.each_job_file_csv(job, headers: false)
167
+ # job_file = job.job_files.present? ? job.job_files.first : nil
168
+ # if job_file
169
+ # data_source = job_file.data_source
170
+ # # GiJob.logger.ap({data_source: data_source})
171
+ #
172
+ # encoding = Getit::CsvUtils.create_encoding_param(data_source)
173
+ # # GiJob.logger.ap({encoding: encoding})
174
+ #
175
+ # csv = CSV.new(data_source, encoding: encoding, headers: headers)
176
+ # row_num = 0
177
+ # csv.each do |row|
178
+ # row_num += 1
179
+ # yield(row, row_num)
180
+ # end
181
+ # end
182
+ # end
183
+ #
184
+ # def increment_progress(job, need_update = true)
185
+ # # 例外時に補足して件数情報をlogに出すために、インスタンスで処理件数を持つ
186
+ # if @tmp_counts_progress.nil?
187
+ # @tmp_counts_progress = job.counts_progress || 0
188
+ # end
189
+ # @tmp_counts_progress += 1
190
+ # # file_upload時は、最後に件数更新を行うため、need_update = falseを想定
191
+ # (job, @tmp_counts_progress) if need_update
192
+ # end
193
+ #
194
+ # def update_counts_progress(job, num)
195
+ # job.update!({counts_progress: num || 0})
196
+ # @tmp_counts_progress = num
197
+ # end
198
+ #
199
+ # def append_error_with(job, tag, message)
200
+ # job.append_error(tag, message)
201
+ # @has_error = true
202
+ # end
203
+ #
204
+ #
205
+ # def self.save_job_file!(job, additional_info)
206
+ # # upload_fileがある場合はDBに格納する
207
+ # if additional_info.has_key?(:uploaded_file)
208
+ # # 一時ファイルパスを作成
209
+ # tmp_file_path = self.create_file_path(job, additional_info[:uploaded_file].original_filename)
210
+ #
211
+ # # 2回読み込むため、一時ファイルとして一旦保存する
212
+ # file_copy(additional_info[:uploaded_file], tmp_file_path)
213
+ #
214
+ # # レコードへのファイル保存
215
+ # encoding = Getit::CsvUtils.create_encoding_param(File.new(tmp_file_path))
216
+ # mode = "rt:#{encoding}"
217
+ # # GiJob.logger.ap({mode: mode})
218
+ # tmp_file = File.open(tmp_file_path, mode = mode).read
219
+ # job_file = JobFile.new({
220
+ # job: job,
221
+ # file: tmp_file,
222
+ # file_name: additional_info[:uploaded_file].original_filename,
223
+ # counts_row: tmp_file.count("\n"),
224
+ # })
225
+ # job.update!({
226
+ # job_files: [job_file],
227
+ # })
228
+ #
229
+ # # jobに対してActionDispatch::Fileみたいなのが渡せないため削除する
230
+ # additional_info.delete(:uploaded_file)
231
+ #
232
+ # # 古い一時ファイルを削除する
233
+ # self.delete_old_files
234
+ # end
235
+ # end
236
+ #
237
+ # def self.relation_master_schedule_job!(job, master_schedule_job)
238
+ # if master_schedule_job.present? && job.respond_to?(:master_schedule_job)
239
+ # # master_schedule_jobを紐づけ
240
+ # job.master_schedule_job = master_schedule_job
241
+ #
242
+ # if master_schedule_job.additional_info.present?
243
+ # # master_schedule_jobに設定されているadditional_infoを引き継ぎ
244
+ # master_schedule_job_additional_info = master_schedule_job.additional_info
245
+ # if master_schedule_job.additional_info[:next_one].present?
246
+ # next_one = master_schedule_job.additional_info[:next_one].delete
247
+ # master_schedule_job.save!
248
+ # master_schedule_job_additional_info.merge!(next_one)
249
+ # end
250
+ # job.additional_info.merge!(master_schedule_job.additional_info)
251
+ # job.save!
252
+ # end
253
+ #
254
+ # master_schedule_job.last_job = job
255
+ # master_schedule_job.save!
256
+ # end
257
+ #
258
+ # job.save!
259
+ # end
260
+ #
261
+ # def self.create_file_path(job, file_name_suffix = ".csv")
262
+ # file_name_prefix = "#{job.job_type}_" || ""
263
+ # "#{self.tmp_dir}/job_file_#{file_name_prefix}#{job.id}#{file_name_suffix}"
264
+ # end
265
+ #
266
+ # def self.delete_old_files(time_limit = 14.days)
267
+ # # 2週間前の一時ファイルを削除する
268
+ # tmp_dir = self.tmp_dir
269
+ # time_now = Time.now
270
+ # threshold = time_now - time_limit
271
+ # GiJob.logger.ap("delete_old_file #{time_now} - #{time_limit} = threshold: #{threshold}")
272
+ # deleted_paths = []
273
+ # Dir.glob("#{tmp_dir}/job_file_*").select do |file_path|
274
+ # File.mtime(file_path) < threshold
275
+ # end.each do |file_path|
276
+ # GiJob.logger.ap("delete_old_file: #{file_path}")
277
+ # FileUtils.rm(file_path)
278
+ # deleted_paths << file_path
279
+ # end
280
+ # deleted_paths
281
+ # end
282
+ #
283
+ # def self.tmp_dir
284
+ # tmp_dir = "#{Dir.tmpdir}/job"
285
+ # unless File.exist?(tmp_dir)
286
+ # FileUtils.mkdir_p(tmp_dir)
287
+ # end
288
+ # tmp_dir
289
+ # end
290
+ #
291
+ # def self.file_copy(file, dst)
292
+ # GiJob.logger.ap({file: file, dst: dst})
293
+ # FileUtils.cp(file.path, dst)
294
+ # # GiJob.logger.ap({rrr: file.read})
295
+ # # File.open(dst, 'w') do |fp|
296
+ # # fp.write(file.read)
297
+ # # end
298
+ # end
299
+ #
300
+ # def self.file_create(content, dst)
301
+ # File.open(dst, 'w') do |fp|
302
+ # fp.write(content)
303
+ # end
304
+ # end
305
+ #
306
+ # #### slack通知系
307
+ # def notifier_slack(job:, time_start:, time_end: Time.zone.now, job_rc: {})
308
+ # timestamp_hash = create_timestamp_hash(
309
+ # time_start: time_start,
310
+ # time_end: time_end,
311
+ # counts_progress: job.counts_progress
312
+ # )
313
+ # attachments = []
314
+ # attachments << create_slack_common_attachment(job, timestamp_hash)
315
+ # slack_notifier_info = notifier_slack_imple(
316
+ # attachments: attachments,
317
+ # job: job,
318
+ # timestamp_hash: timestamp_hash,
319
+ # job_rc: job_rc
320
+ # )
321
+ #
322
+ # if slack_notifier_info.instance_of?(Array)
323
+ # slack_notifier_info.each do |info|
324
+ # slack_api_wrapper = SlackApiWrapper.new(info[:initialize_params] || {})
325
+ # slack_api_wrapper.notifier(info[:notifier_params])
326
+ # end
327
+ # else
328
+ # slack_api_wrapper = SlackApiWrapper.new
329
+ # slack_api_wrapper.notifier(slack_notifier_info)
330
+ # end
331
+ # end
332
+ #
333
+ # def create_slack_common_attachment(job, timestamp_hash)
334
+ # job_info_string = "#{job.shop.shopify_domain} (#{job.shop.id}) #{job.enum_localize(:job_type)} (#{job.id})"
335
+ # GiJob.logger.info("[JOB-INFO] #{job_info_string} End #{timestamp_hash[:time_description]} #{timestamp_hash[:processing_count_per_second]}")
336
+ #
337
+ # main_color = job.error? ? "danger" : "good"
338
+ # text_string = "処理件数: #{job.counts_progress} / #{job.counts_all}"
339
+ # text_string += "\n#{timestamp_hash[:timestamp_text]}"
340
+ #
341
+ # if job.process_status_suspended?
342
+ # text_string += "\n一時停止: #{job.process_progress} 件目"
343
+ # elsif job.process_status_terminated?
344
+ # text_string += "\n途中終了: #{job.process_progress} 件目"
345
+ # end
346
+ #
347
+ # job_logs_hash = job.create_job_logs_hash
348
+ # if 0 < job_logs_hash[:error].size
349
+ # main_color = "warning"
350
+ # text_string += "\nエラー件数: #{job_logs_hash[:error].size}"
351
+ # text_string += "\n先頭エラーメッセージ: #{job_logs_hash[:error].first.description}"
352
+ # end
353
+ #
354
+ # attachment = {
355
+ # title: "#{job_info_string}",
356
+ # color: main_color,
357
+ # text: "#{text_string}",
358
+ # }
359
+ # attachment
360
+ # end
361
+ #
362
+ # def notifier_slack_imple(
363
+ # attachments: [],
364
+ # job:,
365
+ # timestamp_hash: {},
366
+ # job_rc: {}
367
+ # )
368
+ # # 必要であれば派生クラスで拡張
369
+ # create_simple_slack_notifer(
370
+ # attachments: attachments,
371
+ # job: job,
372
+ # timestamp_hash: timestamp_hash,
373
+ # job_rc: job_rc
374
+ # )
375
+ # end
376
+ #
377
+ # def create_timestamp_hash(time_start:, time_end: Time.zone.now, counts_progress: nil)
378
+ # time_format = '%Y/%m/%d %H:%M:%S'
379
+ # time_elapsed = (time_end - time_start).round(2) # 経過時間
380
+ # time_description = "#{time_start.strftime(time_format)} ~ #{time_end.strftime(time_format)} (#{time_elapsed}秒)" # 開始終了(経過時間)
381
+ # processing_count_per_second = "#{(counts_progress && 0 < counts_progress) ? (counts_progress / time_elapsed).round(2) : "-"} 件/秒" # 処理件数(秒間処理件数)
382
+ # timestamp_text = "実行時間: #{time_description}\n処理速度: #{processing_count_per_second}"
383
+ #
384
+ # {
385
+ # time_start: time_start,
386
+ # time_end: time_end,
387
+ # time_elapsed: time_elapsed,
388
+ # counts_progress: counts_progress,
389
+ # time_description: time_description,
390
+ # processing_count_per_second: processing_count_per_second,
391
+ # timestamp_text: timestamp_text,
392
+ # }
393
+ # end
394
+ #
395
+ # def create_simple_slack_notifer(
396
+ # attachments: [],
397
+ # job:,
398
+ # timestamp_hash: {},
399
+ # job_rc: {}
400
+ # )
401
+ # counts_hash = job_rc.is_a?(Hash) ? job_rc[:counts_hash] : {}
402
+ # GiJob.logger.ap({counts_hash: counts_hash})
403
+ # text = "#{job.to_simple[:local][:job_type]} - #{timestamp_hash[:time_start].strftime('%-m月%-d日 %-H:%M')}"
404
+ #
405
+ # job_counts_text = nil
406
+ # color = nil
407
+ # if counts_hash.present?
408
+ # job_counts_text = "処理件数: #{counts_hash[:counts]}件"
409
+ # color = "good"
410
+ # else
411
+ # job_counts_text = "ジョブ結果が正常に取得できませんでした"
412
+ # color = "warning"
413
+ # end
414
+ #
415
+ # info_links = job.job_log_files.map {|job_log_file| job_log_file.slack_link}
416
+ #
417
+ # tmp_attachments = [{
418
+ # title: "処理詳細",
419
+ # color: color,
420
+ # text: "#{job_counts_text}",
421
+ # }, {
422
+ # title: "ログファイル",
423
+ # color: "good",
424
+ # text: "#{info_links.join("\n")}",
425
+ # }]
426
+ #
427
+ # if counts_hash.present? && counts_hash[:errors].present?
428
+ # tmp_attachments << {
429
+ # title: "エラー",
430
+ # color: "danger",
431
+ # text: "#{counts_hash[:errors].join("\n")}",
432
+ # }
433
+ # end
434
+ #
435
+ # attachments |= tmp_attachments
436
+ # {text: text, attachments: attachments}
437
+ # end
438
+ #
439
+
440
+ ##################### シグナル捕捉 & suspended
441
+ def set_signal_trap
442
+ @_is_signal_trapped = false
443
+ Signal.trap('TERM') do
444
+ # シグナル発生
445
+ GiJob.logger.info('trap TERM!')
446
+ @_is_signal_trapped = true
447
+ end
448
+
449
+ Signal.trap('TSTP') do
450
+ # シグナル発生
451
+ GiJob.logger.info('trap TSTP!')
452
+ @_is_signal_trapped = true
453
+ end
454
+ end
455
+
456
+ def process_status_transaction(
457
+ gi_job_transaction: @_gi_job_transaction,
458
+ suspendable: true
459
+ )
460
+ # suspendable = true の場合、suspended する
461
+ # suspendable = false の場合、terminated する
462
+ begin
463
+ yield
464
+
465
+ rescue JobCommandSuspended => e
466
+ # GiJob.logger.ap({rescue: e})
467
+ message = e.message
468
+
469
+ info_string = "#{message} のため #{@_process_info[:counts][:progress]} 件目の処理中に中断しました"
470
+ process_status = :terminated
471
+
472
+ if suspendable
473
+ info_string = "#{message} のため #{@_process_info[:counts][:progress]} 件目の処理中に一時停止しました"
474
+ process_status = :suspended
475
+ end
476
+
477
+ gi_job_transaction.append_log_info(division: "job停止", description: "#{info_string}")
478
+ gi_job_transaction.update!({
479
+ process_status: process_status,
480
+ process_progress: e.process_progress
481
+ })
482
+ end
483
+ end
484
+
485
+ def check_job_command!
486
+ if @_is_signal_trapped
487
+ # ジョブ停止要求あり
488
+ raise JobCommandSuspended.new(@_process_progress || nil), "サーバー停止要求"
489
+ end
490
+ end
491
+
492
+ def with_job_command(process_progress_key: nil)
493
+ if process_progress_key.present?
494
+ @_process_progress = {key: process_progress_key}
495
+ end
496
+ check_job_command!
497
+
498
+ # skip判断
499
+ if @_gi_job_transaction.process_progress.present? && @_process_progress.present?
500
+ # 途中まで処理されている場合は、そこまで skip する
501
+ suspended_key = @_gi_job_transaction.process_progress.try(:key, nil)
502
+ now_key = @_process_progress.try(:key, nil)
503
+
504
+ if process_progress_key == suspended_key
505
+ # 途中まで処理されていたkeyと一致した場合は再開
506
+ # process_progress を削除しておく
507
+ @_gi_job_transaction.update!(process_progress: nil)
508
+
509
+ else
510
+ # 処理済みのため skip
511
+ GiJob.logger.ap("実施済みの為スキップします process_progress_key: #{process_progress_key} != suspended_key: #{suspended_progress_key}")
512
+ return nil
513
+ end
514
+ end
515
+
516
+ yield
517
+
518
+ end
519
+
520
+ def sleep_as_check_job_command(seconds, &block)
521
+
522
+ sleep_seconds = seconds
523
+ sleep_loop_max_counts = 1
524
+ if 1 < seconds
525
+ sleep_seconds = 1
526
+ sleep_loop_max_counts = seconds.ceil
527
+ end
528
+
529
+ sleep_loop_max_counts.times do |index|
530
+ GiJob.logger.ap({method: "sleep_as_check_job_command", loops: "#{index}/#{sleep_loop_max_counts}"})
531
+
532
+ # チェックしながら小分けにして休む
533
+ check_job_command!
534
+ sleep(sleep_seconds)
535
+
536
+ if block_given?
537
+ # ブロックが渡されている場合は評価して、必要に応じて抜ける
538
+ rc = yield
539
+ break if rc.present?
540
+ end
541
+ end
542
+
543
+ end
544
+
545
+
546
+ end # end JobRunner
547
+ end
548
+ end
549
+ end