gi_job 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.travis.yml +6 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +15 -0
- data/Gemfile.lock +34 -0
- data/LICENSE.txt +21 -0
- data/README.md +69 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/gi_job.gemspec +29 -0
- data/lib/generators/gi_job/install/install_generator.rb +49 -0
- data/lib/generators/gi_job/install/templates/models/gi_job_file.rb +28 -0
- data/lib/generators/gi_job/install/templates/models/gi_job_log.rb +29 -0
- data/lib/generators/gi_job/install/templates/models/gi_job_transaction.rb +39 -0
- data/lib/generators/gi_job/install/templates/schemas/gi_job_files.schema +11 -0
- data/lib/generators/gi_job/install/templates/schemas/gi_job_logs.schema +10 -0
- data/lib/generators/gi_job/install/templates/schemas/gi_job_transactions.schema +18 -0
- data/lib/generators/gi_job/install/templates/uploaders/gi_log_file_carrier_wave_uploader.rb +25 -0
- data/lib/gi_job.rb +32 -0
- data/lib/gi_job/jobs/concerns/job_constructor.rb +185 -0
- data/lib/gi_job/jobs/concerns/job_runner.rb +549 -0
- data/lib/gi_job/jobs/concerns/jobable.rb +28 -0
- data/lib/gi_job/jobs/job_command_suspended.rb +11 -0
- data/lib/gi_job/jobs/job_utils.rb +22 -0
- data/lib/gi_job/models/concerns/gi_job_fileable.rb +79 -0
- data/lib/gi_job/models/concerns/gi_job_loggable.rb +26 -0
- data/lib/gi_job/models/concerns/gi_job_ownerble.rb +20 -0
- data/lib/gi_job/models/concerns/gi_job_transactionable.rb +220 -0
- data/lib/gi_job/tasks/gi_job_task.rake +176 -0
- data/lib/gi_job/version.rb +3 -0
- metadata +78 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
create_table "gi_job_files", id: :bigint, unsigned: true, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" do |t|
|
|
2
|
+
t.references "gi_job_transaction"
|
|
3
|
+
t.string "division"
|
|
4
|
+
t.string "name", null: false
|
|
5
|
+
t.json "file"
|
|
6
|
+
t.unsigned_integer "rows"
|
|
7
|
+
t.unsigned_bigint "size"
|
|
8
|
+
t.timestamps
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
add_index "gi_job_files", ["gi_job_transaction_id"], name: "gi_job_files_ix1", using: :btree
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
create_table "gi_job_logs", id: :bigint, unsigned: true, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" do |t|
|
|
2
|
+
t.references "gi_job_transaction"
|
|
3
|
+
t.integer "level", default: 100, null: false
|
|
4
|
+
t.integer "position"
|
|
5
|
+
t.string "division"
|
|
6
|
+
t.text "description"
|
|
7
|
+
t.timestamps
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
add_index "gi_job_logs", ["gi_job_transaction_id", "level", "division"], name: "gi_job_logs_ix1", using: :btree
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
create_table "gi_job_transactions", id: :bigint, unsigned: true, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" do |t|
|
|
2
|
+
t.references "owner", polymorphic: true
|
|
3
|
+
t.string "name", default: "-", null: false
|
|
4
|
+
t.integer "status", default: 100, null: false
|
|
5
|
+
t.integer "counts_all"
|
|
6
|
+
t.integer "counts_progress"
|
|
7
|
+
t.integer "counts_completed"
|
|
8
|
+
t.integer "counts_errord"
|
|
9
|
+
t.integer "command", default: 100, null: false
|
|
10
|
+
t.text "parameter"
|
|
11
|
+
t.string "process_id"
|
|
12
|
+
t.integer "process_status", default: 100, null: false
|
|
13
|
+
t.text "process_progress"
|
|
14
|
+
t.timestamps
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
add_index "gi_job_transactions", ["owner_id", "owner_type", "name", "status"], name: "gi_job_transactions_ix1", using: :btree
|
|
18
|
+
add_index "gi_job_transactions", ["updated_at"], name: "gi_job_transactions_ix2", using: :btree
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
class GiLogFileCarrierWaveUploader < CarrierWave::Uploader::Base
|
|
2
|
+
|
|
3
|
+
def store_dir
|
|
4
|
+
"#{Rails.env}/uploads/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
# def extension_white_list
|
|
8
|
+
# %w(txt csv)
|
|
9
|
+
# end
|
|
10
|
+
|
|
11
|
+
def filename
|
|
12
|
+
if original_filename.present?
|
|
13
|
+
"#{original_filename}"
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def fog_public
|
|
18
|
+
false
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def fog_authenticated_url_expiration
|
|
22
|
+
5.minutes
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
end
|
data/lib/gi_job.rb
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
require "gi_job/version"
|
|
2
|
+
require "gi_job/jobs/job_utils"
|
|
3
|
+
require "gi_job/jobs/job_command_suspended"
|
|
4
|
+
require "gi_job/jobs/concerns/jobable"
|
|
5
|
+
require "gi_job/models/concerns/gi_job_transactionable"
|
|
6
|
+
require "gi_job/models/concerns/gi_job_loggable"
|
|
7
|
+
require "gi_job/models/concerns/gi_job_fileable"
|
|
8
|
+
require "gi_job/models/concerns/gi_job_ownerble"
|
|
9
|
+
|
|
10
|
+
module GiJob
|
|
11
|
+
class Error < StandardError; end
|
|
12
|
+
# Your code goes here...
|
|
13
|
+
|
|
14
|
+
def self.logger
|
|
15
|
+
@logger ||= Rails::Logger.new($stdout, level: Rails::Logger::INFO)
|
|
16
|
+
end
|
|
17
|
+
def self.logger=(logger)
|
|
18
|
+
@logger = logger
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.run
|
|
22
|
+
puts "GiJob call run!!"
|
|
23
|
+
puts "aaa!"
|
|
24
|
+
puts "abc!"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
class GiJobRailtie < Rails::Railtie
|
|
28
|
+
rake_tasks do
|
|
29
|
+
load 'gi_job/tasks/gi_job_task.rake'
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
|
|
2
|
+
module GiJob
|
|
3
|
+
module Jobs
|
|
4
|
+
module Concerns
|
|
5
|
+
module JobConstructor
|
|
6
|
+
|
|
7
|
+
extend ActiveSupport::Concern
|
|
8
|
+
|
|
9
|
+
included do # begin included
|
|
10
|
+
#### job作成
|
|
11
|
+
def self.create_job(**args)
|
|
12
|
+
self.create_job_with(args)
|
|
13
|
+
end
|
|
14
|
+
end # end included
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class_methods do # begin class_methods
|
|
18
|
+
|
|
19
|
+
def create_job_with(
|
|
20
|
+
owner: nil,
|
|
21
|
+
parameter: {},
|
|
22
|
+
gi_job_transaction_params: {},
|
|
23
|
+
option: {
|
|
24
|
+
create_only: false,
|
|
25
|
+
uploaded_file: nil,
|
|
26
|
+
}
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
# transactionレコード作成
|
|
30
|
+
gi_job_transaction = self.create_gi_job_transaction(
|
|
31
|
+
owner: owner,
|
|
32
|
+
parameter: parameter,
|
|
33
|
+
gi_job_transaction_params: gi_job_transaction_params,
|
|
34
|
+
option: option
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
if option[:create_only].present?
|
|
38
|
+
# 作成のみの場合はstartしない
|
|
39
|
+
return {gi_job_transaction: gi_job_transaction}
|
|
40
|
+
else
|
|
41
|
+
self.start_job_with(gi_job_transaction: gi_job_transaction)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def start_job_with(gi_job_transaction:)
|
|
47
|
+
# 前処理
|
|
48
|
+
# GiJob.logger.ap({message: "pre_process"})
|
|
49
|
+
result = job_process_call(process_name: "前処理", gi_job_transaction: gi_job_transaction) do |**args|
|
|
50
|
+
self.job_pre_process_impl(args)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
if result.present? && result[:error].present?
|
|
54
|
+
# 前処理エラー
|
|
55
|
+
return {gi_job_transaction: gi_job_transaction}.merge(result)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# 本処理
|
|
59
|
+
# GiJob.logger.ap({message: "call_process"})
|
|
60
|
+
result = job_process_call(process_name: "呼出処理", gi_job_transaction: gi_job_transaction) do |**args|
|
|
61
|
+
self.job_call_process_impl(args)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
return {gi_job_transaction: gi_job_transaction}.merge(result)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def create_gi_job_transaction(
|
|
68
|
+
owner: nil,
|
|
69
|
+
parameter: {},
|
|
70
|
+
gi_job_transaction_params: {},
|
|
71
|
+
option: {}
|
|
72
|
+
)
|
|
73
|
+
# GiJob.logger.ap({create_job_transaction: {owner: owner, parameter: parameter, gi_job_transaction_params: gi_job_transaction_params, option: option,}})
|
|
74
|
+
|
|
75
|
+
tmp_parameter = parameter.deep_dup
|
|
76
|
+
|
|
77
|
+
# jobレコードを作成
|
|
78
|
+
merged_params = ({
|
|
79
|
+
owner: owner,
|
|
80
|
+
name: "#{self.name.underscore}",
|
|
81
|
+
parameter: parameter.present? ? parameter.deep_symbolize_keys! : {},
|
|
82
|
+
}).merge(gi_job_transaction_params)
|
|
83
|
+
gi_job_transaction = GiJobTransaction.new(merged_params)
|
|
84
|
+
gi_job_transaction.save!
|
|
85
|
+
# GiJob.logger.ap(created: {gi_job_transaction: gi_job_transaction})
|
|
86
|
+
|
|
87
|
+
if option.has_key?(:uploaded_file) && option[:uploaded_file].present?
|
|
88
|
+
# uploaded_fileを保存
|
|
89
|
+
uploaded_file = option.delete(:uploaded_file)
|
|
90
|
+
gi_job_transaction.create_file!(uploaded_file: uploaded_file)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# # その他のadditional_infoを保存
|
|
94
|
+
# gi_job_transaction.update!(additional_info: tmp_additional_info)
|
|
95
|
+
#
|
|
96
|
+
# # master_jobを紐づけ
|
|
97
|
+
# self.relation_master_schedule_job!(gi_job_transaction, master_schedule_job)
|
|
98
|
+
|
|
99
|
+
gi_job_transaction
|
|
100
|
+
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def job_process_call(process_name: nil, **args, &func)
|
|
104
|
+
# GiJob.logger.ap(process_name: process_name, args: args, func: func)
|
|
105
|
+
return_object = {error: nil, messages: [], result: nil}
|
|
106
|
+
begin
|
|
107
|
+
return_object = func.call(args)
|
|
108
|
+
rescue => e
|
|
109
|
+
GiJob.logger.fatal(e.message)
|
|
110
|
+
GiJob.logger.fatal(e.backtrace.join("\n"))
|
|
111
|
+
message = "#{process_name.present? ? "#{process_name}で" : ""}エラーが発生しました。エラーコード: #{e}"
|
|
112
|
+
return_object = {error: e, messages: [message], result: nil}
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
if return_object.present? && return_object[:error].present?
|
|
116
|
+
gi_job_transaction = args[:gi_job_transaction]
|
|
117
|
+
gi_job_transaction.status_error!
|
|
118
|
+
gi_job_transaction.append_log_error(
|
|
119
|
+
division: "#{self.name}",
|
|
120
|
+
description: "#{return_object[:messages].present? ? [return_object[:messages]].flatten.join(" ") : ""}"
|
|
121
|
+
)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
return_object
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def job_pre_process_impl(**args)
|
|
128
|
+
# 前処理が必要であれば派生クラスで拡張する事
|
|
129
|
+
# この処理は、同期/非同期に関わらず、同期処理で行われる。
|
|
130
|
+
# また、ジョブ再開時には再度呼び出しは行われない。
|
|
131
|
+
# エラーの場合は {error: true} とする事
|
|
132
|
+
{error: nil, messages: [], result: nil}
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def job_call_process_impl(**args)
|
|
136
|
+
# 呼び出し処理の変更必要であれば派生クラスで拡張する事
|
|
137
|
+
# エラーの場合は {error: true} とする事
|
|
138
|
+
return self.job_call_process_default(args[:gi_job_transaction])
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def job_call_process_default(gi_job_transaction)
|
|
142
|
+
# GiJob.logger.ap(job_call_process_default: {gi_job_transaction: gi_job_transaction})
|
|
143
|
+
|
|
144
|
+
# TODO 再開時にsuspendをnoneに変更する
|
|
145
|
+
JobUtils.check_job_suspended(gi_job_transaction)
|
|
146
|
+
|
|
147
|
+
application_job = nil
|
|
148
|
+
if gi_job_transaction.parameter[:_perform_now].present?
|
|
149
|
+
# 即時実行指定
|
|
150
|
+
application_job = perform_now({gi_job_transaction: gi_job_transaction})
|
|
151
|
+
else
|
|
152
|
+
application_job = perform_later({gi_job_transaction: gi_job_transaction})
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
{error: nil, messages: [], result: {application_job: application_job}}
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def job_post_process_impl(**args)
|
|
159
|
+
# 後処理必要であれば派生クラスで拡張する事
|
|
160
|
+
# この処理は、同期/非同期にかかわらず、ジョブ終了後に呼び出される
|
|
161
|
+
# エラーの場合は {error: true} とする事
|
|
162
|
+
return self.job_post_process_default(args[:gi_job_transaction])
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def job_post_process_default(gi_job_transaction)
|
|
166
|
+
if gi_job_transaction.parameter[:_one_shot].present?
|
|
167
|
+
# 単発ジョブ指定の場合は過去のjobを削除する
|
|
168
|
+
GiJobTransaction.delete_old_for_one_shot(gi_job_transaction)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
{error: nil, messages: [], result: nil}
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def aaa()
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
end # end class_methods
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
end # end GiJobable
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
|
@@ -0,0 +1,549 @@
|
|
|
1
|
+
|
|
2
|
+
module GiJob
|
|
3
|
+
module Jobs
|
|
4
|
+
module Concerns
|
|
5
|
+
module JobRunner
|
|
6
|
+
|
|
7
|
+
extend ActiveSupport::Concern
|
|
8
|
+
|
|
9
|
+
included do # begin included
|
|
10
|
+
end # end included
|
|
11
|
+
|
|
12
|
+
class_methods do # begin class_methods
|
|
13
|
+
end # end class_methods
|
|
14
|
+
|
|
15
|
+
##################### job実行系
|
|
16
|
+
|
|
17
|
+
def perform(args)
|
|
18
|
+
set_signal_trap
|
|
19
|
+
perform_impl(args)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def perform_impl(**args)
|
|
23
|
+
_perform_default(args)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def _perform_default(**args)
|
|
27
|
+
|
|
28
|
+
@_gi_job_transaction = args[:gi_job_transaction]
|
|
29
|
+
# GiJob.logger.ap({_perform_default: {@_gi_job_transaction: @_gi_job_transaction}})
|
|
30
|
+
|
|
31
|
+
@_process_info = {
|
|
32
|
+
error: @_gi_job_transaction.counts_errord != nil && 0 < @_gi_job_transaction.counts_errord,
|
|
33
|
+
counts: {
|
|
34
|
+
all: @_gi_job_transaction.counts_all || 0,
|
|
35
|
+
progress: @_gi_job_transaction.counts_progress || 0,
|
|
36
|
+
completed: @_gi_job_transaction.counts_completed || 0,
|
|
37
|
+
errord: @_gi_job_transaction.counts_errord || 0,
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
# GiJob.logger.ap({_perform_default: {@_process_info: @_process_info}})
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
begin
|
|
44
|
+
run_job do
|
|
45
|
+
|
|
46
|
+
_suspendable = self.class.respond_to?(:GI_JOB_SUSPENDABLE) && self.class::GI_JOB_SUSPENDABLE
|
|
47
|
+
process_status_transaction(suspendable: _suspendable) do
|
|
48
|
+
run_job_impl(
|
|
49
|
+
gi_job_transaction: @_gi_job_transaction,
|
|
50
|
+
parameter: @_gi_job_transaction.parameter
|
|
51
|
+
)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
rescue => e
|
|
57
|
+
JobUtils.logging_fatal(e)
|
|
58
|
+
@_gi_job_transaction.status_error!
|
|
59
|
+
@_gi_job_transaction.append_log(
|
|
60
|
+
level: :error,
|
|
61
|
+
division: "#{e.class.name}",
|
|
62
|
+
description: "処理中に内部エラーが発生しました。"
|
|
63
|
+
)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def run_job
|
|
69
|
+
# GiJob.logger.ap({run_job: {args: args}})
|
|
70
|
+
|
|
71
|
+
time_start = Time.zone.now
|
|
72
|
+
result = {}
|
|
73
|
+
begin
|
|
74
|
+
@_gi_job_transaction.status_running!
|
|
75
|
+
result = yield(@_gi_job_transaction)
|
|
76
|
+
|
|
77
|
+
rescue => e
|
|
78
|
+
GiJob.logger.fatal(e.message)
|
|
79
|
+
GiJob.logger.fatal(e.backtrace.join("\n"))
|
|
80
|
+
@_process_info[:error] = true
|
|
81
|
+
@_gi_job_transaction.status_error!
|
|
82
|
+
counts_progress = @_process_info[:counts][:progress]
|
|
83
|
+
message = "#{counts_progress}件目の処理で内部エラーが発生しました。"
|
|
84
|
+
@_gi_job_transaction.append_log(
|
|
85
|
+
level: :error,
|
|
86
|
+
division: "#{e.class.name}",
|
|
87
|
+
description: message,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
GiJob.logger.ap({JOB: "done #{self.class.name}"})
|
|
93
|
+
if @_gi_job_transaction.process_status_suspended? || @_gi_job_transaction.status_ended?
|
|
94
|
+
# 以下の場合はステータスを変更しない
|
|
95
|
+
# - 途中でsuspendedが発生している場合
|
|
96
|
+
# - 途中で終了ステータスが設定されている場合
|
|
97
|
+
|
|
98
|
+
else
|
|
99
|
+
if @_process_info[:error]
|
|
100
|
+
@_gi_job_transaction.status_error!
|
|
101
|
+
else
|
|
102
|
+
@_gi_job_transaction.status_completed!
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
syms = @_process_info[:counts].keys
|
|
107
|
+
update_counts_by_process_info(syms)
|
|
108
|
+
|
|
109
|
+
_slack_notifire = self.class.respond_to?(:GI_JOB_SLACK_NOTIFIER) && self.class::GI_JOB_SLACK_NOTIFIER
|
|
110
|
+
if _slack_notifire && @_gi_job_transaction.parameter[:_dont_notify].blank?
|
|
111
|
+
# 通知
|
|
112
|
+
notifier_slack(gi_job_transaction: @_gi_job_transaction, time_start: time_start, result: result)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# 後処理
|
|
116
|
+
post_process_result = self.class.job_process_call(process_name: "後処理", gi_job_transaction: @_gi_job_transaction) do |**args|
|
|
117
|
+
self.class.job_post_process_impl(args)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def run_job_impl(gi_job_transaction:, parameter:)
|
|
123
|
+
# 派生クラスで拡張すること
|
|
124
|
+
GiJob.logger.warning("gi_job_transaction.id: #{gi_job_transaction.id}, ...not implemented run_job_imple in #{self.class.name}")
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
################### counts系
|
|
129
|
+
def increment_counts(sym = :progress, threshold: 1)
|
|
130
|
+
@_process_info[:counts][sym] += 1
|
|
131
|
+
|
|
132
|
+
record_sym = "counts_#{sym.to_s}".to_sym
|
|
133
|
+
# GiJob.logger.ap({method: "increment_counts", threshold: threshold, record_sym: record_sym, record_value: @_gi_job_transaction[record_sym]})
|
|
134
|
+
if ((@_gi_job_transaction[record_sym] || 0) + (threshold || 1)) <= @_process_info[:counts][sym]
|
|
135
|
+
# thresholdが1000の場合は、1000件ごとにdbを更新する
|
|
136
|
+
# 更新するときは折角なので全部更新する
|
|
137
|
+
syms = @_process_info[:counts].keys
|
|
138
|
+
update_counts_by_process_info(syms)
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def update_counts(**counts_hash)
|
|
143
|
+
# GiJob.logger.ap({method: "update_counts", counts_hash: counts_hash})
|
|
144
|
+
syms = []
|
|
145
|
+
counts_hash.each do |sym, value|
|
|
146
|
+
@_process_info[:counts][sym] = value
|
|
147
|
+
syms << sym
|
|
148
|
+
end
|
|
149
|
+
# GiJob.logger.ap({method: "update_counts", syms: syms})
|
|
150
|
+
update_counts_by_process_info(syms)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def update_counts_by_process_info(syms)
|
|
154
|
+
param = {}
|
|
155
|
+
syms.each do |sym|
|
|
156
|
+
value = @_process_info[:counts][sym]
|
|
157
|
+
record_sym = "counts_#{sym.to_s}".to_sym
|
|
158
|
+
param[record_sym] = value
|
|
159
|
+
# GiJob.logger.ap({method: "update_counts_by_process_info", sym: sym, record_sym: record_sym, value: value, param: param})
|
|
160
|
+
end
|
|
161
|
+
# GiJob.logger.ap({method: "update_counts_by_process_info", param: param})
|
|
162
|
+
@_gi_job_transaction.update!(param)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
#
|
|
166
|
+
# def self.each_job_file_csv(job, headers: false)
|
|
167
|
+
# job_file = job.job_files.present? ? job.job_files.first : nil
|
|
168
|
+
# if job_file
|
|
169
|
+
# data_source = job_file.data_source
|
|
170
|
+
# # GiJob.logger.ap({data_source: data_source})
|
|
171
|
+
#
|
|
172
|
+
# encoding = Getit::CsvUtils.create_encoding_param(data_source)
|
|
173
|
+
# # GiJob.logger.ap({encoding: encoding})
|
|
174
|
+
#
|
|
175
|
+
# csv = CSV.new(data_source, encoding: encoding, headers: headers)
|
|
176
|
+
# row_num = 0
|
|
177
|
+
# csv.each do |row|
|
|
178
|
+
# row_num += 1
|
|
179
|
+
# yield(row, row_num)
|
|
180
|
+
# end
|
|
181
|
+
# end
|
|
182
|
+
# end
|
|
183
|
+
#
|
|
184
|
+
# def increment_progress(job, need_update = true)
|
|
185
|
+
# # 例外時に補足して件数情報をlogに出すために、インスタンスで処理件数を持つ
|
|
186
|
+
# if @tmp_counts_progress.nil?
|
|
187
|
+
# @tmp_counts_progress = job.counts_progress || 0
|
|
188
|
+
# end
|
|
189
|
+
# @tmp_counts_progress += 1
|
|
190
|
+
# # file_upload時は、最後に件数更新を行うため、need_update = falseを想定
|
|
191
|
+
# (job, @tmp_counts_progress) if need_update
|
|
192
|
+
# end
|
|
193
|
+
#
|
|
194
|
+
# def update_counts_progress(job, num)
|
|
195
|
+
# job.update!({counts_progress: num || 0})
|
|
196
|
+
# @tmp_counts_progress = num
|
|
197
|
+
# end
|
|
198
|
+
#
|
|
199
|
+
# def append_error_with(job, tag, message)
|
|
200
|
+
# job.append_error(tag, message)
|
|
201
|
+
# @has_error = true
|
|
202
|
+
# end
|
|
203
|
+
#
|
|
204
|
+
#
|
|
205
|
+
# def self.save_job_file!(job, additional_info)
|
|
206
|
+
# # upload_fileがある場合はDBに格納する
|
|
207
|
+
# if additional_info.has_key?(:uploaded_file)
|
|
208
|
+
# # 一時ファイルパスを作成
|
|
209
|
+
# tmp_file_path = self.create_file_path(job, additional_info[:uploaded_file].original_filename)
|
|
210
|
+
#
|
|
211
|
+
# # 2回読み込むため、一時ファイルとして一旦保存する
|
|
212
|
+
# file_copy(additional_info[:uploaded_file], tmp_file_path)
|
|
213
|
+
#
|
|
214
|
+
# # レコードへのファイル保存
|
|
215
|
+
# encoding = Getit::CsvUtils.create_encoding_param(File.new(tmp_file_path))
|
|
216
|
+
# mode = "rt:#{encoding}"
|
|
217
|
+
# # GiJob.logger.ap({mode: mode})
|
|
218
|
+
# tmp_file = File.open(tmp_file_path, mode = mode).read
|
|
219
|
+
# job_file = JobFile.new({
|
|
220
|
+
# job: job,
|
|
221
|
+
# file: tmp_file,
|
|
222
|
+
# file_name: additional_info[:uploaded_file].original_filename,
|
|
223
|
+
# counts_row: tmp_file.count("\n"),
|
|
224
|
+
# })
|
|
225
|
+
# job.update!({
|
|
226
|
+
# job_files: [job_file],
|
|
227
|
+
# })
|
|
228
|
+
#
|
|
229
|
+
# # jobに対してActionDispatch::Fileみたいなのが渡せないため削除する
|
|
230
|
+
# additional_info.delete(:uploaded_file)
|
|
231
|
+
#
|
|
232
|
+
# # 古い一時ファイルを削除する
|
|
233
|
+
# self.delete_old_files
|
|
234
|
+
# end
|
|
235
|
+
# end
|
|
236
|
+
#
|
|
237
|
+
# def self.relation_master_schedule_job!(job, master_schedule_job)
|
|
238
|
+
# if master_schedule_job.present? && job.respond_to?(:master_schedule_job)
|
|
239
|
+
# # master_schedule_jobを紐づけ
|
|
240
|
+
# job.master_schedule_job = master_schedule_job
|
|
241
|
+
#
|
|
242
|
+
# if master_schedule_job.additional_info.present?
|
|
243
|
+
# # master_schedule_jobに設定されているadditional_infoを引き継ぎ
|
|
244
|
+
# master_schedule_job_additional_info = master_schedule_job.additional_info
|
|
245
|
+
# if master_schedule_job.additional_info[:next_one].present?
|
|
246
|
+
# next_one = master_schedule_job.additional_info[:next_one].delete
|
|
247
|
+
# master_schedule_job.save!
|
|
248
|
+
# master_schedule_job_additional_info.merge!(next_one)
|
|
249
|
+
# end
|
|
250
|
+
# job.additional_info.merge!(master_schedule_job.additional_info)
|
|
251
|
+
# job.save!
|
|
252
|
+
# end
|
|
253
|
+
#
|
|
254
|
+
# master_schedule_job.last_job = job
|
|
255
|
+
# master_schedule_job.save!
|
|
256
|
+
# end
|
|
257
|
+
#
|
|
258
|
+
# job.save!
|
|
259
|
+
# end
|
|
260
|
+
#
|
|
261
|
+
# def self.create_file_path(job, file_name_suffix = ".csv")
|
|
262
|
+
# file_name_prefix = "#{job.job_type}_" || ""
|
|
263
|
+
# "#{self.tmp_dir}/job_file_#{file_name_prefix}#{job.id}#{file_name_suffix}"
|
|
264
|
+
# end
|
|
265
|
+
#
|
|
266
|
+
# def self.delete_old_files(time_limit = 14.days)
|
|
267
|
+
# # 2週間前の一時ファイルを削除する
|
|
268
|
+
# tmp_dir = self.tmp_dir
|
|
269
|
+
# time_now = Time.now
|
|
270
|
+
# threshold = time_now - time_limit
|
|
271
|
+
# GiJob.logger.ap("delete_old_file #{time_now} - #{time_limit} = threshold: #{threshold}")
|
|
272
|
+
# deleted_paths = []
|
|
273
|
+
# Dir.glob("#{tmp_dir}/job_file_*").select do |file_path|
|
|
274
|
+
# File.mtime(file_path) < threshold
|
|
275
|
+
# end.each do |file_path|
|
|
276
|
+
# GiJob.logger.ap("delete_old_file: #{file_path}")
|
|
277
|
+
# FileUtils.rm(file_path)
|
|
278
|
+
# deleted_paths << file_path
|
|
279
|
+
# end
|
|
280
|
+
# deleted_paths
|
|
281
|
+
# end
|
|
282
|
+
#
|
|
283
|
+
# def self.tmp_dir
|
|
284
|
+
# tmp_dir = "#{Dir.tmpdir}/job"
|
|
285
|
+
# unless File.exist?(tmp_dir)
|
|
286
|
+
# FileUtils.mkdir_p(tmp_dir)
|
|
287
|
+
# end
|
|
288
|
+
# tmp_dir
|
|
289
|
+
# end
|
|
290
|
+
#
|
|
291
|
+
# def self.file_copy(file, dst)
|
|
292
|
+
# GiJob.logger.ap({file: file, dst: dst})
|
|
293
|
+
# FileUtils.cp(file.path, dst)
|
|
294
|
+
# # GiJob.logger.ap({rrr: file.read})
|
|
295
|
+
# # File.open(dst, 'w') do |fp|
|
|
296
|
+
# # fp.write(file.read)
|
|
297
|
+
# # end
|
|
298
|
+
# end
|
|
299
|
+
#
|
|
300
|
+
# def self.file_create(content, dst)
|
|
301
|
+
# File.open(dst, 'w') do |fp|
|
|
302
|
+
# fp.write(content)
|
|
303
|
+
# end
|
|
304
|
+
# end
|
|
305
|
+
#
|
|
306
|
+
# #### slack通知系
|
|
307
|
+
# def notifier_slack(job:, time_start:, time_end: Time.zone.now, job_rc: {})
|
|
308
|
+
# timestamp_hash = create_timestamp_hash(
|
|
309
|
+
# time_start: time_start,
|
|
310
|
+
# time_end: time_end,
|
|
311
|
+
# counts_progress: job.counts_progress
|
|
312
|
+
# )
|
|
313
|
+
# attachments = []
|
|
314
|
+
# attachments << create_slack_common_attachment(job, timestamp_hash)
|
|
315
|
+
# slack_notifier_info = notifier_slack_imple(
|
|
316
|
+
# attachments: attachments,
|
|
317
|
+
# job: job,
|
|
318
|
+
# timestamp_hash: timestamp_hash,
|
|
319
|
+
# job_rc: job_rc
|
|
320
|
+
# )
|
|
321
|
+
#
|
|
322
|
+
# if slack_notifier_info.instance_of?(Array)
|
|
323
|
+
# slack_notifier_info.each do |info|
|
|
324
|
+
# slack_api_wrapper = SlackApiWrapper.new(info[:initialize_params] || {})
|
|
325
|
+
# slack_api_wrapper.notifier(info[:notifier_params])
|
|
326
|
+
# end
|
|
327
|
+
# else
|
|
328
|
+
# slack_api_wrapper = SlackApiWrapper.new
|
|
329
|
+
# slack_api_wrapper.notifier(slack_notifier_info)
|
|
330
|
+
# end
|
|
331
|
+
# end
|
|
332
|
+
#
|
|
333
|
+
# def create_slack_common_attachment(job, timestamp_hash)
|
|
334
|
+
# job_info_string = "#{job.shop.shopify_domain} (#{job.shop.id}) #{job.enum_localize(:job_type)} (#{job.id})"
|
|
335
|
+
# GiJob.logger.info("[JOB-INFO] #{job_info_string} End #{timestamp_hash[:time_description]} #{timestamp_hash[:processing_count_per_second]}")
|
|
336
|
+
#
|
|
337
|
+
# main_color = job.error? ? "danger" : "good"
|
|
338
|
+
# text_string = "処理件数: #{job.counts_progress} / #{job.counts_all}"
|
|
339
|
+
# text_string += "\n#{timestamp_hash[:timestamp_text]}"
|
|
340
|
+
#
|
|
341
|
+
# if job.process_status_suspended?
|
|
342
|
+
# text_string += "\n一時停止: #{job.process_progress} 件目"
|
|
343
|
+
# elsif job.process_status_terminated?
|
|
344
|
+
# text_string += "\n途中終了: #{job.process_progress} 件目"
|
|
345
|
+
# end
|
|
346
|
+
#
|
|
347
|
+
# job_logs_hash = job.create_job_logs_hash
|
|
348
|
+
# if 0 < job_logs_hash[:error].size
|
|
349
|
+
# main_color = "warning"
|
|
350
|
+
# text_string += "\nエラー件数: #{job_logs_hash[:error].size}"
|
|
351
|
+
# text_string += "\n先頭エラーメッセージ: #{job_logs_hash[:error].first.description}"
|
|
352
|
+
# end
|
|
353
|
+
#
|
|
354
|
+
# attachment = {
|
|
355
|
+
# title: "#{job_info_string}",
|
|
356
|
+
# color: main_color,
|
|
357
|
+
# text: "#{text_string}",
|
|
358
|
+
# }
|
|
359
|
+
# attachment
|
|
360
|
+
# end
|
|
361
|
+
#
|
|
362
|
+
# def notifier_slack_imple(
|
|
363
|
+
# attachments: [],
|
|
364
|
+
# job:,
|
|
365
|
+
# timestamp_hash: {},
|
|
366
|
+
# job_rc: {}
|
|
367
|
+
# )
|
|
368
|
+
# # 必要であれば派生クラスで拡張
|
|
369
|
+
# create_simple_slack_notifer(
|
|
370
|
+
# attachments: attachments,
|
|
371
|
+
# job: job,
|
|
372
|
+
# timestamp_hash: timestamp_hash,
|
|
373
|
+
# job_rc: job_rc
|
|
374
|
+
# )
|
|
375
|
+
# end
|
|
376
|
+
#
|
|
377
|
+
# def create_timestamp_hash(time_start:, time_end: Time.zone.now, counts_progress: nil)
|
|
378
|
+
# time_format = '%Y/%m/%d %H:%M:%S'
|
|
379
|
+
# time_elapsed = (time_end - time_start).round(2) # 経過時間
|
|
380
|
+
# time_description = "#{time_start.strftime(time_format)} ~ #{time_end.strftime(time_format)} (#{time_elapsed}秒)" # 開始終了(経過時間)
|
|
381
|
+
# processing_count_per_second = "#{(counts_progress && 0 < counts_progress) ? (counts_progress / time_elapsed).round(2) : "-"} 件/秒" # 処理件数(秒間処理件数)
|
|
382
|
+
# timestamp_text = "実行時間: #{time_description}\n処理速度: #{processing_count_per_second}"
|
|
383
|
+
#
|
|
384
|
+
# {
|
|
385
|
+
# time_start: time_start,
|
|
386
|
+
# time_end: time_end,
|
|
387
|
+
# time_elapsed: time_elapsed,
|
|
388
|
+
# counts_progress: counts_progress,
|
|
389
|
+
# time_description: time_description,
|
|
390
|
+
# processing_count_per_second: processing_count_per_second,
|
|
391
|
+
# timestamp_text: timestamp_text,
|
|
392
|
+
# }
|
|
393
|
+
# end
|
|
394
|
+
#
|
|
395
|
+
# def create_simple_slack_notifer(
|
|
396
|
+
# attachments: [],
|
|
397
|
+
# job:,
|
|
398
|
+
# timestamp_hash: {},
|
|
399
|
+
# job_rc: {}
|
|
400
|
+
# )
|
|
401
|
+
# counts_hash = job_rc.is_a?(Hash) ? job_rc[:counts_hash] : {}
|
|
402
|
+
# GiJob.logger.ap({counts_hash: counts_hash})
|
|
403
|
+
# text = "#{job.to_simple[:local][:job_type]} - #{timestamp_hash[:time_start].strftime('%-m月%-d日 %-H:%M')}"
|
|
404
|
+
#
|
|
405
|
+
# job_counts_text = nil
|
|
406
|
+
# color = nil
|
|
407
|
+
# if counts_hash.present?
|
|
408
|
+
# job_counts_text = "処理件数: #{counts_hash[:counts]}件"
|
|
409
|
+
# color = "good"
|
|
410
|
+
# else
|
|
411
|
+
# job_counts_text = "ジョブ結果が正常に取得できませんでした"
|
|
412
|
+
# color = "warning"
|
|
413
|
+
# end
|
|
414
|
+
#
|
|
415
|
+
# info_links = job.job_log_files.map {|job_log_file| job_log_file.slack_link}
|
|
416
|
+
#
|
|
417
|
+
# tmp_attachments = [{
|
|
418
|
+
# title: "処理詳細",
|
|
419
|
+
# color: color,
|
|
420
|
+
# text: "#{job_counts_text}",
|
|
421
|
+
# }, {
|
|
422
|
+
# title: "ログファイル",
|
|
423
|
+
# color: "good",
|
|
424
|
+
# text: "#{info_links.join("\n")}",
|
|
425
|
+
# }]
|
|
426
|
+
#
|
|
427
|
+
# if counts_hash.present? && counts_hash[:errors].present?
|
|
428
|
+
# tmp_attachments << {
|
|
429
|
+
# title: "エラー",
|
|
430
|
+
# color: "danger",
|
|
431
|
+
# text: "#{counts_hash[:errors].join("\n")}",
|
|
432
|
+
# }
|
|
433
|
+
# end
|
|
434
|
+
#
|
|
435
|
+
# attachments |= tmp_attachments
|
|
436
|
+
# {text: text, attachments: attachments}
|
|
437
|
+
# end
|
|
438
|
+
#
|
|
439
|
+
|
|
440
|
+
##################### シグナル捕捉 & suspended
|
|
441
|
+
def set_signal_trap
|
|
442
|
+
@_is_signal_trapped = false
|
|
443
|
+
Signal.trap('TERM') do
|
|
444
|
+
# シグナル発生
|
|
445
|
+
GiJob.logger.info('trap TERM!')
|
|
446
|
+
@_is_signal_trapped = true
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
Signal.trap('TSTP') do
|
|
450
|
+
# シグナル発生
|
|
451
|
+
GiJob.logger.info('trap TSTP!')
|
|
452
|
+
@_is_signal_trapped = true
|
|
453
|
+
end
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def process_status_transaction(
|
|
457
|
+
gi_job_transaction: @_gi_job_transaction,
|
|
458
|
+
suspendable: true
|
|
459
|
+
)
|
|
460
|
+
# suspendable = true の場合、suspended する
|
|
461
|
+
# suspendable = false の場合、terminated する
|
|
462
|
+
begin
|
|
463
|
+
yield
|
|
464
|
+
|
|
465
|
+
rescue JobCommandSuspended => e
|
|
466
|
+
# GiJob.logger.ap({rescue: e})
|
|
467
|
+
message = e.message
|
|
468
|
+
|
|
469
|
+
info_string = "#{message} のため #{@_process_info[:counts][:progress]} 件目の処理中に中断しました"
|
|
470
|
+
process_status = :terminated
|
|
471
|
+
|
|
472
|
+
if suspendable
|
|
473
|
+
info_string = "#{message} のため #{@_process_info[:counts][:progress]} 件目の処理中に一時停止しました"
|
|
474
|
+
process_status = :suspended
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
gi_job_transaction.append_log_info(division: "job停止", description: "#{info_string}")
|
|
478
|
+
gi_job_transaction.update!({
|
|
479
|
+
process_status: process_status,
|
|
480
|
+
process_progress: e.process_progress
|
|
481
|
+
})
|
|
482
|
+
end
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
def check_job_command!
|
|
486
|
+
if @_is_signal_trapped
|
|
487
|
+
# ジョブ停止要求あり
|
|
488
|
+
raise JobCommandSuspended.new(@_process_progress || nil), "サーバー停止要求"
|
|
489
|
+
end
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
def with_job_command(process_progress_key: nil)
|
|
493
|
+
if process_progress_key.present?
|
|
494
|
+
@_process_progress = {key: process_progress_key}
|
|
495
|
+
end
|
|
496
|
+
check_job_command!
|
|
497
|
+
|
|
498
|
+
# skip判断
|
|
499
|
+
if @_gi_job_transaction.process_progress.present? && @_process_progress.present?
|
|
500
|
+
# 途中まで処理されている場合は、そこまで skip する
|
|
501
|
+
suspended_key = @_gi_job_transaction.process_progress.try(:key, nil)
|
|
502
|
+
now_key = @_process_progress.try(:key, nil)
|
|
503
|
+
|
|
504
|
+
if process_progress_key == suspended_key
|
|
505
|
+
# 途中まで処理されていたkeyと一致した場合は再開
|
|
506
|
+
# process_progress を削除しておく
|
|
507
|
+
@_gi_job_transaction.update!(process_progress: nil)
|
|
508
|
+
|
|
509
|
+
else
|
|
510
|
+
# 処理済みのため skip
|
|
511
|
+
GiJob.logger.ap("実施済みの為スキップします process_progress_key: #{process_progress_key} != suspended_key: #{suspended_progress_key}")
|
|
512
|
+
return nil
|
|
513
|
+
end
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
yield
|
|
517
|
+
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
def sleep_as_check_job_command(seconds, &block)
|
|
521
|
+
|
|
522
|
+
sleep_seconds = seconds
|
|
523
|
+
sleep_loop_max_counts = 1
|
|
524
|
+
if 1 < seconds
|
|
525
|
+
sleep_seconds = 1
|
|
526
|
+
sleep_loop_max_counts = seconds.ceil
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
sleep_loop_max_counts.times do |index|
|
|
530
|
+
GiJob.logger.ap({method: "sleep_as_check_job_command", loops: "#{index}/#{sleep_loop_max_counts}"})
|
|
531
|
+
|
|
532
|
+
# チェックしながら小分けにして休む
|
|
533
|
+
check_job_command!
|
|
534
|
+
sleep(sleep_seconds)
|
|
535
|
+
|
|
536
|
+
if block_given?
|
|
537
|
+
# ブロックが渡されている場合は評価して、必要に応じて抜ける
|
|
538
|
+
rc = yield
|
|
539
|
+
break if rc.present?
|
|
540
|
+
end
|
|
541
|
+
end
|
|
542
|
+
|
|
543
|
+
end
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
end # end JobRunner
|
|
547
|
+
end
|
|
548
|
+
end
|
|
549
|
+
end
|