inat-channel 0.8.2 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +327 -94
- data/bin/inat-channel +14 -15
- data/lib/inat-channel/api.rb +53 -30
- data/lib/inat-channel/config.rb +52 -13
- data/lib/inat-channel/data.rb +267 -36
- data/lib/inat-channel/data_convert.rb +27 -11
- data/lib/inat-channel/data_types.rb +53 -81
- data/lib/inat-channel/icons.rb +21 -0
- data/lib/inat-channel/lock.rb +6 -6
- data/lib/inat-channel/logger.rb +10 -2
- data/lib/inat-channel/message.rb +18 -3
- data/lib/inat-channel/telegram.rb +30 -12
- data/lib/inat-channel/template.rb +25 -9
- data/lib/inat-channel/version.rb +7 -2
- data/lib/inat-channel.rb +0 -1
- metadata +1 -29
data/lib/inat-channel/api.rb
CHANGED
|
@@ -15,7 +15,7 @@ module INatChannel
|
|
|
15
15
|
PER_PAGE = 200
|
|
16
16
|
PAGE_DELAY = 1.0
|
|
17
17
|
API_ENDPOINT = 'https://api.inaturalist.org/v2/observations'
|
|
18
|
-
LIST_FIELDS = 'uuid'
|
|
18
|
+
LIST_FIELDS = '(uuid:!t,created_at:!t,faves_count:!t,taxon:(id:!t))'
|
|
19
19
|
SINGLE_FIELDS = '(id:!t,uuid:!t,uri:!t,geojson:(all:!t),user:(login:!t,name:!t),taxon:(ancestor_ids:!t,preferred_common_name:!t,name:!t),' +
|
|
20
20
|
'place_ids:!t,place_guess:!t,observed_on_string:!t,description:!t,photos:(url:!t),time_observed_at:!t,' +
|
|
21
21
|
'identifications:(taxon:(ancestors:(name:!t))))'
|
|
@@ -23,66 +23,66 @@ module INatChannel
|
|
|
23
23
|
private_constant :PER_PAGE, :PAGE_DELAY, :API_ENDPOINT, :LIST_FIELDS, :SINGLE_FIELDS
|
|
24
24
|
|
|
25
25
|
def load_news
|
|
26
|
+
load_list(**IC::CONFIG[:base_query], updated_since: (Date.today - IC::CONFIG.dig(:days_back, :fresh)).to_s)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def load_list **query
|
|
26
30
|
result = []
|
|
27
31
|
page = 1
|
|
28
|
-
|
|
32
|
+
|
|
29
33
|
loop do
|
|
30
|
-
|
|
34
|
+
IC::logger.debug "Fetch page #{page} with per_page=#{PER_PAGE}"
|
|
31
35
|
|
|
32
36
|
response = faraday.get API_ENDPOINT do |req|
|
|
33
|
-
req.params[
|
|
34
|
-
req.params[
|
|
35
|
-
req.params[
|
|
36
|
-
req.params.merge!
|
|
37
|
-
req.params['created_d1'] = (Date.today - INatChannel::CONFIG[:days_back]).to_s
|
|
37
|
+
req.params[:page] = page
|
|
38
|
+
req.params[:per_page] = PER_PAGE
|
|
39
|
+
req.params[:fields] = LIST_FIELDS
|
|
40
|
+
req.params.merge! query
|
|
38
41
|
end
|
|
39
42
|
|
|
40
43
|
unless response.success?
|
|
41
|
-
|
|
42
|
-
|
|
44
|
+
IC::logger.error "❌ Failed to fetch observations page #{page}: HTTP #{response.status}"
|
|
45
|
+
IC::notify_admin "❌ Failed to fetch observations page #{page}: HTTP #{response.status}"
|
|
43
46
|
break
|
|
44
47
|
end
|
|
45
48
|
|
|
46
49
|
data = JSON.parse response.body, symbolize_names: true
|
|
47
|
-
|
|
48
|
-
result += uuids
|
|
50
|
+
result += data[:results]
|
|
49
51
|
|
|
50
52
|
total = data[:total_results] || 0
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
break if uuids.empty? || result.size >= total
|
|
53
|
+
IC::logger.debug "Page #{page}: fetched #{data.size} records, total expected #{total}"
|
|
54
|
+
break if data.empty? || result.size >= total
|
|
54
55
|
page += 1
|
|
55
56
|
sleep PAGE_DELAY
|
|
56
57
|
end
|
|
57
58
|
|
|
58
|
-
|
|
59
|
-
result
|
|
59
|
+
IC::logger.debug "Loaded total #{result.size} records"
|
|
60
|
+
result
|
|
60
61
|
rescue => e
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
[]
|
|
62
|
+
IC::logger.error "❌ Exception while loading news: #{e.full_message}"
|
|
63
|
+
IC::notify_admin "❌ Exception while loading news: #{e.message}"
|
|
64
64
|
end
|
|
65
65
|
|
|
66
66
|
def load_observation uuid
|
|
67
67
|
response = faraday.get API_ENDPOINT do |req|
|
|
68
68
|
req.params['uuid'] = uuid
|
|
69
|
-
req.params['locale'] =
|
|
69
|
+
req.params['locale'] = IC::CONFIG[:base_query][:locale] if IC::CONFIG[:base_query][:locale]
|
|
70
70
|
req.params['fields'] = SINGLE_FIELDS
|
|
71
71
|
end
|
|
72
72
|
|
|
73
73
|
if response.success?
|
|
74
74
|
data = JSON.parse response.body, symbolize_names: true
|
|
75
75
|
obs = data[:results]&.first
|
|
76
|
-
|
|
76
|
+
IC::logger.debug "Loaded observation: #{uuid}"
|
|
77
77
|
obs
|
|
78
78
|
else
|
|
79
|
-
|
|
80
|
-
|
|
79
|
+
IC::logger.error "Error loading observation #{uuid}: HTTP #{response.status}"
|
|
80
|
+
IC::notify_admin "Error loading observation #{uuid}: HTTP #{response.status}"
|
|
81
81
|
nil
|
|
82
82
|
end
|
|
83
83
|
rescue => e
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
IC::notify_admin "Exception while loading observation #{uuid}: #{e.message}"
|
|
85
|
+
IC::logger.error e.full_message
|
|
86
86
|
nil
|
|
87
87
|
end
|
|
88
88
|
|
|
@@ -90,12 +90,16 @@ module INatChannel
|
|
|
90
90
|
|
|
91
91
|
def faraday
|
|
92
92
|
@faraday ||= Faraday::new do |f|
|
|
93
|
-
f.request :retry,
|
|
94
|
-
|
|
93
|
+
f.request :retry,
|
|
94
|
+
max: IC::CONFIG.dig(:api, :retries),
|
|
95
|
+
interval: IC::CONFIG.dig(:api, :interval),
|
|
96
|
+
interval_randomness: IC::CONFIG.dig(:api, :randomness),
|
|
97
|
+
backoff_factor: IC::CONFIG.dig(:api, :backoff),
|
|
98
|
+
exceptions: [ Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ClientError ]
|
|
95
99
|
f.request :url_encoded
|
|
96
100
|
|
|
97
|
-
if
|
|
98
|
-
f.response :logger,
|
|
101
|
+
if IC::logger.level == ::Logger::DEBUG
|
|
102
|
+
f.response :logger, IC::logger, bodies: true, headers: true
|
|
99
103
|
end
|
|
100
104
|
|
|
101
105
|
f.adapter Faraday::default_adapter
|
|
@@ -107,3 +111,22 @@ module INatChannel
|
|
|
107
111
|
end
|
|
108
112
|
|
|
109
113
|
end
|
|
114
|
+
|
|
115
|
+
module IC
|
|
116
|
+
|
|
117
|
+
def load_news
|
|
118
|
+
INatChannel::API::load_news
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def load_observation uuid
|
|
122
|
+
INatChannel::API::load_observation uuid
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def load_list **query
|
|
126
|
+
INatChannel::API::load_list(**query)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
module_function :load_news, :load_observation
|
|
130
|
+
|
|
131
|
+
end
|
|
132
|
+
|
data/lib/inat-channel/config.rb
CHANGED
|
@@ -23,9 +23,9 @@ module INatChannel
|
|
|
23
23
|
cfg.merge! options
|
|
24
24
|
cfg[:log_level] ||= :warn
|
|
25
25
|
env = load_env
|
|
26
|
-
cfg
|
|
26
|
+
cfg[:tg_bot] ||= {}
|
|
27
|
+
cfg[:tg_bot].merge! env
|
|
27
28
|
validate_and_fix_config! cfg
|
|
28
|
-
cfg
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
def parse_options
|
|
@@ -33,7 +33,7 @@ module INatChannel
|
|
|
33
33
|
OptionParser.new do |opts|
|
|
34
34
|
opts.banner = 'Usage: inat-channel [options]'
|
|
35
35
|
opts.on '-c', '--config FILE', 'Config file (default: inat-channel.yml)' do |v|
|
|
36
|
-
raise
|
|
36
|
+
raise "Config file not found: #{v}" unless File.exist?(v)
|
|
37
37
|
options[:config] = v
|
|
38
38
|
end
|
|
39
39
|
opts.on '-l', '--log-level LEVEL', [:debug, :info, :warn, :error], 'Log level (default: warn)' do |v|
|
|
@@ -43,7 +43,7 @@ module INatChannel
|
|
|
43
43
|
options[:log_level] = :debug
|
|
44
44
|
end
|
|
45
45
|
opts.on '--version', 'Show version info and exit' do
|
|
46
|
-
puts
|
|
46
|
+
puts IC::VERSION
|
|
47
47
|
exit
|
|
48
48
|
end
|
|
49
49
|
opts.on '-h', '--help', 'Show help and exit' do
|
|
@@ -68,27 +68,66 @@ module INatChannel
|
|
|
68
68
|
|
|
69
69
|
def load_env
|
|
70
70
|
{
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
token: (ENV['TELEGRAM_BOT_TOKEN'] or raise 'TELEGRAM_BOT_TOKEN required'),
|
|
72
|
+
admin_id: (ENV['ADMIN_TELEGRAM_ID'] or raise 'ADMIN_TELEGRAM_ID required')
|
|
73
73
|
}
|
|
74
74
|
end
|
|
75
75
|
|
|
76
76
|
def validate_and_fix_config! cfg
|
|
77
77
|
raise 'Missing or invalid base_query' unless Hash === cfg[:base_query]
|
|
78
|
-
raise 'Missing or invalid days_back' unless Integer === cfg
|
|
79
|
-
raise 'Missing chat_id' unless cfg
|
|
78
|
+
raise 'Missing or invalid days_back' unless Integer === cfg.dig(:days_back, :fresh) && cfg.dig(:days_back, :fresh)
|
|
79
|
+
raise 'Missing chat_id' unless cfg.dig(:tg_bot, :chat_id)
|
|
80
80
|
|
|
81
81
|
basename = File.basename cfg[:config], '.*'
|
|
82
|
-
cfg[:
|
|
83
|
-
cfg[:
|
|
84
|
-
cfg[:
|
|
85
|
-
cfg[:
|
|
82
|
+
cfg[:data_files] ||= {}
|
|
83
|
+
cfg[:data_files][:root] ||= 'data'
|
|
84
|
+
cfg[:data_files][:pool] ||= "#{ cfg[:data_files][:root] }/#{ basename }_pool.json"
|
|
85
|
+
cfg[:data_files][:sent] ||= "#{ cfg[:data_files][:root] }/#{ basename }_sent.json"
|
|
86
|
+
cfg[:data_files][:used] ||= "#{ cfg[:data_files][:root] }/#{ basename }_used.json"
|
|
87
|
+
|
|
88
|
+
cfg[:lock_file] ||= {}
|
|
89
|
+
cfg[:lock_file][:path] ||= "#{ cfg[:data_files][:root] }/#{ basename }__bot.lock"
|
|
90
|
+
cfg[:lock_file][:ttl] ||= 300 # 5 min
|
|
91
|
+
|
|
92
|
+
cfg[:days_back][:pool] ||= 3 * cfg.dig(:days_back, :fresh)
|
|
93
|
+
cfg[:days_back][:sent] ||= cfg[:days_back][:pool] + 1
|
|
94
|
+
cfg[:days_back][:used] ||= 365
|
|
95
|
+
|
|
96
|
+
cfg[:api] ||= {}
|
|
97
|
+
cfg[:api][:retries] ||= 5
|
|
98
|
+
cfg[:api][:interval] ||= 1.0
|
|
99
|
+
cfg[:api][:randomness] ||= 0.5
|
|
100
|
+
cfg[:api][:backoff] ||= 2
|
|
101
|
+
cfg[:api][:page_delay] ||= 1.0
|
|
102
|
+
cfg[:api][:per_page] ||= 200
|
|
103
|
+
|
|
104
|
+
cfg[:tg_bot][:retries] ||= 5
|
|
105
|
+
cfg[:tg_bot][:interval] ||= 1.0
|
|
106
|
+
cfg[:tg_bot][:randomness] ||= 0.5
|
|
107
|
+
cfg[:tg_bot][:backoff] ||= 2
|
|
108
|
+
cfg[:tg_bot][:desc_limit] ||= 512
|
|
109
|
+
cfg[:tg_bot][:link_zoom] ||= 12
|
|
110
|
+
|
|
111
|
+
cfg[:unique_taxon] ||= :ignore
|
|
112
|
+
cfg[:unique_taxon] = cfg[:unique_taxon].to_sym
|
|
113
|
+
|
|
114
|
+
cfg[:log_level] ||= :warn
|
|
115
|
+
cfg[:log_level] = cfg[:log_level].to_sym
|
|
116
|
+
cfg[:notify_level] ||= :warn
|
|
117
|
+
cfg[:notify_level] = cfg[:notify_level].to_sym
|
|
118
|
+
|
|
119
|
+
cfg
|
|
86
120
|
end
|
|
87
121
|
|
|
88
122
|
end
|
|
89
123
|
|
|
90
124
|
end
|
|
91
125
|
|
|
92
|
-
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
module IC
|
|
129
|
+
|
|
130
|
+
CONFIG = INatChannel::Config::config
|
|
93
131
|
|
|
94
132
|
end
|
|
133
|
+
|
data/lib/inat-channel/data.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require 'time'
|
|
2
|
+
require 'date'
|
|
2
3
|
require 'json'
|
|
3
4
|
require 'fileutils'
|
|
4
5
|
require 'set'
|
|
@@ -9,80 +10,288 @@ require_relative 'lock'
|
|
|
9
10
|
|
|
10
11
|
module INatChannel
|
|
11
12
|
|
|
12
|
-
module
|
|
13
|
+
module Storage
|
|
13
14
|
|
|
14
15
|
class << self
|
|
15
16
|
|
|
16
17
|
def select_uuid fresh
|
|
17
|
-
|
|
18
|
+
# 1. Отфильтровываем уже отправленные
|
|
19
|
+
fresh.reject! { |rec| sent?(rec[:uuid]) }
|
|
18
20
|
|
|
19
|
-
|
|
21
|
+
# 2. Если нужно, отфильтровываем по таксонам
|
|
22
|
+
taxon_uniq = IC::CONFIG[:unique_taxon]
|
|
23
|
+
fresh.reject! { |rec| used?(rec.dig :taxon, :id) } if taxon_uniq == :strict
|
|
24
|
+
|
|
25
|
+
if taxon_uniq == :priority
|
|
26
|
+
uniq_fresh = fresh.reject { |rec| used?(rec.dig :taxon, :id) }
|
|
27
|
+
unless uniq_fresh.empty?
|
|
28
|
+
IC::logger.info "Take a fresh & unique (from #{uniq_fresh.size})"
|
|
29
|
+
sample = sample_with_weight uniq_fresh, field: :faves_count
|
|
30
|
+
fresh.reject { |rec| rec[:uuid] == sample[:uuid] }.each do |rec|
|
|
31
|
+
pool[rec[:uuid]] = {
|
|
32
|
+
'created_at' => Date.parse(rec[:created_at]),
|
|
33
|
+
'faves_count' => rec[:faves_count],
|
|
34
|
+
'taxon_id' => rec.dig(:taxon, :id)
|
|
35
|
+
}
|
|
36
|
+
end
|
|
37
|
+
sample[:taxon_id] = sample.dig :taxon, :id
|
|
38
|
+
@in_process = sample
|
|
39
|
+
return sample[:uuid]
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# 3. Если добрались до этого места, берем просто из свежих
|
|
20
44
|
unless fresh.empty?
|
|
21
|
-
|
|
22
|
-
fresh
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
45
|
+
IC::logger.info "Take a fresh (from #{fresh.size})"
|
|
46
|
+
sample = sample_with_weight fresh, field: :faves_count
|
|
47
|
+
fresh.reject { |rec| rec[:uuid] == sample[:uuid] }.each do |rec|
|
|
48
|
+
pool[rec[:uuid]] = {
|
|
49
|
+
"created_at" => Date.parse(rec[:created_at]),
|
|
50
|
+
"faves_count" => rec[:faves_count],
|
|
51
|
+
"taxon_id" => rec.dig(:taxon, :id),
|
|
52
|
+
}
|
|
53
|
+
end
|
|
54
|
+
sample[:taxon_id] = sample.dig :taxon, :id
|
|
55
|
+
@in_process = sample
|
|
56
|
+
return sample[:uuid]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# 4. Если ..., проверяем приоритет уникальных уже для пула
|
|
60
|
+
pool_records = pool.map { |k, v| v.merge({ 'uuid' => k }) }
|
|
61
|
+
if taxon_uniq == :priority
|
|
62
|
+
uniq_pool = pool_records.reject { |rec| used?(rec['taxon_id']) }
|
|
63
|
+
unless uniq_pool.empty?
|
|
64
|
+
IC::logger.info "Take an unique pool record (from #{uniq_pool.size})"
|
|
65
|
+
sample = sample_with_weight uniq_pool, field: 'faves_count'
|
|
66
|
+
sample.transform_keys!(&:to_sym)
|
|
67
|
+
@in_process = sample
|
|
68
|
+
return sample[:uuid]
|
|
69
|
+
end
|
|
26
70
|
end
|
|
27
71
|
|
|
28
|
-
|
|
29
|
-
unless
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
72
|
+
# 5. Если ..., берем из пула
|
|
73
|
+
unless pool_records.empty?
|
|
74
|
+
IC::logger.info "Take a pool record (from #{pool.size})"
|
|
75
|
+
sample = sample_with_weight pool_records, field: 'faves_count'
|
|
76
|
+
sample.transform_keys!(&:to_sym)
|
|
77
|
+
@in_process = sample
|
|
78
|
+
return sample[:uuid]
|
|
34
79
|
end
|
|
35
80
|
|
|
36
|
-
nil
|
|
81
|
+
return nil
|
|
37
82
|
end
|
|
38
83
|
|
|
39
|
-
def
|
|
40
|
-
|
|
84
|
+
def confirm! msg_id
|
|
85
|
+
# Отправка выполнена, фиксируем
|
|
86
|
+
sent[@in_process[:uuid]] = {
|
|
87
|
+
'msg_id' => msg_id,
|
|
88
|
+
'sent_at' => Date.today
|
|
89
|
+
}
|
|
90
|
+
used[@in_process[:taxon_id]] = Date.today
|
|
41
91
|
end
|
|
42
92
|
|
|
43
|
-
def
|
|
44
|
-
|
|
93
|
+
def revert!
|
|
94
|
+
# Отправка не выполнена, возвращаем рабочую запись в пул
|
|
95
|
+
pool[@in_process[:uuid]] = {
|
|
96
|
+
'created_at' => @in_process[:created_at],
|
|
97
|
+
'faves_count' => @in_process[:faves_count],
|
|
98
|
+
'taxon_id' => @in_process[:taxon_id]
|
|
99
|
+
}
|
|
45
100
|
end
|
|
46
101
|
|
|
47
102
|
def save
|
|
48
103
|
save_pool
|
|
49
104
|
save_sent
|
|
50
|
-
|
|
105
|
+
save_used
|
|
106
|
+
IC::logger.info "Saved pool=#{pool.size}, sent=#{sent.size}"
|
|
51
107
|
end
|
|
52
108
|
|
|
53
109
|
private
|
|
54
110
|
|
|
111
|
+
def used
|
|
112
|
+
@used ||= load_used
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def pool
|
|
116
|
+
@pool ||= load_pool
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def sent
|
|
120
|
+
@sent ||= load_sent
|
|
121
|
+
end
|
|
122
|
+
|
|
55
123
|
def sent? uuid
|
|
56
124
|
sent.has_key? uuid
|
|
57
125
|
end
|
|
58
126
|
|
|
127
|
+
def used? taxon_id
|
|
128
|
+
used.has_key? taxon_id
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def sample_with_weight source, field:
|
|
132
|
+
data = []
|
|
133
|
+
source.each do |rec|
|
|
134
|
+
num = rec[field]
|
|
135
|
+
num = 1 unless Integer === num && num > 0
|
|
136
|
+
num.times { data << rec }
|
|
137
|
+
end
|
|
138
|
+
data.sample
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def reload_pool old
|
|
142
|
+
result = {}
|
|
143
|
+
old.each_slice 100 do |items|
|
|
144
|
+
records = IC::load_list uuid: items.join(',')
|
|
145
|
+
records.each do |rec|
|
|
146
|
+
result[rec[:uuid]] = {
|
|
147
|
+
'created_at' => Date.parse(rec[:created_at]),
|
|
148
|
+
'faves_count' => rec[:faves_count],
|
|
149
|
+
'taxon_id' => rec.dig(:taxon, :id)
|
|
150
|
+
}
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
result
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def fetch_new_pool
|
|
157
|
+
result = {}
|
|
158
|
+
dead_date = Date.today - IC::CONFIG.dig(:days_back, :pool)
|
|
159
|
+
records = IC::load_list(**IC::CONFIG[:base_query], created_d1: dead_date.to_s)
|
|
160
|
+
records.each do |rec|
|
|
161
|
+
result[rec[:uuid]] = {
|
|
162
|
+
'created_at' => Date.parse(rec[:created_at]),
|
|
163
|
+
'faves_count' => rec[:faves_count],
|
|
164
|
+
'taxon_id' => rec.dig(:taxon, :id)
|
|
165
|
+
}
|
|
166
|
+
end
|
|
167
|
+
result
|
|
168
|
+
end
|
|
169
|
+
|
|
59
170
|
def load_pool
|
|
60
|
-
file =
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
171
|
+
file = IC::CONFIG.dig(:data_files, :pool)
|
|
172
|
+
if File.exist?(file)
|
|
173
|
+
data = JSON.parse File.read(file), symbolize_names: false
|
|
174
|
+
case data
|
|
175
|
+
when Hash
|
|
176
|
+
data.each do |_, value|
|
|
177
|
+
begin
|
|
178
|
+
value['created_at'] = Date.parse(value['created_at'])
|
|
179
|
+
rescue => e
|
|
180
|
+
IC::logger.error "Error in pool: #{e.message}"
|
|
181
|
+
IC::logger.debug "Value: #{value.inspect}"
|
|
182
|
+
value['created_at'] = Date.today
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
data
|
|
186
|
+
when Array
|
|
187
|
+
IC::logger.warn "❗ Old format of pool ❗"
|
|
188
|
+
IC::notify_admin "❗ Old format of pool ❗"
|
|
189
|
+
reload_pool data
|
|
190
|
+
else
|
|
191
|
+
IC::notify_admin "❌ Unknown format of pool"
|
|
192
|
+
raise "❌ Unknown format of pool"
|
|
193
|
+
end
|
|
194
|
+
else
|
|
195
|
+
fetch_new_pool
|
|
196
|
+
end
|
|
197
|
+
rescue => e
|
|
198
|
+
IC::notify_admin '❌ ' + e.message
|
|
199
|
+
raise e
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def load_used
|
|
203
|
+
file = IC::CONFIG.dig(:data_files, :used)
|
|
204
|
+
if File.exist?(file)
|
|
205
|
+
data = JSON.parse File.read(file), symbolize_names: false
|
|
206
|
+
data.transform_keys!(&:to_i)
|
|
207
|
+
data.transform_values! do |value|
|
|
208
|
+
begin
|
|
209
|
+
Date.parse value
|
|
210
|
+
rescue => e
|
|
211
|
+
IC::logger.error "Error in used: #{e.message}"
|
|
212
|
+
IC::logger.debug "Value: #{value.inspect}"
|
|
213
|
+
Date.today
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
data
|
|
217
|
+
else
|
|
218
|
+
{}
|
|
219
|
+
end
|
|
220
|
+
rescue => e
|
|
221
|
+
IC::notify_admin '❌ ' + e.message
|
|
222
|
+
raise e
|
|
66
223
|
end
|
|
67
224
|
|
|
68
225
|
def load_sent
|
|
69
|
-
file =
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
226
|
+
file = IC::CONFIG.dig(:data_files, :sent)
|
|
227
|
+
if File.exist?(file)
|
|
228
|
+
data = JSON.parse File.read(file), symbolize_names: false
|
|
229
|
+
raise "Invalid format of sent file" unless Hash === data
|
|
230
|
+
# Чистим чего набажили...
|
|
231
|
+
data.each do |_, value|
|
|
232
|
+
begin
|
|
233
|
+
value['sent_at'] = Date.parse(value['sent_at'])
|
|
234
|
+
rescue => e
|
|
235
|
+
IC::logger.error "Error in sent: #{e.message}"
|
|
236
|
+
IC::logger.debug "Value: #{value.inspect}"
|
|
237
|
+
value['sent_at'] = Date.today
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
data
|
|
241
|
+
else
|
|
242
|
+
{}
|
|
243
|
+
end
|
|
244
|
+
rescue => e
|
|
245
|
+
IC::notify_admin '❌ ' + e.message
|
|
246
|
+
raise e
|
|
75
247
|
end
|
|
76
248
|
|
|
77
249
|
def save_pool
|
|
78
|
-
pool.
|
|
79
|
-
|
|
250
|
+
size = pool.size
|
|
251
|
+
|
|
252
|
+
# 1. Удаляем отправленные
|
|
253
|
+
pool.reject! { |uuid, _| sent?(uuid) }
|
|
254
|
+
IC::logger.info "Removed #{size - pool.size} sent records from pool" if pool.size != size
|
|
255
|
+
size = pool.size
|
|
256
|
+
|
|
257
|
+
# 2. Удаляем использованные
|
|
258
|
+
taxon_uniq = IC::CONFIG[:unique_taxon]
|
|
259
|
+
pool.reject! { |_, value| used?(value['taxon_id']) } if taxon_uniq == :strict || Integer === taxon_uniq
|
|
260
|
+
IC::logger.info "Removed #{size - pool.size} used records from pool" if pool.size != size
|
|
261
|
+
size = pool.size
|
|
262
|
+
|
|
263
|
+
# 3. Удаляем устаревшие
|
|
264
|
+
dead_date = Date.today - IC::CONFIG.dig(:days_back, :pool)
|
|
265
|
+
pool.reject! { |_, value| value['created_at'] < dead_date }
|
|
266
|
+
IC::logger.info "Removed #{size - pool.size} outdated records from pool" if pool.size != size
|
|
267
|
+
|
|
268
|
+
file = IC::CONFIG.dig(:data_files, :pool)
|
|
269
|
+
FileUtils.mkdir_p File.dirname(file)
|
|
270
|
+
File.write file, JSON.pretty_generate(pool)
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def save_used
|
|
274
|
+
size = used.size
|
|
275
|
+
|
|
276
|
+
# Удаляем устаревшие, если актуально
|
|
277
|
+
dead_date = Date.today - IC::CONFIG.dig(:days_back, :used)
|
|
278
|
+
used.reject! { |_, value| value < dead_date }
|
|
279
|
+
IC::logger.info "Removed #{size - used.size} outdated records from used" if used.size != size
|
|
280
|
+
|
|
281
|
+
file = IC::CONFIG.dig(:data_files, :used)
|
|
80
282
|
FileUtils.mkdir_p File.dirname(file)
|
|
81
|
-
File.write file, JSON.pretty_generate(
|
|
283
|
+
File.write file, JSON.pretty_generate(used)
|
|
82
284
|
end
|
|
83
285
|
|
|
84
286
|
def save_sent
|
|
85
|
-
|
|
287
|
+
size = sent.size
|
|
288
|
+
|
|
289
|
+
# Удаляем устаревшие
|
|
290
|
+
dead_date = Date.today - IC::CONFIG.dig(:days_back, :sent)
|
|
291
|
+
sent.reject! { |_, value| value['sent_at'] < dead_date }
|
|
292
|
+
IC::logger.info "Removed #{size - sent.size} outdated records from sent" if sent.size != size
|
|
293
|
+
|
|
294
|
+
file = IC::CONFIG.dig(:data_files, :sent)
|
|
86
295
|
FileUtils.mkdir_p File.dirname(file)
|
|
87
296
|
File.write file, JSON.pretty_generate(sent)
|
|
88
297
|
end
|
|
@@ -92,3 +301,25 @@ module INatChannel
|
|
|
92
301
|
end
|
|
93
302
|
|
|
94
303
|
end
|
|
304
|
+
|
|
305
|
+
module IC
|
|
306
|
+
|
|
307
|
+
def select_uuid fresh
|
|
308
|
+
INatChannel::Storage::select_uuid fresh
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
def save_data
|
|
312
|
+
INatChannel::Storage::save
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def confirm_sending! msg_id
|
|
316
|
+
INatChannel::Storage::confirm! msg_id
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def revert_sending!
|
|
320
|
+
INatChannel::Storage::revert!
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
module_function :select_uuid, :save_data, :confirm_sending!, :revert_sending!
|
|
324
|
+
|
|
325
|
+
end
|
|
@@ -11,16 +11,22 @@ module INatChannel
|
|
|
11
11
|
class << self
|
|
12
12
|
|
|
13
13
|
def convert_observation observation_source
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
14
|
+
begin
|
|
15
|
+
id = observation_source[:id]
|
|
16
|
+
url = observation_source[:uri]
|
|
17
|
+
uuid = observation_source[:uuid]
|
|
18
|
+
user = convert_user observation_source[:user]
|
|
19
|
+
taxon = convert_taxon observation_source[:taxon], observation_source[:identifications]
|
|
20
|
+
places = convert_places observation_source[:place_ids]
|
|
21
|
+
datetime = DateTime.parse(observation_source[:time_observed_at] || observation_source[:observed_on_string])
|
|
22
|
+
location = convert_location observation_source[:geojson]
|
|
23
|
+
description = convert_description observation_source[:description]
|
|
24
|
+
place_guess = observation_source[:place_guess]
|
|
25
|
+
rescue => e
|
|
26
|
+
IC::logger.error e.full_message
|
|
27
|
+
IC::logger.info JSON.pretty_generate(observation_source)
|
|
28
|
+
raise e
|
|
29
|
+
end
|
|
24
30
|
Observation::new id: id, url: url, uuid: uuid, user: user, taxon: taxon, places: places, datetime: datetime, location: location, description: description, place_guess: place_guess
|
|
25
31
|
end
|
|
26
32
|
|
|
@@ -70,7 +76,7 @@ module INatChannel
|
|
|
70
76
|
end
|
|
71
77
|
|
|
72
78
|
def convert_places place_ids
|
|
73
|
-
places_config =
|
|
79
|
+
places_config = IC::CONFIG[:places]
|
|
74
80
|
return nil unless places_config
|
|
75
81
|
result = []
|
|
76
82
|
places_config.each do |_, items|
|
|
@@ -90,3 +96,13 @@ module INatChannel
|
|
|
90
96
|
end
|
|
91
97
|
|
|
92
98
|
end
|
|
99
|
+
|
|
100
|
+
module IC
|
|
101
|
+
|
|
102
|
+
def convert_observation source
|
|
103
|
+
INatChannel::DataConvert::convert_observation source
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
module_function :convert_observation
|
|
107
|
+
|
|
108
|
+
end
|