inat-channel 0.8.2 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ module INatChannel
15
15
  PER_PAGE = 200
16
16
  PAGE_DELAY = 1.0
17
17
  API_ENDPOINT = 'https://api.inaturalist.org/v2/observations'
18
- LIST_FIELDS = 'uuid'
18
+ LIST_FIELDS = '(uuid:!t,created_at:!t,faves_count:!t,taxon:(id:!t))'
19
19
  SINGLE_FIELDS = '(id:!t,uuid:!t,uri:!t,geojson:(all:!t),user:(login:!t,name:!t),taxon:(ancestor_ids:!t,preferred_common_name:!t,name:!t),' +
20
20
  'place_ids:!t,place_guess:!t,observed_on_string:!t,description:!t,photos:(url:!t),time_observed_at:!t,' +
21
21
  'identifications:(taxon:(ancestors:(name:!t))))'
@@ -23,66 +23,66 @@ module INatChannel
23
23
  private_constant :PER_PAGE, :PAGE_DELAY, :API_ENDPOINT, :LIST_FIELDS, :SINGLE_FIELDS
24
24
 
25
25
  def load_news
26
+ load_list(**IC::CONFIG[:base_query], updated_since: (Date.today - IC::CONFIG.dig(:days_back, :fresh)).to_s)
27
+ end
28
+
29
+ def load_list **query
26
30
  result = []
27
31
  page = 1
28
-
32
+
29
33
  loop do
30
- INatChannel::LOGGER.debug "Fetch page #{page} with per_page=#{PER_PAGE}"
34
+ IC::logger.debug "Fetch page #{page} with per_page=#{PER_PAGE}"
31
35
 
32
36
  response = faraday.get API_ENDPOINT do |req|
33
- req.params['page'] = page
34
- req.params['per_page'] = PER_PAGE
35
- req.params['fields'] = LIST_FIELDS
36
- req.params.merge! INatChannel::CONFIG[:base_query]
37
- req.params['created_d1'] = (Date.today - INatChannel::CONFIG[:days_back]).to_s
37
+ req.params[:page] = page
38
+ req.params[:per_page] = PER_PAGE
39
+ req.params[:fields] = LIST_FIELDS
40
+ req.params.merge! query
38
41
  end
39
42
 
40
43
  unless response.success?
41
- INatChannel::Telegram::notify_admin "Failed to fetch observations page #{page}: HTTP #{response.status}"
42
- INatChannel::LOGGER.error "HTTP #{response.status} on page #{page}"
44
+ IC::logger.error "Failed to fetch observations page #{page}: HTTP #{response.status}"
45
+ IC::notify_admin " Failed to fetch observations page #{page}: HTTP #{response.status}"
43
46
  break
44
47
  end
45
48
 
46
49
  data = JSON.parse response.body, symbolize_names: true
47
- uuids = data[:results].map { |o| o[:uuid] }
48
- result += uuids
50
+ result += data[:results]
49
51
 
50
52
  total = data[:total_results] || 0
51
- INatChannel::LOGGER.debug "Page #{page}: fetched #{uuids.size} UUIDs, total expected #{total}"
52
-
53
- break if uuids.empty? || result.size >= total
53
+ IC::logger.debug "Page #{page}: fetched #{data.size} records, total expected #{total}"
54
+ break if data.empty? || result.size >= total
54
55
  page += 1
55
56
  sleep PAGE_DELAY
56
57
  end
57
58
 
58
- INatChannel::LOGGER.debug "Loaded total #{result.uniq.size} unique UUIDs"
59
- result.uniq
59
+ IC::logger.debug "Loaded total #{result.size} records"
60
+ result
60
61
  rescue => e
61
- INatChannel::Telegram::notify_admin "Exception while loading news: #{e.message}"
62
- INatChannel::LOGGER.error e.full_message
63
- []
62
+ IC::logger.error "Exception while loading news: #{e.full_message}"
63
+ IC::notify_admin "❌ Exception while loading news: #{e.message}"
64
64
  end
65
65
 
66
66
  def load_observation uuid
67
67
  response = faraday.get API_ENDPOINT do |req|
68
68
  req.params['uuid'] = uuid
69
- req.params['locale'] = INatChannel::CONFIG[:base_query][:locale] if INatChannel::CONFIG[:base_query][:locale]
69
+ req.params['locale'] = IC::CONFIG[:base_query][:locale] if IC::CONFIG[:base_query][:locale]
70
70
  req.params['fields'] = SINGLE_FIELDS
71
71
  end
72
72
 
73
73
  if response.success?
74
74
  data = JSON.parse response.body, symbolize_names: true
75
75
  obs = data[:results]&.first
76
- INatChannel::LOGGER.debug "Loaded observation: #{uuid}"
76
+ IC::logger.debug "Loaded observation: #{uuid}"
77
77
  obs
78
78
  else
79
- INatChannel::LOGGER.error "Error loading observation #{uuid}: HTTP #{response.status}"
80
- INatChannel::Telegram::notify_admin "Error loading observation #{uuid}: HTTP #{response.status}"
79
+ IC::logger.error "Error loading observation #{uuid}: HTTP #{response.status}"
80
+ IC::notify_admin "Error loading observation #{uuid}: HTTP #{response.status}"
81
81
  nil
82
82
  end
83
83
  rescue => e
84
- INatChannel::Telegram::notify_admin "Exception while loading observation #{uuid}: #{e.message}"
85
- INatChannel::LOGGER.error e.full_message
84
+ IC::notify_admin "Exception while loading observation #{uuid}: #{e.message}"
85
+ IC::logger.error e.full_message
86
86
  nil
87
87
  end
88
88
 
@@ -90,12 +90,16 @@ module INatChannel
90
90
 
91
91
  def faraday
92
92
  @faraday ||= Faraday::new do |f|
93
- f.request :retry, max: INatChannel::CONFIG[:retries], interval: 2.0, interval_randomness: 0.5,
94
- exceptions: [ Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ClientError ]
93
+ f.request :retry,
94
+ max: IC::CONFIG.dig(:api, :retries),
95
+ interval: IC::CONFIG.dig(:api, :interval),
96
+ interval_randomness: IC::CONFIG.dig(:api, :randomness),
97
+ backoff_factor: IC::CONFIG.dig(:api, :backoff),
98
+ exceptions: [ Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ClientError ]
95
99
  f.request :url_encoded
96
100
 
97
- if INatChannel::LOGGER.level == ::Logger::DEBUG
98
- f.response :logger, INatChannel::LOGGER, bodies: true, headers: true
101
+ if IC::logger.level == ::Logger::DEBUG
102
+ f.response :logger, IC::logger, bodies: true, headers: true
99
103
  end
100
104
 
101
105
  f.adapter Faraday::default_adapter
@@ -107,3 +111,22 @@ module INatChannel
107
111
  end
108
112
 
109
113
  end
114
+
115
+ module IC
116
+
117
+ def load_news
118
+ INatChannel::API::load_news
119
+ end
120
+
121
+ def load_observation uuid
122
+ INatChannel::API::load_observation uuid
123
+ end
124
+
125
+ def load_list **query
126
+ INatChannel::API::load_list(**query)
127
+ end
128
+
129
+ module_function :load_news, :load_observation
130
+
131
+ end
132
+
@@ -23,9 +23,9 @@ module INatChannel
23
23
  cfg.merge! options
24
24
  cfg[:log_level] ||= :warn
25
25
  env = load_env
26
- cfg.merge! env
26
+ cfg[:tg_bot] ||= {}
27
+ cfg[:tg_bot].merge! env
27
28
  validate_and_fix_config! cfg
28
- cfg
29
29
  end
30
30
 
31
31
  def parse_options
@@ -33,7 +33,7 @@ module INatChannel
33
33
  OptionParser.new do |opts|
34
34
  opts.banner = 'Usage: inat-channel [options]'
35
35
  opts.on '-c', '--config FILE', 'Config file (default: inat-channel.yml)' do |v|
36
- raise ArgumentError, "Config file not found: #{v}" unless File.exist?(v)
36
+ raise "Config file not found: #{v}" unless File.exist?(v)
37
37
  options[:config] = v
38
38
  end
39
39
  opts.on '-l', '--log-level LEVEL', [:debug, :info, :warn, :error], 'Log level (default: warn)' do |v|
@@ -43,7 +43,7 @@ module INatChannel
43
43
  options[:log_level] = :debug
44
44
  end
45
45
  opts.on '--version', 'Show version info and exit' do
46
- puts INatChannel::VERSION
46
+ puts IC::VERSION
47
47
  exit
48
48
  end
49
49
  opts.on '-h', '--help', 'Show help and exit' do
@@ -68,27 +68,66 @@ module INatChannel
68
68
 
69
69
  def load_env
70
70
  {
71
- telegram_bot_token: (ENV['TELEGRAM_BOT_TOKEN'] or raise 'TELEGRAM_BOT_TOKEN required'),
72
- admin_telegram_id: (ENV['ADMIN_TELEGRAM_ID'] or raise 'ADMIN_TELEGRAM_ID required')
71
+ token: (ENV['TELEGRAM_BOT_TOKEN'] or raise 'TELEGRAM_BOT_TOKEN required'),
72
+ admin_id: (ENV['ADMIN_TELEGRAM_ID'] or raise 'ADMIN_TELEGRAM_ID required')
73
73
  }
74
74
  end
75
75
 
76
76
  def validate_and_fix_config! cfg
77
77
  raise 'Missing or invalid base_query' unless Hash === cfg[:base_query]
78
- raise 'Missing or invalid days_back' unless Integer === cfg[:days_back] && cfg[:days_back] > 0
79
- raise 'Missing chat_id' unless cfg[:chat_id]
78
+ raise 'Missing or invalid days_back' unless Integer === cfg.dig(:days_back, :fresh) && cfg.dig(:days_back, :fresh)
79
+ raise 'Missing chat_id' unless cfg.dig(:tg_bot, :chat_id)
80
80
 
81
81
  basename = File.basename cfg[:config], '.*'
82
- cfg[:pool_file] ||= "./data/#{basename}_pool.json"
83
- cfg[:sent_file] ||= "./data/#{basename}_sent.json"
84
- cfg[:lock_file] ||= "./data/#{basename}__bot.lock"
85
- cfg[:retries] ||= 5
82
+ cfg[:data_files] ||= {}
83
+ cfg[:data_files][:root] ||= 'data'
84
+ cfg[:data_files][:pool] ||= "#{ cfg[:data_files][:root] }/#{ basename }_pool.json"
85
+ cfg[:data_files][:sent] ||= "#{ cfg[:data_files][:root] }/#{ basename }_sent.json"
86
+ cfg[:data_files][:used] ||= "#{ cfg[:data_files][:root] }/#{ basename }_used.json"
87
+
88
+ cfg[:lock_file] ||= {}
89
+ cfg[:lock_file][:path] ||= "#{ cfg[:data_files][:root] }/#{ basename }__bot.lock"
90
+ cfg[:lock_file][:ttl] ||= 300 # 5 min
91
+
92
+ cfg[:days_back][:pool] ||= 3 * cfg.dig(:days_back, :fresh)
93
+ cfg[:days_back][:sent] ||= cfg[:days_back][:pool] + 1
94
+ cfg[:days_back][:used] ||= 365
95
+
96
+ cfg[:api] ||= {}
97
+ cfg[:api][:retries] ||= 5
98
+ cfg[:api][:interval] ||= 1.0
99
+ cfg[:api][:randomness] ||= 0.5
100
+ cfg[:api][:backoff] ||= 2
101
+ cfg[:api][:page_delay] ||= 1.0
102
+ cfg[:api][:per_page] ||= 200
103
+
104
+ cfg[:tg_bot][:retries] ||= 5
105
+ cfg[:tg_bot][:interval] ||= 1.0
106
+ cfg[:tg_bot][:randomness] ||= 0.5
107
+ cfg[:tg_bot][:backoff] ||= 2
108
+ cfg[:tg_bot][:desc_limit] ||= 512
109
+ cfg[:tg_bot][:link_zoom] ||= 12
110
+
111
+ cfg[:unique_taxon] ||= :ignore
112
+ cfg[:unique_taxon] = cfg[:unique_taxon].to_sym
113
+
114
+ cfg[:log_level] ||= :warn
115
+ cfg[:log_level] = cfg[:log_level].to_sym
116
+ cfg[:notify_level] ||= :warn
117
+ cfg[:notify_level] = cfg[:notify_level].to_sym
118
+
119
+ cfg
86
120
  end
87
121
 
88
122
  end
89
123
 
90
124
  end
91
125
 
92
- CONFIG = Config::config
126
+ end
127
+
128
+ module IC
129
+
130
+ CONFIG = INatChannel::Config::config
93
131
 
94
132
  end
133
+
@@ -1,4 +1,5 @@
1
1
  require 'time'
2
+ require 'date'
2
3
  require 'json'
3
4
  require 'fileutils'
4
5
  require 'set'
@@ -9,80 +10,288 @@ require_relative 'lock'
9
10
 
10
11
  module INatChannel
11
12
 
12
- module Data
13
+ module Storage
13
14
 
14
15
  class << self
15
16
 
16
17
  def select_uuid fresh
17
- INatChannel::LOGGER.info "Received #{fresh.size} uuids"
18
+ # 1. Отфильтровываем уже отправленные
19
+ fresh.reject! { |rec| sent?(rec[:uuid]) }
18
20
 
19
- fresh.reject! { |uuid| sent?(uuid) }
21
+ # 2. Если нужно, отфильтровываем по таксонам
22
+ taxon_uniq = IC::CONFIG[:unique_taxon]
23
+ fresh.reject! { |rec| used?(rec.dig :taxon, :id) } if taxon_uniq == :strict
24
+
25
+ if taxon_uniq == :priority
26
+ uniq_fresh = fresh.reject { |rec| used?(rec.dig :taxon, :id) }
27
+ unless uniq_fresh.empty?
28
+ IC::logger.info "Take a fresh & unique (from #{uniq_fresh.size})"
29
+ sample = sample_with_weight uniq_fresh, field: :faves_count
30
+ fresh.reject { |rec| rec[:uuid] == sample[:uuid] }.each do |rec|
31
+ pool[rec[:uuid]] = {
32
+ 'created_at' => Date.parse(rec[:created_at]),
33
+ 'faves_count' => rec[:faves_count],
34
+ 'taxon_id' => rec.dig(:taxon, :id)
35
+ }
36
+ end
37
+ sample[:taxon_id] = sample.dig :taxon, :id
38
+ @in_process = sample
39
+ return sample[:uuid]
40
+ end
41
+ end
42
+
43
+ # 3. Если добрались до этого места, берем просто из свежих
20
44
  unless fresh.empty?
21
- result = fresh.sample
22
- fresh.delete result
23
- pool.merge fresh
24
- INatChannel::LOGGER.info "Fresh uuid selected, #{fresh.size} uuids added to pool"
25
- return result
45
+ IC::logger.info "Take a fresh (from #{fresh.size})"
46
+ sample = sample_with_weight fresh, field: :faves_count
47
+ fresh.reject { |rec| rec[:uuid] == sample[:uuid] }.each do |rec|
48
+ pool[rec[:uuid]] = {
49
+ "created_at" => Date.parse(rec[:created_at]),
50
+ "faves_count" => rec[:faves_count],
51
+ "taxon_id" => rec.dig(:taxon, :id),
52
+ }
53
+ end
54
+ sample[:taxon_id] = sample.dig :taxon, :id
55
+ @in_process = sample
56
+ return sample[:uuid]
57
+ end
58
+
59
+ # 4. Если ..., проверяем приоритет уникальных уже для пула
60
+ pool_records = pool.map { |k, v| v.merge({ 'uuid' => k }) }
61
+ if taxon_uniq == :priority
62
+ uniq_pool = pool_records.reject { |rec| used?(rec['taxon_id']) }
63
+ unless uniq_pool.empty?
64
+ IC::logger.info "Take an unique pool record (from #{uniq_pool.size})"
65
+ sample = sample_with_weight uniq_pool, field: 'faves_count'
66
+ sample.transform_keys!(&:to_sym)
67
+ @in_process = sample
68
+ return sample[:uuid]
69
+ end
26
70
  end
27
71
 
28
- pool.reject! { |uuid| sent?(uuid) }
29
- unless pool.empty?
30
- result = pool.to_a.sample
31
- pool.delete result
32
- INatChannel::LOGGER.info "Pool uuid selected, #{pool.size} uuids remain in pool"
33
- return result
72
+ # 5. Если ..., берем из пула
73
+ unless pool_records.empty?
74
+ IC::logger.info "Take a pool record (from #{pool.size})"
75
+ sample = sample_with_weight pool_records, field: 'faves_count'
76
+ sample.transform_keys!(&:to_sym)
77
+ @in_process = sample
78
+ return sample[:uuid]
34
79
  end
35
80
 
36
- nil
81
+ return nil
37
82
  end
38
83
 
39
- def pool
40
- @pool ||= load_pool
84
+ def confirm! msg_id
85
+ # Отправка выполнена, фиксируем
86
+ sent[@in_process[:uuid]] = {
87
+ 'msg_id' => msg_id,
88
+ 'sent_at' => Date.today
89
+ }
90
+ used[@in_process[:taxon_id]] = Date.today
41
91
  end
42
92
 
43
- def sent
44
- @sent ||= load_sent
93
+ def revert!
94
+ # Отправка не выполнена, возвращаем рабочую запись в пул
95
+ pool[@in_process[:uuid]] = {
96
+ 'created_at' => @in_process[:created_at],
97
+ 'faves_count' => @in_process[:faves_count],
98
+ 'taxon_id' => @in_process[:taxon_id]
99
+ }
45
100
  end
46
101
 
47
102
  def save
48
103
  save_pool
49
104
  save_sent
50
- INatChannel::LOGGER.info "Saved pool=#{pool.size}, sent=#{sent.size}"
105
+ save_used
106
+ IC::logger.info "Saved pool=#{pool.size}, sent=#{sent.size}"
51
107
  end
52
108
 
53
109
  private
54
110
 
111
+ def used
112
+ @used ||= load_used
113
+ end
114
+
115
+ def pool
116
+ @pool ||= load_pool
117
+ end
118
+
119
+ def sent
120
+ @sent ||= load_sent
121
+ end
122
+
55
123
  def sent? uuid
56
124
  sent.has_key? uuid
57
125
  end
58
126
 
127
+ def used? taxon_id
128
+ used.has_key? taxon_id
129
+ end
130
+
131
+ def sample_with_weight source, field:
132
+ data = []
133
+ source.each do |rec|
134
+ num = rec[field]
135
+ num = 1 unless Integer === num && num > 0
136
+ num.times { data << rec }
137
+ end
138
+ data.sample
139
+ end
140
+
141
+ def reload_pool old
142
+ result = {}
143
+ old.each_slice 100 do |items|
144
+ records = IC::load_list uuid: items.join(',')
145
+ records.each do |rec|
146
+ result[rec[:uuid]] = {
147
+ 'created_at' => Date.parse(rec[:created_at]),
148
+ 'faves_count' => rec[:faves_count],
149
+ 'taxon_id' => rec.dig(:taxon, :id)
150
+ }
151
+ end
152
+ end
153
+ result
154
+ end
155
+
156
+ def fetch_new_pool
157
+ result = {}
158
+ dead_date = Date.today - IC::CONFIG.dig(:days_back, :pool)
159
+ records = IC::load_list(**IC::CONFIG[:base_query], created_d1: dead_date.to_s)
160
+ records.each do |rec|
161
+ result[rec[:uuid]] = {
162
+ 'created_at' => Date.parse(rec[:created_at]),
163
+ 'faves_count' => rec[:faves_count],
164
+ 'taxon_id' => rec.dig(:taxon, :id)
165
+ }
166
+ end
167
+ result
168
+ end
169
+
59
170
  def load_pool
60
- file = INatChannel::CONFIG[:pool_file]
61
- data = JSON.parse File.read(file), symbolize_names: false
62
- raise "Invalid format of pool file" unless Array === data
63
- Set[*data]
64
- rescue
65
- Set::new
171
+ file = IC::CONFIG.dig(:data_files, :pool)
172
+ if File.exist?(file)
173
+ data = JSON.parse File.read(file), symbolize_names: false
174
+ case data
175
+ when Hash
176
+ data.each do |_, value|
177
+ begin
178
+ value['created_at'] = Date.parse(value['created_at'])
179
+ rescue => e
180
+ IC::logger.error "Error in pool: #{e.message}"
181
+ IC::logger.debug "Value: #{value.inspect}"
182
+ value['created_at'] = Date.today
183
+ end
184
+ end
185
+ data
186
+ when Array
187
+ IC::logger.warn "❗ Old format of pool ❗"
188
+ IC::notify_admin "❗ Old format of pool ❗"
189
+ reload_pool data
190
+ else
191
+ IC::notify_admin "❌ Unknown format of pool"
192
+ raise "❌ Unknown format of pool"
193
+ end
194
+ else
195
+ fetch_new_pool
196
+ end
197
+ rescue => e
198
+ IC::notify_admin '❌ ' + e.message
199
+ raise e
200
+ end
201
+
202
+ def load_used
203
+ file = IC::CONFIG.dig(:data_files, :used)
204
+ if File.exist?(file)
205
+ data = JSON.parse File.read(file), symbolize_names: false
206
+ data.transform_keys!(&:to_i)
207
+ data.transform_values! do |value|
208
+ begin
209
+ Date.parse value
210
+ rescue => e
211
+ IC::logger.error "Error in used: #{e.message}"
212
+ IC::logger.debug "Value: #{value.inspect}"
213
+ Date.today
214
+ end
215
+ end
216
+ data
217
+ else
218
+ {}
219
+ end
220
+ rescue => e
221
+ IC::notify_admin '❌ ' + e.message
222
+ raise e
66
223
  end
67
224
 
68
225
  def load_sent
69
- file = INatChannel::CONFIG[:sent_file]
70
- data = JSON.parse File.read(file), symbolize_names: false
71
- raise "Invalid format of sent file" unless Hash === data
72
- data
73
- rescue
74
- {}
226
+ file = IC::CONFIG.dig(:data_files, :sent)
227
+ if File.exist?(file)
228
+ data = JSON.parse File.read(file), symbolize_names: false
229
+ raise "Invalid format of sent file" unless Hash === data
230
+ # Чистим чего набажили...
231
+ data.each do |_, value|
232
+ begin
233
+ value['sent_at'] = Date.parse(value['sent_at'])
234
+ rescue => e
235
+ IC::logger.error "Error in sent: #{e.message}"
236
+ IC::logger.debug "Value: #{value.inspect}"
237
+ value['sent_at'] = Date.today
238
+ end
239
+ end
240
+ data
241
+ else
242
+ {}
243
+ end
244
+ rescue => e
245
+ IC::notify_admin '❌ ' + e.message
246
+ raise e
75
247
  end
76
248
 
77
249
  def save_pool
78
- pool.reject! { |uuid| sent?(uuid) }
79
- file = INatChannel::CONFIG[:pool_file]
250
+ size = pool.size
251
+
252
+ # 1. Удаляем отправленные
253
+ pool.reject! { |uuid, _| sent?(uuid) }
254
+ IC::logger.info "Removed #{size - pool.size} sent records from pool" if pool.size != size
255
+ size = pool.size
256
+
257
+ # 2. Удаляем использованные
258
+ taxon_uniq = IC::CONFIG[:unique_taxon]
259
+ pool.reject! { |_, value| used?(value['taxon_id']) } if taxon_uniq == :strict || Integer === taxon_uniq
260
+ IC::logger.info "Removed #{size - pool.size} used records from pool" if pool.size != size
261
+ size = pool.size
262
+
263
+ # 3. Удаляем устаревшие
264
+ dead_date = Date.today - IC::CONFIG.dig(:days_back, :pool)
265
+ pool.reject! { |_, value| value['created_at'] < dead_date }
266
+ IC::logger.info "Removed #{size - pool.size} outdated records from pool" if pool.size != size
267
+
268
+ file = IC::CONFIG.dig(:data_files, :pool)
269
+ FileUtils.mkdir_p File.dirname(file)
270
+ File.write file, JSON.pretty_generate(pool)
271
+ end
272
+
273
+ def save_used
274
+ size = used.size
275
+
276
+ # Удаляем устаревшие, если актуально
277
+ dead_date = Date.today - IC::CONFIG.dig(:days_back, :used)
278
+ used.reject! { |_, value| value < dead_date }
279
+ IC::logger.info "Removed #{size - used.size} outdated records from used" if used.size != size
280
+
281
+ file = IC::CONFIG.dig(:data_files, :used)
80
282
  FileUtils.mkdir_p File.dirname(file)
81
- File.write file, JSON.pretty_generate(pool.to_a)
283
+ File.write file, JSON.pretty_generate(used)
82
284
  end
83
285
 
84
286
  def save_sent
85
- file = INatChannel::CONFIG[:sent_file]
287
+ size = sent.size
288
+
289
+ # Удаляем устаревшие
290
+ dead_date = Date.today - IC::CONFIG.dig(:days_back, :sent)
291
+ sent.reject! { |_, value| value['sent_at'] < dead_date }
292
+ IC::logger.info "Removed #{size - sent.size} outdated records from sent" if sent.size != size
293
+
294
+ file = IC::CONFIG.dig(:data_files, :sent)
86
295
  FileUtils.mkdir_p File.dirname(file)
87
296
  File.write file, JSON.pretty_generate(sent)
88
297
  end
@@ -92,3 +301,25 @@ module INatChannel
92
301
  end
93
302
 
94
303
  end
304
+
305
+ module IC
306
+
307
+ def select_uuid fresh
308
+ INatChannel::Storage::select_uuid fresh
309
+ end
310
+
311
+ def save_data
312
+ INatChannel::Storage::save
313
+ end
314
+
315
+ def confirm_sending! msg_id
316
+ INatChannel::Storage::confirm! msg_id
317
+ end
318
+
319
+ def revert_sending!
320
+ INatChannel::Storage::revert!
321
+ end
322
+
323
+ module_function :select_uuid, :save_data, :confirm_sending!, :revert_sending!
324
+
325
+ end
@@ -11,16 +11,22 @@ module INatChannel
11
11
  class << self
12
12
 
13
13
  def convert_observation observation_source
14
- id = observation_source[:id]
15
- url = observation_source[:uri]
16
- uuid = observation_source[:uuid]
17
- user = convert_user observation_source[:user]
18
- taxon = convert_taxon observation_source[:taxon], observation_source[:identifications]
19
- places = convert_places observation_source[:place_ids]
20
- datetime = DateTime.parse observation_source[:time_observed_at]
21
- location = convert_location observation_source[:geojson]
22
- description = convert_description observation_source[:description]
23
- place_guess = observation_source[:place_guess]
14
+ begin
15
+ id = observation_source[:id]
16
+ url = observation_source[:uri]
17
+ uuid = observation_source[:uuid]
18
+ user = convert_user observation_source[:user]
19
+ taxon = convert_taxon observation_source[:taxon], observation_source[:identifications]
20
+ places = convert_places observation_source[:place_ids]
21
+ datetime = DateTime.parse(observation_source[:time_observed_at] || observation_source[:observed_on_string])
22
+ location = convert_location observation_source[:geojson]
23
+ description = convert_description observation_source[:description]
24
+ place_guess = observation_source[:place_guess]
25
+ rescue => e
26
+ IC::logger.error e.full_message
27
+ IC::logger.info JSON.pretty_generate(observation_source)
28
+ raise e
29
+ end
24
30
  Observation::new id: id, url: url, uuid: uuid, user: user, taxon: taxon, places: places, datetime: datetime, location: location, description: description, place_guess: place_guess
25
31
  end
26
32
 
@@ -70,7 +76,7 @@ module INatChannel
70
76
  end
71
77
 
72
78
  def convert_places place_ids
73
- places_config = INatChannel::CONFIG[:places]
79
+ places_config = IC::CONFIG[:places]
74
80
  return nil unless places_config
75
81
  result = []
76
82
  places_config.each do |_, items|
@@ -90,3 +96,13 @@ module INatChannel
90
96
  end
91
97
 
92
98
  end
99
+
100
+ module IC
101
+
102
+ def convert_observation source
103
+ INatChannel::DataConvert::convert_observation source
104
+ end
105
+
106
+ module_function :convert_observation
107
+
108
+ end