inat-channel 0.8.0.14 → 0.9.0

data/lib/inat-channel/api.rb

@@ -15,73 +15,74 @@ module INatChannel
       PER_PAGE = 200
       PAGE_DELAY = 1.0
       API_ENDPOINT = 'https://api.inaturalist.org/v2/observations'
-      LIST_FIELDS = 'uuid'
+      LIST_FIELDS = '(uuid:!t,created_at:!t,faves_count:!t,taxon:(id:!t))'
       SINGLE_FIELDS = '(id:!t,uuid:!t,uri:!t,geojson:(all:!t),user:(login:!t,name:!t),taxon:(ancestor_ids:!t,preferred_common_name:!t,name:!t),' +
-                      'place_ids:!t,place_guess:!t,observed_on_string:!t,description:!t,photos:(url:!t),identifications:(taxon:(ancestors:(name:!t))))'
+                      'place_ids:!t,place_guess:!t,observed_on_string:!t,description:!t,photos:(url:!t),time_observed_at:!t,' +
+                      'identifications:(taxon:(ancestors:(name:!t))))'
 
       private_constant :PER_PAGE, :PAGE_DELAY, :API_ENDPOINT, :LIST_FIELDS, :SINGLE_FIELDS
 
       def load_news 
+        load_list(**IC::CONFIG[:base_query], created_d1: (Date.today - IC::CONFIG.dig(:days_back, :fresh)).to_s)
+      end
+      
+      def load_list **query
         result = []
         page = 1
-
+        
         loop do 
-          INatChannel::LOGGER.debug "Fetch page #{page} with per_page=#{PER_PAGE}"
+          IC::logger.debug "Fetch page #{page} with per_page=#{PER_PAGE}"
 
           response = faraday.get API_ENDPOINT do |req|
-            req.params['page'] = page
-            req.params['per_page'] = PER_PAGE
-            req.params['fields'] = LIST_FIELDS
-            req.params.merge! INatChannel::CONFIG[:base_query]
-            req.params['created_d1'] = (Date.today - INatChannel::CONFIG[:days_back]).to_s
+            req.params[:page] = page
+            req.params[:per_page] = PER_PAGE
+            req.params[:fields] = LIST_FIELDS
+            req.params.merge! query
           end
 
           unless response.success?
-            INatChannel::Telegram::notify_admin "Failed to fetch observations page #{page}: HTTP #{response.status}"
-            INatChannel::LOGGER.error "HTTP #{response.status} on page #{page}"
+            IC::logger.error "❌ Failed to fetch observations page #{page}: HTTP #{response.status}"
+            IC::notify_admin "❌ Failed to fetch observations page #{page}: HTTP #{response.status}"
             break
           end
 
           data = JSON.parse response.body, symbolize_names: true
-          uuids = data[:results].map { |o| o[:uuid] }
-          result += uuids
+          result += data[:results]
 
           total = data[:total_results] || 0
-          INatChannel::LOGGER.debug "Page #{page}: fetched #{uuids.size} UUIDs, total expected #{total}"
-
-          break if uuids.empty? || result.size >= total
+          IC::logger.debug "Page #{page}: fetched #{data.size} records, total expected #{total}"
+          break if data.empty? || result.size >= total
           page += 1
           sleep PAGE_DELAY
         end
 
-        INatChannel::LOGGER.debug "Loaded total #{result.uniq.size} unique UUIDs"
-        result.uniq
+        IC::logger.debug "Loaded total #{result.size} records"
+        result
       rescue => e
-        INatChannel::Telegram::notify_admin "Exception while loading news: #{e.message}"
-        INatChannel::LOGGER.error e.full_message
-        []
+        IC::logger.error "❌ Exception while loading news: #{e.full_message}"
+        IC::notify_admin "❌ Exception while loading news: #{e.message}"
       end
 
       def load_observation uuid
         response = faraday.get API_ENDPOINT do |req|
           req.params['uuid'] = uuid
-          req.params['locale'] = INatChannel::CONFIG[:base_query][:locale] if INatChannel::CONFIG[:base_query][:locale]
+          req.params['locale'] = IC::CONFIG[:base_query][:locale] if IC::CONFIG[:base_query][:locale]
           req.params['fields'] = SINGLE_FIELDS
         end
 
         if response.success?
           data = JSON.parse response.body, symbolize_names: true
           obs = data[:results]&.first
-          INatChannel::LOGGER.debug "Loaded observation: #{uuid}"
+          IC::logger.debug "Loaded observation: #{uuid}"
           obs
         else
-          INatChannel::LOGGER.error "Error loading observation #{uuid}: HTTP #{response.status}"
-          INatChannel::Telegram::notify_admin "Error loading observation #{uuid}: HTTP #{response.status}"
+          IC::logger.error "Error loading observation #{uuid}: HTTP #{response.status}"
+          IC::notify_admin "Error loading observation #{uuid}: HTTP #{response.status}"
           nil
         end
       rescue => e
-        INatChannel::Telegram::notify_admin "Exception while loading observation #{uuid}: #{e.message}"
-        INatChannel::LOGGER.error e.full_message
+        IC::notify_admin "Exception while loading observation #{uuid}: #{e.message}"
+        IC::logger.error e.full_message
         nil
       end
 
@@ -89,12 +90,16 @@ module INatChannel
 
       def faraday
         @faraday ||= Faraday::new do |f|
-          f.request :retry, max: INatChannel::CONFIG[:retries], interval: 2.0, interval_randomness: 0.5,
-                            exceptions: [ Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ClientError ]
+          f.request :retry, 
+                    max: IC::CONFIG.dig(:api, :retries), 
+                    interval: IC::CONFIG.dig(:api, :interval), 
+                    interval_randomness: IC::CONFIG.dig(:api, :randomness),
+                    backoff_factor: IC::CONFIG.dig(:api, :backoff),
+                    exceptions: [ Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ClientError ]
           f.request :url_encoded
 
-          if INatChannel::LOGGER.level == ::Logger::DEBUG
-            f.response :logger, INatChannel::LOGGER, bodies: true, headers: true 
+          if IC::logger.level == ::Logger::DEBUG
+            f.response :logger, IC::logger, bodies: true, headers: true 
           end
 
           f.adapter Faraday::default_adapter
@@ -106,3 +111,22 @@ module INatChannel
   end
 
 end
+
+module IC
+
+  def load_news
+    INatChannel::API::load_news
+  end
+
+  def load_observation uuid
+    INatChannel::API::load_observation uuid
+  end
+
+  def load_list **query
+    INatChannel::API::load_list(**query)
+  end
+
+  module_function :load_news, :load_observation
+
+end
+

data/lib/inat-channel/config.rb

@@ -23,9 +23,9 @@ module INatChannel
         cfg.merge! options
         cfg[:log_level] ||= :warn
         env = load_env
-        cfg.merge! env
+        cfg[:tg_bot] ||= {}
+        cfg[:tg_bot].merge! env
         validate_and_fix_config! cfg
-        cfg
       end
 
       def parse_options
@@ -33,7 +33,7 @@ module INatChannel
         OptionParser.new do |opts|
           opts.banner = 'Usage: inat-channel [options]'
           opts.on '-c', '--config FILE', 'Config file (default: inat-channel.yml)' do |v|
-            raise ArgumentError, "Config file not found: #{v}" unless File.exist?(v)
+            raise "Config file not found: #{v}" unless File.exist?(v)
             options[:config] = v
           end
           opts.on '-l', '--log-level LEVEL', [:debug, :info, :warn, :error], 'Log level (default: warn)' do |v|
@@ -43,7 +43,7 @@ module INatChannel
             options[:log_level] = :debug
           end
           opts.on '--version', 'Show version info and exit' do
-            puts INatChannel::VERSION
+            puts IC::VERSION
             exit
           end
           opts.on '-h', '--help', 'Show help and exit' do
@@ -56,32 +56,78 @@ module INatChannel
 
       def load_config path
         raise "Config file not found: #{path}" unless File.exist?(path)
-        YAML.safe_load_file(path, symbolize_names: true)
+        cfg = YAML.safe_load_file path, symbolize_names: true
+        if String === cfg[:places]
+          path = File.expand_path(path)
+          places_path = File.expand_path(cfg[:places], File.dirname(path))
+          places = YAML.safe_load_file places_path, symbolize_names: true
+          cfg[:places] = places
+        end
+        cfg
       end
 
       def load_env
         { 
-          telegram_bot_token: (ENV['TELEGRAM_BOT_TOKEN'] or raise 'TELEGRAM_BOT_TOKEN required'),
-          admin_telegram_id:  (ENV['ADMIN_TELEGRAM_ID']  or raise 'ADMIN_TELEGRAM_ID required')
+          token:    (ENV['TELEGRAM_BOT_TOKEN'] or raise 'TELEGRAM_BOT_TOKEN required'),
+          admin_id: (ENV['ADMIN_TELEGRAM_ID']  or raise 'ADMIN_TELEGRAM_ID required')
         }
       end
 
       def validate_and_fix_config! cfg
         raise 'Missing or invalid base_query' unless Hash === cfg[:base_query]
-        raise 'Missing or invalid days_back'  unless Integer === cfg[:days_back] && cfg[:days_back] > 0
-        raise 'Missing chat_id'               unless cfg[:chat_id]
+        raise 'Missing or invalid days_back'  unless Integer === cfg.dig(:days_back, :fresh) && cfg.dig(:days_back, :fresh)
+        raise 'Missing chat_id'               unless cfg.dig(:tg_bot, :chat_id)
 
         basename = File.basename cfg[:config], '.*'
-        cfg[:pool_file] ||= "./data/#{basename}_pool.json"
-        cfg[:sent_file] ||= "./data/#{basename}_sent.json"
-        cfg[:lock_file] ||= "./data/#{basename}__bot.lock"
-        cfg[:retries]   ||= 5
+        cfg[:data_files]        ||= {}
+        cfg[:data_files][:root] ||= 'data'
+        cfg[:data_files][:pool] ||= "#{ cfg[:data_files][:root] }/#{ basename }_pool.json"
+        cfg[:data_files][:sent] ||= "#{ cfg[:data_files][:root] }/#{ basename }_sent.json"
+        cfg[:data_files][:used] ||= "#{ cfg[:data_files][:root] }/#{ basename }_used.json"
+
+        cfg[:lock_file]        ||= {}
+        cfg[:lock_file][:path] ||= "#{ cfg[:data_files][:root] }/#{ basename }__bot.json"
+        cfg[:lock_file][:ttl]  ||= 300  # 5 min
+
+        cfg[:days_back][:pool] ||= 3 * cfg.dig(:days_back, :fresh)
+        cfg[:days_back][:sent] ||= cfg[:days_back][:pool] + 1
+        cfg[:days_back][:used] ||= 365
+
+        cfg[:api]              ||= {}
+        cfg[:api][:retries]    ||= 5
+        cfg[:api][:interval]   ||= 1.0
+        cfg[:api][:randomness] ||= 0.5
+        cfg[:api][:backoff]    ||= 2
+        cfg[:api][:page_delay] ||= 1.0
+        cfg[:api][:per_page]   ||= 200
+
+        cfg[:tg_bot][:retries]    ||= 5
+        cfg[:tg_bot][:interval]   ||= 1.0
+        cfg[:tg_bot][:randomness] ||= 0.5
+        cfg[:tg_bot][:backoff]    ||= 2
+        cfg[:tg_bot][:desc_limit] ||= 512
+        cfg[:tg_bot][:link_zoom]  ||= 12
+
+        cfg[:unique_taxon] ||= :ignore
+        cfg[:unique_taxon]   = cfg[:unique_taxon].to_sym
+
+        cfg[:log_level]    ||= :warn
+        cfg[:log_level]      = cfg[:log_level].to_sym
+        cfg[:notify_level] ||= :warn
+        cfg[:notify_level]   = cfg[:notify_level].to_sym
+
+        cfg
       end
 
     end
 
   end
 
-  CONFIG = Config::config
+end
+
+module IC
+
+  CONFIG = INatChannel::Config::config
 
 end
+

data/lib/inat-channel/data.rb

@@ -1,4 +1,5 @@
 require 'time'
+require 'date'
 require 'json'
 require 'fileutils'
 require 'set'
@@ -9,80 +10,288 @@ require_relative 'lock'
 
 module INatChannel
 
-  module Data 
+  module Storage
 
     class << self
 
       def select_uuid fresh
-        INatChannel::LOGGER.info "Received #{fresh.size} uuids"
+        # 1. Отфильтровываем уже отправленные
+        fresh.reject! { |rec| sent?(rec[:uuid]) }
 
-        fresh.reject! { |uuid| sent?(uuid) }
+        # 2. Если нужно, отфильтровываем по таксонам
+        taxon_uniq = IC::CONFIG[:unique_taxon]
+        fresh.reject! { |rec| used?(rec.dig :taxon, :id) } if taxon_uniq == :strict
+        
+        if taxon_uniq == :priority
+          uniq_fresh = fresh.reject { |rec| used?(rec.dig :taxon, :id) }
+          unless uniq_fresh.empty?
+            IC::logger.info "Take a fresh & unique (from #{uniq_fresh.size})"
+            sample = sample_with_weight uniq_fresh, field: :faves_count
+            fresh.reject { |rec| rec[:uuid] == sample[:uuid] }.each do |rec| 
+              pool[rec[:uuid]] = {
+                'created_at' => Date.parse(rec[:created_at]),
+                'faves_count' => rec[:faves_count],
+                'taxon_id' => rec.dig(:taxon, :id)
+              }
+            end
+            sample[:taxon_id] = sample.dig :taxon, :id
+            @in_process = sample
+            return sample[:uuid]
+          end
+        end
+
+        # 3. Если добрались до этого места, берем просто из свежих
         unless fresh.empty?
-          result = fresh.sample
-          fresh.delete result
-          pool.merge fresh
-          INatChannel::LOGGER.info "Fresh uuid selected, #{fresh.size} uuids added to pool"
-          return result
+          IC::logger.info "Take a fresh (from #{fresh.size})"
+          sample = sample_with_weight fresh, field: :faves_count
+          fresh.reject { |rec| rec[:uuid] == sample[:uuid] }.each do |rec|
+            pool[rec[:uuid]] = {
+              "created_at" => Date.parse(rec[:created_at]),
+              "faves_count" => rec[:faves_count],
+              "taxon_id" => rec.dig(:taxon, :id),
+            }
+          end
+          sample[:taxon_id] = sample.dig :taxon, :id
+          @in_process = sample
+          return sample[:uuid]
+        end
+        
+        # 4. Если ..., проверяем приоритет уникальных уже для пула
+        pool_records = pool.map { |k, v| v.merge({ 'uuid' => k }) }
+        if taxon_uniq == :priority
+          uniq_pool = pool_records.reject { |rec| used?(rec['taxon_id']) }
+          unless uniq_pool.empty?
+            IC::logger.info "Take an unique pool record (from #{uniq_pool.size})"
+            sample = sample_with_weight uniq_pool, field: 'faves_count'
+            sample.transform_keys!(&:to_sym)
+            @in_process = sample
+            return sample[:uuid]
+          end
         end
 
-        pool.reject! { |uuid| sent?(uuid) }
-        unless pool.empty?
-          result = pool.to_a.sample
-          pool.delete result
-          INatChannel::LOGGER.info "Pool uuid selected, #{pool.size} uuids remain in pool"
-          return result
+        # 5. Если ..., берем из пула
+        unless pool_records.empty?
+          IC::logger.info "Take a pool record (from #{pool.size})"
+          sample = sample_with_weight pool_records, field: 'faves_count'
+          sample.transform_keys!(&:to_sym)
+          @in_process = sample
+          return sample[:uuid]
         end
 
-        nil
+        return nil
       end
 
-      def pool
-        @pool ||= load_pool
+      def confirm! msg_id
+        # Отправка выполнена, фиксируем
+        sent[@in_process[:uuid]] = {
+          'msg_id' => msg_id,
+          'sent_at' => Date.today
+        }
+        used[@in_process[:taxon_id]] = Date.today
       end
 
-      def sent
-        @sent ||= load_sent
+      def revert!
+        # Отправка не выполнена, возвращаем рабочую запись в пул
+        pool[@in_process[:uuid]] = {
+          'created_at' => @in_process[:created_at],
+          'faves_count' => @in_process[:faves_count],
+          'taxon_id' => @in_process[:taxon_id]
+        }
       end
 
       def save
         save_pool
         save_sent
-        INatChannel::LOGGER.info "Saved pool=#{pool.size}, sent=#{sent.size}"
+        save_used
+        IC::logger.info "Saved pool=#{pool.size}, sent=#{sent.size}"
       end
 
       private
 
+      def used
+        @used ||= load_used
+      end
+
+      def pool
+        @pool ||= load_pool
+      end
+
+      def sent
+        @sent ||= load_sent
+      end
+
       def sent? uuid
         sent.has_key? uuid
       end
 
+      def used? taxon_id
+        used.has_key? taxon_id
+      end
+
+      def sample_with_weight source, field: 
+        data = []
+        source.each do |rec|
+          num = rec[field]
+          num = 1 unless Integer === num && num > 0
+          num.times { data << rec }
+        end
+        data.sample
+      end
+
+      def reload_pool old
+        result = {}
+        old.each_slice 100 do |items|
+          records = IC::load_list uuid: items.join(',')
+          records.each do |rec|
+            result[rec[:uuid]] = {
+              'created_at' => Date.parse(rec[:created_at]),
+              'faves_count' => rec[:faves_count],
+              'taxon_id' => rec.dig(:taxon, :id)
+            }
+          end
+        end
+        result
+      end
+
+      def fetch_new_pool
+        result = {}
+        dead_date = Date.today - IC::CONFIG.dig(:days_back, :pool)
+        records = IC::load_list(**IC::CONFIG[:base_query], created_d1: dead_date.to_s)
+        records.each do |rec|
+          result[rec[:uuid]] = {
+            'created_at' => Date.parse(rec[:created_at]),
+            'faves_count' => rec[:faves_count],
+            'taxon_id' => rec.dig(:taxon, :id)
+          }
+        end
+        result
+      end
+
       def load_pool
-        file = INatChannel::CONFIG[:pool_file]
-        data = JSON.parse File.read(file), symbolize_names: false
-        raise "Invalid format of pool file" unless Array === data
-        Set[*data]
-      rescue
-        Set::new
+        file = IC::CONFIG.dig(:data_files, :pool)
+        if File.exist?(file)
+          data = JSON.parse File.read(file), symbolize_names: false
+          case data
+          when Hash
+            data.each do |_, value|
+              begin
+                value['created_at'] = Date.parse(value['created_at'])
+              rescue => e
+                IC::logger.error "Error in pool: #{e.message}"
+                IC::logger.debug "Value: #{value.inspect}"
+                value['created_at'] = Date.today
+              end
+            end
+            data
+          when Array
+            IC::logger.warn "❗ Old format of pool ❗"
+            IC::notify_admin "❗ Old format of pool ❗"
+            reload_pool data
+          else
+            IC::notify_admin "❌ Unknown format of pool"
+            raise "❌ Unknown format of pool"
+          end
+        else
+          fetch_new_pool
+        end
+      rescue => e
+        IC::notify_admin '❌ ' + e.message
+        raise e
+      end
+
+      def load_used
+        file = IC::CONFIG.dig(:data_files, :used)
+        if File.exist?(file)
+          data = JSON.parse File.read(file), symbolize_names: false
+          data.transform_keys!(&:to_i)
+          data.transform_values! do |value|
+            begin
+              Date.parse value
+            rescue => e
+              IC::logger.error "Error in used: #{e.message}"
+              IC::logger.debug "Value: #{value.inspect}"
+              Date.today
+            end
+          end
+          data
+        else
+          {}
+        end
+      rescue => e
+        IC::notify_admin '❌ ' + e.message
+        raise e
       end
 
       def load_sent
-        file = INatChannel::CONFIG[:sent_file]
-        data = JSON.parse File.read(file), symbolize_names: false
-        raise "Invalid format of sent file" unless Hash === data
-        data
-      rescue
-        {}
+        file = IC::CONFIG.dig(:data_files, :sent)
+        if File.exist?(file)
+          data = JSON.parse File.read(file), symbolize_names: false
+          raise "Invalid format of sent file" unless Hash === data
+          # Чистим чего набажили...
+          data.each do |_, value|
+            begin
+              value['sent_at'] = Date.parse(value['sent_at'])
+            rescue => e
+              IC::logger.error "Error in sent: #{e.message}"
+              IC::logger.debug "Value: #{value.inspect}"
+              value['sent_at'] = Date.today
+            end
+          end
+          data
+        else
+          {}
+        end
+      rescue => e
+        IC::notify_admin '❌ ' + e.message
+        raise e
       end
 
       def save_pool
-        pool.reject! { |uuid| sent?(uuid) }
-        file = INatChannel::CONFIG[:pool_file]
+        size = pool.size
+
+        # 1. Удаляем отправленные
+        pool.reject! { |uuid, _| sent?(uuid) }
+        IC::logger.info "Removed #{size - pool.size} sent records from pool" if pool.size != size
+        size = pool.size
+
+        # 2. Удаляем использованные
+        taxon_uniq = IC::CONFIG[:unique_taxon]
+        pool.reject! { |_, value| used?(value['taxon_id']) } if taxon_uniq == :strict || Integer === taxon_uniq
+        IC::logger.info "Removed #{size - pool.size} used records from pool" if pool.size != size
+        size = pool.size
+
+        # 3. Удаляем устаревшие
+        dead_date = Date.today - IC::CONFIG.dig(:days_back, :pool)
+        pool.reject! { |_, value| value['created_at'] < dead_date }
+        IC::logger.info "Removed #{size - pool.size} outdated records from pool" if pool.size != size
+
+        file = IC::CONFIG.dig(:data_files, :pool)
+        FileUtils.mkdir_p File.dirname(file)
+        File.write file, JSON.pretty_generate(pool)
+      end
+
+      def save_used
+        size = used.size
+
+        # Удаляем устаревшие, если актуально
+        dead_date = Date.today - IC::CONFIG.dig(:days_back, :used)
+        used.reject! { |_, value| value < dead_date }
+        IC::logger.info "Removed #{size - used.size} outdated records from used" if used.size != size
+
+        file = IC::CONFIG.dig(:data_files, :used)
         FileUtils.mkdir_p File.dirname(file)
-        File.write file, JSON.pretty_generate(pool.to_a)
+        File.write file, JSON.pretty_generate(used)
       end
 
       def save_sent
-        file = INatChannel::CONFIG[:sent_file]
+        size = sent.size
+        
+        # Удаляем устаревшие 
+        dead_date = Date.today - IC::CONFIG.dig(:days_back, :sent)
+        sent.reject! { |_, value| value['sent_at'] < dead_date }
+        IC::logger.info "Removed #{size - sent.size} outdated records from sent" if sent.size != size
+
+        file = IC::CONFIG.dig(:data_files, :sent)
         FileUtils.mkdir_p File.dirname(file)
         File.write file, JSON.pretty_generate(sent)
       end
@@ -92,3 +301,25 @@ module INatChannel
   end
 
 end
+
+module IC
+
+  def select_uuid fresh
+    INatChannel::Storage::select_uuid fresh
+  end
+
+  def save_data
+    INatChannel::Storage::save
+  end
+
+  def confirm_sending! msg_id
+    INatChannel::Storage::confirm! msg_id
+  end
+
+  def revert_sending!
+    INatChannel::Storage::revert!
+  end
+
+  module_function :select_uuid, :save_data, :confirm_sending!, :revert_sending!
+
+end