inat-get 0.9.0.12 → 0.9.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1728f650aec437e88f0d9ba844695b9ffd5d4e9410aa3fa72a7d2e855943b9d2
-  data.tar.gz: 22b960c8392bd3b786e4d067877bd10cf7f11594fd240697e53def459403968c
+  metadata.gz: 05a4272992b33172ef754c61f1d1d1c89a11cc7efe51cb84420efda09bfff858
+  data.tar.gz: 702308f230e526f0a10125c2785b258a0417f513937f3111a816e0f727734262
 SHA512:
-  metadata.gz: edf53795eacbf7606eec2172e5129143530c7bb5f3aee34386fb37eda9fdce7472df15d8d01579542ae7b31edaf7d9b5aa3e47f4affd4a6b8c18c22b56d09949
-  data.tar.gz: 5bba1a958b59ae606cdce9a8c1914f0c9a85c29f5f440e673387f868278df17012f48d6fea4c0e653d7bd40ef986c9cc86dfd70a8c2a0cbaf5d20f30f359bef3
+  metadata.gz: 673c18491a5152a65e5d6773f325125fa98824ee8ac6dc543e0d57cd50dafa9cac0751ab4ad522de7ecd9eaa9f87b77e2af56d8b64323c74ee83507aa43c76b4
+  data.tar.gz: a6c4facdb3f9ba680f693e58c9bf788ddf71ae0b0b675d457dc1b6b4ab82ee67dfa50a22a327fc81fad1e8b51710faea27f3b2a145eda9fda2eb8e1532fa5b22

data/lib/inat-get/app/core/api.rb

@@ -2,10 +2,14 @@
 
 require 'faraday'
 require 'faraday/retry'
+require 'faraday/typhoeus'
+# require 'faraday/gzip'
+# require 'faraday-http-cache'
 require 'is-duration'
 
 require_relative 'server'
 require_relative 'console_logger'
+require_relative 'api_cache'
 
 class INatGet::App::Server::API < INatGet::App::Server
 
@@ -21,6 +25,7 @@ class INatGet::App::Server::API < INatGet::App::Server
 
   def get query, **opts
     endpoint = @config.dig(:api, :root) + query[:endpoint].to_s
+    @logger.info query[:endpoint].to_s
     timepoint = Time::now
     if @last_request
       delta = timepoint - @last_request
@@ -33,6 +38,7 @@ class INatGet::App::Server::API < INatGet::App::Server
       rq.params.merge! query[:query]
       rq.headers["User-Agent"] = "iNatGet v#{INatGet::Info::VERSION} (#{ INatGet::Info::VERSION_ALIAS })"
     end
+    @logger.clear
     if response.success?
       begin
         data = JSON.parse response.body, symbolize_names: true
@@ -45,6 +51,8 @@ class INatGet::App::Server::API < INatGet::App::Server
       @logger.error "Error in response: #{response.status}"
       return { status: :error, error: response.status }.freeze
     end
+  # rescue => e
+  #   return { status: :error, error: e.message }.freeze
   end
 
   def faraday
@@ -55,10 +63,19 @@ class INatGet::App::Server::API < INatGet::App::Server
                 interval: IS::Duration::parse(@config.dig(:api, :retry, :interval)),
                 interval_randomness: @config.dig(:api, :retry, :randomness),
                 backoff_factor: @config.dig(:api, :retry, :backoff),
+                retry_block: lambda { |env:, options:, retry_count:, exception:, will_retry_in:| @logger.warn "retry... : #{ retry_count } : #{ exception.class }" },
                 exceptions: [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ClientError]
       f.request :url_encoded
-      # f.response :logger, tmp_logger, bodies: true, headers: true
-      f.adapter Faraday::default_adapter
+      # f.response :logger, tmp_logger, { headers: true, bodies: false }
+
+      # f.use :http_cache, store: INatGet::App::Server::API::Cache::new(@config.dig(:caching, :api) || 100)
+
+      f.adapter :typhoeus do |typhoeus|
+        typhoeus.options[:connecttimeout] = 10 
+        typhoeus.options[:timeout] = 20
+        typhoeus.options[:low_speed_limit] = 10
+        typhoeus.options[:low_speed_time] = 10
+      end
     end
   end
 

data/lib/inat-get/app/core/api_cache.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative '../../info'
+
+module INatGet::App
+  class Server
+    class API < INatGet::App::Server; end
+  end
+end
+
+# @api private
+class INatGet::App::Server::API::Cache
+
+  def initialize limit = 100
+    @limit = limit
+    @data = {}
+  end
+
+  def read key
+    return nil unless @data.has_key?(key)
+    value = @data.delete key
+    @data[key] = value
+    value
+  end
+
+  def write key, value
+    @data.delete key
+    @data[key] = value
+    @data.shift if @data.size > @limit
+    value
+  end
+
+  def delete key
+    @data.delete key
+  end
+
+end

data/lib/inat-get/app/core/console.rb

@@ -65,6 +65,16 @@ class INatGet::App::Server::Console < INatGet::App::Server
     true
   end
 
+  def clear progname, **opts
+    key = prog_key progname
+    pid = opts[:_sender_pid]
+    if key == :wrk
+      @table.update _sender_pid: pid, message: ''
+    else
+      @table.summary message: ''
+    end
+  end
+
   private
 
   def prog_key progname

data/lib/inat-get/app/core/console_logger.rb

@@ -24,4 +24,8 @@ class INatGet::App::ConsoleLogger < Logger
     @console.log severity, msg, (progname || self.progname)
   end
 
+  def clear progname = nil
+    @console.clear(progname || self.progname)
+  end
+
 end

data/lib/inat-get/app/core/server.rb

@@ -66,6 +66,7 @@ class INatGet::App::Server
   end
 
   def on_error exception
+    warn exception.class
     warn exception
     warn exception.backtrace
   end
@@ -92,7 +93,7 @@ class INatGet::App::Server
     end
 
     def method_missing sym, *args, **kwargs
-      kwargs[:_sender_pid] = ::Process::pid
+      kwargs[:_sender_pid] ||= ::Process::pid
       msg = {
         method: sym,
         args: args,
@@ -104,8 +105,8 @@ class INatGet::App::Server
       socket.close
       result
     rescue => e
-      pp e
-      false
+      # pp e
+      { status: :error, error: e.message }
     end
 
   end

data/lib/inat-get/app/setup.rb

@@ -16,8 +16,8 @@ module INatGet::App::Setup
   DEFAULTS = {
     logs: {
       screen: {
-        api: 'warn',
-        sys: 'warn',
+        api: 'info',
+        sys: 'info',
         wrk: 'info'
       },
       file: {
@@ -34,6 +34,7 @@ module INatGet::App::Setup
       password: nil
     },
     caching: {
+      api: 100,
       update: '4h',
       refresh: { 
         interval: '4d',
@@ -62,7 +63,7 @@ module INatGet::App::Setup
       preferred_place: 7161,
       translate_projects: 'umbrella',    # all | umbrella | none
       retry: {
-        max: 5,
+        max: 25,
         interval: '1s',
         randomness: 0.5,
         backoff: 2

data/lib/inat-get/data/models/base.rb

@@ -52,6 +52,19 @@ module INatGet::Data
 
     end
 
+    module Identified
+
+      def eql? other
+        return false unless other.is_a?(self.class)
+        self.id == other.id
+      end
+
+      def hash
+        [Identified, self.class, self.id].hash
+      end
+
+    end
+
   end
 
 end

data/lib/inat-get/data/models/identification.rb

@@ -10,6 +10,8 @@ module INatGet::Data; end
 
 class INatGet::Data::Model::Identification < INatGet::Data::Model
 
+  include INatGet::Data::Model::Identified
+
   set_dataset :identifications
 
   many_to_one :observation, class: :'INatGet::Data::Model::Observation'

data/lib/inat-get/data/models/observation.rb

@@ -18,6 +18,8 @@ module INatGet::Data; end
 
 class INatGet::Data::Model::Observation < INatGet::Data::Model
 
+  include INatGet::Data::Model::Identified
+
   set_dataset :observations
 
   one_to_many :faves,           class: INatGet::Data::Model::Fave

data/lib/inat-get/data/models/photo.rb

@@ -9,6 +9,8 @@ module INatGet::Data; end
 
 class INatGet::Data::Model::Photo < INatGet::Data::Model
 
+  include INatGet::Data::Model::Identified
+
   set_dataset :photos
 
   many_to_many :observations, class: :'INatGet::Data::Model::Observation', join_table: :observation_photos, left_key: :photo_id, right_key: :observation_id

data/lib/inat-get/data/models/place.rb

@@ -9,6 +9,8 @@ module INatGet::Data; end
 
 class INatGet::Data::Model::Place < INatGet::Data::Model
 
+  include INatGet::Data::Model::Identified
+
   set_dataset :places
 
   many_to_many :observations, class: :'INatGet::Data::Model::Observation', join_table: :observation_places, left_key: :place_id, right_key: :observation_id

data/lib/inat-get/data/models/project.rb

@@ -16,6 +16,8 @@ module INatGet::Data; end
 
 class INatGet::Data::Model::Project < INatGet::Data::Model
 
+  include INatGet::Data::Model::Identified
+
   set_dataset :projects
 
   many_to_one :user, class: :'INatGet::Data::Model::User'

data/lib/inat-get/data/models/sound.rb

@@ -6,6 +6,8 @@ require_relative "../../info"
 
 class INatGet::Data::Model::Sound < INatGet::Data::Model
 
+  include INatGet::Data::Model::Identified
+
   set_dataset :sounds
 
   many_to_many :observations, class: :'INatGet::Data::Model::Observation', join_table: :observation_sounds, left_key: :sound_id, right_key: :observation_id

data/lib/inat-get/data/models/taxon.rb

@@ -8,6 +8,8 @@ require_relative 'base'
 
 class INatGet::Data::Model::Taxon < INatGet::Data::Model
 
+  include INatGet::Data::Model::Identified
+
   set_dataset :taxa
 
   one_to_many :observations

data/lib/inat-get/data/models/user.rb

@@ -7,6 +7,8 @@ require_relative 'base'
 
 class INatGet::Data::Model::User < INatGet::Data::Model
 
+  include INatGet::Data::Model::Identified
+
   set_dataset :users
 
   one_to_many :observations

data/lib/inat-get/data/updaters/base.rb

@@ -146,25 +146,30 @@ class INatGet::Data::Updater
     rq_model = INatGet::Data::Model::Request
     record = nil
     found = false
-    rq_model.db.transaction(isolation: :committed, mode: :immediate) do
+    # rq_model.db.transaction(isolation: :committed, mode: :immediate) do
       record = rq_model.with_pk(rq_hash)
       if record
         while record.busy
           raise "Too old business" if start_point - record.busy > HARD_STOP
+          console.update status: 'waiting...'
           sleep 0.1
           record.reload
         end
         return :fresh if record.finished > actual_point
-        record.update busy: start_point
+        rq_model.db.transaction(isolation: :committed, mode: :immediate) do
+          record.update busy: start_point
+        end
         found = true
       else
-        record = rq_model.create full: rq_hash, endless: el_hash, endpoint: endpoint.to_s, query: rq_json, started: start_point, freshed: start_point, busy: start_point
-        set_request_projects record, query[:project_id] if query[:project_id]
-        set_request_places   record, query[:place_id]   if query[:place_id]
-        set_request_taxa     record, query[:taxon_id]   if query[:taxon_id]
-        set_request_users    record, query[:user_id]    if query[:user_id]
+        rq_model.db.transaction(isolation: :committed, mode: :immediate) do
+          record = rq_model.create full: rq_hash, endless: el_hash, endpoint: endpoint.to_s, query: rq_json, started: start_point, freshed: start_point, busy: start_point
+          set_request_projects record, query[:project_id] if query[:project_id]
+          set_request_places   record, query[:place_id]   if query[:place_id]
+          set_request_taxa     record, query[:taxon_id]   if query[:taxon_id]
+          set_request_users    record, query[:user_id]    if query[:user_id]
+        end
       end
-    end
+    # end
 
     # Устанавливаем updated_since
     if allow_updated_since?
@@ -307,6 +312,7 @@ class INatGet::Data::Updater
       until result
         query[:id_above] = id_above if id_above
         check_shutdown!
+        console.update status: 'querying...'
         response = api.get({ endpoint: endpoint, query: query })
         if response[:status] == :error
           result = :error
@@ -335,6 +341,7 @@ class INatGet::Data::Updater
       until result
         query[:page] = page if page
         check_shutdown!
+        console.update status: 'querying...'
         response = api.get({ endpoint: endpoint, query: query })
         if response[:status] == :error
           result = :error

data/lib/inat-get/info.rb

@@ -5,7 +5,7 @@ module INatGet; end
 module INatGet::Info
 
   NAME = 'inat-get'
-  VERSION = '0.9.0.12'
+  VERSION = '0.9.0.14'
   VERSION_ALIAS = 'Carduelis carduelis'
 
   AUTHOR = 'Ivan Shikhalev'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inat-get
 version: !ruby/object:Gem::Version
-  version: 0.9.0.12
+  version: 0.9.0.14
 platform: ruby
 authors:
 - Ivan Shikhalev
@@ -51,6 +51,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: faraday-http-cache
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: faraday-typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: is-dsl
   requirement: !ruby/object:Gem::Requirement
@@ -71,14 +99,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.8.6
+        version: 0.8.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.8.6
+        version: 0.8.10
 - !ruby/object:Gem::Dependency
   name: is-duration
   requirement: !ruby/object:Gem::Requirement
@@ -250,6 +278,7 @@ files:
 - lib/inat-get/app.rb
 - lib/inat-get/app/application.rb
 - lib/inat-get/app/core/api.rb
+- lib/inat-get/app/core/api_cache.rb
 - lib/inat-get/app/core/console.rb
 - lib/inat-get/app/core/console_logger.rb
 - lib/inat-get/app/core/server.rb