inat-get 0.8.0.11 → 0.8.0.13

data/lib/inat/data/api.rb

@@ -1,218 +1,247 @@
 # frozen_string_literal: true
 
-require 'json'
-require 'uri'
-require 'net/http'
+require "json"
+require "uri"
+require "net/http"
 
-require_relative '../app/globals'
-require_relative '../app/status'
+require_relative "../app/globals"
+require_relative "../app/status"
 
-module API
+class INat::API
 
   RECORDS_LIMIT = 200
   FREQUENCY_LIMIT = 1.0
 
-  class << self
+  using INat::Types::Std
 
-    include LogDSL
+  include INat::App::Logger::DSL
+  include INat::App
 
-    def init
-      @mutex = Mutex::new
-    end
+  def initialize
+    @mutex = Mutex::new
+  end
 
-    def get path, part, limit, *ids
-      return [] if ids.empty?
-      if ids.size > limit
-        rest = ids.dup
-        head = rest.shift limit
-        return get(path, *head) + get(path, *rest)
+  def get path, part, limit, *ids
+    return [] if ids.empty?
+    if ids.size > limit
+      rest = ids.dup
+      head = rest.shift limit
+      return get(path, *head) + get(path, *rest)
+    end
+    result = []
+    Status::status "[api]", "#{path} ..."
+    @mutex.synchronize do
+      now = Time::new
+      if @last_call && now - @last_call < FREQUENCY_LIMIT
+        sleep FREQUENCY_LIMIT - (now - @last_call)
       end
-      result = []
-      Status::status '[api]', "#{ path } ..."
-      @mutex.synchronize do
-        now = Time::new
-        if @last_call && now - @last_call < FREQUENCY_LIMIT
-          sleep FREQUENCY_LIMIT - (now - @last_call)
-        end
-        case part
-        when :query
-          url = G.config[:api][:root] + path.to_s + "?id=#{ ids.join(',') }"
-          url += "&per_page=#{ limit }"
-          locale = G.config[:api][:locale]
-          url += "&locale=#{ locale }" if locale
-          preferred_place_id = G.config[:api][:preferred_place_id]
-          url += "&preferred_place_id=#{ preferred_place_id }" if preferred_place_id
-        when :path
-          url = G.config[:api][:root] + path.to_s + "/#{ ids.join(',') }"
-        else
-          raise ArgumentError, "Invalid 'part' argument: #{ part.inspect }!", caller
-        end
-        uri = URI(url)
-        info "GET: URI = #{ uri.inspect }"
-        # Status::status 'GET', uri.to_s
-        https = uri.scheme == 'https'
-        open_timeout = G.config[:api][:open_timeout]
-        read_timeout = G.config[:api][:read_timeout]
-        http_options = {
-          use_ssl: https
-        }
-        http_options[:open_timeout] = open_timeout if open_timeout
-        http_options[:read_timeout] = read_timeout if read_timeout
-        answered = false
-        answer_count = 50
-        last_time = Time::new
-        until answered
-          begin
-            Net::HTTP::start uri.host, uri.port, **http_options do |http|
-              request = Net::HTTP::Get::new uri
-              request['User-Agent'] = G.config[:api][:user_agent] || "INat::Get // Unknown Instance"
-              response = http.request request
-              if Net::HTTPSuccess === response
-                data = JSON.parse(response.body)
-                result = data['results']
-                total = data['total_results']
-                paged = data['per_page']
-                time_diff = Time::new - last_time
-                debug "GET OK: total = #{ total } paged = #{ paged } time = #{ time_diff } "
-                # Status::status 'GET', uri.to_s + ' DONE'
-              else
-                error "Bad response om #{ uri.path }#{ uri.query && !uri.query.empty? && '?' + uri.query || '' }: #{ response.inspect }!"
-                result = [ { 'id' => ids.first } ]
-                # Status::status 'GET', uri.to_s + ' ERROR'
-              end
-            end
-            answered = true
-          rescue Exception
-            if answer_count > 0
-              answer_count -= 1
-              answered = false
-              error "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
-              # Status::status "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
-              sleep 2.0
+      case part
+      when :query
+        url = G.config[:api][:root] + path.to_s + "?id=#{ids.join(",")}"
+        url += "&per_page=#{limit}"
+        locale = G.config[:api][:locale]
+        url += "&locale=#{locale}" if locale
+        preferred_place_id = G.config[:api][:preferred_place_id]
+        url += "&preferred_place_id=#{preferred_place_id}" if preferred_place_id
+      when :path
+        url = G.config[:api][:root] + path.to_s + "/#{ids.join(",")}"
+      else
+        raise ArgumentError, "Invalid 'part' argument: #{part.inspect}!", caller
+      end
+      uri = URI(url)
+      info "GET: URI = #{uri.inspect}"
+      # Status::status 'GET', uri.to_s
+      https = uri.scheme == "https"
+      open_timeout = G.config[:api][:open_timeout]
+      read_timeout = G.config[:api][:read_timeout]
+      http_options = {
+        use_ssl: https,
+      }
+      http_options[:open_timeout] = open_timeout if open_timeout
+      http_options[:read_timeout] = read_timeout if read_timeout
+      answered = false
+      answer_count = 50
+      last_time = Time::new
+      until answered
+        begin
+          Net::HTTP::start uri.host, uri.port, **http_options do |http|
+            request = Net::HTTP::Get::new uri
+            request["User-Agent"] = G.config[:api][:user_agent] || "INat::Get // Unknown Instance"
+            response = http.request request
+            if Net::HTTPSuccess === response
+              data = JSON.parse(response.body)
+              result = data["results"]
+              total = data["total_results"]
+              paged = data["per_page"]
+              time_diff = Time::new - last_time
+              debug "GET OK: total = #{total} paged = #{paged} time = #{time_diff} "
+              # Status::status 'GET', uri.to_s + ' DONE'
             else
-              answered = true
-              error "Error in HTTP request: #{ $!.inspect }!"
-              # Status::status "Error in HTTP request: #{ $!.inspect }!"
+              error "Bad response om #{uri.path}#{uri.query && !uri.query.empty? && "?" + uri.query || ""}: #{response.inspect}!"
+              result = [{ "id" => ids.first }]
+              # Status::status 'GET', uri.to_s + ' ERROR'
             end
           end
+          answered = true
+        rescue Exception
+          if answer_count > 0
+            answer_count -= 1
+            answered = false
+            error "Error in HTTP request: #{$!.inspect}, retry: #{answer_count}."
+            # Status::status "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
+            sleep 2.0
+          else
+            answered = true
+            error "Error in HTTP request: #{$!.inspect}!"
+            # Status::status "Error in HTTP request: #{ $!.inspect }!"
+          end
         end
-        @last_call = Time::new
       end
-      Status::status '[api]', "#{ path } DONE"
-      result
+      @last_call = Time::new
     end
+    Status::status "[api]", "#{path} DONE"
+    result
+  end
 
-    private def make_url path, **params
-      url = G.config[:api][:root] + path.to_s
-      query = []
-      params.each do |key, value|
-        query_param = "#{ key }="
-        if Array === value
-          query_param += value.map(&:to_query).join(',')
-        else
-          query_param += value.to_query
-        end
-        query << query_param
-      end
-      locale = G.config[:api][:locale]
-      query << "locale=#{ locale }" if locale
-      preferred_place_id = G.config[:api][:preferred_place_id]
-      query << "preferred_place_id=#{ preferred_place_id }" if preferred_place_id
-      if !query.empty?
-        url += "?" + query.join('&')
+  private def make_url path, **params
+    url = G.config[:api][:root] + path.to_s
+    query = []
+    params.each do |key, value|
+      query_param = "#{key}="
+      if Array === value
+        query_param += value.map(&:to_query).join(",")
+      else
+        query_param += value.to_query
       end
-      url
+      query << query_param
+    end
+    locale = G.config[:api][:locale]
+    query << "locale=#{locale}" if locale
+    preferred_place_id = G.config[:api][:preferred_place_id]
+    query << "preferred_place_id=#{preferred_place_id}" if preferred_place_id
+    if !query.empty?
+      url += "?" + query.join("&")
     end
+    url
+  end
 
-    def query path, first_only: false, **params, &block
-      Status::status '[api]', "#{ path } ..."
-      para = params.dup
-      para.delete_if { |key, _| key.intern == :page }
-      para[:per_page] = RECORDS_LIMIT
-      para[:order_by] = :id
-      para[:order]    = :asc
-      result = []
-      rest = nil
-      total = 0
-      @mutex.synchronize do
-        now = Time::new
-        if @last_call && now - @last_call < FREQUENCY_LIMIT
-          sleep FREQUENCY_LIMIT - (now - @last_call)
-        end
-        url = make_url path, **para
-        uri = URI(url)
-        info "QUERY: URI = #{ uri.inspect }"
-        # Status::status 'QUERY', uri.to_s
-        https = uri.scheme == 'https'
-        open_timeout = G.config[:api][:open_timeout]
-        read_timeout = G.config[:api][:read_timeout]
-        http_options = {
-          use_ssl: https
-        }
-        http_options[:open_timeout] = open_timeout if open_timeout
-        http_options[:read_timeout] = read_timeout if read_timeout
-        answered = false
-        answer_count = 50
-        last_time = Time::new
-        until answered
-          begin
-            Net::HTTP::start uri.host, uri.port, **http_options do |http|
-              request = Net::HTTP::Get::new uri
-              request["User-Agent"] = G.config[:api][:user_agent] || "INat::Get // Unknown Instance"
-              response = http.request request
-              if Net::HTTPSuccess === response
-                data = JSON.parse(response.body)
-                result = data["results"]
-                total = data["total_results"]
-                paged = data["per_page"]
-                time_diff = Time::new - last_time
-                debug "QUERY OK: total = #{ total } paged = #{ paged } time = #{ time_diff } "
-                if total > paged && !first_only
-                  max = result.map { |o| o["id"] }.max
-                  rest = para
-                  rest[:id_above] = max
-                end
-              else
-                raise RuntimeError, "Invalid response: #{response.inspect}"
+  def query path, first_only: false, **params, &block
+    Status::status "[api]", "#{path} ..."
+    para = params.dup
+    para.delete_if { |key, _| key.intern == :page }
+    para[:per_page] = RECORDS_LIMIT
+    para[:order_by] = :id
+    para[:order] = :asc
+    result = []
+    rest = nil
+    total = 0
+    @mutex.synchronize do
+      now = Time::new
+      if @last_call && now - @last_call < FREQUENCY_LIMIT
+        sleep FREQUENCY_LIMIT - (now - @last_call)
+      end
+      url = make_url path, **para
+      uri = URI(url)
+      info "QUERY: URI = #{uri.inspect}"
+      # Status::status 'QUERY', uri.to_s
+      https = uri.scheme == "https"
+      open_timeout = G.config[:api][:open_timeout]
+      read_timeout = G.config[:api][:read_timeout]
+      http_options = {
+        use_ssl: https,
+      }
+      http_options[:open_timeout] = open_timeout if open_timeout
+      http_options[:read_timeout] = read_timeout if read_timeout
+      answered = false
+      answer_count = 50
+      last_time = Time::new
+      until answered
+        begin
+          Net::HTTP::start uri.host, uri.port, **http_options do |http|
+            request = Net::HTTP::Get::new uri
+            request["User-Agent"] = G.config[:api][:user_agent] || "INat::Get // Unknown Instance"
+            response = http.request request
+            if Net::HTTPSuccess === response
+              data = JSON.parse(response.body)
+              result = data["results"]
+              total = data["total_results"]
+              paged = data["per_page"]
+              time_diff = Time::new - last_time
+              debug "QUERY OK: total = #{total} paged = #{paged} time = #{time_diff} "
+              if total > paged && !first_only
+                max = result.map { |o| o["id"] }.max
+                rest = para
+                rest[:id_above] = max
               end
-            end
-            answered = true
-          rescue Exception
-            if answer_count > 0
-              answer_count -= 1
-              answered = false
-              error "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
-              # Status::status "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
-              sleep 2.0
             else
-              raise
+              raise RuntimeError, "Invalid response: #{response.inspect}"
             end
           end
+          answered = true
+        rescue Exception
+          if answer_count > 0
+            answer_count -= 1
+            answered = false
+            error "Error in HTTP request: #{$!.inspect}, retry: #{answer_count}."
+            # Status::status "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
+            sleep 2.0
+          else
+            raise
+          end
         end
-        @last_call = Time::new
       end
-      Status::status '[api]', "#{ path } DONE"
-      # TODO: переделать рекурсию в итерации
-      if block_given?
-        rr = []
-        result.each do |js_object|
-          rr << yield(js_object, total)
-        end
-        rr += query(path, **rest, &block) if rest
-        rr
-      else
-        result += query(path, **rest) if rest
-        result
+      @last_call = Time::new
+    end
+    Status::status "[api]", "#{path} DONE"
+    # TODO: переделать рекурсию в итерации
+    if block_given?
+      rr = []
+      result.each do |js_object|
+        rr << yield(js_object, total)
       end
+      rr += query(path, **rest, &block) if rest
+      rr
+    else
+      result += query(path, **rest) if rest
+      result
     end
+  end
 
+  # @api private
+  def load_file filename
+    data = JSON.parse File.read(filename)
+    data["results"]
+  end
+
+  class << self
+
+    private :new
+
+    # @api application
+    # Singleton instance
+    # @return [API]
+    def instance
+      @instance ||= new
+      @instance
+    end
+
+    # @api application
+    # Get one or more objects by id
+    # @return [Array<Hash>]
+    def get path, part, limit, *ids
+      instance.get path, part, limit, *ids
+    end
+
+    # @api application
+    def query path, first_only: false, **params, &block
+      instance.query path, first_only: first_only, **params, &block
+    end
+
+    # @api private
     def load_file filename
-      data = JSON.parse File.read(filename)
-      data['results']
+      instance.load_file filename
     end
 
   end
 
 end
-
-API::init

data/lib/inat/data/db.rb

@@ -6,10 +6,13 @@ require 'sqlite3'
 require_relative '../app/globals'
 require_relative 'ddl'
 
-class DB
+class INat::DB
 
-  include LogDSL
+  include INat
+  include INat::App
+  include INat::App::Logger::DSL
 
+  # @private
   def self.get_finalizer *dbs
     proc do
       dbs.each { |db| db.close }
@@ -27,7 +30,7 @@ class DB
       @data.auto_vacuum = 1
       @data.results_as_hash = true
       @data.foreign_keys = true
-      @data.execute_batch DDL.DDL
+      @data.execute_batch Data::DDL.DDL
     end
     ObjectSpace.define_finalizer self, self.class.get_finalizer(@data)
   end
@@ -65,8 +68,10 @@ class DB
 
   class << self
 
+    private :new
+
     def instance
-      @instance ||= DB::new
+      @instance ||= new
       @instance
     end
 

data/lib/inat/data/ddl.rb

@@ -1,16 +1,35 @@
 # frozen_string_literal: true
 
-module DDL
+# TODO: подумать и заменить константой, возможно
+
+class INat::Data::DDL
+
+  def initialize
+    @models = []
+  end
+
+  def <<(model)
+    @models << model
+  end
+
+  def DDL
+    @models.map(&:DDL).join("\n")
+  end
 
   class << self
 
-    def << model
-      @models ||= []
-      @models << model
+    private :new
+
+    def instance
+      @instance ||= new
+    end
+
+    def <<(model)
+      instance << model
     end
 
     def DDL
-      @models.map(&:DDL).join("\n")
+      instance.DDL
     end
 
   end

data/lib/inat/data/entity/comment.rb

@@ -4,11 +4,15 @@ require_relative '../types/std'
 require_relative '../types/extras'
 require_relative '../entity'
 
-autoload :Observation, 'inat/data/entity/observation'
-autoload :User,        'inat/data/entity/user'
-autoload :Flag,        'inat/data/entity/flag'
+module INat::Entity
+  autoload :Observation, 'inat/data/entity/observation'
+  autoload :User,        'inat/data/entity/user'
+  autoload :Flag,        'inat/data/entity/flag'
+end
+
+class INat::Entity::Comment < INat::Data::Entity
 
-class Comment < Entity
+  include INat::Entity
 
   table :comments
 

data/lib/inat/data/entity/flag.rb

@@ -4,10 +4,14 @@ require_relative '../types/std'
 require_relative '../types/extras'
 require_relative '../entity'
 
-autoload :Observation, 'inat/data/entity/observation'
-autoload :User,        'inat/data/entity/user'
+module INat::Entity
+  autoload :Observation, 'inat/data/entity/observation'
+  autoload :User,        'inat/data/entity/user'
+end
+
+class INat::Entity::Flag < INat::Data::Entity
 
-class Flag < Entity
+  include INat::Entity
 
   table :flags
 

data/lib/inat/data/entity/identification.rb

@@ -8,11 +8,16 @@ require_relative '../entity'
 
 require_relative '../enums/identificationcategory'
 
-autoload :Observation, 'inat/data/entity/observation'
-autoload :Taxon,       'inat/data/entity/taxon'
-autoload :Flag,        'inat/data/entity/flag'
+module INat::Entity
+  autoload :Observation, 'inat/data/entity/observation'
+  autoload :Taxon,       'inat/data/entity/taxon'
+  autoload :Flag,        'inat/data/entity/flag'
+end
+
+class INat::Entity::Identification < INat::Data::Entity
 
-class Identification < Entity
+  include INat::Data::Types
+  include INat::Entity
 
   api_path :identifications
   api_part :query

data/lib/inat/data/entity/observation.rb

@@ -8,20 +8,25 @@ require_relative '../enums/qualitygrade'
 require_relative '../enums/licensecode'
 require_relative '../enums/geoprivacy'
 
-autoload :Taxon,            'inat/data/entity/taxon'
-autoload :User,             'inat/data/entity/user'
-autoload :Flag,             'inat/data/entity/flag'
-autoload :ObservationSound, 'inat/data/entity/observationsound'
-autoload :Sound,            'inat/data/entity/sound'
-autoload :ObservationPhoto, 'inat/data/entity/observationphoto'
-autoload :Photo,            'inat/data/entity/photo'
-autoload :Place,            'inat/data/entity/place'
-autoload :Project,          'inat/data/entity/project'
-autoload :Comment,          'inat/data/entity/comment'
-autoload :Identification,   'inat/data/entity/identification'
-autoload :Vote,             'inat/data/entity/vote'
-
-class Observation < Entity
+module INat::Entity
+  autoload :Taxon,            'inat/data/entity/taxon'
+  autoload :User,             'inat/data/entity/user'
+  autoload :Flag,             'inat/data/entity/flag'
+  autoload :ObservationSound, 'inat/data/entity/observationsound'
+  autoload :Sound,            'inat/data/entity/sound'
+  autoload :ObservationPhoto, 'inat/data/entity/observationphoto'
+  autoload :Photo,            'inat/data/entity/photo'
+  autoload :Place,            'inat/data/entity/place'
+  autoload :Project,          'inat/data/entity/project'
+  autoload :Comment,          'inat/data/entity/comment'
+  autoload :Identification,   'inat/data/entity/identification'
+  autoload :Vote,             'inat/data/entity/vote'
+end
+
+class INat::Entity::Observation < INat::Data::Entity
+
+  include INat::Data::Types
+  include INat::Entity
 
   api_path :observations
   api_part :query
@@ -29,6 +34,8 @@ class Observation < Entity
 
   table :observations
 
+  # @!group Fields
+
   field :quality_grade, type: QualityGrade, required: true, index: true
   field :uuid, type: UUID, unique: true
   field :species_guess, type: String
@@ -101,6 +108,10 @@ class Observation < Entity
   field :photo_licensed, type: Boolean, index: true
   field :rank, type: Rank, index: true
 
+  # @!endgroup
+
+  # @!group Callbacks
+
   def post_update
     t = self.taxon
     if t
@@ -116,6 +127,8 @@ class Observation < Entity
     end
   end
 
+  # @!endgroup
+
   def normalized_taxon ranks
     ranks = (ranks .. ranks) if Rank === ranks
     raise TypeError, "Invalid type for ranks: #{ ranks.inspect }!", caller unless Range === ranks

data/lib/inat/data/entity/observationphoto.rb

@@ -9,10 +9,14 @@ require_relative '../entity'
 # require_relative 'observation'
 # require_relative 'photo'
 
-autoload :Observation, 'inat/data/entity/observation'
-autoload :Photo,       'inat/data/entity/photo'
+module INat::Entity
+  autoload :Observation, 'inat/data/entity/observation'
+  autoload :Photo,       'inat/data/entity/photo'
+end
+
+class INat::Entity::ObservationPhoto < INat::Data::Entity
 
-class ObservationPhoto < Entity
+  include INat::Entity
 
   table :observation_photos