inat-get 0.8.0.11

data/lib/inat/app/task.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative 'globals'
+require_relative 'task/context'
+
+class Task
+
+  CHECK_LIST = %w[ . .inat .rb ]
+  FILE_CHECK_LIST = %w[ .inat .iNat .INat .INAT .rb .RB ]
+
+  private_constant :CHECK_LIST, :FILE_CHECK_LIST
+
+  private def existing path
+    if File.exist?(path)
+      path
+    else
+      nil
+    end
+  end
+
+  private def try_extensions base, *extensions
+    FILE_CHECK_LIST.each do |exception|
+      path = base + exception
+      return path if File.exist?(path)
+    end
+    return nil
+  end
+
+  private def name_complete? source
+    s = source.downcase
+    CHECK_LIST.each do |check|
+      return true if s.end_with?(check)
+    end
+    return false
+  end
+
+  # TODO: переписать более внятно
+  private def get_names source
+    path = File.expand_path source
+    basename = File.basename(source, '.*')
+    return [ basename, existing(path) ] if name_complete?(source)
+    base = path.split('/')[..-2].join('/') + '/' + basename
+    name = try_extensions base, *FILE_CHECK_LIST
+    return [ basename, name ]
+  end
+
+  def config
+    @application.config
+  end
+
+  def logger
+    @application.logger
+  end
+
+  def name
+    @context&.name
+  end
+
+  def done?
+    @context&.done?
+  end
+
+  def initialize application, source
+    @application = application
+    @basename, @path = get_names source
+    raise ArgumentError, "File not found: #{source}!" if @path.nil?
+    @context = Task::Context::new self, @basename, @path
+  end
+
+  def execute
+    G.current_task = self
+    @context.execute
+  end
+
+end

data/lib/inat/data/api.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'uri'
+require 'net/http'
+
+require_relative '../app/globals'
+require_relative '../app/status'
+
+module API
+
+  RECORDS_LIMIT = 200
+  FREQUENCY_LIMIT = 1.0
+
+  class << self
+
+    include LogDSL
+
+    def init
+      @mutex = Mutex::new
+    end
+
+    def get path, part, limit, *ids
+      return [] if ids.empty?
+      if ids.size > limit
+        rest = ids.dup
+        head = rest.shift limit
+        return get(path, *head) + get(path, *rest)
+      end
+      result = []
+      Status::status '[api]', "#{ path } ..."
+      @mutex.synchronize do
+        now = Time::new
+        if @last_call && now - @last_call < FREQUENCY_LIMIT
+          sleep FREQUENCY_LIMIT - (now - @last_call)
+        end
+        case part
+        when :query
+          url = G.config[:api][:root] + path.to_s + "?id=#{ ids.join(',') }"
+          url += "&per_page=#{ limit }"
+          locale = G.config[:api][:locale]
+          url += "&locale=#{ locale }" if locale
+          preferred_place_id = G.config[:api][:preferred_place_id]
+          url += "&preferred_place_id=#{ preferred_place_id }" if preferred_place_id
+        when :path
+          url = G.config[:api][:root] + path.to_s + "/#{ ids.join(',') }"
+        else
+          raise ArgumentError, "Invalid 'part' argument: #{ part.inspect }!", caller
+        end
+        uri = URI(url)
+        info "GET: URI = #{ uri.inspect }"
+        # Status::status 'GET', uri.to_s
+        https = uri.scheme == 'https'
+        open_timeout = G.config[:api][:open_timeout]
+        read_timeout = G.config[:api][:read_timeout]
+        http_options = {
+          use_ssl: https
+        }
+        http_options[:open_timeout] = open_timeout if open_timeout
+        http_options[:read_timeout] = read_timeout if read_timeout
+        answered = false
+        answer_count = 50
+        last_time = Time::new
+        until answered
+          begin
+            Net::HTTP::start uri.host, uri.port, **http_options do |http|
+              request = Net::HTTP::Get::new uri
+              request['User-Agent'] = G.config[:api][:user_agent] || "INat::Get // Unknown Instance"
+              response = http.request request
+              if Net::HTTPSuccess === response
+                data = JSON.parse(response.body)
+                result = data['results']
+                total = data['total_results']
+                paged = data['per_page']
+                time_diff = Time::new - last_time
+                debug "GET OK: total = #{ total } paged = #{ paged } time = #{ time_diff } "
+                # Status::status 'GET', uri.to_s + ' DONE'
+              else
+                error "Bad response om #{ uri.path }#{ uri.query && !uri.query.empty? && '?' + uri.query || '' }: #{ response.inspect }!"
+                result = [ { 'id' => ids.first } ]
+                # Status::status 'GET', uri.to_s + ' ERROR'
+              end
+            end
+            answered = true
+          rescue Exception
+            if answer_count > 0
+              answer_count -= 1
+              answered = false
+              error "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
+              # Status::status "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
+              sleep 2.0
+            else
+              answered = true
+              error "Error in HTTP request: #{ $!.inspect }!"
+              # Status::status "Error in HTTP request: #{ $!.inspect }!"
+            end
+          end
+        end
+        @last_call = Time::new
+      end
+      Status::status '[api]', "#{ path } DONE"
+      result
+    end
+
+    private def make_url path, **params
+      url = G.config[:api][:root] + path.to_s
+      query = []
+      params.each do |key, value|
+        query_param = "#{ key }="
+        if Array === value
+          query_param += value.map(&:to_query).join(',')
+        else
+          query_param += value.to_query
+        end
+        query << query_param
+      end
+      locale = G.config[:api][:locale]
+      query << "locale=#{ locale }" if locale
+      preferred_place_id = G.config[:api][:preferred_place_id]
+      query << "preferred_place_id=#{ preferred_place_id }" if preferred_place_id
+      if !query.empty?
+        url += "?" + query.join('&')
+      end
+      url
+    end
+
+    def query path, first_only: false, **params, &block
+      Status::status '[api]', "#{ path } ..."
+      para = params.dup
+      para.delete_if { |key, _| key.intern == :page }
+      para[:per_page] = RECORDS_LIMIT
+      para[:order_by] = :id
+      para[:order]    = :asc
+      result = []
+      rest = nil
+      total = 0
+      @mutex.synchronize do
+        now = Time::new
+        if @last_call && now - @last_call < FREQUENCY_LIMIT
+          sleep FREQUENCY_LIMIT - (now - @last_call)
+        end
+        url = make_url path, **para
+        uri = URI(url)
+        info "QUERY: URI = #{ uri.inspect }"
+        # Status::status 'QUERY', uri.to_s
+        https = uri.scheme == 'https'
+        open_timeout = G.config[:api][:open_timeout]
+        read_timeout = G.config[:api][:read_timeout]
+        http_options = {
+          use_ssl: https
+        }
+        http_options[:open_timeout] = open_timeout if open_timeout
+        http_options[:read_timeout] = read_timeout if read_timeout
+        answered = false
+        answer_count = 50
+        last_time = Time::new
+        until answered
+          begin
+            Net::HTTP::start uri.host, uri.port, **http_options do |http|
+              request = Net::HTTP::Get::new uri
+              request["User-Agent"] = G.config[:api][:user_agent] || "INat::Get // Unknown Instance"
+              response = http.request request
+              if Net::HTTPSuccess === response
+                data = JSON.parse(response.body)
+                result = data["results"]
+                total = data["total_results"]
+                paged = data["per_page"]
+                time_diff = Time::new - last_time
+                debug "QUERY OK: total = #{ total } paged = #{ paged } time = #{ time_diff } "
+                if total > paged && !first_only
+                  max = result.map { |o| o["id"] }.max
+                  rest = para
+                  rest[:id_above] = max
+                end
+              else
+                raise RuntimeError, "Invalid response: #{response.inspect}"
+              end
+            end
+            answered = true
+          rescue Exception
+            if answer_count > 0
+              answer_count -= 1
+              answered = false
+              error "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
+              # Status::status "Error in HTTP request: #{ $!.inspect }, retry: #{ answer_count }."
+              sleep 2.0
+            else
+              raise
+            end
+          end
+        end
+        @last_call = Time::new
+      end
+      Status::status '[api]', "#{ path } DONE"
+      # TODO: переделать рекурсию в итерации
+      if block_given?
+        rr = []
+        result.each do |js_object|
+          rr << yield(js_object, total)
+        end
+        rr += query(path, **rest, &block) if rest
+        rr
+      else
+        result += query(path, **rest) if rest
+        result
+      end
+    end
+
+    def load_file filename
+      data = JSON.parse File.read(filename)
+      data['results']
+    end
+
+  end
+
+end
+
+API::init

data/lib/inat/data/cache.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# class Cache
+
+#   class << self
+
+#   end
+
+# end

data/lib/inat/data/db.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'sqlite3'
+
+require_relative '../app/globals'
+require_relative 'ddl'
+
+class DB
+
+  include LogDSL
+
+  def self.get_finalizer *dbs
+    proc do
+      dbs.each { |db| db.close }
+    end
+  end
+
+  def initialize
+    @mutex = Mutex::new
+    @config = G.config
+    @directory = @config[:data][:directory]
+    FileUtils.mkpath @directory
+    @data = SQLite3::Database::open "#{@directory}/inat-cache.db"
+    @mutex.synchronize do
+      @data.encoding = 'UTF-8'
+      @data.auto_vacuum = 1
+      @data.results_as_hash = true
+      @data.foreign_keys = true
+      @data.execute_batch DDL.DDL
+    end
+    ObjectSpace.define_finalizer self, self.class.get_finalizer(@data)
+  end
+
+  def execute query, *args
+    Status::status '[db]', '...'
+    result = []
+    @mutex.synchronize do
+      last_time = Time::new
+      info "DB: query = #{ query } args = #{ args.inspect }"
+      result = @data.execute query, args
+      time_diff = Time::new - last_time
+      debug "DB OK: count = #{ Array === result && result.size || 'none' } time = #{ (time_diff * 1000000).to_i }ns"
+    end
+    Status::status '[db]', 'DONE'
+    result
+  end
+
+  def execute_batch query
+    Status::status '[db]', '...'
+    @mutex.synchronize do
+      last_time = Time::new
+      info "DB: batch = #{ query }"
+      @data.execute_batch "BEGIN TRANSACTION;\n" + query + "\nCOMMIT;\n"
+      time_diff = Time::new - last_time
+      debug "DB OK: time = #{ (time_diff * 1000000).to_i }ns"
+    end
+    Status::status '[db]', 'DONE'
+  end
+
+  # def transaction &block
+  #   raise ArgumentError, "Block is required?", caller unless block_given?
+  #   @data.transaction(&block)
+  # end
+
+  class << self
+
+    def instance
+      @instance ||= DB::new
+      @instance
+    end
+
+    def execute query, *args
+      instance.execute query, *args
+    end
+
+    def execute_batch query
+      instance.execute_batch query
+    end
+
+    # def transaction &block
+    #   instance.transaction(&block)
+    # end
+
+  end
+
+end

data/lib/inat/data/ddl.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DDL
+
+  class << self
+
+    def << model
+      @models ||= []
+      @models << model
+    end
+
+    def DDL
+      @models.map(&:DDL).join("\n")
+    end
+
+  end
+
+end

data/lib/inat/data/entity/comment.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative '../types/std'
+require_relative '../types/extras'
+require_relative '../entity'
+
+autoload :Observation, 'inat/data/entity/observation'
+autoload :User,        'inat/data/entity/user'
+autoload :Flag,        'inat/data/entity/flag'
+
+class Comment < Entity
+
+  table :comments
+
+  field :observation, type: Observation, index: true
+
+  field :uuid, type: UUID, unique: true
+  field :user, type: User, index: true
+  field :created_at, type: Time, required: true
+  field :body, type: String
+  field :hidden, type: Boolean, index: true
+
+  links :flags, item_type: Flag, owned: true
+
+  ignore :moderator_actions      # TODO: разобраться
+
+  ignore :created_at_details
+
+end

data/lib/inat/data/entity/flag.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative '../types/std'
+require_relative '../types/extras'
+require_relative '../entity'
+
+autoload :Observation, 'inat/data/entity/observation'
+autoload :User,        'inat/data/entity/user'
+
+class Flag < Entity
+
+  table :flags
+
+  field :flag, type: String, required: true
+  field :created_at, type: Time
+  field :updated_at, type: Time
+  field :user, type: User, index: true, required: true
+  field :resolved, type: Boolean, index: true
+  field :resolver, type: User, index: true
+  field :comment, type: String
+
+end

data/lib/inat/data/entity/identification.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'extra/uuid'
+
+require_relative '../types/std'
+require_relative '../types/extras'
+require_relative '../entity'
+
+require_relative '../enums/identificationcategory'
+
+autoload :Observation, 'inat/data/entity/observation'
+autoload :Taxon,       'inat/data/entity/taxon'
+autoload :Flag,        'inat/data/entity/flag'
+
+class Identification < Entity
+
+  api_path :identifications
+  api_part :query
+  api_limit 200
+
+  table :identifications
+
+  field :observation, type: Observation, index: true
+
+  field :uuid, type: UUID, unique: true
+  field :user, type: User, index: true, required: true
+  field :created_at, type: Time, index: true
+  field :body, type: String
+  field :category, type: IdentificationCategory, index: true
+  field :current, type: Boolean
+  field :own_observation, type: Boolean, index: true
+  field :vision, type: Boolean
+  field :disagreement, type: Boolean, index: true
+  field :spam, type: Boolean, index: true
+  field :hidden, type: Boolean, index: true
+  field :taxon, type: Taxon, index: true
+  field :previous_observation_taxon, type: Taxon, index: true
+
+  links :flags, item_type: Flag
+
+  ignore :taxon_change                    # TODO: разобраться
+  ignore :moderator_actions
+  ignore :created_at_details
+
+end

data/lib/inat/data/entity/observation.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require_relative '../types/std'
+require_relative '../types/extras'
+require_relative '../types/location'
+require_relative '../entity'
+require_relative '../enums/qualitygrade'
+require_relative '../enums/licensecode'
+require_relative '../enums/geoprivacy'
+
+autoload :Taxon,            'inat/data/entity/taxon'
+autoload :User,             'inat/data/entity/user'
+autoload :Flag,             'inat/data/entity/flag'
+autoload :ObservationSound, 'inat/data/entity/observationsound'
+autoload :Sound,            'inat/data/entity/sound'
+autoload :ObservationPhoto, 'inat/data/entity/observationphoto'
+autoload :Photo,            'inat/data/entity/photo'
+autoload :Place,            'inat/data/entity/place'
+autoload :Project,          'inat/data/entity/project'
+autoload :Comment,          'inat/data/entity/comment'
+autoload :Identification,   'inat/data/entity/identification'
+autoload :Vote,             'inat/data/entity/vote'
+
+class Observation < Entity
+
+  api_path :observations
+  api_part :query
+  api_limit 200
+
+  table :observations
+
+  field :quality_grade, type: QualityGrade, required: true, index: true
+  field :uuid, type: UUID, unique: true
+  field :species_guess, type: String
+  field :license_code, type: LicenseCode, index: true
+  field :description, type: String
+  field :captive, type: Boolean, index: true
+  field :taxon, type: Taxon, index: true
+  field :community_taxon, type: Taxon, index: true
+  field :uri, type: URI
+  field :obscured, type: Boolean, index: true
+  field :spam, type: Boolean, index: true
+  field :user, type: User, index: true
+  field :mappable, type: Boolean, index: true
+
+  links :flags, item_type: Flag, owned: true
+
+  field :time_observed_at, type: Time, index: true
+  field :created_at, type: Time, index: true
+  field :observed_on, type: Date, index: true
+  field :observed_on_string, type: String
+  field :updated_at, type: Time, index: true
+
+  field :geoprivacy, type: GeoPrivacy, index: true
+  field :taxon_geoprivacy, type: GeoPrivacy, index: true
+  field :location, type: Location, index: true
+  field :positional_accuracy, type: Integer, index: true
+  field :public_positional_accuracy, type: Integer, index: true
+  field :place_guess, type: String
+
+  backs :observation_sounds, item_type: ObservationSound, owned: true
+  links :sounds, item_type: Sound, table_name: :observation_sounds, owned: false
+  backs :observation_photos, item_type: ObservationPhoto, owned: true
+  links :photos, item_type: Photo, table_name: :observation_photos, owned: false
+
+  links :places, item_type: Place, owned: true
+  links :projects, item_type: Project, owned: true                       # Это только ручные
+  links :cached_projects, item_type: Project, table_name: :project_observations, owned: false
+
+  field :owners_identification_from_vision, type: Boolean
+  field :identifications_most_agree, type: Boolean
+  field :identifications_some_agree, type: Boolean
+  field :num_identification_agreements, type: Integer
+  field :identifications_most_disagree, type: Boolean
+  field :num_identification_disagreements, type: Integer
+  backs :comments, item_type: Comment, owned: true
+  backs :identifications, item_type: Identification, owned: true
+
+  links :ident_taxa, item_type: Taxon, ids_field: :ident_taxon_ids, index: true, owned: true
+
+  links :votes, item_type: Vote, owned: true
+  links :faves, item_type: Vote, owned: true
+
+  field :day, type: Integer, index: true
+  field :month, type: Integer, index: true
+  field :year, type: Integer, index: true
+
+  block :observed_on_details, type: Object do |value|
+    if Hash === value
+      self.day   = value['day']
+      self.month = value['month']
+      self.year  = value['year']
+    end
+  end
+
+  field :endemic, type: Boolean, index: true
+  field :introduced, type: Boolean, index: true
+  field :native, type: Boolean, index: true
+  field :taxon_is_active, type: Boolean, index: true
+  field :threatened, type: Boolean, index: true
+  field :photo_licensed, type: Boolean, index: true
+  field :rank, type: Rank, index: true
+
+  def post_update
+    t = self.taxon
+    if t
+      self.endemic = t.endemic
+      self.introduced = t.introduced
+      self.native = t.native
+      self.taxon_is_active = t.is_active
+      self.threatened = t.threatened
+      self.rank = t.rank
+    end
+    if self.observation_photos != nil
+      self.photo_licensed = self.observation_photos.any? { |p| p.photo.license_code != nil }
+    end
+  end
+
+  def normalized_taxon ranks
+    ranks = (ranks .. ranks) if Rank === ranks
+    raise TypeError, "Invalid type for ranks: #{ ranks.inspect }!", caller unless Range === ranks
+    min = ranks.begin
+    # max = ranks.end
+    taxon = self.taxon
+    while true do
+      return nil if taxon == nil
+      return nil if taxon.rank < min
+      return taxon if ranks === taxon.rank
+      taxon = taxon.parent
+    end
+  end
+
+  def sort_key
+    time_observed_at || observed_on&.to_time || Time::at(0)
+  end
+
+  ignore :tags                # TODO: implement
+  ignore :created_time_zone   # TODO: подумать...
+  ignore :observed_time_zone
+  ignore :time_zone_offset
+  ignore :geojson
+  ignore :annotations
+  # ignore :observed_on_details
+  ignore :created_at_details
+  ignore :cached_votes_total
+  ignore :comments_count
+  ignore :site_id
+  ignore :quality_metrics
+  ignore :reviewed_by
+  ignore :oauth_application_id
+  ignore :project_ids_with_curator_id
+  # ignore :place_ids
+  ignore :outlinks             # TODO
+  ignore :faves_count
+  ignore :ofvs
+  ignore :preferences
+  ignore :map_scale
+  ignore :identifications_count
+  ignore :project_ids_without_curator_id
+  ignore :project_observations  # TODO: ???
+  ignore :non_owner_ids
+  ignore :out_of_range
+  ignore :id_please
+  ignore :context_user_geoprivacy   # NEED: обязательно разобраться
+  ignore :context_geoprivacy
+  ignore :context_taxon_geoprivacy
+
+  def to_s
+    "<a href=\"https://www.inaturalist.org/observations/#{ id }\">\##{ id }</a>"
+  end
+
+end