inat-get 0.8.0.11

data/lib/inat/data/entity.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+
+require_relative '../app/globals'
+
+require_relative 'model'
+require_relative 'ddl'
+require_relative 'db'
+require_relative 'api'
+
+class Entity < Model
+
+  include LogDSL
+
+  class << self
+
+    include LogDSL
+
+    def inherited sub
+      sub.send :init
+      DDL << sub
+    end
+
+    private :new
+
+    private def init
+      @mutex = Mutex::new
+    end
+
+    private def update
+      raise ArgumentError, "Block is required!", caller unless block_given?
+      result = nil
+      exception = nil
+      @mutex.synchronize do
+        begin
+          result = yield
+        rescue Exception => e
+          exception = e
+        end
+      end
+      raise exception.class, exception.message, caller, cause: exception if exception
+      result
+    end
+
+    private def get id
+      return nil if id == nil
+      update do
+        @entities ||= {}
+        @entities[id] ||= new id
+        @entities[id]
+      end
+    end
+
+    def fetch *ids
+      return [] if ids.empty?
+      # sids = if ids.size > 7
+      #   ids.take(7).map(&:to_s) + [ 'and more' ]
+      # else
+      #   ids.map(&:to_s)
+      # end
+      # Status::status '[fetch]', "#{ self } : " + sids.join(', ') + ' ...'
+      result = ids.map { |id| get id }.filter { |x| x != nil }
+      nc_ids = result.select { |e| !e.complete? && !e.process? }.map(&:id)
+      read(*nc_ids)
+      nc_ids = result.select { |e| !e.complete? && !e.process? }.map(&:id)
+      load(*nc_ids)
+      nc_ids = result.select { |e| !e.complete? && !e.process? }.map(&:id)
+      warning "Some #{ self } IDs were not fetched: #{ nc_ids.join(', ') }!" unless nc_ids.empty?
+      # result = [ nil ] if result == []
+      # Status::status '[fetch]', "#{ self } : " + sids.join(', ') + ' DONE'
+      result
+    end
+
+    def by_id id
+      fetch(id).first
+    end
+
+    # TODO: подумать о переименовании
+    def from_db_rows data
+      result = []
+      data.each do |row|
+        id = row['id'] || row[:id]
+        raise TypeError, "Invalid data row: no 'id' field!" unless id
+        # check.delete id
+        entity = get id
+        entity.update(from_db: true) do
+          fields.each do |_, field|
+            case field.kind
+            when :value
+              name, value = field.from_db row
+              if name != nil && value != nil
+                entity.send "#{ name }=", value
+              end
+            when :links
+              ids = DB.execute("SELECT #{ field.link_field } FROM #{ field.table_name } WHERE #{ field.back_field } = ?", entity.id).map { |x| x[field.link_field.to_s] }
+              entity.send "#{ field.id_field }=", ids
+            when :backs
+              # TODO: подумать над тем, чтобы сразу загрузить: вынести парсинг отдельно...
+              ids = DB.execute("SELECT id FROM #{ field.type.table } WHERE #{ field.back_field } = ?", entity.id).map { |x| x['id'] }
+              entity.send "#{ field.id_field }=", ids
+            end
+          end
+        end
+        result << entity
+      end
+      result
+    end
+
+    def read *ids
+      return [] if ids.empty?
+      # check = ids.dup
+      # fields = self.fields
+      data = DB.execute "SELECT * FROM #{ self.table } WHERE id IN (#{ (['?'] * ids.size).join(',') })", *ids
+      from_db_rows data
+    end
+
+    def load *ids
+      return [] if ids.empty? || @api_path.nil?
+      data = API.get @api_path, @api_part, @api_limit, *ids
+      data.map { |obj| parse obj }
+    end
+
+    def load_file filename
+      data = API.load_file filename
+      data.map { |obj| parse obj }
+    end
+
+    def parse src
+      return nil if src == nil
+      raise TypeError, "Source must be a Hash! (#{ src.inspect })" unless Hash === src
+      # if !(Hash === src)
+      #   warning "INVALID SOURCE for #{ self }: #{ src.inspect }"
+      #   return nil
+      # end
+      id = src[:id] || src['id']
+      raise ArgumentError, "Source must have an Integer 'id' value!", caller unless Integer === id
+      fields = self.fields
+      entity = self.get id
+      entity.update do
+        src.each do |key, value|
+          key = key.intern if String === key
+          field = fields[key] || fields.values.find { |f| f.id_field == key }
+          raise ArgumentError, "Field not found in #{ self.name }: '#{ key }'!", caller unless field
+          if field.write?
+            unless (field.type === value) || (field.id_field == key && Integer === value)
+              if field.id_field == key
+                # do nothing
+              elsif field.type.respond_to?(:parse)
+                if Array === value
+                  if field.kind == :backs
+                    value = value.map { |v| field.type.parse(v.merge(field.back_field => entity.id)) }
+                  else
+                    value = value.map { |v| field.type.parse(v) }
+                  end
+                else
+                  value = field.type.parse(value)
+                end
+              else
+                raise TypeError, "Invalid '#{ key }' value type: #{ value.inspect }!", caller
+              end
+            end
+            entity.send "#{ key }=", value
+          end
+        end
+      end
+      entity
+      # entity.save
+    end
+
+    def ddl
+      "INTEGER REFERENCES #{ self.table } (id)"
+    end
+
+  end
+
+  field :id, type: Integer, primary_key: true
+
+  def initialize id
+    super()
+    self.id = id
+    # @saved = true
+    @s_count = 0
+  end
+
+  def complete?
+    fields = self.class.fields.values.select { |f| f.required? }
+    # values = fields.map { |f| [ f.name, send(f.name) ] }
+    fields.all? { |f| send(f.name) != nil }
+  end
+
+  def save
+    return self if @saved
+    # debug "Save #{ self.class.name } id = #{ self.id } saved = #{ @saved.inspect }"
+    @s_count += 1
+    debug "Saving count = #{ @s_count } [#{ self.class }: #{ self.id }]" if @s_count > 1
+    names = []
+    values = []
+    links = []
+    backs = []
+    # update do
+      self.class.fields.each do |_, field|
+        case field.kind
+        when :value
+          value = self.send(field.name)
+          if Entity === value && value != self # && !value.process?
+            value.save
+          end
+          name, value = field.to_db value
+          if name != nil && value != nil
+            names << name
+            values << value
+          end
+        when :links
+          links << { field: field, values: self.send(field.name) } if field.owned?
+        when :backs
+          backs << { field: field, values: self.send(field.name) } if field.owned?
+        end
+      end
+    # end
+    names = names.flatten
+    values = values.flatten
+    # DB.transaction do |db|
+      DB.execute "INSERT OR REPLACE INTO #{ self.class.table } (#{ names.join(',') }) VALUES (#{ (['?'] * values.size).join(',') });", *values
+      @saved = true
+      links.each do |link|
+        field = link[:field]
+        values = link[:values]
+        olinks = []
+        values.each do |value|
+          value.save if value != self
+          # DB.execute "INSERT OR REPLACE INTO #{ field.table_name } (#{ field.back_field }, #{ field.link_field }) VALUES (?, ?);", self.id, value.id
+          olinks << "INSERT OR REPLACE INTO #{ field.table_name } (#{ field.back_field }, #{ field.link_field }) VALUES (#{ self.id }, #{ value.id });"
+        end
+        DB.execute_batch olinks.join("\n")
+        news = values.map(&:id)
+        olds = DB.execute("SELECT #{ field.link_field } as id FROM #{ field.table_name } WHERE #{ field.back_field } = ?;", self.id).map { |r| r['id'] }
+        diff = olds.filter { |o| !news.include?(o) }
+        if !diff.empty?
+          DB.execute "DELETE FROM #{ field.table_name } WHERE #{ field.back_field } = ? AND #{ field.link_field } IN (#{ (['?'] * diff.size).join(',') });",
+                      self.id, *diff
+        end
+      end
+      backs.each do |back|
+        field = back[:field]
+        values = back[:values]
+        values.each do |value|
+          value.send "#{ field.back_field }=", self.id
+          value.save if value != self
+        end
+        news = values.map(&:id)
+        olds = DB.execute("SELECT id FROM #{ field.type.table } WHERE #{ field.back_field } = ?;", self.id).map { |r| r['id'] }
+        diff = olds.filter { |o| !news.include?(o) }
+        if !diff.empty?
+          DB.execute "DELETE FROM #{ field.type.table } WHERE #{ field.back_field } = ? AND id IN (#{ (['?'] * diff.size).join(',') });",
+                      self.id, *diff
+        end
+      end
+    # end
+    # @saved = true
+    # TODO: разобраться и почистить двойное присваивание
+    self
+  end
+
+  def to_db
+    self.id
+  end
+
+end
+
+module BySLUG
+
+  def by_slug slug
+    # Status::status '[fetch]', "#{ self } : #{ slug } ..."
+    @entities ||= {}
+    results = @entities.values.select { |e| e.slug == slug.intern }
+    if results.empty?
+      data = DB.execute "SELECT * FROM #{ table } WHERE slug = ?", slug.to_s
+      results = from_db_rows data
+    end
+    if results.empty?
+      data = API.get @api_path, :path, 1, slug
+      results = data.map { |d| parse(d) }
+    end
+    # Status::status '[fetch]', "#{ self } : #{ slug } DONE"
+    if results.empty?
+      nil
+    else
+      results.first
+    end
+  end
+
+end

data/lib/inat/data/enums/conservationstatus.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class CS < Enum
+
+  item :NE, data:  0
+  item :DD, data:  5
+  item :LC, data: 10
+  item :NT, data: 20
+  item :VU, data: 30
+  item :EN, data: 40
+  item :CR, data: 50
+  item :EW, data: 60
+  item :EX, data: 70
+
+  item_alias :not_evaluated         => :NE,
+             :data_deficient        => :DD,
+             :least_concern         => :LC,
+             :near_threatened       => :NT,
+             :vulnerable            => :VU,
+             :endangered            => :EN,
+             :critically_endangered => :CR,
+             :extinct_in_the_wild   => :EW,
+             :extinct               => :EX
+
+  freeze
+end
+
+ConservationStatus = CS

data/lib/inat/data/enums/geoprivacy.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class GeoPrivacy < Enum
+
+  items :open,
+        :obscured,
+        :obscured_private
+
+  item_alias :private => :obscured_private
+
+  freeze
+end

data/lib/inat/data/enums/iconictaxa.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class IconicTaxa < Enum
+
+  items :Aves,
+        :Amphibia,
+        :Reptilia,
+        :Mammalia,
+        :Actinopterygii,
+        :Mollusca,
+        :Arachnida,
+        :Insecta,
+        :Animalia,
+        :Plantae,
+        :Fungi,
+        :Protozoa,
+        :Chromista,
+        :unknown
+
+  freeze
+end

data/lib/inat/data/enums/identificationcategory.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class IdentificationCategory < Enum
+
+  items :improving,
+        :supporting,
+        :leading,
+        :maverick
+
+  freeze
+end

data/lib/inat/data/enums/licensecode.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class LicenseCode < Enum
+
+  items :'cc0',
+        :'cc-by',
+        :'cc-by-nc',
+        :'cc-by-nd',
+        :'cc-by-sa',
+        :'cc-by-nc-nd',
+        :'cc-by-nc-sa'
+
+  freeze
+end

data/lib/inat/data/enums/projectadminrole.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class ProjectAdminRole < Enum
+
+  items :curator,
+        :manager
+
+  freeze
+end

data/lib/inat/data/enums/projecttype.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class ProjectType < Enum
+
+  items :collection,
+        :umbrella,
+        :contest,
+        :bioblitz,
+        :assessment,
+        :manual
+
+  # TODO: переделать тип во что-то универсальное. наверное.
+
+  class << self
+
+    def parse src
+      if src == ''
+        return ProjectType::MANUAL
+      else
+        super src
+      end
+    end
+
+  end
+
+  def to_s
+    if self == ProjectType::MANUAL
+      return ''
+    else
+      super
+    end
+  end
+
+  freeze
+end

data/lib/inat/data/enums/qualitygrade.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class QualityGrade < Enum
+
+  items :research,
+        :needs_id,
+        :casual
+
+  freeze
+end

data/lib/inat/data/enums/rank.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class Rank < Enum
+
+  item :stateofmatter, data: 100
+  item :kingdom,       data:  70
+  item :phylum,        data:  60
+  item :subphylum,     data:  57
+  item :superclass,    data:  53
+  item :class,         data:  50
+  item :subclass,      data:  47
+  item :infraclass,    data:  45
+  item :subterclass,   data:  44
+  item :superorder,    data:  43
+  item :order,         data:  40
+  item :suborder,      data:  37
+  item :infraorder,    data:  35
+  item :parvorder,     data:  34.5
+  item :zoosection,    data:  34
+  item :zoosubsection, data:  33.5
+  item :superfamily,   data:  33
+  item :epifamily,     data:  32
+  item :family,        data:  30
+  item :subfamily,     data:  27
+  item :supertribe,    data:  26
+  item :tribe,         data:  25
+  item :subtribe,      data:  24
+  item :genus,         data:  20
+  item :genushybrid,   data:  20
+  item :subgenus,      data:  15
+  item :section,       data:  13
+  item :subsection,    data:  12
+  item :complex,       data:  11
+  item :species,       data:  10
+  item :hybrid,        data:  10
+  item :subspecies,    data:   5
+  item :variety,       data:   5
+  item :form,          data:   5
+  item :infrahybrid,   data:   5
+
+  item_alias :division       => :phylum,
+             :'sub-class'    => :subclass,
+             :'super-order'  => :superorder,
+             :'sub-order'    => :suborder,
+             :'super-family' => :superfamily,
+             :'sub-family'   => :subfamily,
+             :gen            => :genus,
+             :sp             => :species,
+             :spp            => :species,
+             :infraspecies   => :subspecies,
+             :ssp            => :subspecies,
+             :'sub-species'  => :subspecies,
+             :subsp          => :subspecies,
+             :trinomial      => :subspecies,
+             :var            => :variety
+
+  freeze
+end