inat-get 0.9.0.8 → 0.9.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 187fbbcea790e41c40ec4d14dd08c52fed9a6c207f5b4f512db425a63361e875
-  data.tar.gz: ed262c666cf618fa77a2223738583fc7b6ad7817cf1a5f32e94b3fc65fd8a75f
+  metadata.gz: 375524ed9be224c8d856b640812d33d449cc0306c7e3ae6f1366b720aa8f5156
+  data.tar.gz: d1917c1ef4b018535912c464ce491ac620068389f5c6f7107c24a0a5def577be
 SHA512:
-  metadata.gz: 1a7ad7ef87681e1852e4dbb23a39683da38a9c262b5e2bd9b883f3702b6744df5dc610d8014bd5a5ba793b916965ead871267235975c686e44b23d6c3757fb3f
-  data.tar.gz: 1cb4ac5c3a6c12b9e321189a9207d347f7a962cc3992c48b7a7e39187ec7993672eda415a708d88163c1cc563c012efc38d671e681475f4cbb02a5ba09216c8d
+  metadata.gz: b7800a7d299c507d2db2931e5cf7e617bdac8dac27b02b3461363b7487f4dfa5ed62f94fb15d4dd910a8bdeb01a470f1b1f6d345069cf618d8f7f151af414e17
+  data.tar.gz: 4b66a9790d97d26197d412f042e253fa2ccd4ed9fddfe4c0251127eafc13b8b9cefa0edd87a254fb42980d6c46543cfbe80a190ed878147cafb96db487c46d4d

data/.yardopts

@@ -0,0 +1,12 @@
+--title "INatGet"
+--markup markdown
+--readme README.md
+--files DSL.md
+--asset coverage-badge.svg
+--files share/inat-get/docs/*.md
+--protected
+--private
+--no-private
+--load .yardpreamble.rb
+--plugin is-sequel
+lib/**/*.rb

data/.yardpreamble.rb

@@ -0,0 +1,15 @@
+ENV['SEQUEL_MIGRATIONS_DIR'] = './share/inat-get/db/migrations/'
+
+module WF
+
+  def warn message, category = nil, **kwargs
+    case message
+    when /literal string will be frozen in the future/, /character class has duplicated range/, /yardpreamble/
+      # do nothing
+    else
+      super message
+    end
+  end
+
+end
+Warning.extend WF

data/coverage-badge.svg

@@ -0,0 +1,6 @@
+<?xml version="1.0"?><svg xmlns="http://www.w3.org/2000/svg" width="100" height="20">
+<rect width="100" height="20" fill="#555"/>
+<rect x="63" width="37" height="20" fill="red"/>
+<text x="8" y="14" fill="#fff" font-family="Verdana" font-size="11">coverage</text>
+<text x="66" y="14" fill="#fff" font-family="Verdana" font-size="11">68%</text>
+</svg>

data/lib/inat-get/app/maintenance.rb

@@ -69,7 +69,7 @@ module INatGet::App::Maintenance
     end
 
     def db_update config
-      run_migration config
+      run_migrations config
       exit Errno::NOERROR::Errno
     end
 
@@ -79,7 +79,7 @@ module INatGet::App::Maintenance
         $stderr.puts "❌ \e[1mVersion must be an integer\e[0m"
         exit Errno::ECANCELED::Errno
       end
-      run_migration config, target: target
+      run_migrations config, target: target
       exit Errno::NOERROR::Errno
     end
 

data/lib/inat-get/data/dsl/dataset.rb

@@ -85,7 +85,20 @@ class INatGet::Data::DSL::Dataset
   def % field
     # field = field.to_sym
     values = get_field_values field
+    total = if values.respond_to?(:count)
+      values.count
+    else
+      values.size
+    end
+    Thread::current[:total] ||= 0
+    Thread::current[:total] += total
+    console.update status: "grouping by #{ field }...", total: Thread::current[:total]
+    current = 0
+    Thread::current[:current] ||= 0
     dss = values.map do |value|
+      current += 1
+      Thread::current[:current] += 1
+      console.update status: "grouping by #{ field }...", current: Thread::current[:current]
       if value.is_a?(INatGet::Data::Model::Taxon)
         query = Q(self.condition.model, :taxon => value )
       else
@@ -94,6 +107,12 @@ class INatGet::Data::DSL::Dataset
       INatGet::Data::DSL::Dataset::new(value, self.condition & query, self.updated?)
     end
     INatGet::Data::DSL::List::new(*dss)
+  ensure
+    if current < total
+      Thread::current[:total] -= total - current
+      console.update total: Thread::current[:total]
+    end
+    console.update status: 'grouped'
   end
 
   # @return [Dataset]
@@ -112,13 +131,29 @@ class INatGet::Data::DSL::Dataset
   # @yield Block
   # @yieldparam [Sequel::Model] obj
   # @return [void]
-  def each &block
+  def each
     return to_enum(__method__) unless block_given?
     connect!
+    total = @dataset.count
+    Thread::current[:total] ||= 0
+    Thread::current[:total] += total
+    console.update status: 'processing...', total: Thread::current[:total]
+    current = 0
+    Thread::current[:current] ||= 0
     @dataset.each do |item|
       check_shutdown!
-      block.call item
+      current += 1
+      Thread::current[:current] += 1
+      console.update status: 'processing...', current: Thread::current[:current]
+      yield item
+    end
+  ensure
+    # Цикл может быть прерван досрочно, тогда нам нужно уменьшить total, чтобы результат сходился
+    if current < total
+      Thread::current[:total] -= total - current
+      console.update total: Thread::current[:total]
     end
+    console.update status: 'processed'
   end
 
   # @return [Integer]
@@ -171,7 +206,7 @@ class INatGet::Data::DSL::Dataset
       ranks = INatGet::Data::Enum::Rank::select { |r| r.level == field.level }
       subquery = taxon_id_at_ranks(*ranks)
       ids = query.distinct.select_map(subquery.as(:taxon_at_rank)).compact
-      INatGet::Data::Model::Taxon.where(id: ids).all
+      INatGet::Data::Model::Taxon.where(id: ids)
     elsif model.associations.include?(field)
       reflection = model.association_reflection(field)
       target = reflection.associated_class
@@ -179,14 +214,14 @@ class INatGet::Data::DSL::Dataset
       case reflection[:type]
       when :many_to_one
         ids = query.distinct.select_map(reflection[:key])
-        target.where(id: ids).all
+        target.where(id: ids)
       when :one_to_many, :many_to_many
         # Для всех типов "много" используем ассоциативный датасет
         # association_join делает join на основе метаданных связи
         ids = query.association_join(field)
                    .distinct
                    .select_map(reflection.qualified_right_key)
-        target.where(id: ids).all
+        target.where(id: ids)
       end
     else
       # field может быть как символом :column, так и Sequel.function(:month, :created_at)

data/lib/inat-get/data/dsl/dsl.rb

@@ -41,6 +41,17 @@ module INatGet::Data::DSL
     true
   end
 
+  # @private
+  class DummyConsole
+    def update(**kwargs)
+      # do nothing
+    end
+  end
+
+  def console
+    @console ||= Thread::current[:console] || DummyConsole::new
+  end
+
   # @endgroup
 
   # @group Date Utils

data/lib/inat-get/data/dsl/list.rb

@@ -48,6 +48,7 @@ class INatGet::Data::DSL::List
 
   # @return [self]
   def add! other
+    other = [ other ] if other.is_a?(INatGet::Data::DSL::Dataset)
     other.each do |ds|
       if @datasets.has_key?(ds.key)
         @datasets[ds.key] += ds
@@ -135,12 +136,30 @@ class INatGet::Data::DSL::List
   # @yieldparam [Dataset] ds
   def each
     return to_enum(__method__) unless block_given?
-    if @sorter
-      @datasets.values.sort_by(&@sorter).each { |ds| yield ds }
+    sorted = if @sorter
+      @datasets.values.sort_by(&@sorter)
     else
-      @datasets.each { |_, ds| yield ds }
+      @datasets.values
+    end
+    total = sorted.size
+    Thread::current[:total] ||= 0
+    Thread::current[:total] += total
+    console.update status: 'listing...', total: Thread::current[:total]
+    current = 0
+    Thread::current[:current] ||= 0
+    sorted.each do |ds|
+      current += 1
+      Thread::current[:current] += 1
+      console.update status: 'listing...', current: Thread::current[:current]
+      yield ds
     end
     self
+  ensure
+    if current < total
+      Thread::current[:total] -= total - current
+      console.update total: Thread::current[:total]
+    end
+    console.update status: 'listed'
   end
 
   # @return [Integer]

data/lib/inat-get/data/models/base.rb

@@ -48,6 +48,8 @@ module INatGet::Data
 
       def updater() = self.manager&.updater
 
+      def endpoint() = self.manager&.endpoint
+
     end
 
   end

data/lib/inat-get/data/models/place.rb

@@ -23,7 +23,7 @@ class INatGet::Data::Model::Place < INatGet::Data::Model
   def <=> other
     return nil unless other.is_a?(INatGet::Data::Model::Place)
     return 0 if self.id == other.id
-    self.slug <=> other.slug
+    (self.display_name || self.name || self.slug) <=> (other.display_name || other.name || other.slug)
   end
 
 end

data/lib/inat-get/data/models/project.rb

@@ -96,7 +96,7 @@ class INatGet::Data::Model::Project < INatGet::Data::Model
   def <=> other
     return nil unless other.is_a?(INatGet::Data::Model::Project)
     return 0 if self.id == other.id
-    self.slug <=> other.slug
+    (self.title || self.slug) <=> (other.title || other.slug)
   end
 
 end

data/lib/inat-get/data/updaters/base.rb

@@ -83,11 +83,30 @@ class INatGet::Data::Updater
   # @private
   def update_by_ids! *ids
     return [] if ids.empty?
-    interval = parse_duration(@config.dig(:caching, :refs, self.manager.endpoint) || @config.dig(:caching, :refs, :default))
+    interval = parse_duration(@config.dig(:caching, :refs, self.manager&.endpoint) || @config.dig(:caching, :refs, :default))
     if interval
       point = (Time::now.to_time - interval).to_time
-      # TODO: учесть slugs и uuids
-      fresh = self.model.where(id: ids, cached: (point .. )).select_map(:id)
+      ints = uuids = slugs = []
+      if self.manager&.uuid? || self.manager&.sid
+        ints, other = ids.compact.partition { |i| i.is_a?(Integer) }
+        if !self.manager&.uuid?
+          slugs = other
+        elsif !self.manager&.sid
+          uuids = other
+        else
+          uuids, slugs = other.partition { |i| i.is_a?(String) && i =~ INatGet::Data::Helper::UUID_PATTERN }
+        end
+      else
+        ints = ids
+      end
+      condition = { id: ints }
+      # condition = Sequel.|(condition, { id: ints }) unless ints.empty?
+      condition = Sequel.|(condition, { uuid: uuids }) unless uuids.empty?
+      condition = Sequel.|(condition, { self.manager&.sid => slugs }) unless slugs.empty?
+      condition = Sequel.&(condition, { cached: (point ..  ) })
+      fresh = self.model.where(condition).select_map(:id)
+      fresh += self.model.where(condition).select_map(:uuid) unless uuids.empty?
+      fresh += self.model.where(condition).select_map(self.manager&.sid) unless slugs.empty?
       ids -= fresh
     end
     ids.each_slice(self.slice_size) do |slice|

data/lib/inat-get/info.rb

@@ -5,7 +5,7 @@ module INatGet; end
 module INatGet::Info
 
   NAME = 'inat-get'
-  VERSION = '0.9.0.8'
+  VERSION = '0.9.0.10'
   VERSION_ALIAS = 'Carduelis carduelis'
 
   AUTHOR = 'Ivan Shikhalev'

data/share/inat-get/db/migrations/002_nullable_slug.rb

@@ -0,0 +1,13 @@
+# frozen_string_literals: true
+
+Sequel.migration do
+
+  change do
+
+    alter_table :places do
+      set_column_allow_null :slug, true
+    end
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inat-get
 version: !ruby/object:Gem::Version
-  version: 0.9.0.8
+  version: 0.9.0.10
 platform: ruby
 authors:
 - Ivan Shikhalev
@@ -240,9 +240,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".yardopts"
+- ".yardpreamble.rb"
 - LICENSE
 - README.md
 - bin/inat-get
+- coverage-badge.svg
 - lib/inat-get.rb
 - lib/inat-get/app.rb
 - lib/inat-get/app/application.rb
@@ -373,6 +376,7 @@ files:
 - lib/inat-get/utils/json.rb
 - lib/inat-get/utils/simple_singular.rb
 - share/inat-get/db/migrations/001_start_database.rb
+- share/inat-get/db/migrations/002_nullable_slug.rb
 - share/inat-get/demo/01_user_stat.rb
 - share/inat-get/demo/02_underfound.rb
 - share/inat-get/demo/03_newcomers.rb