inat-get 0.8.0.11

data/README.md

@@ -0,0 +1,16 @@
+<img src="docs/logo.png" align="right" style="float:right;width:57%;">
+
+# INat::Get
+
+A toolset for fetching and processing data from **[iNaturalist.org][inat]**.
+
+The author of this software is not associated or affiliated with the iNaturalist.
+Only the [public API][api] is used in accordance with [the conditions][cond].
+
+[inat]: https://www.inaturalist.org/
+[api]: https://api.inaturalist.org/v1/docs/
+[cond]: https://api.inaturalist.org/v1/docs/#terms-of-use
+
+## Current status
+
+Early alpha version.

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+task default: %i[]

data/bin/inat-get

@@ -0,0 +1,59 @@
+#!/usr/bin/ruby -w
+
+require 'date'
+require 'pp'
+
+# require 'extra/enum'
+
+require 'inat/app/application'
+require 'inat/app/globals'
+require 'inat/data/entity/observation'
+require 'inat/data/entity/request'
+require 'inat/data/db'
+require 'inat/data/query'
+
+app = Application.new
+app.run
+# PP::pp Globals.config, $>, 64
+
+# puts
+# puts DDL.DDL
+
+# PP::pp DB::instance, $>, 64
+
+# data = API::query(:observations, user_login: 'shikhalev', month: 11)
+
+# PP::pp data.map { |d| Observation::parse(d) }.size, $>, 64
+
+# https://www.inaturalist.org/projects/174222
+# https://www.inaturalist.org/places/193592
+# https://www.inaturalist.org/places/193881 (Ачитский район)
+# https://www.inaturalist.org/places/194104 (Богдановичский район)
+# https://www.inaturalist.org/places/194414 (Качканарский район)
+# https://www.inaturalist.org/places/194023 (Берёзовский район)
+# https://api.inaturalist.org/v1/observations?project_id=176067&updated_since=2023-10-31T19:36:29+05:00&per_page=200&order_by=id&order=asc&locale=ru&preferred_place_id=11829
+
+# W, [2023-11-02T01:54:41.818021 #8118]  WARN -- ‹main›: Some Taxon IDs were not fetched: 48460, 1, 47120, 245097, 47119, 47118, 120474, 342614, 319384, 67599, 495875, 153683, 153680, 367182, 48893, 1252003, 60920!
+# W, [2023-11-02T03:49:58.509031 #8118]  WARN -- ‹main›: Some Taxon IDs were not fetched: 1, 47118, 47119, 47120, 48460, 48893, 60920, 67599, 120474, 153680, 153683, 245097, 319384, 342614, 367182, 495875, 1252003!
+
+# query = Query::new project_id: 180212
+# PP::pp query.observations.size, $>, 64
+
+# require 'inat/data/types/apiquery'
+
+# pp Time::new.to_date.to_s
+
+# a = ApiQuery::new ololo: 1, lalala: UUID.generate
+
+# PP::pp a, $>, 64
+# puts a.to_query
+
+# Request::create ''
+
+# pp URI::parse('https://api.inaturalist.org/v1/observations?project_id=176067&updated_since=2023-10-31T19:36:29+05:00&per_page=200&order_by=id&order=asc&locale=ru&preferred_place_id=11829')
+
+# d = Date::parse '2023-11-01'
+# t = d.to_time
+# i = t.to_i
+
+# pp [ d, t, i ]

data/inat-get.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative 'lib/inat/app/info'
+
+Gem::Specification.new do |spec|
+  spec.name = "inat-get"
+  spec.version = AppInfo::VERSION
+  spec.authors = [ AppInfo::AUTHOR ]
+  spec.email = [ AppInfo::EMAIL ]
+
+  spec.summary = "Client for iNaturalist API."
+  # spec.description = "TODO: Write a longer description or delete this line."
+  spec.homepage = AppInfo::HOMEPAGE
+  spec.license = "GPL-3.0"
+  spec.required_ruby_version = ">= 3.1.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = AppInfo::SOURCE_URL
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[test/ spec/ features/ samples/ .git .circleci appveyor Gemfile])
+    end
+  end
+  spec.bindir = "bin"
+  spec.executables = [ 'inat-get' ]
+  spec.require_paths = [ "lib" ]
+
+  spec.add_dependency "sqlite3", "~> 1.6.6"
+  spec.add_dependency "tzinfo", "~> 2.0.6"
+
+end

data/lib/extra/enum.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+class Enum
+
+  class << self
+
+    def inherited sub
+      if self != ::Enum
+        raise TypeError, "Enum classes are closed!", caller
+      end
+    end
+
+    private :new
+
+    private def item name, order = nil, description: nil, data: nil
+      raise TypeError, "Name of enum value must be a Symbol!", caller unless Symbol === name
+      raise TypeError, "Order of enum value must be a Integer!", caller unless nil === order || Integer === order
+      raise TypeError, "Description of enum value must be a String!", caller unless nil === description || String === description
+      @values ||= []
+      @by_name ||= {}
+      @by_order ||= {}
+      order ||= (@values.map(&:order).max || 0) + 1
+      value = new name, order, description, data
+      value.freeze
+      @values << value
+      @by_name[name] = value
+      @by_order[order] = value
+      const_name = if name[0] != name[0].upcase
+        name.upcase
+      else
+        name
+      end
+      const_name = const_name.to_s.gsub '-', '_'
+      self.const_set const_name, value
+      return value
+    end
+
+    private def items *names, **names_with_order
+      names.each do |name|
+        item name
+      end
+      names_with_order.each do |name, order|
+        item name, order
+      end
+      return values
+    end
+
+    private def item_alias **params
+      @aliases ||= {}
+      params.each do |name, value|
+        raise TypeError, "Alias name must be a Symbol!", caller unless Symbol === name
+        value = self[value] if Symbol === value
+        raise TypeError, "Alias value must be a Symbol or #{ self.name }!", caller unless self === value
+        @aliases[name] = value
+        @by_name[name] = value
+        const_name = if name[0] != name[0].upcase
+          name.upcase
+        else
+          name
+        end
+        const_name = const_name.to_s.gsub '-', '_'
+        self.const_set const_name, value
+      end
+      return aliases
+    end
+
+    def values
+      @values ||= []
+      @values.sort_by(&:order).dup.freeze
+    end
+
+    def aliases
+      @aliases ||= {}
+      @aliases.dup.freeze
+    end
+
+    def to_a
+      values
+    end
+
+    def to_h
+      @by_name.dup.freeze
+    end
+
+    include Enumerable
+
+    def each
+      if block_given?
+        @values.sort_by(&:order).each do |item|
+          yield item
+        end
+      else
+        to_enum :each
+      end
+    end
+
+    private def get name_or_order
+      return nil if name == nil
+      result = @by_name[name_or_order] || @by_order[name_or_order]
+      raise ArgumentError, "Invalid name or order: #{ name_or_order.inspect }!", caller if result == nil
+      return result
+    end
+
+    def [] name_or_order
+      return name_or_order if self === name_or_order
+      case name_or_order
+      when Symbol, Integer
+        @by_name[name_or_order] || @by_order[name_or_order]
+      when Range
+        Range::new get(name_or_order.begin), get(name_or_order.end), name_or_order.exclude_end?
+      else
+        raise TypeError, "Invalid key: #{ name_or_order.inspect }!", caller
+      end
+    end
+
+    def parse src, case_sensitive: true
+      return src if self === src
+      raise TypeError, "Source must be a String!", caller unless String === src
+      src = src.strip
+      prefix = self.name + '::'
+      src = src[prefix.length ..] if src.start_with?(prefix)
+      return nil if src.empty?
+      key = if case_sensitive
+        @by_name.keys.find { |v| v.to_s == src }
+      else
+        src = src.downcase
+        @by_name.keys.find { |v| v.to_s.downcase == src }
+      end
+      raise NameError, "Invalid name: #{ src.inspect }!", caller if key.nil?
+      @by_name[key]
+    end
+
+  end
+
+  attr_reader :name, :order, :description, :data
+
+  def initialize name, order, description, data
+    @name = name
+    @order = order
+    @description = description
+    @data = data
+  end
+
+  def intern
+    @name
+  end
+
+  def to_s
+    @name.to_s
+  end
+
+  def to_i
+    @order
+  end
+
+  def inspect
+    const_name = @name
+    add = ''
+    if @name[0] != @name[0].upcase
+      const_name = "#{@name.upcase}"
+      add = " = #{ @name.inspect }"
+    end
+    add += " @data=#{ @data.inspect }" if @data
+    add += " @description=#{ @description.inspect }" if @description
+    const_name = const_name.to_s.gsub '-', '_'
+    "\#<#{ self.class.name }::#{ const_name }#{ add }>"
+  end
+
+  include Comparable
+
+  def <=> other
+    return nil if !(self.class === other)
+    @order <=> other.order
+  end
+
+  def succ
+    self.class.values.select { |v| v.order > @order }.first
+  end
+
+  def pred
+    self.class.values.select { |v| v.order < @order }.last
+  end
+
+end

data/lib/extra/period.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'time'
+require 'date'
+
+class Period
+
+  attr_reader :value
+
+  def initialize value
+    @value = value
+  end
+
+  class << self
+
+    private :new
+
+    def parse src
+      return nil if src == nil
+      return src if Period === src
+      raise TypeError, "Source (#{ src.inspect }) must be a String!", caller unless String === src
+      src = src.strip
+      value = 0
+      case src
+      when /^\d+$/
+        value = src.to_i
+      when /^(\d+[wW])?(\d+[dD])?(\d+[hH])?(\d+[mM])?(\d+[sS])?$/
+        weeks = /(\d+)[wW]/.match(src)
+        value += weeks[1].to_i * 7 * 24 * 60 * 60 if weeks
+        days = /(\d+)[dD]/.match(src)
+        value += days[1].to_i * 24 * 60 * 60 if days
+        hours = /(\d+)[hH]/.match(src)
+        value += hours[1].to_i * 60 * 60 if hours
+        minutes = /(\d+)[mM]/.match(src)
+        value += minutes[1].to_i * 60 if minutes
+        seconds = /(\d+)[sS]/.match(src)
+        value += seconds[1].to_i if seconds
+      else
+        raise ArgumentError, "Invalid source: #{ src }!", caller
+      end
+      new(value).freeze
+    end
+
+    def make weeks: 0, days: 0, hours: 0, minutes: 0, seconds: 0
+      raise TypeError, "Parameter 'weeks' must be an Integer!", caller   unless Integer === weeks
+      raise TypeError, "Parameter 'days' must be an Integer!", caller    unless Integer === days
+      raise TypeError, "Parameter 'hours' must be an Integer!", caller   unless Integer === hours
+      raise TypeError, "Parameter 'minutes' must be an Integer!", caller unless Integer === minutes
+      raise TypeError, "Parameter 'seconds' must be an Integer!", caller unless Integer === seconds
+      value =  seconds
+      value += minutes             * 60
+      value += hours          * 60 * 60
+      value += days      * 24 * 60 * 60
+      value += weeks * 7 * 24 * 60 * 60
+      new(value).freeze
+    end
+
+    def diff first, second
+      first = first.to_time if Date === first
+      second = second.to_time if Date === second
+      raise TypeError, "Parameter 'first' must be a Time or Date!", caller  unless Time === first
+      raise TypeError, "Parameter 'second' must be a Time or Date!", caller unless Time === second
+      value = first.to_i - second.to_i
+      new(value).freeze
+    end
+
+  end
+
+  WEEK   = make weeks:   1
+  DAY    = make days:    1
+  HOUR   = make hours:   1
+  MINUTE = make minutes: 1
+  SECOND = make seconds: 1
+  ZERO   = make seconds: 0
+
+  def weeks
+    g = sgn
+    w = @value.abs / (7 * 24 * 60 * 60)
+    g * w
+  end
+
+  def days all: true
+    g = sgn
+    d = @value.abs / (24 * 60 * 60)
+    all && g * d || g * (d % 7)
+  end
+
+  def hours all: false
+    g = sgn
+    h = @value.abs / (60 * 60)
+    all && g * h || g * (h % 24)
+  end
+
+  def minutes all: false
+    g = sgn
+    m = @value.abs / 60
+    all && g * m || g * (m % 60)
+  end
+
+  def seconds all: false
+    g = sgn
+    s = @value.abs
+    all && g * s || g * (s % 60)
+  end
+
+  def sgn
+    if @value == 0
+      0
+    elsif @value > 0
+      1
+    else
+      -1
+    end
+  end
+
+  def abs
+    self.class.make seconds: @value.abs
+  end
+
+  def to_i
+    @value
+  end
+
+  def to_s with_weeks: true
+    return '0' if @value == 0
+    return "-#{abs.to_s}" if @value < 0
+    w = with_weeks && weeks || 0
+    d = days all: !with_weeks
+    h = hours
+    m = minutes
+    s = seconds
+    result = ''
+    result += "#{ w }w" if w != 0
+    result += "#{ d }d" if d != 0
+    result += "#{ h }h" if h != 0
+    result += "#{ m }m" if m != 0
+    result += "#{ s }s" if s != 0
+    result
+  end
+
+  def to_hs
+    return "-#{abs.to_hs}" if @value < 0
+    h = hours all: true
+    m = minutes
+    s = seconds
+    format "%d:%02d:%02d", h, m, s
+  end
+
+  def inspect
+    "\#<Period: #{to_s}>"
+  end
+
+  include Comparable
+
+  def <=> other
+    return nil if other.nil?
+    @value <=> other.value
+  end
+
+  def +@
+    self
+  end
+
+  def -@
+    self.class.make(seconds: -@value)
+  end
+
+  def + other
+    raise TypeError, "Second argument must be a Period!", caller unless Period === other
+    self.class.make(seconds: @value + other.value)
+  end
+
+  def - other
+    raise TypeError, "Second argument must be a Period!", caller unless Period === other
+    self.class.make(seconds: @value - other.value)
+  end
+
+  def * num
+    raise TypeError, "Second argument must be a number!", caller unless Numeric === num
+    self.class.make(seconds: (@value * num).to_i)
+  end
+
+  def / num
+    raise TypeError, "Second argument must be a number!", caller unless Numeric === num
+    self.class.make(seconds: (@value / num).to_i)
+  end
+
+  class Num
+
+    attr_reader :value
+
+    def initialize num
+      @value = num
+    end
+
+    def * period
+      period * @value
+    end
+
+  end
+
+  def coerce num
+    case num
+    when Integer, Float, Rational
+      [ Period::Num::new(num), self ]
+    else
+      raise TypeError, "Cannot coerce Period to a #{ num.class }!", caller
+    end
+  end
+
+end
+
+class Time
+
+  alias :preperiod_plus :+
+  alias :preperiod_minus :-
+
+  def + arg
+    return self + arg.value if Period === arg
+    preperiod_plus arg
+  end
+
+  def - arg
+    return self + (-arg.value) if Period === arg
+    preperiod_minus arg
+  end
+
+end
+
+class Numeric
+
+  def weeks
+    Period::make(weeks: 1) * self
+  end
+
+  def days
+    Period::make(days: 1) * self
+  end
+
+  def hours
+    Period::make(hours: 1) * self
+  end
+
+  def minutes
+    Period::make(minutes: 1) * self
+  end
+
+  def seconds
+    Period::make(seconds: 1) * self
+  end
+
+end

data/lib/extra/uuid.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+class UUID
+
+  BASE   = 16
+  BYTES  = 16
+  DIGITS = 32
+
+  attr_reader :value
+
+  def initialize value
+    @value = value
+  end
+
+  class << self
+
+    private :new
+
+    def parse src
+      return nil if src == nil
+      raise TypeError, "Source must be a String!", caller unless String === src
+      src = src.strip
+      raise ArgumentError, "Invalid UUID source: #{ src.inspect }!", caller unless /^\h{8}\-\h{4}\-\h{4}\-\h{4}\-\h{12}$/ === src || /^\h{32}$/ === src
+      value = src.gsub('-', '').to_i(BASE)
+      new(value).freeze
+    end
+
+    def generate count = 1, prefix: nil, prefix_length: nil
+      if prefix == nil
+        if prefix_length != nil
+          raise ArgumentError, "Prefix length must be Integer < #{ BYTES }!", caller unless Integer === prefix_length && prefix_length < BYTES
+          prefix = SecureRandom.hex prefix_length
+        else
+          prefix = ''
+          prefix_length = 0
+        end
+      else
+        raise TypeError, "Prefix must be a String!", caller unless String === prefix
+        prefix = prefix.gsub('-', '')
+        prefix_length = prefix.length / 2
+      end
+      result = count.times.map { parse(prefix + SecureRandom.hex(BYTES - prefix_length)) }
+      if count == 1
+        result[0]
+      else
+        result
+      end
+    end
+
+    def make *args
+      raise ArgumentError, "Method receives 1..5 arguments!", caller unless (1..5) === args.size
+      raise ArgumentError, "Arguments must be Integers!", caller     unless args.all? { |a| Integer === a }
+      last = args.pop
+      value = 0
+      value += args[0] << 12 * 8 if args.size > 0
+      value += args[1] << 10 * 8 if args.size > 1
+      value += args[2] <<  8 * 8 if args.size > 2
+      value += args[3] <<  6 * 8 if args.size > 3
+      value += last
+      new(value).freeze
+    end
+
+  end
+
+  ZERO = make 0
+
+  def to_i
+    @value
+  end
+
+  def to_s
+    raw = @value.to_s(BASE)
+    raw = '0' * (DIGITS - raw.length) + raw if raw.length < DIGITS
+    "#{ raw[0..7] }-#{ raw[8..11] }-#{ raw[12..15] }-#{ raw[16..19] }-#{ raw[20..31] }"
+  end
+
+  def inspect
+    to_s
+  end
+
+  include Comparable
+
+  def <=> other
+    return nil unless UUID === other
+    @value <=> other.value
+  end
+
+end

data/lib/inat/app/application.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative 'config'
+require_relative 'logging'
+require_relative 'preamble'
+require_relative 'globals'
+require_relative 'task'
+
+class Application
+
+  include AppPreamble
+
+  def logger
+    G.logger
+  end
+
+  def initialize
+    setup!
+    G.config = @config.freeze
+    G.logger = DualLogger::new self
+  end
+
+  private def tasks!
+    @tasks = Queue::new
+    @files.each do |file|
+      @tasks.push Task::new(self, file)
+    end
+  end
+
+  private def start!
+    @threads = []
+    count = [ @tasks.size, G.config[:threads][:tasks] ].min
+    count.times do
+      @threads << Thread.start do
+        while !@tasks.empty?
+          task = @tasks.pop
+          task.execute
+        end
+      end
+    end
+    @threads.each { |th| th.join }
+  end
+
+  def run
+    preamble!
+    tasks!
+    start!
+  end
+
+end