inat-get 0.8.0.11

data/lib/inat/app/config/messagelevel.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class MessageLevel < Enum
+
+  item :TRACE, -1
+  item :DEBUG,   data: 0
+  item :INFO,    data: 1
+  item :WARNING, data: 2
+  item :ERROR,   data: 3
+  item :FATAL,   data: 4
+  item :UNKNOWN, data: 5
+
+  item_alias :WARN => :WARNING
+
+  def severity
+    self.data
+  end
+
+  freeze
+end

data/lib/inat/app/config/shiftage.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module ShiftAge
+
+  class << self
+
+    def parse source
+      source = source.to_i if /\d+/ === source
+      source = source.to_s.downcase.intern if Symbol === source || String === source
+      case source
+      when nil
+        0
+      when Integer
+        source
+      when :daily, :weekly, :monthly
+        source
+      else
+        raise TypeError, "Invalid shift age value: #{source}!"
+      end
+    end
+
+  end
+
+end

data/lib/inat/app/config/updatemode.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'extra/enum'
+
+class UpdateMode < Enum
+
+  items :UPDATE,
+        :FORCE,
+        :RELOAD,
+        :MINIMAL,
+        :OFFLINE
+
+  item_alias :DEFAULT       => :UPDATE,
+             :FORCE_UPDATE  => :FORCE,
+             :FORCE_RELOAD  => :RELOAD,
+             :SKIP_EXISTING => :MINIMAL,
+             :NO_UPDATE     => :OFFLINE
+
+  freeze
+end

data/lib/inat/app/config.rb

@@ -0,0 +1,296 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'optparse'
+
+require_relative '../utils/deep'
+require_relative '../../extra/period'
+require_relative 'info'
+require_relative 'config/messagelevel'
+require_relative 'config/updatemode'
+require_relative 'config/shiftage'
+
+class Application
+
+  include AppInfo
+
+  API_DEFAULT = 'https://api.inaturalist.org/v1/'
+  DEFAULT_LOG = "./#{ NAME }.log"
+
+  DEFAULTS = {
+    threads: {
+      enable: true,
+      tasks: 3,
+    },
+    data: {
+      cache: true,
+      directory: File.expand_path("~/.cache/#{ NAME }/"),
+      update: UpdateMode::UPDATE,
+      update_period: Period::DAY,
+      outdate_period: Period::WEEK,
+    },
+    verbose: MessageLevel::ERROR,
+    log: {
+      enable: false,
+      file: DEFAULT_LOG,
+      level: MessageLevel::WARNING,
+      shift: {
+        age: nil,
+        size: nil,
+      },
+    },
+    api: {
+      root: API_DEFAULT,
+      locale: nil,
+      preferred_place_id: nil,
+      open_timeout: nil,
+      read_timeout: nil,
+    },
+    preamble: [],
+  }
+
+  CONFIG_FILE = File.expand_path "~/.config/#{ NAME }.yml"
+
+  private def parse_files files
+    files.map do |file|
+      if file.start_with?('@')
+        list = file[1..]
+        File.readlines(list, chomp: true).filter { |l| !l.empty? }
+      else
+        file
+      end
+    end.flatten
+  end
+
+  private def parse_args!
+    options = {}
+    opts = OptionParser::new USAGE do |o|
+
+      o.accept UpdateMode do |mode|
+        UpdateMode::parse mode, case_sensitive: false
+      end
+
+      o.accept MessageLevel do |level|
+        MessageLevel::parse level, case_sensitive: false
+      end
+
+      o.accept ShiftAge do |shift_age|
+        ShiftAge::parse shift_age
+      end
+
+      o.accept Period do |period|
+        Period::parse period
+      end
+
+      o.separator ''
+      o.separator "\e[1mInformation:\e[0m"
+
+      o.on '-h', '--help', 'Show help and exit.' do
+        puts ABOUT
+        puts
+        puts opts.help
+        exit 0
+      end
+
+      o.on '-?', '--usage', 'Show short help and exit.' do
+        puts opts.help
+        exit 0
+      end
+
+      o.on '--about', 'Show program info and exit.' do
+        puts ABOUT
+        exit 0
+      end
+
+      o.on '--version', 'Show version and exit.' do
+        puts VERSION
+        exit 0
+      end
+
+      o.separator ''
+      o.separator "\e[1mConfiguration:\e[0m"
+
+      o.on '-c', '--config FILE', String, "Configuration file instead standard (~/.config/#{ NAME }.yml)." do |value|
+        options[:config_file] = value
+      end
+
+      o.separator ''
+      o.separator "\e[1mFlow Control:\e[0m"
+
+      o.on '-1', '--no-threads', 'Disable threads.' do
+        options[:threads] ||= {}
+        options[:threads][:enable] = false
+      end
+
+      o.on '-t', '--tasks NUMBER', Integer, 'Limit number of concurrent tasks.' do |value|
+        options[:threads] ||= {}
+        options[:threads][:tasks] = value
+      end
+
+      o.separator ''
+      o.separator "\e[1mCache Control:\e[0m"
+
+      o.on '-u', '--update MODE', UpdateMode, "Update mode:", "  'default' (or 'update')",
+                                                              "  'force-update' (or 'force')",
+                                                              "  'force-reload' (or 'reload')",
+                                                              "  'skip-existing' (or 'minimal')",
+                                                              "  'no-update' (or 'offline')" do |value|
+        options[:data] ||= {}
+        options[:data][:update] = value
+      end
+
+      o.on '-f', '--force-update', 'Force update.' do
+        options[:data] ||= {}
+        options[:data][:update] = UpdateMode::FORCE
+      end
+
+      o.on '-F', '--force-reload', '--reload', 'Force reload.' do
+        options[:data] ||= {}
+        options[:data][:update] = UpdateMode::RELOAD
+      end
+
+      o.on '--skip-existing', 'Skip updating if exist.' do
+        options[:data] ||= {}
+        options[:data][:update] = UpdateMode::MINIMAL
+      end
+
+      o.on '-n', '--offline', '--no-update', 'No updating.' do
+        options[:data] ||= {}
+        options[:data][:update] = UpdateMode::OFFLINE
+      end
+
+      o.separator ''
+
+      o.on '--update-period PERIOD', Period, 'Update period.' do |value|
+        options[:data] ||= {}
+        options[:data][:update_period] = value
+      end
+
+      o.on '--obsolete-period PERIOD', Period, 'Obsolete period.' do |value|
+        options[:data] ||= {}
+        options[:data][:obsolete_period] = value
+      end
+
+      o.on '--cache-directory PATH', String, 'Cache directory path.' do |value|
+        options[:data] ||= {}
+        options[:data][:directory] = value
+      end
+
+      o.separator ''
+
+      o.on '--clean-requests', 'Clean outdated requests in data cache.' do
+        options[:preamble] << :clean_requests
+      end
+
+      o.on '--clean-observations', 'Clean observations without request in data cache.' do
+        options[:preamble] << :clean_observations
+      end
+
+      o.on '--clean-orphans', 'Clean orphan objects in data cache.' do
+        options[:preamble] << :clean_orphans
+      end
+
+      o.on '--clean-all', 'Clean data cache.' do
+        options[:preamble] << :clean_all
+      end
+
+      o.separator ''
+      o.separator "\e[1mAPI Parameters:\e[0m"
+
+      o.on '-A', '--api-root URI', String, 'API root' do |value|
+        options[:api] ||= {}
+        options[:api][:root] = value
+      end
+
+      o.on '-L', '--locale LOCALE', String, 'Set the locale.' do |value|
+        value = nil if value.downcase == 'none'
+        options[:api] ||= {}
+        options[:api][:locale] = value
+      end
+
+      o.on '-P', '--preferred-place ID', Integer, 'Preferred place for API.' do |value|
+        value = nil if value == 0
+        options[:api] ||= {}
+        options[:api][:preferred_place_id] = value
+      end
+
+      o.separator ''
+      o.separator "\e[1mLogging Control:\e[0m"
+
+      o.on '--verbose-level LEVEL', MessageLevel, 'Set the logging level for $STDERR.', "  'fatal'",
+                                                                                            "  'error'",
+                                                                                            "  'warn' (or 'warning')",
+                                                                                            "  'info'",
+                                                                                            "  'debug'" do |value|
+        options[:verbose] = value
+      end
+
+      o.on '-v', '--verbose', 'Set verbose level to INFO.' do
+        options[:verbose] = MessageLevel::INFO
+      end
+
+      o.on '--log-level LEVEL', MessageLevel, 'Set the logging level for file log' do |value|
+        options[:log] ||= {}
+        options[:log][:level] = value
+      end
+
+      o.on '--log-file FILE', String, 'File name for log.' do |value|
+        value = DEFAULT_LOG if /^default$/i === value
+        options[:log] ||= {}
+        options[:log][:file] = value
+      end
+
+      o.on '-l', '--log [FILE]', String, "Enable logging to file. If file is not specified './#{ NAME }.log' used.",
+                                         "  'none', 'no' or 'false' to disabling." do |value|
+        options[:log] ||= {}
+        case value
+        when nil, /^(true|yes|default)$/i
+          options[:log][:enable] = true
+          options[:log][:file] = DEFAULT_LOG
+        when /^(false|no|none)$/i
+          options[:log][:enable] = false
+          options[:log][:file] = DEFAULT_LOG
+        else
+          options[:log][:enable] = true
+          options[:log][:file] = value.to_s
+        end
+      end
+
+      o.on '--log-shift-age AGE', ShiftAge, 'Log file shift age.' do |value|
+        value = nil if value == 0
+        options[:log] ||= {}
+        options[:log][:shift] ||= {}
+        options[:log][:shift][:age] = value
+      end
+
+      o.on '--log-shift-size SIZE', Integer, 'Log file shift size.' do |value|
+        value = nil if value == 0
+        options[:log] ||= {}
+        options[:log][:shift] ||= {}
+        options[:log][:shift][:size] = value
+      end
+
+      o.separator ''
+
+      o.separator "\e[1mFile Arguments:\e[0m"
+      o.separator "\t‹task[, ...]›\t\t     One or more names of task files.\n" +
+                  "\t\t\t\t     If name has not extension try to read '‹task›' than '‹task›.inat' than '‹task›.rb'."
+
+    end
+    files = parse_files(opts.parse(ARGV))
+    [ options, files ]
+  end
+
+  private def setup!
+    @config = DEFAULTS
+    options, @files = parse_args!
+    config_file = options[:config_file] || CONFIG_FILE
+    @config.deep_merge! YAML.load_file(config_file, symbolize_names: true, freeze: true) if File.exist?(config_file)
+    @config.deep_merge! options
+    @config[:verbose] = MessageLevel::parse @config[:verbose]
+    @config[:log][:level] = MessageLevel::parse @config[:log][:level]
+  end
+
+  attr_reader :config
+
+end

data/lib/inat/app/globals.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Globals
+
+  class << self
+
+    def logger
+      @@logger
+    end
+
+    def logger= value
+      @@logger = value
+    end
+
+    def current_task
+      Thread.current.thread_variable_get 'current_task'
+    end
+
+    def current_task= value
+      Thread.current.thread_variable_set 'current_task', value
+    end
+
+    def config
+      @@config
+    end
+
+    def config= value
+      @@config = value
+    end
+
+    def status
+      @@status
+    end
+
+    def status= value
+      @@status = value
+    end
+
+  end
+
+end
+
+G = Globals
+
+module LogDSL
+
+  def included mod
+    mod.extend LogDSL
+  end
+
+  def log message, level: MessageLevel::INFO
+    G.status.wrap do
+      G.logger.log(G.current_task&.name || '‹main›', level, message)
+    end
+  end
+
+  def error message, exception: nil
+    log message, level: MessageLevel::ERROR
+    raise exception, message if Class === exception && Exception >= exception
+  end
+
+  def warning message
+    log message, level: MessageLevel::WARNING
+  end
+
+  def info message
+    log message, level: MessageLevel::INFO
+  end
+
+  def debug message
+    log message, level: MessageLevel::DEBUG
+  end
+
+  def echo message, level: MessageLevel::UNKNOWN
+    log message, level: level
+  end
+
+  module_function :log, :error, :warning, :info, :debug, :echo
+
+end

data/lib/inat/app/info.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module AppInfo
+
+  AUTHOR = 'Ivan Shikhalev'
+  EMAIL = 'shkikhalev@gmail.com'
+
+  VERSION = '0.8.0.11'
+  HOMEPAGE = 'https://github.com/shikhalev/inat-get'
+  SOURCE_URL = 'https://github.com/shikhalev/inat-get'
+
+  EXE = File.basename $0
+  NAME = File.basename $0, '.rb'
+
+  USAGE = "\e[1mUsage $\e[0m #{EXE} [options] ‹task[, ...]›"
+  ABOUT = "\e[1mINat::Get v#{VERSION}\e[0m — A toolset for fetching and processing data from \e[4miNaturalist.org\e[0m.\n" +
+          "\n" +
+          "\e[1mSource:\e[0m \e[4m#{HOMEPAGE}\e[0m\n" +
+          "\e[1mAuthor:\e[0m #{AUTHOR} ‹\e[4m#{EMAIL}\e[0m›"
+
+end

data/lib/inat/app/logging.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+class DualLogger
+
+  def config
+    @application.config
+  end
+
+  def initialize application
+    @application = application
+    v_para = {
+      level: config[:verbose].severity
+    }
+    @verb = Logger::new STDERR, **v_para
+    if config[:log][:enable]
+      f_para = {
+        level: config[:log][:level].severity,
+      }
+      f_para[:shift_age] = config[:log][:shift][:age] if config[:log][:shift][:age]
+      f_para[:shift_level] = config[:log][:shift][:level] if config[:log][:shift][:level]
+      @file = Logger::new config[:log][:file], **f_para
+    else
+      @file = nil
+    end
+  end
+
+  def log task, level, message
+    task = task.name if task.respond_to?(:name)
+    @verb.log(level.severity, message, task)
+    @file.log(level.severity, message, task) if @file
+  end
+
+end

data/lib/inat/app/preamble.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module AppPreamble
+
+  private def do_clean_requests
+    # NEED: implement
+  end
+
+  private def do_clean_observations
+    # NEED: implement
+  end
+
+  private def do_clean_orphans
+    # NEED: implement
+  end
+
+  private def do_clean_all
+    # NEED: implement
+  end
+
+  private def preamble!
+    config[:preamble].each do |name|
+      self.send "do_#{name}"
+    end
+  end
+
+end

data/lib/inat/app/status.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative 'globals'
+
+# TODO: переделать на отдельный поток и очередь статусов. Ввести отдельно ключ, как идентификатор запроса, например.
+
+module Status
+
+  class << self
+
+    private def ellipsis src, len
+      return '' unless String === src
+      if src.length <= len
+        src
+      else
+        src[.. len - 4] + '...'
+      end
+    end
+
+    # private def get_num
+    #   @num ||= 0
+    #   @num += 1
+    #   @num
+    # end
+
+    def init
+      @lines = {}
+      @mutex = Mutex::new
+      @shift = 0
+    end
+
+    def up
+      $stderr.print "\e[#{ @shift }A\r" if @shift > 0
+    end
+
+    def out
+      $stderr.print "\e[0K\n"
+      @lines.each do |key, value|
+        $stderr.print "#{ value }\e[0K\n"
+      end
+      $stderr.print "\e[0K\n"
+      @shift = @lines.size + 2
+    end
+
+    def wrap
+      @mutex.synchronize do
+        up
+        yield
+        out
+      end
+    end
+
+    def status key, value
+      task = ellipsis G.current_task&.name, 16
+      key = task if key == nil
+      line = format("%16s | %s", key.to_s, value)
+      @mutex.synchronize do
+        @lines ||= {}
+        @lines[key] = line
+        up
+        out
+      end
+    end
+
+  end
+
+  private def status str
+    self.class.status str
+  end
+
+end
+
+Status::init
+G.status = Status

data/lib/inat/app/task/context.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative 'dsl'
+require_relative '../../report/table'
+
+class Task::Context
+
+  include Task::DSL
+  include TableDSL
+
+  attr_accessor :name
+
+  def config
+    @task.config
+  end
+
+  def logger
+    @task.logger
+  end
+
+  def done?
+    @done
+  end
+
+  def initialize task, name, path
+    @task = task
+    @name = name
+    @done = false
+    source_code = File.readlines(path).map { |line| "    #{line}" }.join('')
+    full_code = "define_singleton_method :execute do\n" +
+                "  begin\n" +
+                "    #{source_code}\n" +
+                "    Status::status nil, 'DONE'\n" +
+                "  rescue Exception => e\n" +
+                "    error \"\#{e.inspect}\"\n" +
+                "    error \"\#{e.backtrace.join(\"\\n\\t\")}\"\n" +
+                "    Status::status nil, 'ERROR : ' + e.inspect\n" +
+                "  rescue\n" +
+                "    error 'Unknown error!'\n" +
+                "    Status::status nil, 'ERROR'\n" +
+                "  end\n" +
+                "  @done = true\n" +
+                "end"
+    self.instance_eval full_code
+  end
+
+end

data/lib/inat/app/task/dsl.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative '../globals'
+require_relative '../config/messagelevel'
+
+require_relative '../../data/entity/observation'
+require_relative '../../data/query'
+require_relative '../../data/sets/dataset'
+require_relative '../../data/sets/list'
+
+class Task; end
+
+module Task::DSL
+
+  include LogDSL
+
+  def select **params
+    query = Query::new(**params)
+    DataSet::new nil, query.observations
+  end
+
+  module_function :select
+
+end