metacrunch 3.1.4 → 4.0.1

data/lib/metacrunch/db.rb

@@ -1,8 +0,0 @@
-require "sequel"
-
-module Metacrunch
-  class Db
-    require_relative "db/reader"
-    require_relative "db/writer"
-  end
-end

data/lib/metacrunch/db/reader.rb

@@ -1,33 +0,0 @@
-require "metacrunch/db"
-
-module Metacrunch
-  class Db::Reader
-
-    def initialize(database_connection_or_url, dataset_proc, options = {})
-      @rows_per_fetch = options.delete(:rows_per_fetch) || 1000
-
-      @db = if database_connection_or_url.is_a?(String)
-        Sequel.connect(database_connection_or_url, options)
-      else
-        database_connection_or_url
-      end
-
-      @dataset = dataset_proc.call(@db).unlimited
-
-      unless @dataset.opts[:order]
-        raise ArgumentError, "Metacrunch::Db::Reader requires the dataset be ordered."
-      end
-    end
-
-    def each(&block)
-      return enum_for(__method__) unless block_given?
-
-      @dataset.paged_each(rows_per_fetch: @rows_per_fetch, strategy: :filter) do |row|
-        yield(row)
-      end
-
-      self
-    end
-
-  end
-end

data/lib/metacrunch/db/writer.rb

@@ -1,55 +0,0 @@
-require "metacrunch/db"
-
-module Metacrunch
-  class Db::Writer
-
-    def initialize(database_connection_or_url, dataset_proc, options = {})
-      @use_upsert          = options.delete(:use_upsert)          || false
-      @id_key              = options.delete(:id_key)              || :id
-      @isolation_level     = options.delete(:isolation_level)     || :repeatable
-      @transaction_retries = options.delete(:transaction_retries) || 5
-
-      @db = if database_connection_or_url.is_a?(String)
-        Sequel.connect(database_connection_or_url, options)
-      else
-        database_connection_or_url
-      end
-
-      @dataset = dataset_proc.call(@db)
-    end
-
-    def write(data)
-      if data.is_a?(Array)
-        @db.transaction(isolation: @isolation_level, num_retries: @transaction_retries) do
-          data.each{|d| insert_or_upsert(d) }
-        end
-      else
-        insert_or_upsert(data)
-      end
-    end
-
-    def close
-      @db.disconnect
-    end
-
-  private
-
-    def insert_or_upsert(data)
-      @use_upsert ? upsert(data) : insert(data)
-    end
-
-    def insert(data)
-      @dataset.insert(data) if data
-    end
-
-    def upsert(data)
-      if data
-        rec = @dataset.where(@id_key => data[@id_key])
-        if 1 != rec.update(data)
-          insert(data)
-        end
-      end
-    end
-
-  end
-end

data/lib/metacrunch/fs.rb

@@ -1,6 +0,0 @@
-module Metacrunch
-  module Fs
-    require_relative "fs/reader"
-    require_relative "fs/entry"
-  end
-end

data/lib/metacrunch/fs/entry.rb

@@ -1,17 +0,0 @@
-module Metacrunch
-  class Fs::Entry
-
-    attr_reader :filename, :archive_filename, :contents
-
-    def initialize(filename:, archive_filename: nil, contents: nil)
-      @filename         = filename
-      @archive_filename = archive_filename.presence
-      @contents         = contents
-    end
-
-    def from_archive?
-      @archive_filename != nil
-    end
-
-  end
-end

data/lib/metacrunch/fs/reader.rb

@@ -1,63 +0,0 @@
-require "metacrunch/fs"
-require "rubygems/package"
-
-module Metacrunch
-  class Fs::Reader
-    include Metacrunch::ParallelProcessableReader
-
-    def initialize(filenames = nil)
-      @filenames = [*filenames].map{|f| f.presence}.compact
-    end
-
-    def each(&block)
-      return enum_for(__method__) unless block_given?
-
-      offset = 0 + process_index
-
-      while offset < @filenames.count do
-        _filename = @filenames[offset]
-
-        if is_archive?(_filename)
-         read_archive(_filename, &block)
-        else
-         read_regular_file(_filename, &block)
-        end
-
-        offset += number_of_processes
-      end
-    end
-
-  private
-
-    def is_archive?(filename)
-      filename.ends_with?(".tar") || filename.ends_with?(".tar.gz") || filename.ends_with?(".tgz")
-    end
-
-    def is_gzip_file?(filename)
-      filename.ends_with?(".gz") || filename.ends_with?(".tgz")
-    end
-
-    def read_regular_file(filename, &block)
-      if File.file?(filename)
-        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : File.open(filename, "r")
-        yield Fs::Entry.new(filename: filename, archive_filename: nil, contents: io.read)
-      end
-    end
-
-    def read_archive(filename, &block)
-      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : File.open(filename, "r")
-      tarReader = Gem::Package::TarReader.new(io)
-
-      tarReader.each do |_tar_entry|
-        if _tar_entry.file?
-          yield Fs::Entry.new(
-            filename: filename,
-            archive_filename: _tar_entry.full_name,
-            contents: _tar_entry.read
-          )
-        end
-      end
-    end
-
-  end
-end

data/lib/metacrunch/job/dsl/option_support.rb

@@ -1,102 +0,0 @@
-module Metacrunch
-  class Job::Dsl::OptionSupport
-
-    def register_options(args, require_args: false, &block)
-      options = {}
-      registry.instance_eval(&block)
-
-      registry.each do |key, opt_def|
-        # Set default value
-        options[key] = opt_def[:default]
-
-        # Register with OptionParser
-        if opt_def[:args].present?
-          option = parser.define(*opt_def[:args]) { |value| options[key] = value }
-
-          option.desc << "REQUIRED" if opt_def[:required]
-          option.desc << "DEFAULT: #{opt_def[:default]}" if opt_def[:default].present?
-
-          parser_options[key] = option
-        end
-      end
-
-      # Finally parse CLI options with OptionParser
-      args = parser.parse(args || [])
-
-      # Make sure required options are present
-      ensure_required_options!(options)
-
-      # Make sure args are present if required
-      ensure_required_args!(args) if require_args
-
-      options
-    end
-
-  private
-
-    def parser
-      @parser ||= OptionParser.new do |parser|
-        parser.banner = "Usage: metacrunch [options] JOB_FILE @@ [job-options] [ARGS]\nJob options:"
-      end
-    end
-
-    def parser_options
-      @parser_options ||= {}
-    end
-
-    def registry
-      @registry ||= OptionRegistry.new
-    end
-
-    def ensure_required_options!(options)
-      registry.each do |key, opt_def|
-        if opt_def[:required] && options[key].blank?
-          long_option = parser_options[key].long.try(:[], 0)
-          short_option = parser_options[key].short.try(:[], 0)
-
-          puts "Error: Required job option `#{long_option || short_option}` missing."
-          puts parser.help
-
-          exit(1)
-        end
-      end
-    end
-
-    def ensure_required_args!(args)
-      if args.blank?
-        puts "Error: Required ARGS are missing."
-        puts parser.help
-
-        exit(1)
-      end
-    end
-
-  private
-
-    class OptionRegistry
-
-      def add(name, *args, default: nil, required: false)
-        if default && required
-          raise ArgumentError, "You can't use `default` and `required` option at the same time."
-        end
-
-        options[name.to_sym] = {
-          args: args,
-          default: default,
-          required: required
-        }
-      end
-
-      def each(&block)
-        options.each(&block)
-      end
-
-    private
-
-      def options
-        @options ||= {}
-      end
-    end
-
-  end
-end

data/lib/metacrunch/parallel_processable_reader.rb

@@ -1,21 +0,0 @@
-module Metacrunch
-  module ParallelProcessableReader
-
-    def set_parallel_process_options(number_of_processes: 1, process_index: 0)
-      raise ArgumentError, "number_of_processes must be >= 1" if number_of_processes < 1
-      raise ArgumentError, "process_index must be >= 0" if process_index < 0
-
-      @number_of_processes = number_of_processes
-      @process_index = process_index
-    end
-
-    def number_of_processes
-      @number_of_processes || 1
-    end
-
-    def process_index
-      @process_index || 0
-    end
-
-  end
-end

data/lib/metacrunch/redis.rb

@@ -1,8 +0,0 @@
-require "redis"
-
-module Metacrunch
-  class Redis
-    require_relative "redis/queue_reader"
-    require_relative "redis/queue_writer"
-  end
-end

data/lib/metacrunch/redis/queue_reader.rb

@@ -1,43 +0,0 @@
-require "metacrunch/redis"
-
-module Metacrunch
-  class Redis::QueueReader
-    include Metacrunch::ParallelProcessableReader
-
-    def initialize(redis_connection_or_url, queue_name, options = {})
-      @queue_name = queue_name
-      raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
-
-      @blocking_mode = options.delete(:blocking) || false
-      @blocking_timeout = options.delete(:blocking_timeout) || 0
-
-      @redis = if redis_connection_or_url.is_a?(String)
-        ::Redis.new(url: redis_connection_or_url)
-      else
-        redis_connection_or_url
-      end
-    end
-
-    def each(&block)
-      return enum_for(__method__) unless block_given?
-
-      if @blocking_mode
-        while true
-          list, result = @redis.blpop(@queue_name, timeout: @blocking_timeout)
-          if result.present?
-            yield JSON.parse(result)
-          else
-            yield nil
-          end
-        end
-      else
-        while result = @redis.lpop(@queue_name)
-          yield JSON.parse(result)
-        end
-      end
-
-      self
-    end
-
-  end
-end

data/lib/metacrunch/redis/queue_writer.rb

@@ -1,39 +0,0 @@
-require "metacrunch/redis"
-
-module Metacrunch
-  class Redis::QueueWriter
-
-    def initialize(redis_connection_or_url, queue_name, options = {})
-      @queue_name = queue_name
-      raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
-
-      @save_on_close = options.delete(:save_on_close) || true
-
-      @redis = if redis_connection_or_url.is_a?(String)
-        ::Redis.new(url: redis_connection_or_url)
-      else
-        redis_connection_or_url
-      end
-    end
-
-    def write(data)
-      @redis.rpush(@queue_name, data)
-    rescue RuntimeError => e
-      if e.message =~ /maxmemory/
-        puts "Redis has reached maxmemory. Waiting 10 seconds and trying again..."
-        sleep(10)
-        retry
-      else
-        raise e
-      end
-    end
-
-    def close
-      if @redis
-        @redis.bgsave if @save_on_close
-        @redis.close
-      end
-    end
-
-  end
-end

data/lib/metacrunch/redis/writer.rb

@@ -1,33 +0,0 @@
-require "metacrunch/redis"
-
-module Metacrunch
-  class Redis::Writer
-
-    def initialize(redis_connection_or_url, options = {})
-      @save_on_close = options.delete(:save_on_close) || true
-
-      @key = options.delete(:key) || :key
-
-      @redis = if redis_connection_or_url.is_a?(String)
-        ::Redis.new(url: redis_connection_or_url)
-      else
-        redis_connection_or_url
-      end
-    end
-
-    def write(data)
-      key = data[@key]
-      raise ArgumentError, "No key found in data. Tried '#{@key}' but didn't found a value." unless key
-
-      @redis.set(key.to_s, data.to_json)
-    end
-
-    def close
-      if @redis
-        @redis.bgsave if @save_on_close
-        @redis.close
-      end
-    end
-
-  end
-end