teplohod 0.0.1.alpha1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0d50cedb72cdebc67d5e1f53545099061783de7d
+  data.tar.gz: 5af88cd316675cc4f24419de22a01b776500ff92
+SHA512:
+  metadata.gz: 06ff005b1d8c27575757b9cf6e48aed4b0137ec2caf0959e2b8b13b8b2e9c9fec4e70bd5a10d821de694929f906d310d07f02a677b9511b7a7b0a58ea9382b22
+  data.tar.gz: e07d47c681221a83639ba9b7a0d0738e7644e1e3fbbee1230611b3ab73ce9320fc47aaf33951cb377d1e532c3402a95a3dcf3411427ff79a78a2fafd7571a669

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rubocop.yml

@@ -0,0 +1,3 @@
+AllCops:
+  Exclude:
+    - 'dummy/**/*'

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.13.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in scrap.gemspec
+gemspec

data/README.md

@@ -0,0 +1,5 @@
+# Teplohod
+
+run example with `teplohod -r ./example/main.rb -w -v`.
+
+dashboard will work on `4567` port.

data/Rakefile

@@ -0,0 +1,21 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'rubocop/rake_task'
+require 'yard'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+RuboCop::RakeTask.new do |task|
+  task.fail_on_error = false
+  task.formatters = %w(simple)
+end
+
+YARD::Rake::YardocTask.new do |t|
+  t.files = ['lib/**/*.rb']
+end
+
+task default: %i(rubocop test yard)

data/bin/teplohod

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/teplohod/cli'
+
+begin
+  cli = Teplohod::CLI.instance
+  cli.prepare ARGV
+  cli.run
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/example/main.rb

@@ -0,0 +1,50 @@
+require 'teplohod'
+require 'teplohod/queue/memory/base'
+
+class StartQueue < Teplohod::Queue::Memory::Base
+end
+
+class ResultQueue < Teplohod::Queue::Memory::Base
+end
+
+class InitialPipeline < Teplohod::Pipeline::Base
+  self.queue = StartQueue.new
+
+  def name
+    "Initial Pipeline"
+  end
+
+  def desription
+    "Initial data there"
+  end
+
+  def on_empty_load
+    10.times do
+      queue.push rand(100)
+    end
+  end
+
+  def on_finish
+    on_empty_load
+  end
+end
+
+class ResultPipeline < Teplohod::Pipeline::Base
+  self.parent = InitialPipeline.new
+  self.queue = ResultQueue.new
+
+  def name
+    "Result Pipeline"
+  end
+
+  def result
+    "Result data there"
+  end
+end
+
+class DemoProvider < Teplohod::Provider::Base
+  self.tail = ResultPipeline.new
+end
+
+Teplohod::Manager.reload_providers!
+Teplohod::Manager.register_provider DemoProvider.new

data/lib/teplohod/cli.rb

@@ -0,0 +1,63 @@
+$stdout.sync = true
+
+require 'singleton'
+require 'optparse'
+require 'logger'
+
+require 'teplohod/utils'
+require 'teplohod/initializers'
+require 'teplohod/signal_handler'
+require 'teplohod'
+
+module Teplohod
+  class CLI
+    include Singleton
+    include Utils
+
+    def prepare(_args = ARGV)
+      Teplohod::Initializers::Options.run!
+      Teplohod::Initializers::Redis.run!
+      Teplohod::Initializers::Logger.run! unless options[:daemonize]
+      Teplohod::Initializers::Daemonizer.run! if options[:daemonize]
+      Teplohod::Initializers::Pid.run!
+    end
+
+    def run
+      Teplohod::Initializers::Boot.run!
+      Teplohod::Initializers::Web.run! if options[:web]
+      Teplohod::Initializers::Banner.run!
+
+      self_read, self_write = IO.pipe
+
+      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+        begin
+          trap sig do
+            self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "Signal #{sig} not supported"
+        end
+      end
+
+      require 'teplohod/launcher'
+      launcher = Teplohod::Launcher.new
+
+      begin
+        launcher.run
+
+        while readable_io = IO.select([self_read])
+          signal = readable_io.first[0].gets.strip
+          Teplohod::SignalHandler.handle(signal)
+        end
+      rescue Interrupt
+        logger.tagged('system') { logger.info 'Shutting down' }
+        launcher.stop
+        logger.tagged('system') { logger.info 'Bye!' }
+        exit 0
+      end
+    end
+
+    def handle_signal(sig)
+    end
+  end
+end

data/lib/teplohod/initializers/banner.rb

@@ -0,0 +1,31 @@
+module Teplohod
+  module Initializers
+    class Banner
+      include Initializable
+
+      def run
+        puts "\e[32m"
+        puts banner
+        puts "\e[0m"
+      end
+
+      private
+
+      # TODO: Add teplohod
+      def banner
+        %(
+  Welcome to #{Teplohod::NAME}!
+  Current version: #{Teplohod::VERSION}
+
+  Options:
+    Environment: #{environment}
+    Mount Point: #{options[:require]}
+    Log Level: #{logger.level}
+    Pid File Path: #{options[:pidfile]}
+    Daemonize: #{options[:daemonize]}
+    Web: #{options[:web]}
+        )
+      end
+    end
+  end
+end

data/lib/teplohod/initializers/boot.rb

@@ -0,0 +1,15 @@
+module Teplohod
+  class NoRequiredFile < ArgumentError
+  end
+
+  module Initializers
+    class Boot
+      include Initializable
+
+      def run
+        raise NoRequiredFile, "#{options[:require]} does not exist" unless File.exist?(options[:require])
+        require options[:require]
+      end
+    end
+  end
+end

data/lib/teplohod/initializers/daemonizer.rb

@@ -0,0 +1,51 @@
+require 'teplohod/initializers/logger'
+
+module Teplohod
+  class NoLogfileError < ArgumentError
+  end
+
+  module Initializers
+    class Daemonizer
+      include Initializable
+
+      def run
+        validate!
+
+        files_to_reopen = []
+        ObjectSpace.each_object(File) do |file|
+          files_to_reopen << file unless file.closed?
+        end
+
+        ::Process.daemon(true, true)
+
+        files_to_reopen.each do |file|
+          begin
+            file.reopen file.path, 'a+'
+            file.sync = true
+          rescue ::Exception
+          end
+        end
+
+        [$stdout, $stderr].each do |io|
+          File.open(options[:logfile], 'ab') do |f|
+            io.reopen(f)
+          end
+          io.sync = true
+        end
+        $stdin.reopen('/dev/null')
+
+        Teplohod::Initializers::Logger.run!
+      end
+
+      private
+
+      def logfile?
+        options[:logfile]
+      end
+
+      def validate!
+        raise NoLogfileError, "You should specify a logfile if you're going to daemonize" unless logfile?
+      end
+    end
+  end
+end

data/lib/teplohod/initializers/logger.rb

@@ -0,0 +1,14 @@
+require 'teplohod/logging'
+
+module Teplohod
+  module Initializers
+    class Logger
+      include Initializable
+
+      def run
+        Teplohod::Logging.initialize_logger(options[:logfile]) if options[:logfile]
+        Teplohod.logger.level = ::Logger::DEBUG if options[:verbose]
+      end
+    end
+  end
+end

data/lib/teplohod/initializers/options.rb

@@ -0,0 +1,57 @@
+require 'optparse'
+
+require 'teplohod'
+
+module Teplohod
+  module Initializers
+    class Options
+      include Initializable
+
+      def self.run!
+        new.run
+      end
+
+      def run
+        cli_options = parse
+        Teplohod.options = options.merge cli_options
+        puts "Start #{Teplohod::NAME} with #{options}"
+      end
+
+      private
+
+      def parse
+        opts = {}
+        parser = ::OptionParser.new do |o|
+          o.on '-d', '--daemon', 'Daemonize process' do |arg|
+            opts[:daemonize] = arg
+          end
+          o.on '-r', '--require [PATH|DIR]', 'Location to require' do |arg|
+            opts[:require] = arg
+          end
+          o.on '-e', '--environment ENV', 'Application environment' do |arg|
+            opts[:environment] = arg
+          end
+          o.on '-v', '--verbose', 'Print more verbose output' do |arg|
+            opts[:verbose] = arg
+          end
+          o.on '-L', '--logfile PATH', 'path to writable logfile' do |arg|
+            opts[:logfile] = arg
+          end
+          o.on '-P', '--pidfile PATH', 'path to pidfile' do |arg|
+            opts[:pidfile] = arg
+          end
+          o.on '-w', '--web', 'run web dashboard' do |arg|
+            opts[:web] = arg
+          end
+        end
+        parser.banner = 'teplohod [options]'
+        parser.on_tail '-h', '--help', 'Show help' do
+          puts parser.to_s
+          exit 1
+        end
+        parser.parse! ARGV
+        opts
+      end
+    end
+  end
+end

data/lib/teplohod/initializers/pid.rb

@@ -0,0 +1,16 @@
+module Teplohod
+  module Initializers
+    class Pid
+      include Initializable
+
+      def run
+        if path = options[:pidfile]
+          pidfile = File.expand_path(path)
+          File.open(pidfile, 'w') do |f|
+            f.puts ::Process.pid
+          end
+        end
+      end
+    end
+  end
+end

data/lib/teplohod/initializers/redis.rb

@@ -0,0 +1,14 @@
+require 'redis'
+
+module Teplohod
+  module Initializers
+    class Redis
+      include Initializable
+
+      def run
+        # TODO: add configuration
+        Teplohod.redis= ::Redis.new
+      end
+    end
+  end
+end

data/lib/teplohod/initializers/web.rb

@@ -0,0 +1,13 @@
+require 'teplohod/web'
+
+module Teplohod
+  module Initializers
+    class Web
+      include Initializable
+
+      def run
+        fork { Teplohod::Web.run! }
+      end
+    end
+  end
+end

data/lib/teplohod/initializers.rb

@@ -0,0 +1,25 @@
+require 'active_support/concern'
+
+require 'teplohod/utils'
+
+module Teplohod
+  module Initializable
+    extend ActiveSupport::Concern
+    include Utils
+
+    module ClassMethods
+      def run!
+        new.run
+      end
+    end
+  end
+end
+
+require 'teplohod/initializers/options'
+require 'teplohod/initializers/banner'
+require 'teplohod/initializers/logger'
+require 'teplohod/initializers/daemonizer'
+require 'teplohod/initializers/pid'
+require 'teplohod/initializers/redis'
+require 'teplohod/initializers/boot'
+require 'teplohod/initializers/web'

data/lib/teplohod/launcher.rb

@@ -0,0 +1,28 @@
+require 'teplohod/utils'
+
+module Teplohod
+  class Launcher
+    include Utils
+
+    def initialize
+      @providers = Teplohod::Manager.providers
+    end
+
+    def run
+      log { logger.info { 'Running..' } }
+      @providers.each(&:run)
+    end
+
+    def stop
+      log { logger.info { 'Stopping..' } }
+    end
+
+    private
+
+    def log
+      logger.tagged 'system' do
+        yield
+      end
+    end
+  end
+end

data/lib/teplohod/logging.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+require 'time'
+require 'logger'
+require 'fcntl'
+require 'active_support/tagged_logging'
+
+module Teplohod
+  module Logging
+    class Formatter
+      include ActiveSupport::TaggedLogging::Formatter
+
+      def call(severity, time, _program_name, message)
+        tags = current_tags.join('/')
+        "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)} #{severity}: [#{tags}] #{message}\n"
+      end
+    end
+
+    def self.initialize_logger(log_target = STDOUT)
+      oldlogger = defined?(@logger) ? @logger : nil
+      @logger = ActiveSupport::TaggedLogging.new(Logger.new(log_target))
+      @logger.formatter = Formatter.new
+      @logger.level = Logger::INFO
+      oldlogger.close if oldlogger
+      @logger
+    end
+
+    def self.logger
+      defined?(@logger) ? @logger : initialize_logger
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new(File::NULL))
+    end
+
+    # This reopens ALL logfiles in the process that have been rotated
+    # using logrotate(8) (without copytruncate) or similar tools.
+    # A +File+ object is considered for reopening if it is:
+    #   1) opened with the O_APPEND and O_WRONLY flags
+    #   2) the current open file handle does not match its original open path
+    #   3) unbuffered (as far as userspace buffering goes, not O_SYNC)
+    # Returns the number of files reopened
+    def self.reopen_logs
+      to_reopen = []
+      append_flags = File::WRONLY | File::APPEND
+
+      ObjectSpace.each_object(File) do |fp|
+        begin
+          if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
+            to_reopen << fp
+          end
+        rescue IOError, Errno::EBADF
+        end
+      end
+
+      nr = 0
+      to_reopen.each do |fp|
+        orig_st = begin
+          fp.stat
+        rescue IOError, Errno::EBADF
+          next
+        end
+
+        begin
+          b = File.stat(fp.path)
+          next if orig_st.ino == b.ino && orig_st.dev == b.dev
+        rescue Errno::ENOENT
+        end
+
+        begin
+          File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
+          fp.sync = true
+          nr += 1
+        rescue IOError, Errno::EBADF
+          # not much we can do...
+        end
+      end
+      nr
+    rescue RuntimeError => ex
+      # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
+      puts "Unable to reopen logs: #{ex.message}"
+    end
+
+    def logger
+      Teplohod::Logging.logger
+    end
+  end
+end

data/lib/teplohod/manager.rb

@@ -0,0 +1,19 @@
+require 'teplohod/stats'
+
+module Teplohod
+  class Manager
+    class << self
+      def providers
+        @providers ||= []
+      end
+
+      def register_provider(provider)
+        providers << provider
+      end
+
+      def reload_providers!
+        @providers = []
+      end
+    end
+  end
+end

data/lib/teplohod/pipeline/base.rb

@@ -0,0 +1,89 @@
+require 'teplohod/utils'
+require 'teplohod/processor/equality'
+
+module Teplohod
+  module Pipeline
+    class Base
+      include Utils
+      attr_reader :key, :timeout
+      attr_accessor :provider
+
+      def initialize(key: self.class.to_s.underscore, timeout: 5, loop: true)
+        @key = key
+        @timeout = timeout
+				@loop = loop
+      end
+
+      def parent
+        @parent ||= self.class.parent
+      end
+
+      def queue
+        @queue ||= self.class.queue
+      end
+
+      def processor
+        @processor = @processor || self.class.processor || Teplohod::Processor::Equality.new
+      end
+
+      def run
+        on_empty_load if queue.empty?
+
+        return if parent.nil?
+        threads thread_name do
+          log { logger.info { 'loop started' } }
+          loop do
+            while element = parent.queue.pop
+              queue.push process(element)
+							Teplohod::Stats.increment(:processed_jobs, self)
+            end
+            log { logger.warn { "parent queue is clear, sleep #{timeout} sec"} }
+            parent.on_finish
+						break unless @loop
+            sleep timeout
+          end
+        end
+      end
+
+			def name
+				"#{self.class}#name is not implemented"
+			end
+
+			def description
+				"#{self.class}#description is not implemented"
+			end
+
+			protected
+
+      def process(data)
+        result = processor.process data
+        log { logger.info { "processed #{data}" } }
+				result
+      end
+
+      def on_finish
+        log { logger.warn { '#on_finish does nothing' } }
+      end
+
+      def on_empty_load
+        log { logger.warn { '#on_empty_load does nothing' } }
+      end
+
+      private
+
+      def thread_name
+        [provider&.key, key].join('/')
+      end
+
+      class << self
+        attr_accessor :parent, :queue, :processor
+      end
+
+      def log
+        logger.tagged provider&.key, key do
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/teplohod/processor/base.rb

@@ -0,0 +1,9 @@
+module Teplohod
+  module Processor
+    class Base
+      def process(_input)
+        raise NotImplemented
+      end
+    end
+  end
+end

data/lib/teplohod/processor/equality.rb

@@ -0,0 +1,11 @@
+require 'teplohod/processor/base'
+
+module Teplohod
+  module Processor
+    class Equality < Base
+      def process(input)
+        input
+      end
+    end
+  end
+end