teplohod 0.0.1.alpha1

data/lib/teplohod/provider/base.rb

@@ -0,0 +1,48 @@
+require 'active_support/core_ext/string/inflections'
+require 'teplohod/utils'
+
+module Teplohod
+  module Provider
+    class Base
+      include Utils
+
+      attr_reader :key
+
+      def initialize(key: self.class.to_s.underscore)
+        @key = key
+      end
+
+      def tail
+        @tail ||= self.class.tail
+      end
+
+      def pipelines
+        @pipelines ||= []
+        if @pipelines.empty?
+          pipeline = tail
+          while pipeline
+            @pipelines << pipeline
+            pipeline = pipeline.parent
+          end
+        end
+        @pipelines
+      end
+
+      def run
+        logger.tagged key do
+          logger.info { 'starting...' }
+        end
+        pipeline = tail
+        while pipeline
+          pipeline.provider = self
+          pipeline.run
+          pipeline = pipeline.parent
+        end
+      end
+
+      class << self
+        attr_accessor :tail
+      end
+    end
+  end
+end

data/lib/teplohod/queue/base.rb

@@ -0,0 +1,25 @@
+module Teplohod
+  module Queue
+    class Base
+      def pop
+        raise NotImplementedError
+      end
+
+      def push(_data)
+        raise NotImplementedError
+      end
+
+      def length
+        raise NotImplementedError
+      end
+
+      def empty?
+        length.zero?
+      end
+
+      def last(count)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/teplohod/queue/memory/base.rb

@@ -0,0 +1,33 @@
+require 'teplohod/queue/base'
+
+module Teplohod
+  module Queue
+    module Memory
+      class Base < ::Teplohod::Queue::Base
+        def initialize
+          @data = []
+        end
+
+        def all
+          @data
+        end
+
+        def push(value)
+          @data.push value
+        end
+
+        def pop
+          @data.pop
+        end
+
+        def length
+          @data.length
+        end
+
+        def last(count)
+          @data.take(count)
+        end
+      end
+    end
+  end
+end

data/lib/teplohod/signal_handler.rb

@@ -0,0 +1,39 @@
+require 'teplohod/utils'
+
+module Teplohod
+  module SignalHandler
+    class << self
+      include Utils
+
+      def handle(signal)
+        logger.debug "Got #{signal} signal"
+        case signal
+        when 'INT'
+          # Handle Ctrl-C in JRuby like MRI
+          # http://jira.codehaus.org/browse/JRUBY-4637
+          raise Interrupt
+        when 'TERM'
+          # Heroku sends TERM and then waits 10 seconds for process to exit.
+          raise Interrupt
+        when 'USR1'
+          logger.info 'Received USR1, no longer accepting new work'
+          launcher.quiet
+        when 'USR2'
+          if options[:logfile]
+            logger.info 'Received USR2, reopening log file'
+            Teplohod::Logging.reopen_logs
+          end
+        when 'TTIN'
+          Thread.list.each do |thread|
+            logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+            if thread.backtrace
+              logger.warn thread.backtrace.join("\n")
+            else
+              logger.warn '<no backtrace available>'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/teplohod/stats.rb

@@ -0,0 +1,36 @@
+require 'teplohod/utils'
+
+# TODO: use statsd or something more flexible
+module Teplohod
+  module Stats
+    include Utils
+
+    class << self
+      PROVIDERS = 'providers'.freeze
+
+      def all(target)
+        key = target.key
+        processed = redis.with { |conn| conn.get "#{key}:processed" } || 0
+        {
+          processed: processed
+        }
+      end
+
+      def increment(type, target)
+        send type, target.key
+      end
+
+      private
+
+      def processed_jobs(key)
+        redis.with do |conn|
+          conn.incr "#{key}:processed"
+        end
+      end
+
+      def redis
+        @redis ||= Teplohod.redis
+      end
+    end
+  end
+end

data/lib/teplohod/utils.rb

@@ -0,0 +1,31 @@
+module Teplohod
+  module Utils
+    def threads(_name, count: 1)
+      threads = []
+      count.times do
+        thread = Thread.new do
+          yield
+        end
+        thread.abort_on_exception = true
+        threads << thread
+      end
+      threads
+    end
+
+    def environment
+      Teplohod.environment
+    end
+
+    def logger
+      Teplohod.logger
+    end
+
+    def options
+      Teplohod.options
+    end
+
+    def redis
+      Teplohod.redis
+    end
+  end
+end

data/lib/teplohod/version.rb

@@ -0,0 +1,3 @@
+module Teplohod
+  VERSION = '0.0.1.alpha1'.freeze
+end

data/lib/teplohod/web.rb

@@ -0,0 +1,19 @@
+require 'sinatra'
+require 'erb'
+
+require 'teplohod'
+
+module Teplohod
+  class Web < Sinatra::Base
+    set :views, File.expand_path("#{File.dirname(__FILE__)}/../../web/views")
+    set :public_folder, File.expand_path("#{File.dirname(__FILE__)}/../../web/assets")
+
+    set :server, :thin
+    set :logging, false
+
+    get '/' do
+      providers = Teplohod::Manager.providers
+      erb :index, locals: { providers: providers }
+    end
+  end
+end

data/lib/teplohod.rb

@@ -0,0 +1,57 @@
+require 'active_support/core_ext/class/attribute_accessors'
+require 'redis'
+require 'connection_pool'
+require 'redis-namespace'
+
+require 'teplohod/version'
+require 'teplohod/utils'
+require 'teplohod/logging'
+
+require 'teplohod/manager'
+
+require 'teplohod/queue/base'
+require 'teplohod/provider/base'
+require 'teplohod/pipeline/base'
+require 'teplohod/processor/base'
+
+module Teplohod
+  NAME = 'Teplohod'.freeze
+  REDIS_NAMESPACE = 'teplohod'.freeze
+
+  class << self
+    DEFAULTS = {
+      environment: (ENV['RACK_ENV'] || ENV['RAILS_ENV'] || 'development'),
+      require: './parser.rb',
+      verbose: false,
+      daemonize: false
+    }.freeze
+
+    def options
+      @options ||= DEFAULTS.dup
+    end
+
+    attr_writer :options
+    attr_reader :redis
+
+    def reload_options!
+      @options = DEFAULTS.dup
+    end
+
+    def logger
+      Teplohod::Logging.logger
+    end
+
+    def logger=(log)
+      Teplohod::Logging.logger = log
+    end
+
+    # TODO add configuartion from options
+    def redis=(redis)
+      @redis ||= ConnectionPool.new(options) { Redis::Namespace.new(REDIS_NAMESPACE, redis: redis) }
+    end
+
+    def environment
+      options[:environment]
+    end
+  end
+end

data/teplohod.gemspec

@@ -0,0 +1,38 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'teplohod/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'teplohod'
+  spec.version       = Teplohod::VERSION
+  spec.authors       = ['Sergey Kuchmistov']
+  spec.email         = ['sergey.kuchmistov@gmail.com']
+
+  spec.summary       = 'Framework for persisted ongoing data processing'
+  spec.description   = 'Framework for persisted ongoing data processing like scraping or analyzing.'
+  spec.homepage      = 'http://raisintech.com/teplohod'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test)/})
+  end
+  spec.executables   = ['teplohod']
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'sinatra', '~> 2.0.0.beta2'
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'redis'
+  spec.add_dependency 'connection_pool'
+  spec.add_dependency 'redis-namespace'
+  spec.add_dependency 'thin'
+
+  spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'simplecov-console'
+  spec.add_development_dependency 'minitest-reporters'
+  spec.add_development_dependency 'fakeredis'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'yard'
+end