akane 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0bd862e8bde263f78e295317635f8bb4eb0256cd
-  data.tar.gz: 4b64ada85e74c5018bc7f013c3f37d850cccb7e5
+  metadata.gz: de30517cd16b9dd52bf5628602e20e22dfcb594a
+  data.tar.gz: 509875d93f4c1bcebfb950f289b206d69916f2ea
 SHA512:
-  metadata.gz: 2015deae33ec6b5598a17743dc308f01ce8cf452819f1f2ca2334b1a37365897fb79c7f7d8676bb1618ebe5bc9ac98d3e224d39cd34738bad89cecb038029343
-  data.tar.gz: fe3250c3bc69de4bafcad0f68ea3d488efdf0ee75d7718e04eabe27f3ce2a400a200fab02bb1fe124858dab17d430fa2555277278b38cf161ae38696e4f681da
+  metadata.gz: 98ec5786e6587eac236b868309b85fc9a8d1fc445b761f927e1085c04629fb7534105bee28bec99796af86b7aa2ba44ea9e87fb5f6a738370b5ca97e36a91038
+  data.tar.gz: 3c6ab75d276a089ce1a0486dfb608c3a09f029ece9096a94a98caf2927447ef275c9f64338e7f707ae96dd273da997b18e0d1b9cea35935e4d8e59c6720c4d2a

data/akane.example.yml

@@ -0,0 +1,21 @@
+---
+consumer:
+  token: CONSUMER_TOKEN
+  secret: CONSUMER_SECRET
+accounts:
+  your_account_name1:
+    token: ACCESS_TOKEN1
+    secret: SECRET_TOKEN1
+  your_account_name2:
+    token: ACCESS_TOKEN2
+    secret: SECRET_TOKEN2
+storages:
+  - stdout
+  - file:
+      dir: /tmp/akane
+      sync_io: false
+  - elasticsearch:
+      host: localhost
+      index: akane
+      kuromoji: false
+      enable_es_log: false

data/akane.gemspec

@@ -21,6 +21,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "elasticsearch", "~> 0.4.1"
   spec.add_dependency "twitter", "~> 5.5.1"
   spec.add_dependency "oauth", ">= 0.4.7"
+  spec.add_dependency "sigdump"
+
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"

data/lib/akane/config.rb

@@ -29,5 +29,15 @@ module Akane
     def logger
       Logger.new(@hash["log"] || $stdout)
     end
+
+    def log_direct(line)
+      if @hash["log"]
+        open(@hash["log"], 'a') do |io|
+          io.puts line
+        end
+      else
+        $stdout.puts line
+      end
+    end
   end
 end

data/lib/akane/manager.rb

@@ -35,7 +35,7 @@ module Akane
           [[definition, {}]]
         end
       end.map do |kind, config|
-        @logger.info "Preparing... storgae - #{kind}"
+        @logger.info "Preparing... storage - #{kind}"
         require "akane/storages/#{kind}"
         Akane::Storages.const_get(kind.gsub(/(?:\A|_)(.)/) { $1.upcase }).new(
           config: config,
@@ -43,7 +43,11 @@ module Akane
         )
       end
 
-      @recorder = Akane::Recorder.new(@storages, logger: @config.logger)
+      @recorder = Akane::Recorder.new(
+        @storages,
+        timeout: @config["timeout"] || 20,
+        logger: @config.logger
+      )
 
       @logger.info "Prepared with #{@storages.size} storage(s) and #{@receivers.size} receiver(s)"
     end
@@ -51,8 +55,38 @@ module Akane
     def start
       @logger.info "Starting receivers..."
       @receivers.each(&:start)
+
+      @logger.info "Assigning signal handlers..."
+      handle_signals
+
       @logger.info "Starting recorder..."
       @recorder.run
+
+      @logger.info "Recorder stopped. Waiting for storages..."
+      stop_storages
+    end
+
+    def handle_signals
+      @terminating = false
+
+      begin
+        require 'sigdump/setup'
+      rescue LoadError
+      end
+
+      on_interrupt = proc do
+        if @terminating
+          @config.log_direct "Terminating forcely..."
+          exit
+        else
+          @terminating = true
+          @config.log_direct "Gracefully stopping..."
+          @recorder.stop!
+        end
+      end
+
+      trap(:INT, on_interrupt)
+      trap(:TERM, on_interrupt)
     end
 
     def run
@@ -62,6 +96,22 @@ module Akane
       start()
     end
 
+    def stop_storages
+      @storages.each(&:stop!)
+      loop do
+        not_exitable = @storages.any? do |storage|
+          if storage.exitable?
+            false
+          else
+            @logger.debug "[status] #{storage.name}: #{storage.status || 'not exitable'.freeze}"
+            true
+          end
+        end
+        break unless not_exitable
+        sleep 1
+      end
+    end
+
     private
 
     def on_tweet(account, tweet)

data/lib/akane/receivers/stream.rb

@@ -26,19 +26,26 @@ module Akane
         @logger.info "Stream : Starting"
 
         @thread = Thread.new do
-          stream.user do |obj|
-            case obj
-            when Twitter::Tweet
-              invoke(:tweet, obj)
-            when Twitter::DirectMessage
-              invoke(:message, obj)
-            when Twitter::Streaming::DeletedTweet
-              invoke(:delete, obj.user_id, obj.id)
-            when Twitter::Streaming::Event
-              invoke(:event,
-                     'event' => obj.name, 'source' => obj.source,
-                     'target' => obj.target, 'target_object' => obj.target_object)
+          begin
+            stream.user do |obj|
+              case obj
+              when Twitter::Tweet
+                invoke(:tweet, obj)
+              when Twitter::DirectMessage
+                invoke(:message, obj)
+              when Twitter::Streaming::DeletedTweet
+                invoke(:delete, obj.user_id, obj.id)
+              when Twitter::Streaming::Event
+                invoke(:event,
+                       'event' => obj.name, 'source' => obj.source,
+                       'target' => obj.target, 'target_object' => obj.target_object)
+              end
             end
+          rescue Exception => e
+            raise e if defined?(Twitter::Streaming::MockClient)
+            @logger.error 'Error on stream'
+            @logger.error e.inspect
+            @logger.error e.backtrace
           end
         end
 

data/lib/akane/recorder.rb

@@ -1,12 +1,17 @@
 require 'thread'
+require 'timeout'
 
 module Akane
   class Recorder
-    def initialize(storages, logger: Logger.new(nil))
+    class Stop < Exception; end # :nodoc:
+
+    def initialize(storages, timeout: 20, logger: Logger.new(nil))
       @storages = storages
       @logger = logger
       @queue = Queue.new
       @recently_performed = RoundrobinFlags.new(1000)
+      @timeout = timeout
+      @stop = false
     end
 
     def queue_length
@@ -14,21 +19,25 @@ module Akane
     end
 
     def record_tweet(account, tweet)
+      return self if @stop
       @queue << [:record_tweet, account, tweet]
       self
     end
 
     def mark_as_deleted(account, user_id, tweet_id)
+      return self if @stop
       @queue << [:mark_as_deleted, account, user_id, tweet_id]
       self
     end
 
     def record_message(account, message)
+      return self if @stop
       @queue << [:record_message, account, message]
       self
     end
 
     def record_event(account, event)
+      return self if @stop
       @queue << [:record_event, account, event]
       self
     end
@@ -50,9 +59,18 @@ module Akane
 
       @storages.each do |storage|
         begin
-          storage.__send__(action, account, *payload)
+          timeout(@timeout) do
+            storage.__send__(action, account, *payload)
+          end
+
+        rescue Timeout::Error => e
+          raise e if raise_errors
+          @logger.warn "#{storage.name} (#{action}) timed out"
+
+        rescue Interrupt, SignalException, SystemExit => e
+          raise e
+
         rescue Exception => e
-          raise e if e === Interrupt
           raise e if raise_errors
           @logger.error "Error while recorder performing to #{storage.inspect}:  #{e.inspect}"
           @logger.error e.backtrace
@@ -61,16 +79,36 @@ module Akane
     end
 
     def run(raise_errors = false)
-      loop do
-        begin
-          self.dequeue(raise_errors)
-        rescue Exception => e
-          raise e if Interrupt === e
-          raise e if raise_errors
-          @logger.error "Error while recorder dequing: #{e.inspect}"
-          @logger.error e.backtrace
+      @running_thread = Thread.new do 
+        loop do
+          begin
+            begin
+              self.dequeue(raise_errors)
+            rescue Interrupt, SignalException, Stop
+            end
+
+            if @stop
+              break if self.queue_length.zero?
+              @logger.info "processing queue: #{self.queue_length} remaining."
+            end
+          rescue Exception => e
+            raise e if raise_errors
+            @logger.error "Error while recorder dequing: #{e.inspect}"
+            @logger.error e.backtrace
+          end
         end
+
+        @logger.info "Recorder stopped."
+        @stop = false
       end
+
+      @running_thread.join
+      nil
+    end
+
+    def stop!
+      @stop = true
+      @running_thread.raise Stop
     end
 
     class RoundrobinFlags

data/lib/akane/storages/abstract_storage.rb

@@ -4,6 +4,11 @@ module Akane
       def initialize(config: raise(ArgumentError, 'missing config'), logger: Logger.new($stdout))
         @config = config
         @logger = logger
+        @stop = false
+      end
+
+      def name
+        self.class.name
       end
 
       def record_tweet(account, tweet)
@@ -21,6 +26,18 @@ module Akane
       def record_message(account, message)
         raise NotImplementedError
       end
+
+      def stop!
+        @stop = true
+      end
+
+      def exitable?
+        true
+      end
+
+      def status
+        nil
+      end
     end
   end
 end

data/lib/akane/storages/elasticsearch.rb

@@ -15,6 +15,10 @@ module Akane
         set_elasticsearch_up
       end
 
+      def name
+        @name ||= "#{self.class.name}:#{@config["host"]}/#{@index_name}"
+      end
+
       def record_tweet(account, tweet)
         tweet_hash = tweet.attrs
         tweet_hash[:deleted] = false

data/lib/akane/storages/file.rb

@@ -16,6 +16,10 @@ module Akane
         end
       end
 
+      def name
+        @name ||= "#{self.class.name}:#{@config["dir"]}"
+      end
+
       def record_tweet(account, tweet)
         timeline_io.puts "[#{tweet[:created_at].xmlschema}][#{account}] #{tweet[:user][:screen_name]}: " \
                          "#{tweet[:text].gsub(/\r?\n/,' ')} (#{tweet[:user][:id]},#{tweet[:id]})"

data/lib/akane/version.rb

@@ -1,3 +1,3 @@
 module Akane
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/spec/manager_spec.rb

@@ -22,7 +22,8 @@ describe Akane::Manager do
         "token" => "consumer-token", "secret" => "consumer-secret"
       },
       "accounts" => conf_accounts,
-      "storages" => conf_storages
+      "storages" => conf_storages,
+      "timeout" => 72,
     ).tap { |_| _.stub(logger: Logger.new(nil)) }
   end
 
@@ -49,7 +50,7 @@ describe Akane::Manager do
     it "creates recorder with storages" do
       storage = double("storage")
       Akane::Storages::Mock.stub(new: storage)
-      Akane::Recorder.should_receive(:new).with([storage], logger: config.logger).and_call_original
+      Akane::Recorder.should_receive(:new).with([storage], timeout: 72, logger: config.logger).and_call_original
 
       subject.prepare
     end

data/spec/recorder_spec.rb

@@ -66,21 +66,39 @@ describe Akane::Recorder do
   end
 
   describe "#run" do
-    before do
-      @th = Thread.new { subject.run(true) }
-      @th.abort_on_exception = true
-    end
-
     it "continues dequeuing the queue" do
       storages[0].should_receive(:record_tweet).with('a', {:id => 42})
       storages[0].should_receive(:record_tweet).with('b', {:id => 43})
+
+      @th = Thread.new { subject.run(true) }
+      @th.abort_on_exception = true
+
+      15.times { break if @th.status == "sleep"; sleep 0.1 }
+
       subject.record_tweet('a', :id => 42)
       subject.record_tweet('b', :id => 43)
-      10.times { break if subject.queue_length.zero?; sleep 0.1 }
-    end
 
-    after do
+      15.times { break if subject.queue_length.zero?; sleep 0.1 }
       @th.kill if @th && @th.alive?
     end
   end
+
+  describe "#stop!" do
+    it "stops gracefully" do
+      storages[0].should_receive(:record_tweet).with('a', {:id => 42})
+      allow(storages[0]).to receive(:exitable?).and_return(true)
+
+      @th = Thread.new { subject.run(true) }
+      @th.abort_on_exception = true
+      15.times { break if @th.status == "sleep"; sleep 0.1 }
+
+      subject.record_tweet('a', :id => 42)
+      subject.stop!
+      subject.record_tweet('b', :id => 42)
+
+      15.times { break unless @th.alive?; sleep 0.1 }
+
+      expect(@th).not_to be_alive
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: akane
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Shota Fukumori (sora_h)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-15 00:00:00.000000000 Z
+date: 2014-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.7
+- !ruby/object:Gem::Dependency
+  name: sigdump
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- akane.example.yml
 - akane.gemspec
 - bin/akane
 - lib/akane.rb
@@ -169,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Log your timeline to something
@@ -185,4 +200,3 @@ test_files:
 - spec/support/mock_tweetstream.rb
 - spec/support/mock_twitter_streaming.rb
 - spec/util_spec.rb
-has_rdoc: