akane 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6e75ec5065f36b1c0c716011de7d5a27cc1dc74
-  data.tar.gz: 592bf1058bd1b7c14adb72a9a63d155ae37ccfc6
+  metadata.gz: 9eb63675c2d34bca45cf1a084e2cb1d2383dd57f
+  data.tar.gz: 0aaa3ad0e12340ebffd567ac2b805722fc7f57e4
 SHA512:
-  metadata.gz: 9cd8b857e19b6923561a41e5ac45823c5d18580ae30cfe918919b6793743f117159aa760326a850612973770e750ff1e2fadae21659aed954e10fbd4b15299e9
-  data.tar.gz: b950d9ab866c3934656a9f7ce69e63c1f08872e89b2d579d5aca1d499baf67b2d8f53f78d44ed121afe3ca2312db1811232a2ff9c3fc3095d1dd376dc2a15a25
+  metadata.gz: 557db9f3bd65cf55a32f9fced460f584f89af5ce9499be9a36f87965f1f54c97c106d395d6518d5227a9c721322386599ace35c05f9e2f0684f3387bff8dec5d
+  data.tar.gz: 49633ba74f99864a73308fd63d4cb9aae0e6ec5a57e0e8ff36f4afebfc84ff82a5c76edc321b67c017e531e87cc7ac3800c44ea81fbdf405074e6bc5686878f6

data/akane.gemspec

@@ -18,8 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "elasticsearch", "~> 0.4.1"
-  spec.add_dependency "twitter", "~> 5.5.1"
+  spec.add_dependency "elasticsearch", ">= 0.4.1"
+  spec.add_dependency "twitter", ">= 5.9.0"
   spec.add_dependency "oauth", ">= 0.4.7"
   spec.add_dependency "sigdump"
 

data/lib/akane/receivers/stream.rb

@@ -1,9 +1,12 @@
 require 'akane/receivers/abstract_receiver'
+require 'socket'
 require 'twitter'
 
 module Akane
   module Receivers
     class Stream < AbstractReceiver
+      class TimeoutError < StandardError; end
+
       def initialize(*)
         super
         @thread = nil
@@ -36,18 +39,24 @@ module Akane
           consumer_key: @consumer[:token],
           consumer_secret: @consumer[:secret],
           access_token: @account[:token],
-          access_token_secret: @account[:secret]
+          access_token_secret: @account[:secret],
+          ssl_socket_class: CustomSSLSocketFactory.new(self),
         )
       end
 
       attr_reader :thread
+      attr_accessor :last
 
       def start
         @logger.info "Stream : Starting"
 
+        @last = Time.now
+        @retry_count = 0
         @thread = Thread.new do
           begin
             stream.send(@stream_method, @stream_options) do |obj|
+              @retry_count = 0
+
               case obj
               when Twitter::Tweet
                 invoke(:tweet, obj)
@@ -59,6 +68,8 @@ module Akane
                 invoke(:event,
                        'event' => obj.name, 'source' => obj.source,
                        'target' => obj.target, 'target_object' => obj.target_object)
+              else
+                next
               end
             end
           rescue Exception => e
@@ -66,9 +77,53 @@ module Akane
             @logger.error 'Error on stream'
             @logger.error e.inspect
             @logger.error e.backtrace
+
+            @retry_count += 1
+
+            # reconnecting https://dev.twitter.com/streaming/overview/connecting
+            case e
+            when Twitter::Error::EnhanceYourCalm # 420
+              interval = 5 ** @retry_count
+            when Twitter::Error
+              interval = [320, 5 ** @retry_count].min
+            else
+              interval = [16, 0.25 * @retry_count].min
+            end
+
+            @logger.info "stream will reconnect after #{interval} sec (retry_count=#{@retry_count})"
+            sleep interval
+            @logger.info 'stream reconnecting'
+            retry
+          end
+        end
+
+        @watchdog = Thread.new do
+          th = @thread
+          begin
+            loop do
+              break unless @thread
+              # @logger.debug "watchdog last #{@last} #{Time.now - @last}"
+              if (Time.now - @last) > 90
+                @last = Time.now
+                @logger.error 'watchdog timeout'
+                th.raise(TimeoutError)
+              end
+              sleep 1
+            end
+            @logger.info 'watchdog stop'
+          rescue Exception => e
+            @logger.error 'Error on watchdog'
+            @logger.error e.inspect
+            @logger.error e.backtrace
+
+            sleep 5
+            @logger.info 'watchdog restarting'
+            retry
           end
         end
 
+        @thread.abort_on_exception = true
+
         self
       end
 
@@ -77,6 +132,28 @@ module Akane
         @thread = nil
         self
       end
+
+      class CustomSSLSocketFactory
+        def initialize(target)
+          @target = target
+        end
+
+        def new(*args)
+          OpenSSL::SSL::SSLSocket.new(*args).tap do |sock|
+            class << sock
+              def last_target=(x)
+                @akane_ext_last_target = x
+              end
+              def readpartial(*)
+                super.tap do |x|
+                  @akane_ext_last_target.last = Time.now
+                end
+              end
+            end
+            sock.last_target = @target
+          end
+        end
+      end
     end
   end
 end

data/lib/akane/recorder.rb

@@ -48,12 +48,12 @@ module Akane
 
     def perform(action, account, *payload, raise_errors: false)
       if action == :record_tweet
-        return if @recently_performed[payload.last[:id]]
-        @recently_performed.flag!(payload.last[:id])
+        tweet = payload.last
+        return if @recently_performed[tweet.id]
+        @recently_performed.flag!(tweet.id)
 
-        # WTF: Twitter::NullObject
-        unless payload.last[:retweeted_status].nil?
-          perform(:record_tweet, account, payload.last[:retweeted_status], raise_errors: raise_errors)
+        if tweet.retweet?
+          perform(:record_tweet, account, tweet.retweeted_status, raise_errors: raise_errors)
         end
       end
 

data/lib/akane/storages/file.rb

@@ -21,10 +21,10 @@ module Akane
       end
 
       def record_tweet(account, tweet)
-        timeline_io.puts "[#{tweet[:created_at].xmlschema}][#{account}] #{tweet[:user][:screen_name]}: " \
-                         "#{tweet[:text].gsub(/\r?\n/,' ')} (#{tweet[:user][:id]},#{tweet[:id]})"
+        timeline_io.puts "[#{tweet.created_at.xmlschema}][#{account}] #{tweet.user.screen_name}: " \
+                         "#{tweet.text.gsub(/\r?\n/,' ')} (#{tweet.user.id},#{tweet.id})"
 
-        tweets_io_for_user(tweet[:user][:id], tweet[:user][:screen_name]) do |io|
+        tweets_io_for_user(tweet.user.id, tweet.user.screen_name) do |io|
           io.puts tweet.attrs.to_json
         end
       end
@@ -41,12 +41,12 @@ module Akane
       end
 
       def record_message(account, message)
-        messages_raw_io_for_user(message[:sender][:id], message[:sender][:screen_name]) do |io|
+        messages_raw_io_for_user(message.sender.id, message.sender.screen_name) do |io|
           io.puts message.attrs.to_json
         end
-        messages_io_for_user(message[:sender][:id], message[:sender][:screen_name]) do |io|
-          io.puts "[#{message[:created_at].xmlschema}] #{message[:sender][:screen_name]} -> #{message[:recipient][:screen_name]}:" \
-                           " #{message[:text]} (#{message[:sender][:id]} -> #{message[:recipient][:id]},#{message[:id]})"
+        messages_io_for_user(message.sender.id, message.sender.screen_name) do |io|
+          io.puts "[#{message.created_at.xmlschema}] #{message.sender.screen_name} -> #{message.recipient.screen_name}:" \
+                  " #{message.text} (#{message.sender.id} -> #{message.recipient.id},#{message.id})"
         end
       end
 

data/lib/akane/storages/stdout.rb

@@ -4,7 +4,7 @@ module Akane
   module Storages
     class Stdout < AbstractStorage
       def record_tweet(account, tweet)
-        $stdout.puts "[#{account}] #{tweet["user"]["screen_name"]}: #{tweet["text"]}"
+        $stdout.puts "[#{account}] #{tweet.user.screen_name}: #{tweet.text}"
       end
 
       def mark_as_deleted(account, user_id, tweet_id)
@@ -12,11 +12,11 @@ module Akane
       end
 
       def record_event(account, event)
-        $stdout.puts "[#{account}](EVENT) #{event["event"]}: #{event["source"]["screen_name"]}-> #{event["target"]["screen_name"]}"
+        $stdout.puts "[#{account}](EVENT) #{event.event}: #{event.source.screen_name}-> #{event.target.screen_name}"
       end
 
       def record_message(account, message)
-        $stdout.puts "[#{account}](DM) #{message["user"]["screen_name"]}: #{message["text"]}"
+        $stdout.puts "[#{account}](DM) #{message.user.screen_name}: #{message.text}"
       end
     end
   end

data/lib/akane/version.rb

@@ -1,3 +1,3 @@
 module Akane
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/spec/recorder_spec.rb

@@ -11,26 +11,29 @@ describe Akane::Recorder do
 
   describe "recording tweets" do
     it "records tweet" do
-      storages[0].should_receive(:record_tweet).with('a', {id: 42})
-      subject.record_tweet('a', id: 42)
+      tweet = Twitter::Tweet.new(id: 42)
+      storages[0].should_receive(:record_tweet).with('a', tweet)
+      subject.record_tweet('a', tweet)
       subject.dequeue(true)
     end
 
     it "doesn't record tweets which already recorded recently" do
-      storages[0].should_receive(:record_tweet).with('a', {id: 40})
-      storages[0].should_receive(:record_tweet).with('a', {id: 42})
+      tweet0 = Twitter::Tweet.new(id: 40)
+      tweet1 = Twitter::Tweet.new(id: 42)
+      storages[0].should_receive(:record_tweet).with('a', tweet0)
+      storages[0].should_receive(:record_tweet).with('a', tweet1)
 
-      subject.record_tweet('a', id: 40)
-      subject.record_tweet('a', id: 42)
+      subject.record_tweet('a', tweet0)
+      subject.record_tweet('a', tweet1)
       subject.dequeue(true)
       subject.dequeue(true)
-      subject.record_tweet('a', id: 42)
+      subject.record_tweet('a', tweet1)
       subject.dequeue(true)
     end
 
     it "records retweeted tweet" do
-      tweet = {id: 42, text: "foo", user: {id: 1, screen_name: "a"}}
-      retweet = {id: 43, text: "RT @a: foo", user: {id: 2, screen_name: "b"}, retweeted_status: tweet}
+      tweet = Twitter::Tweet.new(id: 42, text: "foo", user: {id: 1, screen_name: "a"})
+      retweet = Twitter::Tweet.new(id: 43, text: "RT @a: foo", user: {id: 2, screen_name: "b"}, retweeted_status: tweet.to_hash)
       storages[0].should_receive(:record_tweet).with('a', retweet)
       storages[0].should_receive(:record_tweet).with('a', tweet)
       subject.record_tweet('a', retweet)
@@ -67,16 +70,19 @@ describe Akane::Recorder do
 
   describe "#run" do
     it "continues dequeuing the queue" do
-      storages[0].should_receive(:record_tweet).with('a', {:id => 42})
-      storages[0].should_receive(:record_tweet).with('b', {:id => 43})
+      tweet0 = Twitter::Tweet.new(id: 42)
+      tweet1 = Twitter::Tweet.new(id: 43)
+
+      storages[0].should_receive(:record_tweet).with('a', tweet0)
+      storages[0].should_receive(:record_tweet).with('b', tweet1)
 
       @th = Thread.new { subject.run(true) }
       @th.abort_on_exception = true
 
       15.times { break if @th.status == "sleep"; sleep 0.1 }
 
-      subject.record_tweet('a', :id => 42)
-      subject.record_tweet('b', :id => 43)
+      subject.record_tweet('a', tweet0)
+      subject.record_tweet('b', tweet1)
 
       15.times { break if subject.queue_length.zero?; sleep 0.1 }
       @th.kill if @th && @th.alive?
@@ -85,16 +91,18 @@ describe Akane::Recorder do
 
   describe "#stop!" do
     it "stops gracefully" do
-      storages[0].should_receive(:record_tweet).with('a', {:id => 42})
+      tweet = Twitter::Tweet.new(id: 40)
+
+      storages[0].should_receive(:record_tweet).with('a', tweet)
       allow(storages[0]).to receive(:exitable?).and_return(true)
 
       @th = Thread.new { subject.run(true) }
       @th.abort_on_exception = true
       15.times { break if @th.status == "sleep"; sleep 0.1 }
 
-      subject.record_tweet('a', :id => 42)
+      subject.record_tweet('a', tweet)
       subject.stop!
-      subject.record_tweet('b', :id => 42)
+      subject.record_tweet('b', tweet)
 
       15.times { break unless @th.alive?; sleep 0.1 }
 

data/spec/support/mock_twitter_streaming.rb

@@ -56,6 +56,10 @@ class Twitter::Streaming::MockClient
     @mutex = Mutex.new
   end
 
+  def ssl_socket_class=(x)
+    return x
+  end
+
   attr_accessor :options, :hooks
 
   def invoke(*args)

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: akane
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Shota Fukumori (sora_h)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-28 00:00:00.000000000 Z
+date: 2016-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: twitter
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.5.1
+        version: 5.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.5.1
+        version: 5.9.0
 - !ruby/object:Gem::Dependency
   name: oauth
   requirement: !ruby/object:Gem::Requirement