fluent-plugin-twitter 0.5.4 → 0.6.0

data/.travis.yml

@@ -2,8 +2,9 @@ sudo: false
 language: ruby
 
 rvm:
-  - 2.2
-  - 2.1
+  - 2.3.1
+  - 2.2.5
+  - 2.1.10
 
 before_install:
   - gem update bundler
@@ -12,17 +13,18 @@ gemfile:
   - Gemfile
   - gemfiles/fluentd_v0.10.gemfile
   - gemfiles/fluentd_v0.12.gemfile
+  - gemfiles/fluentd_v0.14.gemfile
 
 matrix:
   include:
-    - rvm: 2.3.1
-      gemfile: Gemfile
     - rvm: 2.0.0
       gemfile: gemfiles/fluentd_v0.10.gemfile
     - rvm: 2.0.0
       gemfile: gemfiles/fluentd_v0.12.gemfile
+  exclude:
+    - rvm: 2.3.1
+      gemfile: gemfiles/fluentd_v0.10.gemfile
     - rvm: 1.9.3
       gemfile: gemfiles/fluentd_v0.10.gemfile
     - rvm: 1.9.3
       gemfile: gemfiles/fluentd_v0.12.gemfile
-

data/Appraisals

@@ -6,3 +6,6 @@ appraise "fluentd v0.12" do
   gem "fluentd", "~> 0.12.0"
 end
 
+appraise "fluentd v0.14" do
+  gem "fluentd", "~> 0.14.0"
+end

data/README.md

@@ -27,11 +27,11 @@ $ gem install fluent-plugin-twitter
 
 # for td-agent
 $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install eventmachine
-$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-twitter
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-twitter -v 0.6.0
 
 # for td-agent2
 $ sudo td-agent-gem install eventmachine
-$ sudo td-agent-gem install fluent-plugin-twitter
+$ sudo td-agent-gem install fluent-plugin-twitter -v 0.6.0
 ```
 
 ## Input Configuration
@@ -42,22 +42,23 @@ It require td-agent2 (fluentd v0.12) to use keyword with hashtag.
 
 `````
 <source>
-  type twitter
-  consumer_key        YOUR_CONSUMER_KEY       # Required
-  consumer_secret     YOUR_CONSUMER_SECRET    # Required
-  oauth_token         YOUR_OAUTH_TOKEN        # Required
-  oauth_token_secret  YOUR_OAUTH_TOKEN_SECRET # Required
-  tag                 input.twitter.sampling  # Required
-  timeline            tracking                # Required (tracking or sampling or location or userstream)
-  keyword             'Ruby,Python,#fleuntd'  # Optional (keyword has priority than follow_ids)
-  follow_ids          14252,53235             # Optional (integers, not screen names)
+  @type twitter
+  consumer_key        YOUR_CONSUMER_KEY        # Required
+  consumer_secret     YOUR_CONSUMER_SECRET     # Required
+  access_token        YOUR_ACCESS_TOKEN        # Required
+  access_token_secret YOUR_ACCESS_TOKEN_SECRET # Required
+  tag                 input.twitter.sampling   # Required
+  timeline            tracking                 # Required (tracking or sampling or location or userstream)
+  keyword             'Ruby,Python,#fleuntd'   # Optional (keyword has priority than follow_ids)
+  follow_ids          14252,53235              # Optional (integers, not screen names)
   locations           31.110283, 129.431631, 45.619283, 145.510175  # Optional (bounding boxes; first pair specifies longitude/latitude of southwest corner)
-  lang                ja,en                   # Optional
-  output_format       nest                    # Optional (nest or flat or simple[default])
+  lang                ja,en                    # Optional
+  output_format       nest                     # Optional (nest or flat or simple[default])
+  flatten_separator   _                        # Optional
 </source>
 
 <match input.twitter.sampling>
-  type stdout
+  @type stdout
 </match>
 `````
 
@@ -72,16 +73,16 @@ $ tail -f /var/log/td-agent/td-agent.log
 ### Output Sample
 `````
 <source>
-  type http
+  @type http
   port 8888
 </source>
 
 <match notify.twitter>
-  type twitter
+  @type twitter
   consumer_key        YOUR_CONSUMER_KEY
   consumer_secret     YOUR_CONSUMER_SECRET
-  access_token         YOUR_OAUTH_TOKEN
-  access_token_secret  YOUR_OAUTH_TOKEN_SECRET
+  access_token        YOUR_ACCESS_TOKEN
+  access_token_secret YOUR_ACCESS_TOKEN_SECRET
 </match>
 `````
 

data/fluent-plugin-twitter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-twitter"
-  s.version     = "0.5.4"
+  s.version     = "0.6.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-twitter"
@@ -19,13 +19,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "test-unit", ">= 3.1.0"
   s.add_development_dependency "appraisal"
 
-  # Since Fluentd v0.14 requires ruby 2.1 or later.
-  if defined?(RUBY_VERSION) && RUBY_VERSION < "2.1"
-    s.add_runtime_dependency "fluentd", "< 0.14"
-  else
-    s.add_runtime_dependency "fluentd", ">= 0.10.46"
-  end
-  s.add_runtime_dependency "twitter", ">= 5.0.0"
-  s.add_runtime_dependency "tweetstream", ">= 2.6.1"
-  s.add_runtime_dependency "string-scrub", ">= 0.0.3" if RUBY_VERSION.to_f < 2.1
+  s.add_runtime_dependency "fluentd", [">= 0.10.46", "< 2"]
+  s.add_runtime_dependency "twitter", "~> 5.0"
+  s.add_runtime_dependency "string-scrub", [">= 0.0.3", "<= 0.0.5"]
 end

data/gemfiles/fluentd_v0.14.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "http://rubygems.org"
+
+gem "fluentd", "~> 0.14.0"
+
+gemspec :path => "../"

data/lib/fluent/plugin/in_twitter.rb

@@ -1,3 +1,7 @@
+require 'twitter'
+require 'nkf'
+require 'string/scrub' if RUBY_VERSION.to_f < 2.1
+
 require "fluent/input"
 
 module Fluent
@@ -13,8 +17,8 @@ module Fluent
 
     config_param :consumer_key, :string, :secret => true
     config_param :consumer_secret, :string, :secret => true
-    config_param :oauth_token, :string, :secret => true
-    config_param :oauth_token_secret, :string, :secret => true
+    config_param :access_token, :string, :secret => true
+    config_param :access_token_secret, :string, :secret => true
     config_param :tag, :string
     config_param :timeline, :string
     config_param :keyword, :string, :default => nil
@@ -26,9 +30,6 @@ module Fluent
 
     def initialize
       super
-      require 'tweetstream'
-      require 'nkf'
-      require 'string/scrub' if RUBY_VERSION.to_f < 2.1
     end
 
     def configure(conf)
@@ -43,20 +44,16 @@ module Fluent
 
       @keyword = @keyword.gsub('${hashtag}', '#') unless @keyword.nil?
 
-      TweetStream.configure do |config|
+      @client = Twitter::Streaming::Client.new do |config|
         config.consumer_key = @consumer_key
         config.consumer_secret = @consumer_secret
-        config.oauth_token = @oauth_token
-        config.oauth_token_secret = @oauth_token_secret
-        config.auth_method = :oauth
+        config.access_token = @access_token
+        config.access_token_secret = @access_token_secret
       end
     end
 
     def start
       @thread = Thread.new(&method(:run))
-      @any = Proc.new do |hash|
-          get_message(hash) if is_message?(hash)
-      end
     end
 
     def shutdown
@@ -64,64 +61,58 @@ module Fluent
     end
 
     def run
-      client = get_twitter_connection
+      notice = "twitter: starting Twitter Streaming API for #{@timeline}."
+      notice << " tag:#{@tag}"
+      notice << " lang:#{@lang}" unless @lang.nil?
+      notice << " keyword:#{@keyword}" unless @keyword.nil?
+      notice << " follow:#{@follow_ids}" unless @follow_ids.nil? && !@keyword.nil?
+      $log.info notice
+
       if ['sampling', 'tracking'].include?(@timeline) && @keyword
-        client.track(@keyword)
+        @client.filter(track: @keyword, &method(:handle_object))
       elsif @timeline == 'tracking' && @follow_ids
-        client.follow(@follow_ids)
-      elsif @timeline == 'location' && @locations
-        client.locations(@locations)
+        @client.filter(follow: @follow_ids, &method(:handle_object))
       elsif @timeline == 'sampling' && @keyword.nil? && @follow_ids.nil?
-        client.sample
+        @client.sample(&method(:handle_object))
       elsif @timeline == 'userstream'
-        client.userstream
+        @client.user(&method(:handle_object))
       end
     end
 
-    def get_twitter_connection
-      notice = "twitter: starting Twitter Streaming API for #{@timeline}."
-      notice << " tag:#{@tag}"
-      notice << " lang:#{@lang}" unless @lang.nil?
-      notice << " keyword:#{@keyword}" unless @keyword.nil?
-      notice << " follow:#{@follow_ids}" unless @follow_ids.nil? && !@keyword.nil?
-      $log.info notice
-      client = TweetStream::Client.new
-      client.on_anything(&@any)
-      client.on_error do |message|
-        $log.info "twitter: unexpected error has occured. #{message}"
+    def handle_object(object)
+      if is_message?(object)
+        get_message(object)
       end
-      return client
     end
 
-    def is_message?(status)
-      return false if !status.include?(:text)
-      return false if !status.include?(:user)
-      return false if (!@lang.nil? && @lang != '') && !@lang.include?(status[:user][:lang])
+    def is_message?(tweet)
+      return false if !tweet.is_a?(Twitter::Tweet)
+      return false if (!@lang.nil? && @lang != '') && !@lang.include?(tweet.user.lang)
       if @timeline == 'userstream' && (!@keyword.nil? && @keyword != '')
         pattern = NKF::nkf('-WwZ1', @keyword).gsub(/,\s?/, '|')
-        tweet = NKF::nkf('-WwZ1', status[:text])
+        tweet = NKF::nkf('-WwZ1', tweet.text)
         return false if !Regexp.new(pattern, Regexp::IGNORECASE).match(tweet)
       end
       return true
     end
 
-    def get_message(status)
+    def get_message(tweet)
       case @output_format
       when 'nest'
-        record = hash_key_to_s(status)
+        record = hash_key_to_s(tweet.to_h)
       when 'flat'
-        record = hash_flatten(status)
+        record = hash_flatten(tweet.to_h)
       when 'simple'
         record = Hash.new
-        record.store('message', status[:text]).scrub('')
-        record.store('coordinates', status[:coordinates])
-        record.store('place', status[:place])
-        record.store('created_at', status[:created_at])
-        record.store('user_name', status[:user][:name])
-        record.store('user_screen_name', status[:user][:screen_name])
-        record.store('user_profile_image_url', status[:user][:profile_image_url])
-        record.store('user_time_zone', status[:user][:time_zone])
-        record.store('user_lang', status[:user][:lang])
+        record.store('message', tweet.text).scrub('')
+        record.store('geo', tweet.geo)
+        record.store('place', tweet.place)
+        record.store('created_at', tweet.created_at)
+        record.store('user_name', tweet.user.name)
+        record.store('user_screen_name', tweet.user.screen_name)
+        record.store('user_profile_image_url', tweet.user.profile_image_url)
+        record.store('user_time_zone', tweet.user.time_zone)
+        record.store('user_lang', tweet.user.lang)
       end
       router.emit(@tag, Engine.now, record)
     end
@@ -170,3 +161,12 @@ module Fluent
     end
   end
 end
+
+# TODO: Remove this monkey patch after release new version of twitter gem
+#
+# See: https://github.com/sferik/twitter/pull/815
+class Twitter::NullObject
+  def to_json(*args)
+    nil.to_json(*args)
+  end
+end

data/test/helper.rb

@@ -1,12 +1,4 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
+require 'bundler/setup'
 require 'test/unit'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))

data/test/plugin/test_in_twitter.rb

@@ -8,8 +8,8 @@ class TwitterInputTest < Test::Unit::TestCase
   CONFIG = %[
     consumer_key        CONSUMER_KEY
     consumer_secret     CONSUMER_SECRET
-    oauth_token         OAUTH_TOKEN
-    oauth_token_secret  OAUTH_TOKEN_SECRET
+    access_token        ACCESS_TOKEN
+    access_token_secret ACCESS_TOKEN_SECRET
     tag                 input.twitter
     timeline            sampling
   ]
@@ -25,16 +25,16 @@ class TwitterInputTest < Test::Unit::TestCase
     d = create_driver %[
       consumer_key        CONSUMER_KEY
       consumer_secret     CONSUMER_SECRET
-      oauth_token         OAUTH_TOKEN
-      oauth_token_secret  OAUTH_TOKEN_SECRET
+      access_token        ACCESS_TOKEN
+      access_token_secret ACCESS_TOKEN_SECRET
       tag                 input.twitter
       timeline            tracking
       keyword             ${hashtag}fluentd,fluentd lang:ja
     ]
     assert_equal 'CONSUMER_KEY', d.instance.consumer_key
     assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
-    assert_equal 'OAUTH_TOKEN', d.instance.oauth_token
-    assert_equal 'OAUTH_TOKEN_SECRET', d.instance.oauth_token_secret
+    assert_equal 'ACCESS_TOKEN', d.instance.access_token
+    assert_equal 'ACCESS_TOKEN_SECRET', d.instance.access_token_secret
     assert_equal '#fluentd,fluentd lang:ja', d.instance.keyword
   end
 
@@ -45,16 +45,16 @@ class TwitterInputTest < Test::Unit::TestCase
     d = create_driver(%[
       consumer_key        CONSUMER_KEY
       consumer_secret     CONSUMER_SECRET
-      oauth_token         OAUTH_TOKEN
-      oauth_token_secret  OAUTH_TOKEN_SECRET
+      access_token        ACCESS_TOKEN
+      access_token_secret ACCESS_TOKEN_SECRET
       tag                 input.twitter
       timeline            tracking
       keyword             'treasuredata,treasure data,#treasuredata,fluentd,#fluentd'
     ], 'test', true)
     assert_equal 'CONSUMER_KEY', d.instance.consumer_key
     assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
-    assert_equal 'OAUTH_TOKEN', d.instance.oauth_token
-    assert_equal 'OAUTH_TOKEN_SECRET', d.instance.oauth_token_secret
+    assert_equal 'ACCESS_TOKEN', d.instance.access_token
+    assert_equal 'ACCESS_TOKEN_SECRET', d.instance.access_token_secret
     assert_equal 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd', d.instance.keyword
   end
 end

metadata

@@ -1,99 +1,124 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-twitter
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.0
+  prerelease: 
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-07 00:00:00.000000000 Z
+date: 2017-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.10.46
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.10.46
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: twitter
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: tweetstream
+  name: string-scrub
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.6.1
+        version: 0.0.3
+    - - <=
+      - !ruby/object:Gem::Version
+        version: 0.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+    - - <=
       - !ruby/object:Gem::Version
-        version: 2.6.1
+        version: 0.0.5
 description: 
 email:
 - y.ken.studio@gmail.com
@@ -101,8 +126,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
+- .gitignore
+- .travis.yml
 - Appraisals
 - Gemfile
 - LICENSE.txt
@@ -111,6 +136,7 @@ files:
 - fluent-plugin-twitter.gemspec
 - gemfiles/fluentd_v0.10.gemfile
 - gemfiles/fluentd_v0.12.gemfile
+- gemfiles/fluentd_v0.14.gemfile
 - lib/fluent/plugin/in_twitter.rb
 - lib/fluent/plugin/out_twitter.rb
 - test/helper.rb
@@ -119,26 +145,27 @@ files:
 homepage: https://github.com/y-ken/fluent-plugin-twitter
 licenses:
 - Apache-2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Fluentd Input/Output plugin to collect/process tweets with Twitter Streaming
   API.
 test_files:

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 8085b7a29724df994e46bccf4c1c69751820e187
-  data.tar.gz: 62930fb36628f16d9a179c3704ace0ef1e515cb5
-SHA512:
-  metadata.gz: 5fbb52eab855d47e897d8dc9c8ff2b8df9b856e392057c8bcf2fe775e7b7a2d7a23be803252abec70435e116a8ad355b68a75cc8a1942b22e300d93943457e52
-  data.tar.gz: 7bcdf380a5c0d8d737adc78dcd8b7ea1c42424f6334676085c9a96e57d2ca374a84c2b8e8ab8eb6489c63c08dcaab6af2512d243f40aa13d906a91f701856bc9