fluent-plugin-twitter 0.3.2 → 0.4.0

data/README.md

@@ -8,9 +8,9 @@ Fluentd Input/Output plugin to process tweets with Twitter Streaming API.
 
 before use, install dependent library as:
 
-```bash
+```
 # for RHEL/CentOS
-$ sudo yum install openssl-devel
+$ sudo yum -y install openssl-devel libcurl libcurl-devel
 
 # for Ubuntu/Debian
 $ sudo apt-get install libssl-dev
@@ -18,20 +18,26 @@ $ sudo apt-get install libssl-dev
 
 ## Installation
 
-### native gem
+install with `gem` or `fluent-gem` command as:
 
-`````
-gem install fluent-plugin-twitter
-`````
+```
+# for fluentd
+$ gem install eventmachine
+$ gem install fluent-plugin-twitter
 
-### td-agent gem
-`````
-/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-twitter
-`````
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install eventmachine
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-twitter
+
+# for td-agent2
+$ sudo td-agent-gem install eventmachine
+$ sudo td-agent-gem install fluent-plugin-twitter
+```
 
 ## Input Configuration
 
 ### Input Sample
+
 `````
 <source>
   type twitter
@@ -40,8 +46,9 @@ gem install fluent-plugin-twitter
   oauth_token         YOUR_OAUTH_TOKEN        # Required
   oauth_token_secret  YOUR_OAUTH_TOKEN_SECRET # Required
   tag                 input.twitter.sampling  # Required
-  timeline            sampling                # Required (sampling or userstream)
-  keyword             Ruby,Python             # Optional
+  timeline            tracking                # Required (tracking or sampling or userstream)
+  keyword             Ruby,Python             # Optional (keyword is priority than follow_ids)
+  follow_ids          14252,53235             # Optional (integers, not screen names)
   lang                ja,en                   # Optional
   output_format       nest                    # Optional (nest or flat or simple[default])
 </source>
@@ -51,7 +58,8 @@ gem install fluent-plugin-twitter
 </match>
 `````
 
-### Debug
+### Testing
+
 `````
 $ tail -f /var/log/td-agent/td-agent.log
 `````
@@ -74,7 +82,8 @@ $ tail -f /var/log/td-agent/td-agent.log
 </match>
 `````
 
-### Debug
+### Testing
+
 `````
 $ curl http://localhost:8888/notify.twitter -F 'json={"message":"foo"}'
 `````
@@ -88,15 +97,9 @@ http://pocketstudio.jp/log3/2012/02/12/how_to_get_twitter_apikey_and_token/
 http://qiita.com/items/fe4258b394190f23fece
 
 ## TODO
-* support fetching twitter search
-* support fetching specific user userstream
 
 patches welcome!
 
-## Known Issue
-On starting fluentd, appearing alert message below. Please tell me how to fix up.
-`/usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/eventmachine-1.0.0/lib/eventmachine.rb:1530: warning: already initialized constant EM`
-
 ## Copyright
 
 Copyright © 2012- Kentaro Yoshida (@yoshi_ken)

data/fluent-plugin-twitter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-twitter"
-  s.version     = "0.3.2"
+  s.version     = "0.4.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-twitter"
@@ -18,4 +18,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "fluentd", [">= 0.10.46"]
   s.add_runtime_dependency "twitter", ">= 5.0.0"
   s.add_runtime_dependency "tweetstream", [">= 2.6.1"]
+  s.add_runtime_dependency "string-scrub" if RUBY_VERSION < "2.1"
 end

data/lib/fluent/plugin/in_twitter.rb

@@ -1,6 +1,6 @@
 module Fluent
   class TwitterInput < Fluent::Input
-    TIMELINE_TYPE = %w(userstream sampling)
+    TIMELINE_TYPE = %w(userstream sampling tracking)
     OUTPUT_FORMAT_TYPE = %w(nest flat simple)
     Plugin.register_input('twitter', self)
 
@@ -11,6 +11,7 @@ module Fluent
     config_param :tag, :string
     config_param :timeline, :string
     config_param :keyword, :string, :default => nil
+    config_param :follow_ids, :string, :default => nil
     config_param :lang, :string, :default => nil
     config_param :output_format, :string, :default => 'simple'
     config_param :flatten_separator, :string, :default => '_'
@@ -55,14 +56,14 @@ module Fluent
 
     def run
       client = get_twitter_connection
-      if @timeline == 'sampling' && @keyword
+      if ['sampling', 'tracking'].include?(@timeline) && @keyword
         client.track(@keyword)
-      elsif @timeline == 'sampling' && @keyword.nil?
+      elsif @timeline == 'tracking' && @follow_ids
+        client.follow(@follow_ids)
+      elsif @timeline == 'sampling' && @keyword.nil? && @follow_ids.nil?
         client.sample
       elsif @timeline == 'userstream'
         client.userstream
-      #elsif @timeline == 'follow'
-      #  client.follow(@follow_ids)
       end
     end
 
@@ -71,6 +72,7 @@ module Fluent
       notice << " tag:#{@tag}"
       notice << " lang:#{@lang}" unless @lang.nil?
       notice << " keyword:#{@keyword}" unless @keyword.nil?
+      notice << " follow:#{@follow_ids}" unless @follow_ids.nil? && !@keyword.nil?
       $log.info notice
       client = TweetStream::Client.new
       client.on_anything(&@any)
@@ -100,7 +102,7 @@ module Fluent
         record = hash_flatten(status)
       when 'simple'
         record = Hash.new
-        record.store('message', status[:text])
+        record.store('message', status[:text]).scrub('')
         record.store('geo', status[:geo])
         record.store('place', status[:place])
         record.store('created_at', status[:created_at])
@@ -116,8 +118,10 @@ module Fluent
     def hash_flatten(record, prefix = nil)
       record.inject({}) do |d, (k, v)|
         k = prefix.to_s + k.to_s
-        if v.is_a?(Hash)
+        if v.instance_of?(Hash)
           d.merge(hash_flatten(v, k + @flatten_separator))
+        elsif v.instance_of?(String)
+          d.merge(k => v.scrub(""))
         else
           d.merge(k => v)
         end
@@ -131,6 +135,8 @@ module Fluent
           newhash[k.to_s] = hash_key_to_s(v)
         elsif v.instance_of?(Array) then
           newhash[k.to_s] = array_key_to_s(v)
+        elsif v.instance_of?(String)
+          newhash[k.to_s] = v.scrub('')
         else
           newhash[k.to_s] = v
         end
@@ -144,6 +150,8 @@ module Fluent
           hash_key_to_s(v)
         elsif v.instance_of?(Array) then
           array_key_to_s(v)
+        elsif v.instance_of?(String) then
+          v.scrub('')
         else
           v
         end

data/test/plugin/test_in_twitter.rb

@@ -28,7 +28,7 @@ class TwitterInputTest < Test::Unit::TestCase
       oauth_token         OAUTH_TOKEN
       oauth_token_secret  OAUTH_TOKEN_SECRET
       tag                 input.twitter
-      timeline            sampling
+      timeline            tracking
       keyword             ${hashtag}ruby lang:ja "happy hour" :)
     ]
     assert_equal 'CONSUMER_KEY', d.instance.consumer_key

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-twitter
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-02 00:00:00.000000000 Z
+date: 2014-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -75,6 +75,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.6.1
+- !ruby/object:Gem::Dependency
+  name: string-scrub
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - y.ken.studio@gmail.com