fluent-plugin-twitter 0.1.1 → 0.2.1

data/README.md

@@ -2,7 +2,7 @@ fluent-plugin-twitter
 =====================
 
 ## Component
-Fluentd Input/Output plugin. You can create your own "Twitter Bot" with fluentd messaging system.
+Fluentd Input/Output plugin to process tweets with Twitter Streaming API.
 
 ## Installation
 
@@ -29,8 +29,9 @@ gem install fluent-plugin-twitter
   oauth_token_secret  YOUR_OAUTH_TOKEN_SECRET # Required
   tag                 input.twitter.sampling  # Required
   timeline            sampling                # Required (sampling or userstream)
-  keyword             Ruby,Python             # Optional (userstream not supported yet)
+  keyword             Ruby,Python             # Optional (only work with `timeline` is sampling)
   lang                ja,en                   # Optional
+  output_format       nest                    # Optional (nest or flat or simple[default])
 </source>
 
 <match input.twitter.sampling>
@@ -71,6 +72,9 @@ $ curl http://localhost:8888/notify.twitter -F 'json={"message":"foo"}'
 ### Twitter OAuth Guide
 http://pocketstudio.jp/log3/2012/02/12/how_to_get_twitter_apikey_and_token/
 
+### Quick Tour to count a tweet with fluent-plugin-twitter and fluent-plugin-datacounter
+http://qiita.com/items/fe4258b394190f23fece
+
 ## TODO
 patches welcome!
 

data/fluent-plugin-twitter.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-twitter"
-  s.version     = "0.1.1"
+  s.version     = "0.2.1"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-twitter"
-  s.summary     = %q{Fluentd Input/Output plugin. You can create your own "Twitter Bot" with fluentd messaging system.}
+  s.summary     = %q{Fluentd Input/Output plugin to process tweets with Twitter Streaming API.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/in_twitter.rb

@@ -1,6 +1,7 @@
 module Fluent
   class TwitterInput < Fluent::Input
     TIMELINE_TYPE = %w(userstream sampling)
+    OUTPUT_FORMAT_TYPE = %w(nest flat simple)
     Plugin.register_input('twitter', self)
 
     config_param :consumer_key, :string
@@ -10,7 +11,9 @@ module Fluent
     config_param :tag, :string
     config_param :timeline, :string
     config_param :keyword, :string, :default => nil
-    config_param :lang, :string, :default => ''
+    config_param :lang, :string, :default => nil
+    config_param :output_format, :string, :default => 'simple'
+    config_param :flatten_separator, :string, :default => '_'
 
     def initialize
       super
@@ -19,7 +22,14 @@ module Fluent
 
     def configure(conf)
       super
-      raise Fluent::ConfigError, "timeline value undefined #{@timeline}" if !TIMELINE_TYPE.include?(@timeline)
+
+      if !TIMELINE_TYPE.include?(@timeline)
+        raise Fluent::ConfigError, "timeline value undefined #{@timeline}"
+      end
+      if !OUTPUT_FORMAT_TYPE.include?(@output_format)
+        raise Fluent::ConfigError, "output_format value undefined #{@output_format}"
+      end
+
       TweetStream.configure do |config|
         config.consumer_key = @consumer_key
         config.consumer_secret = @consumer_secret
@@ -31,6 +41,9 @@ module Fluent
 
     def start
       @thread = Thread.new(&method(:run))
+      @any = Proc.new do |hash|
+          get_message(hash) if is_message?(hash)
+      end
     end
 
     def shutdown
@@ -38,57 +51,69 @@ module Fluent
     end
 
     def run
+      client = get_twitter_connection
       if @timeline == 'sampling' && @keyword
-        start_twitter_track
+        client.track(@keyword)
       elsif @timeline == 'sampling' && @keyword.nil?
-        start_twitter_sample
+        client.sample
       elsif @timeline == 'userstream'
-        start_twitter_userstream
+        client.userstream
+      #elsif @timeline == 'follow'
+      #  client.follow(@follow_ids)
       end
     end
 
-    def start_twitter_track
-      $log.info "starting twitter keyword tracking. tag:#{@tag} lang:#{@lang} keyword:#{@keyword}"
+    def get_twitter_connection
+      notice = "twitter: starting Twitter Streaming API for #{@timeline}."
+      notice << " tag:#{@tag}"
+      notice << " lang:#{@lang}" unless @lang.nil?
+      notice << " keyword:#{@keyword}" unless @keyword.nil?
+      $log.info notice
       client = TweetStream::Client.new
-      client.track(@keyword) do |status|
-        next unless status.text
-        next unless @lang.include?(status.user.lang)
-        get_message(status)
+      client.on_anything(&@any)
+      client.on_error do |message|
+        $log.info "twitter: unexpected error has occured. #{message}"
       end
+      return client
     end
 
-    def start_twitter_sample
-      $log.info "starting twitter sampled streaming. tag:#{@tag} lang:#{@lang}"
-      client = TweetStream::Client.new
-      client.sample do |status|
-        next unless status.text
-        next unless @lang.include?(status.user.lang)
-        get_message(status)
-      end
+    def is_message?(status)
+      return false if !status.include?(:text)
+      return false if !status.include?(:user)
+      return false if (!@lang.nil? && @lang != '') && !@lang.include?(status[:user][:lang])
+      return true
     end
 
-    def start_twitter_userstream
-      $log.info "starting twitter userstream tracking. tag:#{@tag} lang:#{@lang}"
-      client = TweetStream::Client.new
-      client.userstream do |status|
-        next unless status.text
-        next unless @lang.include?(status.user.lang)
-        get_message(status)
+    def get_message(status)
+      case @output_format
+      when 'nest'
+        record = status.inject({}){|f,(k,v)| f[k.to_s] = v; f}
+      when 'flat'
+        record = hash_flatten(status)
+      when 'simple'
+        record = Hash.new
+        record.store('message', status[:text])
+        record.store('geo', status[:geo])
+        record.store('place', status[:place])
+        record.store('created_at', status[:place])
+        record.store('user_name', status[:user][:name])
+        record.store('user_screen_name', status[:user][:screen_name])
+        record.store('user_profile_image_url', status[:user][:profile_image_url])
+        record.store('user_time_zone', status[:user][:time_zone])
+        record.store('user_lang', status[:user][:lang])
       end
+      Engine.emit(@tag, Engine.now, record)
     end
 
-    def get_message(status)
-      record = Hash.new
-      record.store('message', status.text)
-      record.store('geo', status.geo)
-      record.store('place', status.place)
-      record.store('created_at', status.created_at)
-      record.store('user_name', status.user.name)
-      record.store('user_screen_name', status.user.screen_name)
-      record.store('user_profile_image_url', status.user.profile_image_url)
-      record.store('user_time_zone', status.user.time_zone)
-      record.store('user_lang', status.user.lang)
-      Engine.emit(@tag, Engine.now, record)
+    def hash_flatten(record, prefix = nil)
+      record.inject({}) do |d, (k, v)|
+        k = prefix.to_s + k.to_s
+        if v.is_a?(Hash)
+          d.merge(hash_flatten(v, k + @flatten_separator))
+        else
+          d.merge(k => v)
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-twitter
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-14 00:00:00.000000000 Z
+date: 2013-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &11259400 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *11259400
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &11258160 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11258160
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: twitter
-  requirement: &11257360 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +53,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11257360
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: tweetstream
-  requirement: &11283100 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +69,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11283100
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - y.ken.studio@gmail.com
@@ -93,11 +113,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Fluentd Input/Output plugin. You can create your own "Twitter Bot" with fluentd
-  messaging system.
+summary: Fluentd Input/Output plugin to process tweets with Twitter Streaming API.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_twitter.rb