fluent-plugin-twitter 0.0.1 → 0.1.1

data/.gitignore

@@ -1,18 +1,4 @@
 *.gem
-*.rbc
 .bundle
-.config
-coverage
-InstalledFiles
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp
-
-# YARD artifacts
-.yardoc
-_yardoc
-doc/
+Gemfile.lock
+pkg/*

data/README.md

@@ -1,8 +1,8 @@
 fluent-plugin-twitter
 =====================
 
-## Overview
-create your own twitter bot with fluentd
+## Component
+Fluentd Input/Output plugin. You can create your own "Twitter Bot" with fluentd messaging system.
 
 ## Installation
 
@@ -17,22 +17,67 @@ gem install fluent-plugin-twitter
 /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-twitter
 `````
 
-## Configuration
+## Input Configuration
 
-### Sample
+### Input Sample
 `````
+<source>
+  type twitter
+  consumer_key        YOUR_CONSUMER_KEY       # Required
+  consumer_secret     YOUR_CONSUMER_SECRET    # Required
+  oauth_token         YOUR_OAUTH_TOKEN        # Required
+  oauth_token_secret  YOUR_OAUTH_TOKEN_SECRET # Required
+  tag                 input.twitter.sampling  # Required
+  timeline            sampling                # Required (sampling or userstream)
+  keyword             Ruby,Python             # Optional (userstream not supported yet)
+  lang                ja,en                   # Optional
+</source>
+
+<match input.twitter.sampling>
+  type stdout
+</match>
+`````
+
+### Debug
+`````
+$ tail -f /var/log/td-agent/td-agent.log
+`````
+
+## Output Configuration
+
+### Output Sample
+`````
+<source>
+  type http
+  port 8888
+</source>
+
 <match notify.twitter>
   type twitter
   consumer_key        YOUR_CONSUMER_KEY
   consumer_secret     YOUR_CONSUMER_SECRET
-  oauth_token         YOUR_OAUTH_TOEKN
-  oauth_token_secret  YOUR_OAUTH_TOEKN_SECRET
+  oauth_token         YOUR_OAUTH_TOKEN
+  oauth_token_secret  YOUR_OAUTH_TOKEN_SECRET
 </match>
 `````
 
+### Debug
+`````
+$ curl http://localhost:8888/notify.twitter -F 'json={"message":"foo"}'
+`````
+
+## Reference
+
+### Twitter OAuth Guide
+http://pocketstudio.jp/log3/2012/02/12/how_to_get_twitter_apikey_and_token/
+
 ## TODO
 patches welcome!
 
+## Known Issue
+On starting fluentd, appearing alert message below. Please tell me how to fix up.
+`/usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/eventmachine-1.0.0/lib/eventmachine.rb:1530: warning: already initialized constant EM`
+
 ## Copyright
 
 Copyright © 2012- Kentaro Yoshida (@yoshi_ken)

data/fluent-plugin-twitter.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-twitter"
-  s.version     = "0.0.1"
+  s.version     = "0.1.1"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-twitter"
-  s.summary     = %q{Fluentd Output filter plugin. You can create your own "Twitter Bot" with fluentd messaging system.}
+  s.summary     = %q{Fluentd Input/Output plugin. You can create your own "Twitter Bot" with fluentd messaging system.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -17,4 +17,6 @@ Gem::Specification.new do |s|
   # specify any dependencies here; for example:
   s.add_development_dependency "fluentd"
   s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "twitter"
+  s.add_runtime_dependency "tweetstream"
 end

data/lib/fluent/plugin/in_twitter.rb

@@ -0,0 +1,94 @@
+module Fluent
+  class TwitterInput < Fluent::Input
+    TIMELINE_TYPE = %w(userstream sampling)
+    Plugin.register_input('twitter', self)
+
+    config_param :consumer_key, :string
+    config_param :consumer_secret, :string
+    config_param :oauth_token, :string
+    config_param :oauth_token_secret, :string
+    config_param :tag, :string
+    config_param :timeline, :string
+    config_param :keyword, :string, :default => nil
+    config_param :lang, :string, :default => ''
+
+    def initialize
+      super
+      require 'tweetstream'
+    end
+
+    def configure(conf)
+      super
+      raise Fluent::ConfigError, "timeline value undefined #{@timeline}" if !TIMELINE_TYPE.include?(@timeline)
+      TweetStream.configure do |config|
+        config.consumer_key = @consumer_key
+        config.consumer_secret = @consumer_secret
+        config.oauth_token = @oauth_token
+        config.oauth_token_secret = @oauth_token_secret
+        config.auth_method = :oauth
+      end
+    end
+
+    def start
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      Thread.kill(@thread)
+    end
+
+    def run
+      if @timeline == 'sampling' && @keyword
+        start_twitter_track
+      elsif @timeline == 'sampling' && @keyword.nil?
+        start_twitter_sample
+      elsif @timeline == 'userstream'
+        start_twitter_userstream
+      end
+    end
+
+    def start_twitter_track
+      $log.info "starting twitter keyword tracking. tag:#{@tag} lang:#{@lang} keyword:#{@keyword}"
+      client = TweetStream::Client.new
+      client.track(@keyword) do |status|
+        next unless status.text
+        next unless @lang.include?(status.user.lang)
+        get_message(status)
+      end
+    end
+
+    def start_twitter_sample
+      $log.info "starting twitter sampled streaming. tag:#{@tag} lang:#{@lang}"
+      client = TweetStream::Client.new
+      client.sample do |status|
+        next unless status.text
+        next unless @lang.include?(status.user.lang)
+        get_message(status)
+      end
+    end
+
+    def start_twitter_userstream
+      $log.info "starting twitter userstream tracking. tag:#{@tag} lang:#{@lang}"
+      client = TweetStream::Client.new
+      client.userstream do |status|
+        next unless status.text
+        next unless @lang.include?(status.user.lang)
+        get_message(status)
+      end
+    end
+
+    def get_message(status)
+      record = Hash.new
+      record.store('message', status.text)
+      record.store('geo', status.geo)
+      record.store('place', status.place)
+      record.store('created_at', status.created_at)
+      record.store('user_name', status.user.name)
+      record.store('user_screen_name', status.user.screen_name)
+      record.store('user_profile_image_url', status.user.profile_image_url)
+      record.store('user_time_zone', status.user.time_zone)
+      record.store('user_lang', status.user.lang)
+      Engine.emit(@tag, Engine.now, record)
+    end
+  end
+end

data/test/helper.rb

@@ -23,6 +23,7 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_twitter'
+require 'fluent/plugin/in_twitter'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_in_twitter.rb

@@ -0,0 +1,40 @@
+require 'helper'
+
+class TwitterInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    consumer_key        CONSUMER_KEY
+    consumer_secret     CONSUMER_SECRET
+    oauth_token         OAUTH_TOKEN
+    oauth_token_secret  OAUTH_TOKEN_SECRET
+    tag                 input.twitter
+    timeline            sampling
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::TwitterInput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    d = create_driver %[
+      consumer_key        CONSUMER_KEY
+      consumer_secret     CONSUMER_SECRET
+      oauth_token         OAUTH_TOKEN
+      oauth_token_secret  OAUTH_TOKEN_SECRET
+      tag                 input.twitter
+      timeline            sampling
+    ]
+    d.instance.inspect
+    assert_equal 'CONSUMER_KEY', d.instance.consumer_key
+    assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
+    assert_equal 'OAUTH_TOKEN', d.instance.oauth_token
+    assert_equal 'OAUTH_TOKEN_SECRET', d.instance.oauth_token_secret
+  end
+end
+

data/test/plugin/test_out_twitter.rb

@@ -8,8 +8,8 @@ class TwitterOutputTest < Test::Unit::TestCase
   CONFIG = %[
     consumer_key        CONSUMER_KEY
     consumer_secret     CONSUMER_SECRET
-    oauth_token         OAUTH_TOEKN
-    oauth_token_secret  OAUTH_TOEKN_SECRET
+    oauth_token         OAUTH_TOKEN
+    oauth_token_secret  OAUTH_TOKEN_SECRET
   ]
 
   def create_driver(conf=CONFIG,tag='test')
@@ -23,14 +23,14 @@ class TwitterOutputTest < Test::Unit::TestCase
     d = create_driver %[
       consumer_key        CONSUMER_KEY
       consumer_secret     CONSUMER_SECRET
-      oauth_token         OAUTH_TOEKN
-      oauth_token_secret  OAUTH_TOEKN_SECRET
+      oauth_token         OAUTH_TOKEN
+      oauth_token_secret  OAUTH_TOKEN_SECRET
     ]
     d.instance.inspect
     assert_equal 'CONSUMER_KEY', d.instance.consumer_key
     assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
-    assert_equal 'OAUTH_TOEKN', d.instance.oauth_token
-    assert_equal 'OAUTH_TOEKN_SECRET', d.instance.oauth_token_secret
+    assert_equal 'OAUTH_TOKEN', d.instance.oauth_token
+    assert_equal 'OAUTH_TOKEN_SECRET', d.instance.oauth_token_secret
   end
 
   def test_emit

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-twitter
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-02 00:00:00.000000000 Z
+date: 2012-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &16126320 !ruby/object:Gem::Requirement
+  requirement: &11259400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *16126320
+  version_requirements: *11259400
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &16125060 !ruby/object:Gem::Requirement
+  requirement: &11258160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,29 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *16125060
+  version_requirements: *11258160
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: &11257360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *11257360
+- !ruby/object:Gem::Dependency
+  name: tweetstream
+  requirement: &11283100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *11283100
 description: 
 email:
 - y.ken.studio@gmail.com
@@ -46,8 +68,10 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-twitter.gemspec
+- lib/fluent/plugin/in_twitter.rb
 - lib/fluent/plugin/out_twitter.rb
 - test/helper.rb
+- test/plugin/test_in_twitter.rb
 - test/plugin/test_out_twitter.rb
 homepage: https://github.com/y-ken/fluent-plugin-twitter
 licenses: []
@@ -72,8 +96,9 @@ rubyforge_project:
 rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
-summary: Fluentd Output filter plugin. You can create your own "Twitter Bot" with
-  fluentd messaging system.
+summary: Fluentd Input/Output plugin. You can create your own "Twitter Bot" with fluentd
+  messaging system.
 test_files:
 - test/helper.rb
+- test/plugin/test_in_twitter.rb
 - test/plugin/test_out_twitter.rb