embulk-plugin-twitterstream 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: add26419b4ae29992705f6d30f3d8c668cdb64ce
+  data.tar.gz: b28e1559c7e6e95b8f47a446584c635dcf50fb81
+SHA512:
+  metadata.gz: 2bf1266efe1988b791fcbee980896576414d9f1b9e31a6c69285889843148e29b15fcf7bef4ff312b50e9c70d63cb5f83862ae83310e7b816bce0d7a2579f2cc
+  data.tar.gz: 8ea781ebf0b26d7ed2f1c67979f44ed5ebb40405c065f2cd3b221a497bc054a06227366552ce848ada7778731287ca1137196ccdbfddc7e7db3ec98a71642dd5

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+gemspec
+
+gem 'twitter'

data/README.md

@@ -0,0 +1,31 @@
+# Embulk plugin for Twitter Stream input
+
+This [Embulk](https://github.com/embulk/embulk) plugin read records from Twitter Stream!
+
+## Configuration
+
+```yaml
+exec: {}
+in:
+  type: twitterstream
+  consumer_key:        XXXXXXXXXXXXXXXXXXXXXX
+  consumer_secret:     XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+  access_token:        XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+  access_token_secret: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+  count: 3
+  columns:
+    - id_str
+    - name
+    - user.screen_name
+out: {type: stdout}
+```
+
+`count` and `columns` are omittable.
+
+## License
+
+MIT
+
+## Author
+
+Yasuhiro Matsumoto (a.k.a mattn)

data/Rakefile

@@ -0,0 +1,4 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+task :default => [:build]

data/embulk-plugin-twitterstream.gemspec

@@ -0,0 +1,21 @@
+
+Gem::Specification.new do |gem|
+  gem.name          = "embulk-plugin-twitterstream"
+  gem.version       = "0.0.1"
+
+  gem.summary       = %q{Embulk plugin for TwitterStream input}
+  gem.description   = gem.summary
+  gem.authors       = ["Yasuhiro Matsumoto"]
+  gem.email         = ["mattn.jp@gmail.com"]
+  gem.license       = "MIT"
+  gem.homepage      = "https://github.com/mattn/embulk-plugin-twitterstream"
+
+  gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc      = false
+
+  gem.add_development_dependency 'twitter', ['>= 5.13.0']
+  gem.add_development_dependency 'bundler', ['~> 1.0']
+  gem.add_development_dependency 'rake', ['>= 0.9.2']
+end

data/lib/embulk/input_twitterstream.rb

@@ -0,0 +1,119 @@
+require 'twitter'
+require 'json'
+
+module Embulk
+  class StopStreamException < Exception; end
+
+  class InputTwitterStream < InputPlugin
+    Plugin.register_input('twitterstream', self)
+
+      COLUMN_NAMES = [
+        'text',
+        'created_at',
+        'id_str',
+        'user.follow_request_sent',
+        'user.following',
+        'user.time_zone',
+        'user.created_at',
+        'user.profile_sidebar_fill_color',
+        'user.profile_image_url',
+        'user.default_profile',
+        'user.geo_enabled',
+        'user.profile_sidebar_border_color',
+        'user.is_translator',
+        'user.url',
+        'user.profile_image_url_https',
+        'user.description',
+        'user.listed_count',
+        'user.profile_use_background_image',
+        'user.friends_count',
+        'user.followers_count',
+        'user.profile_text_color',
+        'user.profile_background_image_url',
+        'user.location',
+        'user.profile_link_color',
+        'user.protected',
+        'user.default_profile_image',
+        'user.lang',
+        'user.statuses_count',
+        'user.verified',
+        'user.name',
+        'user.id_str',
+        'user.show_all_inline_media',
+        'user.contributors_enabled',
+        'user.notifications',
+        'user.profile_background_image_url_https',
+        'user.profile_background_color',
+        'user.id',
+        'user.profile_background_tile',
+        'user.utc_offset',
+        'user.favourites_count',
+        'user.screen_name',
+        'in_reply_to_user_id',
+        'id',
+        'contributors',
+        'truncated'
+      ]
+
+    def self.transaction(config, &control)
+      task = config
+      threads = 1
+      cols = config.param('columns', :array)
+      if cols.empty?
+        columns = COLUMN_NAMES.map.with_index  {|column, index|
+          Column.new(index, column, :string)
+        }
+      else
+        columns = config.param('columns', :array).map.with_index { |column, index|
+          Column.new(i, column, :string)
+        }
+      end
+      commit_reports = yield(task, columns, threads)
+      return {}
+    end
+
+    def initialize(task, schema, index, page_builder)
+      super
+    end
+
+    def dot_flatten(hash, path = '')
+      hash.each_with_object({}) do |(k, v), ret|
+        key = path + k.to_s
+        if v.is_a? Hash
+          ret.merge! dot_flatten(v, key + '.')
+        else
+          ret[key] = v
+        end
+      end
+    end
+
+    def run
+      client = Twitter::Streaming::Client.new(
+        consumer_key:         @task['consumer_key'],
+        consumer_secret:      @task['consumer_secret'],
+        access_token:         @task['access_token'],
+        access_token_secret:  @task['access_token_secret'],
+      )
+      count = @task['count'] ? @task['count'].to_i : 0
+      begin
+        client.user do |item|
+          case item
+          when Twitter::Tweet
+            tweet = dot_flatten(item.to_hash)
+            @page_builder.add(@schema.map {|column| tweet.has_key?(column.name) ? tweet[column.name].to_s : ''})
+            if count > 0
+              count -= 1
+              if count == 0
+                raise StopStreamException if count == 0
+              end
+            end
+          end
+        end
+      rescue StopStreamException
+      end
+      @page_builder.finish
+      commit_report = {}
+      return commit_report
+    end
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: embulk-plugin-twitterstream
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Yasuhiro Matsumoto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.13.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Embulk plugin for TwitterStream input
+email:
+- mattn.jp@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- embulk-plugin-twitterstream.gemspec
+- lib/embulk/input_twitterstream.rb
+homepage: https://github.com/mattn/embulk-plugin-twitterstream
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Embulk plugin for TwitterStream input
+test_files: []