mihatter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 16370366bc95fe808cd74431832fdb9e996eeb62
+  data.tar.gz: 9fd5df7779f57f9c75dec597dcf86a2988aeb235
+SHA512:
+  metadata.gz: 5cb17d507e29e7f3b5679edd47aa9224eb99c10f500aad16f0974b0a694c5924824e4ace5bd63cf2da039beccb5f39228ef7e87842864b400d4ea48bdd822539
+  data.tar.gz: 4ea59e4ae3e23974056918f6cebc42baced72cfc57b25d3ea39de7980aadb6c0b61ae3592608c46a8b620f8ea88bddea533322dcf240e2a11de5468837e78f91

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mihatter.gemspec
+gemspec

data/README.md

@@ -0,0 +1,53 @@
+# Mihatter
+
+Mihatter is a simple Twitter crawler.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'mihatter'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mihatter
+
+## Usage
+
+```ruby
+require 'mihatter'
+
+Mihatter.configuration do |config|
+  config.consumer_key = '.....'
+  config.consumer_secret = '.....'
+  config.access_token = '.....'
+  config.access_token_secret = '.....'
+
+  config.keyword = 'sushi'
+end
+
+# use REST API
+watcher = Mihatter::RestWatcher.new
+watcher.run! do |tweet|
+  puts tweet.text
+end
+
+# use Streaming API (only English, cannot use CJK)
+watcher = Mihatter::StreamingWatcher.new
+watcher.run! do |tweet|
+  puts tweet.text
+end
+```
+
+## License
+MIT
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/rutan/mihatter.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler/gem_tasks'
+task :default => :spec

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'mihatter'
+require 'pry'
+Pry.start
+

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/mihatter.rb

@@ -0,0 +1,12 @@
+require 'mihatter/version'
+require 'mihatter/configuration'
+require 'mihatter/rest_watcher'
+require 'mihatter/streaming_watcher'
+
+module Mihatter
+  def self.configuration
+    @configuration ||= Mihatter::Configuration.new
+    yield @configuration if block_given?
+    @configuration
+  end
+end

data/lib/mihatter/configuration.rb

@@ -0,0 +1,24 @@
+module Mihatter
+  class Configuration
+    def initialize
+      self.keyword = 'Twitter'
+      self.wait_time = 30
+    end
+
+    attr_accessor :consumer_key
+    attr_accessor :consumer_secret
+    attr_accessor :access_token
+    attr_accessor :access_token_secret
+
+    attr_accessor :keyword
+    attr_accessor :lang
+    attr_accessor :since_id
+
+    attr_accessor :wait_time
+
+    def merge(config = {})
+      config.each { |k, v| self.public_send("#{k}=", v) }
+      self
+    end
+  end
+end

data/lib/mihatter/rest_watcher.rb

@@ -0,0 +1,67 @@
+require 'twitter'
+
+module Mihatter
+  class RestWatcher
+    def initialize(config = {})
+      @config = Mihatter.configuration.dup.merge(config)
+    end
+
+    attr_reader :config
+
+    def run!
+      raise ArgumentError, '`Mihatter::RestWatcher#run!` require block' unless block_given?
+
+      begin
+        connect
+        assign_since_id unless @config.since_id
+        loop do
+          search.each do |tweet|
+            yield tweet
+            @config.since_id = tweet.id
+          end
+          sleep @config.wait_time
+        end
+      rescue Twitter::Error::TooManyRequests => e
+        sleep e.rate_limit.reset_in
+      end
+    end
+
+    def since_id
+      @config.since_id
+    end
+
+    private
+
+    def connect
+      @client = Twitter::REST::Client.new do |config|
+        config.consumer_key = @config.consumer_key
+        config.consumer_secret = @config.consumer_secret
+        config.access_token = @config.access_token
+        config.access_token_secret = @config.access_token_secret
+      end
+    end
+
+    def assign_since_id
+      result = @client.search(@config.keyword, {
+        result_type: 'recent',
+        count: 1,
+        lang: @config.lang,
+      }.delete_if { |_, v| v.nil? }).first
+      @config.since_id = (result ? result.id : 0)
+    end
+
+    def search(max_id: nil)
+      results = @client.search(@config.keyword, {
+        result_type: 'recent',
+        count: 100,
+        lang: @config.lang,
+        max_id: max_id,
+        since_id: @config.since_id,
+      }.delete_if {|_, v| v.nil?} ).take(100).sort do |a, b|
+        a.id <=> b.id
+      end
+      results = search(max_id: results.first.id - 1) + results if results.size == 100
+      results
+    end
+  end
+end

data/lib/mihatter/streaming_watcher.rb

@@ -0,0 +1,30 @@
+require 'twitter'
+
+module Mihatter
+  class StreamingWatcher
+    def initialize(config = {})
+      @config = Mihatter.configuration.dup.merge(config)
+    end
+
+    attr_reader :config
+
+    def run!
+      raise ArgumentError, '`Mihatter::StreamingWatcher#run!` require block' unless block_given?
+      connect
+      @client.filter(track: @config.keyword) do |obj|
+        yield obj if obj.is_a?(Twitter::Tweet)
+      end
+    end
+
+    private
+
+    def connect
+      @client = Twitter::Streaming::Client.new do |config|
+        config.consumer_key = @config.consumer_key
+        config.consumer_secret = @config.consumer_secret
+        config.access_token = @config.access_token
+        config.access_token_secret = @config.access_token_secret
+      end
+    end
+  end
+end

data/lib/mihatter/version.rb

@@ -0,0 +1,3 @@
+module Mihatter
+  VERSION = '0.1.0'
+end

data/mihatter.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mihatter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'mihatter'
+  spec.version       = Mihatter::VERSION
+  spec.authors       = ['ru_shalm']
+  spec.email         = ['ru_shalm@hazimu.com']
+  spec.summary       = %q{Simple Twitter crawler}
+  spec.homepage      = 'https://github.com/rutan/mihatter'
+  spec.licenses      = ['MIT']
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'twitter', '~> 5'
+  spec.add_development_dependency 'bundler', '~> 1.12'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'pry'
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: mihatter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- ru_shalm
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-05-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ru_shalm@hazimu.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/mihatter.rb
+- lib/mihatter/configuration.rb
+- lib/mihatter/rest_watcher.rb
+- lib/mihatter/streaming_watcher.rb
+- lib/mihatter/version.rb
+- mihatter.gemspec
+homepage: https://github.com/rutan/mihatter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Simple Twitter crawler
+test_files: []