twitter_to_csv 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.idea

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.2@twitter_to_csv --create

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in twitter_to_csv.gemspec
+gemspec
+

data/README.markdown

@@ -0,0 +1,9 @@
+# Twitter To CSV
+
+## Usage
+
+      twitter_to_csv --username <your twitter username> --password <your twitter password> \
+                     --json hi.json --filter zit,zits,pimple,pimples,acne
+
+
+Verbosity and actually outputting to a CSV coming soon :)

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/twitter_to_csv

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby 
+require 'rubygems'
+require 'open-uri'
+require 'optparse'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'twitter_to_csv'))
+
+options = { :csv => STDOUT }
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($0)} [options]"
+  opts.separator ""
+  opts.separator "Specific options:"
+
+  opts.on("-u", "--username USERNAME", "Twitter username") do |username|
+    options[:username] = username
+  end
+
+  opts.on("-p", "--password PASSWORD", "Twitter password") do |password|
+    options[:password] = password
+  end
+
+  opts.on("-c", "--csv [FILE]", "The CSV file to write to") do |csv|
+    options[:csv] = File.open(csv, 'a')
+  end
+  
+  opts.on("-j", "--json [FILE]", "The file to log all Twitter JSON to") do |json|
+    options[:json] = File.open(json, 'a')
+  end
+
+  opts.on("-f", "--filter [KEYWORDS]", "Keywords to ask Twitter to filter on") do |filter|
+    options[:filter] = filter.split(/\s*,\s*/)
+  end
+
+  opts.on("-e", "--require-english", "Attempt to filter out non-English tweets.", "This will have both false positives and false negatives.") do |e|
+    options[:require_english] = e
+  end
+
+  opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+    options[:verbose] = v
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    STDERR.puts opts
+    exit
+  end
+  
+  opts.on_tail("--version", "Show version") do
+    STDERR.puts "twitter_to_csv version #{TwitterToCsv::VERSION}"
+    exit
+  end
+end
+
+parser.parse!
+
+unless options[:username] && options[:password]
+  STDERR.puts "Error: Twitter username and password are required fields.\n\n"
+  STDERR.puts parser
+  exit 1
+end
+
+TwitterToCsv::CsvBuilder.new(options).run

data/lib/twitter_to_csv.rb

@@ -0,0 +1,13 @@
+require 'rubygems'
+require File.expand_path(File.join(File.dirname(__FILE__), "twitter_to_csv", "version"))
+require 'fastercsv'
+require 'pp'
+require 'json'
+require 'twitter/json_stream'
+require 'em-http-request'
+require File.expand_path(File.join(File.dirname(__FILE__), "twitter_to_csv", "twitter_watcher"))
+require File.expand_path(File.join(File.dirname(__FILE__), "twitter_to_csv", "csv_builder"))
+require 'unsupervised-language-detection'
+
+module TwitterToCsv
+end

data/lib/twitter_to_csv/csv_builder.rb

@@ -0,0 +1,53 @@
+module TwitterToCsv
+  class CsvBuilder
+    attr_accessor :options
+
+    def initialize(options = {})
+      @options = options
+    end
+
+    def is_english?(status)
+      if status.has_key?('delete')
+        STDERR.puts "Skipping Tweet with delete." if options[:verbose]
+        return false
+      end
+
+      if status['text'] =~ /[^[:ascii:]]/
+        STDERR.puts "Skipping \"#{status['text']}\" due to non-ascii text." if options[:verbose]
+        return false
+      end
+
+      unless status['user']['lang'] == "en"
+        STDERR.puts "Skipping \"#{status['text']}\" due to lang of #{status['user']['lang']}." if options[:verbose]
+        return false
+      end
+
+      unless UnsupervisedLanguageDetection.is_english_tweet?(status['text'])
+        STDERR.puts "Skipping \"#{status['text']}\" due to UnsupervisedLanguageDetection guessing non-English" if options[:verbose]
+        return false
+      end
+
+      true
+    end
+
+    def run
+      begin
+        TwitterWatcher.new(options).run do |status|
+          if (options[:require_english] && is_english?(status)) || !options[:require_english]
+            if options[:json]
+              options[:json].puts JSON.dump(status) #JSON.pretty_generate(status)
+              options[:json].puts "------SEPERATOR------"
+              options[:json].flush
+            end
+            STDERR.puts "Logging: #{status['text']}" if options[:verbose]
+          end
+        end
+      rescue StandardError => e
+        STDERR.puts "\nException #{e.message}:\n#{e.backtrace.join("\n")}\n\n"
+        STDERR.puts "Waiting for a couple of minutes..."
+        sleep 120
+        retry
+      end
+    end
+  end
+end

data/lib/twitter_to_csv/twitter_watcher.rb

@@ -0,0 +1,49 @@
+module TwitterToCsv
+  class TwitterWatcher
+    attr_accessor :username, :password, :filter, :fetch_errors
+
+    def initialize(options)
+      @username = options[:username]
+      @password = options[:password]
+      @filter = options[:filter]
+      @fetch_errors = 0
+    end
+
+    def progress(str)
+      STDERR.print "#{str}..."
+      STDERR.flush
+      yield
+      STDERR.puts "done."
+    end
+
+    def run(&block)
+      EventMachine::run do
+        stream = Twitter::JSONStream.connect(
+          :path    => "/1/statuses/#{(filter && filter.length > 0) ? 'filter' : 'sample'}.json#{"?track=#{filter.join(",")}" if filter && filter.length > 0}",
+          :auth    => "#{username}:#{password}",
+          :ssl     => true
+        )
+
+        stream.each_item do |item|
+          handle_status JSON.parse(item), block
+        end
+
+        stream.on_error do |message|
+          STDERR.puts " --> Twitter error: #{message} <--"
+        end
+
+        stream.on_max_reconnects do |timeout, retries|
+          STDERR.puts " --> Oops, tried too many times! <--"
+          EventMachine::stop_event_loop
+        end
+      end
+    end
+
+    def handle_status(status, block)
+      return unless status
+      return if status.has_key?('delete')
+      status['text'] = status['text'].gsub(/&lt;/, "<").gsub(/&gt;/, ">").gsub(/[\t\n\r]/, '  ')
+      block.call(status)
+    end
+  end
+end

data/lib/twitter_to_csv/version.rb

@@ -0,0 +1,3 @@
+module TwitterToCsv
+  VERSION = "0.0.1"
+end

data/twitter_to_csv.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "twitter_to_csv/version"
+
+Gem::Specification.new do |s|
+  s.name        = "twitter_to_csv"
+  s.version     = TwitterToCsv::VERSION
+  s.authors     = ["Andrew Cantino"]
+  s.email       = ["andrew@iterationlabs.com"]
+  s.homepage    = ""
+  s.summary     = %q{Dump the Twitter streaming API to a CSV or JSON file}
+  s.description = %q{}
+
+  s.rubyforge_project = "twitter_to_csv"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # s.add_development_dependency "rspec"
+  s.add_runtime_dependency 'fastercsv'
+  s.add_runtime_dependency 'twitter-stream'
+  s.add_runtime_dependency 'em-http-request'
+  s.add_runtime_dependency 'unsupervised-language-detection'
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: twitter_to_csv
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Andrew Cantino
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-17 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fastercsv
+  requirement: &83775030 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *83775030
+- !ruby/object:Gem::Dependency
+  name: twitter-stream
+  requirement: &83774820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *83774820
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: &83774610 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *83774610
+- !ruby/object:Gem::Dependency
+  name: unsupervised-language-detection
+  requirement: &83774400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *83774400
+description: ''
+email:
+- andrew@iterationlabs.com
+executables:
+- twitter_to_csv
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- README.markdown
+- Rakefile
+- bin/twitter_to_csv
+- lib/twitter_to_csv.rb
+- lib/twitter_to_csv/csv_builder.rb
+- lib/twitter_to_csv/twitter_watcher.rb
+- lib/twitter_to_csv/version.rb
+- twitter_to_csv.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: twitter_to_csv
+rubygems_version: 1.8.16
+signing_key: 
+specification_version: 3
+summary: Dump the Twitter streaming API to a CSV or JSON file
+test_files: []