twitter_to_csv 0.0.2 → 0.0.3

data/README.markdown

@@ -1,12 +1,182 @@
 # Twitter To CSV
 
+A tool for exporting the Twitter stream into a CSV file.
+
+    (sudo) gem install twitter_to_csv
+
 ## Usage
 
-      twitter_to_csv --username <your twitter username> --password <your twitter password> \
-                     --json hi.json --filter zit,zits,pimple,pimples,acne \
-                     --csv out.csv --fields text,
-                     --fields text,retweeted_status.id,retweeted_status.favorited,...
-                     
-Use `--sample-fields 1000`` to output the occurrence count of different Twitter fields.
+You might start by first running the script for a while to dump the Twitter stream into a JSON file:
+
+    twitter_to_csv --username <your twitter username> --password <your twitter password> \
+                   --json out.json --filter your,keywords,of,interest
+
+Then, later, you could export to CSV:
+
+    twitter_to_csv --replay-from-file out.json --csv out.csv \
+                   --fields text,created_at,user.name,retweeted_status.id,retweeted_status.favorited,...
+
+Alternatively, you can always stream directly to CSV:
+
+    twitter_to_csv --username <your twitter username> --password <your twitter password> \
+                   --filter your,keywords,of,interest --csv out.csv \
+                   --fields text,created_at,user.name,retweeted_status.id,retweeted_status.favorited,...
+
+## Requiring English
+
+You may want to limit to Tweets that appear to be writen in English.
+
+    twitter_to_csv --username <your twitter username> --password <your twitter password> \
+                   --require-english --fields ...
+
+This filter isn't perfect and will have both false positives and false negatives, but it works pretty well.
+
+## URLS
+
+You can extract URLs from the tweet into their own columns by including `--url-columns 3`, for example, to get up to 3 extracted URLs in their own columns.
+
+## Field names
+
+Use `--sample-fields 1000`` to output the occurrence count of different Twitter fields, like so:
+
+    twitter_to_csv --username <your twitter username> --password <your twitter password> --sample-fields 1000
+
+Here's a partial list:
 
-You can also `--replay-from-file` if you have a JSON output file and you want to run it back through the exporter.
+    in_reply_to_screen_name
+    favorited
+    text
+    entities.urls
+    entities.user_mentions
+    entities.hashtags
+    in_reply_to_user_id
+    contributors
+    place
+    coordinates
+    source
+    geo
+    retweeted
+    retweet_count
+    in_reply_to_status_id
+    in_reply_to_status_id_str
+    id_str
+    user.default_profile_image
+    user.verified
+    user.notifications
+    user.profile_sidebar_border_color
+    user.screen_name
+    user.lang
+    user.favourites_count
+    user.contributors_enabled
+    user.profile_use_background_image
+    user.friends_count
+    user.location
+    user.profile_text_color
+    user.followers_count
+    user.profile_image_url
+    user.description
+    user.statuses_count
+    user.following
+    user.profile_background_image_url
+    user.show_all_inline_media
+    user.listed_count
+    user.profile_link_color
+    user.is_translator
+    user.default_profile
+    user.time_zone
+    user.profile_background_color
+    user.protected
+    user.id_str
+    user.geo_enabled
+    user.profile_background_tile
+    user.name
+    user.profile_background_image_url_https
+    user.created_at
+    user.profile_sidebar_fill_color
+    user.id
+    user.follow_request_sent
+    user.utc_offset
+    user.url
+    user.profile_image_url_https
+    truncated
+    id
+    created_at
+    in_reply_to_user_id_str
+    retweeted_status.in_reply_to_screen_name
+    retweeted_status.favorited
+    retweeted_status.text
+    retweeted_status.entities.urls
+    retweeted_status.entities.user_mentions
+    retweeted_status.entities.hashtags
+    retweeted_status.in_reply_to_user_id
+    retweeted_status.contributors
+    retweeted_status.place
+    retweeted_status.coordinates
+    retweeted_status.source
+    retweeted_status.geo
+    retweeted_status.retweeted
+    retweeted_status.retweet_count
+    retweeted_status.in_reply_to_status_id
+    retweeted_status.in_reply_to_status_id_str
+    retweeted_status.id_str
+    retweeted_status.user.default_profile_image
+    retweeted_status.user.verified
+    retweeted_status.user.notifications
+    retweeted_status.user.profile_sidebar_border_color
+    retweeted_status.user.screen_name
+    retweeted_status.user.lang
+    retweeted_status.user.favourites_count
+    retweeted_status.user.contributors_enabled
+    retweeted_status.user.profile_use_background_image
+    retweeted_status.user.friends_count
+    retweeted_status.user.location
+    retweeted_status.user.profile_text_color
+    retweeted_status.user.followers_count
+    retweeted_status.user.profile_image_url
+    retweeted_status.user.description
+    retweeted_status.user.statuses_count
+    retweeted_status.user.following
+    retweeted_status.user.profile_background_image_url
+    retweeted_status.user.show_all_inline_media
+    retweeted_status.user.listed_count
+    retweeted_status.user.profile_link_color
+    retweeted_status.user.is_translator
+    retweeted_status.user.default_profile
+    retweeted_status.user.time_zone
+    retweeted_status.user.profile_background_color
+    retweeted_status.user.protected
+    retweeted_status.user.id_str
+    retweeted_status.user.geo_enabled
+    retweeted_status.user.profile_background_tile
+    retweeted_status.user.name
+    retweeted_status.user.profile_background_image_url_https
+    retweeted_status.user.created_at
+    retweeted_status.user.profile_sidebar_fill_color
+    retweeted_status.user.id
+    retweeted_status.user.follow_request_sent
+    retweeted_status.user.utc_offset
+    retweeted_status.user.url
+    retweeted_status.user.profile_image_url_https
+    retweeted_status.truncated
+    retweeted_status.id
+    retweeted_status.created_at
+    retweeted_status.in_reply_to_user_id_str
+    possibly_sensitive
+    possibly_sensitive_editable
+    retweeted_status.possibly_sensitive
+    retweeted_status.possibly_sensitive_editable
+    place.country_code
+    place.place_type
+    place.country
+    place.bounding_box.type
+    place.bounding_box.coordinates
+    place.full_name
+    place.name
+    place.id
+    place.url
+    coordinates.type
+    coordinates.coordinates
+    geo.type
+    geo.coordinates
+    retweeted_status.entities.media
+    entities.media

data/bin/twitter_to_csv

@@ -1,10 +1,10 @@
-#!/usr/bin/env ruby 
+#!/usr/bin/env ruby
 require 'rubygems'
 require 'open-uri'
 require 'optparse'
 require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'twitter_to_csv'))
 
-options = { :csv => STDOUT, :fields => %w[text user.lang] }
+options = { :csv => STDOUT, :fields => %w[text] }
 parser = OptionParser.new do |opts|
   opts.banner = "Usage: #{File.basename($0)} [options]"
   opts.separator ""
@@ -18,20 +18,20 @@ parser = OptionParser.new do |opts|
     options[:password] = password
   end
 
-  opts.on("-c", "--csv [FILE]", "The CSV file to append to") do |csv|
+  opts.on("-c", "--csv FILE", "The CSV file to append to") do |csv|
     options[:csv_appending] = File.exists?(csv)
     options[:csv] = File.open(csv, 'a')
   end
-  
-  opts.on("-j", "--json [FILE]", "The JSON file to append to") do |json|
+
+  opts.on("-j", "--json FILE", "The JSON file to append to") do |json|
     options[:json] = File.open(json, 'a')
   end
 
-  opts.on("-f", "--filter [KEYWORDS]", "Keywords to ask Twitter to filter on") do |filter|
+  opts.on("-f", "--filter KEYWORDS", "Keywords to ask Twitter to filter on") do |filter|
     options[:filter] = filter.split(/\s*,\s*/)
   end
 
-  opts.on("-x", "--fields [FIELDS]", "Fields to include in the CSV") do |fields|
+  opts.on("-x", "--fields FIELDS", "Fields to include in the CSV") do |fields|
     options[:fields] = fields.split(/\s*,\s*/)
   end
 
@@ -43,19 +43,23 @@ parser = OptionParser.new do |opts|
     options[:verbose] = v
   end
 
-  opts.on_tail("", "--sample-fields [NUMBER_OF_SAMPLES]", "Sample fields from Twitter, display them, and then exit.") do |samples|
+  opts.on_tail(nil, "--sample-fields NUMBER_OF_SAMPLES", "Record NUMBER_OF_SAMPLES tweets and then print out all","of the field names seen.  Use to find out what can be passed to.") do |samples|
     options[:sample_fields] = samples && samples.to_i
   end
 
-  opts.on_tail("", "--replay-from-file FILENAME", "Replay status from a JSON dump file") do |replay_file|
+  opts.on_tail("", "--replay-from-file FILENAME", "Replay tweets from a JSON dump file") do |replay_file|
     options[:replay_from_file] = replay_file
   end
 
+  opts.on_tail("", "--url-columns NUMBER_OF_COLUMNS", "Extract up to NUMBER_OF_COLUMNS urls from the status and include them in the CSV") do |url_columns|
+    options[:url_columns] = url_columns.to_i
+  end
+
   opts.on_tail("-h", "--help", "Show this message") do
     STDERR.puts opts
     exit
   end
-  
+
   opts.on_tail("--version", "Show version") do
     STDERR.puts "twitter_to_csv version #{TwitterToCsv::VERSION}"
     exit

data/lib/twitter_to_csv/csv_builder.rb

@@ -1,9 +1,13 @@
+# encoding: UTF-8
 require 'pp'
 
 module TwitterToCsv
   class CsvBuilder
     attr_accessor :options, :sampled_fields
 
+    # http://daringfireball.net/2010/07/improved_regex_for_matching_urls
+    URL_REGEX = %r"\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s\(\)<>]+|\((?:[^\s\(\)<>]+|(?:\([^\s\(\)<>]+\)))*\))+(?:\((?:[^\s\(\)<>]+|(?:\([^\s\(\)<>]+\)))*\)|[^\s\`\!\(\)\[\]\{\};:'\".,<>\?«»“”‘’]))"i
+
     def initialize(options = {})
       @options = options
       @sampled_fields = {}
@@ -20,6 +24,7 @@ module TwitterToCsv
             handle_status status
           end
         rescue SignalException, SystemExit
+          EventMachine::stop_event_loop
           exit
         rescue StandardError => e
           STDERR.puts "\nException #{e.message}:\n#{e.backtrace.join("\n")}\n\n"
@@ -40,7 +45,13 @@ module TwitterToCsv
     end
 
     def log_csv_header
-      options[:csv].puts options[:fields].to_csv(:encoding => 'UTF-8', :force_quotes => true)
+      header_labels = options[:fields].dup
+
+      if options[:url_columns] && options[:url_columns] > 0
+        options[:url_columns].times { |i| header_labels << "url_#{i+1}" }
+      end
+
+      options[:csv].puts header_labels.to_csv(:encoding => 'UTF-8', :force_quotes => true)
     end
 
     def log_csv(status)
@@ -48,8 +59,14 @@ module TwitterToCsv
         field.split(".").inject(status) { |memo, segment|
           memo && memo[segment]
         }.to_s
-      end.to_csv(:encoding => 'UTF-8', :force_quotes => true)
-      options[:csv].puts csv_row
+      end
+
+      if options[:url_columns] && options[:url_columns] > 0
+        urls = status['text'].scan(URL_REGEX).flatten.compact
+        options[:url_columns].times { |i| csv_row << urls[i].to_s }
+      end
+
+      options[:csv].puts csv_row.to_csv(:encoding => 'UTF-8', :force_quotes => true)
     end
 
     def replay_from(filename)

data/lib/twitter_to_csv/version.rb

@@ -1,3 +1,3 @@
 module TwitterToCsv
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/spec/csv_builder_spec.rb

@@ -17,6 +17,24 @@ describe TwitterToCsv::CsvBuilder do
       end
     end
 
+    describe "log_csv_header" do
+      it "outputs the fields as header labels" do
+        string_io = StringIO.new
+        csv_builder = TwitterToCsv::CsvBuilder.new(:csv => string_io, :fields => %w[something something_else.a])
+        csv_builder.log_csv_header
+        string_io.rewind
+        string_io.read.should == '"something","something_else.a"' + "\n"
+      end
+
+      it "includes urls if requested" do
+        string_io = StringIO.new
+        csv_builder = TwitterToCsv::CsvBuilder.new(:csv => string_io, :fields => %w[something], :url_columns => 2)
+        csv_builder.log_csv_header
+        string_io.rewind
+        string_io.read.should == '"something","url_1","url_2"' + "\n"
+      end
+    end
+
     describe "logging to a CSV" do
       it "outputs the requested fields when requested in dot-notation" do
         string_io = StringIO.new
@@ -35,6 +53,22 @@ describe TwitterToCsv::CsvBuilder do
         string_io.rewind
         string_io.read.should == "\"hello\",\"b\",\"foo\"\n"
       end
+
+      it "can extract URLs" do
+        string_io = StringIO.new
+        csv_builder = TwitterToCsv::CsvBuilder.new(:csv => string_io, :fields => %w[something], :url_columns => 2)
+        csv_builder.handle_status({
+            'something' => "hello",
+            'text' => 'this is http://a.com/url and http://a.com/nother'
+        })
+        csv_builder.handle_status({
+            'something' => "hello",
+            'text' => 'this is http://a.com/url/again'
+        })
+        string_io.rewind
+        string_io.read.should == "\"hello\",\"http://a.com/url\",\"http://a.com/nother\"\n" +
+                                 "\"hello\",\"http://a.com/url/again\",\"\"\n"
+      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitter_to_csv
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-21 00:00:00.000000000Z
+date: 2012-02-28 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &71632880 !ruby/object:Gem::Requirement
+  requirement: &70236843171320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *71632880
+  version_requirements: *70236843171320
 - !ruby/object:Gem::Dependency
   name: twitter-stream
-  requirement: &71632670 !ruby/object:Gem::Requirement
+  requirement: &70236843170900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *71632670
+  version_requirements: *70236843170900
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &71632460 !ruby/object:Gem::Requirement
+  requirement: &70236843170480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *71632460
+  version_requirements: *70236843170480
 - !ruby/object:Gem::Dependency
   name: unsupervised-language-detection
-  requirement: &71632250 !ruby/object:Gem::Requirement
+  requirement: &70236843170060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *71632250
+  version_requirements: *70236843170060
 description: ''
 email:
 - andrew@iterationlabs.com
@@ -97,8 +97,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: twitter_to_csv
-rubygems_version: 1.8.16
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
 summary: Dump the Twitter streaming API to a CSV or JSON file
-test_files: []
+test_files:
+- spec/csv_builder_spec.rb
+- spec/spec_helper.rb