twitter_to_csv 0.0.1 → 0.0.2

data/.gitignore

@@ -3,3 +3,4 @@
 Gemfile.lock
 pkg/*
 .idea
+data

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/README.markdown

@@ -3,7 +3,10 @@
 ## Usage
 
       twitter_to_csv --username <your twitter username> --password <your twitter password> \
-                     --json hi.json --filter zit,zits,pimple,pimples,acne
+                     --json hi.json --filter zit,zits,pimple,pimples,acne \
+                     --csv out.csv --fields text,
+                     --fields text,retweeted_status.id,retweeted_status.favorited,...
+                     
+Use `--sample-fields 1000`` to output the occurrence count of different Twitter fields.
 
-
-Verbosity and actually outputting to a CSV coming soon :)
+You can also `--replay-from-file` if you have a JSON output file and you want to run it back through the exporter.

data/Rakefile

@@ -1 +1,5 @@
 require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+task :default => :spec

data/bin/twitter_to_csv

@@ -4,7 +4,7 @@ require 'open-uri'
 require 'optparse'
 require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'twitter_to_csv'))
 
-options = { :csv => STDOUT }
+options = { :csv => STDOUT, :fields => %w[text user.lang] }
 parser = OptionParser.new do |opts|
   opts.banner = "Usage: #{File.basename($0)} [options]"
   opts.separator ""
@@ -18,11 +18,12 @@ parser = OptionParser.new do |opts|
     options[:password] = password
   end
 
-  opts.on("-c", "--csv [FILE]", "The CSV file to write to") do |csv|
+  opts.on("-c", "--csv [FILE]", "The CSV file to append to") do |csv|
+    options[:csv_appending] = File.exists?(csv)
     options[:csv] = File.open(csv, 'a')
   end
   
-  opts.on("-j", "--json [FILE]", "The file to log all Twitter JSON to") do |json|
+  opts.on("-j", "--json [FILE]", "The JSON file to append to") do |json|
     options[:json] = File.open(json, 'a')
   end
 
@@ -30,6 +31,10 @@ parser = OptionParser.new do |opts|
     options[:filter] = filter.split(/\s*,\s*/)
   end
 
+  opts.on("-x", "--fields [FIELDS]", "Fields to include in the CSV") do |fields|
+    options[:fields] = fields.split(/\s*,\s*/)
+  end
+
   opts.on("-e", "--require-english", "Attempt to filter out non-English tweets.", "This will have both false positives and false negatives.") do |e|
     options[:require_english] = e
   end
@@ -38,6 +43,14 @@ parser = OptionParser.new do |opts|
     options[:verbose] = v
   end
 
+  opts.on_tail("", "--sample-fields [NUMBER_OF_SAMPLES]", "Sample fields from Twitter, display them, and then exit.") do |samples|
+    options[:sample_fields] = samples && samples.to_i
+  end
+
+  opts.on_tail("", "--replay-from-file FILENAME", "Replay status from a JSON dump file") do |replay_file|
+    options[:replay_from_file] = replay_file
+  end
+
   opts.on_tail("-h", "--help", "Show this message") do
     STDERR.puts opts
     exit
@@ -51,7 +64,7 @@ end
 
 parser.parse!
 
-unless options[:username] && options[:password]
+unless (options[:username] && options[:password]) || options[:replay_from_file]
   STDERR.puts "Error: Twitter username and password are required fields.\n\n"
   STDERR.puts parser
   exit 1

data/lib/twitter_to_csv.rb

@@ -1,6 +1,6 @@
 require 'rubygems'
 require File.expand_path(File.join(File.dirname(__FILE__), "twitter_to_csv", "version"))
-require 'fastercsv'
+require 'csv'
 require 'pp'
 require 'json'
 require 'twitter/json_stream'

data/lib/twitter_to_csv/csv_builder.rb

@@ -1,9 +1,95 @@
+require 'pp'
+
 module TwitterToCsv
   class CsvBuilder
-    attr_accessor :options
+    attr_accessor :options, :sampled_fields
 
     def initialize(options = {})
       @options = options
+      @sampled_fields = {}
+      @num_samples = 0
+    end
+
+    def run
+      log_csv_header if options[:csv] && !options[:csv_appending]
+      if options[:replay_from_file]
+        replay_from options[:replay_from_file]
+      else
+        begin
+          TwitterWatcher.new(options).run do |status|
+            handle_status status
+          end
+        rescue SignalException, SystemExit
+          exit
+        rescue StandardError => e
+          STDERR.puts "\nException #{e.message}:\n#{e.backtrace.join("\n")}\n\n"
+          STDERR.puts "Waiting for a couple of minutes..."
+          sleep 120
+          retry
+        end
+      end
+    end
+
+    def handle_status(status)
+      if (options[:require_english] && is_english?(status)) || !options[:require_english]
+        log_json(status) if options[:json]
+        log_csv(status) if options[:csv]
+        sample_fields(status) if options[:sample_fields]
+        STDERR.puts "Logging: #{status['text']}" if options[:verbose]
+      end
+    end
+
+    def log_csv_header
+      options[:csv].puts options[:fields].to_csv(:encoding => 'UTF-8', :force_quotes => true)
+    end
+
+    def log_csv(status)
+      csv_row = options[:fields].map do |field|
+        field.split(".").inject(status) { |memo, segment|
+          memo && memo[segment]
+        }.to_s
+      end.to_csv(:encoding => 'UTF-8', :force_quotes => true)
+      options[:csv].puts csv_row
+    end
+
+    def replay_from(filename)
+      File.open(filename, "r") do |file|
+        until file.eof?
+          line = file.readline
+          next if line =~ /\A------SEP.RATOR------\Z/i
+          handle_status JSON.parse(line)
+        end
+      end
+    end
+
+    def sample_fields(status)
+      extract_fields(status, sampled_fields)
+      @num_samples += 1
+      if @num_samples > options[:sample_fields]
+        puts "Sampled fields from Twitter:"
+        sampled_fields.each do |field, count|
+          puts " #{field} #{' ' * [60 - field.length, 0].max} #{count}"
+        end
+        exit 1
+      end
+    end
+
+    def extract_fields(object, fields, current_path = [])
+      if object.is_a?(Hash)
+        object.each do |k, v|
+          extract_fields v, fields, current_path + [k]
+        end
+      else
+        path = current_path.join(".")
+        fields[path] ||= 0
+        fields[path] += 1
+      end
+    end
+
+    def log_json(status)
+      options[:json].puts JSON.dump(status) #JSON.pretty_generate(status)
+      options[:json].puts "------SEPARATOR------"
+      options[:json].flush
     end
 
     def is_english?(status)
@@ -29,25 +115,5 @@ module TwitterToCsv
 
       true
     end
-
-    def run
-      begin
-        TwitterWatcher.new(options).run do |status|
-          if (options[:require_english] && is_english?(status)) || !options[:require_english]
-            if options[:json]
-              options[:json].puts JSON.dump(status) #JSON.pretty_generate(status)
-              options[:json].puts "------SEPERATOR------"
-              options[:json].flush
-            end
-            STDERR.puts "Logging: #{status['text']}" if options[:verbose]
-          end
-        end
-      rescue StandardError => e
-        STDERR.puts "\nException #{e.message}:\n#{e.backtrace.join("\n")}\n\n"
-        STDERR.puts "Waiting for a couple of minutes..."
-        sleep 120
-        retry
-      end
-    end
   end
 end

data/lib/twitter_to_csv/version.rb

@@ -1,3 +1,3 @@
 module TwitterToCsv
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/csv_builder_spec.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe TwitterToCsv::CsvBuilder do
+  describe "#handle_status" do
+    describe "when :english is set" do
+      it "skips non-English tweets" do
+        string_io = StringIO.new
+        csv_builder = TwitterToCsv::CsvBuilder.new(:require_english => true, :csv => string_io, :fields => %w[text])
+        csv_builder.handle_status('text' => "This is English", 'user' =>  { 'lang' => 'en' })
+        csv_builder.handle_status('text' => "هذه الجملة باللغة الإنجليزية.", 'user' =>  { 'lang' => 'en' })
+        csv_builder.handle_status('text' => "Esta frase se encuentra en Ingles.", 'user' =>  { 'lang' => 'en' })
+        csv_builder.handle_status('text' => "This is still English", 'user' =>  { 'lang' => 'en' })
+        csv_builder.handle_status('text' => "The lang code can lie, but we trust it for now.", 'user' =>  { 'lang' => 'fr' })
+        string_io.rewind
+        string_io.read.should == "\"This is English\"\n\"This is still English\"\n"
+      end
+    end
+
+    describe "logging to a CSV" do
+      it "outputs the requested fields when requested in dot-notation" do
+        string_io = StringIO.new
+        csv_builder = TwitterToCsv::CsvBuilder.new(:csv => string_io, :fields => %w[something something_else.a something_else.c.d])
+        csv_builder.handle_status({
+            'something' => "hello",
+            'something_else' => {
+                'a' => 'b',
+                'c' => {
+                    'd' => "foo",
+                    'e' => 'bar'
+                },
+                'blah' => 'hi'
+            }
+        })
+        string_io.rewind
+        string_io.read.should == "\"hello\",\"b\",\"foo\"\n"
+      end
+    end
+  end
+
+  describe "#extract_fields" do
+    it "finds all the paths through a hash" do
+      obj = {
+          :a => :b,
+          :b => "c",
+          :d => {
+              :e => :f,
+              :g => {
+                  :h => :i,
+                  :j => {
+                      :k => "l"
+                  }
+              },
+              :m => "n"
+          }
+      }
+      fields = { "a" => 1 }
+      TwitterToCsv::CsvBuilder.new.extract_fields(obj, fields)
+      fields.should == { "a" => 2, "b" => 1, "d.e" => 1, "d.g.h" => 1, "d.g.j.k" => 1, "d.m" => 1 }
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'twitter_to_csv'
+
+RSpec.configure do |config|
+  # some (optional) config here
+end

data/twitter_to_csv.gemspec

@@ -18,8 +18,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  # s.add_development_dependency "rspec"
-  s.add_runtime_dependency 'fastercsv'
+  s.add_development_dependency "rspec"
   s.add_runtime_dependency 'twitter-stream'
   s.add_runtime_dependency 'em-http-request'
   s.add_runtime_dependency 'unsupervised-language-detection'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitter_to_csv
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-17 00:00:00.000000000Z
+date: 2012-02-21 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fastercsv
-  requirement: &83775030 !ruby/object:Gem::Requirement
+  name: rspec
+  requirement: &71632880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
-  version_requirements: *83775030
+  version_requirements: *71632880
 - !ruby/object:Gem::Dependency
   name: twitter-stream
-  requirement: &83774820 !ruby/object:Gem::Requirement
+  requirement: &71632670 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *83774820
+  version_requirements: *71632670
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &83774610 !ruby/object:Gem::Requirement
+  requirement: &71632460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *83774610
+  version_requirements: *71632460
 - !ruby/object:Gem::Dependency
   name: unsupervised-language-detection
-  requirement: &83774400 !ruby/object:Gem::Requirement
+  requirement: &71632250 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *83774400
+  version_requirements: *71632250
 description: ''
 email:
 - andrew@iterationlabs.com
@@ -64,6 +64,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
 - .rvmrc
 - Gemfile
 - README.markdown
@@ -73,6 +74,8 @@ files:
 - lib/twitter_to_csv/csv_builder.rb
 - lib/twitter_to_csv/twitter_watcher.rb
 - lib/twitter_to_csv/version.rb
+- spec/csv_builder_spec.rb
+- spec/spec_helper.rb
 - twitter_to_csv.gemspec
 homepage: ''
 licenses: []