cablegator 0.9.2 → 0.9.9

data/.gitignore

@@ -1,3 +1,4 @@
 pkg/*
 *.gem
 .bundle
+.rvmrc

data/Gemfile

@@ -1,3 +1,6 @@
 source 'http://gemcutter.org'
 gem 'nokogiri'
 gem 'httparty'
+gem 'twitter'
+gem 'oauth'
+gem 'launchy'

data/Gemfile.lock

@@ -1,14 +1,43 @@
 GEM
   remote: http://gemcutter.org/
   specs:
+    addressable (2.2.2)
+    configuration (1.2.0)
     crack (0.1.8)
+    faraday (0.5.3)
+      addressable (~> 2.2.2)
+      multipart-post (~> 1.0.1)
+      rack (>= 1.1.0, < 2)
+    faraday_middleware (0.3.1)
+      faraday (~> 0.5.3)
+    hashie (0.4.0)
     httparty (0.6.1)
       crack (= 0.1.8)
+    launchy (0.3.7)
+      configuration (>= 0.0.5)
+      rake (>= 0.8.1)
+    multi_json (0.0.5)
+    multi_xml (0.2.0)
+    multipart-post (1.0.1)
     nokogiri (1.4.4)
+    oauth (0.4.4)
+    rack (1.2.1)
+    rake (0.8.7)
+    simple_oauth (0.1.3)
+    twitter (1.0.0)
+      faraday (~> 0.5.3)
+      faraday_middleware (~> 0.3.0)
+      hashie (~> 0.4.0)
+      multi_json (~> 0.0.5)
+      multi_xml (~> 0.2.0)
+      simple_oauth (~> 0.1.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   httparty
+  launchy
   nokogiri
+  oauth
+  twitter

data/README.md

@@ -1,20 +1,28 @@
 # cablegator
 Downloads WikiLeaks CableGate cables for offline viewing and archiving 
+Tweets link to Wikileaks CableGate cables- making the information within easy reach of the twitterverse.
 
 Install via:
 <code>
 > gem install cablegator
 </code>
 
-
 Run Via:
 <code>
-$ download_cables ..
-Downloading cables to /Users/csquared/projects/cablegator
-You already have 66BUENOSAIRES2481 in /Users/csquared/projects/cablegator/cable/1966/12/66BUENOSAIRES2481.html
-You already have 72TEHRAN1164 in /Users/csquared/projects/cablegator/cable/1972/02/72TEHRAN1164.html
-You already have 72TEHRAN5055 in /Users/csquared/projects/cablegator/cable/1972/08/72TEHRAN5055.htm
-Downloading cable 75TEHRAN2069 to /Users/csquared/projects/cablegator//cable/1975/03/75TEHRAN2069.html
-...
+>     download_cables <path>
+</code>
 
+And
+<code>
+>     tweet_cables <delay in seconds>
 </code>
+You will then be prompted to login to Twitter from the cmd line.
+From here the program tweets links to all the cables, waiting between posts.  Default delay is 1 second.
+
+#TODO:
+* DONE - Don't tweet if already tweeted
+* Allow tweet customization
+* Create webapp to run this
+
+#Features:
+* Disqualifies you from employment at various federal agencies.

data/TODO

@@ -0,0 +1,5 @@
+
+* tweet the rest of the cables 
+  -- search for if tag has been already posted
+  -- otherwise figure out some way to resume posting
+* tweet the subjects with references to the hashtag

data/bin/download_cables

@@ -1,25 +1,19 @@
 #! /usr/bin/env ruby
 require 'cablegator'
-require 'cablegator/wikileaks'
+require 'fileutils'
 
 dir_prefix = ARGV[0] || Dir.pwd
 
 puts "Downloading cables to #{File.expand_path(dir_prefix)}"
-doc = Nokogiri::HTML(WikiLeaks.home)
+Wikileaks.with_each_cable do |cable_url|
+  file_location = File.expand_path(dir_prefix + cable_url)
+  reference_id = Wikileaks.reference_id(cable_url) 
 
-doc.css(%{a[href^='/date']}).each do |link|
-  page_with_cables = Nokogiri::HTML(WikiLeaks.get(link.attributes['href'].value))
-  page_with_cables.css(%{a[href^='/cable']}).each do |cable|
-    cable_url = cable.attributes['href'].value
-    file_location = File.expand_path(dir_prefix + cable_url)
-    reference_id = File.basename(cable_url).gsub(File.extname(cable_url),'') 
-
-    if !File.exist?(file_location)
-      FileUtils.mkdir_p(File.dirname(file_location))
-      STDOUT.puts "Downloading cable #{reference_id} to #{file_location}"
-      File.open(file_location, 'w') { |f| f << WikiLeaks.get(cable_url) } 
-    else
-      STDOUT.puts "You already have #{reference_id} in #{file_location}"
-    end
+  if !File.exist?(file_location)
+    FileUtils.mkdir_p(File.dirname(file_location))
+    STDOUT.puts "Downloading cable #{reference_id} to #{file_location}"
+    File.open(file_location, 'w') { |f| f << WikiLeaks.get(cable_url) } 
+  else
+    STDOUT.puts "You already have #{reference_id} in #{file_location}"
   end
 end

data/bin/tweet_cables

@@ -0,0 +1,26 @@
+#! /usr/bin/env ruby
+require 'twitter'
+require 'cablegator'
+require 'cablegator/twitter'
+
+Twitter.command_line_login
+tweeted = CableGator.save_file("~/.cablegator/tweeted")
+
+puts "Tweeting cables" 
+WikiLeaks.with_each_cable do |cable_url|
+  reference_id = WikiLeaks.reference_id(cable_url) 
+  if tweeted.include?(reference_id)
+    puts "Already tweeted #{reference_id}"
+  else
+    tweet = "#cablegate ##{reference_id} #{WikiLeaks.base_uri + cable_url}"
+    puts "Tweeting: #{tweet}"
+    begin 
+      Twitter.update(tweet) 
+      tweeted << reference_id
+      delay = ARGV[0].to_i rescue nil
+      sleep(delay || 1)
+    rescue 
+      "Tweet #{reference_id} failed"
+    end
+  end
+end

data/bin/tweet_cables_format

@@ -0,0 +1,31 @@
+#! /usr/bin/env ruby
+require 'twitter'
+require 'cablegator'
+require 'cablegator/twitter'
+
+Twitter.command_line_login
+tweeted = CableGator.save_file("~/.cablegator/tweeted_fmt_#{ARGV[0].to_s.gsub(/:| /,'')}")
+
+puts "Tweeting cables" 
+WikiLeaks.with_each_cable_data do |cable_data|
+  reference_id = cable_data['reference_id']
+
+  if tweeted.include?(reference_id)
+    puts "Already tweeted #{reference_id}"
+  else
+    format_string = ARGV[0].to_s.dup
+    cable_data.each { |k,v| format_string.gsub!(/\:#{k}/,v) }
+
+    tweet = "#cablegate ##{reference_id} #{format_string}"
+    
+    puts "Tweeting: #{tweet}"
+    begin 
+      Twitter.update(tweet) 
+      tweeted << reference_id
+      delay = ARGV[1].to_i rescue nil
+      sleep(delay || 10)
+    rescue 
+      "Tweet #{reference_id} failed"
+    end
+  end
+end

data/cablegator.gemspec

@@ -16,6 +16,9 @@ Gem::Specification.new do |s|
 
   s.add_dependency('nokogiri')
   s.add_dependency('httparty')
+  s.add_dependency('twitter')
+  s.add_dependency('oauth')
+  s.add_dependency('launchy')
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/cablegator.rb

@@ -1,12 +1,22 @@
 # library loader
-require 'rubygems'
-#require 'bundler'
-#Bundler.setup
+# require 'rubygems'
+# require 'bundler'
+# Bundler.setup
 
 $LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
 
-# gems
-require 'open-uri'
-require 'nokogiri'
-require 'httparty'
-require 'fileutils'
+require 'cablegator/wikileaks'
+
+module CableGator
+  def self.save_file(pathname)
+    require 'fileutils'
+    save_file = File.expand_path(pathname)
+    FileUtils.mkdir_p(File.dirname(save_file))
+
+    tweeted = File.read(save_file).split(',') rescue []
+    at_exit do
+      File.open(save_file, 'w') { |f| f << tweeted.join(',') }
+    end
+    tweeted
+  end
+end

data/lib/cablegator/twitter.rb

@@ -0,0 +1,35 @@
+require 'oauth'
+require 'launchy'
+
+module Twitter
+  def self.command_line_login
+    consumer_key = 'ddxhhHSHys7210VR8lhYag'
+    consumer_secret = 'FN5kIDAvwWU4jb54JMfWWTbtpI30JKeEmRWMYSMYk'
+
+    oauth_client = OAuth::Consumer.new(consumer_key, consumer_secret, 
+                        :site => 'http://api.twitter.com', 
+                        :request_endpoint => 'http://api.twitter.com', 
+                        :sign_in => true)
+
+    req_token = oauth_client.get_request_token
+    Launchy.open(req_token.authorize_url)
+
+    puts "\nAllow access to get pin\n"
+
+    puts "Enter Pin: "
+    pin = STDIN.gets.strip 
+
+    begin 
+      access_token = req_token.get_access_token(:oauth_verifier => pin)
+
+      Twitter.configure do |config|
+        config.consumer_key = access_token.consumer.key
+        config.consumer_secret = access_token.consumer.secret
+        config.oauth_token = access_token.token 
+        config.oauth_token_secret = access_token.secret
+      end
+    rescue
+      STDOUT.puts "Twitter Authentication failed - please try again"
+    end
+  end
+end

data/lib/cablegator/version.rb

@@ -1,3 +1,3 @@
 module Cablegator
-  VERSION = "0.9.2"
+  VERSION = "0.9.9"
 end

data/lib/cablegator/wikileaks.rb

@@ -1,12 +1,66 @@
+require 'socket'
+require 'timeout'
+require 'nokogiri'
+require 'httparty'
+
 class WikiLeaks 
   include HTTParty
+  PROXY_URL = 'localhost'
+  PROXY_PORT = 8118
   base_uri 'http://wikileaks.ch'
 
+  #let's check for tor on privoxy
+  Timeout::timeout(1) do
+    TCPSocket.new(PROXY_URL, PROXY_PORT).close
+    http_proxy 'localhost', 8118
+    puts "Using TOR!"
+  end rescue nil
+    
   HOME = '/cablegate.html'
 
   class << self
-    def home
-      self.get(HOME)
+    def reference_id(cable_url)
+      File.basename(cable_url).gsub(File.extname(cable_url),'') 
+    end
+
+    def with_each_cable
+      doc = Nokogiri::HTML(self.get(HOME))
+      doc.css(%{a[href^='/date']}).each do |link|
+        cable_url = link.attributes['href'].value
+        begin
+          page_with_cables = Nokogiri::HTML(WikiLeaks.get(cable_url))
+          page_with_cables.css(%{a[href^='/cable']}).each do |cable|
+            cable_url = cable.attributes['href'].value
+            yield cable_url
+          end
+        rescue
+          STDOUT.puts "error GETtin page #{cable_url}"
+        end
+      end
+    end
+
+    def with_each_cable_data
+      doc = Nokogiri::HTML(self.get(HOME))
+      doc.css(%{a[href^='/date']}).each do |link|
+        cable_url = link.attributes['href'].value
+        begin
+          page_with_cables = Nokogiri::HTML(WikiLeaks.get(cable_url))
+          page_with_cables.css(%{table.cable tr:has(td)}).each do |cable|
+            cable_hash = {}	
+            #cable_hash['cable_url'] = cable.css(%{a[href='/cable']}).first.attributes['href'].value
+            children = cable.element_children.map{ |x| x.text.strip }
+            cable_hash['reference_id'] = children[0]	  
+            cable_hash['subject'] = children[1]	  
+            cable_hash['origin_date'] =  children[2]	  
+            cable_hash['release_date'] = children[3]	  
+            cable_hash['classification'] = children[4]
+            cable_hash['location'] = children[5]
+            yield cable_hash
+          end
+        rescue
+          STDOUT.puts "error GETtin page #{cable_url}"
+        end
+      end
     end
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 9
-  - 2
-  version: 0.9.2
+  - 9
+  version: 0.9.9
 platform: ruby
 authors: 
 - csquared
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-11 00:00:00 -06:00
+date: 2010-12-13 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -43,11 +43,52 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: twitter
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: oauth
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: launchy
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
 description: Downloads Wikileaks Cables to current directory
 email: 
 - christopher.continanza@gmail.com
 executables: 
 - download_cables
+- tweet_cables
+- tweet_cables_format
 extensions: []
 
 extra_rdoc_files: []
@@ -58,9 +99,13 @@ files:
 - Gemfile.lock
 - README.md
 - Rakefile
+- TODO
 - bin/download_cables
+- bin/tweet_cables
+- bin/tweet_cables_format
 - cablegator.gemspec
 - lib/cablegator.rb
+- lib/cablegator/twitter.rb
 - lib/cablegator/version.rb
 - lib/cablegator/wikileaks.rb
 has_rdoc: true