vanity-source 0.6 → 0.7

data/CHANGELOG

@@ -1,3 +1,19 @@
+2010-08-24 v0.7 Added test suite and resources
+
+Each source can have an associated resources file (YAML), e.g. ruby_gems.rb
+would have ruby_gems.yml. The default implementation of methods like name,
+description and display pull content from the resource. Resources can support
+multiple languages, but we're starting with just EN.
+
+Some sources also need API keys. These are accessed using the method api_key
+that returns the API key value for the current source (often a string, but can
+also be hash or array). There's an api_keys.yml file which contains fake API
+keys. You can put a real key in there while generating a cassette for your test
+case.
+
+Test suite is here, using WebMock and VCR to rest API calls. Cassettes go in
+test/cassettes, fixtures in test/fixtures.
+
 2010-08-19 v0.6 Added Backtweet
 
 2010-08-19 v0.5 Added Github and Github issues

data/Gemfile

@@ -7,6 +7,8 @@ group :development do
 end
 
 group :test do
-  gem "rack", "1.1.0"
+  gem "shoulda"
+  gem "timecop"
+  gem "vcr"
   gem "webmock"
 end

data/Rakefile

@@ -1,4 +1,5 @@
 require "rake/testtask"
+require "shoulda/tasks"
 
 # -- Building stuff --
 
@@ -25,4 +26,10 @@ task :push=>["test", "build"] do
   sh "gem push #{spec.name}-#{spec.version}.gem"
 end
 
-task :test
+
+task :default=>:test
+Rake::TestTask.new do |task|
+  task.test_files = FileList['test/**/*_test.rb']
+  #task.warning = true
+  task.verbose = true
+end

data/lib/vanity/source.rb

@@ -1,5 +1,8 @@
-require "uri"
+require "active_support"
+require "json"
 require "net/http"
+require "rack"
+require "uri"
 
 module Vanity
 
@@ -108,28 +111,49 @@ module Vanity
           logger.info "Loaded source #{id}: #{klass}"
         end
       end
+
+      # API keys (see instance method api_key).
+      attr_accessor :api_keys
+
+      def included(klass)
+        klass.extend ClassMethods
+      end
     end
 
+
+    module ClassMethods
+      # Source identifier.
+      def source_id
+        @source_id ||= name.demodulize.underscore
+      end
+    end
+
+
     # Returns the display name for this source.
     #
-    # Default implementation uses the class name, e.g Source::OneUp becomes "One
-    # Up".
+    # Uses the resource 'name', and fallbacks on the class name (e.g
+    # Source::OneUp becomes "One Up").
     def name
-      self.class.name.demodulize.titleize
+      resources["name"] || source_id.titleize
     end
 
     # Returns additional information about this source.
     #
     # A good description helps the user find this source and decide whether or
     # not to use it.
+    #
+    # Uses the resource 'description'.
     def description
+      resources["description"]
     end
 
     # This method returns a hash with two values:
     # * inputs -- HTML fragment for a setup form
     # * notes -- HTML fragment for setup notes
+    #
+    # Uses the resources 'inputs' and 'notes'.
     def display
-      {}
+      { :inputs=>resources["inputs"], :notes=>resources["notes"] }
     end
 
     # Called to setup a new collector with parameters from the HTML form (see
@@ -168,9 +192,31 @@ module Vanity
       []
     end
 
+  protected
+
+    # Source identifier.
+    def source_id
+      self.class.source_id
+    end
+
     # Logger. Dump messages that can help with troubleshooting.
     def logger
-      self.class.logger
+      Vanity::Source.logger
+    end
+
+    # Returns I18n resources for this gem.
+    def resources
+      unless @resources
+        file_name = File.dirname(__FILE__) + "/sources/#{source_id}.yml"
+        @resources = File.exist?(file_name) ? YAML.load_file(file_name)["en"] : {}
+      end
+      @resources
+    end
+
+    # Returns API key for this source. May return string or hash, depending on
+    # the API.
+    def api_key
+      @api_key ||= Vanity::Source.api_keys[source_id] or raise "No API key for #{source_id}"
     end
   end
 end

data/lib/vanity/sources/backtweets.rb

@@ -4,20 +4,8 @@ module Vanity
     class Backtweets
       include Vanity::Source
 
-      INPUTS = <<-HTML
-        <label>URL <input type="text" name="source[url]" size="30"></label>
-      HTML
-
-      def description
-        "Finds Tweets for any URL"
-      end
-
-      def display
-        { :inputs=>INPUTS }
-      end
-
       def setup(context, params)
-        url = params["url"].strip
+        url = params["url"].strip.downcase.sub(/^http(s?):\/\//, "")
         context["metric.name"] = "Tweets for #{url}"
         context["metric.columns"] = [{ :id=>"tweets", :label=>"Tweets" }]
         context["metric.totals"] = true
@@ -30,14 +18,31 @@ module Vanity
 
       def update(context, webhook, &block)
         Net::HTTP.start "backtweets.com" do |http|
-          get = Net::HTTP::Get.new("/search.json?key=#{::API["backtweet"]}&q=#{URI.escape context["url"]}")
-          case response = http.request(get)
-          when Net::HTTPOK
-            json = JSON.parse(response.body)
-            block.call :set=>{ :tweets=>json["totalresults"] }
-          else
-            raise "#{response.code}: #{response.message}"
+          get = Net::HTTP::Get.new("/search.json?key=#{api_key}&q=#{Rack::Utils.escape context["url"]}")
+          response = http.request(get)
+          json = JSON.parse(response.body) rescue nil if Net::HTTPOK === response
+          unless json
+            logger.error "Backtweets: #{response.code} #{response.message}"
+            raise "Last request didn't go as expected, trying again later"
+          end
+         
+          if tweets = json["tweets"]
+            last_tweet_id = context["last_tweet_id"]
+            if last_tweet_id
+              new_ones = tweets.take_while { |tweet| tweet["tweet_id"] != last_tweet_id }.reverse
+              new_ones.each do |tweet|
+                handle, id = tweet["tweet_from_user"], tweet["tweet_id"]
+                block.call :activity=>{ :id=>id, :body=>tweet["tweet_text"], :url=>"http://twitter.com/#{handle}/#{id}",
+                                        :timestamp=>Time.parse(tweet["tweet_created_at"]).utc,
+                                        :person=>{ :name=>handle, :url=>"http://twitter.com/#{handle}", :photo=>tweet["tweet_profile_image_url"] } }
+              end
+            end
+            if recent = tweets.first
+              context["last_tweet_id"] = recent["tweet_id"]
+            end
           end
+
+          block.call :set=>{ :tweets=>json["totalresults"].to_i }
         end
       end
 

data/lib/vanity/sources/backtweets.yml

@@ -0,0 +1,9 @@
+en:
+  description: "Finds Tweets linking to any URL"
+  inputs: |-
+    <label>URL <input type="text" name="source[url]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>If this is a new search, it may take some time before we get the first results (including historical data). Check back in 8 hours.</li>
+      <li>Data provided by <a href="http://backtweets.com">Backtweets</a></li>
+    </ul>

data/lib/vanity/sources/github.rb

@@ -4,45 +4,21 @@ module Vanity
     class Github
       include Vanity::Source
 
-      INPUTS = <<-HTML
-        <label>User/repository <input type="text" name="source[repo]" size="30"></label>
-        <label>Branch <input type="text" name="source[branch]" size="30" value="master"></label>
-        <p>For private repositories:</p>
-        <label>Username <input type="text" name="source[username]" size="30"></label>
-        <label><a href="https://github.com/account#admin_bucket" target="github">API Token</a> <input type="password" name="source[api_token]" size="30"></label>
-      HTML
-
-      NOTES = <<-HTML
-        <ul>
-          <li>Enter user/repository like this: <code>assaf/vanity</code></li>
-          <li>To access private repositories, authenticate using your username and API token.
-              You can find your API token in <a href="https://github.com/account#admin_bucket" target="github">Account Settings</a>.
-          </li>
-        </ul>
-      HTML
-
-      def description
-        "Tracks Github commits, watchers and forks"
-      end
-
-      def display
-        { :inputs=>INPUTS, :notes=>NOTES }
-      end
-
       def setup(context, params)
         repo = params["repo"].strip
         context["metric.name"] = "Github: #{repo}"
-        context["metric.columns"] = [{ :id=>"commits", :label=>"Commits" }, { :id=>"watchers", :label=>"Watchers" }, { :id=>"forks", :label=>"forks" }]
+        context["metric.columns"] = [{ :id=>"commits", :label=>"Commits" }, { :id=>"watchers", :label=>"Watchers" }, { :id=>"forks", :label=>"Forks" }]
         context["metric.totals"] = true
         context["repo"] = repo
-        context["branch"] = params["branch"].strip
+        branch = params["branch"].strip
+        context["branch"] = branch.blank? ? "master" : branch
         context["username"] = params["username"]
         context["api_token"] = params["api_token"]
       end
 
       def validate(context)
-        raise "Missing user/repository" if context["repo"].blank?
-        raise "Missing branch name" if context["branch"].blank?
+        raise "Missing repository name" if context["repo"].blank?
+        raise "Need user name and repository name, e.g. assaf/vanity" unless context["repo"][/^[\w-]+\/[\w-]+$/]
       end
 
       def update(context, webhook, &block)
@@ -51,37 +27,52 @@ module Vanity
         http.start do
           case response = request(http, context, "/api/v2/json/repos/show/:repo")
           when Net::HTTPOK
-            json = JSON.parse(response.body)["repository"]
-            forks = json["forks"]
-            watchers = json["watchers"]
-            block.call :set=>{ :forks=>forks, :watchers=>watchers }
-            context.update json.slice(*%w{description url homepage})
+            json = JSON.parse(response.body)["repository"] rescue nil
           when Net::HTTPNotFound, Net::HTTPBadRequest
-            raise "Could not find the repository \"#{context["repo"]}\""
+            raise "Could not find the repository #{context["repo"]}"
           when Net::HTTPUnauthorized
             raise "You are not authorized to access this repository, or invalid username/password"
+          end
+          if json
+            context.update json.slice(*%w{description url homepage})
+            block.call :set=>{ :forks=>json["forks"], :watchers=>json["watchers"] }
           else
-            raise "#{response.code}: #{response.message}"
+            logger.error "Backtweets: #{response.code} #{response.message}"
+            error = true
           end
 
           case response = request(http, context, "/api/v2/json/commits/list/:repo/:branch")
           when Net::HTTPOK
-            commits = JSON.parse(response.body)["commits"]
-            if context["last_commit_at"]
-              since = commits.count { |commit| commit["committed_date"] > context["last_commit_at"] }
-              block.call :inc=>{ :commits=>since }
+            commits = JSON.parse(response.body)["commits"] rescue nil
+          when Net::HTTPNotFound, Net::HTTPBadRequest
+            raise "Could not find the branch #{context["branch"]}"
+          end
+
+          if commits
+            last_seen_id = context["last_commit"]["id"] if context["last_commit"]
+            if last_seen_id
+              new_ones = commits.take_while { |commit| commit["id"] != last_seen_id }
+              new_ones.reverse.each do |commit|
+                committer = commit["committer"]
+                block.call :activity=>{ :id=>commit["id"], :body=>commit["message"], :url=>commit["url"],
+                                        :timestamp=>Time.parse(commit["committed_date"]).utc,
+                                        :person=>{ :name=>committer["name"], :url=>"http://github.com/#{committer["name"]}",
+                                                   :email=>committer["email"] } }
+              end
+              block.call :inc=>{ :commits=>new_ones.count }
             else
               block.call :set=>{ :commits=>commits.count }
             end
             if last_commit = commits.first
-              context["last_commit"] = last_commit["message"]
-              context["last_commit_url"] = last_commit["url"]
-              context["last_commit_at"] = last_commit["committed_date"]
+              context.update "last_commit"=>last_commit.slice("id", "message", "url").
+                merge("timestamp"=>Time.parse(last_commit["committed_date"]).utc)
             end
-          when Net::HTTPNotFound, Net::HTTPBadRequest
-            raise "Could not find the branch \"#{context["branch"]}\""
+          else
+            logger.error "Backtweets: #{response.code} #{response.message}"
+            error = true
           end
 
+          raise "Last request didn't go as expected, trying again later" if error
         end
       end
 
@@ -92,10 +83,14 @@ module Vanity
       end
 
       def meta(context)
-        [ { :title=>"Repository", :text=>context["repo"], :url=>context["url"] },
-          { :text=>context["description"] },
-          { :title=>"Home page", :url=>context["homepage"] },
-          { :title=>"Commit", :text=>context["last_commit"], :url=>context["last_commit_url"] }]
+        meta = [ { :title=>"Repository", :text=>context["repo"], :url=>context["url"] },
+                 { :title=>"Branch", :text=>context["branch"] },
+                 { :text=>context["description"] },
+                 { :title=>"Home page", :url=>context["homepage"] } ]
+        if last_commit = context["last_commit"]
+          meta << { :title=>"Commit", :text=>last_commit["message"], :url=>last_commit["url"] }
+        end
+        meta
       end
     end
   end

data/lib/vanity/sources/github.yml

@@ -0,0 +1,15 @@
+en:
+  description: "Tracks Github commits, watchers and forks"
+  inputs: |-
+    <label>User/repository <input type="text" name="source[repo]" size="30"></label>
+    <label>Branch <input type="text" name="source[branch]" size="30" value="master"></label>
+    <p>For private repositories:</p>
+    <label>Username <input type="text" name="source[username]" size="30"></label>
+    <label><a href="https://github.com/account#admin_bucket" target="github">API Token</a> <input type="password" name="source[api_token]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>Enter user name/repository like this: <code>assaf/vanity</code></li>
+      <li>To access private repositories, authenticate using your username and API token.
+          You can find your API token in <a href="https://github.com/account#admin_bucket" target="github">Account Settings</a>.
+      </li>
+    </ul>

data/lib/vanity/sources/github_issues.rb

@@ -4,34 +4,10 @@ module Vanity
     class GithubIssues
       include Vanity::Source
 
-      INPUTS = <<-HTML
-        <label>User/repository <input type="text" name="source[repo]" size="30"></label>
-        <p>For private repositories:</p>
-        <label>Username <input type="text" name="source[username]" size="30"></label>
-        <label><a href="https://github.com/account#admin_bucket" target="github">API Token</a> <input type="password" name="source[api_token]" size="30"></label>
-      HTML
-
-      NOTES = <<-HTML
-        <ul>
-          <li>Enter user/repository like this: <code>assaf/vanity</code></li>
-          <li>To access private repositories, authenticate using your username and API token.
-              You can find your API token in <a href="https://github.com/account#admin_bucket" target="github">Account Settings</a>.
-          </li>
-        </ul>
-      HTML
-
-      def description
-        "Tracks open/closed Github issues"
-      end
-
-      def display
-        { :inputs=>INPUTS, :notes=>NOTES }
-      end
-
       def setup(context, params)
         repo = params["repo"].to_s.strip
         context["metric.name"] = "Github Issues for #{repo}"
-        context["metric.columns"] = [{ :id=>"open", :label=>"Opened" }, { :id=>"closed", :label=>"Closed" }]
+        context["metric.columns"] = [{ :id=>"open", :label=>"Open issues" }, { :id=>"closed", :label=>"Closed issues" }]
         context["metric.totals"] = true
         context["repo"] = repo
         context["username"] = params["username"]
@@ -40,27 +16,54 @@ module Vanity
 
       def validate(context)
         raise "Missing user/repository" if context["repo"].blank?
+        raise "Need user name and repository name, e.g. assaf/vanity" unless context["repo"][/^[\w-]+\/[\w-]+$/]
       end
 
       def update(context, webhook, &block)
         http = Net::HTTP.new("github.com", 443)
         http.use_ssl = true
         http.start do
+          update = {}
           case response = request(http, context, "open")
           when Net::HTTPOK
-            open = JSON.parse(response.body)["issues"].length
+            open = JSON.parse(response.body)["issues"] rescue nil
           when Net::HTTPNotFound, Net::HTTPBadRequest
-            raise "Could not find the repository \"#{context["repo"]}\""
+            raise "Could not find the repository #{context["repo"]}"
           when Net::HTTPUnauthorized
             raise "You are not authorized to access this repository, or invalid username/password"
           end
+          if open
+            update[:open] = open.count
+            if open_ids = context["open-ids"]
+              open.reject { |issue| open_ids.include?(issue["number"]) }.reverse.each do |issue|
+                sha = Digest::SHA1.hexdigest([context["repo"], "open", issue["number"], issue["updated_at"]].join(":"))
+                block.call :activity=>{ :id=>sha, :body=>"New issue opened: #{issue["title"]}",
+                  :url=>"http://github.com/#{context["repo"]}/issues#issue/#{issue["number"]}",
+                  :timestamp=>Time.parse(issue["created_at"]).utc }
+              end
+            end
+            context["open-ids"] = open.map { |issue| issue["number"] }
+          end
 
           case response = request(http, context, "closed")
           when Net::HTTPOK
-            closed = JSON.parse(response.body)["issues"].length
+            closed = JSON.parse(response.body)["issues"] rescue nil
+          end
+          if closed
+            update[:closed] = closed.count
+            if closed_ids = context["closed-ids"]
+              closed.reject { |issue| closed_ids.include?(issue["number"]) }.reverse.each do |issue|
+                sha = Digest::SHA1.hexdigest([context["repo"], "closed", issue["number"], issue["updated_at"]].join(":"))
+                block.call :activity=>{ :id=>sha, :body=>"Issue closed: #{issue["title"]}",
+                  :url=>"http://github.com/#{context["repo"]}/issues#issue/#{issue["number"]}",
+                  :timestamp=>Time.parse(issue["closed_at"]).utc }
+              end
+            end
+            context["closed-ids"] = closed.map { |issue| issue["number"] }
           end
 
-          block.call :set=>{ :open=>open, :closed=>closed } if open || closed
+          raise "Last request didn't go as expected, trying again later" if update.empty?
+          block.call :set=>update
         end
       end
 

data/lib/vanity/sources/github_issues.yml

@@ -0,0 +1,14 @@
+en:
+  description: "Tracks open/closed Github issues"
+  inputs: |-
+    <label>User/repository <input type="text" name="source[repo]" size="30"></label>
+    <p>For private repositories:</p>
+    <label>Username <input type="text" name="source[username]" size="30"></label>
+    <label><a href="https://github.com/account#admin_bucket" target="github">API Token</a> <input type="password" name="source[api_token]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>Enter user name/repository like this: <code>assaf/vanity</code></li>
+      <li>To access private repositories, authenticate using your username and API token.
+          You can find your API token in <a href="https://github.com/account#admin_bucket" target="github">Account Settings</a>.
+      </li>
+    </ul>

data/lib/vanity/sources/ruby_gems.rb

@@ -4,22 +4,6 @@ module Vanity
     class RubyGems
       include Vanity::Source
 
-      INPUTS = <<-HTML
-        <label>Gem name <input type="text" name="source[gem_name]" size="30"></label>
-      HTML
-
-      NOTES = <<-HTML
-        Gem names are case sensitive
-      HTML
-
-      def description
-        "Tracks activity (downloads and releases) of a Ruby Gem from rubygems.org"
-      end
-
-      def display
-        { :inputs=>INPUTS, :notes=>NOTES }
-      end
-
       def setup(context, params)
         gem_name = params["gem_name"].to_s.strip
         context["metric.name"] = "RubyGems: #{gem_name}"
@@ -33,16 +17,29 @@ module Vanity
       end
 
       def update(context, webhook)
-        uri = URI.parse("http://rubygems.org/api/v1/gems/#{URI.escape context["gem_name"]}.json")
+        gem_name = context["gem_name"]
+        uri = URI.parse("http://rubygems.org/api/v1/gems/#{Rack::Utils.escape gem_name}.json")
         case response = Net::HTTP.get_response(uri)
         when Net::HTTPOK
-          json = JSON.parse(response.body)
-          context["version"] = json["version"]
-          context.update json.slice(*%w{homepage_uri project_uri info authors info})
-          yield :set=>{ :downloads=>json["downloads"] }
+          json = JSON.parse(response.body) rescue nil
         when Net::HTTPNotFound
-          raise "Could not find the Gem \"#{context["gem_name"]}\""
+          raise "Could not find the Gem #{gem_name}"
+        end
+        unless json
+          logger.error "RubyGems: #{response.code} #{response.message}"
+          raise "Last request didn't go as expected, trying again later"
+        end  
+
+        if context["version"]
+          current, latest = Gem::Version.new(context["version"]), Gem::Version.new(json["version"])
+          if latest > current
+            yield :activity=>{ :id=>"#{gem_name}-#{latest}", :url=>"http://rubygems.org/gems/#{gem_name}",
+                               :body=>"Released <a href=\"http://rubygems.org/gems/#{gem_name}/versions/#{latest}\">#{gem_name} version #{latest}</a>" }
+          end
         end
+        context["version"] = json["version"]
+        context.update json.slice(*%w{homepage_uri project_uri info authors info})
+        yield :set=>{ :downloads=>json["downloads"] }
       end
 
       def meta(context)

data/lib/vanity/sources/ruby_gems.yml

@@ -0,0 +1,6 @@
+en:
+  description: "Tracks downloads and releases of a Ruby Gem on rubygems.org"
+  inputs: |-
+    <label>Gem name <input type="text" name="source[gem_name]" size="30"></label>
+  notes: |-
+    Gem names are case sensitive.

data/test/api_keys.yml

@@ -0,0 +1 @@
+backtweets: "4554feeeeeeeeeeeeeee"