dash-mario 0.15

data/lib/dash-fu/marios/backtweets.rb

@@ -0,0 +1,57 @@
+module DashFu::Mario
+  # Track Twitter links using the Backtype API.
+  class Backtweets
+    include DashFu::Mario
+
+    def setup(source, params)
+      url = params["url"].strip.downcase.sub(/^http(s?):\/\//, "")
+      source["metric.name"] = "Tweets for #{url}"
+      source["metric.columns"] = [{ :id=>"tweets", :label=>"Tweets" }]
+      source["metric.totals"] = true
+      source["url"] = url
+    end
+
+    def validate(source)
+      raise "Missing URL" if source["url"].blank?
+    end
+
+    def update(source, request, &block)
+      Net::HTTP.start "backtweets.com" do |http|
+        get = Net::HTTP::Get.new("/search.json?key=#{api_key}&q=#{Rack::Utils.escape source["url"]}")
+        response = http.request(get)
+        json = JSON.parse(response.body) rescue nil if Net::HTTPOK === response
+        unless json
+          logger.error "Backtweets: #{response.code} #{response.message}"
+          raise "Last request didn't go as expected, trying again later"
+        end
+       
+        if tweets = json["tweets"]
+          last_tweet_id = source["last_tweet_id"]
+          if last_tweet_id
+            new_ones = tweets.take_while { |tweet| tweet["tweet_id"] != last_tweet_id }.reverse
+            new_ones.each do |tweet|
+              handle, id = tweet["tweet_from_user"], tweet["tweet_id"]
+              url = "http://twitter.com/#{handle}/#{id}"
+              html = <<-HTML
+<a href="#{url}">tweeted</a>:
+<blockquote>#{tweet["tweet_text"]}</blockquote>
+              HTML
+              person = { :fullname=>handle, :identities=>%W{twitter.com:#{handle}}, :photo_url=>tweet["tweet_profile_image_url"] }
+              block.call :activity=>{ :uid=>id, :url=>url, :html=>html, :tags=>%w{twitter mention},
+                                      :timestamp=>Time.parse(tweet["tweet_created_at"]).utc, :person=>person }
+            end
+          end
+          if recent = tweets.first
+            source["last_tweet_id"] = recent["tweet_id"]
+          end
+        end
+
+        block.call :set=>{ :tweets=>json["totalresults"].to_i }
+      end
+    end
+
+    def meta(source)
+      [ { :text=>"Search yourself", :url=>"http://backtweets.com/search?q=#{URI.escape source["url"]}" } ]
+    end
+  end
+end

data/lib/dash-fu/marios/backtweets.yml

@@ -0,0 +1,9 @@
+en:
+  description: "Finds Tweets linking to any URL"
+  inputs: |-
+    <label>URL <input type="text" name="source[url]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>If this is a new search, it may take some time before we get the first results (including historical data). Check back in 8 hours.</li>
+      <li>Data provided by <a href="http://backtweets.com">Backtweets</a></li>
+    </ul>

data/lib/dash-fu/marios/github.rb

@@ -0,0 +1,110 @@
+module DashFu::Mario
+  # Track Github commits, watchers and forks.
+  class Github
+    include DashFu::Mario
+
+    def setup(source, params)
+      repo = params["repo"].strip
+      source["metric.name"] = "Github: #{repo}"
+      source["metric.columns"] = [{ :id=>"commits", :label=>"Commits" }, { :id=>"watchers", :label=>"Watchers" }, { :id=>"forks", :label=>"Forks" }]
+      source["metric.totals"] = true
+      source["repo"] = repo
+      branch = params["branch"].strip
+      source["branch"] = branch.blank? ? "master" : branch
+      source["username"] = params["username"]
+      source["api_token"] = params["api_token"]
+    end
+
+    def validate(source)
+      raise "Missing repository name" if source["repo"].blank?
+      raise "Need user name and repository name, e.g. assaf/vanity" unless source["repo"][/^[\w-]+\/[\w-]+$/]
+    end
+
+    def update(source, request, &block)
+      http = Net::HTTP.new("github.com", 443)
+      http.use_ssl = true
+      http.start do
+        case response = http_request(http, source, "/api/v2/json/repos/show/:repo")
+        when Net::HTTPOK
+          json = JSON.parse(response.body)["repository"] rescue nil
+        when Net::HTTPNotFound, Net::HTTPBadRequest
+          raise "Could not find the repository #{source["repo"]}"
+        when Net::HTTPUnauthorized
+          raise "You are not authorized to access this repository, or invalid username/password"
+        end
+        if json
+          source.update json.slice(*%w{description url homepage})
+          block.call :set=>{ :forks=>json["forks"], :watchers=>json["watchers"] }
+        else
+          logger.error "Github: #{response.code} #{response.message}"
+          error = true
+        end
+
+        case response = http_request(http, source, "/api/v2/json/commits/list/:repo/:branch")
+        when Net::HTTPOK
+          commits = JSON.parse(response.body)["commits"] rescue nil
+        when Net::HTTPNotFound, Net::HTTPBadRequest
+          raise "Could not find the branch #{source["branch"]}"
+        end
+
+        if commits
+          last_seen_id = source["last_commit"]["id"] if source["last_commit"]
+          if last_seen_id
+            new_ones = commits.take_while { |commit| commit["id"] != last_seen_id }
+            merged = new_ones.inject([]) do |all, commit|
+              last = all.last.last unless all.empty?
+              if last && last["committer"]["email"] == commit["committer"]["email"]
+                all.last << commit
+              else
+                all << [commit]
+              end
+              all
+            end
+
+            merged.reverse.each do |commits|
+              first = commits.first
+              committer = first["committer"]
+              person = { :fullname=>committer["name"], :identities=>%W{github.com:#{committer["login"]}}, :email=>committer["email"] }
+              messages = commits.map { |commit| %{<blockquote><a href="#{commit["url"]}">#{commit["id"][0,7]}</a> #{h commit["message"].strip.split(/[\n\r]/).first[0,50]}</blockquote>} }
+              html = %{pushed to #{h source["branch"]} at <a href="http://github.com/#{source["repo"]}">#{h source["repo"]}</a>:\n#{messages.join("\n")}}
+              block.call :activity=>{ :uid=>first["id"], :html=>html, :url=>first["url"], :tags=>%w{push},
+                                      :timestamp=>Time.parse(first["committed_date"]).utc, :person=>person }
+            end
+            block.call :inc=>{ :commits=>new_ones.count }
+          else
+            block.call :set=>{ :commits=>commits.count }
+          end
+          if last_commit = commits.first
+            source.update "last_commit"=>last_commit.slice("id", "message", "url").
+              merge("timestamp"=>Time.parse(last_commit["committed_date"]).utc)
+          end
+        else
+          logger.error "Github: #{response.code} #{response.message}"
+          error = true
+        end
+
+        raise "Last request didn't go as expected, trying again later" if error
+      end
+    end
+
+    def meta(source)
+      meta = [ { :title=>"Repository", :text=>source["repo"], :url=>source["url"] },
+               { :title=>"Branch", :text=>source["branch"] },
+               { :text=>source["description"] },
+               { :title=>"Home page", :url=>source["homepage"] } ]
+      if last_commit = source["last_commit"]
+        meta << { :title=>"Commit", :text=>last_commit["message"], :url=>last_commit["url"] }
+      end
+      meta
+    end
+
+  protected
+
+    def http_request(http, source, path)
+      get = Net::HTTP::Get.new(path.gsub(":repo", URI.escape(source["repo"])).gsub(":branch", URI.escape(source["branch"])))
+      get.basic_auth "#{source["username"]}/token", source["api_token"] unless source["username"].blank? && source["api_token"].blank?
+      http.request(get)
+    end
+
+  end
+end

data/lib/dash-fu/marios/github.yml

@@ -0,0 +1,15 @@
+en:
+  description: "Tracks Github commits, watchers and forks"
+  inputs: |-
+    <label>User/repository <input type="text" name="source[repo]" size="30"></label>
+    <label>Branch <input type="text" name="source[branch]" size="30" value="master"></label>
+    <p>For private repositories:</p>
+    <label>Username <input type="text" name="source[username]" size="30"></label>
+    <label><a href="https://github.com/account#admin_bucket" target="github">API Token</a> <input type="password" name="source[api_token]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>Enter user name/repository like this: <code>assaf/vanity</code></li>
+      <li>To access private repositories, authenticate using your username and API token.
+          You can find your API token in <a href="https://github.com/account#admin_bucket" target="github">Account Settings</a>.
+      </li>
+    </ul>

data/lib/dash-fu/marios/github_issues.rb

@@ -0,0 +1,91 @@
+module DashFu::Mario
+  # Track open/closed Github issues.
+  class GithubIssues
+    include DashFu::Mario
+
+    def setup(source, params)
+      repo = params["repo"].to_s.strip
+      source["metric.name"] = "Github Issues for #{repo}"
+      source["metric.columns"] = [{ :id=>"open", :label=>"Open issues" }, { :id=>"closed", :label=>"Closed issues" }]
+      source["metric.totals"] = true
+      source["repo"] = repo
+      source["username"] = params["username"]
+      source["api_token"] = params["api_token"]
+    end
+
+    def validate(source)
+      raise "Missing user/repository" if source["repo"].blank?
+      raise "Need user name and repository name, e.g. assaf/vanity" unless source["repo"][/^[\w-]+\/[\w-]+$/]
+    end
+
+    def update(source, request, &block)
+      http = Net::HTTP.new("github.com", 443)
+      http.use_ssl = true
+      http.start do
+        update = {}
+        case response = http_request(http, source, "open")
+        when Net::HTTPOK
+          open = JSON.parse(response.body)["issues"] rescue nil
+        when Net::HTTPNotFound, Net::HTTPBadRequest
+          raise "Could not find the repository #{source["repo"]}"
+        when Net::HTTPUnauthorized
+          raise "You are not authorized to access this repository, or invalid username/password"
+        end
+        if open
+          update[:open] = open.count
+          if open_ids = source["open-ids"]
+            open.reject { |issue| open_ids.include?(issue["number"]) }.reverse.each do |issue|
+              sha = Digest::SHA1.hexdigest([source["repo"], "open", issue["number"], issue["updated_at"]].join(":"))
+              url = "http://github.com/#{source["repo"]}/issues#issue/#{issue["number"]}"
+              html = <<-HTML
+opened <a href="#{url}">issue #{issue["number"]}</a> on #{source["repo"]}:
+<blockquote>#{h issue["title"]}</blockquote>
+              HTML
+              block.call :activity=>{ :uid=>sha, :url=>url, :html=>html, :tags=>%w{issue opened},
+                                      :timestamp=>Time.parse(issue["created_at"]).utc }
+            end
+          end
+          source["open-ids"] = open.map { |issue| issue["number"] }
+        end
+
+        case response = http_request(http, source, "closed")
+        when Net::HTTPOK
+          closed = JSON.parse(response.body)["issues"] rescue nil
+        end
+        if closed
+          update[:closed] = closed.count
+          if closed_ids = source["closed-ids"]
+            closed.reject { |issue| closed_ids.include?(issue["number"]) }.reverse.each do |issue|
+              sha = Digest::SHA1.hexdigest([source["repo"], "closed", issue["number"], issue["updated_at"]].join(":"))
+              url = "http://github.com/#{source["repo"]}/issues#issue/#{issue["number"]}"
+              html = <<-HTML
+closed <a href="#{url}">issue #{issue["number"]}</a> on #{source["repo"]}:
+<blockquote>#{h issue["title"]}</blockquote>
+              HTML
+              block.call :activity=>{ :uid=>sha, :url=>url, :html=>html, :tags=>%w{issue closed},
+                                      :timestamp=>Time.parse(issue["closed_at"]).utc }
+            end
+          end
+          source["closed-ids"] = closed.map { |issue| issue["number"] }
+        end
+
+        raise "Last request didn't go as expected, trying again later" if update.empty?
+        block.call :set=>update
+      end
+    end
+
+    def meta(source)
+      [ { :title=>"On Github", :url=>"http://github.com/#{source["repo"]}/issues" } ]
+    end
+
+  protected
+
+    def http_request(http, source, state)
+      path = "/api/v2/json/issues/list/#{URI.escape source["repo"]}/#{state}"
+      get = Net::HTTP::Get.new(path)
+      get.basic_auth "#{source["username"]}/token", source["api_token"] unless source["username"].blank? && source["api_token"].blank?
+      http.request(get)
+    end
+
+  end
+end

data/lib/dash-fu/marios/github_issues.yml

@@ -0,0 +1,14 @@
+en:
+  description: "Tracks open/closed Github issues"
+  inputs: |-
+    <label>User/repository <input type="text" name="source[repo]" size="30"></label>
+    <p>For private repositories:</p>
+    <label>Username <input type="text" name="source[username]" size="30"></label>
+    <label><a href="https://github.com/account#admin_bucket" target="github">API Token</a> <input type="password" name="source[api_token]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>Enter user name/repository like this: <code>assaf/vanity</code></li>
+      <li>To access private repositories, authenticate using your username and API token.
+          You can find your API token in <a href="https://github.com/account#admin_bucket" target="github">Account Settings</a>.
+      </li>
+    </ul>

data/lib/dash-fu/marios/ruby_gems.rb

@@ -0,0 +1,55 @@
+module DashFu::Mario
+  # Track activity (downloads and releases) of a Ruby Gem from rubygems.org.
+  class RubyGems
+    include DashFu::Mario
+
+    def setup(source, params)
+      gem_name = params["gem_name"].to_s.strip
+      source["metric.name"] = "RubyGems: #{gem_name}"
+      source["metric.columns"] = [{ :id=>"downloads", :label=>"Downloads" }]
+      source["metric.totals"] = true
+      source["gem_name"] = gem_name
+    end
+
+    def validate(source)
+      raise "Missing gem name" if source["gem_name"].blank?
+    end
+
+    def update(source, request)
+      gem_name = source["gem_name"]
+      uri = URI.parse("http://rubygems.org/api/v1/gems/#{Rack::Utils.escape gem_name}.json")
+      case response = Net::HTTP.get_response(uri)
+      when Net::HTTPOK
+        json = JSON.parse(response.body) rescue nil
+      when Net::HTTPNotFound
+        raise "Could not find the Gem #{gem_name}"
+      end
+      unless json
+        logger.error "RubyGems: #{response.code} #{response.message}"
+        raise "Last request didn't go as expected, trying again later"
+      end  
+
+      if source["version"]
+        current, latest = Gem::Version.new(source["version"]), Gem::Version.new(json["version"])
+        if latest > current
+          html = "released <a href=\"http://rubygems.org/gems/#{gem_name}/versions/#{latest}\">#{gem_name} version #{latest}</a>."
+          person = { :fullname=>"RubyGems", :identities=>"url:http://rubygems.org/", :photo_url=>"http://dash-fu.com/images/sources/ruby.png" }
+          yield :activity=>{ :uid=>"#{gem_name}-#{latest}", :url=>"http://rubygems.org/gems/#{gem_name}",
+                             :html=>html, :tags=>%w{release}, :person=>person }
+        end
+      end
+      source["version"] = json["version"]
+      source.update json.slice(*%w{homepage_uri project_uri info authors info})
+      yield :set=>{ :downloads=>json["downloads"] }
+    end
+
+    def meta(source)
+      [ { :title=>"Project", :text=>source["gem_name"], :url=>source["homepage_uri"] || source["project_uri"] },
+        { :text=>source["info"] },
+        { :title=>"Version", :text=>source["version"] },
+        { :title=>"Authors", :text=>source["authors"] },
+        { :title=>"Source", :text=>"RubyGems", :url=>source["project_uri"] } ]
+    end
+
+  end
+end

data/lib/dash-fu/marios/ruby_gems.yml

@@ -0,0 +1,6 @@
+en:
+  description: "Tracks downloads and releases of a Ruby Gem on rubygems.org"
+  inputs: |-
+    <label>Gem name <input type="text" name="source[gem_name]" size="30"></label>
+  notes: |-
+    Gem names are case sensitive.

data/test/api_keys.yml

@@ -0,0 +1 @@
+backtweets: "4554feeeeeeeeeeeeeee"

data/test/backtweets_test.rb

@@ -0,0 +1,151 @@
+require_relative "setup"
+
+test DashFu::Mario::Backtweets do
+  context "setup" do
+    setup { setup_source "url"=>"vanity.labnotes.org" }
+
+    should "use normalize URL and remove scheme" do
+      setup_source "url"=>"http://Vanity.Labnotes.Org"
+      assert_equal "Tweets for vanity.labnotes.org", metric.name
+    end
+
+    context "metric" do
+      subject { metric }
+
+      should "use URL" do
+        assert_equal "Tweets for vanity.labnotes.org", subject.name
+      end
+
+      should "measure totals" do
+        assert subject.totals
+      end
+
+      should "capture tweets" do
+        assert subject.columns.include?(:id=>"tweets", :label=>"Tweets")
+      end
+    end
+  end
+
+
+  context "validation" do
+    should "raise error if URL missing" do
+      assert_raise(RuntimeError) { setup_source "url"=>" " }
+    end
+
+    should "create valid metric" do
+      setup_source "url"=>"vanity.labnotes.org"
+      assert metric.valid?
+    end
+  end
+
+
+  context "update" do
+    setup { setup_source "url"=>"vanity.labnotes.org" }
+
+    should "include API key" do
+      update_source
+      assert_requested :get, "http://backtweets.com/search.json?key=4554feeeeeeeeeeeeeee&q=vanity.labnotes.org"
+    end
+
+    should "properly encode URL" do
+      setup_source "url"=>"need&this=encoded"
+      update_source rescue nil
+      assert_requested :get, "http://backtweets.com/search.json?key=#{mario.send :api_key}&q=need%26this%3Dencoded"
+    end
+
+    should "handle errors" do
+      stub_request(:get, interactions.first.uri).to_return :status=>500
+      assert_raise(RuntimeError) { update_source }
+      assert_equal "Last request didn't go as expected, trying again later", last_error
+    end
+
+    should "handle invlid document entity" do
+      stub_request(:get, interactions.first.uri).to_return :body=>"Not JSON"
+      assert_raise(RuntimeError) { update_source }
+      assert_equal "Last request didn't go as expected, trying again later", last_error
+    end
+
+    should "capture number of tweets" do
+      update_source
+      assert_equal({ :tweets=>11 }, totals)
+    end
+
+    should "ignore previous tweets" do
+      update_source
+      assert activities.empty?
+    end
+
+    context "repeating" do
+      setup do
+        update_source
+        second = interactions.last
+        stub_request(:get, second.uri).to_return :body=>second.response.body
+        update_source
+      end
+
+      should "capture new tweets" do
+        assert_equal 2, activities.count
+      end
+
+      context "activity" do
+        subject { activity }
+
+        should "capture tweet identifier" do
+          assert_equal "20959239300", subject.uid
+        end
+
+        should "capture tweet text" do
+          assert_equal <<-HTML, subject.html
+<a href="http://twitter.com/assaf/20959239300">tweeted</a>:
+<blockquote>Super awesome <a href="http://vanity.labnotes.org/">http://j.mp/aOrUnsz</a></blockquote>
+          HTML
+        end
+
+        should "capture tweet URL" do
+          assert_equal "http://twitter.com/assaf/20959239300", subject.url
+        end
+
+        should "capture tweet timestamp" do
+          assert_equal Time.parse("2010-8-22T05:00:04").utc, subject.timestamp
+        end
+
+        should "tag as tweeter and mention" do
+          assert_contains subject.tags, "twitter"
+          assert_contains subject.tags, "mention"
+        end
+
+        should "be valid" do
+          assert subject.valid?
+        end
+
+        context "author" do
+          subject { activity.person }
+
+          should "capture full name" do
+            assert_equal "assaf", subject.fullname
+          end
+
+          should "capture screen name" do
+            assert_contains subject.identities, "twitter.com:assaf"
+          end
+
+          should "capture photo" do
+            assert_equal "http://twitter.com/account/profile_image/assaf", subject.photo_url
+          end
+        end
+      end
+
+    end
+
+  end
+
+
+  context "meta" do
+    setup { setup_source "url"=>"vanity.labnotes.org" }
+    subject { meta }
+
+    should "link to search results" do
+      assert subject.include?(:text=>"Search yourself", :url=>"http://backtweets.com/search?q=vanity.labnotes.org")
+    end
+  end
+end