dash-bees 0.18

data/lib/dash-fu/bees/backtweets.rb

@@ -0,0 +1,49 @@
+module DashFu::Bee
+  # Track Twitter links using the Backtype API.
+  class Backtweets
+    include DashFu::Bee
+
+    def setup(source, params)
+      url = params["url"].strip.downcase.sub(/^http(s?):\/\//, "")
+      source["source.name"] = "Tweets for #{url}"
+      source["metric.columns"] = [{ id: "tweets", label: "Tweets" }]
+      source["metric.totals"] = true
+      source["url"] = url
+    end
+
+    def validate(source)
+      raise "Missing URL" if source["url"].blank?
+    end
+
+    def update(source, callback)
+      session "backtweets.com" do |http|
+        http.get_json "/search.json?key=#{api_key}&q=#{Rack::Utils.escape source["url"]}" do |status, json|
+          if status == 200 && tweets = json["tweets"]
+            last_tweet_id = source["last_tweet_id"]
+            new_ones = tweets.take_while { |tweet| tweet["tweet_id"] != last_tweet_id }.reverse
+            new_ones.each do |tweet|
+              screen_name, id = tweet["tweet_from_user"], tweet["tweet_id"]
+              url = "http://twitter.com/#{screen_name}/#{id}"
+              html = <<-HTML
+<a href="#{url}">tweeted</a>:
+<blockquote>#{tweet["tweet_text"]}</blockquote>
+              HTML
+              person = { fullname: screen_name, identities: %W{twitter.com:#{screen_name}},
+                         photo_url: "http://img.tweetimag.es/i/#{screen_name}_n" }
+              callback.activity! uid: id, url: url, html: html, tags: %w{twitter mention},
+                                 timestamp: Time.parse(tweet["tweet_created_at"]).utc, person: person
+              source["last_tweet_id"] = tweet["tweet_id"]
+            end
+            callback.set! tweets: json["totalresults"].to_i
+          else
+            callback.error! "Last request didn't go as expected, trying again later"
+          end
+        end
+      end
+    end
+
+    def meta(source)
+      [ { text: "Search yourself", url: "http://backtweets.com/search?q=#{URI.escape source["url"]}" } ]
+    end
+  end
+end

data/lib/dash-fu/bees/backtweets.yml

@@ -0,0 +1,9 @@
+en:
+  description: "Finds Tweets linking to any URL"
+  inputs: |-
+    <label>URL <input type="text" name="source[url]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>If this is a new search, it may take some time before we get the first results (including historical data). Check back in 8 hours.</li>
+      <li>Data provided by <a href="http://backtweets.com">Backtweets</a></li>
+    </ul>

data/lib/dash-fu/bees/github.rb

@@ -0,0 +1,103 @@
+module DashFu::Bee
+  # Track Github commits, watchers and forks.
+  class Github
+    include DashFu::Bee
+
+    def setup(source, params)
+      repo = params["repo"].strip
+      source["source.name"] = "Github: #{repo}"
+      source["metric.columns"] = [{ id: "commits", label: "Commits" }, { id: "watchers", label: "Watchers" }, { id: "forks", label: "Forks" }]
+      source["metric.totals"] = true
+      source["repo"] = repo
+      branch = params["branch"].strip
+      source["branch"] = branch.blank? ? "master" : branch
+      source["username"] = params["username"]
+      source["api_token"] = params["api_token"]
+    end
+
+    def validate(source)
+      raise "Missing repository name" if source["repo"].blank?
+      raise "Need user name and repository name, e.g. assaf/vanity" unless source["repo"][/^[\w-]+\/[\w-]+$/]
+    end
+
+    def update(source, callback)
+      session "github.com", 443 do |http|
+        auth = { username: "#{source["username"]}/token", password: source["api_token"] }
+        http.get_json "/api/v2/json/repos/show/#{source["repo"]}", auth do |status, json|
+          case status
+          when 200
+            if repository = json["repository"]
+              source.update repository.slice(*%w{description url homepage})
+              callback.set! forks: repository["forks"], watchers: repository["watchers"]
+            end
+          when 404, 400
+            callback.error! "Could not find the repository #{source["repo"]}"
+          when 401
+            callback.error! "You are not authorized to access this repository, or invalid username/password"
+          else
+            callback.error! "Last request didn't go as expected, trying again later"
+          end
+        end
+
+        http.get_json "/api/v2/json/commits/list/#{source["repo"]}/#{source["branch"]}", auth do |status, json|
+          case status
+          when 200
+            if commits = json["commits"]
+              last_seen_id = source["last_commit"]["id"] if source["last_commit"]
+              new_ones = commits.take_while { |commit| commit["id"] != last_seen_id }
+              merged = new_ones.inject([]) do |all, commit|
+                last = all.last.last unless all.empty?
+                if last && last["committer"]["email"] == commit["committer"]["email"] &&
+                  Time.parse(last["committed_date"]) - Time.parse(commit["committed_date"]) < 1.hour
+                  all.last << commit
+                else
+                  all << [commit]
+                end
+                all
+              end
+
+              merged.reverse.each do |commits|
+                first = commits.first
+                committer = first["committer"]
+                person = { fullname: committer["name"], identities: %W{github.com:#{committer["login"]}}, email: committer["email"] }
+                messages = commits.map { |commit| %{<blockquote><a href="#{commit["url"]}">#{commit["id"][0,7]}</a> #{h commit["message"].strip.split(/[\n\r]/).first[0,50]}</blockquote>} }
+                html = %{pushed to #{h source["branch"]} at <a href="http://github.com/#{source["repo"]}">#{h source["repo"]}</a>:\n#{messages.join("\n")}}
+                callback.activity! uid: first["id"], html: html, url: first["url"], tags: %w{push},
+                                   timestamp: Time.parse(first["committed_date"]).utc, person: person
+              end
+              callback.inc! commits: new_ones.count
+              if last_commit = commits.first
+                source.update "last_commit"=>last_commit.slice("id", "message", "url").
+                  merge("timestamp"=>Time.parse(last_commit["committed_date"]).utc)
+              end
+            end
+          when 404, 400
+            callback.error! "Could not find the branch #{source["branch"]}"
+          else
+            callback.error! "Github: #{response.code} #{response.message}"
+          end
+        end
+      end
+    end
+
+    def meta(source)
+      meta = [ { title: "Repository", text: source["repo"], url: source["url"] },
+               { title: "Branch", text: source["branch"] },
+               { text: source["description"] },
+               { title: "Home page", url: source["homepage"] } ]
+      if last_commit = source["last_commit"]
+        meta << { title: "Commit", text: last_commit["message"], url: last_commit["url"] }
+      end
+      meta
+    end
+
+  protected
+
+    def http_request(http, source, path)
+      get = Net::HTTP::Get.new(path.gsub(":repo", URI.escape(source["repo"])).gsub(":branch", URI.escape(source["branch"])))
+      get.basic_auth "#{source["username"]}/token", source["api_token"] unless source["username"].blank? && source["api_token"].blank?
+      http.request(get)
+    end
+
+  end
+end

data/lib/dash-fu/bees/github.yml

@@ -0,0 +1,15 @@
+en:
+  description: "Tracks Github commits, watchers and forks"
+  inputs: |-
+    <label>User/repository <input type="text" name="source[repo]" size="30"></label>
+    <label>Branch <input type="text" name="source[branch]" size="30" value="master"></label>
+    <p>For private repositories:</p>
+    <label>Username <input type="text" name="source[username]" size="30"></label>
+    <label><a href="https://github.com/account#admin_bucket" target="github">API Token</a> <input type="password" name="source[api_token]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>Enter user name/repository like this: <code>assaf/vanity</code></li>
+      <li>To access private repositories, authenticate using your username and API token.
+          You can find your API token in <a href="https://github.com/account#admin_bucket" target="github">Account Settings</a>.
+      </li>
+    </ul>

data/lib/dash-fu/bees/github_issues.rb

@@ -0,0 +1,90 @@
+module DashFu::Bee
+  # Track open/closed Github issues.
+  class GithubIssues
+    include DashFu::Bee
+
+    def setup(source, params)
+      repo = params["repo"].to_s.strip
+      source["source.name"] = "Github Issues for #{repo}"
+      source["metric.columns"] = [{ id: "open", label: "Open issues" }, { id: "closed", label: "Closed issues" }]
+      source["metric.totals"] = true
+      source["repo"] = repo
+      source["username"] = params["username"]
+      source["api_token"] = params["api_token"]
+    end
+
+    def validate(source)
+      raise "Missing user/repository" if source["repo"].blank?
+      raise "Need user name and repository name, e.g. assaf/vanity" unless source["repo"][/^[\w-]+\/[\w-]+$/]
+    end
+
+    def update(source, callback)
+      session "github.com", 443 do |http|
+        auth = { username: "#{source["username"]}/token", password: source["api_token"] }
+        http.get_json "/api/v2/json/issues/list/#{source["repo"]}/open" do |status, json|
+          case status
+          when 200
+            if open = json["issues"]
+              callback.set! open: open.count
+              open_ids = Set.new(source["open_ids"])
+              open.reject { |issue| open_ids.include?(issue["number"]) }.reverse.each do |issue|
+                sha = Digest::SHA1.hexdigest([source["repo"], "open", issue["number"], issue["updated_at"]].join(":"))
+                url = "http://github.com/#{source["repo"]}/issues#issue/#{issue["number"]}"
+                html = <<-HTML
+opened <a href="#{url}">issue #{issue["number"]}</a> on #{source["repo"]}:
+<blockquote>#{h issue["title"]}</blockquote>
+                HTML
+                callback.activity! uid: sha, url: url, html: html, tags: %w{issue opened},
+                                   timestamp: Time.parse(issue["created_at"]).utc
+                open_ids << issue["number"]
+              end
+              source["open_ids"] = open_ids.to_a
+            end
+          when 404, 400
+            callback.error! "Could not find the repository #{source["repo"]}"
+          when 401
+            callback.error! "You are not authorized to access this repository, or invalid username/password"
+          else
+            callback.error! "Last request didn't go as expected, trying again later"
+          end
+        end
+
+        http.get_json "/api/v2/json/issues/list/#{source["repo"]}/closed" do |status, json|
+          case status
+          when 200
+            if closed = json["issues"]
+              callback.set! closed: closed.count
+              closed_ids = Set.new(source["closed_ids"])
+              closed.reject { |issue| closed_ids.include?(issue["number"]) }.reverse.each do |issue|
+                sha = Digest::SHA1.hexdigest([source["repo"], "closed", issue["number"], issue["updated_at"]].join(":"))
+                url = "http://github.com/#{source["repo"]}/issues#issue/#{issue["number"]}"
+                html = <<-HTML
+closed <a href="#{url}">issue #{issue["number"]}</a> on #{source["repo"]}:
+<blockquote>#{h issue["title"]}</blockquote>
+                HTML
+                callback.activity! uid: sha, url: url, html: html, tags: %w{issue closed},
+                                   timestamp: Time.parse(issue["closed_at"]).utc
+                closed_ids << issue["number"]
+              end
+              source["closed_ids"] = closed_ids.to_a
+            end
+          end
+        end
+      end
+    end
+
+    def meta(source)
+      [ { title: "On Github", url: "http://github.com/#{source["repo"]}/issues" } ]
+    end
+
+  protected
+
+    def http_request(http, source, state)
+      path = "/api/v2/json/issues/list/#{URI.escape source["repo"]}/#{state}"
+      get = Net::HTTP::Get.new(path)
+      get.basic_auth "#{source["username"]}/token", source["api_token"] unless source["username"].blank? && source["api_token"].blank?
+      http.request(get)
+    end
+
+  end
+end

data/lib/dash-fu/bees/github_issues.yml

@@ -0,0 +1,14 @@
+en:
+  description: "Tracks open/closed Github issues"
+  inputs: |-
+    <label>User/repository <input type="text" name="source[repo]" size="30"></label>
+    <p>For private repositories:</p>
+    <label>Username <input type="text" name="source[username]" size="30"></label>
+    <label><a href="https://github.com/account#admin_bucket" target="github">API Token</a> <input type="password" name="source[api_token]" size="30"></label>
+  notes: |-
+    <ul>
+      <li>Enter user name/repository like this: <code>assaf/vanity</code></li>
+      <li>To access private repositories, authenticate using your username and API token.
+          You can find your API token in <a href="https://github.com/account#admin_bucket" target="github">Account Settings</a>.
+      </li>
+    </ul>

data/lib/dash-fu/bees/ruby_gems.rb

@@ -0,0 +1,54 @@
+module DashFu::Bee
+  # Track activity (downloads and releases) of a Ruby Gem from rubygems.org.
+  class RubyGems
+    include DashFu::Bee
+
+    def setup(source, params)
+      gem_name = params["gem_name"].to_s.strip
+      source["source.name"] = "RubyGems: #{gem_name}"
+      source["metric.columns"] = [{ id: "downloads", label: "Downloads" }]
+      source["metric.totals"] = true
+      source["gem_name"] = gem_name
+    end
+
+    def validate(source)
+      raise "Missing gem name" if source["gem_name"].blank?
+    end
+
+    def update(source, callback)
+      gem_name = source["gem_name"]
+      session "rubygems.org" do |http|
+        http.get_json "/api/v1/gems/#{gem_name}.json" do |status, json|
+          case status
+          when 200
+            if source["version"]
+              current, latest = Gem::Version.new(source["version"]), Gem::Version.new(json["version"])
+              if latest > current
+                html = "released <a href=\"http://rubygems.org/gems/#{gem_name}/versions/#{latest}\">#{gem_name} version #{latest}</a>."
+                person = { fullname: "RubyGems", identities: "url:http://rubygems.org/", photo_url: "http://dash-fu.com/images/sources/ruby.png" }
+                callback.activity! uid: "#{gem_name}-#{latest}", url: "http://rubygems.org/gems/#{gem_name}",
+                                   html: html, tags: %w{release}, person: person
+              end
+            end
+            source["version"] = json["version"]
+            source.update json.slice(*%w{homepage_uri project_uri info authors info})
+            callback.set! downloads: json["downloads"]
+          when 404
+            callback.error! "Could not find the Gem #{gem_name}"
+          else
+            callback.error! "Last request didn't go as expected, trying again later"
+          end
+        end
+      end
+    end
+
+    def meta(source)
+      [ { title: "Project", text: source["gem_name"], url: source["homepage_uri"] || source["project_uri"] },
+        { text: source["info"] },
+        { title: "Version", text: source["version"] },
+        { title: "Authors", text: source["authors"] },
+        { title: "Source", text: "RubyGems", url: source["project_uri"] } ]
+    end
+
+  end
+end

data/lib/dash-fu/bees/ruby_gems.yml

@@ -0,0 +1,6 @@
+en:
+  description: "Tracks downloads and releases of a Ruby Gem on rubygems.org"
+  inputs: |-
+    <label>Gem name <input type="text" name="source[gem_name]" size="30"></label>
+  notes: |-
+    Gem names are case sensitive.

data/test/api_keys.yml

@@ -0,0 +1 @@
+backtweets: "4554feeeeeeeeeeeeeee"

data/test/backtweets_test.rb

@@ -0,0 +1,148 @@
+require_relative "setup"
+
+test DashFu::Bee::Backtweets do
+  context "setup" do
+    setup { source.setup "url"=>"vanity.labnotes.org" }
+
+    should "use normalize URL and remove scheme" do
+      source.setup "url"=>"http://Vanity.Labnotes.Org"
+      assert_equal "Tweets for vanity.labnotes.org", source.name
+    end
+
+    context "metric" do
+      subject { source.metric }
+
+      should "use URL" do
+        assert_equal "Tweets for vanity.labnotes.org", subject.name
+      end
+
+      should "measure totals" do
+        assert subject.totals
+      end
+
+      should "capture tweets" do
+        assert subject.columns.include?(id: "tweets", label: "Tweets")
+      end
+    end
+  end
+
+
+  context "validation" do
+    should "raise error if URL missing" do
+      assert_raise(RuntimeError) { source.setup "url"=>" " }
+    end
+
+    should "create valid metric" do
+      source.setup "url"=>"vanity.labnotes.org"
+      assert source.valid?
+    end
+  end
+
+
+  context "update" do
+    setup { source.setup "url"=>"vanity.labnotes.org" }
+
+    should "include API key" do
+      source.update
+      assert_requested :get, "http://backtweets.com/search.json?key=4554feeeeeeeeeeeeeee&q=vanity.labnotes.org"
+    end
+
+    should "properly encode URL" do
+      stub_request(:get, /backtweets.com/).to_return body: interactions.first.response.body
+      source.setup "url"=>"need&this=encoded"
+      source.update
+      assert_requested :get, "http://backtweets.com/search.json?key=#{bee.send :api_key}&q=need%26this%3Dencoded"
+    end
+
+    should "handle errors" do
+      stub_request(:get, interactions.first.uri).to_return status: 500
+      source.update
+      assert_equal "Last request didn't go as expected, trying again later", source.last_error
+    end
+
+    should "handle invalid document entity" do
+      stub_request(:get, interactions.first.uri).to_return body: "Not JSON"
+      source.update
+      assert_equal "Last request didn't go as expected, trying again later", source.last_error
+    end
+
+    should "capture number of tweets" do
+      source.update
+      assert_equal({ tweets: 2 }, source.metric.values)
+    end
+
+    context "activity" do
+      setup { source.update }
+      subject { source.activity }
+
+      should "capture tweet identifier" do
+        assert_equal "20959239143", subject.uid
+      end
+
+      should "capture tweet text" do
+        assert_equal <<-HTML, subject.html
+<a href="http://twitter.com/dude/20959239143">tweeted</a>:
+<blockquote>Super awesome <a href="http://vanity.labnotes.org/">http://j.mp/aOrUnsz</a></blockquote>
+        HTML
+      end
+
+      should "capture tweet URL" do
+        assert_equal "http://twitter.com/dude/20959239143", subject.url
+      end
+
+      should "capture tweet timestamp" do
+        assert_equal Time.parse("2010-8-12T08:30:04").utc, subject.timestamp
+      end
+
+      should "tag as tweeter and mention" do
+        assert_contains subject.tags, "twitter"
+        assert_contains subject.tags, "mention"
+      end
+
+      should "be valid" do
+        assert subject.valid?
+      end
+
+      context "author" do
+        subject { source.activity.person }
+
+        should "capture full name" do
+          assert_equal "dude", subject.fullname
+        end
+
+        should "capture screen name" do
+          assert_contains subject.identities, "twitter.com:dude"
+        end
+
+        should "capture photo" do
+          assert_equal "http://img.tweetimag.es/i/dude_n", subject.photo_url
+        end
+      end
+    end
+
+    context "repeating" do
+      setup do
+        source.update
+        second = interactions.last
+        stub_request(:get, second.uri).to_return body: second.response.body
+        source.update
+      end
+
+      should "capture new tweets" do
+        assert_equal 3, source.activities.count
+        assert_match "The last of its kind", source.activity.html
+      end
+
+    end
+  end
+
+
+  context "meta" do
+    setup { source.setup "url"=>"vanity.labnotes.org" }
+    subject { source.meta }
+
+    should "link to search results" do
+      assert subject.include?(text: "Search yourself", url: "http://backtweets.com/search?q=vanity.labnotes.org")
+    end
+  end
+end