dash-bees 0.21 → 0.22

data/CHANGELOG

@@ -1,3 +1,16 @@
+2010-09-23 v0.22 Icons, origins and bug fixes.
+
+API change: using special field names (DashFu::Bee::Fields).
+
+Added icon URL, and each source gets to report text/URL to the origin (e.g.
+service, search page). These will be shown separate from metadata.
+
+Fixed tweet count for Backtweet source.
+
+Fixed handling of feeds that lack a title.
+
+GitHub/issues now accept repository URL.
+
 2010-09-20 v0.21 Improvements and bug fixes for feed source
 
 Truncates long contents.

data/Gemfile

@@ -9,7 +9,6 @@ end
 group :test do
   gem "awesome_print"
   gem "builder"
-  gem "shoulda"
   gem "timecop"
   gem "vcr"
   gem "webmock"

data/dash-bees.gemspec

@@ -1,5 +1,5 @@
 $: << File.dirname(__FILE__) + "/lib"
-require "dash-fu/bee"
+require "dash-fu/bee/version"
 
 Gem::Specification.new do |spec|
   spec.name           = "dash-bees"

data/lib/dash-fu/bee.rb

@@ -3,14 +3,18 @@ require "net/http"
 require "uri"
 require "open-uri"
 require "rack"
+require "nokogiri"
+require "sanitize"
+require "dash-fu/bee/version"
+require "dash-fu/bee/http_session"
+require "dash-fu/bee/fields"
 
 # See http://dash-fu.com
 module DashFu
 
   # The README covers it all.
   module Bee
-
-    VERSION = "0.21"
+    include DashFu::Bee::Fields
 
     class << self
       attr_accessor :logger
@@ -132,47 +136,7 @@ module DashFu
       http = Net::HTTP.new(host, port)
       http.use_ssl = true if port == 443
       http.start do |http|
-        yield Session.new(http)
-      end
-    end
-
-    # HTTP Session.
-    class Session
-      def initialize(http) #:nodoc:
-        @http = http
-      end
-
-      # Make a GET request and yield response to the block. Response consists of
-      # three arguments: status code, response body and response headers. The
-      # block may be called asynchronoulsy.
-      def get(path, headers = {}, &block)
-        response = @http.request(get_request(path, headers || {}))
-        yield response.code.to_i, response.body, {}
-      end
-
-      # Make a GET request and yield response to the block. If the response
-      # status is 200 the second argument is the response JSON object. The block
-      # may be called asynchronously.
-      def get_json(path, headers = {}, &block)
-        response = @http.request(get_request(path, headers || {}))
-        if Net::HTTPOK === response
-          json = JSON.parse(response.body) rescue nil
-          if json
-            yield response.code.to_i, json, {}
-          else
-            yield 500, "Not a JSON document", {}
-          end
-        else
-          yield response.code.to_i, response.message, {}
-        end
-      end
-
-    protected
-
-      def get_request(path, headers)
-        request = Net::HTTP::Get.new(path)
-        request.basic_auth headers[:username], headers[:password] if headers[:username] && headers[:password]
-        request
+        yield DashFu::HTTPSession.new(http)
       end
     end
 

data/lib/dash-fu/bee/fields.rb

@@ -0,0 +1,17 @@
+module DashFu::Bee
+  module Fields
+
+    # Suggested name for the source.
+    SOURCE_NAME = "_name"
+    # Information about the origin of the data, using two fields, text and URL.
+    ORIGIN = "_origin"
+    # URL for an icon for this feed. Icon should have 1:1 aspect ration,
+    # 48px/48px is the ideal isize.
+    ICON = "_icon"
+
+    # If this source has a metric, enumerates the columns.
+    METRIC_COLUMNS = "_metric_columns"
+    # True if there's a metric and it collects totals
+    METRIC_TOTALS = "_metric_totals"
+  end
+end

data/lib/dash-fu/bee/http_session.rb

@@ -0,0 +1,44 @@
+require "json"
+require "net/http"
+
+module DashFu
+  # HTTP Session.
+  class HTTPSession
+    def initialize(http) #:nodoc:
+      @http = http
+    end
+
+    # Make a GET request and yield response to the block. Response consists of
+    # three arguments: status code, response body and response headers. The
+    # block may be called asynchronoulsy.
+    def get(path, headers = {}, &block)
+      response = @http.request(get_request(path, headers || {}))
+      yield response.code.to_i, response.body, {}
+    end
+
+    # Make a GET request and yield response to the block. If the response
+    # status is 200 the second argument is the response JSON object. The block
+    # may be called asynchronously.
+    def get_json(path, headers = {}, &block)
+      response = @http.request(get_request(path, headers || {}))
+      if Net::HTTPOK === response
+        json = JSON.parse(response.body) rescue nil
+        if json
+          yield response.code.to_i, json, {}
+        else
+          yield 500, "Not a JSON document", {}
+        end
+      else
+        yield response.code.to_i, response.message, {}
+      end
+    end
+
+  protected
+
+    def get_request(path, headers)
+      request = Net::HTTP::Get.new(path)
+      request.basic_auth headers[:username], headers[:password] if headers[:username] && headers[:password]
+      request
+    end
+  end
+end

data/lib/dash-fu/bee/version.rb

@@ -0,0 +1,5 @@
+module DashFu
+  module Bee
+    VERSION = "0.22"
+  end
+end

data/lib/dash-fu/bees/backtweets.rb

@@ -4,15 +4,21 @@ module DashFu::Bee
     include DashFu::Bee
 
     def setup(source, params)
-      url = params["url"].strip.downcase.sub(/^http(s?):\/\//, "")
-      source["source.name"] = "Tweets for #{url}"
-      source["metric.columns"] = [{ id: "tweets", label: "Tweets" }]
-      source["metric.totals"] = true
-      source["url"] = url
+      uri = URI.parse(params["url"]).normalize
+      uri = URI.parse("http://#{uri}").normalize if uri.scheme.nil?
+      source["url"] = uri.to_s
+      short = source["url"].gsub(/^https?:\/\//, "")
+      short.gsub!(/\/$/, "") if uri.path == "/"
+
+      source[SOURCE_NAME] = "Tweets for #{short}"
+      source[ORIGIN] = { text: "Backtweets for #{short}", url: "http://backtweets.com/search?q=#{URI.escape source["url"]}" }
+      source[METRIC_COLUMNS] = [{ id: "tweets", label: "Tweets" }]
+      source[METRIC_TOTALS] = true
     end
 
     def validate(source)
-      raise "Missing URL" if source["url"].blank?
+      uri = URI.parse(source["url"])
+      raise "This doesn't look like a valid page name, please check your URL" unless uri.absolute? && uri.scheme == "http" && !uri.host.blank?
     end
 
     def update(source, callback)
@@ -20,6 +26,9 @@ module DashFu::Bee
         http.get_json "/search.json?key=#{api_key}&q=#{Rack::Utils.escape source["url"]}" do |status, json|
           if status == 200 && tweets = json["tweets"]
             last_tweet_id = source["last_tweet_id"]
+            # Since we don't get much in terms of history, we only look at this
+            # count once; afterwards, we increment for each tweet we find.
+            callback.set! tweets: json["totalresults"].to_i unless last_tweet_id
             new_ones = tweets.take_while { |tweet| tweet["tweet_id"] != last_tweet_id }.reverse
             new_ones.each do |tweet|
               screen_name, id = tweet["tweet_from_user"], tweet["tweet_id"]
@@ -30,11 +39,11 @@ module DashFu::Bee
               HTML
               person = { fullname: screen_name, identities: %W{twitter.com:#{screen_name}},
                          photo_url: "http://img.tweetimag.es/i/#{screen_name}_n" }
-              callback.activity! uid: id, url: url, html: html, tags: %w{twitter mention},
+              callback.activity! uid: id, url: url, html: html, tags: %w{twitter mention}, public: true,
                                  timestamp: Time.parse(tweet["tweet_created_at"]).utc, person: person
               source["last_tweet_id"] = tweet["tweet_id"]
             end
-            callback.set! tweets: json["totalresults"].to_i
+            callback.inc! tweets: new_ones.count if last_tweet_id
           else
             callback.error! "Last request didn't go as expected, trying again later"
           end
@@ -42,8 +51,5 @@ module DashFu::Bee
       end
     end
 
-    def meta(source)
-      [ { text: "Search yourself", url: "http://backtweets.com/search?q=#{URI.escape source["url"]}" } ]
-    end
   end
 end

data/lib/dash-fu/bees/feed.rb

@@ -1,37 +1,40 @@
-require "sanitize"
-require "nokogiri"
-
 module DashFu::Bee
   # Track Web feed (Atom or RSS).
   class Feed
     include DashFu::Bee
 
     def setup(source, params)
-      source["url"] = params["url"].strip
-      source["source.name"] = "Web feed"
-    end
+      unfeed = params["url"].strip.gsub(/^feed:\/\//i, "http://").gsub(/^feed:/i, "")
+      uri = URI.parse(unfeed).normalize
+      uri = URI.parse("http://#{uri}").normalize if uri.scheme.nil?
+      source["url"] = uri.to_s
+      return unless valid_uri?(uri)
 
-    def validate(source)
-      unfeed = source["url"].gsub(/^feed:\/\//, "http://").gsub(/^feed:/, "")
-      uri = URI.parse(unfeed) rescue nil
-      raise "Not a valid URL" unless uri && uri.absolute?
-      raise "Only HTTP/S URLs supported" unless uri.scheme == "http" || uri.scheme == "https"
       begin
         uri.open read_timeout: 3, redirect: true do |io|
           code = io.status.first
           raise "Cannot read this feed, got status code #{code}" unless code == "200"
           feed = (Nokogiri::XML(io.read)>"feed").first
-          source["source.name"] = source["title"] = get_text(feed>"title").strip
+          title = get_text(feed>"title").strip
+          title = uri.to_s if title.blank?
+
+          source[SOURCE_NAME] = title
           alt = (feed>"link[rel=alternate]").first
-          source["permalink"] = alt["href"] if alt
-          source["logo"] = (feed>"logo").text
-          source["url"] = uri.to_s 
+          permalink = alt ? alt["href"].strip : uri.to_s
+          source[ORIGIN] = { text: title, url: permalink }
+          source[ICON] = (feed>"icon").text
         end
       rescue
+        puts $!
         raise "Cannot read this feed: is it down for you or just for us?"
       end
     end
 
+    def validate(source)
+      uri = URI.parse(source["url"])
+      raise "This doesn't look like a valid feed, please check your URL" unless valid_uri?(uri)
+    end
+
     def update(source, callback)
       uri = URI.parse(source["url"])
       session uri.host, uri.port do |http|
@@ -77,9 +80,7 @@ module DashFu::Bee
     end
 
     def meta(source)
-      meta = []
-      meta << { title: "Source", text: source["title"], url: source["permalink"] }
-      meta
+      []
     end
 
     def get_text(elements)
@@ -102,5 +103,9 @@ module DashFu::Bee
       Sanitize.clean(truncate_html(html.to_s.strip, length), Sanitize::Config::BASIC.merge(output: :html)).strip
     end
 
+    def valid_uri?(uri)
+      uri.absolute? && (uri.scheme == "http" || uri.scheme == "https") && !uri.host.blank?
+    end
+
   end
 end

data/lib/dash-fu/bees/feed.yml

@@ -1,7 +1,7 @@
 en:
   description: "Gather activities from an Atom feed"
   inputs: |-
-    <label>Feed URL <input type="text" name="source[url]" size="50"></label>
+    <label>Feed URL <input type="text" name="source[url]" size="70"></label>
   notes: |-
     <ul>
     </ul>

data/lib/dash-fu/bees/github.rb

@@ -3,16 +3,22 @@ module DashFu::Bee
   class Github
     include DashFu::Bee
 
+    ICON_URL = "http://dash-fu.com/images/sources/github.png"
+
     def setup(source, params)
       repo = params["repo"].strip
-      source["source.name"] = "GitHub: #{repo}"
-      source["metric.columns"] = [{ id: "commits", label: "Commits" }, { id: "watchers", label: "Watchers" }, { id: "forks", label: "Forks" }]
-      source["metric.totals"] = true
+      repo = $1 if repo =~ /^http:\/\/github\.com\/([\w-]+\/[\w-]+)/i
       source["repo"] = repo
       branch = params["branch"].strip
       source["branch"] = branch.blank? ? "master" : branch
       source["username"] = params["username"]
       source["api_token"] = params["api_token"]
+
+      source[SOURCE_NAME] = "GitHub: #{repo}"
+      source[METRIC_COLUMNS] = [{ id: "commits", label: "Commits" }, { id: "watchers", label: "Watchers" }, { id: "forks", label: "Forks" }]
+      source[METRIC_TOTALS] = true
+      source[ORIGIN] = { text: repo, url: "http://github.com/#{repo}" }
+      source[ICON] = ICON_URL
     end
 
     def validate(source)
@@ -22,7 +28,7 @@ module DashFu::Bee
 
     def update(source, callback)
       session "github.com", 443 do |http|
-        auth = { username: "#{source["username"]}/token", password: source["api_token"] }
+        auth = { username: "#{source["username"]}/token", password: source["api_token"] } unless source["username"].blank?
         http.get_json "/api/v2/json/repos/show/#{source["repo"]}", auth do |status, json|
           case status
           when 200
@@ -63,7 +69,7 @@ module DashFu::Bee
                 person = { fullname: committer["name"], identities: Array.wrap(identity), email: committer["email"] }
                 messages = commits.map { |commit| %{<blockquote><a href="#{commit["url"]}">#{commit["id"][0,7]}</a> #{h commit["message"].strip.split(/[\n\r]/).first[0,50]}</blockquote>} }
                 html = %{pushed to #{h source["branch"]} at <a href="http://github.com/#{source["repo"]}">#{h source["repo"]}</a>:\n#{messages.join("\n")}}
-                callback.activity! uid: first["id"], html: html, url: first["url"], tags: %w{push},
+                callback.activity! uid: first["id"], html: html, url: first["url"], tags: %w{push}, public: auth.nil?,
                                    timestamp: Time.parse(first["committed_date"]).utc, person: person
               end
               callback.inc! commits: new_ones.count
@@ -82,8 +88,7 @@ module DashFu::Bee
     end
 
     def meta(source)
-      meta = [ { title: "Repository", text: source["repo"], url: source["url"] },
-               { title: "Branch", text: source["branch"] },
+      meta = [ { title: "Branch", text: source["branch"] },
                { text: source["description"] },
                { title: "Home page", url: source["homepage"] } ]
       if last_commit = source["last_commit"]

data/lib/dash-fu/bees/github_issues.rb

@@ -3,14 +3,20 @@ module DashFu::Bee
   class GithubIssues
     include DashFu::Bee
 
+    ICON_URL = "http://dash-fu.com/images/sources/github.png"
+
     def setup(source, params)
       repo = params["repo"].to_s.strip
-      source["source.name"] = "Github Issues for #{repo}"
-      source["metric.columns"] = [{ id: "open", label: "Open issues" }, { id: "closed", label: "Closed issues" }]
-      source["metric.totals"] = true
+      repo = $1 if repo =~ /^http:\/\/github\.com\/([\w-]+\/[\w-]+)/i
       source["repo"] = repo
       source["username"] = params["username"]
       source["api_token"] = params["api_token"]
+
+      source[SOURCE_NAME] = "GitHub issues for #{repo}"
+      source[METRIC_COLUMNS] = [{ id: "open", label: "Open issues" }, { id: "closed", label: "Closed issues" }]
+      source[METRIC_TOTALS] = true
+      source[ORIGIN] = { text: repo, url: "http://github.com/#{repo}" }
+      source[ICON] = ICON_URL
     end
 
     def validate(source)
@@ -19,9 +25,9 @@ module DashFu::Bee
     end
 
     def update(source, callback)
-      github = { fullname: "GitHub", identities: %w{site:github.com}, photo_url: "http://github.com/images/error/octocat_happy.gif" }
+      github = { fullname: "GitHub", identities: %w{site:github.com}, photo_url: ICON_URL }
       session "github.com", 443 do |http|
-        auth = { username: "#{source["username"]}/token", password: source["api_token"] }
+        auth = { username: "#{source["username"]}/token", password: source["api_token"] } unless source["username"].blank?
         http.get_json "/api/v2/json/issues/list/#{source["repo"]}/open" do |status, json|
           case status
           when 200
@@ -35,7 +41,7 @@ module DashFu::Bee
 opened <a href="#{url}">issue #{issue["number"]}</a> on #{source["repo"]}:
 <blockquote>#{h issue["title"]}</blockquote>
                 HTML
-                callback.activity! uid: sha, url: url, html: html, tags: %w{issue opened},
+                callback.activity! uid: sha, url: url, html: html, tags: %w{issue opened}, public: !auth,
                                    person: github, timestamp: Time.parse(issue["created_at"]).utc
                 open_ids << issue["number"]
               end
@@ -63,7 +69,7 @@ opened <a href="#{url}">issue #{issue["number"]}</a> on #{source["repo"]}:
 closed <a href="#{url}">issue #{issue["number"]}</a> on #{source["repo"]}:
 <blockquote>#{h issue["title"]}</blockquote>
                 HTML
-                callback.activity! uid: sha, url: url, html: html, tags: %w{issue closed},
+                callback.activity! uid: sha, url: url, html: html, tags: %w{issue closed}, public: !auth,
                                    person: github, timestamp: Time.parse(issue["closed_at"]).utc
                 closed_ids << issue["number"]
               end
@@ -74,9 +80,5 @@ closed <a href="#{url}">issue #{issue["number"]}</a> on #{source["repo"]}:
       end
     end
 
-    def meta(source)
-      [ { title: "On GitHub", url: "http://github.com/#{source["repo"]}/issues" } ]
-    end
-
   end
 end

data/lib/dash-fu/bees/ruby_gems.rb

@@ -3,12 +3,17 @@ module DashFu::Bee
   class RubyGems
     include DashFu::Bee
 
+    ICON_URL = "http://dash-fu.com/images/sources/ruby.png"
+
     def setup(source, params)
       gem_name = params["gem_name"].to_s.strip
-      source["source.name"] = "RubyGems: #{gem_name}"
-      source["metric.columns"] = [{ id: "downloads", label: "Downloads" }]
-      source["metric.totals"] = true
       source["gem_name"] = gem_name
+
+      source[SOURCE_NAME] = "RubyGems: #{gem_name}"
+      source[METRIC_COLUMNS] = [{ id: "downloads", label: "Downloads" }]
+      source[METRIC_TOTALS] = true
+      source[ORIGIN] = { text: "RubyGems: #{gem_name}", url: "http://rubygems.org/gems/#{gem_name}" }
+      source[ICON] = ICON_URL
     end
 
     def validate(source)
@@ -25,8 +30,8 @@ module DashFu::Bee
               current, latest = Gem::Version.new(source["version"]), Gem::Version.new(json["version"])
               if latest > current
                 html = "released <a href=\"http://rubygems.org/gems/#{gem_name}/versions/#{latest}\">#{gem_name} version #{latest}</a>."
-                person = { fullname: "RubyGems", identities: "url:http://rubygems.org/", photo_url: "http://dash-fu.com/images/sources/ruby.png" }
-                callback.activity! uid: "#{gem_name}-#{latest}", url: "http://rubygems.org/gems/#{gem_name}",
+                person = { fullname: "RubyGems", identities: "url:http://rubygems.org/", photo_url: ICON_URL }
+                callback.activity! uid: "#{gem_name}-#{latest}", url: "http://rubygems.org/gems/#{gem_name}", public: true,
                                    html: html, tags: %w{release}, person: person
               end
             end
@@ -46,8 +51,7 @@ module DashFu::Bee
       [ { title: "Project", text: source["gem_name"], url: source["homepage_uri"] || source["project_uri"] },
         { text: source["info"] },
         { title: "Version", text: source["version"] },
-        { title: "Authors", text: source["authors"] },
-        { title: "Source", text: "RubyGems", url: source["project_uri"] } ]
+        { title: "Authors", text: source["authors"] } ]
     end
 
   end