livedoor-feeddiscover 1.0.0

data/History.txt

@@ -0,0 +1,3 @@
+== 0.1.0 / 2007-06-16
+
+* Initial release

data/License.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2007 MIYAMUKO Katsuyuki <mailto:miyamuko@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,23 @@
+History.txt
+License.txt
+Manifest.txt
+README.txt
+Rakefile
+examples/benchmark.rb
+examples/mkopml.rb
+helper/helper.rb
+helper/rake.rb
+helper/rake_sh_filter.rb
+helper/util.rb
+lib/livedoor/api/feed_discover.rb
+lib/livedoor/api/feed_discover/client.rb
+lib/livedoor/api/feed_discover/feed.rb
+lib/livedoor/api/feed_discover/feed_list.rb
+lib/livedoor/api/feed_discover/version.rb
+setup.rb
+spec/livedoor/api/feed_discover/client_spec.rb
+spec/livedoor/api/feed_discover/feed_list_spec.rb
+spec/livedoor/api/feed_discover/feed_spec.rb
+spec/livedoor/api/feed_discover/version_spec.rb
+spec/spec.opts
+spec/spec_helper.rb

data/README.txt

@@ -0,0 +1,112 @@
+== livedoor-feeddiscover - The fastest feed auto-discovery
+
+=== DESCRIPTION
+
+livedoor-feeddiscover performs feed autodiscovery using the livedoor Feed Discover API.
+
+livedoor Feed Discover API find a Atom/RSS feed(s) from the livedoor Reader crawler database.
+So, livedoor-feeddiscover do not access the target URL.
+
+* {livedoor Feed Discover API}[http://wiki.livedoor.jp/staff_reader/d/Feed%20Discover%20API] (API Spec, Japanese)
+* {livedoor Reader}[http://reader.livedoor.com/reader/] (Fastest RSS reader, Japanese)
+
+
+=== SYNOPSIS
+
+  require "rubygems"
+  require "livedoor/api/feed_discover"
+
+  # find feed of matz blog.
+  matzblog = Livedoor::API::FeedDiscover.find(
+    "http://www.rubyist.net/~matz/",
+    "User-Agent" => "livedoor-feeddiscover/#{Livedoor::API::FeedDiscover::VERSION::STRING}")
+  if matzblog.feeds?
+    puts matzblog.feed.source             # specified argument
+    puts matzblog.feed.link               # blog link
+    puts matzblog.feed.feedlink           # feed link
+    puts matzblog.feed.image              # blog logo (channel image)
+    puts matzblog.feed.icon               # favicon
+    puts matzblog.feed.title              # blog title (UTF-8 string)
+    puts matzblog.feed.description        # blog description (UTF-8 string)
+    puts matzblog.feed.official?          # auto discover-able?
+    puts matzblog.feed.thirdparty?        # thirdparty feeds? (not official)
+    puts matzblog.feed.subscribers_count  # subscribers in livedoor Reader
+    puts matzblog.feed.avg_rate           # average rate in livedoor Reader
+  end
+
+  # FeedDiscover.find can take multiple links (max 1000).
+  feedlist = Livedoor::API::FeedDiscover.find("http://rubyforge.org/", "http://raa.ruby-lang.org/")
+  puts feedlist.feedlinks
+
+  # generate OPML for Japanese rubyist blogs linked from Ruby hotlinks.
+  RUBY_HOT_LINKS = "http://www.rubyist.net/~kazu/samidare/"
+  rubyist_jp = Livedoor::API::FeedDiscover.find_from(RUBY_HOT_LINKS)
+
+  open("rubyist-jp.opml", "w") do |w|
+    w.puts rubyist_jp.opml
+  end
+
+  rubyist_jp.each do |feed|
+    puts feed.link
+    puts "  ==> #{feed.feedlink} (#{feed.thirdparty? ? 'thirdpary' : 'official'})"
+  end
+
+
+=== BENCHMARK
+
+vs feed-discover[http://rubyforge.org/projects/feed-discover/].
+
+  > ruby examples\benchmark.rb
+  Start auto-discover for 4 blogs ------------------------------------
+  http://www.rubyist.net/~matz/
+  http://rubyforge.org/
+  http://raa.ruby-lang.org/
+  http://jruby.codehaus.org/
+  
+                                 user     system      total        real
+  feed-discover              5.234000   0.234000   5.468000 ( 13.797000)
+  livedoor-feed-discover     0.000000   0.000000   0.000000 (  0.078000)
+  
+  Results ------------------------------------------------------------
+  feed-discover: found 3 feeds.
+  livedoor-f-d:  found 3 feeds.
+  
+  Diffs --------------------------------------------------------------
+  http://www.rubyist.net/~matz/
+    feed-discover: http://www.rubyist.net/~matz/comments.rdf, http://www.rubyist.net/~matz/index.rdf
+    livedoor-f-d:  http://www.rubyist.net/~matz/index.rdf
+
+
+=== REQUIREMENTS
+
+* json[http://rubyforge.org/projects/json/]
+
+
+=== SEE ALSO
+
+* feed-discover[http://rubyforge.org/projects/feed-discover/]
+
+
+=== INSTALL
+
+[Unix]
+    sudo gem install -y livedoor-feeddiscover
+
+[Windows]
+    gem install -y livedoor-feeddiscover
+
+
+=== COPYRIGHT
+
+Copyright (c) 2007 MIYAMUKO Katsuyuki.
+
+livedoor-feeddiscover is released under an MIT license.
+See {License.txt}[link:files/License_txt.html] for full license.
+
+
+=== OTHER STUFF
+
+Author::          MIYAMUKO Katsuyuki <mailto:miyamuko@gmail.com>
+Home URL::        http://feeddiscoverapi.rubyforge.org
+Project URL::     http://rubyforge.org/projects/feeddiscoverapi
+Blog (Japanese):: {id:miyamuko}[http://d.hatena.ne.jp/miyamuko/]

data/Rakefile

@@ -0,0 +1,202 @@
+# -*- mode: ruby -*-
+
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/contrib/rubyforgepublisher'
+require 'fileutils'
+require 'hoe'
+begin
+  require 'spec/rake/spectask'
+rescue LoadError
+  puts 'To use rspec for testing you must install rspec gem:'
+  puts '$ sudo gem install rspec'
+  exit
+end
+
+require 'helper/helper'
+
+include FileUtils
+require File.join(File.dirname(__FILE__), 'lib', 'livedoor', 'api', 'feed_discover', 'version')
+
+AUTHOR = 'MIYAMUKO Katsuyuki'  # can also be an array of Authors
+EMAIL = "miyamuko@gmail.com"
+DESCRIPTION = "Ruby interface to the livedoor Feed Discover API"
+GEM_NAME = 'livedoor-feeddiscover'
+NEWS_FILE = "news.txt"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts "ERROR: No rubyforge config file found: #{@config_file}"
+      puts "Run 'rubyforge setup' to prepare your env for access to Rubyforge"
+      exit
+    end
+  end
+  @rubyforge_username ||= @config["username"]
+end
+
+RUBYFORGE_PROJECT = 'feeddiscoverapi' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+NAME = "livedoor-feeddiscover"
+REV = nil
+# UNCOMMENT IF REQUIRED:
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Livedoor::API::FeedDiscover::VERSION::STRING + (REV ? ".#{REV}" : "")
+CLEAN.include ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store', '**/.cache', NEWS_FILE]
+RDOC_OPTS = ['--quiet', '--title', 'livedoor-feeddiscover documentation',
+    "--opname", "index.html",
+    "--line-numbers",
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps
+    @extra_deps.reject!{|x| Array(x).first == 'hoe' }
+    @extra_deps
+  end
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR
+  p.email = EMAIL
+  p.url = HOMEPATH
+
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["spec/**/*_spec.rb"]
+  p.clean_globs |= CLEAN  #An array of file patterns to delete on clean.
+
+  p.summary = p.paragraphs_of('README.txt', 2..2)[0]
+  p.description = p.paragraphs_of('README.txt', 2..3).join("\n\n")
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+
+  p.extra_deps << "json"
+  p.need_zip = true
+  p.need_tar = true
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\n\n")
+desc 'Release the website and new gem version'
+task :deploy => [:check_manifest!, :check_version, :publish_docs, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+task :check_manifest! => [:check_manifest] do
+  unless $?.success?
+    puts "----"
+    puts "Please update Manifest.txt."
+    exit 1
+  end
+end
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+desc "Run the specs under spec/models"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+desc "Default task is to run specs"
+task :default => :spec
+
+#
+# new task
+#
+
+desc 'Uninstall the gem package'
+task_for_windows :uninstall_gem do
+  sh "gem.bat uninstall #{GEM_NAME}"
+end
+
+
+#
+# overwrite hoe defined tasks.
+#
+
+task_for_windows :install_gem do
+  sh "gem.bat install pkg/*.gem"
+end
+
+task_for_windows :ridocs do
+  sh "rdoc.bat --ri -o ri ."
+end
+
+# remove HTML comment for stupid IE6.
+task :docs do
+  Dir.glob("**/*.html").each do |html|
+    c = File.read(html)
+    File.open(html, "w") do |w|
+      w.puts c.gsub(/<!--.*?-->\n/m, "")
+    end
+  end
+end
+
+task_for_windows :publish_docs do
+  config = YAML.load(File.read(File.expand_path("~/.rubyforge/user-config.yml")))
+  with_temporary_rename("doc", "html") {
+    Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, config["username"]).upload
+  }
+end
+
+# rake publish_docs SCP=pscp
+if ENV["SCP"] and ENV["SCP"] != "scp"
+  add_sh_filter_for(/\Ascp /) {|cmd|
+    cmd[0].sub!(/\Ascp/, ENV["SCP"])
+    cmd[0].sub!(/-rq/, "-r")
+    cmd
+  }
+end
+
+
+task :create_news do
+  subject, title, body, urls = hoe.announcement
+  open(NEWS_FILE, "w") do |w|
+    w.puts subject
+    w.puts "#{title}\n\n#{body}\n\n"
+    w.puts "--"
+    w.puts "Home URL:    #{HOMEPATH}"
+    w.puts "Project URL: #{DOWNLOAD_PATH}"
+  end
+  puts File.read(NEWS_FILE)
+end
+
+override_task :post_news do
+  subject, body = File.read(NEWS_FILE).split(/\n/, 2)
+  require 'rubyforge'
+  rf = RubyForge.new
+  rf.login
+  rf.post_news(RUBYFORGE_PROJECT, subject, body)
+  puts "Posted to rubyforge `#{subject}'."
+  rm_f NEWS_FILE
+end
+task :clean do
+  rm_f NEWS_FILE
+end
+
+# vim: syntax=Ruby

data/examples/benchmark.rb

@@ -0,0 +1,64 @@
+require "benchmark"
+require "open-uri"
+require "uri"
+
+require "rubygems"
+require "feed_discover"
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), "../lib")
+require "livedoor/api/feed_discover"
+
+
+def feed_discover(blogs, acc)
+  blogs.each do |blog|
+    acc[blog] = nil
+    begin
+      fd = FeedDiscover.new(blog)
+      acc[blog] = fd.feeds.join(", ") if fd.feeds?
+    rescue OpenURI::HTTPError
+    end
+  end
+end
+
+def livedoor_feed_discover(blogs, acc)
+  feedlist = Livedoor::API::FeedDiscover.find(*blogs)
+  feedlist.each do |feed|
+    acc[feed.source] = feed.feedlink
+  end
+end
+
+def header(label)
+  puts "#{label} #{'-' * (68 - 1 - label.length)}"
+end
+
+blogs = %W{
+  http://www.rubyist.net/~matz/
+  http://rubyforge.org/
+  http://raa.ruby-lang.org/
+  http://jruby.codehaus.org/
+}
+feeds_a = blogs.inject({}) {|acc,e| acc[e] = nil; acc }
+feeds_b = feeds_a.dup
+
+header("Start auto-discover for #{blogs.size} blogs")
+puts blogs
+puts
+Benchmark.bm(25) do |x|
+  x.report("feed-discover")           { feed_discover(blogs, feeds_a) }
+  x.report("livedoor-feed-discover")  { livedoor_feed_discover(blogs, feeds_b) }
+end
+puts
+
+header("Results")
+puts "feed-discover: found #{feeds_a.values.compact.size} feeds."
+puts "livedoor-f-d:  found #{feeds_b.values.compact.size} feeds."
+puts
+
+header("Diffs")
+blogs.each do |blog|
+  if feeds_a[blog] != feeds_b[blog]
+    puts blog
+    puts "  feed-discover: #{feeds_a[blog]}"
+    puts "  livedoor-f-d:  #{feeds_b[blog]}"
+  end
+end

data/examples/mkopml.rb

@@ -0,0 +1,48 @@
+require "optparse"
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), "../lib")
+require "livedoor/api/feed_discover"
+
+def with_output_stream(out)
+  case out
+  when "-"
+    yield $stdout
+  when String
+    open(out, "w"){|w| yield w }
+  else
+    yield out
+  end
+end
+
+def main
+  find_from = nil
+  output = $stdout
+  header = {}
+
+  ARGV.options do |opt|
+    opt.banner = "Usage: #{File.basename($0)} [options] [URL...]"
+    opt.on("-f", "--find-from URL"){|v| find_from = v }
+    opt.on("-o", "--output OUTPUT"){|v| output = v }
+    opt.on("-U", "--user-agent AGENT"){|v| header["User-Agent"] = v }
+    opt.parse!
+  end
+
+  if find_from
+    feedlist = Livedoor::API::FeedDiscover.find_from(find_from, header)
+  elsif not ARGV.empty?
+    feedlist = Livedoor::API::FeedDiscover.find(ARGV, header)
+  else
+    puts ARGV.options
+    exit 1
+  end
+
+  unless feedlist.feeds?
+    exit 2
+  end
+
+  with_output_stream(output) do |w|
+    w.puts feedlist.opml
+  end
+end
+
+main

data/helper/helper.rb

@@ -0,0 +1,3 @@
+require "helper/util"
+require "helper/rake"
+require "helper/rake_sh_filter"

data/helper/rake.rb

@@ -0,0 +1,58 @@
+require "rubygems"
+require "rake"
+
+def tasks
+  Rake.application.instance_eval { @tasks }
+end
+
+def current_scope
+  Rake.application.instance_eval { @scope.last }
+end
+
+def task_defined?(task_name)
+  Rake::Task.task_defined?(task_name)
+end
+
+def remove_tasks(*task_names)
+  task_names.flatten.each do |task_name|
+    remove_task(task_name)
+  end
+end
+
+def remove_task(task_name)
+  tasks.delete(task_name.to_s)
+end
+
+def lookup_task(task_name)
+  Rake::Task[task_name] rescue nil
+end
+
+def rakecall(task_name)
+  lookup_task(task_name).invoke
+end
+
+def override_task(task_args, &block)
+  task_name, deps = Rake.application.resolve_args(task_args)
+  original = lookup_task(task_name)
+  orgproc = lambda {} # nop
+  if original
+    Rake.application.last_comment = original.comment
+    deps |= original.prerequisites
+    orgproc = lambda{ original.execute }
+  end
+
+  remove_task(task_name)
+  Rake::Task.define_task(task_name => deps) do |t|
+    block.call(t, orgproc)
+  end
+end
+
+def task_for_windows(task_args, &block)
+  override_task(task_args) do |t, org|
+    if windows?
+      block.call(t)
+    else
+      org.call
+    end
+  end
+end

data/helper/rake_sh_filter.rb

@@ -0,0 +1,23 @@
+module FileUtils
+  SHELL_COMMAND_FILTER = []
+
+  alias sh_original sh
+  def sh(*cmd, &block)
+    cmd = SHELL_COMMAND_FILTER.inject(cmd){|c,filter| filter.call(c) or c }
+    sh_original(*cmd, &block)
+  end
+
+  def add_sh_filter(&block)
+    SHELL_COMMAND_FILTER << block
+  end
+
+  def add_sh_filter_for(pattern, &block)
+    add_sh_filter {|cmd|
+      if cmd[0] =~ pattern
+        block.call(cmd)
+      else
+        cmd
+      end
+    }
+  end
+end

data/helper/util.rb

@@ -0,0 +1,19 @@
+def windows?
+  RUBY_PLATFORM =~ /win/
+end
+
+def with_temporary_rename(from, to)
+  mv from, to
+  yield
+ensure
+  mv to, from
+end
+
+def has_dot?
+  if windows?
+    v = `dot -V 2>&1` rescue nil
+    v and v =~ /Graphviz/
+  else
+    `which dot` =~ /\/dot/
+  end
+end

data/lib/livedoor/api/feed_discover/client.rb

@@ -0,0 +1,77 @@
+require 'open-uri'
+require 'cgi'
+require 'uri'
+
+begin
+  require 'json'
+rescue LoadError
+  require 'rubygems'
+  require 'json'
+end
+
+
+require 'livedoor/api/feed_discover/feed'
+require 'livedoor/api/feed_discover/feed_list'
+
+
+module Livedoor::API::FeedDiscover
+  class Client
+
+    BASE_URI = URI("http://rpc.reader.livedoor.com/feed/discover").freeze
+
+    def initialize(header = nil)
+      @http_header = (header || {}).dup
+    end
+
+    def http_header
+      @http_header.dup
+    end
+
+    def find(*args)
+      urls = (args || []).flatten
+      header = (Hash === urls.last ? urls.pop : {})
+      raise ArgumentError, "empty link" if urls.empty?
+      raise ArgumentError, "to many links `#{urls.size}' (max 1000)" if 1000 < urls.size
+      FeedList.new(discover_lazy(http_header.update(header), :links => urls.join("\n")))
+    end
+
+    def find_from(url, header = nil)
+      raise ArgumentError, "nil url" if url.nil?
+      header ||= {}
+      FeedList.new(discover_lazy(http_header.update(header), :url => url))
+    end
+
+    private
+
+    def discover_lazy(header, param)
+      {
+        :json => lambda { request_json(header, param) },
+        :opml => lambda { request_opml(header, param) }
+      }
+    end
+
+    def request_json(header, param)
+      JSON.parse(request("json", header, param))
+    end
+
+    def request_opml(header, param)
+      request("opml", header, param)
+    end
+
+    def request(format, header, param)
+      query = param.dup
+      query[:format] = format
+      get(build_uri(query), header)
+    end
+
+    def get(uri, header)
+      open(uri, header){|f| f.read }
+    end
+
+    def build_uri(query)
+      r = BASE_URI.dup
+      r.query = query.map {|k,v| "#{k}=#{CGI.escape(v.to_s)}"}.join('&')
+      r
+    end
+  end
+end