wuclan 0.2.0

data/examples/twitter/scrape_twitter_hosebird/edamame-killall

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+Daemonz::ProcTable.ps_emulation.find_all{|proc| proc.cmdline =~ %r{ruby \S+/test_spewer\.rb} }.each do |proc|
+      Process.kill 15, proc.pid.to_i
+    end

data/examples/twitter/scrape_twitter_hosebird/foo.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'sys/proctable'
+include Sys
+
+# Sys::ProcTable.ps{ |proc|
+#   puts proc.to_a.join("\t")
+# }
+
+process_exe = File.dirname(__FILE__)+'/test_spewer.rb'
+require 'ps_emulation'
+
+# p [Daemonz::ProcTable.ps_emulation, Sys::ProcTable.ps.first]
+
+p process_exe
+Daemonz::ProcTable.ps_emulation.each do |proc|
+  next unless (proc.cmdline =~ %r{ruby \S+/test_spewer\.rb})
+  puts proc.to_a.join("\t")
+end

data/examples/twitter/scrape_twitter_hosebird/ps_emulation.rb

@@ -0,0 +1,111 @@
+#
+#  This code taken from the daemonz gem
+#    http://github.com/costan/daemonz
+#  from the file lib/daemonz/process.rb:
+#    http://github.com/costan/daemonz/raw/master/lib/daemonz/process.rb
+#
+module Daemonz
+
+  # Mocks the sys-proctable gem using ps.
+  #
+  # This is useful even if sys-proctable is available, because it may fail for
+  # random reasons.
+  module ProcTable
+    #
+    # Pure-ruby equivalent to Sys::Proctable's ProcInfo struct.  This
+    # is somewhat wasteful, as we only fill in a few of the fields.
+    #
+    class FakeProcInfo < Struct.new(
+        :pid,    :ppid,      :pgid,    :ruid,    :rgid,    :comm,     :state,  :pctcpu,
+        :oncpu,  :tnum,      :tdev,    :wmesg,   :rtime,   :priority, :usrpri, :nice,
+        :cmdline,:starttime, :maxrss,  :ixrss,   :idrss,   :isrss,    :minflt, :majflt,
+        :nswap,  :inblock,   :oublock, :msgsnd,  :msgrcv,  :nsignals, :nvcsw,
+        :nivcsw, :utime,     :stime )
+    end
+
+    #
+    # Emulate the Sys::ProcTable.ps method by shelling out to
+    #
+    def self.ps_emulation
+      retval = []
+      ps_output = `ps ax`
+      ps_output.each_line do |pline|
+        pdata = pline.chomp.split(nil, 5)
+        pinfo = FakeProcInfo.new
+        pinfo.pid     = pdata[0].strip!
+        pinfo.cmdline = pdata[4].strip!
+        retval << pinfo
+      end
+      retval.shift
+      return retval
+    end
+
+
+    # ===========================================================================
+    #
+    # Know when to hold em and when to fold em:
+    #
+    # We want to use sys/proctable if it is available, succeeds, and has a
+    # complete implementation on our platform.
+    #
+    # * sys/proctable is broken on OSX (possibly just 10.5 Leopard): fake it.
+    # * if require 'sys/proctable' fails, then fake it.
+    # * Otherwise, as a first resort use sys/proctable;
+    # * but if the call fails, fake it.
+    #
+    def self.require_sys_proctable_or_die_trying!
+      begin
+        require 'sys/proctable'
+        require 'sys/uname'
+        return false if Sys::Uname.sysname =~ /Darwin/
+        return true
+      rescue Exception
+        return false
+      end
+    end
+
+    # ===========================================================================
+    #
+    # Define either the fake ProcTable.ps or a defensive-driver ProcTable.ps
+    #
+    unless self.require_sys_proctable_or_die_trying!
+      # The accelerated version is not available, use the slow version all the time.
+      def self.ps
+        self.ps_emulation
+      end
+    else
+      #
+      # If sys/proctable exists
+      #
+      def self.ps
+        # We don't use ps_emulation all the time because sys-proctable is
+        # faster. We only pay the performance penalty when sys-proctable fails.
+        begin            ; Sys::ProcTable.ps
+        rescue Exception ; self.ps_emulation ; end
+      end
+    end
+
+    # ===========================================================================
+    #
+    # returns information about a process or all the running processes
+    #
+    def self.process_info(pid = nil)
+      info = Hash.new
+      Daemonz::ProcTable.ps.each do |process|
+        item = { :cmdline => process.cmdline, :pid => process.pid.to_s }
+        if pid.nil?
+          info[process.pid.to_s] = item
+        else
+          return item if item[:pid].to_s == pid.to_s
+        end
+      end
+
+      if pid.nil?
+        return info
+      else
+        return nil
+      end
+    end
+
+  end
+end

data/examples/twitter/scrape_twitter_hosebird/scrape_twitter_hosebird.rb

@@ -0,0 +1,110 @@
+$: << File.dirname(__FILE__)+'/../../lib'
+$: << File.dirname(__FILE__)+'/../../../wukong/lib'
+$: << File.dirname(__FILE__)+'/../../../monkeyshines/lib'
+require 'edamame'
+require 'edamame/monitoring'
+require 'ps_emulation'
+
+class ScraperGod < GodProcess
+  DEFAULT_OPTIONS = {
+    :max_file_age      => 21600,               # 6 hours
+    :max_file_size     => 1000000,           # About 10% less than 128MB, so it should mostly fill one or two blocks on a hadoop file system
+    :max_file_size_interval => 10.seconds,
+    :process_exe       => File.dirname(__FILE__)+'/test_spewer.rb',
+  }
+  def self.default_options() super.deep_merge(ScraperGod::DEFAULT_OPTIONS)           ; end
+  def self.site_options()    super.deep_merge(global_site_options[:scraper_god]||{}) ; end
+  def self.kind
+    :scraper
+  end
+  def handle
+    self.class.kind.to_s + '_' + (options[:handle] || File.basename(options[:process_exe])).to_s
+  end
+
+  def mkdirs!
+    super()
+    FileUtils.mkdir_p options[:data_dir]
+    p options[:process_exe]
+  end
+
+  def setup_restart watcher
+    super(watcher)
+    watcher.restart_if do |restart|
+      restart.condition(:file_size_of_oldest_datestamped) do |c|
+        c.data_dir = options[:data_dir]
+        c.max_size = options[:max_file_size]
+        c.interval = options[:max_file_size_interval] if options[:max_file_size_interval]
+      end
+    end
+  end
+
+  def start_command
+    [
+      options[:process_exe],
+      "--handle=#{handle}",
+    ].flatten.compact.join(" ")
+  end
+  def restart_command() nil ; end
+  def stop_command()    nil ; end
+end
+
+module God
+  module Conditions
+
+    #
+    # a
+    #
+    class FileSize < PollCondition
+      attr_accessor :path, :max_size
+
+      def initialize
+        super
+        self.path = nil
+        self.max_size = nil
+      end
+
+      def valid?
+        valid = true
+        valid &= complain("Attribute 'path' must be specified", self)     if self.path.nil?
+        valid &= complain("Attribute 'max_size' must be specified", self) if self.max_size.nil?
+        valid
+      end
+
+      def test
+        return false if self.path.blank? || (! File.exists?(self.path))
+        File.size(self.path) >= self.max_size
+      end
+    end
+
+    #
+    # Look for the last-datestamped file in the last-datestamped directory
+    #
+    class FileSizeOfOldestDatestamped < FileSize
+      attr_accessor :data_dir
+
+      def initialize
+        super
+        self.data_dir = nil
+      end
+
+      def valid?
+        valid = super()
+        valid &= complain("Attribute 'data_dir' must be specified", self) if self.data_dir.nil?
+        valid
+      end
+
+      def path
+        last_dir = Dir[data_dir+'/[0-9]*'].
+          find_all{|dir| File.directory?(dir) && (File.basename(dir) =~ /^\d+$/)}.
+          sort.last
+        Dir["#{last_dir}/*"].sort.last || ''
+      end
+
+    end
+
+  end
+end
+
+WORK_DIR = File.dirname(__FILE__)+'/work'
+
+ScraperGod.create :data_dir => WORK_DIR

data/examples/twitter/scrape_twitter_hosebird/test_spewer.rb

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines'
+require 'wuclan/twitter' ; include Wuclan::Twitter::Scrape
+# Setup
+WORK_DIR = Subdir[__FILE__,'work'].expand_path.to_s
+Monkeyshines.load_global_options!
+Monkeyshines.load_cmdline_options!
+# Monkeyshines::CONFIG[:handle] = :test
+
+# set up a chunked store -- will save timestamped files
+# in the form :rootdir/:date/:handle+:timestamp-:pid.tsv",
+dest = Monkeyshines::Store::ChunkedFlatFileStore.new(:rootdir => WORK_DIR)
+
+# make those files fill up at about 40k/sec
+loop do
+  dest << ( Time.now.utc.to_s + " " + "*"*4064)
+  sleep 0.5
+end

data/examples/twitter/scrape_twitter_hosebird/twitter_hosebird_god.yaml

@@ -0,0 +1,10 @@
+:process_groups:
+  :twitter_hosebird:
+    :type:              :scraper
+    :max_file_age:      21600               # 6 hours
+    :max_file_size:     120000000           # About 10% less than 128MB, so it should mostly fill one or two blocks on a hadoop file system
+    :hosebird_level:    spritzer            # spritzer, gardenhose, etc.
+    :user_pass_file:    ...
+    :data_dir:          /data/ripd/com.tw/com.twitter.stream
+    :log_dir:           /data/log/com.tw/com.twitter.stream
+    :filename_pattern:  ":date/stream.twitter.com+:datetime-:pid-:hostname.json"

data/examples/twitter/scrape_twitter_search/dump_twitter_search_jobs.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines'
+require 'wuclan/twitter' ; include Wuclan::Twitter::Scrape
+require 'monkeyshines/utils/filename_pattern'
+# Setup
+WORK_DIR = Subdir[__FILE__,'work'].expand_path.to_s
+Monkeyshines.load_global_options!
+Monkeyshines.load_cmdline_options!
+Monkeyshines::CONFIG[:fetcher] = Monkeyshines::CONFIG[:twitter_api]
+include Edamame::Scheduling
+
+#
+# You can also specify these with --source-fetches on the command line
+#
+class TwitterScraper < Monkeyshines::Runner
+  def self.define_cmdline_options &block
+    super(&block)
+    yield(:dest_filename, "URI for scrape store to dump into", :type => String)
+  end
+end
+
+Monkeyshines::CONFIG[:dest][:filename] ||= Monkeyshines::Utils::FilenamePattern.new(":dest_dir/seed/jobs-:handle-:hostname-:timestamp.tsv", :dest_dir => WORK_DIR ).to_s
+default_tube = Monkeyshines::CONFIG[:handle].to_s.gsub(/\W+/,'').gsub(/_/,'-')
+
+source = Edamame::PersistentQueue.new( :tube => default_tube,
+  :queue => { :uris => ['localhost:11220'], },
+  :store => { :uri =>            ':11221',  }
+  )
+dest = Monkeyshines::Store::FlatFileStore.new :filename => Monkeyshines::CONFIG[:dest][:filename], :filemode => 'w'
+Log.info "Sucking from tube '#{default_tube}' on s#{source.options[:store][:uri]}/q#{source.options[:queue][:uris]}, dumping into #{dest.filename}"
+source.each do |job|
+#   dest << ( [Monkeyshines.url_decode(job.obj[:key])] + job.values_of(:priority, :prev_rate, :prev_items, :prev_span_min, :prev_span_max) )
+  # last_run = Time.parse(job.last_run)
+  last_run = job.last_run ? job.last_run.strftime("%Y%m%d%H%M%S") : nil
+  dest << ( [Monkeyshines.url_decode(job.obj[:key])] + job.values_of(:priority, :prev_items_rate, :prev_max, :prev_items, :delay) + [last_run] )
+end

data/examples/twitter/scrape_twitter_search/load_twitter_search_jobs.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines'
+require 'wuclan/twitter' ; include Wuclan::Twitter::Scrape
+# Setup
+WORK_DIR = Subdir[__FILE__,'work'].expand_path.to_s
+Monkeyshines.load_global_options!
+Monkeyshines.load_cmdline_options!
+Monkeyshines::CONFIG[:fetcher] = Monkeyshines::CONFIG[:twitter_api]
+
+include Edamame::Scheduling
+
+#
+# You can also specify these with --source-fetches on the command line
+#
+class TwitterScraper < Monkeyshines::Runner
+  def self.define_cmdline_options &block
+    super(&block)
+    yield(:source_filename, "URI for scrape store to load from", :type => String, :required => true)
+  end
+end
+
+default_tube = Monkeyshines::CONFIG[:handle].to_s.gsub(/\W+/,'').gsub(/_/,'-')
+p default_tube
+
+DEFAULT_PRIORITY        = 65536
+DEFAULT_TTR             = nil
+DEFAULT_RESERVE_TIMEOUT = 15
+IMMEDIATELY             = 0
+source = Monkeyshines::Store::FlatFileStore.new :filename => Monkeyshines::CONFIG[:source][:filename]
+dest = Edamame::PersistentQueue.new( :tube => default_tube,
+  :queue => { :uris => ['localhost:11220'], },
+  :store => { :uri =>            ':11221',  }
+  )
+source.each do |query_term, priority, prev_items_rate, prev_max, prev_items, delay, last_run|
+  # |query_term, priority, prev_rate, prev_items, prev_span_min, prev_span_max|
+  query_term.strip!
+  query_term       = Monkeyshines.url_encode(query_term)
+  priority         = priority.to_i
+  priority         = DEFAULT_PRIORITY     if (priority  == 0  )
+  prev_items_rate  = prev_items_rate.to_f
+  prev_items_rate  = nil                  if (prev_items_rate < 1e-6)
+  prev_max         = prev_max.to_i
+  prev_items       = prev_items.to_i
+  delay            = delay.to_f
+  last_run         = last_run ? DateTime.parse(last_run) : nil
+
+  twitter_search = { :type => 'TwitterSearchRequest', :key => query_term }
+
+  job = Edamame::Job.new(default_tube, priority, nil, 1,
+    Recurring.new(delay, prev_max, prev_items, prev_items_rate),
+    twitter_search
+    )
+  dest.put job, job.priority, IMMEDIATELY
+end
+
+
+# loop do
+#   job    = dest.reserve(DEFAULT_RESERVE_TIMEOUT) or break
+#   p [job, job.priority, job.scheduling, job.obj]
+#   # dest.delete job
+# end

data/examples/twitter/scrape_twitter_search/scrape_twitter_search.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines'
+require 'wuclan/twitter' ; include Wuclan::Twitter::Scrape
+# Setup
+WORK_DIR = Subdir[__FILE__,'work'].expand_path.to_s
+Monkeyshines.load_global_options!
+Monkeyshines.load_cmdline_options!
+Monkeyshines::CONFIG[:fetcher] = Monkeyshines::CONFIG[:twitter_api]
+
+#
+# * jobs stream from an edamame job queue.
+# * each job generates 1 to 15 paginated requests, stopping when a response
+#   overlaps the prev_max item.
+# * Each request is fetched with the standard HTTP fetcher.
+# * jobs are rescheduled based on the observed item rate
+# * results are sent to a ChunkedFlatFileStore
+#
+
+#
+# Run scraper
+#
+loop do
+  begin
+    #
+    # Create scraper
+    #
+    scraper = Monkeyshines::Runner.new({
+        :log     => { :iters => 600, :time => 150, :dest => nil }, # Monkeyshines::CONFIG[:handle]
+        :source  => { :type  => TwitterSearchRequestStream, :queue_request_timeout => (10 * 60),
+          :queue => { :uris => ['localhost:11220'], },
+          :store => { :uri =>            ':11221',  }, },
+        :dest    => { :type  => :chunked_flat_file_store, :rootdir => WORK_DIR, :filemode => 'a' },
+        # :dest    => { :type  => :flat_file_store, :filename => WORK_DIR+"/test_output.tsv" },
+        # :fetcher => { :type => TwitterSearchFakeFetcher },
+        :sleep_time  => 1 ,
+      })
+    Log.info "Starting a run!"
+    scraper.run
+  rescue Beanstalk::NotFoundError => e
+    warn e
+  end
+end

data/examples/twitter/scrape_twitter_search/twitter_search_daemons.god

@@ -0,0 +1,25 @@
+$: << File.dirname(__FILE__)+'/../../../../edamame/lib'
+require 'edamame/monitoring'
+WORK_DIR = File.dirname(__FILE__)+'/work'
+
+#
+# For debugging:
+#
+#   sudo god -c this_file.god -D
+#
+# (for production, use the etc/initc.d script in this directory)
+#
+# TODO: define an EdamameDirector that lets us name these collections.
+#
+THE_FAITHFUL = [
+  # twitter_search
+  [BeanstalkdGod, { :port => 11240, :max_mem_usage => 100.megabytes,  }],
+  [TyrantGod,     { :port => 11241, :db_dirname => WORK_DIR, :db_name => "twitter_search-queue.tct" }],
+  #
+  # [TyrantGod,     { :port => 11249, :db_dirname => WORK_DIR, :db_name => "twitter_search-flat.tct" }],
+]
+
+THE_FAITHFUL.each do |klass, config|
+  proc = klass.create(config.merge :flapping_notify => 'default')
+  proc.mkdirs!
+end

data/lib/old/twitter_api.rb

@@ -0,0 +1,88 @@
+module TwitterApi
+  #
+  # The URI for a given resource
+  #
+  def gen_url
+    case context
+    when :user, :followers, :friends, :favorites, :timeline
+      "http://twitter.com/#{resource_path}/#{identifier}.json?page=#{page}"
+    when :followers_ids, :friends_ids
+      "http://twitter.com/#{resource_path}/#{identifier}.json"
+    when :user_timeline
+      "http://twitter.com/#{resource_path}/#{identifier}.json?page=#{page}&count=200"
+    # when :public_timeline
+    # when :search
+    #  "http://search.twitter.com/search.json?q=#{query}"
+    else
+      raise "Don't know how to retrieve #{context} yet"
+    end
+  end
+
+  # Regular expression to grok resource from uri
+  GROK_URI_RE = %r{http://twitter.com/(\w+/\w+)/(\w+)\.json\?page=(\d+)}
+
+  # Context <=> resource mapping
+  #
+  # aka. repairing the non-REST uri's
+  RESOURCE_PATH_FROM_CONTEXT = {
+    :user            => 'users/show',
+    :followers_ids   => 'followers/ids',
+    :friends_ids     => 'friends/ids',
+    :followers       => 'statuses/followers',
+    :friends         => 'statuses/friends',
+    :favorites       => 'favorites',
+    :timeline        => 'statuses/user_timeline',
+    :user_timeline   => 'statuses/user_timeline',
+    :public_timeline => 'statuses/public_timeline'
+  }
+  # Get url resource for context
+  def resource_path
+    RESOURCE_PATH_FROM_CONTEXT[context.to_sym]
+  end
+
+  def self.pages_from_count per_page, count, max=nil
+    num = [ (count.to_f / per_page.to_f).ceil, 0 ].max
+    [num, max].compact.min
+  end
+  def self.pages context, thing
+    case context
+    when :favorites       then pages_from_count( 20, thing.favourites_count, 20)
+    when :friends         then pages_from_count(100, thing.friends_count,    10)
+    when :followers       then pages_from_count(100, thing.followers_count,  10)
+    when :followers_ids   then thing.followers_count == 0 ? 0 : 1
+    when :friends_ids     then thing.friends_count   == 0 ? 0 : 1
+    when :user            then 1
+    when :public_timeline then 1
+    when :user_timeline   then pages_from_count(200, thing.statuses_count,   20)
+    when :search          then pages_from_count(100, 1500)
+    else raise "need to define pages for context #{context}"
+    end
+  end
+
+  module ClassMethods
+    # Get context from url resource
+    def context_for_resource(resource)
+      RESOURCE_PATH_FROM_CONTEXT.invert[resource] or raise("Wrong resource specification #{resource}")
+    end
+  end
+
+  def self.included base
+    base.extend ClassMethods
+  end
+end
+
+# language: http://en.wikipedia.org/wiki/ISO_639-1
+#
+# * Find tweets containing a word:         http://search.twitter.com/search.atom?q=twitter
+# * Find tweets from a user:               http://search.twitter.com/search.atom?q=from%3Aalexiskold
+# * Find tweets to a user:                 http://search.twitter.com/search.atom?q=to%3Atechcrunch
+# * Find tweets referencing a user:        http://search.twitter.com/search.atom?q=%40mashable
+# * Find tweets containing a hashtag:      http://search.twitter.com/search.atom?q=%23haiku
+# * Combine any of the operators together: http://search.twitter.com/search.atom?q=movie+%3A%29
+#
+# * lang:      restricts tweets to the given language, given by an ISO 639-1 code. Ex: http://search.twitter.com/search.atom?lang=en&q=devo
+# * rpp:       the number of tweets to return per page, up to a max of 100. Ex: http://search.twitter.com/search.atom?lang=en&q=devo&rpp=15
+# * page:      the page number (starting at 1) to return, up to a max of roughly 1500 results (based on rpp * page)
+# * since_id:  returns tweets with status ids greater than the given id.
+# * geocode:   returns tweets by users located within a given radius of the given latitude/longitude, where the user's location is taken from their Twitter profile. The parameter value is specified by "latitide,longitude,radius", where radius units must be specified as either "mi" (miles) or "km" (kilometers). Ex: http://search.twitter.com/search.atom?geocode=40.757929%2C-73.985506%2C25km. Note that you cannot use the near operator via the API to geocode arbitrary locations; however you can use this geocode parameter to search near geocodes directly.
+# * show_user: when "true", adds "<user>:" to the beginning of the tweet. This is useful for readers that do not display Atom's author field. The default is "false".

data/lib/wuclan/delicious/delicious_html_request.rb

@@ -0,0 +1,31 @@
+module Wuclan
+  module Delicious
+    module DeliciousRequest
+    end
+
+    # Recent bookmarks by tag:                                                    http://delicious.com/tag/{tag[+tag+...+tag]}?detail=3
+    # Popular bookmarks by tag:                                                   http://delicious.com/popular/{tag}?detail=3
+    # Bookmarks for a specific user by tag(s):                                    http://delicious.com/{username}/{tag[+tag+...+tag]}?detail=3
+    # Bookmarks for a specific user:                                              http://delicious.com/{username}?detail=3
+    class TagRequest
+      attr_accessor :scope
+      def initialize scope
+      end
+
+      # pages
+      # count
+    end
+
+    # A list of all public tags for a user:                                       http://delicious.com/tags/{username}?view=all
+    class UserTagsRequest
+    end
+
+    # Recent bookmarks for a URL:                                                 http://delicious.com/url/{url md5}
+    class UrlInfoRequest
+    end
+
+    # A list of a user's network members:                                         http://delicious.com/network/{username}
+    class FriendsFollowersRequest
+    end
+  end
+end

data/lib/wuclan/delicious/delicious_models.rb

@@ -0,0 +1,26 @@
+module Wuclan
+  module Models
+
+    #
+    # Models for the delicious.com (formerly del.icio.us) social network
+    #
+    # Link:         has tags,   tagged by socialites
+    # Socialite:                describes links with tabs,  uses tags,         follows/followedby socialites
+    # Tag:          tags links,                             used by socialites
+
+    class DeliciousLink < Struct.new(
+        :delicious_link_id, :url, :title, :taggers_count)
+    end
+    class DeliciousTag < Struct.new(
+        :name )
+    end
+    class DeliciousUser < Struct.new(
+        :id, :scraped_at, :screen_name, :protected, :followers_count, :friends_count, :taggings_count, :name, :description, :bio_url )
+    end
+
+    class DeliciousTagging < Struct.new(
+        :tag_name, :delicious_link_id, :screen_name, :created_at, :text, :description)
+    end
+
+  end
+end

data/lib/wuclan/delicious/delicious_request.rb

@@ -0,0 +1,65 @@
+module Wuclan
+  module Delicious
+    module DeliciousRequest
+    end
+
+    # Recent bookmarks by tag:                                                    http://feeds.delicious.com/v2/{format}/tag/{tag[+tag+...+tag]}
+    # Popular bookmarks by tag:                                                   http://feeds.delicious.com/v2/{format}/popular/{tag}
+    # Bookmarks for a specific user by tag(s):                                    http://feeds.delicious.com/v2/{format}/{username}/{tag[+tag+...+tag]}
+    # Bookmarks for a specific user:                                              http://feeds.delicious.com/v2/{format}/{username}
+    class TagRequest
+      attr_accessor :scope
+      def initialize scope
+      end
+
+      # pages
+      # count
+    end
+
+    # A list of all public tags for a user:                                       http://feeds.delicious.com/v2/{format}/tags/{username}
+    class UserTagsRequest
+    end
+
+    # Public summary information about a user (as seen in the network badge):     http://feeds.delicious.com/v2/{format}/userinfo/{username}
+    class UserInfoRequest
+    end
+
+    # Recent bookmarks for a URL:                                                 http://feeds.delicious.com/v2/{format}/url/{url md5}
+    class UrlInfoRequest
+    end
+
+    # A list of a user's network members:                                         http://feeds.delicious.com/v2/{format}/networkmembers/{username}
+    class FollowersRequest
+    end
+
+    # A list of a user's network fans:                                            http://feeds.delicious.com/v2/{format}/networkfans/{username}
+    class FriendsRequest
+    end
+  end
+end
+
+
+# Recent bookmarks by tag:                                                        http://feeds.delicious.com/v2/{format}/tag/{tag[+tag+...+tag]}
+# Popular bookmarks by tag:                                                       http://feeds.delicious.com/v2/{format}/popular/{tag}
+# Bookmarks for a specific user:                                                  http://feeds.delicious.com/v2/{format}/{username}
+# Bookmarks for a specific user by tag(s):                                        http://feeds.delicious.com/v2/{format}/{username}/{tag[+tag+...+tag]}
+# Public summary information about a user (as seen in the network badge):         http://feeds.delicious.com/v2/{format}/userinfo/{username}
+# A list of all public tags for a user:                                           http://feeds.delicious.com/v2/{format}/tags/{username}
+# Recent bookmarks for a URL:                                                     http://feeds.delicious.com/v2/{format}/url/{url md5}
+# A list of a user's network members:                                             http://feeds.delicious.com/v2/{format}/networkmembers/{username}
+# A list of a user's network fans:                                                http://feeds.delicious.com/v2/{format}/networkfans/{username}
+
+# Bookmarks from the hotlist:                                                     http://feeds.delicious.com/v2/{format}
+# Recent bookmarks:                                                               http://feeds.delicious.com/v2/{format}/recent
+# Popular bookmarks:                                                              http://feeds.delicious.com/v2/{format}/popular
+# Recent site alerts (as seen in the top-of-page alert bar on the site):          http://feeds.delicious.com/v2/{format}/alerts
+# Bookmarks from a user's subscriptions:                                          http://feeds.delicious.com/v2/{format}/subscriptions/{username}
+# Bookmarks from members of a user's network:                                     http://feeds.delicious.com/v2/{format}/network/{username}
+# Bookmarks from members of a user's network by tag:                              http://feeds.delicious.com/v2/{format}/network/{username}/{tag[+tag+...+tag]}
+# Summary information about a URL (as seen in the tagometer):                     http://feeds.delicious.com/v2/json/urlinfo/{url md5}
+
+# Private bookmarks for a specific user:                                          http://feeds.delicious.com/v2/{format}/{username}?private={key}
+# Private bookmarks for a specific user by tag(s):                                http://feeds.delicious.com/v2/{format}/{username}/{tag[+tag+...+tag]}?private={key}
+# Private feed for a user's inbox bookmarks from others:                          http://feeds.delicious.com/v2/{format}/inbox/{username}?private={key}
+# Bookmarks from members of a user's private network:                             http://feeds.delicious.com/v2/{format}/network/{username}?private={key}
+# Bookmarks from members of a user's private network by tag:                      http://feeds.delicious.com/v2/{format}/network/{username}/{tag[+tag+...+tag]}?private={key}