wuclan 0.2.0

data/lib/wuclan/friendfeed/scrape/friendfeed_search_request.rb

@@ -0,0 +1,60 @@
+# module Wuclan
+#     module Twitter
+#       module Scrape
+#         #
+#         # ScrapeRequest for the twitter Search API.
+#         #
+#         # Examines the parsed contents to describe result
+#         #
+#         class TwitterSearchRequest < Monkeyshines::ScrapeRequest
+#           include Monkeyshines::RawJsonContents
+#           # Extract the actual search items returned
+#           def items
+#             parsed_contents['results'] if parsed_contents
+#           end
+#           # Checks that the response parses and has the right data structure.
+#           # if healthy? is true things should generally work
+#           def healthy?
+#             items && items.is_a?(Array)
+#           end
+#           # Number of items returned in this request
+#           def num_items()
+#             items ? items.length : 0
+#           end
+#           # Span of IDs. Assumes the response has the ids in sort order oldest to newest
+#           # (which the twitter API provides)
+#           def span
+#             [items.last['id'], items.first['id']] rescue nil
+#           end
+#           # Span of created_at times covered by this request.
+#           # Useful for rate estimation.
+#           def timespan
+#             [Time.parse(items.last['created_at']), Time.parse(items.first['created_at'])] rescue nil
+#           end
+#         end
+#       end
+#     end
+#   end
+# end
+
+
+
+# class FriendFeedSearchFetcher
+#   def search_url query, page, max_id
+#     start = (page-1)*100
+#     %Q{http://friendfeed.com/api/feed/search?q=#{query}&service=twitter&start=#{start}&num=100}
+#   end
+#   def max_results_per_page
+#     100
+#   end
+#   def max_page
+#     4
+#   end
+#   def terminate? response, page
+#     terminate = super || page >= max_page
+#   end
+#   def get_results response
+#     response['entries'] if response
+#   end
+# end
+

data/lib/wuclan/friendster.rb

@@ -0,0 +1,7 @@
+require 'monkeyshines/scrape_request/raw_json_contents'
+module Wuclan
+  module Friendster
+    autoload :Scrape, 'wuclan/friendster/scrape.rb'
+    autoload :Parse,  'wuclan/friendster/parse.rb'
+  end
+end

data/lib/wuclan/lastfm/model/base.rb

@@ -0,0 +1,49 @@
+require 'monkeyshines/scrape_request/raw_json_contents'
+module Wuclan
+  module Lastfm
+
+    class LastfmUser
+    end
+
+    class LastfmShout
+    end
+
+    class LastfmArtist
+    end
+
+    class LastfmAlbum
+    end
+
+    class LastfmTrack
+    end
+
+    class LastfmGroup
+    end
+
+    class LastfmTag
+    end
+
+    class LastfmEvent
+    end
+
+    class LastfmVenue
+    end
+
+    # Interactions
+
+    class AShoutsB
+    end
+
+    # Relationships
+
+    class LastfmTagging < Struct.new(:user_id, :taggable_type, :taggable_id)
+    end
+
+    class LastfmListen
+    end
+
+    class LastfmAttending
+    end
+
+  end
+end

data/lib/wuclan/lastfm/model/sample_responses.txt

@@ -0,0 +1,16 @@
+// lastfm_artist_info_request	Linkin%20Park	1		http://ws.audioscrobbler.com/2.0/?method=Artist.getInfo&artist=Linkin%20Park&limit=100&page=1&api_key=12717a7e1a0ec8a053026b813fd59a7c&format=json	20090817020535	200	OK	
+{ "artist": {
+    "name":   "Linkin Park",
+    "mbid":   "f59c5520-5f46-4d2c-b2c4-822eabf53419",
+    "stats":  {"listeners":"1473188","playcount":"93021323"},
+    "url":    "http:\/\/www.last.fm\/music\/Linkin+Park",
+    "tags":   {"tag":[{"name":"rock","url":"http:\/\/www.last.fm\/tag\/rock"},{"name":"nu metal","url":"http:\/\/www.last.fm\/tag\/nu%20metal"},{"name":"alternative","url":"http:\/\/www.last.fm\/tag\/alternative"},{"name":"alternative rock","url":"http:\/\/www.last.fm\/tag\/alternative%20rock"},{"name":"metal","url":"http:\/\/www.last.fm\/tag\/metal"}]},
+    "bio":    {"published":"Thu, 2 Jul 2009 21:19:19 +0000","summary":"Linkin Park is a band from Agoura Hills, California. The band consists of Joe &quot;<a href=\"http:\/\/ws.audioscrobbler.com\/music\/Mr.+Hahn\" class=\"bbcode_artist\">Mr. Hahn<\/a>&quot; Hahn, <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Mike+Shinoda\" class=\"bbcode_artist\">Mike Shinoda<\/a>, Dave Farrell (a.k.a. Phoenix), <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Chester+Bennington\" class=\"bbcode_artist\">Chester Bennington<\/a>, <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Brad+Delson\" class=\"bbcode_artist\">Brad Delson<\/a> and <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Rob+Bourdon\" class=\"bbcode_artist\">Rob Bourdon<\/a>.  Released under the band name &quot;Hybrid Theory&quot;, the band's first album, <a title=\"Linkin Park - Hybrid Theory EP\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/Hybrid+Theory+EP\" class=\"bbcode_album\">Hybrid Theory EP<\/a> was released in 1995. Only 1000 copies of this very rare original EP were made.  The band's first full studio album, <a title=\"Linkin Park - Hybrid Theory\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/Hybrid+Theory\" class=\"bbcode_album\">Hybrid Theory<\/a>, released in 2001, had a huge success, selling 42 million copies worldwide as of October 2009.","content":"<strong>Linkin Park<\/strong> is a band from Agoura Hills, California. The band consists of Joe &quot;<a href=\"http:\/\/ws.audioscrobbler.com\/music\/Mr.+Hahn\" class=\"bbcode_artist\">Mr. Hahn<\/a>&quot; Hahn, <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Mike+Shinoda\" class=\"bbcode_artist\">Mike Shinoda<\/a>, Dave Farrell (a.k.a. Phoenix), <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Chester+Bennington\" class=\"bbcode_artist\">Chester Bennington<\/a>, <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Brad+Delson\" class=\"bbcode_artist\">Brad Delson<\/a> and <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Rob+Bourdon\" class=\"bbcode_artist\">Rob Bourdon<\/a>.\n \n Released under the band name &quot;Hybrid Theory&quot;, the band's first album, <a title=\"Linkin Park - Hybrid Theory EP\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/Hybrid+Theory+EP\" class=\"bbcode_album\">Hybrid Theory EP<\/a> was released in 1995. Only 1000 copies of this very rare original EP were made.\n \n The band's first full studio album, <a title=\"Linkin Park - Hybrid Theory\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/Hybrid+Theory\" class=\"bbcode_album\">Hybrid Theory<\/a>, released in 2001, had a huge success, selling 42 million copies worldwide as of October 2009. Among its singles were <a title=\"Linkin Park &ndash; One Step Closer\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/One+Step+Closer\" class=\"bbcode_track\">One Step Closer<\/a>, <a title=\"Linkin Park &ndash; Crawling\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/Crawling\" class=\"bbcode_track\">Crawling<\/a>, and the band's highest-selling single, <a title=\"Linkin Park &ndash; In the End\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/In+the+End\" class=\"bbcode_track\">In the End<\/a>. In 2001, the band released &quot;Frat party at the pancake festival&quot;. A dvd with live stuff, extra footage of the band and the music video's from Hybrid Theory. A remix album, <a title=\"Linkin Park - Reanimation\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/Reanimation\" class=\"bbcode_album\">Reanimation<\/a> was released in 2002, and was based around Hybrid Theory songs.\n \n The band's second studio album, <a title=\"Linkin Park - Meteora\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/Meteora\" class=\"bbcode_album\">Meteora<\/a> (released 2003), was not so successful as &quot;Hybrid Theory&quot;. Still, it sold 10 million copies worldwide, spawned the worldwide hit <a title=\"Linkin Park &ndash; Breaking the Habit\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/Breaking+the+Habit\" class=\"bbcode_track\">Breaking the Habit<\/a>, and was the first Linkin Park album that was not fully rap driven.\n \n A limited edition live album and DVD came next: <a title=\"Linkin Park - Live in Texas\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/Live+in+Texas\" class=\"bbcode_album\">Live in Texas<\/a>. It was an opportunity for fans who have not been to the band's gigs to see their performance live. \n \n Linkin Park and <a href=\"http:\/\/ws.audioscrobbler.com\/music\/Jay-Z\" class=\"bbcode_artist\">Jay-Z<\/a> then released <a title=\"Jay-Z and Linkin Park - Collision Course\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Jay-Z+and+Linkin+Park\/Collision+Course\" class=\"bbcode_album\">Collision Course<\/a> in which previously-released songs from the two artists were mixed together. Some time after this was a major label attempt to combat the success of illegal <a href=\"http:\/\/ws.audioscrobbler.com\/tag\/mashup\" class=\"bbcode_tag\" rel=\"tag\">mashup<\/a> albums popular in 2004 and 2005, most of which also used Jay-Z's <a title=\"Jay-Z - Black Album\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Jay-Z\/Black+Album\" class=\"bbcode_album\">Black Album<\/a> a cappellas. The album introduced fans of both artists to each other and marked the beginning of Jay-Z's return from retirement.\n \n The band's newest album is <a title=\"Linkin Park - Minutes To Midnight\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/Minutes+To+Midnight\" class=\"bbcode_album\">Minutes To Midnight<\/a>, often abbreviated MTM. The album was co-produced by Mike Shinoda and legendary Grammy wining producer Rick Rubin (Slayer, Red Hot Chili Peppers, Johnny Cash) The release date, originally somewhere between July and September of 2006, was pushed to May of 2007. The band cites their reasoning as having fun experimenting with new styles and sounds in the studio, saying it was their most fun recording session to date. The new album was finally released on iTunes on May 11th 2007 and in stores worldwide three days later. The sounds are new, more hip-hop based beats and rock song dynamics. It is 12 tracks long, including the band's 6th consecutive #1 hit, <a title=\"Linkin Park &ndash; What I've Done\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/What+I%27ve+Done\" class=\"bbcode_track\">What I've Done<\/a>. The video for the song was directed by Joseph Hahn.\n \n Four more singles from Minutes to Midnight were released: \n <a title=\"Linkin Park &ndash; Bleed It Out\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/Bleed+It+Out\" class=\"bbcode_track\">Bleed It Out<\/a> (released August 2007), <a title=\"Linkin Park &ndash; Shadow of the Day\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/Shadow+of+the+Day\" class=\"bbcode_track\">Shadow of the Day<\/a> (released October 2007), <a title=\"Linkin Park &ndash; Given Up\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/Given+Up\" class=\"bbcode_track\">Given Up<\/a> (released March 2008), and the final single, <a title=\"Linkin Park &ndash; Leave Out All the Rest\" href=\"http:\/\/ws.audioscrobbler.com\/music\/Linkin+Park\/_\/Leave+Out+All+the+Rest\" class=\"bbcode_track\">Leave Out All the Rest<\/a> (released June 2008).\n \n The band began touring again in late April in 2007 with a show in Berlin, and then continuing through Europe. 2007 also marks the return of the Projekt Revolution tour, which is always headlined by Linkin Park.\n \n On 29 June 2008, the band played their biggest Projekt Revolution show to date at the Milton Keynes Bowl, the venue Green Day's 2005 live DVD was recorded at (a 60,000+ capacity venue). The PR UK lineup included Jay-Z who again performed some &quot;Collision Course&quot; tracks with Linkin Park. Other bands included Pendulum, N*E*R*D, Enter Shikari, The Bravery and Innerpartysystem.\n \n The band's second live CD\/DVD, Road To Revolution: Live At Milton Keynes, was released November 24th 2008. It included at least one song of each Linkin Park album including Collision Course with Jay-Z and featured excerpts from Fort Minor's The Rising Tied in Points Of Authority. "}}}
+    "image":  [{"#text":"http:\/\/userserve-ak.last.fm\/serve\/34\/21871115.jpg","size":"small"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/64\/21871115.jpg","size":"medium"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/126\/21871115.jpg","size":"large"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/252\/21871115.jpg","size":"extralarge"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/_\/21871115\/Linkin+Park.jpg","size":"mega"}],"streamable":"1",
+    "similar": {
+      "artist":[
+        {"name":"Fort Minor","url":"http:\/\/www.last.fm\/music\/Fort+Minor","image":[{"#text":"http:\/\/userserve-ak.last.fm\/serve\/34\/108914.jpg","size":"small"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/64\/108914.jpg>","size":"medium"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/126\/108914.jpg","size":"large"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/252\/108914.jpg","size":"extralarge"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/500\/108914\/Fort+Minor.jpg","size":"mega"}]},
+        {"name":"Jay-Z and Linkin Park","url":"http:\/\/www.last.fm\/music\/Jay-Z+and+Linkin+Park","image":[{"#text":"http:\/\/userserve-ak.last.fm\/serve\/34\/16170933.jpg","size":"small"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/64\/16170933.jpg>","size":"medium"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/126\/16170933.jpg","size":"large"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/252\/16170933.jpg","size":"extralarge"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/500\/16170933\/JayZ+and+Linkin+Park+LP++JAYZ.jpg","size":"mega"}]},
+        {"name":"Chester Bennington","url":"http:\/\/www.last.fm\/music\/Chester+Bennington","image":[{"#text":"http:\/\/userserve-ak.last.fm\/serve\/34\/33909.jpg","size":"small"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/64\/33909.jpg>","size":"medium"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/126\/33909.jpg","size":"large"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/252\/33909.jpg","size":"extralarge"},{"#text":"","size":"mega"}]},
+        {"name":"Xero","url":"http:\/\/www.last.fm\/music\/Xero","image":[{"#text":"http:\/\/userserve-ak.last.fm\/serve\/34\/8582695.jpg","size":"small"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/64\/8582695.jpg>","size":"medium"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/126\/8582695.jpg","size":"large"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/252\/8582695.jpg","size":"extralarge"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/500\/8582695\/Xero.jpg","size":"mega"}]},
+        {"name":"Papa Roach","url":"http:\/\/www.last.fm\/music\/Papa+Roach", "image":[{"#text":"http:\/\/userserve-ak.last.fm\/serve\/34\/23013493.jpg","size":"small"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/64\/23013493.jpg>","size":"medium"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/126\/23013493.jpg","size":"large"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/252\/23013493.jpg","size":"extralarge"},{"#text":"http:\/\/userserve-ak.last.fm\/serve\/500\/23013493\/Papa+Roach+lifeline_presspic.jpg","size":"mega"}]}]},

data/lib/wuclan/lastfm/scrape/base.rb

@@ -0,0 +1,195 @@
+require 'monkeyshines/scrape_request/raw_json_contents'
+module Wuclan
+  module Lastfm
+    module Scrape
+
+      #
+      # Base class for Lastfm API requests
+      #
+      #
+      # For parsing, note the different form for empty responses.
+      #
+      class Base < TypedStruct.new(
+          [:identifier,       Integer],
+          [:page,             Integer],
+          [:moreinfo,         String],
+          [:url,              String],
+          [:scraped_at,       Bignum],
+          [:response_code,    Integer],
+          [:response_message, String],
+          [:contents,         String]
+          )
+        # Basic ScrapeRequest functionality
+        include Monkeyshines::ScrapeRequestCore
+        # Contents are JSON
+        include Monkeyshines::RawJsonContents
+        # API Authentication
+        cattr_accessor :api_key
+        self.api_key = Monkeyshines::CONFIG[:lastfm_api][:api_key] rescue nil
+
+        #
+        # call with fields in order:
+        #
+        #    identifier page moreinfo url scraped_at response_code response_message contents
+        #
+        # you only have to fill in what you know -- in fact, it's typical to
+        # pass in only the identifier
+        def initialize *args
+          super *args
+          self.page = (page.to_i < 1 ? 1 : page.to_i)
+        end
+
+        # ===========================================================================
+        #
+        # API
+        #
+
+        # Slug for the URL to let lastfm know what resource is requested
+        class_inheritable_accessor :api_method
+        # Responses are a JSON hash with one
+        class_inheritable_accessor :main_fieldname
+        class_inheritable_accessor :response_type
+
+        #
+        # Generate request URL from other attributes
+        #
+        def make_url
+          [ "http://ws.audioscrobbler.com/2.0/?method=#{api_method}#{identifier}",
+            "&limit=#{max_items}&api_key=#{self.class.api_key}&format=json",
+            "&page=#{page}" ].join("")
+        end
+
+        #
+        # Call with a hash of identifier attr-val pairs (including an optional page)
+        #
+        # Ex.
+        #     LastfmAlbumInfoRequest.from_identifier_hash(
+        #       :album => 'Rum+Sodomy+%26+the+Lash',
+        #       :artist => 'The+Pogues',
+        #       :mbid => 'ba1f6641-9085-36a0-8962-65ad6e48afd1')
+        # Calls
+        #     LastfmAlbumInfoRequest.new(
+        #       'album=Rum+Sodomy+%26+the+Lash&artist=The+Pogues&mbid=ba1f6641-9085-36a0-8962-65ad6e48afd1')
+        #
+        # FIXME -- this is stupid in taking a special 'id' and a just-as-necessary hash
+        #
+        def self.from_identifier_hash name, hsh={}
+          name = url_encode(name)
+          page = hsh.delete(:page)
+          rest = self.class.make_url_query(hsh)
+          new("#{name}&#{rest}", page)
+        end
+
+        # ===========================================================================
+        #
+        # Parsing
+        #
+
+        # Healthy if it's well-formed and not an error
+        def healthy?
+          super && ( (contents !~ %r{^\{"error":}) )
+        end
+
+        #
+        # Last.fm responses typically have the form
+        #
+        #   { "requested_objs": {
+        #       "requested_type": [ { ... obj ... }, { ... obj ... } ... ],
+        #       "@attr":{ "key": "val" }
+        #     }
+        #   }
+        #
+        # e.g.
+        #
+        #   {"similarartists":{
+        #      "artist": [
+        #        ...
+        #        {"name":"Pillar","mbid":"..","match":"5.66",..."streamable":"1"}],
+        #     "@attr":{"artist":"Dead by Sunrise"}
+        #     }}
+        #
+        def main_result
+          return @main_result if @main_result
+          return unless healthy?
+          @main_result = parsed_contents[main_fieldname] rescue {}
+        end
+
+        #
+        # Information about the request itself.
+        #
+        # (see doc for result_attrs)
+        def result_attrs
+          main_result["@attr"] rescue {}
+        end
+
+        # Extract the actual search items returned
+        def items
+          # [obj_or_array].flatten makes this always be an array (lastfm gives
+          # the single object when it's not an array)
+          [main_result[response_type]].flatten.compact rescue []
+        end
+
+        #
+        # Pagination and Rescheduling
+        #
+        # Pagination
+        include Monkeyshines::ScrapeRequestCore::Paginated
+        include Monkeyshines::ScrapeRequestCore::Paginating
+        include Monkeyshines::ScrapeRequestCore::PaginatedWithLimit
+        # include Monkeyshines::ScrapeRequestCore::PaginatedWithRate
+
+        # API max items per response
+        self.max_items = 50
+        # API max pages
+        self.hard_request_limit = 15
+        # Items to get each re-visit. If there are up to 50 items per page,
+        # target_items_per_job of 1000 will try to reschedule so that its return visit
+        # makes about twenty page requests.
+        # #self.target_items_per_job = 150
+
+        # creates the paginated request
+        def request_for_page page, pageinfo=nil
+          (page.to_i > 1) ? self.class.new(identifier, page) : self
+        end
+
+        #
+        # Max pages for this resource
+        #
+        def max_pages
+          return 1 if (result_attrs.blank? || result_attrs['totalPages'].blank?)
+          attrs['totalPages'].to_i.clamp(1,self.hard_request_limit)
+        end
+
+        #
+        # For stream-type objects
+        #
+
+        # Span of IDs. Assumes the response has the ids in sort order oldest to newest
+        # (which the twitter API provides)
+        def span
+          [items.last['id'], items.first['id']] rescue nil
+        end
+        # Span of created_at times covered by this request.
+        # Useful for rate estimation.
+        def timespan
+          [Time.parse(items.last['created_at']).utc, Time.parse(items.first['created_at']).utc] rescue nil
+        end
+
+        class_inheritable_accessor :requestables
+        self.requestables = []
+        def recursive_requests &block
+          return unless healthy?
+          #
+          # requestables
+          #
+          requestables.each do |klass|
+            req = klass.new(identifier)
+            req.generation = generation.to_i + 1
+            yield req
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/wuclan/lastfm/scrape/concrete.rb

@@ -0,0 +1,143 @@
+module Wuclan
+  module Lastfm
+    module Scrape
+
+      #
+      # Defines the concrete terms of the API
+      #
+
+      class LastfmAlbumInfoRequest              < Base ; self.api_method = 'album.getinfo&album='              ; end
+      class LastfmAlbumTagsRequest              < Base ; self.api_method = 'album.gettags&album='              ; end
+      class LastfmArtistEventsRequest           < Base ; self.api_method = 'artist.getevents&artist='          ; end
+      class LastfmArtistImagesRequest           < Base ; self.api_method = 'artist.getimages&artist='          ; end
+      class LastfmArtistInfoRequest             < Base ; self.api_method = 'artist.getinfo&artist='            ; end
+      class LastfmArtistPodcastRequest          < Base ; self.api_method = 'artist.getpodcast&artist='         ; end
+      class LastfmArtistShoutsRequest           < Base ; self.api_method = 'artist.getshouts&artist='          ; end
+      class LastfmArtistSimilarRequest          < Base ; self.api_method = 'artist.getsimilar&artist='         ; end
+      class LastfmArtistTagsRequest             < Base ; self.api_method = 'artist.gettags&artist='            ; end
+      class LastfmArtistTopAlbumsRequest        < Base ; self.api_method = 'artist.gettopalbums&artist='       ; end
+      class LastfmArtistTopFansRequest          < Base ; self.api_method = 'artist.gettopfans&artist='         ; end
+      class LastfmArtistTopTagsRequest          < Base ; self.api_method = 'artist.gettoptags&artist='         ; end
+      class LastfmArtistTopTracksRequest        < Base ; self.api_method = 'artist.gettoptracks&artist='       ; end
+      class LastfmEventAttendeesRequest         < Base ; self.api_method = 'event.getattendees&event='         ; end
+      class LastfmEventInfoRequest              < Base ; self.api_method = 'event.getinfo&event='              ; end
+      class LastfmEventShoutsRequest            < Base ; self.api_method = 'event.getshouts&event='            ; end
+      class LastfmGeoEventsRequest              < Base ; self.api_method = 'geo.getevents&geo='                ; end
+      class LastfmGeoTopArtistsRequest          < Base ; self.api_method = 'geo.gettopartists&geo='            ; end
+      class LastfmGeoTopTracksRequest           < Base ; self.api_method = 'geo.gettoptracks&geo='             ; end
+      class LastfmGroupMembersRequest           < Base ; self.api_method = 'group.getmembers&group='           ; end
+      class LastfmGroupWeeklyAlbumChartRequest  < Base ; self.api_method = 'group.getweeklyalbumchart&group='  ; end
+      class LastfmGroupWeeklyArtistChartRequest < Base ; self.api_method = 'group.getweeklyartistchart&group=' ; end
+      class LastfmGroupWeeklyChartListRequest   < Base ; self.api_method = 'group.getweeklychartlist&group='   ; end
+      class LastfmGroupWeeklyTrackChartRequest  < Base ; self.api_method = 'group.getweeklytrackchart&group='  ; end
+      class LastfmPlaylistfetchRequest          < Base ; self.api_method = 'playlist.fetch&playlist='          ; end
+      class LastfmTagSimilarRequest             < Base ; self.api_method = 'tag.getsimilar&tag='               ; end
+      class LastfmTagTopAlbumsRequest           < Base ; self.api_method = 'tag.gettopalbums&tag='             ; end
+      class LastfmTagTopArtistsRequest          < Base ; self.api_method = 'tag.gettopartists&tag='            ; end
+      class LastfmTagTopTagsRequest             < Base ; self.api_method = 'tag.gettoptags&tag='               ; end
+      class LastfmTagTopTracksRequest           < Base ; self.api_method = 'tag.gettoptracks&tag='             ; end
+      class LastfmTagWeeklyArtistChartRequest   < Base ; self.api_method = 'tag.getweeklyartistchart&tag='     ; end
+      class LastfmTagWeeklyChartListRequest     < Base ; self.api_method = 'tag.getweeklychartlist&tag='       ; end
+      class LastfmTasteometercompareRequest     < Base ; self.api_method = 'tasteometer.compare&tasteometer='  ; end
+      class LastfmTrackInfoRequest              < Base ; self.api_method = 'track.getinfo&track='              ; end
+      class LastfmTrackSimilarRequest           < Base ; self.api_method = 'track.getsimilar&track='           ; end
+      class LastfmTrackTagsRequest              < Base ; self.api_method = 'track.gettags&track='              ; end
+      class LastfmTrackTopFansRequest           < Base ; self.api_method = 'track.gettopfans&track='           ; end
+      class LastfmTrackTopTagsRequest           < Base ; self.api_method = 'track.gettoptags&track='           ; end
+      class LastfmUserEventsRequest             < Base ; self.api_method = 'user.getevents&user='              ; end
+      class LastfmUserFriendsRequest            < Base ; self.api_method = 'user.getfriends&recenttracks=1&user='             ; end
+      class LastfmUserInfoRequest               < Base ; self.api_method = 'user.getinfo&user='                ; end
+      class LastfmUserLovedTracksRequest        < Base ; self.api_method = 'user.getlovedtracks&user='         ; end
+      class LastfmUserNeighboursRequest         < Base ; self.api_method = 'user.getneighbours&user='          ; end
+      class LastfmUserPastEventsRequest         < Base ; self.api_method = 'user.getpastevents&user='          ; end
+      class LastfmUserPlaylistsRequest          < Base ; self.api_method = 'user.getplaylists&user='           ; end
+      class LastfmUserRecentStationsRequest     < Base ; self.api_method = 'user.getrecentstations&user='      ; end
+      class LastfmUserRecentTracksRequest       < Base ; self.api_method = 'user.getrecenttracks&user='        ; end
+      class LastfmUserRecommendedArtistsRequest < Base ; self.api_method = 'user.getrecommendedartists&user='  ; end
+      class LastfmUserRecommendedEventsRequest  < Base ; self.api_method = 'user.getrecommendedevents&user='   ; end
+      class LastfmUserShoutsRequest             < Base ; self.api_method = 'user.getshouts&user='              ; end
+      class LastfmUserTopAlbumsRequest          < Base ; self.api_method = 'user.gettopalbums&user='           ; end
+      class LastfmUserTopArtistsRequest         < Base ; self.api_method = 'user.gettopartists&user='          ; end
+      class LastfmUserTopTagsRequest            < Base ; self.api_method = 'user.gettoptags&user='             ; end
+      class LastfmUserTopTracksRequest          < Base ; self.api_method = 'user.gettoptracks&user='           ; end
+      class LastfmUserWeeklyAlbumChartRequest   < Base ; self.api_method = 'user.getweeklyalbumchart&user='    ; end
+      class LastfmUserWeeklyArtistChartRequest  < Base ; self.api_method = 'user.getweeklyartistchart&user='   ; end
+      class LastfmUserWeeklyChartListRequest    < Base ; self.api_method = 'user.getweeklychartlist&user='     ; end
+      class LastfmUserWeeklyTrackChartRequest   < Base ; self.api_method = 'user.getweeklytrackchart&user='    ; end
+      class LastfmVenueEventsRequest            < Base ; self.api_method = 'venue.getevents&venue='            ; end
+      class LastfmVenuePastEventsRequest        < Base ; self.api_method = 'venue.getpastevents&venue='        ; end
+
+      # Singular
+      class LastfmAlbumInfoRequest              < Base ; self.main_fieldname = 'album'                 ; self.response_type = 'album'             ; end
+      class LastfmArtistInfoRequest             < Base ; self.main_fieldname = 'artist'                ; self.response_type = 'artist'            ; end
+      class LastfmEventInfoRequest              < Base ; self.main_fieldname = 'event'                 ; self.response_type = 'event'             ; end
+      class LastfmTrackInfoRequest              < Base ; self.main_fieldname = 'track'                 ; self.response_type = 'track'             ; end
+      class LastfmUserInfoRequest               < Base ; self.main_fieldname = 'user'                  ; self.response_type = 'user'              ; end
+      # Albums
+      class LastfmArtistTopAlbumsRequest        < Base ; self.main_fieldname = 'topalbums'             ; self.response_type = 'album'             ; end
+      class LastfmTagTopAlbumsRequest           < Base ; self.main_fieldname = 'topalbums'             ; self.response_type = 'album'             ; end
+      class LastfmUserTopAlbumsRequest          < Base ; self.main_fieldname = 'topalbums'             ; self.response_type = 'album'             ; end
+      # Tracks
+      class LastfmArtistTopTracksRequest        < Base ; self.main_fieldname = 'toptracks'             ; self.response_type = 'track'             ; end
+      class LastfmGeoTopTracksRequest           < Base ; self.main_fieldname = 'toptracks'             ; self.response_type = 'track'             ; end
+      class LastfmTagTopTracksRequest           < Base ; self.main_fieldname = 'toptracks'             ; self.response_type = 'track'             ; end
+      class LastfmUserTopTracksRequest          < Base ; self.main_fieldname = 'toptracks'             ; self.response_type = 'track'             ; end
+      class LastfmUserLovedTracksRequest        < Base ; self.main_fieldname = 'lovedtracks'           ; self.response_type = 'track'             ; end
+      class LastfmUserRecentTracksRequest       < Base ; self.main_fieldname = 'recenttracks'          ; self.response_type = 'track'             ; end
+      class LastfmTrackSimilarRequest           < Base ; self.main_fieldname = 'similartracks'         ; self.response_type = 'track'             ; end
+      # Artists
+      class LastfmArtistSimilarRequest          < Base ; self.main_fieldname = 'artists'               ; self.response_type = 'artist'            ; end
+      class LastfmUserRecommendedArtistsRequest < Base ; self.main_fieldname = 'recommendedartists'    ; self.response_type = 'artist'            ; end
+      class LastfmGeoTopArtistsRequest          < Base ; self.main_fieldname = 'topartists'            ; self.response_type = 'artist'            ; end
+      class LastfmTagTopArtistsRequest          < Base ; self.main_fieldname = 'topartists'            ; self.response_type = 'artist'            ; end
+      class LastfmUserTopArtistsRequest         < Base ; self.main_fieldname = 'topartists'            ; self.response_type = 'artist'            ; end
+      # Events
+      class LastfmArtistEventsRequest           < Base ; self.main_fieldname = 'events'                ; self.response_type = 'event'             ; end
+      class LastfmGeoEventsRequest              < Base ; self.main_fieldname = 'events'                ; self.response_type = 'event'             ; end
+      class LastfmUserEventsRequest             < Base ; self.main_fieldname = 'events'                ; self.response_type = 'event'             ; end
+      class LastfmVenueEventsRequest            < Base ; self.main_fieldname = 'events'                ; self.response_type = 'event'             ; end
+      class LastfmUserPastEventsRequest         < Base ; self.main_fieldname = 'pastevents'            ; self.response_type = 'event'             ; end
+      class LastfmVenuePastEventsRequest        < Base ; self.main_fieldname = 'pastevents'            ; self.response_type = 'event'             ; end
+      class LastfmUserRecommendedEventsRequest  < Base ; self.main_fieldname = 'recommendedevents'     ; self.response_type = 'event'             ; end
+      # Users
+      class LastfmEventAttendeesRequest         < Base ; self.main_fieldname = 'attendees'             ; self.response_type = 'user'              ; end
+      class LastfmUserFriendsRequest            < Base ; self.main_fieldname = 'friends'               ; self.response_type = 'user'              ; end
+      class LastfmArtistTopFansRequest          < Base ; self.main_fieldname = 'topfans'               ; self.response_type = 'user'              ; end
+      class LastfmTrackTopFansRequest           < Base ; self.main_fieldname = 'topfans'               ; self.response_type = 'user'              ; end
+      class LastfmGroupMembersRequest           < Base ; self.main_fieldname = 'members'               ; self.response_type = 'user'              ; end
+      class LastfmUserNeighboursRequest         < Base ; self.main_fieldname = 'neighbours'            ; self.response_type = 'user'              ; end
+      # Shouts
+      class LastfmArtistShoutsRequest           < Base ; self.main_fieldname = 'shouts'                ; self.response_type = 'shout'             ; end
+      class LastfmEventShoutsRequest            < Base ; self.main_fieldname = 'shouts'                ; self.response_type = 'shout'             ; end
+      class LastfmUserShoutsRequest             < Base ; self.main_fieldname = 'shouts'                ; self.response_type = 'shout'             ; end
+      # Tags
+      class LastfmArtistTagsRequest             < Base ; self.main_fieldname = 'similartags'           ; self.response_type = 'tag'               ; end
+      class LastfmTagSimilarRequest             < Base ; self.main_fieldname = 'similartags'           ; self.response_type = 'tag'               ; end
+      class LastfmAlbumTagsRequest              < Base ; self.main_fieldname = 'tags'                  ; self.response_type = 'tag'               ; end
+      class LastfmTrackTagsRequest              < Base ; self.main_fieldname = 'tags'                  ; self.response_type = 'tag'               ; end
+      class LastfmArtistTopTagsRequest          < Base ; self.main_fieldname = 'toptags'               ; self.response_type = 'tag'               ; end
+      class LastfmTagTopTagsRequest             < Base ; self.main_fieldname = 'toptags'               ; self.response_type = 'tag'               ; end
+      class LastfmTrackTopTagsRequest           < Base ; self.main_fieldname = 'toptags'               ; self.response_type = 'tag'               ; end
+      class LastfmUserTopTagsRequest            < Base ; self.main_fieldname = 'toptags'               ; self.response_type = 'tag'               ; end
+      # Charts
+      class LastfmGroupWeeklyAlbumChartRequest  < Base ; self.main_fieldname = 'weeklyalbumchart'      ; self.response_type = 'weeklyalbumchart'  ; end
+      class LastfmUserWeeklyAlbumChartRequest   < Base ; self.main_fieldname = 'weeklyalbumchart'      ; self.response_type = 'weeklyalbumchart'  ; end
+      class LastfmGroupWeeklyArtistChartRequest < Base ; self.main_fieldname = 'weeklyartistchart'     ; self.response_type = 'weeklyartistchart' ; end
+      class LastfmTagWeeklyArtistChartRequest   < Base ; self.main_fieldname = 'weeklyartistchart'     ; self.response_type = 'weeklyartistchart' ; end
+      class LastfmUserWeeklyArtistChartRequest  < Base ; self.main_fieldname = 'weeklyartistchart'     ; self.response_type = 'weeklyartistchart' ; end
+      class LastfmGroupWeeklyChartListRequest   < Base ; self.main_fieldname = 'weeklychartlist'       ; self.response_type = 'weeklychartlist'   ; end
+      class LastfmTagWeeklyChartListRequest     < Base ; self.main_fieldname = 'weeklychartlist'       ; self.response_type = 'weeklychartlist'   ; end
+      class LastfmUserWeeklyChartListRequest    < Base ; self.main_fieldname = 'weeklychartlist'       ; self.response_type = 'weeklychartlist'   ; end
+      class LastfmGroupWeeklyTrackChartRequest  < Base ; self.main_fieldname = 'weeklytrackchart'      ; self.response_type = 'weeklytrackchart'  ; end
+      class LastfmUserWeeklyTrackChartRequest   < Base ; self.main_fieldname = 'weeklytrackchart'      ; self.response_type = 'weeklytrackchart'  ; end
+
+      # class LastfmArtistImagesRequest           < Base ; self.main_fieldname = 'images'                ; end
+      # class LastfmUserPlaylistsRequest          < Base ; self.main_fieldname = 'playlists'             ; end
+      # class LastfmArtistPodcastRequest          < Base ; self.main_fieldname = 'podcast'               ; end
+      # class LastfmUserRecentStationsRequest     < Base ; self.main_fieldname = 'recentstations'        ; end
+      # class LastfmPlaylistfetchRequest          < Base ; self.main_fieldname ='''                ; end
+      # class LastfmTasteometercompareRequest     < Base ; self.main_fieldname =''         ; end
+
+    end
+  end
+end

data/lib/wuclan/lastfm/scrape/lastfm_job.rb

@@ -0,0 +1,12 @@
+require 'wuclan/lastfm/scrape/recursive_requests'
+
+module Wuclan
+  module Lastfm
+    module Scrape
+
+      Base.class_eval do
+      end
+
+    end
+  end
+end

data/lib/wuclan/lastfm/scrape/lastfm_request_stream.rb

@@ -0,0 +1,17 @@
+require 'wuclan/lastfm/scrape/base'
+module Wuclan
+  module Lastfm
+    module Scrape
+
+      class LastfmRequestStream < Monkeyshines::RequestStream::KlassRequestStream
+        include Wuclan::Lastfm::Scrape
+
+      end
+
+    end
+  end
+end
+
+
+# if req.is_a?(LastfmArtistShoutsRequest) then p([req.parsed_contents["shouts"]["shout"].length, req.parsed_contents["shouts"]["@attr"]]) rescue nil
+# end