livejournal 0.0.1

data/lib/livejournal/request.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/ruby
+#--
+# ljrb -- LiveJournal Ruby module
+# Copyright (c) 2005 Evan Martin <martine@danga.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#++
+
+
+require 'livejournal/basic'
+require 'cgi'
+require 'net/http'
+require 'date'
+
+module LiveJournal
+  module Request
+    class ProtocolException < RuntimeError; end
+
+    # ljtimes look like 2005-12-04 10:24:00.
+    # Convert a Time to an ljtime.
+    def self.time_to_ljtime time
+      time.strftime '%Y-%m-%d %H:%M:%S'
+    end
+    # Convert an ljtime to a Time.
+    def self.ljtime_to_time str
+      dt = DateTime.strptime(str, '%Y-%m-%d %H:%M')
+      Time.gm(dt.year, dt.mon, dt.day, dt.hour, dt.min, 0, 0)
+    end
+
+    # wrapper around a given hash, prefixing all key lookups with base
+    class HashStrip #:nodoc:
+      def initialize(base, hash)
+        @base = base
+        @hash = hash
+      end
+      def [](key)
+        @hash[@base + key]
+      end
+    end
+
+    # Superclass for all LiveJournal requests.
+    class Req #:nodoc:
+      def initialize(user, mode)
+        @user = user
+        @reqparams = { "user"          => user.username, 
+                       "password"      => user.password,
+                       "mode"          => mode,
+                       "clientversion" => "Ruby",
+                       "ver"           => 1 }
+        @reqparams['usejournal'] = user.usejournal if user.usejournal
+        @results = {}
+        @verbose = false
+        @dryrun = false
+      end
+
+      def verbose!; @verbose = true; end
+      def dryrun!; @dryrun = true; end
+
+      def []=(key, value)
+        @reqparams[key] = value
+      end
+
+      def [](key)
+        @result[key]
+      end
+
+      def run
+        h = Net::HTTP.new('www.livejournal.com')
+        h.set_debug_output $stderr if @verbose
+        request = @reqparams.collect { |key, value|
+          "#{CGI.escape(key)}=#{CGI.escape(value.to_s)}"
+        }.join("&")
+        p request if @verbose
+        return if @dryrun
+        response, data = h.post('/interface/flat', request)
+        parseresponse(data)
+        dumpresponse if @verbose
+        if self['success'] != "OK"
+          raise ProtocolException, self['errmsg']
+        end
+      end
+
+      def dumpresponse
+        @result.keys.sort.each { |key| puts "#{key} -> #{@result[key]}" }
+      end
+
+      protected
+      def parseresponse(data)
+        lines = data.split(/\r?\n/)
+        @result = {}
+        0.step(lines.length-1, 2) do |i|
+          @result[lines[i]] = lines[i+1]
+        end
+      end
+
+      def each_in_array(base)
+        for i in 1..(self["#{base}_count"].to_i) do
+          yield HashStrip.new("#{base}_#{i.to_s}_", self)
+        end
+      end
+      def build_array(base)
+        array = []
+        each_in_array(base) { |x| array << yield(x) }
+        array
+      end
+    end
+  end
+end
+
+# vim: ts=2 sw=2 et :

data/lib/livejournal/sync.rb

@@ -0,0 +1,185 @@
+#!/usr/bin/ruby
+#--
+# ljrb -- LiveJournal Ruby module
+# Copyright (c) 2005 Evan Martin <martine@danga.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#++
+
+# A full sync involves both the LiveJournal flat protocol, for entries:
+require 'livejournal/request'
+# As well as a custom XML format via REST, for comments:
+# http://www.livejournal.com/developer/exporting.bml
+require 'net/http'
+require 'uri'
+require 'livejournal/comments-xml'
+
+require 'livejournal/entry'
+require 'livejournal/comment'
+
+module LiveJournal
+  module Request
+    class SyncItems < Req
+      attr_reader :syncitems, :fetched, :total
+      def initialize(user, syncitems=nil, lastsync=nil)
+        super(user, 'syncitems')
+        @syncitems = syncitems || {}
+        self['lastsync'] = lastsync if lastsync
+      end
+
+      def run
+        super
+        lasttime = nil
+        @fetched = 0
+        @total = self['sync_total'].to_i
+        each_in_array('sync') do |item|
+          item, time = item['item'], item['time']
+          next if @syncitems.has_key? item
+          @fetched += 1
+          lasttime = time if lasttime.nil? or time > lasttime
+          @syncitems[item] = time
+        end
+        lasttime
+      end
+
+      def self.subset_items(syncitems, want_type='L')
+        items = {}
+        syncitems.each do |item, time|
+          next unless item =~ /^(.)-(\d+)$/
+          type, id = $1, $2.to_i
+          items[id] = time if type == want_type
+        end
+        items
+      end
+    end
+
+    class SessionGenerate < Req
+      def initialize(user)
+        super(user, 'sessiongenerate')
+      end
+      def run
+        super
+        self['ljsession']
+      end
+    end
+  end
+
+  # Journal export.  A full export involves syncing both entries and comments.
+  # See <tt>samples/export</tt> for a full example.
+  module Sync
+
+    # To run a sync, create a Sync::Entries object, then call
+    # Entries#run_syncitems to fetch the sync metadata, then call
+    # Entries#run_sync to get the actual entries.
+    class Entries
+      # To resume from a previous sync, pass in its lastsync value.
+      attr_reader :lastsync
+
+      def initialize(user, lastsync=nil)
+        @user = user
+        @logitems = {}
+        @lastsync = lastsync
+      end
+      def run_syncitems  # :yields: cur, total
+        cur = 0
+        total = nil
+        items = {}
+        lastsync = @lastsync
+        while total.nil? or cur < total
+          req = Request::SyncItems.new(@user, items, lastsync)
+          lastsync = req.run
+          cur += req.fetched
+          total = req.total unless total
+          yield cur, total if block_given?
+        end
+        @logitems = Request::SyncItems::subset_items(items, 'L')
+        return (not @logitems.empty?)
+      end
+
+      def run_sync  # :yields: entries_hash, lastsync, remaining_count
+        return if @logitems.empty?
+
+        lastsync = @lastsync
+        while @logitems.size > 0
+          req = Request::GetEvents.new(@user, :lastsync => lastsync)
+          entries = req.run
+          # pop off all items that we now have entries for
+          entries.each do |itemid, entry|
+            time = @logitems.delete itemid
+            lastsync = time if lastsync.nil? or time > lastsync
+          end
+          yield entries, lastsync, @logitems.size
+        end
+      end
+    end
+    
+    class Comments
+      def initialize(user)
+        @user = user
+        @session = nil
+        @maxid = nil
+      end
+
+      def run_GET(mode, start)
+        unless @session
+          req = Request::SessionGenerate.new(@user)
+          @session = req.run
+        end
+
+        path = "/export_comments.bml?get=comment_#{mode}&startid=#{start}"
+        # authas:  hooray for features discovered by reading source!
+        path += "&authas=#{@user.usejournal}" if @user.usejournal
+        url = URI.parse(@user.server.url + path)
+                        
+        http = Net::HTTP.new(url.host, url.port)
+        resp, data = http.get(url, 'Cookie' => "ljsession=#{@session}")
+        # XXX we'd like to stream this data to the XML parser.
+        # is this possible?
+        raise RuntimeError, "XXX" if resp.code != '200'
+        return data
+      end
+      private :run_GET
+
+      def run_metadata(start=0)  # :yields: cur, total, comments_hash
+        while @maxid.nil? or start < @maxid
+          data = run_GET('meta', start)
+          parsed = LiveJournal::Sync::CommentsXML::Parser.new(data)
+          @maxid ||= parsed.maxid
+          break if parsed.comments.empty?
+          cur = parsed.comments.keys.max
+          yield cur, @maxid, parsed
+          start = cur + 1
+        end
+      end
+      
+      def run_body(start=0)  # :yields: cur, total, comments_hash
+        while start < @maxid
+          data = run_GET('body', start)
+          parsed = LiveJournal::Sync::CommentsXML::Parser.new(data)
+          break if parsed.comments.empty?
+          cur = parsed.comments.keys.max
+          yield cur, @maxid, parsed
+          start = cur + 1
+        end
+      end
+    end
+  end
+end
+
+# vim: ts=2 sw=2 et :

data/sample/export

@@ -0,0 +1,154 @@
+#!/usr/bin/ruby
+
+require 'livejournal/sync'
+require 'livejournal/database'
+require 'optparse'
+
+class ProgressBar
+  def initialize(caption)
+    @cur = 0
+    @width = 40
+    print caption
+    print("[" + " "*40 + "]" + "\x08"*41)
+    $stdout.flush
+  end
+
+  def fill_to(pos)
+    dots = pos - @cur
+    print "."*dots
+    $stdout.flush
+    @cur = pos
+  end
+
+  def update(cur, max)
+    fill_to(@width*cur/max)
+  end
+
+  def finish(error=false)
+    return unless @cur >= 0
+    fill_to(@width) unless error
+    puts
+    @cur = -1
+  end
+
+  def self.with_progress(caption)
+    bar = ProgressBar.new(caption)
+    begin
+      yield bar
+    rescue => e
+      bar.finish(true)
+      raise e
+    else
+      bar.finish(false)
+    end
+  end
+end
+
+username = nil
+password = nil
+usejournal = nil
+dbfile = nil
+get_entries = true
+get_comments = true
+opts = OptionParser.new do |opts|
+  opts.on('-u', '--user USERNAME',
+          'Login username') { |username| }
+  opts.on('-p', '--password PASSWORD',
+          'Login password') { |password| }
+  opts.on('-a', '--usejournal JOURNAL',
+          'Journal to sync (USERNAME must have access)') { |usejournal| }
+  opts.on('-d', '--db FILENAME',
+          'Filename for output database') { |dbfile| }
+  opts.on('--nocomments',
+          "Don't fetch comments (only entries)") { get_comments = false }
+  opts.on('--noentries',
+          "Don't fetch entries (only comments)") { get_entries = false }
+end
+opts.parse!(ARGV)
+
+unless username
+  puts opts
+  puts "ERROR: Must specify username."
+  exit 1
+end
+
+unless dbfile
+  puts opts
+  puts "ERROR: Must specify database file."
+  exit 1
+end
+
+unless password
+  print "Enter password (WARNING: echoed to screen): "
+  $stdout.flush
+  password = gets.strip
+end
+
+user = LiveJournal::User.new(username, password)
+user.usejournal = usejournal
+db = LiveJournal::Database.new(dbfile, true)
+db.username = user.usejournal || user.username
+
+if get_entries
+  puts "Fetching entries..."
+  lastsync = db.lastsync
+  puts "Resuming from #{lastsync}." if lastsync
+  sync = LiveJournal::Sync::Entries.new(user, lastsync)
+
+  ProgressBar::with_progress("Fetching metadata: ") do |bar|
+    sync.run_syncitems do |cur, total|
+      bar.update(cur, total)
+    end
+  end
+
+  ProgressBar::with_progress("Fetching bodies: ") do |bar|
+    cur = 0
+    sync.run_sync do |entries, lastsync, remaining|
+      db.transaction do
+        entries.each do |itemid, entry|
+          db.store_entry entry
+        end
+      end
+      db.lastsync = lastsync
+
+      cur += entries.length
+      bar.update(cur, remaining+cur)
+    end
+  end
+end
+
+if get_comments
+  puts "Fetching comments..."
+
+  cs = LiveJournal::Sync::Comments.new(user)
+  next_meta = db.last_comment_meta
+  if next_meta
+    next_meta += 1
+  else
+    next_meta = 0
+  end
+
+  ProgressBar::with_progress("Fetching metadata: ") do |bar|
+    cs.run_metadata(next_meta) do |cur, max, data|
+      db.store_comments_meta data.comments
+      db.store_usermap data.usermap
+      bar.update(cur, max)
+    end
+  end
+
+  next_full = db.last_comment_full
+  if next_full
+    next_full += 1
+  else
+    next_full = 0
+  end
+
+  ProgressBar::with_progress("Fetching bodies: ") do |bar|
+    cs.run_body(next_full) do |cur, max, data|
+      bar.update(cur, max)
+      db.store_comments_full data.comments
+    end
+  end
+end
+
+# vim: ts=2 sw=2 et :