livejournal2 0.4.0

data/sample/export

@@ -0,0 +1,144 @@
+#!/usr/bin/ruby -I../lib
+
+require 'livejournal/sync'
+require 'livejournal/database'
+require 'optparse'
+require 'progressbar'
+
+create = false
+username = nil
+password = nil
+usejournal = nil
+dbfile = nil
+get_entries = true
+get_comments = true
+opts = OptionParser.new do |opts|
+  opts.on('-c', '--create',
+          'Create a new database if necessary') { |create| }
+  opts.on('-u', '--user USERNAME',
+          'Login username') { |username| }
+  opts.on('-p', '--password PASSWORD',
+          'Login password') { |password| }
+  opts.on('-a', '--usejournal JOURNAL',
+          'Journal to sync (USERNAME must have access)') { |usejournal| }
+  opts.on('-d', '--db FILENAME',
+          'Filename for output database') { |dbfile| }
+  opts.on('--nocomments',
+          "Don't fetch comments (only entries)") { get_comments = false }
+  opts.on('--noentries',
+          "Don't fetch entries (only comments)") { get_entries = false }
+end
+opts.parse!(ARGV)
+
+unless dbfile
+  puts opts
+  puts "ERROR: Must specify database file."
+  exit 1
+end
+
+begin
+  db = LiveJournal::Database.new(dbfile, create)
+rescue Errno::ENOENT
+  puts "Use the --create flag to create a new database."
+  raise
+end
+username ||= db.username
+usejournal ||= db.usejournal
+
+unless username
+  puts opts
+  puts "ERROR: Must specify username."
+  exit 1
+end
+
+if usejournal
+  puts "Journal: #{usejournal} (syncing as #{username})."
+else
+  puts "Journal: #{username}."
+end
+
+unless password
+  noecho = system('stty -echo')
+  print "Enter password"
+  print "(WARNING: echoed to screen)" unless noecho
+  print ": "
+  $stdout.flush
+
+  begin
+    password = gets.strip
+  ensure
+    if noecho
+      system('stty sane')
+      puts  # since the user input didn't add the newline for us...
+    end
+  end
+end
+
+user = LiveJournal::User.new(username, password)
+user.usejournal = usejournal
+db.username = user.username
+db.usejournal = usejournal if usejournal
+
+if get_entries
+  puts "Fetching entries..."
+  lastsync = db.lastsync
+  puts "Resuming from #{lastsync}." if lastsync
+  sync = LiveJournal::Sync::Entries.new(user, lastsync)
+
+  ProgressBar::with_progress("Fetching metadata: ") do |bar|
+    sync.run_syncitems do |cur, total|
+      bar.update(cur, total)
+    end
+  end
+
+  ProgressBar::with_progress("Fetching bodies: ") do |bar|
+    cur = 0
+    sync.run_sync do |entries, lastsync, remaining|
+      db.transaction do
+        entries.each do |itemid, entry|
+          db.store_entry entry
+        end
+      end
+      db.lastsync = lastsync
+
+      cur += entries.length
+      bar.update(cur, remaining+cur)
+    end
+  end
+end
+
+if get_comments
+  puts "Fetching comments..."
+
+  cs = LiveJournal::Sync::Comments.new(user)
+  next_meta = db.last_comment_meta
+  if next_meta
+    next_meta += 1
+  else
+    next_meta = 0
+  end
+
+  ProgressBar::with_progress("Fetching metadata: ") do |bar|
+    cs.run_metadata(next_meta) do |cur, max, data|
+      db.store_comments_meta data.comments
+      db.store_usermap data.usermap
+      bar.update(cur, max)
+    end
+  end
+
+  next_full = db.last_comment_full
+  if next_full
+    next_full += 1
+  else
+    next_full = 0
+  end
+
+  ProgressBar::with_progress("Fetching bodies: ") do |bar|
+    cs.run_body(next_full) do |cur, max, data|
+      bar.update(cur, max)
+      db.store_comments_full data.comments
+    end
+  end
+end
+
+# vim: ts=2 sw=2 et :

data/sample/fuse

@@ -0,0 +1,198 @@
+#!/usr/bin/ruby -I../lib
+
+require 'fusefs'
+require 'livejournal/database'
+require 'livejournal/entry'
+require 'stringio'
+
+class DBQuery
+  def initialize
+    @db = $db
+  end
+  def has_any?(where, *args)
+    exists = @db.db.get_first_value("SELECT itemid #{where} LIMIT 1", *args)
+    exists != nil
+  end
+  def has_year? year
+    has_any?("FROM entry WHERE year=?", year)
+  end
+  def has_month? year, month
+    has_any?("FROM entry WHERE year=? AND month=?", year, month)
+  end
+  def has_day? year, month, day
+    has_any?("FROM entry WHERE year=? AND month=? AND day=?", year, month, day)
+  end
+  def has_entry? itemid
+    has_any?("FROM entry WHERE itemid=?", itemid)
+  end
+
+  def get_array(sql, *args)
+    array = []
+    @db.db.execute("SELECT DISTINCT #{sql}", *args) do |row|
+      array << row[0]
+    end
+    array
+  end
+
+  def years
+    get_array('year FROM entry')
+  end
+  def months year
+    get_array('month FROM entry WHERE year=?', year).map { |m| "%02d" % m }
+  end
+  def days year, month
+    get_array('day FROM entry WHERE year=? AND month=?', year, month).map { |d| "%02d" % d }
+  end
+
+  def day_entries year, month, day
+    get_array('itemid FROM entry WHERE year=? AND month=? AND day=?',
+              year, month, day)
+  end
+end
+
+class Dispatcher < FuseFS::FuseDir
+  def initialize
+    @matches = []
+  end
+  def register(match, target)
+    @matches << [match, target]
+  end
+
+  def dispatch(sym, path, *args)
+    path, rest = split_path path if path
+    if path
+      @matches.each do |match, target|
+        if path =~ match
+          return target.dispatch(sym, rest, *(args+[path]))
+        end
+      end
+    end
+
+    # otherwise, dispatch it to the current object
+    begin
+      s = self.send(sym, path, *args)
+    rescue
+      p $!
+    end
+    return s
+  end
+end
+
+class DispatchDir < FuseFS::FuseDir
+  def initialize dispatcher
+    @dispatcher = dispatcher
+  end
+
+  def self.add_dispatch(*args)
+    args.each do |sym|
+      class_eval %{def #{sym}(path, *args)
+        @dispatcher.dispatch(#{sym.inspect}, path, *args)
+      end}
+    end
+  end
+
+  add_dispatch :directory?, :file?
+  add_dispatch :contents, :read_file
+end
+
+class LJFS < Dispatcher
+  class Day < Dispatcher
+    def contents path, year, mon, day
+      entries = $dbq.day_entries year, mon, day
+      entries.map do |itemid|
+        "#{itemid}.txt"
+      end
+    end
+    def directory? path, year, mon, day
+      path == nil and $dbq.has_day? year, mon, day
+    end
+    def file? path, year, mon, day
+      return false unless $dbq.has_day? year, mon, day
+      return false unless path =~ /^(\d+)\.txt$/
+      itemid = $1
+      return false unless $dbq.has_entry? itemid
+      true
+    end
+    def read_file path, year, mon, day
+      return false unless $dbq.has_day? year, mon, day
+      return false unless path =~ /^(\d+)\.txt$/
+      itemid = $1
+      return false unless $dbq.has_entry? itemid
+      entry = $db.get_entry itemid
+      out = StringIO.new
+      if entry.subject
+        out.puts entry.subject
+        out.puts("=" * entry.subject.length)
+        out.puts
+      end
+      out.puts entry.event
+      out.rewind
+      return out.read
+    end
+  end
+  class Month < Dispatcher
+    def initialize
+      super
+      register(/^\d{2}/, Day.new)
+    end
+    def directory? path, year, mon
+      path == nil and $dbq.has_month? year, mon
+    end
+    def file? path, year, mon
+      false
+    end
+    def contents path, year, mon
+      $dbq.days year, mon
+    end
+  end
+
+  class Year < Dispatcher
+    def initialize
+      super
+      register(/^\d{2}/, Month.new)
+    end
+    def directory? path, year
+      path == nil and $dbq.has_year? year
+    end
+    def file? path, year
+      false
+    end
+    def contents path, year
+      $dbq.months year
+    end
+  end
+
+  def initialize
+    super
+    register(/^\d{4}$/, Year.new)
+  end
+
+  def directory? path
+    return false
+  end
+  def file? path
+    return path == 'username'
+  end
+  def read_file path
+    return $db.username
+  end
+  def contents *args
+    return ['username'] + $dbq.years
+  end
+end
+
+unless ARGV.length == 2
+  puts "usage: #{$0} dbfile mountpoint"
+  exit 1
+end
+dbfile, mountpoint = ARGV
+
+$db = LiveJournal::Database.new dbfile
+$dbq = DBQuery.new  # temp hack
+root = DispatchDir.new LJFS.new
+
+FuseFS.set_root root
+FuseFS.mount_under mountpoint
+FuseFS.run
+
+# vim: ts=2 sw=2 et :

data/sample/graph

@@ -0,0 +1,208 @@
+#!/usr/bin/ruby -I../lib
+
+begin
+  require 'rubygems'
+  require 'gruff'
+rescue LoadError => e
+  puts <<EOT
+Error loading libraries: #{e}
+Though ljrb itself doesn't require them, the graph sample uses
+the gruff library: http://nubyonrails.com/pages/gruff ,
+which itself requires rubygems.
+EOT
+  exit 1
+end
+require 'livejournal/database'
+require 'livejournal/entry'
+require 'optparse'
+require 'time'
+
+class Options
+  # required options
+  attr_reader :dbfile, :outfile, :mode
+  # optional options
+  attr_reader :smooth, :normalize, :geom, :starttime, :endtime
+  # mode-specific options
+  attr_reader :tags
+
+  def initialize
+    @smooth = 1
+    @dbfile = nil
+    @outfile = nil
+    @normalize = false
+    @mode = nil
+    @geom = 800
+    @opts = OptionParser.new do |o|
+      o.banner = "usage: graph [options] [modeparam [modeparam ...]]"
+      o.on('-d', '--db=FILE', 'database (required)') { |@dbfile| }
+      o.on('-o', '--out=FILENAME.PNG', 'output file (required)') { |@outfile| }
+      o.on('-g', '--geom=WxH', 'output geometry') { |@geom| }
+      o.on('--smooth=N',
+           'merge every set of n consecutive datapoints') do |smooth|
+        @smooth = smooth.to_i
+      end
+      o.on('--normalize',
+           'normalize relative to entry counts') { |@normalize| }
+      o.on('--start=TIME', 'start time') { |t| @starttime = Time.parse t }
+      o.on('--end=TIME',   'end time')   { |t| @endtime = Time.parse t }
+      o.on('-m', '--mode=MODE',
+           'mode (required) one of {tags, security}') do |mode|
+        case mode
+        when 'tags'
+          @mode = :tags
+        when 'security'
+          @mode = :security
+        else
+          die "bad mode #{mode}"
+        end
+      end
+    end
+    self
+  end
+  def parse!(argv)
+    @opts.parse! ARGV
+    check_params
+    self
+  end
+  def die(msg)
+    puts msg
+    puts @opts
+    exit 1
+  end
+  def check_params
+    die "must specify database." unless @dbfile
+    die "must specify output file." unless @outfile
+    die "must specify mode." unless @mode
+
+    case @mode
+    when :tags
+      @tags = ARGV
+      unless @tags.length > 0
+        die "must specify at least one tag as an extra command-line parameter."
+      end
+    end
+  end
+end
+$opts = Options.new.parse! ARGV
+
+class DataSet
+  # store a series of (time, class, count) observations.
+
+  def initialize
+    # data:  klass -> time -> count
+    @data = Hash.new { |h,k| h[k] = Hash.new { |h,k| h[k] = 0 } }
+    @firsttime = @lasttime = nil
+    @bucketing = :month
+  end
+
+  def bucket(time)
+    case @bucketing
+    when :month
+      mon = time.month
+      if $opts.smooth != 1
+        # when smooth = 3,
+        #   1 -> 1
+        #   3 -> 1
+        #   4 -> 4
+        #   5 -> 4
+        #  12 -> 10
+        mon = ((mon-1)/$opts.smooth * $opts.smooth) + 1
+      end
+      "%d-%02d" % [time.year, mon]
+    end
+  end
+
+  def each_bucket
+    case @bucketing
+    when :month
+      time = Date.parse(@firsttime.strftime('%Y-%m-01'))
+      lasttime = Date.parse(@lasttime.strftime('%Y-%m-01'))
+      lastbucket = nil
+      while time < lasttime
+        b = bucket(time)
+        yield b if b != lastbucket
+        lastbucket = b
+        time = time >> 1
+      end
+    end
+  end
+
+  def labels(count=5)
+    total = 0
+    each_bucket { total += 1 }
+    skip = (total / count.to_f).round
+
+    labels = {}
+    i = 0
+    each_bucket do |bucket|
+      labels[i] = bucket if i % skip == 0
+      i += 1
+    end
+    labels
+  end
+
+  def get_class(klass)
+    buckets = []
+    data = @data[klass]
+    each_bucket do |bucket|
+      buckets << if data.has_key? bucket
+                   if $opts.normalize
+                     data[bucket] / @data[:total][bucket].to_f
+                   else
+                     data[bucket]
+                   end
+                 else
+                   0
+                 end
+    end
+    buckets
+  end
+
+  def add(klass, time, count=1)
+    @firsttime = time if @firsttime.nil? or time < @firsttime
+    @lasttime  = time if @lasttime.nil?  or time > @lasttime
+    @data[klass][bucket(time)] += count
+  end
+end
+
+dataset = DataSet.new
+db = LiveJournal::Database.new $opts.dbfile
+total_entries = 0
+
+db.each_entry do |entry|
+  next if $opts.starttime and $opts.starttime > entry.time 
+  next if $opts.endtime and $opts.endtime < entry.time 
+
+  dataset.add(:total, entry.time, 1)
+  case $opts.mode
+  when :tags
+    entry.taglist.each do |tag|
+      dataset.add(tag, entry.time)
+    end
+  when :security
+    case entry.security
+    when :public;  dataset.add(:public, entry.time)
+    when :friends; dataset.add(:friends, entry.time)
+    else; dataset.add(:other, entry.time)
+    end
+  end
+  total_entries += 1
+end
+
+g = Gruff::Line.new($opts.geom)
+g.labels = dataset.labels
+case $opts.mode
+when :tags
+  g.title = 'Tag Rate'
+  $opts.tags.each do |tag|
+    g.data(tag, dataset.get_class(tag))
+  end
+when :security
+  g.title = 'Entry Security'
+  [:public, :friends, :other].each do |sec|
+    g.data(sec.to_s, dataset.get_class(sec))
+  end
+end
+g.write $opts.outfile
+
+# vim: ts=2 sw=2 et :