rtlog 0.1.0

data/ChangeLog

@@ -0,0 +1,4 @@
+== 0.0.1 / 2009-10-02
+
+* initial release
+

data/README.mdown

@@ -0,0 +1,72 @@
+rtlog
+=====
+
+Rtlog is a creating html archive tools from Twitter.
+
+Installation
+------------
+
+Rtlog is a collection of Ruby scripts, so a Ruby interpreter is required
+(tested with 1.8.7), and the following gems:
+
+-   oauth
+-   rubytter
+-   activesupport
+-   json\_pure
+
+### Gem Installation
+
+    $ gem install rtlog
+
+### Features/Problems
+
+
+### Synopsis
+
+    $ rtlog-create\
+      --log-level debug\
+      -i yuanying\
+      -p password\
+      -t '~/Sites/lifelog'\
+      -u 'http://192.168.10.8/~yuanying/lifelog'\
+      --temp-dir '~/.lifelog/temp'
+
+-   -i: twitter account id
+-   -p: twitter account password
+-   -t: target directory for generated html
+-   -u: url prefix on generated html
+-   --temp-dir: temporary directory for downloaded tweets
+
+-   -r: re-construct html (optional)
+-   -d: re-download all tweets (optional)
+-   --log-level: log level (fatal / error / warn / info / debug)
+
+### Copyright
+
+-   Author::    yuanying <yuanying at fraction dot jp>
+-   Copyright:: Copyright (c) 2009-2010 yuanying
+
+### LICENSE
+
+    (The MIT License)
+    
+    Copyright (c) 2009 yuanying
+    
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    'Software'), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+    
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+    
+    THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+    IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+    CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+    TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+    SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,127 @@
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'spec'
+require 'spec/rake/spectask'
+# require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/contrib/rubyforgepublisher'
+require 'rake/contrib/sshpublisher'
+require 'fileutils'
+include FileUtils
+
+NAME              = "rtlog"
+AUTHOR            = "yuanying"
+EMAIL             = "yuanying at fraction dot jp"
+DESCRIPTION       = ""
+RUBYFORGE_PROJECT = "rtlog"
+HOMEPATH          = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+BIN_FILES         = %w( rtlog-create )
+VERS              = "0.1.0"
+
+REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
+CLEAN.include ['**/.*.sw?', '*.gem', '.config']
+RDOC_OPTS = [
+  '--title', "#{NAME} documentation",
+  "--charset", "utf-8",
+  "--opname", "index.html",
+  "--line-numbers",
+  "--main", "README.mdown",
+  "--inline-source",
+]
+
+task :default => [:spec]
+task :package => [:clean]
+
+desc "Run the specs under spec/models"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+spec = Gem::Specification.new do |s|
+  s.name              = NAME
+  s.version           = VERS
+  s.platform          = Gem::Platform::RUBY
+  s.has_rdoc          = true
+  s.extra_rdoc_files  = ["README.mdown", "ChangeLog"]
+  s.rdoc_options     += RDOC_OPTS + ['--exclude', '^(examples|extras)/']
+  s.summary           = DESCRIPTION
+  s.description       = DESCRIPTION
+  s.author            = AUTHOR
+  s.email             = EMAIL
+  s.homepage          = HOMEPATH
+  s.executables       = BIN_FILES
+  s.rubyforge_project = RUBYFORGE_PROJECT
+  s.bindir            = "bin"
+  s.require_path      = "lib"
+  s.autorequire       = ""
+  s.test_files        = Dir["test/test_*.rb"]
+
+  s.add_dependency('activesupport', '>=2.3.4')
+  s.add_dependency('json_pure', '>=1.1.9')
+  s.add_dependency('rubytter', '>=0.8.0')
+  #s.add_dependency('activesupport', '>=1.3.1')
+  #s.required_ruby_version = '>= 1.8.2'
+
+  s.files = %w(README.mdown ChangeLog Rakefile) +
+    Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") + 
+    Dir.glob("ext/**/*.{h,c,rb}") +
+    Dir.glob("examples/**/*.rb") +
+    Dir.glob("tools/*.rb")
+
+  s.extensions = FileList["ext/**/extconf.rb"].to_a
+end
+
+Rake::GemPackageTask.new(spec) do |p|
+  p.need_tar = true
+  p.gem_spec = spec
+end
+
+task :install do
+  name = "#{NAME}-#{VERS}.gem"
+  sh %{rake package}
+  sh %{sudo gem install pkg/#{name}}
+end
+
+task :uninstall => [:clean] do
+  sh %{sudo gem uninstall #{NAME}}
+end
+
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'html'
+  rdoc.options += RDOC_OPTS
+  rdoc.template = "resh"
+  #rdoc.template = "#{ENV['template']}.rb" if ENV['template']
+  if ENV['DOC_FILES']
+    rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/))
+  else
+    rdoc.rdoc_files.include('README.mdown', 'ChangeLog')
+    rdoc.rdoc_files.include('lib/**/*.rb')
+    rdoc.rdoc_files.include('ext/**/*.c')
+  end
+end
+
+desc "Publish to RubyForge"
+task :rubyforge => [:rdoc, :package] do
+  require 'rubyforge'
+  Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'yuanying').upload
+end
+
+desc 'Package and upload the release to gemcutter.'
+task :release => [:clean, :package] do |t|
+  v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z"
+  abort "Versions don't match #{v} vs #{VERS}" unless v == VERS
+  pkg = "pkg/#{NAME}-#{VERS}"
+
+  files = [
+    "#{pkg}.tgz",
+    "#{pkg}.gem"
+  ].compact
+
+  puts "Releasing #{NAME} v. #{VERS}"
+  sh %{gem push #{pkg}.gem}
+end

data/bin/rtlog-create

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby -KU -W0
+
+RTLOG_ROOT = File.join(File.dirname(__FILE__), '..')
+
+$:.insert(0, *[
+  File.join(RTLOG_ROOT, 'lib')
+])
+
+require 'rubygems'
+require 'optparse'
+require 'yaml'
+require 'logger'
+require 'rtlog/archives'
+require 'rtlog/pages'
+
+config        = {}
+
+config_file   = nil #File.join( File.dirname(__FILE__), '..', 'lib', 'rtlog', 'example', 'config', 'config.yml' )
+download_all  = false
+re_construct  = false
+loglevel      = 'warn'
+
+opt = OptionParser.new
+opt.on('-c', '--config-file CONFIG_FILE') { |v| config_file = v }
+opt.on('-d') { |boolean| download_all = boolean }
+opt.on('-r') { |boolean| re_construct = boolean }
+opt.on("--log-level LOGLEVEL", 'fatal / error / warn / info / debug') { |v| loglevel = v }
+
+opt.on('-i', '--twitter-id TWITTER_ID') { |v| config['twitter_id'] = v }
+opt.on('-p', '--twitter-password TWITTER_PASSWORD') { |v| config['twitter_password'] = v }
+opt.on('-t', '--target-dir GENERATED_HTML_TARGET_DIR') { |v| config['target_dir'] = v }
+opt.on('--config-dir CONFIG_DIR') { |v| config['config_dir'] = v }
+opt.on('--temp-dir TEMP_DIR') { |v| config['temp_dir'] = v }
+opt.on('-u', '--url-prefix URL_PREFIX') { |v| config['url_prefix'] = v }
+opt.on('--time-zone TIME_ZONE') { |v| config['time_zone'] = v }
+opt.on('--consumer-key CONSUMER_KEY') { |v| config['consumer-key'] = v }
+opt.on('--consumer-secret CONSUMER_SECRET') { |v| config['consumer-secret'] = v }
+opt.on('--access-token ACCESS_TOKEN') { |v| config['access-token'] = v }
+opt.on('--access-token-secret ACCESS_TOKEN_SECRET') { |v| config['access-token-secret'] = v }
+opt.parse!
+
+def constantize(camel_cased_word)
+  unless /\A(?:::)?([A-Z]\w*(?:::[A-Z]\w*)*)\z/ =~ camel_cased_word
+    raise NameError, "#{camel_cased_word.inspect} is not a valid constant name!"
+  end
+
+  Object.module_eval("::#{$1}", __FILE__, __LINE__)
+end
+
+logger = Logger.new(STDOUT)
+logger.level = constantize("Logger::#{loglevel.upcase}")
+Rtlog.logger = logger
+
+if config_file
+  logger.debug("Loading config file: #{config_file}")
+  File.open(config_file) { |file| config = YAML.load(file).update(config) }
+end
+
+## default setting
+config = {
+  'target_dir'  => './lifelog',
+  'config_dir'  => '~/.lifelog/config',
+  'temp_dir'    => '~/.lifelog/temp',
+  'url_prefix'  => 'http://localhost'
+}.update(config)
+
+log = Rtlog::Archive.new(config)
+
+update_months = []
+update_days   = []
+if log.recent_entry_id == nil || download_all
+  log.download_all
+  download_all = true
+else
+  log.download( 'count' => 200, 'since_id' => log.recent_entry_id ) do |tweet|
+    tweet = Rtlog::Tweet.new(config, tweet)
+    month = [tweet.created_at.year, tweet.created_at.month]
+    update_months << month unless update_months.include?(month)
+    day   = [tweet.created_at.year, tweet.created_at.month, tweet.created_at.day]
+    update_days   << day   unless update_days.include?(day)
+  end
+end
+
+if re_construct || download_all || (update_months.size > 0)
+  index = Rtlog::IndexPage.new(config, log)
+  index.generate
+  rss   = Rtlog::RssPage.new(config, log)
+  rss.generate
+end
+
+if re_construct || download_all
+  index.month_pages.each do |month|
+    begin
+      month.generate
+      month.current_day_pages.each do |day|
+        day.generate
+      end
+    end while month = month.next
+  end
+else
+  update_months.each do |month|
+    month = Time.zone.local(*month)
+    month_entry = log.month_entry(month)
+    month_page = Rtlog::MonthPage.new(config, log, month_entry)
+    begin
+      month_page.generate
+    end while month_page = month_page.next
+  end
+  
+  update_days.each do |day|
+    day = Time.zone.local(*day)
+    day_entry = log.day_entry(day)
+    day_page = Rtlog::DayPage.new(config, log, day_entry)
+    day_page.generate
+  end
+end

data/lib/rtlog/archives.rb

@@ -0,0 +1,387 @@
+require 'rubygems'
+require 'rtlog'
+require 'oauth'
+require 'rubytter'
+require 'active_support'
+require 'fileutils'
+require 'json/pure'
+require 'open-uri'
+require 'erb'
+
+class Rubytter
+  def create_request(req, basic_auth = true)
+    @header.each {|k, v| req.add_field(k, v) }
+    req.basic_auth(@login, @password) if @login && @password
+    req
+  end
+end
+
+module Rtlog
+
+module DirUtils
+  def entries(path)
+    Dir.entries(path).delete_if{|d| !(/\d+/ =~ d) }.reverse.map!{ |file| File.join(path, file) }
+  end
+end
+
+class TwitPic
+  TWIT_REGEXP = /http\:\/\/twitpic\.com\/([0-9a-zA-Z]+)/
+  
+  attr_reader :id
+  attr_reader :config
+  attr_reader :url
+  
+  def initialize config, url
+    @config = config
+    @url    = url
+    @id     = TWIT_REGEXP.match(url)[1]
+  end
+  
+  def original_thumbnail_url
+    "http://twitpic.com/show/thumb/#{id}"
+  end
+  
+  def local_url
+    "#{config['url_prefix']}#{path}"
+  end
+  
+  def download
+    file_path   = File.expand_path( File.join(config['target_dir'], path) )
+    folder_path = File.dirname(file_path)
+    FileUtils.mkdir_p(folder_path) unless File.exist?(folder_path)
+    open(original_thumbnail_url) do |f|
+      extname = File.extname( f.base_uri.path )
+      file_path = file_path + extname
+      open(file_path, 'w') do |io|
+        io.write f.read
+      end
+    end
+    sleep 1
+  end
+  
+  protected
+  def path
+    prefix = id[0, 2]
+    "/twitpic/#{prefix}/#{id}"
+  end
+end
+
+class Tweet
+  include DirUtils
+  include ERB::Util
+  attr_reader :data
+  attr_reader :config
+  
+  def initialize config, path_or_data
+    @config = config
+    if path_or_data.is_a?(String)
+      open(path_or_data) do |io|
+        @data = JSON.parse(io.read)
+      end
+    else
+      @data = path_or_data
+    end
+  end
+  
+  def method_missing sym, *args, &block
+    return super unless @data.key?(sym.to_s)
+    return @data[sym.to_s]
+  end
+  
+  URL_REGEXP    = /https?(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)/
+  REPRY_REGEXP  = /@(\w+)/
+  
+  def formatted_text
+    t = h(text)
+    t = t.gsub(URL_REGEXP)    { "<a href='#{$&}'>#{$&}</a>" }
+    t = t.gsub(REPRY_REGEXP)  { "<a href='http://twitter.com/#{$1}'>@#{$1}</a>" }
+    t
+  end
+  
+  def id
+    @data['id']
+  end
+  
+  def medias
+    unless defined?(@medias) && @medias
+      @medias = []
+      text.gsub(TwitPic::TWIT_REGEXP) do |m|
+        @medias << TwitPic.new(config, m)
+      end
+    end
+    @medias
+  end
+  
+  def created_at
+    Time.zone.parse(@data['created_at'])
+  end
+end
+
+class Entry
+  include DirUtils
+  attr_reader :config
+  attr_reader :path
+  attr_writer :logger
+  
+  def initialize config, path
+    @config = config
+    @path   = path
+    @date   = nil
+  end
+  
+  def date
+    unless @date
+      @date = Time.zone.local( *path.split('/').last(date_split_size) )
+    end
+    @date
+  end
+  
+  def ==(v)
+    return false if v.respond_to?(:path)
+    self.path == v.path
+  end
+  
+  def logger
+    defined?(@logger) ? @logger : Rtlog.logger
+  end
+end
+
+class DayEntry < Entry
+  def tweets
+    entries(@path).each do |path|
+      yield Tweet.new(config, path)
+    end
+  end
+  
+  def size
+    entries(@path).size
+  end
+  
+  protected
+  def date_split_size
+    3
+  end
+end
+
+class MonthEntry < Entry
+  
+  def day_entries
+    unless defined?(@day_entries) && @day_entries
+      @day_entries = []
+      entries(@path).each do |path|
+        @day_entries << DayEntry.new(config, path)
+      end
+    end
+    @day_entries
+  end
+  
+  def size
+    unless defined?(@size) && @size
+      @size = 0
+      day_entries.each do |d|
+        @size += d.size
+      end
+    end
+    @size
+  end
+  
+  protected
+  def date_split_size
+    2
+  end
+end
+
+class YearEntry < Entry
+  
+  def month_entries
+    unless defined?(@month_entries) && @month_entries
+      @month_entries = []
+      entries(@path).each do |path|
+        @month_entries << MonthEntry.new(config, path)
+      end
+    end
+    @month_entries
+  end
+  
+  protected
+  def date_split_size
+    1
+  end
+end
+
+class Archive
+  include DirUtils
+
+  attr_accessor :config
+  attr_writer :logger
+
+  def initialize config
+    @config       = config
+    if config['twitter_id'] && config['twitter_password']
+      @tw = Rubytter.new(config['twitter_id'], config['twitter_password'])
+    elsif config['consumer-key'] && config['consumer-secret'] && config['access-token'] && config['access-token-secret']
+      consumer = OAuth::Consumer.new(
+        config['consumer-key'],
+        config['consumer-secret'],
+        :site => 'http://twitter.com'
+      )
+      access_token = OAuth::AccessToken.new(
+        consumer,
+        config['access-token'],
+        config['access-token-secret']
+      )
+      @tw = OAuthRubytter.new(access_token)
+    else
+      @tw = Rubytter.new
+    end
+
+    @year_entries = nil
+    Time.zone     = @config['time_zone'] || user_info.time_zone
+  end
+  
+  def logger
+    defined?(@logger) ? @logger : Rtlog.logger
+  end
+  
+  def user_info
+    @userinfo ||= @tw.user( twitter_id )
+    @userinfo
+  end
+  alias :user :user_info
+  
+  def recent_entry_id
+    year_entries.first.month_entries.first.day_entries.first.tweets do |tweet|
+      return tweet.id
+    end
+  rescue
+    nil
+  end
+  
+  def recent_day_entries size=7
+    unless defined?(@recent_day_entries) && @recent_day_entries
+      @recent_day_entries = []
+      year_entries.each do |y|
+        y.month_entries.each do |m|
+          m.day_entries.each do |d|
+            @recent_day_entries << d
+            return @recent_day_entries if @recent_day_entries.size == size
+          end
+        end
+      end
+    end
+    @recent_day_entries
+  end
+  
+  def year_entries
+    unless @year_entries
+      @year_entries = []
+      entries(temp_dir).each do |path|
+        @year_entries << YearEntry.new(config, path)
+      end
+    end
+    @year_entries
+  end
+  
+  def previous_month_entry month_entry
+    previous_year   = nil
+    previous_month  = nil
+    year_entries.each do |y|
+      index = y.month_entries.index(month_entry)
+      if index == 0
+        if previous_year && previous_year.month_entries.size > 0
+          return previous_year.month_entries.last
+        else
+          return nil
+        end
+      elsif index
+        return y.month_entries[index-1]
+      end
+      previous_year = y
+    end
+    return nil
+  end
+  
+  def next_month_entry month_entry
+    return_next_entry = false
+    year_entries.each do |y|
+      return y.month_entries.first if return_next_entry && y.month_entries.size > 0
+      index = y.month_entries.index(month_entry)
+      if index == nil
+        next
+      elsif y.month_entries.size == (index + 1)
+        return_next_entry = true
+      else
+        return y.month_entries[index+1]
+      end
+    end
+    return nil
+  end
+  
+  def month_entry month
+    path = File.join( temp_dir, sprintf('%04d', month.year), sprintf('%02d', month.month) )
+    MonthEntry.new(config, path)
+  end
+  
+  def day_entry day
+    path = File.join( temp_dir, sprintf('%04d', day.year), sprintf('%02d', day.month), sprintf('%02d', day.day) )
+    DayEntry.new(config, path)
+  end
+
+  def temp_dir
+    @temp_dir ||= File.expand_path(@config['temp_dir'])
+    @temp_dir
+  end
+
+  def twitter_id
+    @config['twitter_id']
+  end
+
+  def download option={}
+    option['count'] ||= (@config['count'] || 200)
+    option.reject! { |k,v|  v==nil }
+
+    timeline = @tw.user_timeline( twitter_id, option )
+    return false if timeline.size == 0
+    return false if timeline.last.id == option['max_id']
+
+    timeline.each do |status|
+      status = JSON.parse(status.to_json)
+      save status
+      yield status if block_given?
+    end
+
+    @year_entries       = nil
+    @recent_day_entries = nil
+    return timeline.last.id
+  end
+
+  def download_all
+    max_id = nil
+    while true
+      max_id = download('max_id' => max_id) unless block_given?
+      max_id = download('max_id' => max_id) { |status| yield status } if block_given?
+      break unless max_id
+      sleep 10
+    end
+  end
+
+  def save status
+    Tweet.new(config, status).medias.each do |m|
+      logger.debug("Download media: #{m.class}, #{m.original_thumbnail_url}")
+      m.download
+      sleep 2
+    end
+    date = Time.zone.parse(status['created_at'])
+    date = DateTime.parse(date.to_s)
+    
+    path = "#{temp_dir}/#{date.strftime('%Y/%m/%d')}/#{status['id']}.json"
+    FileUtils.mkdir_p( File.dirname(path) ) unless File.exist?( File.dirname(path) )
+    File.open(path, "w") { |file| file.write(status.to_json) }
+    logger.debug("Tweet is saved: #{path}")
+  end
+  
+end
+
+end
+
+