rtlog 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +4 -0
- data/README.mdown +72 -0
- data/Rakefile +127 -0
- data/bin/rtlog-create +116 -0
- data/lib/rtlog/archives.rb +387 -0
- data/lib/rtlog/example/config/config.yml +22 -0
- data/lib/rtlog/example/config/day.html.erb +21 -0
- data/lib/rtlog/example/config/index.html.erb +19 -0
- data/lib/rtlog/example/config/layout.html.erb +229 -0
- data/lib/rtlog/example/config/month.html.erb +70 -0
- data/lib/rtlog/example/config/rss.xml.erb +50 -0
- data/lib/rtlog/example/public/styles/site.css +82 -0
- data/lib/rtlog/pages.rb +276 -0
- data/lib/rtlog.rb +15 -0
- metadata +109 -0
data/ChangeLog
ADDED
data/README.mdown
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
rtlog
|
2
|
+
=====
|
3
|
+
|
4
|
+
Rtlog is a creating html archive tools from Twitter.
|
5
|
+
|
6
|
+
Installation
|
7
|
+
------------
|
8
|
+
|
9
|
+
Rtlog is a collection of Ruby scripts, so a Ruby interpreter is required
|
10
|
+
(tested with 1.8.7), and the following gems:
|
11
|
+
|
12
|
+
- oauth
|
13
|
+
- rubytter
|
14
|
+
- activesupport
|
15
|
+
- json\_pure
|
16
|
+
|
17
|
+
### Gem Installation
|
18
|
+
|
19
|
+
$ gem install rtlog
|
20
|
+
|
21
|
+
### Features/Problems
|
22
|
+
|
23
|
+
|
24
|
+
### Synopsis
|
25
|
+
|
26
|
+
$ rtlog-create\
|
27
|
+
--log-level debug\
|
28
|
+
-i yuanying\
|
29
|
+
-p password\
|
30
|
+
-t '~/Sites/lifelog'\
|
31
|
+
-u 'http://192.168.10.8/~yuanying/lifelog'\
|
32
|
+
--temp-dir '~/.lifelog/temp'
|
33
|
+
|
34
|
+
- -i: twitter account id
|
35
|
+
- -p: twitter account password
|
36
|
+
- -t: target directory for generated html
|
37
|
+
- -u: url prefix on generated html
|
38
|
+
- --temp-dir: temporary directory for downloaded tweets
|
39
|
+
|
40
|
+
- -r: re-construct html (optional)
|
41
|
+
- -d: re-download all tweets (optional)
|
42
|
+
- --log-level: log level (fatal / error / warn / info / debug)
|
43
|
+
|
44
|
+
### Copyright
|
45
|
+
|
46
|
+
- Author:: yuanying <yuanying at fraction dot jp>
|
47
|
+
- Copyright:: Copyright (c) 2009-2010 yuanying
|
48
|
+
|
49
|
+
### LICENSE
|
50
|
+
|
51
|
+
(The MIT License)
|
52
|
+
|
53
|
+
Copyright (c) 2009 yuanying
|
54
|
+
|
55
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
56
|
+
a copy of this software and associated documentation files (the
|
57
|
+
'Software'), to deal in the Software without restriction, including
|
58
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
59
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
60
|
+
permit persons to whom the Software is furnished to do so, subject to
|
61
|
+
the following conditions:
|
62
|
+
|
63
|
+
The above copyright notice and this permission notice shall be
|
64
|
+
included in all copies or substantial portions of the Software.
|
65
|
+
|
66
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
67
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
68
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
69
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
70
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
71
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
72
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'rake/clean'
|
4
|
+
require 'spec'
|
5
|
+
require 'spec/rake/spectask'
|
6
|
+
# require 'rake/testtask'
|
7
|
+
require 'rake/packagetask'
|
8
|
+
require 'rake/gempackagetask'
|
9
|
+
require 'rake/rdoctask'
|
10
|
+
require 'rake/contrib/rubyforgepublisher'
|
11
|
+
require 'rake/contrib/sshpublisher'
|
12
|
+
require 'fileutils'
|
13
|
+
include FileUtils
|
14
|
+
|
15
|
+
NAME = "rtlog"
|
16
|
+
AUTHOR = "yuanying"
|
17
|
+
EMAIL = "yuanying at fraction dot jp"
|
18
|
+
DESCRIPTION = ""
|
19
|
+
RUBYFORGE_PROJECT = "rtlog"
|
20
|
+
HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
|
21
|
+
BIN_FILES = %w( rtlog-create )
|
22
|
+
VERS = "0.1.0"
|
23
|
+
|
24
|
+
REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
|
25
|
+
CLEAN.include ['**/.*.sw?', '*.gem', '.config']
|
26
|
+
RDOC_OPTS = [
|
27
|
+
'--title', "#{NAME} documentation",
|
28
|
+
"--charset", "utf-8",
|
29
|
+
"--opname", "index.html",
|
30
|
+
"--line-numbers",
|
31
|
+
"--main", "README.mdown",
|
32
|
+
"--inline-source",
|
33
|
+
]
|
34
|
+
|
35
|
+
task :default => [:spec]
|
36
|
+
task :package => [:clean]
|
37
|
+
|
38
|
+
desc "Run the specs under spec/models"
|
39
|
+
Spec::Rake::SpecTask.new do |t|
|
40
|
+
t.spec_opts = ['--options', "spec/spec.opts"]
|
41
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
42
|
+
end
|
43
|
+
|
44
|
+
spec = Gem::Specification.new do |s|
|
45
|
+
s.name = NAME
|
46
|
+
s.version = VERS
|
47
|
+
s.platform = Gem::Platform::RUBY
|
48
|
+
s.has_rdoc = true
|
49
|
+
s.extra_rdoc_files = ["README.mdown", "ChangeLog"]
|
50
|
+
s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/']
|
51
|
+
s.summary = DESCRIPTION
|
52
|
+
s.description = DESCRIPTION
|
53
|
+
s.author = AUTHOR
|
54
|
+
s.email = EMAIL
|
55
|
+
s.homepage = HOMEPATH
|
56
|
+
s.executables = BIN_FILES
|
57
|
+
s.rubyforge_project = RUBYFORGE_PROJECT
|
58
|
+
s.bindir = "bin"
|
59
|
+
s.require_path = "lib"
|
60
|
+
s.autorequire = ""
|
61
|
+
s.test_files = Dir["test/test_*.rb"]
|
62
|
+
|
63
|
+
s.add_dependency('activesupport', '>=2.3.4')
|
64
|
+
s.add_dependency('json_pure', '>=1.1.9')
|
65
|
+
s.add_dependency('rubytter', '>=0.8.0')
|
66
|
+
#s.add_dependency('activesupport', '>=1.3.1')
|
67
|
+
#s.required_ruby_version = '>= 1.8.2'
|
68
|
+
|
69
|
+
s.files = %w(README.mdown ChangeLog Rakefile) +
|
70
|
+
Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") +
|
71
|
+
Dir.glob("ext/**/*.{h,c,rb}") +
|
72
|
+
Dir.glob("examples/**/*.rb") +
|
73
|
+
Dir.glob("tools/*.rb")
|
74
|
+
|
75
|
+
s.extensions = FileList["ext/**/extconf.rb"].to_a
|
76
|
+
end
|
77
|
+
|
78
|
+
Rake::GemPackageTask.new(spec) do |p|
|
79
|
+
p.need_tar = true
|
80
|
+
p.gem_spec = spec
|
81
|
+
end
|
82
|
+
|
83
|
+
task :install do
|
84
|
+
name = "#{NAME}-#{VERS}.gem"
|
85
|
+
sh %{rake package}
|
86
|
+
sh %{sudo gem install pkg/#{name}}
|
87
|
+
end
|
88
|
+
|
89
|
+
task :uninstall => [:clean] do
|
90
|
+
sh %{sudo gem uninstall #{NAME}}
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
Rake::RDocTask.new do |rdoc|
|
95
|
+
rdoc.rdoc_dir = 'html'
|
96
|
+
rdoc.options += RDOC_OPTS
|
97
|
+
rdoc.template = "resh"
|
98
|
+
#rdoc.template = "#{ENV['template']}.rb" if ENV['template']
|
99
|
+
if ENV['DOC_FILES']
|
100
|
+
rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/))
|
101
|
+
else
|
102
|
+
rdoc.rdoc_files.include('README.mdown', 'ChangeLog')
|
103
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
104
|
+
rdoc.rdoc_files.include('ext/**/*.c')
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
desc "Publish to RubyForge"
|
109
|
+
task :rubyforge => [:rdoc, :package] do
|
110
|
+
require 'rubyforge'
|
111
|
+
Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'yuanying').upload
|
112
|
+
end
|
113
|
+
|
114
|
+
desc 'Package and upload the release to gemcutter.'
|
115
|
+
task :release => [:clean, :package] do |t|
|
116
|
+
v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z"
|
117
|
+
abort "Versions don't match #{v} vs #{VERS}" unless v == VERS
|
118
|
+
pkg = "pkg/#{NAME}-#{VERS}"
|
119
|
+
|
120
|
+
files = [
|
121
|
+
"#{pkg}.tgz",
|
122
|
+
"#{pkg}.gem"
|
123
|
+
].compact
|
124
|
+
|
125
|
+
puts "Releasing #{NAME} v. #{VERS}"
|
126
|
+
sh %{gem push #{pkg}.gem}
|
127
|
+
end
|
data/bin/rtlog-create
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
#!/usr/bin/env ruby -KU -W0
|
2
|
+
|
3
|
+
RTLOG_ROOT = File.join(File.dirname(__FILE__), '..')
|
4
|
+
|
5
|
+
$:.insert(0, *[
|
6
|
+
File.join(RTLOG_ROOT, 'lib')
|
7
|
+
])
|
8
|
+
|
9
|
+
require 'rubygems'
|
10
|
+
require 'optparse'
|
11
|
+
require 'yaml'
|
12
|
+
require 'logger'
|
13
|
+
require 'rtlog/archives'
|
14
|
+
require 'rtlog/pages'
|
15
|
+
|
16
|
+
config = {}
|
17
|
+
|
18
|
+
config_file = nil #File.join( File.dirname(__FILE__), '..', 'lib', 'rtlog', 'example', 'config', 'config.yml' )
|
19
|
+
download_all = false
|
20
|
+
re_construct = false
|
21
|
+
loglevel = 'warn'
|
22
|
+
|
23
|
+
opt = OptionParser.new
|
24
|
+
opt.on('-c', '--config-file CONFIG_FILE') { |v| config_file = v }
|
25
|
+
opt.on('-d') { |boolean| download_all = boolean }
|
26
|
+
opt.on('-r') { |boolean| re_construct = boolean }
|
27
|
+
opt.on("--log-level LOGLEVEL", 'fatal / error / warn / info / debug') { |v| loglevel = v }
|
28
|
+
|
29
|
+
opt.on('-i', '--twitter-id TWITTER_ID') { |v| config['twitter_id'] = v }
|
30
|
+
opt.on('-p', '--twitter-password TWITTER_PASSWORD') { |v| config['twitter_password'] = v }
|
31
|
+
opt.on('-t', '--target-dir GENERATED_HTML_TARGET_DIR') { |v| config['target_dir'] = v }
|
32
|
+
opt.on('--config-dir CONFIG_DIR') { |v| config['config_dir'] = v }
|
33
|
+
opt.on('--temp-dir TEMP_DIR') { |v| config['temp_dir'] = v }
|
34
|
+
opt.on('-u', '--url-prefix URL_PREFIX') { |v| config['url_prefix'] = v }
|
35
|
+
opt.on('--time-zone TIME_ZONE') { |v| config['time_zone'] = v }
|
36
|
+
opt.on('--consumer-key CONSUMER_KEY') { |v| config['consumer-key'] = v }
|
37
|
+
opt.on('--consumer-secret CONSUMER_SECRET') { |v| config['consumer-secret'] = v }
|
38
|
+
opt.on('--access-token ACCESS_TOKEN') { |v| config['access-token'] = v }
|
39
|
+
opt.on('--access-token-secret ACCESS_TOKEN_SECRET') { |v| config['access-token-secret'] = v }
|
40
|
+
opt.parse!
|
41
|
+
|
42
|
+
def constantize(camel_cased_word)
|
43
|
+
unless /\A(?:::)?([A-Z]\w*(?:::[A-Z]\w*)*)\z/ =~ camel_cased_word
|
44
|
+
raise NameError, "#{camel_cased_word.inspect} is not a valid constant name!"
|
45
|
+
end
|
46
|
+
|
47
|
+
Object.module_eval("::#{$1}", __FILE__, __LINE__)
|
48
|
+
end
|
49
|
+
|
50
|
+
logger = Logger.new(STDOUT)
|
51
|
+
logger.level = constantize("Logger::#{loglevel.upcase}")
|
52
|
+
Rtlog.logger = logger
|
53
|
+
|
54
|
+
if config_file
|
55
|
+
logger.debug("Loading config file: #{config_file}")
|
56
|
+
File.open(config_file) { |file| config = YAML.load(file).update(config) }
|
57
|
+
end
|
58
|
+
|
59
|
+
## default setting
|
60
|
+
config = {
|
61
|
+
'target_dir' => './lifelog',
|
62
|
+
'config_dir' => '~/.lifelog/config',
|
63
|
+
'temp_dir' => '~/.lifelog/temp',
|
64
|
+
'url_prefix' => 'http://localhost'
|
65
|
+
}.update(config)
|
66
|
+
|
67
|
+
log = Rtlog::Archive.new(config)
|
68
|
+
|
69
|
+
update_months = []
|
70
|
+
update_days = []
|
71
|
+
if log.recent_entry_id == nil || download_all
|
72
|
+
log.download_all
|
73
|
+
download_all = true
|
74
|
+
else
|
75
|
+
log.download( 'count' => 200, 'since_id' => log.recent_entry_id ) do |tweet|
|
76
|
+
tweet = Rtlog::Tweet.new(config, tweet)
|
77
|
+
month = [tweet.created_at.year, tweet.created_at.month]
|
78
|
+
update_months << month unless update_months.include?(month)
|
79
|
+
day = [tweet.created_at.year, tweet.created_at.month, tweet.created_at.day]
|
80
|
+
update_days << day unless update_days.include?(day)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
if re_construct || download_all || (update_months.size > 0)
|
85
|
+
index = Rtlog::IndexPage.new(config, log)
|
86
|
+
index.generate
|
87
|
+
rss = Rtlog::RssPage.new(config, log)
|
88
|
+
rss.generate
|
89
|
+
end
|
90
|
+
|
91
|
+
if re_construct || download_all
|
92
|
+
index.month_pages.each do |month|
|
93
|
+
begin
|
94
|
+
month.generate
|
95
|
+
month.current_day_pages.each do |day|
|
96
|
+
day.generate
|
97
|
+
end
|
98
|
+
end while month = month.next
|
99
|
+
end
|
100
|
+
else
|
101
|
+
update_months.each do |month|
|
102
|
+
month = Time.zone.local(*month)
|
103
|
+
month_entry = log.month_entry(month)
|
104
|
+
month_page = Rtlog::MonthPage.new(config, log, month_entry)
|
105
|
+
begin
|
106
|
+
month_page.generate
|
107
|
+
end while month_page = month_page.next
|
108
|
+
end
|
109
|
+
|
110
|
+
update_days.each do |day|
|
111
|
+
day = Time.zone.local(*day)
|
112
|
+
day_entry = log.day_entry(day)
|
113
|
+
day_page = Rtlog::DayPage.new(config, log, day_entry)
|
114
|
+
day_page.generate
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,387 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rtlog'
|
3
|
+
require 'oauth'
|
4
|
+
require 'rubytter'
|
5
|
+
require 'active_support'
|
6
|
+
require 'fileutils'
|
7
|
+
require 'json/pure'
|
8
|
+
require 'open-uri'
|
9
|
+
require 'erb'
|
10
|
+
|
11
|
+
class Rubytter
|
12
|
+
def create_request(req, basic_auth = true)
|
13
|
+
@header.each {|k, v| req.add_field(k, v) }
|
14
|
+
req.basic_auth(@login, @password) if @login && @password
|
15
|
+
req
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
module Rtlog
|
20
|
+
|
21
|
+
module DirUtils
|
22
|
+
def entries(path)
|
23
|
+
Dir.entries(path).delete_if{|d| !(/\d+/ =~ d) }.reverse.map!{ |file| File.join(path, file) }
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
class TwitPic
|
28
|
+
TWIT_REGEXP = /http\:\/\/twitpic\.com\/([0-9a-zA-Z]+)/
|
29
|
+
|
30
|
+
attr_reader :id
|
31
|
+
attr_reader :config
|
32
|
+
attr_reader :url
|
33
|
+
|
34
|
+
def initialize config, url
|
35
|
+
@config = config
|
36
|
+
@url = url
|
37
|
+
@id = TWIT_REGEXP.match(url)[1]
|
38
|
+
end
|
39
|
+
|
40
|
+
def original_thumbnail_url
|
41
|
+
"http://twitpic.com/show/thumb/#{id}"
|
42
|
+
end
|
43
|
+
|
44
|
+
def local_url
|
45
|
+
"#{config['url_prefix']}#{path}"
|
46
|
+
end
|
47
|
+
|
48
|
+
def download
|
49
|
+
file_path = File.expand_path( File.join(config['target_dir'], path) )
|
50
|
+
folder_path = File.dirname(file_path)
|
51
|
+
FileUtils.mkdir_p(folder_path) unless File.exist?(folder_path)
|
52
|
+
open(original_thumbnail_url) do |f|
|
53
|
+
extname = File.extname( f.base_uri.path )
|
54
|
+
file_path = file_path + extname
|
55
|
+
open(file_path, 'w') do |io|
|
56
|
+
io.write f.read
|
57
|
+
end
|
58
|
+
end
|
59
|
+
sleep 1
|
60
|
+
end
|
61
|
+
|
62
|
+
protected
|
63
|
+
def path
|
64
|
+
prefix = id[0, 2]
|
65
|
+
"/twitpic/#{prefix}/#{id}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
class Tweet
|
70
|
+
include DirUtils
|
71
|
+
include ERB::Util
|
72
|
+
attr_reader :data
|
73
|
+
attr_reader :config
|
74
|
+
|
75
|
+
def initialize config, path_or_data
|
76
|
+
@config = config
|
77
|
+
if path_or_data.is_a?(String)
|
78
|
+
open(path_or_data) do |io|
|
79
|
+
@data = JSON.parse(io.read)
|
80
|
+
end
|
81
|
+
else
|
82
|
+
@data = path_or_data
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def method_missing sym, *args, &block
|
87
|
+
return super unless @data.key?(sym.to_s)
|
88
|
+
return @data[sym.to_s]
|
89
|
+
end
|
90
|
+
|
91
|
+
URL_REGEXP = /https?(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)/
|
92
|
+
REPRY_REGEXP = /@(\w+)/
|
93
|
+
|
94
|
+
def formatted_text
|
95
|
+
t = h(text)
|
96
|
+
t = t.gsub(URL_REGEXP) { "<a href='#{$&}'>#{$&}</a>" }
|
97
|
+
t = t.gsub(REPRY_REGEXP) { "<a href='http://twitter.com/#{$1}'>@#{$1}</a>" }
|
98
|
+
t
|
99
|
+
end
|
100
|
+
|
101
|
+
def id
|
102
|
+
@data['id']
|
103
|
+
end
|
104
|
+
|
105
|
+
def medias
|
106
|
+
unless defined?(@medias) && @medias
|
107
|
+
@medias = []
|
108
|
+
text.gsub(TwitPic::TWIT_REGEXP) do |m|
|
109
|
+
@medias << TwitPic.new(config, m)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
@medias
|
113
|
+
end
|
114
|
+
|
115
|
+
def created_at
|
116
|
+
Time.zone.parse(@data['created_at'])
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
class Entry
|
121
|
+
include DirUtils
|
122
|
+
attr_reader :config
|
123
|
+
attr_reader :path
|
124
|
+
attr_writer :logger
|
125
|
+
|
126
|
+
def initialize config, path
|
127
|
+
@config = config
|
128
|
+
@path = path
|
129
|
+
@date = nil
|
130
|
+
end
|
131
|
+
|
132
|
+
def date
|
133
|
+
unless @date
|
134
|
+
@date = Time.zone.local( *path.split('/').last(date_split_size) )
|
135
|
+
end
|
136
|
+
@date
|
137
|
+
end
|
138
|
+
|
139
|
+
def ==(v)
|
140
|
+
return false if v.respond_to?(:path)
|
141
|
+
self.path == v.path
|
142
|
+
end
|
143
|
+
|
144
|
+
def logger
|
145
|
+
defined?(@logger) ? @logger : Rtlog.logger
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
class DayEntry < Entry
|
150
|
+
def tweets
|
151
|
+
entries(@path).each do |path|
|
152
|
+
yield Tweet.new(config, path)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def size
|
157
|
+
entries(@path).size
|
158
|
+
end
|
159
|
+
|
160
|
+
protected
|
161
|
+
def date_split_size
|
162
|
+
3
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
class MonthEntry < Entry
|
167
|
+
|
168
|
+
def day_entries
|
169
|
+
unless defined?(@day_entries) && @day_entries
|
170
|
+
@day_entries = []
|
171
|
+
entries(@path).each do |path|
|
172
|
+
@day_entries << DayEntry.new(config, path)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
@day_entries
|
176
|
+
end
|
177
|
+
|
178
|
+
def size
|
179
|
+
unless defined?(@size) && @size
|
180
|
+
@size = 0
|
181
|
+
day_entries.each do |d|
|
182
|
+
@size += d.size
|
183
|
+
end
|
184
|
+
end
|
185
|
+
@size
|
186
|
+
end
|
187
|
+
|
188
|
+
protected
|
189
|
+
def date_split_size
|
190
|
+
2
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
class YearEntry < Entry
|
195
|
+
|
196
|
+
def month_entries
|
197
|
+
unless defined?(@month_entries) && @month_entries
|
198
|
+
@month_entries = []
|
199
|
+
entries(@path).each do |path|
|
200
|
+
@month_entries << MonthEntry.new(config, path)
|
201
|
+
end
|
202
|
+
end
|
203
|
+
@month_entries
|
204
|
+
end
|
205
|
+
|
206
|
+
protected
|
207
|
+
def date_split_size
|
208
|
+
1
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
class Archive
|
213
|
+
include DirUtils
|
214
|
+
|
215
|
+
attr_accessor :config
|
216
|
+
attr_writer :logger
|
217
|
+
|
218
|
+
def initialize config
|
219
|
+
@config = config
|
220
|
+
if config['twitter_id'] && config['twitter_password']
|
221
|
+
@tw = Rubytter.new(config['twitter_id'], config['twitter_password'])
|
222
|
+
elsif config['consumer-key'] && config['consumer-secret'] && config['access-token'] && config['access-token-secret']
|
223
|
+
consumer = OAuth::Consumer.new(
|
224
|
+
config['consumer-key'],
|
225
|
+
config['consumer-secret'],
|
226
|
+
:site => 'http://twitter.com'
|
227
|
+
)
|
228
|
+
access_token = OAuth::AccessToken.new(
|
229
|
+
consumer,
|
230
|
+
config['access-token'],
|
231
|
+
config['access-token-secret']
|
232
|
+
)
|
233
|
+
@tw = OAuthRubytter.new(access_token)
|
234
|
+
else
|
235
|
+
@tw = Rubytter.new
|
236
|
+
end
|
237
|
+
|
238
|
+
@year_entries = nil
|
239
|
+
Time.zone = @config['time_zone'] || user_info.time_zone
|
240
|
+
end
|
241
|
+
|
242
|
+
def logger
|
243
|
+
defined?(@logger) ? @logger : Rtlog.logger
|
244
|
+
end
|
245
|
+
|
246
|
+
def user_info
|
247
|
+
@userinfo ||= @tw.user( twitter_id )
|
248
|
+
@userinfo
|
249
|
+
end
|
250
|
+
alias :user :user_info
|
251
|
+
|
252
|
+
def recent_entry_id
|
253
|
+
year_entries.first.month_entries.first.day_entries.first.tweets do |tweet|
|
254
|
+
return tweet.id
|
255
|
+
end
|
256
|
+
rescue
|
257
|
+
nil
|
258
|
+
end
|
259
|
+
|
260
|
+
def recent_day_entries size=7
|
261
|
+
unless defined?(@recent_day_entries) && @recent_day_entries
|
262
|
+
@recent_day_entries = []
|
263
|
+
year_entries.each do |y|
|
264
|
+
y.month_entries.each do |m|
|
265
|
+
m.day_entries.each do |d|
|
266
|
+
@recent_day_entries << d
|
267
|
+
return @recent_day_entries if @recent_day_entries.size == size
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
end
|
272
|
+
@recent_day_entries
|
273
|
+
end
|
274
|
+
|
275
|
+
def year_entries
|
276
|
+
unless @year_entries
|
277
|
+
@year_entries = []
|
278
|
+
entries(temp_dir).each do |path|
|
279
|
+
@year_entries << YearEntry.new(config, path)
|
280
|
+
end
|
281
|
+
end
|
282
|
+
@year_entries
|
283
|
+
end
|
284
|
+
|
285
|
+
def previous_month_entry month_entry
|
286
|
+
previous_year = nil
|
287
|
+
previous_month = nil
|
288
|
+
year_entries.each do |y|
|
289
|
+
index = y.month_entries.index(month_entry)
|
290
|
+
if index == 0
|
291
|
+
if previous_year && previous_year.month_entries.size > 0
|
292
|
+
return previous_year.month_entries.last
|
293
|
+
else
|
294
|
+
return nil
|
295
|
+
end
|
296
|
+
elsif index
|
297
|
+
return y.month_entries[index-1]
|
298
|
+
end
|
299
|
+
previous_year = y
|
300
|
+
end
|
301
|
+
return nil
|
302
|
+
end
|
303
|
+
|
304
|
+
def next_month_entry month_entry
|
305
|
+
return_next_entry = false
|
306
|
+
year_entries.each do |y|
|
307
|
+
return y.month_entries.first if return_next_entry && y.month_entries.size > 0
|
308
|
+
index = y.month_entries.index(month_entry)
|
309
|
+
if index == nil
|
310
|
+
next
|
311
|
+
elsif y.month_entries.size == (index + 1)
|
312
|
+
return_next_entry = true
|
313
|
+
else
|
314
|
+
return y.month_entries[index+1]
|
315
|
+
end
|
316
|
+
end
|
317
|
+
return nil
|
318
|
+
end
|
319
|
+
|
320
|
+
def month_entry month
|
321
|
+
path = File.join( temp_dir, sprintf('%04d', month.year), sprintf('%02d', month.month) )
|
322
|
+
MonthEntry.new(config, path)
|
323
|
+
end
|
324
|
+
|
325
|
+
def day_entry day
|
326
|
+
path = File.join( temp_dir, sprintf('%04d', day.year), sprintf('%02d', day.month), sprintf('%02d', day.day) )
|
327
|
+
DayEntry.new(config, path)
|
328
|
+
end
|
329
|
+
|
330
|
+
def temp_dir
|
331
|
+
@temp_dir ||= File.expand_path(@config['temp_dir'])
|
332
|
+
@temp_dir
|
333
|
+
end
|
334
|
+
|
335
|
+
def twitter_id
|
336
|
+
@config['twitter_id']
|
337
|
+
end
|
338
|
+
|
339
|
+
def download option={}
|
340
|
+
option['count'] ||= (@config['count'] || 200)
|
341
|
+
option.reject! { |k,v| v==nil }
|
342
|
+
|
343
|
+
timeline = @tw.user_timeline( twitter_id, option )
|
344
|
+
return false if timeline.size == 0
|
345
|
+
return false if timeline.last.id == option['max_id']
|
346
|
+
|
347
|
+
timeline.each do |status|
|
348
|
+
status = JSON.parse(status.to_json)
|
349
|
+
save status
|
350
|
+
yield status if block_given?
|
351
|
+
end
|
352
|
+
|
353
|
+
@year_entries = nil
|
354
|
+
@recent_day_entries = nil
|
355
|
+
return timeline.last.id
|
356
|
+
end
|
357
|
+
|
358
|
+
def download_all
|
359
|
+
max_id = nil
|
360
|
+
while true
|
361
|
+
max_id = download('max_id' => max_id) unless block_given?
|
362
|
+
max_id = download('max_id' => max_id) { |status| yield status } if block_given?
|
363
|
+
break unless max_id
|
364
|
+
sleep 10
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
368
|
+
def save status
|
369
|
+
Tweet.new(config, status).medias.each do |m|
|
370
|
+
logger.debug("Download media: #{m.class}, #{m.original_thumbnail_url}")
|
371
|
+
m.download
|
372
|
+
sleep 2
|
373
|
+
end
|
374
|
+
date = Time.zone.parse(status['created_at'])
|
375
|
+
date = DateTime.parse(date.to_s)
|
376
|
+
|
377
|
+
path = "#{temp_dir}/#{date.strftime('%Y/%m/%d')}/#{status['id']}.json"
|
378
|
+
FileUtils.mkdir_p( File.dirname(path) ) unless File.exist?( File.dirname(path) )
|
379
|
+
File.open(path, "w") { |file| file.write(status.to_json) }
|
380
|
+
logger.debug("Tweet is saved: #{path}")
|
381
|
+
end
|
382
|
+
|
383
|
+
end
|
384
|
+
|
385
|
+
end
|
386
|
+
|
387
|
+
|