itgwiki_mirror 0.0.18.pre
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/itgwiki_mirror_backup +64 -0
- data/bin/itgwiki_mirror_deploy +57 -0
- data/lib/itgwiki_mirror.rb +3 -0
- data/lib/itgwiki_mirror/backuper.rb +163 -0
- data/lib/itgwiki_mirror/deployer.rb +211 -0
- metadata +70 -0
@@ -0,0 +1,64 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'itgwiki_mirror'
|
4
|
+
require 'itgwiki_mirror/backuper'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
USAGE = 'Usage: itgwiki_mirror_backup [options] user@host[:port]'
|
8
|
+
|
9
|
+
def usage
|
10
|
+
puts USAGE
|
11
|
+
end
|
12
|
+
|
13
|
+
options = {}
|
14
|
+
OptionParser.new do |opts|
|
15
|
+
|
16
|
+
opts.banner = USAGE
|
17
|
+
|
18
|
+
opts.on '-v', '--verbose', 'Run verbosely (off by default)' do |v|
|
19
|
+
options[:verbose] = v
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on '-u', '--db-username USERNAME', 'USERNAME for mysqldump' do |u|
|
23
|
+
options[:db_user] = u
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on '-p', '--dp-password PASSWORD', 'PASSWORD for mysqldump' do |p|
|
27
|
+
options[:db_pass] = p
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on '-d', '--db-name NAME', 'NAME of the database for mysqldump' do |d|
|
31
|
+
options[:db_name] = d
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on '-w', '--wiki-root ROOT', 'The ROOT of the wiki (e.g. /var/www/w)' do |w|
|
35
|
+
options[:wiki_root] = w.sub(/\/$/, '') # remove trailing slash
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on '-t', '--remote-rsync-target TARGET', 'The remote rsync TARGET (i.e. destination directory)' do |t|
|
39
|
+
options[:rsync_target] = t
|
40
|
+
end
|
41
|
+
|
42
|
+
end.parse!
|
43
|
+
|
44
|
+
# set user, host, and port from ARGV (port is optional)
|
45
|
+
#
|
46
|
+
begin
|
47
|
+
options[:user] = /^(.+)@/.match(ARGV[0])[1]
|
48
|
+
options[:host] = /@([^:]+)/.match(ARGV[0])[1]
|
49
|
+
options[:port] = /:(\d+)/.match(ARGV[0])[1].to_i if /:/.match ARGV[0]
|
50
|
+
rescue
|
51
|
+
usage
|
52
|
+
exit 1
|
53
|
+
end
|
54
|
+
|
55
|
+
if $DEBUG
|
56
|
+
p options
|
57
|
+
end
|
58
|
+
|
59
|
+
if ITGwikiMirror::Backuper.new(options).backup
|
60
|
+
puts 'Success! No problems encountered during backup.' if options[:verbose]
|
61
|
+
else
|
62
|
+
STDERR.puts 'Errors were encountered. Backup was not completed.'
|
63
|
+
end
|
64
|
+
|
@@ -0,0 +1,57 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'itgwiki_mirror'
|
4
|
+
require 'itgwiki_mirror/deployer'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
USAGE = 'Usage: itgwiki_mirror_backup [options] user@host[:port]'
|
8
|
+
|
9
|
+
def usage
|
10
|
+
puts USAGE
|
11
|
+
end
|
12
|
+
|
13
|
+
options = {}
|
14
|
+
OptionParser.new do |opts|
|
15
|
+
|
16
|
+
opts.banner = USAGE
|
17
|
+
|
18
|
+
opts.on '-v', '--verbose', 'Run verbosely (off by default)' do |v|
|
19
|
+
options[:verbose] = v
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on '-u', '--db-username USERNAME', 'USERNAME for mysql import' do |u|
|
23
|
+
options[:db_user] = u
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on '-p', '--dp-password PASSWORD', 'PASSWORD for mysql import' do |p|
|
27
|
+
options[:db_pass] = p
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on '-d', '--db-name NAME', 'NAME of the database for mysql import' do |d|
|
31
|
+
options[:db_name] = d
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on '-w', '--wiki-root ROOT', 'The ROOT of the wiki (e.g. /var/www/w)' do |w|
|
35
|
+
options[:wiki] = w.sub(/\/$/, '') # remove trailing slash
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on '-m', '--mirror-directory DIRECTORY', 'Target of rsync from ITGwiki' do |m|
|
39
|
+
options[:mirror] = m
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.on '-f', '--foreground', '--no-daemon', 'Run in foreground (do not daemonize)' do |f|
|
43
|
+
options[:foreground] = f
|
44
|
+
end
|
45
|
+
|
46
|
+
end.parse!
|
47
|
+
|
48
|
+
if $DEBUG
|
49
|
+
p options
|
50
|
+
end
|
51
|
+
|
52
|
+
#
|
53
|
+
# FIXME should this have some kind of "exec" or something? Ready about threads
|
54
|
+
# in Ruby. Be sure to handle signal processing (kill, term, etc.)
|
55
|
+
#
|
56
|
+
ITGwikiMirror::Deployer.new(options).run
|
57
|
+
|
@@ -0,0 +1,163 @@
|
|
1
|
+
require 'tempfile'
|
2
|
+
|
3
|
+
module ITGwikiMirror
|
4
|
+
|
5
|
+
|
6
|
+
class Backuper
|
7
|
+
|
8
|
+
|
9
|
+
def initialize opts
|
10
|
+
|
11
|
+
# verbose mode
|
12
|
+
#
|
13
|
+
@verbose = opts[:verbose]
|
14
|
+
|
15
|
+
# mysql options
|
16
|
+
#
|
17
|
+
@db_user = opts[:db_user]
|
18
|
+
@db_pass = opts[:db_pass]
|
19
|
+
@db_name = opts[:db_name]
|
20
|
+
|
21
|
+
# ssh options
|
22
|
+
#
|
23
|
+
@user = opts[:user]
|
24
|
+
@host = opts[:host]
|
25
|
+
@port = opts[:port]
|
26
|
+
|
27
|
+
# MediaWiki root
|
28
|
+
#
|
29
|
+
@wiki_root = opts[:wiki_root]
|
30
|
+
|
31
|
+
# rsync target
|
32
|
+
#
|
33
|
+
@rsync_target = opts[:rsync_target]
|
34
|
+
|
35
|
+
# mysqldump temporary file
|
36
|
+
#
|
37
|
+
@dumpfile = Tempfile.new('itgwiki_mirror_mysqldump').path
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
#
|
42
|
+
# Perform the backup
|
43
|
+
#
|
44
|
+
# Dump the database, rsync the database, the wiki directory, and any other
|
45
|
+
# specified files (configuration, etc.) to the remote host. If any of these
|
46
|
+
# steps fails, stop and return false. If they're all successful, return
|
47
|
+
# true.
|
48
|
+
#
|
49
|
+
def backup
|
50
|
+
|
51
|
+
#
|
52
|
+
# FIXME I repeat the $?.success? block over and over. Probably extract to a method.
|
53
|
+
#
|
54
|
+
|
55
|
+
# print immediately so we get the nice 'msg...' -> 'msg...OK' transition
|
56
|
+
# instead of waiting until the jobs is finished to print the message.
|
57
|
+
#
|
58
|
+
STDOUT.sync = true
|
59
|
+
|
60
|
+
|
61
|
+
print 'mysqldump...' if @verbose
|
62
|
+
mysqldump
|
63
|
+
if $?.success?
|
64
|
+
puts 'OK' if @verbose
|
65
|
+
else
|
66
|
+
STDERR.puts 'mysqldump failed'
|
67
|
+
return false
|
68
|
+
end
|
69
|
+
|
70
|
+
print 'rsyncing mysqldump...' if @verbose
|
71
|
+
rsync_mysqldump
|
72
|
+
if $?.success?
|
73
|
+
puts 'OK' if @verbose
|
74
|
+
else
|
75
|
+
STDERR.puts 'rsyncing mysqldump failed'
|
76
|
+
return false
|
77
|
+
end
|
78
|
+
|
79
|
+
|
80
|
+
print 'rsyncing wiki directory...' if @verbose
|
81
|
+
rsync_wiki_dir
|
82
|
+
if $?.success?
|
83
|
+
puts 'OK' if @verbose
|
84
|
+
else
|
85
|
+
STDERR.puts 'rsyncing wiki directory failed'
|
86
|
+
return false
|
87
|
+
end
|
88
|
+
|
89
|
+
print 'Notifying mirror that backup is complete...' if @verbose
|
90
|
+
notify_complete
|
91
|
+
if $?.success?
|
92
|
+
puts 'OK' if @verbose
|
93
|
+
else
|
94
|
+
STDERR.puts 'Notifying mirror failed'
|
95
|
+
return false
|
96
|
+
end
|
97
|
+
|
98
|
+
return true
|
99
|
+
end
|
100
|
+
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
|
105
|
+
def mysqldump
|
106
|
+
|
107
|
+
# XXX You CANNOT have a space between -p and the password
|
108
|
+
#
|
109
|
+
%x( mysqldump -u#{@db_user} -p#{@db_pass} #{@db_name} > #{@dumpfile} )
|
110
|
+
|
111
|
+
puts %( mysqldump -u#{@db_user} -p#{@db_pass} #{@db_name} > #{@dumpfile} ) if $DEBUG
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
#
|
116
|
+
# rsync from src to dst with options
|
117
|
+
# -a (archive mode)
|
118
|
+
# -vP (verbose mode along with partial and progress display) if @verbose
|
119
|
+
# --rsh='ssh -p###' (use ### as ssh port for rsync ) if an alternate port is defined
|
120
|
+
#
|
121
|
+
def rsync src, dst
|
122
|
+
|
123
|
+
# archive mode
|
124
|
+
#
|
125
|
+
opts = '-a'
|
126
|
+
|
127
|
+
# verbose transfer if requested
|
128
|
+
#
|
129
|
+
opts += ' -vP' if @verbose
|
130
|
+
|
131
|
+
# specify an alternate port if requested
|
132
|
+
#
|
133
|
+
opts += " --rsh='ssh -p#{@port}'" if @port
|
134
|
+
|
135
|
+
puts %( rsync #{opts} #{src} #{dst} ) if $DEBUG
|
136
|
+
|
137
|
+
%x( rsync #{opts} #{src} #{dst} )
|
138
|
+
end
|
139
|
+
|
140
|
+
|
141
|
+
def rsync_mysqldump
|
142
|
+
rsync @dumpfile, "#{@user}@#{@host}:#{@rsync_target}/itgwiki.mysqldump"
|
143
|
+
end
|
144
|
+
|
145
|
+
|
146
|
+
def rsync_wiki_dir
|
147
|
+
rsync @wiki_root, "#{@user}@#{@host}:#{@rsync_target}"
|
148
|
+
end
|
149
|
+
|
150
|
+
|
151
|
+
def notify_complete
|
152
|
+
|
153
|
+
# specify port if defined
|
154
|
+
#
|
155
|
+
port = "-p#{@port}" if @port
|
156
|
+
|
157
|
+
# just touch a file called "complete" in the mirror target
|
158
|
+
#
|
159
|
+
%x( ssh #{@user}@#{@host} #{port} touch #{@rsync_target}/complete )
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
@@ -0,0 +1,211 @@
|
|
1
|
+
module ITGwikiMirror
|
2
|
+
|
3
|
+
class Deployer < SimpleDaemon::Base
|
4
|
+
|
5
|
+
|
6
|
+
READ_ONLY_MSG = 'This is a mirror of the wiki. Visit the live site to make changes.'
|
7
|
+
|
8
|
+
|
9
|
+
def initialize opts
|
10
|
+
|
11
|
+
# verbose mode
|
12
|
+
#
|
13
|
+
@verbose = opts[:verbose]
|
14
|
+
|
15
|
+
# mysql options
|
16
|
+
#
|
17
|
+
@db_user = opts[:db_user]
|
18
|
+
@db_pass = opts[:db_pass]
|
19
|
+
@db_name = opts[:db_name]
|
20
|
+
|
21
|
+
# MediaWiki root
|
22
|
+
#
|
23
|
+
@wiki = opts[:wiki]
|
24
|
+
|
25
|
+
# mirror directory (target of rsync from ITGwiki)
|
26
|
+
#
|
27
|
+
@mirror = opts[:mirror]
|
28
|
+
end
|
29
|
+
|
30
|
+
def run
|
31
|
+
|
32
|
+
# Watch the complete file. When it's touched, deploy again.
|
33
|
+
notifier = INotify:Notifier.new
|
34
|
+
notifier.watch("#{@wiki}/complete", :modify) { deploy }
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
|
41
|
+
def success
|
42
|
+
puts 'Success! No problems encountered during deployment.' if @verbose
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
def failure
|
47
|
+
STDERR.puts 'Errors were encountered. Deployment was not completed.'
|
48
|
+
end
|
49
|
+
|
50
|
+
#
|
51
|
+
# Deploy the newly synced content to the mirror server
|
52
|
+
#
|
53
|
+
# Drop the old database, create a new one, import the mysqldump, modify the
|
54
|
+
# local MediaWiki configuration:
|
55
|
+
#
|
56
|
+
# * disable the $wgServer string
|
57
|
+
# * disable Sphinx search
|
58
|
+
# * set maintenance mode (read-only)
|
59
|
+
#
|
60
|
+
# Finally, rsync the local modified version into the live mirror MediaWiki
|
61
|
+
# root.
|
62
|
+
#
|
63
|
+
def deploy
|
64
|
+
|
65
|
+
#
|
66
|
+
# FIXME I repeat the $?.success? block over and over. Probably extract to a method.
|
67
|
+
#
|
68
|
+
|
69
|
+
# print immediately so we get the nice 'msg...' -> 'msg...OK' transition
|
70
|
+
# instead of waiting until the jobs is finished to print the message.
|
71
|
+
#
|
72
|
+
STDOUT.sync = true
|
73
|
+
|
74
|
+
print 'Dropping database...' if @verbose
|
75
|
+
drop_db
|
76
|
+
if $?.success?
|
77
|
+
puts 'OK' if @verbose
|
78
|
+
else
|
79
|
+
STDERR.puts 'Dropping database failed'
|
80
|
+
failure
|
81
|
+
end
|
82
|
+
|
83
|
+
print 'Creating database...' if @verbose
|
84
|
+
create_db
|
85
|
+
if $?.success?
|
86
|
+
puts 'OK' if @verbose
|
87
|
+
else
|
88
|
+
STDERR.puts 'Creating database failed'
|
89
|
+
failure
|
90
|
+
end
|
91
|
+
|
92
|
+
print 'Importing mysqldump...' if @verbose
|
93
|
+
import_mysqldump
|
94
|
+
if $?.success?
|
95
|
+
puts 'OK' if @verbose
|
96
|
+
else
|
97
|
+
STDERR.puts 'Importing mysqldump failed'
|
98
|
+
failure
|
99
|
+
end
|
100
|
+
|
101
|
+
print 'Disabling $wgServer string...' if @verbose
|
102
|
+
disable_server_string
|
103
|
+
if $?.success?
|
104
|
+
puts 'OK' if @verbose
|
105
|
+
else
|
106
|
+
STDERR.puts 'Disabling $wgServer string failed'
|
107
|
+
failure
|
108
|
+
end
|
109
|
+
|
110
|
+
print 'Disabling sphinx search...' if @verbose
|
111
|
+
disable_sphinx_search
|
112
|
+
if $?.success?
|
113
|
+
puts 'OK' if @verbose
|
114
|
+
else
|
115
|
+
STDERR.puts 'Disabling sphinx search failed'
|
116
|
+
failure
|
117
|
+
end
|
118
|
+
|
119
|
+
print 'Setting maintenance mode...' if @verbose
|
120
|
+
set_maintenance_mode
|
121
|
+
if $?.success?
|
122
|
+
puts 'OK' if @verbose
|
123
|
+
else
|
124
|
+
STDERR.puts 'Setting maintenance mode failed'
|
125
|
+
failure
|
126
|
+
end
|
127
|
+
|
128
|
+
print 'rsyncing modified mirror content to mirror site content...' if @verbose
|
129
|
+
rsync
|
130
|
+
if $?.success?
|
131
|
+
puts 'OK' if @verbose
|
132
|
+
else
|
133
|
+
STDERR.puts 'rsyncing modified mirror content to mirror site content failed' if @verbose
|
134
|
+
failure
|
135
|
+
end
|
136
|
+
|
137
|
+
success
|
138
|
+
end
|
139
|
+
|
140
|
+
|
141
|
+
def drop_db
|
142
|
+
|
143
|
+
# XXX no space allowed between -p and password
|
144
|
+
#
|
145
|
+
%x( mysql -u#{@db_user} -p#{@db_pass} -e 'drop database #{@db_name}' )
|
146
|
+
end
|
147
|
+
|
148
|
+
|
149
|
+
def create_db
|
150
|
+
|
151
|
+
# XXX no space allowed between -p and password
|
152
|
+
#
|
153
|
+
%x( mysql -u#{@db_user} -p#{@db_pass} -e 'create database #{@db_name}' )
|
154
|
+
end
|
155
|
+
|
156
|
+
|
157
|
+
def import_mysqldump
|
158
|
+
|
159
|
+
# XXX no space allowed between -p and password
|
160
|
+
#
|
161
|
+
%x( mysql -u#{@db_user} -p#{@db_pass} #{@db_name} < #{@mirror}/itgwiki.mysqldump )
|
162
|
+
end
|
163
|
+
|
164
|
+
|
165
|
+
def disable_server_string
|
166
|
+
|
167
|
+
#
|
168
|
+
# The server string is necessary in the production instance because it's
|
169
|
+
# the only way to tell the service which is behind a proxy that it should
|
170
|
+
# have a certain domain name and that it should use https. For the mirror,
|
171
|
+
# it can successfully be ascertained automatically by the MediaWiki
|
172
|
+
# software.
|
173
|
+
#
|
174
|
+
|
175
|
+
# comment the wgServer line
|
176
|
+
#
|
177
|
+
%x( sed -i '/wgServer/ s/^/#/' #{@mirror}/w/LocalSettings.php )
|
178
|
+
end
|
179
|
+
|
180
|
+
|
181
|
+
def disable_sphinx_search
|
182
|
+
|
183
|
+
# comment lines with the word sphinx (case insensitive) in them that are
|
184
|
+
# not already commented
|
185
|
+
#
|
186
|
+
%x( sed -i '/^[^#].*sphinx/I s/^/#/' #{@mirror}/w/LocalSettings.php )
|
187
|
+
end
|
188
|
+
|
189
|
+
|
190
|
+
def set_maintenance_mode
|
191
|
+
|
192
|
+
# append readonly message to config file
|
193
|
+
#
|
194
|
+
%x( echo '$wgReadOnly = "#{READ_ONLY_MSG}";' >> #{@mirror}/w/LocalSettings.php )
|
195
|
+
end
|
196
|
+
|
197
|
+
|
198
|
+
def rsync
|
199
|
+
|
200
|
+
# XXX being a local rsync (a simple file copy) we're not going to bother with
|
201
|
+
# allowing verbose mode or other options. Just a straight archive (-a)
|
202
|
+
# copy.
|
203
|
+
#
|
204
|
+
# XXX There is a trailing slash on the rsync source because we want to
|
205
|
+
# sync the contents of the directory into the wiki directory. `man rsync`
|
206
|
+
# for more details.
|
207
|
+
#
|
208
|
+
%x( rsync -a #{@mirror}/w/ #{@wiki} )
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
metadata
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: itgwiki_mirror
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.18.pre
|
5
|
+
prerelease: 7
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Justin Force
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-01-19 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rb-inotify
|
16
|
+
requirement: &78672600 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *78672600
|
25
|
+
description: ! "\n Use two scripts, itgwiki_mirror_backup and itgwiki_mirror_deploy,
|
26
|
+
to\n maintain a read-only mirror of ITGwiki. itgwiki_mirror_backup runs\n periodically
|
27
|
+
and automatically on the live instance of ITGwiki to create a\n backup and copy
|
28
|
+
it to the mirror. itgwiki_mirror_deploy runs on the mirror\n and is triggered
|
29
|
+
by itgwiki_mirror_backup, imports and adjusts the backup\n to be read-only, then
|
30
|
+
deploys it on the mirror server. Requires rsync and\n mysqldump.\n "
|
31
|
+
email: justin.force@gmail.com
|
32
|
+
executables:
|
33
|
+
- itgwiki_mirror_backup
|
34
|
+
- itgwiki_mirror_deploy
|
35
|
+
extensions: []
|
36
|
+
extra_rdoc_files: []
|
37
|
+
files:
|
38
|
+
- lib/itgwiki_mirror.rb
|
39
|
+
- lib/itgwiki_mirror/backuper.rb
|
40
|
+
- lib/itgwiki_mirror/deployer.rb
|
41
|
+
- bin/itgwiki_mirror_backup
|
42
|
+
- bin/itgwiki_mirror_deploy
|
43
|
+
homepage: https://github.com/sidewaysmilk/itgwiki_mirror
|
44
|
+
licenses: []
|
45
|
+
post_install_message:
|
46
|
+
rdoc_options: []
|
47
|
+
require_paths:
|
48
|
+
- lib
|
49
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
56
|
+
none: false
|
57
|
+
requirements:
|
58
|
+
- - ! '>'
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 1.3.1
|
61
|
+
requirements:
|
62
|
+
- mysqldump
|
63
|
+
- rsync
|
64
|
+
- sed
|
65
|
+
rubyforge_project:
|
66
|
+
rubygems_version: 1.8.15
|
67
|
+
signing_key:
|
68
|
+
specification_version: 3
|
69
|
+
summary: Maintain a read-only mirror of ITGwiki
|
70
|
+
test_files: []
|