omgdav 0.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +3 -0
- data/.gitignore +17 -0
- data/.manifest +53 -0
- data/.wrongdoc.yml +6 -0
- data/COPYING +661 -0
- data/ChangeLog +185 -0
- data/GIT-VERSION-FILE +1 -0
- data/GIT-VERSION-GEN +33 -0
- data/GNUmakefile +35 -0
- data/LATEST +1 -0
- data/NEWS +1 -0
- data/README +154 -0
- data/bin/omgdav-setup +4 -0
- data/bin/omgdav-sync +32 -0
- data/lib/omgdav/app.rb +70 -0
- data/lib/omgdav/copy.rb +100 -0
- data/lib/omgdav/copy_move.rb +54 -0
- data/lib/omgdav/db.rb +258 -0
- data/lib/omgdav/delete.rb +66 -0
- data/lib/omgdav/get.rb +35 -0
- data/lib/omgdav/http_get.rb +146 -0
- data/lib/omgdav/input_wrapper.rb +32 -0
- data/lib/omgdav/migrations/0001_initial.rb +45 -0
- data/lib/omgdav/migrations/0002_contenttype.rb +15 -0
- data/lib/omgdav/migrations/0003_synctmp.rb +14 -0
- data/lib/omgdav/mkcol.rb +28 -0
- data/lib/omgdav/move.rb +74 -0
- data/lib/omgdav/options.rb +21 -0
- data/lib/omgdav/propfind.rb +46 -0
- data/lib/omgdav/propfind_response.rb +150 -0
- data/lib/omgdav/proppatch.rb +116 -0
- data/lib/omgdav/put.rb +110 -0
- data/lib/omgdav/rack_util.rb +56 -0
- data/lib/omgdav/setup.rb +16 -0
- data/lib/omgdav/sync.rb +78 -0
- data/lib/omgdav/version.rb +2 -0
- data/lib/omgdav.rb +27 -0
- data/omgdav.gemspec +35 -0
- data/pkg.mk +175 -0
- data/setup.rb +1586 -0
- data/test/integration.rb +232 -0
- data/test/test_copy.rb +121 -0
- data/test/test_delete.rb +15 -0
- data/test/test_litmus.rb +61 -0
- data/test/test_move.rb +66 -0
- data/test/test_omgdav_app.rb +102 -0
- data/test/test_propfind.rb +30 -0
- data/test/test_proppatch.rb +156 -0
- data/test/test_put.rb +31 -0
- data/test/test_readonly.rb +22 -0
- data/test/test_sync.rb +49 -0
- data/test/test_urlmap.rb +59 -0
- data/test/test_worm.rb +26 -0
- metadata +342 -0
data/ChangeLog
ADDED
@@ -0,0 +1,185 @@
|
|
1
|
+
ChangeLog from http://bogomips.org/omgdav.git
|
2
|
+
|
3
|
+
commit 678249295f8d03b85073f157a6e91318942c439e
|
4
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
5
|
+
Date: Thu Nov 15 01:37:52 2012 +0000
|
6
|
+
|
7
|
+
doc: more updates
|
8
|
+
|
9
|
+
Ensure README readers can easily refer back to the README
|
10
|
+
file. Also include a link to MogileFS in case users need
|
11
|
+
MogileFS knowledge.
|
12
|
+
|
13
|
+
commit e728bd23000dc3bb9c99d281aefa903c6af2985c
|
14
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
15
|
+
Date: Thu Nov 15 01:23:04 2012 +0000
|
16
|
+
|
17
|
+
pkg: ensure bin/* is properly installed
|
18
|
+
|
19
|
+
We need setup commands for users.
|
20
|
+
|
21
|
+
commit ff98f5bd1664d35e801945ef09647bc232f669a9
|
22
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
23
|
+
Date: Thu Nov 15 00:59:40 2012 +0000
|
24
|
+
|
25
|
+
README: flesh this out with basic information
|
26
|
+
|
27
|
+
Hopefully it's enough for folks to get started.
|
28
|
+
|
29
|
+
commit 1a66258801b9fffd937464445cccf57197425f58
|
30
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
31
|
+
Date: Thu Nov 15 00:45:07 2012 +0000
|
32
|
+
|
33
|
+
import -> sync: import is now bidirectional
|
34
|
+
|
35
|
+
Deleted keys on the MogileFS side may now be reflected
|
36
|
+
by the omgdav database after running "omgdav-sync".
|
37
|
+
|
38
|
+
commit d62d85f0c282d91b292c092601e36aece371c892
|
39
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
40
|
+
Date: Wed Nov 14 23:27:41 2012 +0000
|
41
|
+
|
42
|
+
gemspec: add sequel dependency
|
43
|
+
|
44
|
+
We generally want the latest Sequel, but we do not enforce
|
45
|
+
database drivers.
|
46
|
+
|
47
|
+
commit 93e8dda9f0e082ba70cd798824b22b401db6721a
|
48
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
49
|
+
Date: Wed Nov 14 23:14:28 2012 +0000
|
50
|
+
|
51
|
+
GNUmakefile: "make check" runs all tests
|
52
|
+
|
53
|
+
This might be slightly more familiar to folks used to the same
|
54
|
+
target from autotools.
|
55
|
+
|
56
|
+
commit d0088069b48b0ce5abdf4c49805546c63e8b35b8
|
57
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
58
|
+
Date: Wed Oct 31 23:00:53 2012 +0000
|
59
|
+
|
60
|
+
documentation cleanups
|
61
|
+
|
62
|
+
Namely, there is no internal API documentation, this
|
63
|
+
is a front-facing Rack application.
|
64
|
+
|
65
|
+
commit 5bf0ea2fec1eafd169a84365212f405761758fe0
|
66
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
67
|
+
Date: Fri Oct 26 21:45:36 2012 +0000
|
68
|
+
|
69
|
+
force HTTP/1.0 keepalive connections with kcar
|
70
|
+
|
71
|
+
This reduces the number of TIME-WAIT sockets and should improve
|
72
|
+
performance on high-latency networks. We force HTTP/1.0 instead
|
73
|
+
of 1.1 to avoid the possibility of a server sending us chunked
|
74
|
+
responses (saving us the trouble of parsing)
|
75
|
+
|
76
|
+
Unlike net-http-persistent or curb, this keepalive
|
77
|
+
implementation is built to allow sharing of idle sockets between
|
78
|
+
multiple threads, improving socket reusability in multi-threaded
|
79
|
+
servers.
|
80
|
+
|
81
|
+
Also unlike net-http-persistent or curb, kcar was designed to
|
82
|
+
stream HTTP response bodies as a Rack response body. I could
|
83
|
+
not figure out a way to stream response bodies via nhp or curb
|
84
|
+
without resorting to Fibers or Threads.
|
85
|
+
|
86
|
+
commit 406bb0ba5bf51df583b792c7ef987e018226359d
|
87
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
88
|
+
Date: Fri Oct 26 01:35:46 2012 +0000
|
89
|
+
|
90
|
+
omgdav/app: fix shadow bug with error logging
|
91
|
+
|
92
|
+
Oops :x
|
93
|
+
|
94
|
+
commit 2c6b607ed55629ebc5d12580fe4ddb1d84201f7b
|
95
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
96
|
+
Date: Thu Oct 25 21:20:48 2012 +0000
|
97
|
+
|
98
|
+
app: support for :worm mode (write-once, read-many)
|
99
|
+
|
100
|
+
I find write-once, read-many filesytems a good compromise
|
101
|
+
between read-only and full read-write systems.
|
102
|
+
|
103
|
+
commit d4aad422e0d19b7626f09ac87741105186c0bf6c
|
104
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
105
|
+
Date: Wed Oct 24 21:56:22 2012 +0000
|
106
|
+
|
107
|
+
app: allow methods: argument to limit HTTP methods
|
108
|
+
|
109
|
+
This means it's easy for users to setup a read-only instance
|
110
|
+
for sharing files to users they do not trust to write.
|
111
|
+
|
112
|
+
OMGDAV::App.new(db, mogc, methods: :ro)
|
113
|
+
|
114
|
+
commit ddb98ea8df8f577bc7c4debbf12f7fec0e39be3b
|
115
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
116
|
+
Date: Wed Oct 24 08:53:27 2012 +0000
|
117
|
+
|
118
|
+
get: remove If-Modified-Since handling
|
119
|
+
|
120
|
+
We already track mtime in our database, so we could potentially
|
121
|
+
handle If-Modified-Since without ever hitting the tracker. We
|
122
|
+
can live without If-Modified-Since for now since WebDAV clients
|
123
|
+
like cadaver/fusedav do not seem to use it...
|
124
|
+
|
125
|
+
commit faed5bf668d2ae6eb47259637632c4fc39a468c4
|
126
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
127
|
+
Date: Wed Oct 24 08:44:21 2012 +0000
|
128
|
+
|
129
|
+
omgdav/app: common logger method
|
130
|
+
|
131
|
+
We hope rack.logger to exists, but it is an optional part
|
132
|
+
of the Rack SPEC, so just log to stderr if we must.
|
133
|
+
|
134
|
+
commit 326f6a661b1974f1d72edb324fd5c38b4e116c02
|
135
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
136
|
+
Date: Wed Oct 24 08:41:09 2012 +0000
|
137
|
+
|
138
|
+
omgdav/app: make the rack app fork-friendly
|
139
|
+
|
140
|
+
Some of our users may use unicorn...
|
141
|
+
|
142
|
+
commit e860ab06b39819fa571e9dc579cdc598986d4956
|
143
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
144
|
+
Date: Tue Oct 23 02:45:55 2012 +0000
|
145
|
+
|
146
|
+
support for the getcontenttype attribute
|
147
|
+
|
148
|
+
This live attribute may be overridden by the user using
|
149
|
+
PROPPATCH. fusedav supports this attribute as user.mime_type.
|
150
|
+
|
151
|
+
commit cdf97ba819da87f84c6440a7b1331a3469efe568
|
152
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
153
|
+
Date: Tue Oct 23 00:17:07 2012 +0000
|
154
|
+
|
155
|
+
use IO.copy_stream wherever available for PUT/COPY
|
156
|
+
|
157
|
+
We've never cared about about Ruby 1.8, so we don't need
|
158
|
+
to worry about IO.copy_stream being unavailable. This saves
|
159
|
+
us the trouble of having to worry about writing our own copy
|
160
|
+
loops and managing our own buffers.
|
161
|
+
|
162
|
+
commit 0c353ce9e9637206ecebf5d8f44b6c84bd4633b4
|
163
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
164
|
+
Date: Mon Oct 22 22:59:24 2012 +0000
|
165
|
+
|
166
|
+
app: stream large file uploads by default
|
167
|
+
|
168
|
+
This means we won't have to buffer large files in memory
|
169
|
+
and possibly OOM ourselves in the process.
|
170
|
+
|
171
|
+
commit 0eaedefc69ce6be31161e1cbb9e314dffc6ac809
|
172
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
173
|
+
Date: Mon Oct 22 22:47:46 2012 +0000
|
174
|
+
|
175
|
+
uri_request: cache packed addresses
|
176
|
+
|
177
|
+
Packing addresses generates a bit of garbage and costs
|
178
|
+
us several system calls on glibc for netlink, so cache
|
179
|
+
that result.
|
180
|
+
|
181
|
+
commit ab74d32ab68391aa6dac0472ee4aa8e64c9a9057
|
182
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
183
|
+
Date: Fri Oct 19 19:36:33 2012 +0000
|
184
|
+
|
185
|
+
initial commit
|
data/GIT-VERSION-FILE
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
GIT_VERSION = 0.0.0
|
data/GIT-VERSION-GEN
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
CONSTANT = "OMGDAV::VERSION"
|
3
|
+
RVF = "lib/omgdav/version.rb"
|
4
|
+
GVF = "GIT-VERSION-FILE"
|
5
|
+
DEF_VER = "v0.0.0"
|
6
|
+
vn = DEF_VER
|
7
|
+
|
8
|
+
# First see if there is a version file (included in release tarballs),
|
9
|
+
# then try git-describe, then default.
|
10
|
+
if File.exist?(".git")
|
11
|
+
describe = `git describe --abbrev=4 HEAD 2>/dev/null`.strip
|
12
|
+
case describe
|
13
|
+
when /\Av[0-9]*/
|
14
|
+
vn = describe
|
15
|
+
system(*%w(git update-index -q --refresh))
|
16
|
+
unless `git diff-index --name-only HEAD --`.chomp.empty?
|
17
|
+
vn << "-dirty"
|
18
|
+
end
|
19
|
+
vn.tr!('-', '.')
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
vn = vn.sub!(/\Av/, "")
|
24
|
+
new_ruby_version = "#{CONSTANT} = '#{vn}'\n"
|
25
|
+
cur_ruby_version = File.readlines(RVF)[0] rescue nil
|
26
|
+
if new_ruby_version != cur_ruby_version
|
27
|
+
File.open(GVF, "w") { |fp| fp.write("GIT_VERSION = #{vn}\n") }
|
28
|
+
File.open(RVF, "w") do |fp|
|
29
|
+
fp.write("# :enddoc:\n")
|
30
|
+
fp.write(new_ruby_version)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
puts vn if $0 == __FILE__
|
data/GNUmakefile
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
all::
|
2
|
+
RSYNC_DEST := bogomips.org:/srv/bogomips/omgdav
|
3
|
+
rfproject := rainbows
|
4
|
+
rfpackage := omgdav
|
5
|
+
pkg_extra += lib/omgdav/version.rb
|
6
|
+
include pkg.mk
|
7
|
+
|
8
|
+
LITMUS_VER = 0.13
|
9
|
+
LITMUS_TAR = litmus-$(LITMUS_VER).tar.gz
|
10
|
+
LITMUS_DIR = litmus-$(LITMUS_VER)
|
11
|
+
LITMUS_URL = http://www.webdav.org/neon/litmus/$(LITMUS_TAR)
|
12
|
+
LITMUS_SHA1 = 42ad603035d15798facb3be79b1c51376820cb19
|
13
|
+
CURL = curl
|
14
|
+
|
15
|
+
$(LITMUS_TAR):
|
16
|
+
$(CURL) -vsSf $(LITMUS_URL) > $@+
|
17
|
+
test $$(expr "$$(sha1sum $@+)" : '\([a-f0-9]\{40\}\)') = $(LITMUS_SHA1)
|
18
|
+
mv $@+ $@
|
19
|
+
|
20
|
+
|
21
|
+
$(LITMUS_DIR)/.omgdav: $(LITMUS_TAR)
|
22
|
+
tar xvf $<
|
23
|
+
> $@
|
24
|
+
$(LITMUS_DIR)/basic: $(LITMUS_DIR)/.omgdav
|
25
|
+
cd $(@D) && ./configure --prefix=$$PWD/i
|
26
|
+
$(MAKE) -C $(@D)
|
27
|
+
|
28
|
+
test-litmus: $(LITMUS_DIR)/basic
|
29
|
+
$(MAKE) LITMUS_DIR=$(LITMUS_DIR) MAKE="$(MAKE)" test/test_litmus.rb
|
30
|
+
|
31
|
+
check: test test-litmus
|
32
|
+
|
33
|
+
RSYNC = rsync --exclude '*.html' --exclude '*.html.gz' \
|
34
|
+
--exclude images --exclude '*.css' --exclude '*.css.gz' \
|
35
|
+
--exclude created.*
|
data/LATEST
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Currently unreleased
|
data/README
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
= omgdav - Rack app for bridging WebDAV and MogileFS
|
2
|
+
|
3
|
+
omgdav exposes an existing MogileFS domain over WebDAV. There is
|
4
|
+
absolutely no commitment or modification needed to your existing
|
5
|
+
MogileFS installation to try omgdav in read-only mode.
|
6
|
+
|
7
|
+
omgdav uses its own database and can import key listings from with an
|
8
|
+
existing MogileFS domain.
|
9
|
+
|
10
|
+
omgdav (barely :P) meets class 1 WebDAV compliance according to
|
11
|
+
litmus[1]. Class 2 compliance is planned.
|
12
|
+
|
13
|
+
*** WARNING ***
|
14
|
+
---------------
|
15
|
+
|
16
|
+
Locking of any type is not yet implemented. Thus:
|
17
|
+
|
18
|
+
1) Do not attempt concurrent invocations of COPY/MOVE commands.
|
19
|
+
|
20
|
+
2) Do not use PUT or DELETE while a COPY or MOVE command is running.
|
21
|
+
|
22
|
+
Read-only operation is currently the safest way to use omgdav.
|
23
|
+
|
24
|
+
Features
|
25
|
+
--------
|
26
|
+
|
27
|
+
* Allows read-only access to existing MogileFS data with a WebDAV client.
|
28
|
+
|
29
|
+
* Infers WebDAV collections based on existing MogileFS keys which
|
30
|
+
look like path names (e.g. "/parts/separated/by/slashes").
|
31
|
+
|
32
|
+
* Implemented for Rack, so it is compatible with existing middlewares
|
33
|
+
(for authentication, logging, etc...) and Rack web servers.
|
34
|
+
|
35
|
+
Requirements
|
36
|
+
------------
|
37
|
+
|
38
|
+
* an existing MogileFS instance
|
39
|
+
|
40
|
+
* Ruby 1.9.3 or later (any Ruby implementation compatible with C extensions)
|
41
|
+
|
42
|
+
* a Free, Unix-like operating system (GNU/Linux preferred)
|
43
|
+
|
44
|
+
* a Free database supported by Sequel[2]
|
45
|
+
|
46
|
+
Install (via RubyGems)
|
47
|
+
----------------------
|
48
|
+
|
49
|
+
gem install omgdav
|
50
|
+
|
51
|
+
Usage
|
52
|
+
-----
|
53
|
+
|
54
|
+
1. Setup your database:
|
55
|
+
|
56
|
+
# You may substitute the sqlite:/// database for any other
|
57
|
+
# Free database type Sequel supports. Be sure to have the
|
58
|
+
# appropriate database driver installed (e.g. "sqlite3").
|
59
|
+
omgdav-setup sqlite://test.sqlite
|
60
|
+
|
61
|
+
omgdav-setup runs idempotently. It will automatically upgrade your
|
62
|
+
schema as new versions of omgdav may require database updates
|
63
|
+
|
64
|
+
2. Sync metadata from your existing MogileFS domain:
|
65
|
+
|
66
|
+
omgdav-sync -t $TRACKER_HOST_PORT -d $DOMAIN sqlite://test.sqlite
|
67
|
+
|
68
|
+
omgdav-sync runs idempotently. It may be used to resync changes
|
69
|
+
to the omgdav database if keys are added/removed from MogileFS.
|
70
|
+
|
71
|
+
3. Configure your rackup config file, below is an example config.ru:
|
72
|
+
|
73
|
+
------------------------------------- 8<------------------------------------
|
74
|
+
require "omgdav/app"
|
75
|
+
# replace hosts: and domain: with values suitable for your cluster
|
76
|
+
mogc = MogileFS::MogileFS.new(hosts: %w(127.0.0.1:7001), domain: "test")
|
77
|
+
db = Sequel.connect("sqlite://test.sqlite")
|
78
|
+
|
79
|
+
# for the brave: :ro ("read-only") may be swapped for :rw ("read-write")
|
80
|
+
# :worm is also supported ("write-once, read-many")
|
81
|
+
run OMGDAV::App.new(db, mogc, methods: :ro)
|
82
|
+
------------------------------------- 8<------------------------------------
|
83
|
+
|
84
|
+
4. Start your Rack HTTP server and point your WebDAV client at it.
|
85
|
+
|
86
|
+
rackup -s webrick -p $PORT config.ru
|
87
|
+
|
88
|
+
Code
|
89
|
+
----
|
90
|
+
|
91
|
+
Source code is available via git:
|
92
|
+
|
93
|
+
git clone git://bogomips.org/omgdav.git
|
94
|
+
git clone git://repo.or.cz/omgdav.git
|
95
|
+
|
96
|
+
And viewable with a WWW browser via cgit or gitweb:
|
97
|
+
|
98
|
+
http://bogomips.org/omgdav.git (cgit)
|
99
|
+
http://repo.or.cz/w/omgdav.git (gitweb)
|
100
|
+
|
101
|
+
Contact
|
102
|
+
-------
|
103
|
+
|
104
|
+
Bug reports, user/development discussion, patches, pull requests are
|
105
|
+
greatly appreciated via plain-text email.
|
106
|
+
|
107
|
+
We currently piggy-back onto the public MogileFS
|
108
|
+
mailing list at mailto:mogile@googlegroups.com for feedback.
|
109
|
+
|
110
|
+
If you do not want to deal with the corporate host of the MogileFS
|
111
|
+
mailing list or if you wish to keep your issue secret, feel free to
|
112
|
+
email Eric Wong directly at mailto:normalperson@yhbt.net
|
113
|
+
|
114
|
+
HTML email will be mercilessly deleted.
|
115
|
+
|
116
|
+
License
|
117
|
+
-------
|
118
|
+
omgdav is copyrighted Free Software by all contributors, see logs in
|
119
|
+
revision control for names and email addresses of all of them.
|
120
|
+
|
121
|
+
You can redistribute omgdav and/or modify it under the terms of the GNU
|
122
|
+
Affero General Public License, version 3 or later as published by the
|
123
|
+
Free Software Foundation.
|
124
|
+
|
125
|
+
omgdav is distributed in the hope that it will be useful, but WITHOUT
|
126
|
+
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
127
|
+
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
|
128
|
+
License for more details.
|
129
|
+
|
130
|
+
You should have received a copy of the GNU Affero General Public License
|
131
|
+
along with omgdav; if not, see https://www.gnu.org/licenses/agpl.txt
|
132
|
+
|
133
|
+
fusedav
|
134
|
+
-------
|
135
|
+
|
136
|
+
Some testing and use of omgdav is done on fusedav[3]. Since fusedav
|
137
|
+
upstream appears dormant, we have publicized some bug reports and
|
138
|
+
patches on the Debian bug tracker[4]. Our fusedav patches are
|
139
|
+
also available via various branches in our git repository:
|
140
|
+
|
141
|
+
git clone git://bogomips.org/fusedav
|
142
|
+
|
143
|
+
Links
|
144
|
+
-----
|
145
|
+
MogileFS - http://mogilefs.org/
|
146
|
+
omgdav homepage - http://bogomips.org/omgdav/README
|
147
|
+
omgdav Atom feed - http://bogomips.org/omgdav/NEWS.atom.xml
|
148
|
+
|
149
|
+
References
|
150
|
+
----------
|
151
|
+
[1] litmus - http://www.webdav.org/neon/litmus
|
152
|
+
[2] Sequel - http://sequel.rubyforge.org/
|
153
|
+
[3] fusedav - http://0pointer.de/lennart/projects/fusedav/
|
154
|
+
[4] fusedav (Debian BTS) - http://bugs.debian.org/fusedav
|
data/bin/omgdav-setup
ADDED
data/bin/omgdav-sync
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$stderr.sync = $stdout.sync = true
|
3
|
+
require "optparse"
|
4
|
+
require "sequel"
|
5
|
+
require "mogilefs"
|
6
|
+
require "omgdav/sync"
|
7
|
+
|
8
|
+
mfs_opts = { hosts: %w(127.0.0.1:7001), domain: "test" }
|
9
|
+
|
10
|
+
opts = OptionParser.new do |opts|
|
11
|
+
opts.banner = "Usage: omgdav-sync [options] <uri|path>"
|
12
|
+
opts.separator "Examples:"
|
13
|
+
opts.separator " omgdav-sync -t 127.0.0.1:7001 -d domain sqlite://foo.db"
|
14
|
+
|
15
|
+
opts.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
|
16
|
+
'hostnames/IP addresses of trackers') do |hosts|
|
17
|
+
mfs_opts[:hosts] = hosts
|
18
|
+
end
|
19
|
+
opts.on('-d', '--domain=s', 'domain') do |domain|
|
20
|
+
mfs_opts[:domain] = domain
|
21
|
+
end
|
22
|
+
opts.parse!
|
23
|
+
end
|
24
|
+
|
25
|
+
db = ARGV.shift
|
26
|
+
db = Sequel.connect(db)
|
27
|
+
db.pragma_set(:synchronous, :off) if db.respond_to?(:pragma_set)
|
28
|
+
db.transaction_mode = :immediate if db.respond_to?(:transaction_mode=)
|
29
|
+
mogc = MogileFS::MogileFS.new(mfs_opts)
|
30
|
+
db.transaction do
|
31
|
+
OMGDAV::Sync.new(db, mogc).sync
|
32
|
+
end
|
data/lib/omgdav/app.rb
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# :stopdoc:
|
3
|
+
# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
|
4
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
|
+
require 'logger'
|
6
|
+
require 'omgdav'
|
7
|
+
require 'omgdav/db'
|
8
|
+
require 'omgdav/http_get'
|
9
|
+
# :startdoc:
|
10
|
+
|
11
|
+
class OMGDAV::App # :nodoc:
|
12
|
+
include OMGDAV::DB
|
13
|
+
|
14
|
+
attr_accessor :create_full_put_path
|
15
|
+
attr_accessor :new_file_opts
|
16
|
+
attr_accessor :get_path_opts
|
17
|
+
|
18
|
+
# Example (for Rack config.ru):
|
19
|
+
#
|
20
|
+
# db = Sequel.connect("sqlite://foo.sqlite")
|
21
|
+
# mogc = MogileFS::MogileFS.new(hosts: %w(127.0.0.1:7001), domain: "test")
|
22
|
+
# run OMGDAV::App.new(db, mogc, methods: :worm)
|
23
|
+
def initialize(db, mogc, opts = {})
|
24
|
+
@call_map = {}
|
25
|
+
@db = db
|
26
|
+
@mogc = mogc
|
27
|
+
@domain_id = ensure_domain(mogc.domain)
|
28
|
+
@create_full_put_path = opts[:create_full_put_path] || false
|
29
|
+
@new_file_opts = opts[:new_file_opts] || { largefile: :stream }
|
30
|
+
@get_path_opts = opts[:get_path_opts] || {}
|
31
|
+
@sql_limit = 100
|
32
|
+
@root_node = nil
|
33
|
+
|
34
|
+
# avoid contention by loading this first
|
35
|
+
root_node
|
36
|
+
@db.disconnect # unicorn may load this before forking, don't share
|
37
|
+
@worm = false
|
38
|
+
|
39
|
+
ro_methods = %w(GET PROPFIND OPTIONS)
|
40
|
+
rw_methods = %w(PUT DELETE MKCOL COPY MOVE PROPPATCH)
|
41
|
+
case opts[:methods]
|
42
|
+
when nil, :rw
|
43
|
+
rmethods = ro_methods + rw_methods
|
44
|
+
when :ro
|
45
|
+
rmethods = ro_methods
|
46
|
+
when :worm
|
47
|
+
@worm = true
|
48
|
+
rmethods = ro_methods + %w(PUT MKCOL COPY)
|
49
|
+
else
|
50
|
+
rmethods = opts[:methods]
|
51
|
+
end
|
52
|
+
rmethods.each do |m|
|
53
|
+
m = m.to_s
|
54
|
+
require "omgdav/#{m.downcase}"
|
55
|
+
extend OMGDAV.const_get(m.capitalize)
|
56
|
+
@call_map[m.upcase] = method("call_#{m.downcase}")
|
57
|
+
end
|
58
|
+
get = @call_map["GET"] and @call_map["HEAD"] = get
|
59
|
+
@call_map.default = proc { |_| r(405) }
|
60
|
+
end
|
61
|
+
|
62
|
+
def call(env) # :nodoc:
|
63
|
+
@call_map[env["REQUEST_METHOD"]].call(env)
|
64
|
+
rescue => e
|
65
|
+
l = logger(env)
|
66
|
+
l.error("#{e.message} (#{e.class})")
|
67
|
+
e.backtrace.each { |line| l.error(line) }
|
68
|
+
r(500)
|
69
|
+
end
|
70
|
+
end
|
data/lib/omgdav/copy.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# :enddoc:
|
3
|
+
# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
|
4
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
|
+
require "omgdav/copy_move"
|
6
|
+
require "omgdav/rack_util"
|
7
|
+
require "omgdav/db"
|
8
|
+
|
9
|
+
module OMGDAV::Copy
|
10
|
+
include OMGDAV::DB
|
11
|
+
include OMGDAV::RackUtil
|
12
|
+
include OMGDAV::CopyMove
|
13
|
+
|
14
|
+
def call_copy(env)
|
15
|
+
src, dst = {}, {}
|
16
|
+
err = copy_move_prepare!(env, src, dst) and return err
|
17
|
+
|
18
|
+
if src[:node][:collection]
|
19
|
+
case env["HTTP_DEPTH"]
|
20
|
+
when "0"
|
21
|
+
dst_node = col_ensure(dst[:parent][:id], dst[:basename])
|
22
|
+
when nil, "infinity"
|
23
|
+
dst_node = col_ensure(dst[:parent][:id], dst[:basename])
|
24
|
+
copy_collection(src[:node], dst_node)
|
25
|
+
else
|
26
|
+
r(400, "invalid Depth: #{env['HTTP_DEPTH']}")
|
27
|
+
end
|
28
|
+
else
|
29
|
+
copy_to_col(src[:node], dst[:parent], {}, dst[:basename])
|
30
|
+
end
|
31
|
+
|
32
|
+
# overwrite or new file?
|
33
|
+
r(dst[:node] ? 204 : 201)
|
34
|
+
end
|
35
|
+
|
36
|
+
# recursively copies src_node to dst_node
|
37
|
+
def copy_collection(src_node, dst_node)
|
38
|
+
cache = {}
|
39
|
+
paths = @db[:paths]
|
40
|
+
queue = [ [ 0, src_node, dst_node ] ]
|
41
|
+
max_id = paths.max(:id)
|
42
|
+
q = { domain_id: @domain_id }
|
43
|
+
|
44
|
+
while cur_job = queue.pop
|
45
|
+
min_id, cur_src, dst_col = cur_job
|
46
|
+
q[:parent_id] = cur_src[:id]
|
47
|
+
next if min_id == max_id
|
48
|
+
begin
|
49
|
+
continue = false
|
50
|
+
q[:id] = ((min_id+1)..max_id)
|
51
|
+
paths.order(:id).where(q).limit(@sql_limit).each do |child_node|
|
52
|
+
min_id = child_node[:id]
|
53
|
+
if child_node[:collection]
|
54
|
+
queue << [ min_id, cur_src, dst_col ]
|
55
|
+
dst_parent = col_ensure(dst_col[:id], child_node[:name])
|
56
|
+
queue << [ 0, child_node, dst_parent ]
|
57
|
+
continue = false
|
58
|
+
break
|
59
|
+
else
|
60
|
+
copy_to_col(child_node, dst_col, cache)
|
61
|
+
continue = true
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end while continue
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def copy_to_col(src_file, dst_col, cache, dst_name = src_file[:name])
|
69
|
+
src_key = node_to_key(src_file, cache)
|
70
|
+
|
71
|
+
dst_parts = node_to_parts(dst_col, cache) << dst_name
|
72
|
+
dst_key = dst_parts.join("/")
|
73
|
+
|
74
|
+
opts = @new_file_opts.dup
|
75
|
+
info = @mogc.file_info(src_key)
|
76
|
+
opts[:content_length] = src_file[:length]
|
77
|
+
opts[:class] = info["class"]
|
78
|
+
checksum = info["checksum"] and opts[:checksum] = checksum
|
79
|
+
bytes = @mogc.new_file(dst_key, opts) do |io|
|
80
|
+
@mogc.get_uris(src_key, @get_path_opts).each do |uri|
|
81
|
+
case res = OMGDAV::HttpGet.run(nil, uri)
|
82
|
+
when Array
|
83
|
+
res[2].stream_to(io)
|
84
|
+
break
|
85
|
+
else
|
86
|
+
logger(env).error("#{uri}: #{res.message} (#{res.class})")
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
if src_file[:length] != bytes
|
92
|
+
warn "length mismatch copying(#{src_key.inspect}>#{dst_key.inspect})" \
|
93
|
+
"(expected(#{src_file[:length]}) != copied(#{bytes}))"
|
94
|
+
end
|
95
|
+
|
96
|
+
info = { "length" => bytes }
|
97
|
+
dst_node = file_ensure(dst_col[:id], dst_name, info)
|
98
|
+
dst_node
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# :enddoc:
|
3
|
+
# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
|
4
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
|
+
require "omgdav/rack_util"
|
6
|
+
require "omgdav/db"
|
7
|
+
require "omgdav/delete"
|
8
|
+
|
9
|
+
module OMGDAV::CopyMove
|
10
|
+
include OMGDAV::DB
|
11
|
+
include OMGDAV::RackUtil
|
12
|
+
include OMGDAV::Delete
|
13
|
+
|
14
|
+
def copy_move_prepare!(env, src, dst)
|
15
|
+
dst_uri = env["HTTP_DESTINATION"] or return r(400)
|
16
|
+
dst_uri = URI(dst_uri)
|
17
|
+
req_uri = URI(Rack::Request.new(env).url)
|
18
|
+
return r(502) if dst_uri.host != req_uri.host
|
19
|
+
return r(502) if dst_uri.port != req_uri.port
|
20
|
+
|
21
|
+
script_name = Regexp.quote(env["SCRIPT_NAME"])
|
22
|
+
dst_path = dst_uri.path.dup
|
23
|
+
return r(502) unless dst_path.gsub!(%r{\A#{script_name}/}, "/")
|
24
|
+
|
25
|
+
dst[:parts] = path_split("PATH_INFO" => dst_path)
|
26
|
+
dst[:basename] = dst[:parts].pop
|
27
|
+
dst[:parent] = col_resolve(dst[:parts]) or return r(409)
|
28
|
+
dst[:node] = node_lookup(dst[:parent][:id], dst[:basename])
|
29
|
+
dst[:parts] << dst[:basename]
|
30
|
+
dst[:key] = dst[:parts].join("/")
|
31
|
+
|
32
|
+
case env["HTTP_OVERWRITE"]
|
33
|
+
when "F"
|
34
|
+
dst[:node] and return r(412)
|
35
|
+
when "T", nil
|
36
|
+
(@worm && dst[:node]) and return r(409)
|
37
|
+
else
|
38
|
+
return r(400, "Overwrite: #{env['HTTP_OVERWRITE']} not understood")
|
39
|
+
end
|
40
|
+
|
41
|
+
src[:parts] = path_split(env)
|
42
|
+
src[:basename] = src[:parts].pop
|
43
|
+
src[:parent] = col_resolve(src[:parts]) or return r(404)
|
44
|
+
src[:parts] << src[:basename]
|
45
|
+
src[:node] = node_lookup(src[:parent][:id], src[:basename]) or return r(404)
|
46
|
+
src[:key] = src[:parts].join("/")
|
47
|
+
|
48
|
+
return r(403) if src[:key] == dst[:key]
|
49
|
+
|
50
|
+
delete_entry(dst[:node], {}) if dst[:node]
|
51
|
+
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
end
|