omgdav 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +3 -0
- data/.gitignore +17 -0
- data/.manifest +53 -0
- data/.wrongdoc.yml +6 -0
- data/COPYING +661 -0
- data/ChangeLog +185 -0
- data/GIT-VERSION-FILE +1 -0
- data/GIT-VERSION-GEN +33 -0
- data/GNUmakefile +35 -0
- data/LATEST +1 -0
- data/NEWS +1 -0
- data/README +154 -0
- data/bin/omgdav-setup +4 -0
- data/bin/omgdav-sync +32 -0
- data/lib/omgdav/app.rb +70 -0
- data/lib/omgdav/copy.rb +100 -0
- data/lib/omgdav/copy_move.rb +54 -0
- data/lib/omgdav/db.rb +258 -0
- data/lib/omgdav/delete.rb +66 -0
- data/lib/omgdav/get.rb +35 -0
- data/lib/omgdav/http_get.rb +146 -0
- data/lib/omgdav/input_wrapper.rb +32 -0
- data/lib/omgdav/migrations/0001_initial.rb +45 -0
- data/lib/omgdav/migrations/0002_contenttype.rb +15 -0
- data/lib/omgdav/migrations/0003_synctmp.rb +14 -0
- data/lib/omgdav/mkcol.rb +28 -0
- data/lib/omgdav/move.rb +74 -0
- data/lib/omgdav/options.rb +21 -0
- data/lib/omgdav/propfind.rb +46 -0
- data/lib/omgdav/propfind_response.rb +150 -0
- data/lib/omgdav/proppatch.rb +116 -0
- data/lib/omgdav/put.rb +110 -0
- data/lib/omgdav/rack_util.rb +56 -0
- data/lib/omgdav/setup.rb +16 -0
- data/lib/omgdav/sync.rb +78 -0
- data/lib/omgdav/version.rb +2 -0
- data/lib/omgdav.rb +27 -0
- data/omgdav.gemspec +35 -0
- data/pkg.mk +175 -0
- data/setup.rb +1586 -0
- data/test/integration.rb +232 -0
- data/test/test_copy.rb +121 -0
- data/test/test_delete.rb +15 -0
- data/test/test_litmus.rb +61 -0
- data/test/test_move.rb +66 -0
- data/test/test_omgdav_app.rb +102 -0
- data/test/test_propfind.rb +30 -0
- data/test/test_proppatch.rb +156 -0
- data/test/test_put.rb +31 -0
- data/test/test_readonly.rb +22 -0
- data/test/test_sync.rb +49 -0
- data/test/test_urlmap.rb +59 -0
- data/test/test_worm.rb +26 -0
- metadata +342 -0
data/ChangeLog
ADDED
@@ -0,0 +1,185 @@
|
|
1
|
+
ChangeLog from http://bogomips.org/omgdav.git
|
2
|
+
|
3
|
+
commit 678249295f8d03b85073f157a6e91318942c439e
|
4
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
5
|
+
Date: Thu Nov 15 01:37:52 2012 +0000
|
6
|
+
|
7
|
+
doc: more updates
|
8
|
+
|
9
|
+
Ensure README readers can easily refer back to the README
|
10
|
+
file. Also include a link to MogileFS in case users need
|
11
|
+
MogileFS knowledge.
|
12
|
+
|
13
|
+
commit e728bd23000dc3bb9c99d281aefa903c6af2985c
|
14
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
15
|
+
Date: Thu Nov 15 01:23:04 2012 +0000
|
16
|
+
|
17
|
+
pkg: ensure bin/* is properly installed
|
18
|
+
|
19
|
+
We need setup commands for users.
|
20
|
+
|
21
|
+
commit ff98f5bd1664d35e801945ef09647bc232f669a9
|
22
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
23
|
+
Date: Thu Nov 15 00:59:40 2012 +0000
|
24
|
+
|
25
|
+
README: flesh this out with basic information
|
26
|
+
|
27
|
+
Hopefully it's enough for folks to get started.
|
28
|
+
|
29
|
+
commit 1a66258801b9fffd937464445cccf57197425f58
|
30
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
31
|
+
Date: Thu Nov 15 00:45:07 2012 +0000
|
32
|
+
|
33
|
+
import -> sync: import is now bidirectional
|
34
|
+
|
35
|
+
Deleted keys on the MogileFS side may now be reflected
|
36
|
+
by the omgdav database after running "omgdav-sync".
|
37
|
+
|
38
|
+
commit d62d85f0c282d91b292c092601e36aece371c892
|
39
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
40
|
+
Date: Wed Nov 14 23:27:41 2012 +0000
|
41
|
+
|
42
|
+
gemspec: add sequel dependency
|
43
|
+
|
44
|
+
We generally want the latest Sequel, but we do not enforce
|
45
|
+
database drivers.
|
46
|
+
|
47
|
+
commit 93e8dda9f0e082ba70cd798824b22b401db6721a
|
48
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
49
|
+
Date: Wed Nov 14 23:14:28 2012 +0000
|
50
|
+
|
51
|
+
GNUmakefile: "make check" runs all tests
|
52
|
+
|
53
|
+
This might be slightly more familiar to folks used to the same
|
54
|
+
target from autotools.
|
55
|
+
|
56
|
+
commit d0088069b48b0ce5abdf4c49805546c63e8b35b8
|
57
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
58
|
+
Date: Wed Oct 31 23:00:53 2012 +0000
|
59
|
+
|
60
|
+
documentation cleanups
|
61
|
+
|
62
|
+
Namely, there is no internal API documentation, this
|
63
|
+
is a front-facing Rack application.
|
64
|
+
|
65
|
+
commit 5bf0ea2fec1eafd169a84365212f405761758fe0
|
66
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
67
|
+
Date: Fri Oct 26 21:45:36 2012 +0000
|
68
|
+
|
69
|
+
force HTTP/1.0 keepalive connections with kcar
|
70
|
+
|
71
|
+
This reduces the number of TIME-WAIT sockets and should improve
|
72
|
+
performance on high-latency networks. We force HTTP/1.0 instead
|
73
|
+
of 1.1 to avoid the possibility of a server sending us chunked
|
74
|
+
responses (saving us the trouble of parsing)
|
75
|
+
|
76
|
+
Unlike net-http-persistent or curb, this keepalive
|
77
|
+
implementation is built to allow sharing of idle sockets between
|
78
|
+
multiple threads, improving socket reusability in multi-threaded
|
79
|
+
servers.
|
80
|
+
|
81
|
+
Also unlike net-http-persistent or curb, kcar was designed to
|
82
|
+
stream HTTP response bodies as a Rack response body. I could
|
83
|
+
not figure out a way to stream response bodies via nhp or curb
|
84
|
+
without resorting to Fibers or Threads.
|
85
|
+
|
86
|
+
commit 406bb0ba5bf51df583b792c7ef987e018226359d
|
87
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
88
|
+
Date: Fri Oct 26 01:35:46 2012 +0000
|
89
|
+
|
90
|
+
omgdav/app: fix shadow bug with error logging
|
91
|
+
|
92
|
+
Oops :x
|
93
|
+
|
94
|
+
commit 2c6b607ed55629ebc5d12580fe4ddb1d84201f7b
|
95
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
96
|
+
Date: Thu Oct 25 21:20:48 2012 +0000
|
97
|
+
|
98
|
+
app: support for :worm mode (write-once, read-many)
|
99
|
+
|
100
|
+
I find write-once, read-many filesytems a good compromise
|
101
|
+
between read-only and full read-write systems.
|
102
|
+
|
103
|
+
commit d4aad422e0d19b7626f09ac87741105186c0bf6c
|
104
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
105
|
+
Date: Wed Oct 24 21:56:22 2012 +0000
|
106
|
+
|
107
|
+
app: allow methods: argument to limit HTTP methods
|
108
|
+
|
109
|
+
This means it's easy for users to setup a read-only instance
|
110
|
+
for sharing files to users they do not trust to write.
|
111
|
+
|
112
|
+
OMGDAV::App.new(db, mogc, methods: :ro)
|
113
|
+
|
114
|
+
commit ddb98ea8df8f577bc7c4debbf12f7fec0e39be3b
|
115
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
116
|
+
Date: Wed Oct 24 08:53:27 2012 +0000
|
117
|
+
|
118
|
+
get: remove If-Modified-Since handling
|
119
|
+
|
120
|
+
We already track mtime in our database, so we could potentially
|
121
|
+
handle If-Modified-Since without ever hitting the tracker. We
|
122
|
+
can live without If-Modified-Since for now since WebDAV clients
|
123
|
+
like cadaver/fusedav do not seem to use it...
|
124
|
+
|
125
|
+
commit faed5bf668d2ae6eb47259637632c4fc39a468c4
|
126
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
127
|
+
Date: Wed Oct 24 08:44:21 2012 +0000
|
128
|
+
|
129
|
+
omgdav/app: common logger method
|
130
|
+
|
131
|
+
We hope rack.logger to exists, but it is an optional part
|
132
|
+
of the Rack SPEC, so just log to stderr if we must.
|
133
|
+
|
134
|
+
commit 326f6a661b1974f1d72edb324fd5c38b4e116c02
|
135
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
136
|
+
Date: Wed Oct 24 08:41:09 2012 +0000
|
137
|
+
|
138
|
+
omgdav/app: make the rack app fork-friendly
|
139
|
+
|
140
|
+
Some of our users may use unicorn...
|
141
|
+
|
142
|
+
commit e860ab06b39819fa571e9dc579cdc598986d4956
|
143
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
144
|
+
Date: Tue Oct 23 02:45:55 2012 +0000
|
145
|
+
|
146
|
+
support for the getcontenttype attribute
|
147
|
+
|
148
|
+
This live attribute may be overridden by the user using
|
149
|
+
PROPPATCH. fusedav supports this attribute as user.mime_type.
|
150
|
+
|
151
|
+
commit cdf97ba819da87f84c6440a7b1331a3469efe568
|
152
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
153
|
+
Date: Tue Oct 23 00:17:07 2012 +0000
|
154
|
+
|
155
|
+
use IO.copy_stream wherever available for PUT/COPY
|
156
|
+
|
157
|
+
We've never cared about about Ruby 1.8, so we don't need
|
158
|
+
to worry about IO.copy_stream being unavailable. This saves
|
159
|
+
us the trouble of having to worry about writing our own copy
|
160
|
+
loops and managing our own buffers.
|
161
|
+
|
162
|
+
commit 0c353ce9e9637206ecebf5d8f44b6c84bd4633b4
|
163
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
164
|
+
Date: Mon Oct 22 22:59:24 2012 +0000
|
165
|
+
|
166
|
+
app: stream large file uploads by default
|
167
|
+
|
168
|
+
This means we won't have to buffer large files in memory
|
169
|
+
and possibly OOM ourselves in the process.
|
170
|
+
|
171
|
+
commit 0eaedefc69ce6be31161e1cbb9e314dffc6ac809
|
172
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
173
|
+
Date: Mon Oct 22 22:47:46 2012 +0000
|
174
|
+
|
175
|
+
uri_request: cache packed addresses
|
176
|
+
|
177
|
+
Packing addresses generates a bit of garbage and costs
|
178
|
+
us several system calls on glibc for netlink, so cache
|
179
|
+
that result.
|
180
|
+
|
181
|
+
commit ab74d32ab68391aa6dac0472ee4aa8e64c9a9057
|
182
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
183
|
+
Date: Fri Oct 19 19:36:33 2012 +0000
|
184
|
+
|
185
|
+
initial commit
|
data/GIT-VERSION-FILE
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
GIT_VERSION = 0.0.0
|
data/GIT-VERSION-GEN
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
CONSTANT = "OMGDAV::VERSION"
|
3
|
+
RVF = "lib/omgdav/version.rb"
|
4
|
+
GVF = "GIT-VERSION-FILE"
|
5
|
+
DEF_VER = "v0.0.0"
|
6
|
+
vn = DEF_VER
|
7
|
+
|
8
|
+
# First see if there is a version file (included in release tarballs),
|
9
|
+
# then try git-describe, then default.
|
10
|
+
if File.exist?(".git")
|
11
|
+
describe = `git describe --abbrev=4 HEAD 2>/dev/null`.strip
|
12
|
+
case describe
|
13
|
+
when /\Av[0-9]*/
|
14
|
+
vn = describe
|
15
|
+
system(*%w(git update-index -q --refresh))
|
16
|
+
unless `git diff-index --name-only HEAD --`.chomp.empty?
|
17
|
+
vn << "-dirty"
|
18
|
+
end
|
19
|
+
vn.tr!('-', '.')
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
vn = vn.sub!(/\Av/, "")
|
24
|
+
new_ruby_version = "#{CONSTANT} = '#{vn}'\n"
|
25
|
+
cur_ruby_version = File.readlines(RVF)[0] rescue nil
|
26
|
+
if new_ruby_version != cur_ruby_version
|
27
|
+
File.open(GVF, "w") { |fp| fp.write("GIT_VERSION = #{vn}\n") }
|
28
|
+
File.open(RVF, "w") do |fp|
|
29
|
+
fp.write("# :enddoc:\n")
|
30
|
+
fp.write(new_ruby_version)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
puts vn if $0 == __FILE__
|
data/GNUmakefile
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
all::
|
2
|
+
RSYNC_DEST := bogomips.org:/srv/bogomips/omgdav
|
3
|
+
rfproject := rainbows
|
4
|
+
rfpackage := omgdav
|
5
|
+
pkg_extra += lib/omgdav/version.rb
|
6
|
+
include pkg.mk
|
7
|
+
|
8
|
+
LITMUS_VER = 0.13
|
9
|
+
LITMUS_TAR = litmus-$(LITMUS_VER).tar.gz
|
10
|
+
LITMUS_DIR = litmus-$(LITMUS_VER)
|
11
|
+
LITMUS_URL = http://www.webdav.org/neon/litmus/$(LITMUS_TAR)
|
12
|
+
LITMUS_SHA1 = 42ad603035d15798facb3be79b1c51376820cb19
|
13
|
+
CURL = curl
|
14
|
+
|
15
|
+
$(LITMUS_TAR):
|
16
|
+
$(CURL) -vsSf $(LITMUS_URL) > $@+
|
17
|
+
test $$(expr "$$(sha1sum $@+)" : '\([a-f0-9]\{40\}\)') = $(LITMUS_SHA1)
|
18
|
+
mv $@+ $@
|
19
|
+
|
20
|
+
|
21
|
+
$(LITMUS_DIR)/.omgdav: $(LITMUS_TAR)
|
22
|
+
tar xvf $<
|
23
|
+
> $@
|
24
|
+
$(LITMUS_DIR)/basic: $(LITMUS_DIR)/.omgdav
|
25
|
+
cd $(@D) && ./configure --prefix=$$PWD/i
|
26
|
+
$(MAKE) -C $(@D)
|
27
|
+
|
28
|
+
test-litmus: $(LITMUS_DIR)/basic
|
29
|
+
$(MAKE) LITMUS_DIR=$(LITMUS_DIR) MAKE="$(MAKE)" test/test_litmus.rb
|
30
|
+
|
31
|
+
check: test test-litmus
|
32
|
+
|
33
|
+
RSYNC = rsync --exclude '*.html' --exclude '*.html.gz' \
|
34
|
+
--exclude images --exclude '*.css' --exclude '*.css.gz' \
|
35
|
+
--exclude created.*
|
data/LATEST
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Currently unreleased
|
data/README
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
= omgdav - Rack app for bridging WebDAV and MogileFS
|
2
|
+
|
3
|
+
omgdav exposes an existing MogileFS domain over WebDAV. There is
|
4
|
+
absolutely no commitment or modification needed to your existing
|
5
|
+
MogileFS installation to try omgdav in read-only mode.
|
6
|
+
|
7
|
+
omgdav uses its own database and can import key listings from with an
|
8
|
+
existing MogileFS domain.
|
9
|
+
|
10
|
+
omgdav (barely :P) meets class 1 WebDAV compliance according to
|
11
|
+
litmus[1]. Class 2 compliance is planned.
|
12
|
+
|
13
|
+
*** WARNING ***
|
14
|
+
---------------
|
15
|
+
|
16
|
+
Locking of any type is not yet implemented. Thus:
|
17
|
+
|
18
|
+
1) Do not attempt concurrent invocations of COPY/MOVE commands.
|
19
|
+
|
20
|
+
2) Do not use PUT or DELETE while a COPY or MOVE command is running.
|
21
|
+
|
22
|
+
Read-only operation is currently the safest way to use omgdav.
|
23
|
+
|
24
|
+
Features
|
25
|
+
--------
|
26
|
+
|
27
|
+
* Allows read-only access to existing MogileFS data with a WebDAV client.
|
28
|
+
|
29
|
+
* Infers WebDAV collections based on existing MogileFS keys which
|
30
|
+
look like path names (e.g. "/parts/separated/by/slashes").
|
31
|
+
|
32
|
+
* Implemented for Rack, so it is compatible with existing middlewares
|
33
|
+
(for authentication, logging, etc...) and Rack web servers.
|
34
|
+
|
35
|
+
Requirements
|
36
|
+
------------
|
37
|
+
|
38
|
+
* an existing MogileFS instance
|
39
|
+
|
40
|
+
* Ruby 1.9.3 or later (any Ruby implementation compatible with C extensions)
|
41
|
+
|
42
|
+
* a Free, Unix-like operating system (GNU/Linux preferred)
|
43
|
+
|
44
|
+
* a Free database supported by Sequel[2]
|
45
|
+
|
46
|
+
Install (via RubyGems)
|
47
|
+
----------------------
|
48
|
+
|
49
|
+
gem install omgdav
|
50
|
+
|
51
|
+
Usage
|
52
|
+
-----
|
53
|
+
|
54
|
+
1. Setup your database:
|
55
|
+
|
56
|
+
# You may substitute the sqlite:/// database for any other
|
57
|
+
# Free database type Sequel supports. Be sure to have the
|
58
|
+
# appropriate database driver installed (e.g. "sqlite3").
|
59
|
+
omgdav-setup sqlite://test.sqlite
|
60
|
+
|
61
|
+
omgdav-setup runs idempotently. It will automatically upgrade your
|
62
|
+
schema as new versions of omgdav may require database updates
|
63
|
+
|
64
|
+
2. Sync metadata from your existing MogileFS domain:
|
65
|
+
|
66
|
+
omgdav-sync -t $TRACKER_HOST_PORT -d $DOMAIN sqlite://test.sqlite
|
67
|
+
|
68
|
+
omgdav-sync runs idempotently. It may be used to resync changes
|
69
|
+
to the omgdav database if keys are added/removed from MogileFS.
|
70
|
+
|
71
|
+
3. Configure your rackup config file, below is an example config.ru:
|
72
|
+
|
73
|
+
------------------------------------- 8<------------------------------------
|
74
|
+
require "omgdav/app"
|
75
|
+
# replace hosts: and domain: with values suitable for your cluster
|
76
|
+
mogc = MogileFS::MogileFS.new(hosts: %w(127.0.0.1:7001), domain: "test")
|
77
|
+
db = Sequel.connect("sqlite://test.sqlite")
|
78
|
+
|
79
|
+
# for the brave: :ro ("read-only") may be swapped for :rw ("read-write")
|
80
|
+
# :worm is also supported ("write-once, read-many")
|
81
|
+
run OMGDAV::App.new(db, mogc, methods: :ro)
|
82
|
+
------------------------------------- 8<------------------------------------
|
83
|
+
|
84
|
+
4. Start your Rack HTTP server and point your WebDAV client at it.
|
85
|
+
|
86
|
+
rackup -s webrick -p $PORT config.ru
|
87
|
+
|
88
|
+
Code
|
89
|
+
----
|
90
|
+
|
91
|
+
Source code is available via git:
|
92
|
+
|
93
|
+
git clone git://bogomips.org/omgdav.git
|
94
|
+
git clone git://repo.or.cz/omgdav.git
|
95
|
+
|
96
|
+
And viewable with a WWW browser via cgit or gitweb:
|
97
|
+
|
98
|
+
http://bogomips.org/omgdav.git (cgit)
|
99
|
+
http://repo.or.cz/w/omgdav.git (gitweb)
|
100
|
+
|
101
|
+
Contact
|
102
|
+
-------
|
103
|
+
|
104
|
+
Bug reports, user/development discussion, patches, pull requests are
|
105
|
+
greatly appreciated via plain-text email.
|
106
|
+
|
107
|
+
We currently piggy-back onto the public MogileFS
|
108
|
+
mailing list at mailto:mogile@googlegroups.com for feedback.
|
109
|
+
|
110
|
+
If you do not want to deal with the corporate host of the MogileFS
|
111
|
+
mailing list or if you wish to keep your issue secret, feel free to
|
112
|
+
email Eric Wong directly at mailto:normalperson@yhbt.net
|
113
|
+
|
114
|
+
HTML email will be mercilessly deleted.
|
115
|
+
|
116
|
+
License
|
117
|
+
-------
|
118
|
+
omgdav is copyrighted Free Software by all contributors, see logs in
|
119
|
+
revision control for names and email addresses of all of them.
|
120
|
+
|
121
|
+
You can redistribute omgdav and/or modify it under the terms of the GNU
|
122
|
+
Affero General Public License, version 3 or later as published by the
|
123
|
+
Free Software Foundation.
|
124
|
+
|
125
|
+
omgdav is distributed in the hope that it will be useful, but WITHOUT
|
126
|
+
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
127
|
+
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
|
128
|
+
License for more details.
|
129
|
+
|
130
|
+
You should have received a copy of the GNU Affero General Public License
|
131
|
+
along with omgdav; if not, see https://www.gnu.org/licenses/agpl.txt
|
132
|
+
|
133
|
+
fusedav
|
134
|
+
-------
|
135
|
+
|
136
|
+
Some testing and use of omgdav is done on fusedav[3]. Since fusedav
|
137
|
+
upstream appears dormant, we have publicized some bug reports and
|
138
|
+
patches on the Debian bug tracker[4]. Our fusedav patches are
|
139
|
+
also available via various branches in our git repository:
|
140
|
+
|
141
|
+
git clone git://bogomips.org/fusedav
|
142
|
+
|
143
|
+
Links
|
144
|
+
-----
|
145
|
+
MogileFS - http://mogilefs.org/
|
146
|
+
omgdav homepage - http://bogomips.org/omgdav/README
|
147
|
+
omgdav Atom feed - http://bogomips.org/omgdav/NEWS.atom.xml
|
148
|
+
|
149
|
+
References
|
150
|
+
----------
|
151
|
+
[1] litmus - http://www.webdav.org/neon/litmus
|
152
|
+
[2] Sequel - http://sequel.rubyforge.org/
|
153
|
+
[3] fusedav - http://0pointer.de/lennart/projects/fusedav/
|
154
|
+
[4] fusedav (Debian BTS) - http://bugs.debian.org/fusedav
|
data/bin/omgdav-setup
ADDED
data/bin/omgdav-sync
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$stderr.sync = $stdout.sync = true
|
3
|
+
require "optparse"
|
4
|
+
require "sequel"
|
5
|
+
require "mogilefs"
|
6
|
+
require "omgdav/sync"
|
7
|
+
|
8
|
+
mfs_opts = { hosts: %w(127.0.0.1:7001), domain: "test" }
|
9
|
+
|
10
|
+
opts = OptionParser.new do |opts|
|
11
|
+
opts.banner = "Usage: omgdav-sync [options] <uri|path>"
|
12
|
+
opts.separator "Examples:"
|
13
|
+
opts.separator " omgdav-sync -t 127.0.0.1:7001 -d domain sqlite://foo.db"
|
14
|
+
|
15
|
+
opts.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
|
16
|
+
'hostnames/IP addresses of trackers') do |hosts|
|
17
|
+
mfs_opts[:hosts] = hosts
|
18
|
+
end
|
19
|
+
opts.on('-d', '--domain=s', 'domain') do |domain|
|
20
|
+
mfs_opts[:domain] = domain
|
21
|
+
end
|
22
|
+
opts.parse!
|
23
|
+
end
|
24
|
+
|
25
|
+
db = ARGV.shift
|
26
|
+
db = Sequel.connect(db)
|
27
|
+
db.pragma_set(:synchronous, :off) if db.respond_to?(:pragma_set)
|
28
|
+
db.transaction_mode = :immediate if db.respond_to?(:transaction_mode=)
|
29
|
+
mogc = MogileFS::MogileFS.new(mfs_opts)
|
30
|
+
db.transaction do
|
31
|
+
OMGDAV::Sync.new(db, mogc).sync
|
32
|
+
end
|
data/lib/omgdav/app.rb
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# :stopdoc:
|
3
|
+
# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
|
4
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
|
+
require 'logger'
|
6
|
+
require 'omgdav'
|
7
|
+
require 'omgdav/db'
|
8
|
+
require 'omgdav/http_get'
|
9
|
+
# :startdoc:
|
10
|
+
|
11
|
+
class OMGDAV::App # :nodoc:
|
12
|
+
include OMGDAV::DB
|
13
|
+
|
14
|
+
attr_accessor :create_full_put_path
|
15
|
+
attr_accessor :new_file_opts
|
16
|
+
attr_accessor :get_path_opts
|
17
|
+
|
18
|
+
# Example (for Rack config.ru):
|
19
|
+
#
|
20
|
+
# db = Sequel.connect("sqlite://foo.sqlite")
|
21
|
+
# mogc = MogileFS::MogileFS.new(hosts: %w(127.0.0.1:7001), domain: "test")
|
22
|
+
# run OMGDAV::App.new(db, mogc, methods: :worm)
|
23
|
+
def initialize(db, mogc, opts = {})
|
24
|
+
@call_map = {}
|
25
|
+
@db = db
|
26
|
+
@mogc = mogc
|
27
|
+
@domain_id = ensure_domain(mogc.domain)
|
28
|
+
@create_full_put_path = opts[:create_full_put_path] || false
|
29
|
+
@new_file_opts = opts[:new_file_opts] || { largefile: :stream }
|
30
|
+
@get_path_opts = opts[:get_path_opts] || {}
|
31
|
+
@sql_limit = 100
|
32
|
+
@root_node = nil
|
33
|
+
|
34
|
+
# avoid contention by loading this first
|
35
|
+
root_node
|
36
|
+
@db.disconnect # unicorn may load this before forking, don't share
|
37
|
+
@worm = false
|
38
|
+
|
39
|
+
ro_methods = %w(GET PROPFIND OPTIONS)
|
40
|
+
rw_methods = %w(PUT DELETE MKCOL COPY MOVE PROPPATCH)
|
41
|
+
case opts[:methods]
|
42
|
+
when nil, :rw
|
43
|
+
rmethods = ro_methods + rw_methods
|
44
|
+
when :ro
|
45
|
+
rmethods = ro_methods
|
46
|
+
when :worm
|
47
|
+
@worm = true
|
48
|
+
rmethods = ro_methods + %w(PUT MKCOL COPY)
|
49
|
+
else
|
50
|
+
rmethods = opts[:methods]
|
51
|
+
end
|
52
|
+
rmethods.each do |m|
|
53
|
+
m = m.to_s
|
54
|
+
require "omgdav/#{m.downcase}"
|
55
|
+
extend OMGDAV.const_get(m.capitalize)
|
56
|
+
@call_map[m.upcase] = method("call_#{m.downcase}")
|
57
|
+
end
|
58
|
+
get = @call_map["GET"] and @call_map["HEAD"] = get
|
59
|
+
@call_map.default = proc { |_| r(405) }
|
60
|
+
end
|
61
|
+
|
62
|
+
def call(env) # :nodoc:
|
63
|
+
@call_map[env["REQUEST_METHOD"]].call(env)
|
64
|
+
rescue => e
|
65
|
+
l = logger(env)
|
66
|
+
l.error("#{e.message} (#{e.class})")
|
67
|
+
e.backtrace.each { |line| l.error(line) }
|
68
|
+
r(500)
|
69
|
+
end
|
70
|
+
end
|
data/lib/omgdav/copy.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# :enddoc:
|
3
|
+
# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
|
4
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
|
+
require "omgdav/copy_move"
|
6
|
+
require "omgdav/rack_util"
|
7
|
+
require "omgdav/db"
|
8
|
+
|
9
|
+
module OMGDAV::Copy
|
10
|
+
include OMGDAV::DB
|
11
|
+
include OMGDAV::RackUtil
|
12
|
+
include OMGDAV::CopyMove
|
13
|
+
|
14
|
+
def call_copy(env)
|
15
|
+
src, dst = {}, {}
|
16
|
+
err = copy_move_prepare!(env, src, dst) and return err
|
17
|
+
|
18
|
+
if src[:node][:collection]
|
19
|
+
case env["HTTP_DEPTH"]
|
20
|
+
when "0"
|
21
|
+
dst_node = col_ensure(dst[:parent][:id], dst[:basename])
|
22
|
+
when nil, "infinity"
|
23
|
+
dst_node = col_ensure(dst[:parent][:id], dst[:basename])
|
24
|
+
copy_collection(src[:node], dst_node)
|
25
|
+
else
|
26
|
+
r(400, "invalid Depth: #{env['HTTP_DEPTH']}")
|
27
|
+
end
|
28
|
+
else
|
29
|
+
copy_to_col(src[:node], dst[:parent], {}, dst[:basename])
|
30
|
+
end
|
31
|
+
|
32
|
+
# overwrite or new file?
|
33
|
+
r(dst[:node] ? 204 : 201)
|
34
|
+
end
|
35
|
+
|
36
|
+
# recursively copies src_node to dst_node
|
37
|
+
def copy_collection(src_node, dst_node)
|
38
|
+
cache = {}
|
39
|
+
paths = @db[:paths]
|
40
|
+
queue = [ [ 0, src_node, dst_node ] ]
|
41
|
+
max_id = paths.max(:id)
|
42
|
+
q = { domain_id: @domain_id }
|
43
|
+
|
44
|
+
while cur_job = queue.pop
|
45
|
+
min_id, cur_src, dst_col = cur_job
|
46
|
+
q[:parent_id] = cur_src[:id]
|
47
|
+
next if min_id == max_id
|
48
|
+
begin
|
49
|
+
continue = false
|
50
|
+
q[:id] = ((min_id+1)..max_id)
|
51
|
+
paths.order(:id).where(q).limit(@sql_limit).each do |child_node|
|
52
|
+
min_id = child_node[:id]
|
53
|
+
if child_node[:collection]
|
54
|
+
queue << [ min_id, cur_src, dst_col ]
|
55
|
+
dst_parent = col_ensure(dst_col[:id], child_node[:name])
|
56
|
+
queue << [ 0, child_node, dst_parent ]
|
57
|
+
continue = false
|
58
|
+
break
|
59
|
+
else
|
60
|
+
copy_to_col(child_node, dst_col, cache)
|
61
|
+
continue = true
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end while continue
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def copy_to_col(src_file, dst_col, cache, dst_name = src_file[:name])
|
69
|
+
src_key = node_to_key(src_file, cache)
|
70
|
+
|
71
|
+
dst_parts = node_to_parts(dst_col, cache) << dst_name
|
72
|
+
dst_key = dst_parts.join("/")
|
73
|
+
|
74
|
+
opts = @new_file_opts.dup
|
75
|
+
info = @mogc.file_info(src_key)
|
76
|
+
opts[:content_length] = src_file[:length]
|
77
|
+
opts[:class] = info["class"]
|
78
|
+
checksum = info["checksum"] and opts[:checksum] = checksum
|
79
|
+
bytes = @mogc.new_file(dst_key, opts) do |io|
|
80
|
+
@mogc.get_uris(src_key, @get_path_opts).each do |uri|
|
81
|
+
case res = OMGDAV::HttpGet.run(nil, uri)
|
82
|
+
when Array
|
83
|
+
res[2].stream_to(io)
|
84
|
+
break
|
85
|
+
else
|
86
|
+
logger(env).error("#{uri}: #{res.message} (#{res.class})")
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
if src_file[:length] != bytes
|
92
|
+
warn "length mismatch copying(#{src_key.inspect}>#{dst_key.inspect})" \
|
93
|
+
"(expected(#{src_file[:length]}) != copied(#{bytes}))"
|
94
|
+
end
|
95
|
+
|
96
|
+
info = { "length" => bytes }
|
97
|
+
dst_node = file_ensure(dst_col[:id], dst_name, info)
|
98
|
+
dst_node
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# :enddoc:
|
3
|
+
# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
|
4
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
|
+
require "omgdav/rack_util"
|
6
|
+
require "omgdav/db"
|
7
|
+
require "omgdav/delete"
|
8
|
+
|
9
|
+
module OMGDAV::CopyMove
|
10
|
+
include OMGDAV::DB
|
11
|
+
include OMGDAV::RackUtil
|
12
|
+
include OMGDAV::Delete
|
13
|
+
|
14
|
+
def copy_move_prepare!(env, src, dst)
|
15
|
+
dst_uri = env["HTTP_DESTINATION"] or return r(400)
|
16
|
+
dst_uri = URI(dst_uri)
|
17
|
+
req_uri = URI(Rack::Request.new(env).url)
|
18
|
+
return r(502) if dst_uri.host != req_uri.host
|
19
|
+
return r(502) if dst_uri.port != req_uri.port
|
20
|
+
|
21
|
+
script_name = Regexp.quote(env["SCRIPT_NAME"])
|
22
|
+
dst_path = dst_uri.path.dup
|
23
|
+
return r(502) unless dst_path.gsub!(%r{\A#{script_name}/}, "/")
|
24
|
+
|
25
|
+
dst[:parts] = path_split("PATH_INFO" => dst_path)
|
26
|
+
dst[:basename] = dst[:parts].pop
|
27
|
+
dst[:parent] = col_resolve(dst[:parts]) or return r(409)
|
28
|
+
dst[:node] = node_lookup(dst[:parent][:id], dst[:basename])
|
29
|
+
dst[:parts] << dst[:basename]
|
30
|
+
dst[:key] = dst[:parts].join("/")
|
31
|
+
|
32
|
+
case env["HTTP_OVERWRITE"]
|
33
|
+
when "F"
|
34
|
+
dst[:node] and return r(412)
|
35
|
+
when "T", nil
|
36
|
+
(@worm && dst[:node]) and return r(409)
|
37
|
+
else
|
38
|
+
return r(400, "Overwrite: #{env['HTTP_OVERWRITE']} not understood")
|
39
|
+
end
|
40
|
+
|
41
|
+
src[:parts] = path_split(env)
|
42
|
+
src[:basename] = src[:parts].pop
|
43
|
+
src[:parent] = col_resolve(src[:parts]) or return r(404)
|
44
|
+
src[:parts] << src[:basename]
|
45
|
+
src[:node] = node_lookup(src[:parent][:id], src[:basename]) or return r(404)
|
46
|
+
src[:key] = src[:parts].join("/")
|
47
|
+
|
48
|
+
return r(403) if src[:key] == dst[:key]
|
49
|
+
|
50
|
+
delete_entry(dst[:node], {}) if dst[:node]
|
51
|
+
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
end
|