lsst-git-lfs-s3 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +18 -1
- data/git-lfs-s3.gemspec +3 -3
- data/lib/git-lfs-s3/application.rb +151 -0
- data/lib/git-lfs-s3/services/upload/object_exists.rb +1 -1
- data/lib/git-lfs-s3/services/upload/upload_required.rb +2 -2
- data/lib/git-lfs-s3/version.rb +1 -1
- metadata +17 -17
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 68caf4d6acf139d3cc4f120bc7715b7fc129a444
|
|
4
|
+
data.tar.gz: 78d03cd5f3d5428049b1f9f6918e78109bd97208
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1d1d01d2052bda7b8a93b82f0f9c62e387225f3c46dbfb0eef75d6b581814af58c805364c67c7a1a74cbf91b476cca929a860c4b442214a614183b9793b8e4a6
|
|
7
|
+
data.tar.gz: ce0faca488560ff2f38fdaa6589c56d6c8c33d578af4d3be7a1f8a15c474523118392161ec2582b61ff81f32889845edeb9f61e34a1aa0ea96aafb7d82e506f7
|
data/README.md
CHANGED
|
@@ -29,6 +29,9 @@ All configuration is done via environment variables. All of these configuration
|
|
|
29
29
|
* `AWS_SECRET_ACCESS_KEY` - your AWS secret key.
|
|
30
30
|
* `S3_BUCKET` - the bucket you wish to use for LFS storage. While not required, I recommend using a dedicated bucket for this.
|
|
31
31
|
* `LFS_SERVER_URL` - the URL where this server can be reached; needed to fetch download URLs.
|
|
32
|
+
* `LFS_PUBLIC_SERVER` - (Optional) Support anonymous users for safe operations such as git-clone and git-pull.
|
|
33
|
+
* `LFS_CEPH_S3` - (Optional) support [Ceph S3](http://ceph.com/) (Hammer version, through radosgw).
|
|
34
|
+
* `LFS_CEPH_ENDPOINT` - (Optional) the Ceph S3 endpoint URL. Required when using `LFS_CEPH_S3`.
|
|
32
35
|
|
|
33
36
|
You can (and should) also set authentication information. When you push for the first time from git, you will be prompted to enter a username and password when authentication is enabled. You can configure these with environment variables as well.
|
|
34
37
|
|
|
@@ -65,6 +68,7 @@ If you are new to Git LFS, make sure you read the [Getting Started](https://git-
|
|
|
65
68
|
``` git
|
|
66
69
|
[lfs]
|
|
67
70
|
url = "http://yourserver.com"
|
|
71
|
+
batch = false
|
|
68
72
|
```
|
|
69
73
|
|
|
70
74
|
Once that is done, you can tell Git LFS to track files with `git lfs track "*.psd"`, for example.
|
|
@@ -83,6 +87,19 @@ However, because this is a Sinatra application, it can also be mounted within ot
|
|
|
83
87
|
mount GitLfsS3::Application => '/lfs'
|
|
84
88
|
```
|
|
85
89
|
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
Allow anonymous clones and pulls of the git-lfs git repository.
|
|
93
|
+
|
|
94
|
+
## Ceph S3 Server
|
|
95
|
+
|
|
96
|
+
Through the `LFS_CEPH_S3` and `LFS_CEPH_ENDPOINT` environment variables use a Ceph S3 storage service (Hammer version, through radosgw) instead of AWS S3.
|
|
97
|
+
|
|
98
|
+
## More Complex Server Example
|
|
99
|
+
|
|
100
|
+
* https://github.com/lsst-sqre/git-lfs-s3-server
|
|
101
|
+
|
|
86
102
|
## TODO
|
|
87
103
|
|
|
88
|
-
* Cloudfront support
|
|
104
|
+
* Cloudfront support.
|
|
105
|
+
* Batch API support.
|
data/git-lfs-s3.gemspec
CHANGED
|
@@ -8,9 +8,9 @@ Gem::Specification.new do |gem|
|
|
|
8
8
|
gem.version = GitLfsS3::VERSION
|
|
9
9
|
gem.authors = ["Ryan LeFevre", "J. Matt Peterson"]
|
|
10
10
|
gem.email = ["meltingice8917@gmail.com", "jmatt@lsst.org"]
|
|
11
|
-
gem.description = %q{
|
|
12
|
-
gem.summary = %q{
|
|
13
|
-
gem.homepage = "https://github.com/
|
|
11
|
+
gem.description = %q{A Git LFS server that uses S3 for the storage backend.}
|
|
12
|
+
gem.summary = %q{A Git LFS server that uses S3 for the storage backend by providing presigned S3 URLs.}
|
|
13
|
+
gem.homepage = "https://github.com/meltingice/git-lfs-s3"
|
|
14
14
|
gem.license = 'MIT'
|
|
15
15
|
|
|
16
16
|
gem.files = `git ls-files`.split($/)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require 'date'
|
|
2
|
+
|
|
1
3
|
module GitLfsS3
|
|
2
4
|
class Application < Sinatra::Application
|
|
3
5
|
include AwsHelpers
|
|
@@ -47,6 +49,145 @@ module GitLfsS3
|
|
|
47
49
|
"Git LFS S3 is online."
|
|
48
50
|
end
|
|
49
51
|
|
|
52
|
+
def valid_object?(object)
|
|
53
|
+
begin
|
|
54
|
+
valid = object[:size] >= 0
|
|
55
|
+
rescue
|
|
56
|
+
valid = false
|
|
57
|
+
end
|
|
58
|
+
begin
|
|
59
|
+
if valid
|
|
60
|
+
oid = object[:oid].hex
|
|
61
|
+
valid = oid.size == 32 && object[:oid].size == 64
|
|
62
|
+
end
|
|
63
|
+
rescue
|
|
64
|
+
valid = false
|
|
65
|
+
end
|
|
66
|
+
valid
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def object_download(authenticated, object, object_json)
|
|
70
|
+
oid = object_json[:oid]
|
|
71
|
+
size = object_json[:size]
|
|
72
|
+
{
|
|
73
|
+
'oid' => oid,
|
|
74
|
+
'size' => size,
|
|
75
|
+
'authenticated' => authenticated,
|
|
76
|
+
'actions' => {
|
|
77
|
+
'download' => {
|
|
78
|
+
'href' => object.presigned_url(:get, :expires_in => 86400)
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
'expires_at' => DateTime.now.next_day.to_time.utc.iso8601
|
|
82
|
+
}
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def object_upload(authenticated, object, object_json)
|
|
86
|
+
# Format a single upload object.
|
|
87
|
+
oid = object_json[:oid]
|
|
88
|
+
size = object_json[:size]
|
|
89
|
+
{
|
|
90
|
+
'oid' => oid,
|
|
91
|
+
'size' => size,
|
|
92
|
+
'authenticated' => authenticated,
|
|
93
|
+
'actions' => {
|
|
94
|
+
'upload' => {
|
|
95
|
+
'href' => object.presigned_url(:put, acl: 'public-read', :expires_in => 86400)
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
'expires_at' => DateTime.now.next_day.to_time.utc.iso8601
|
|
99
|
+
}
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def object_error(error, message, object, object_json)
|
|
103
|
+
{
|
|
104
|
+
'oid' => object_json[:oid],
|
|
105
|
+
'size' => object_json[:size],
|
|
106
|
+
'error' => {
|
|
107
|
+
'code' => error,
|
|
108
|
+
'message' => message
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def download(authenticated, params)
|
|
114
|
+
objects = Array.new
|
|
115
|
+
params[:objects].each do |object_json|
|
|
116
|
+
object_json = indifferent_params object_json
|
|
117
|
+
object = object_data object_json[:oid]
|
|
118
|
+
if valid_object? object_json
|
|
119
|
+
if object.exists?
|
|
120
|
+
objects.push object_download(authenticated, object, object_json)
|
|
121
|
+
else
|
|
122
|
+
objects.push object_error(404, 'Object does not exist', object, object_json)
|
|
123
|
+
end
|
|
124
|
+
else
|
|
125
|
+
objects.push object_error(422, 'Validation error', object, object_json)
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
objects
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def upload(authenticated, params)
|
|
132
|
+
objects = Array.new
|
|
133
|
+
params[:objects].each do |object_json|
|
|
134
|
+
object_json = indifferent_params object_json
|
|
135
|
+
object = object_data object_json[:oid]
|
|
136
|
+
if valid_object? object_json
|
|
137
|
+
if object.exists?
|
|
138
|
+
objects.push object_download(authenticated, object, object_json)
|
|
139
|
+
else
|
|
140
|
+
objects.push object_upload(authenticated, object, object_json)
|
|
141
|
+
end
|
|
142
|
+
else
|
|
143
|
+
objects.push object_error(422, 'Validation error', object, object_json)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
objects
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def lfs_resp(objects)
|
|
150
|
+
status 200
|
|
151
|
+
resp = {
|
|
152
|
+
'transfer' => 'basic',
|
|
153
|
+
'objects' => objects
|
|
154
|
+
}
|
|
155
|
+
logger.debug resp
|
|
156
|
+
body MultiJson.dump(resp)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def error_resp(status_code, message)
|
|
160
|
+
status status_code
|
|
161
|
+
resp = {
|
|
162
|
+
'message' => message,
|
|
163
|
+
'request_id' => SecureRandom::uuid
|
|
164
|
+
}
|
|
165
|
+
logger.debug resp
|
|
166
|
+
body MultiJson.dump(resp)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
post '/objects/batch', provides: 'application/vnd.git-lfs+json' do
|
|
170
|
+
# Git LFS Batch API
|
|
171
|
+
authenticated = authorized?
|
|
172
|
+
params = indifferent_params(JSON.parse(request.body.read))
|
|
173
|
+
|
|
174
|
+
if params[:operation] == 'download'
|
|
175
|
+
objects = download(authenticated, params)
|
|
176
|
+
elsif params[:operation] == 'upload'
|
|
177
|
+
if authenticated
|
|
178
|
+
objects = upload(authenticated, params)
|
|
179
|
+
else
|
|
180
|
+
objects = nil
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
if objects
|
|
185
|
+
lfs_resp(objects)
|
|
186
|
+
else
|
|
187
|
+
error_resp(401, 'Credentials needed')
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
50
191
|
get "/objects/:oid", provides: 'application/vnd.git-lfs+json' do
|
|
51
192
|
object = object_data(params[:oid])
|
|
52
193
|
|
|
@@ -94,6 +235,16 @@ module GitLfsS3
|
|
|
94
235
|
body MultiJson.dump(service.response)
|
|
95
236
|
end
|
|
96
237
|
|
|
238
|
+
post "/objects/batch", provides: 'application/vnd.git-lfs+json' do
|
|
239
|
+
logger.debug headers.inspect
|
|
240
|
+
service = UploadBatchService.service_for(request.body)
|
|
241
|
+
logger.debug service.response
|
|
242
|
+
|
|
243
|
+
status service.status
|
|
244
|
+
body MultiJson.dump(service.response)
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
|
|
97
248
|
post '/verify', provides: 'application/vnd.git-lfs+json' do
|
|
98
249
|
data = MultiJson.load(request.body.tap { |b| b.rewind }.read)
|
|
99
250
|
object = object_data(data['oid'])
|
|
@@ -32,9 +32,9 @@ module GitLfsS3
|
|
|
32
32
|
GitLfsS3::CephPresignerService::signed_url(object)
|
|
33
33
|
else
|
|
34
34
|
if GitLfsS3::Application.settings.public_server
|
|
35
|
-
object.presigned_url(:put, acl: 'public-read')
|
|
35
|
+
object.presigned_url(:put, acl: 'public-read', :expires_in => 86400)
|
|
36
36
|
else
|
|
37
|
-
object.presigned_url(:put)
|
|
37
|
+
object.presigned_url(:put, :expires_in => 86400)
|
|
38
38
|
end
|
|
39
39
|
end
|
|
40
40
|
end
|
data/lib/git-lfs-s3/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: lsst-git-lfs-s3
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ryan LeFevre
|
|
@@ -9,65 +9,65 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2017-02-02 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: aws-sdk
|
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
|
17
17
|
requirements:
|
|
18
|
-
- - ~>
|
|
18
|
+
- - "~>"
|
|
19
19
|
- !ruby/object:Gem::Version
|
|
20
20
|
version: '2'
|
|
21
21
|
type: :runtime
|
|
22
22
|
prerelease: false
|
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
|
24
24
|
requirements:
|
|
25
|
-
- - ~>
|
|
25
|
+
- - "~>"
|
|
26
26
|
- !ruby/object:Gem::Version
|
|
27
27
|
version: '2'
|
|
28
28
|
- !ruby/object:Gem::Dependency
|
|
29
29
|
name: sinatra
|
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
|
31
31
|
requirements:
|
|
32
|
-
- - ~>
|
|
32
|
+
- - "~>"
|
|
33
33
|
- !ruby/object:Gem::Version
|
|
34
34
|
version: '1'
|
|
35
35
|
type: :runtime
|
|
36
36
|
prerelease: false
|
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
|
38
38
|
requirements:
|
|
39
|
-
- - ~>
|
|
39
|
+
- - "~>"
|
|
40
40
|
- !ruby/object:Gem::Version
|
|
41
41
|
version: '1'
|
|
42
42
|
- !ruby/object:Gem::Dependency
|
|
43
43
|
name: multi_json
|
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
|
45
45
|
requirements:
|
|
46
|
-
- - ~>
|
|
46
|
+
- - "~>"
|
|
47
47
|
- !ruby/object:Gem::Version
|
|
48
48
|
version: '1'
|
|
49
49
|
type: :runtime
|
|
50
50
|
prerelease: false
|
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
|
52
52
|
requirements:
|
|
53
|
-
- - ~>
|
|
53
|
+
- - "~>"
|
|
54
54
|
- !ruby/object:Gem::Version
|
|
55
55
|
version: '1'
|
|
56
56
|
- !ruby/object:Gem::Dependency
|
|
57
57
|
name: rake
|
|
58
58
|
requirement: !ruby/object:Gem::Requirement
|
|
59
59
|
requirements:
|
|
60
|
-
- - ~>
|
|
60
|
+
- - "~>"
|
|
61
61
|
- !ruby/object:Gem::Version
|
|
62
62
|
version: '10'
|
|
63
63
|
type: :development
|
|
64
64
|
prerelease: false
|
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
|
66
66
|
requirements:
|
|
67
|
-
- - ~>
|
|
67
|
+
- - "~>"
|
|
68
68
|
- !ruby/object:Gem::Version
|
|
69
69
|
version: '10'
|
|
70
|
-
description:
|
|
70
|
+
description: A Git LFS server that uses S3 for the storage backend.
|
|
71
71
|
email:
|
|
72
72
|
- meltingice8917@gmail.com
|
|
73
73
|
- jmatt@lsst.org
|
|
@@ -76,7 +76,7 @@ executables:
|
|
|
76
76
|
extensions: []
|
|
77
77
|
extra_rdoc_files: []
|
|
78
78
|
files:
|
|
79
|
-
- .gitignore
|
|
79
|
+
- ".gitignore"
|
|
80
80
|
- Gemfile
|
|
81
81
|
- README.md
|
|
82
82
|
- Rakefile
|
|
@@ -91,7 +91,7 @@ files:
|
|
|
91
91
|
- lib/git-lfs-s3/services/upload/object_exists.rb
|
|
92
92
|
- lib/git-lfs-s3/services/upload/upload_required.rb
|
|
93
93
|
- lib/git-lfs-s3/version.rb
|
|
94
|
-
homepage: https://github.com/
|
|
94
|
+
homepage: https://github.com/meltingice/git-lfs-s3
|
|
95
95
|
licenses:
|
|
96
96
|
- MIT
|
|
97
97
|
metadata: {}
|
|
@@ -101,19 +101,19 @@ require_paths:
|
|
|
101
101
|
- lib
|
|
102
102
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
103
103
|
requirements:
|
|
104
|
-
- -
|
|
104
|
+
- - ">="
|
|
105
105
|
- !ruby/object:Gem::Version
|
|
106
106
|
version: '0'
|
|
107
107
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
108
108
|
requirements:
|
|
109
|
-
- -
|
|
109
|
+
- - ">="
|
|
110
110
|
- !ruby/object:Gem::Version
|
|
111
111
|
version: '0'
|
|
112
112
|
requirements: []
|
|
113
113
|
rubyforge_project:
|
|
114
|
-
rubygems_version: 2.
|
|
114
|
+
rubygems_version: 2.6.10
|
|
115
115
|
signing_key:
|
|
116
116
|
specification_version: 4
|
|
117
|
-
summary:
|
|
117
|
+
summary: A Git LFS server that uses S3 for the storage backend by providing presigned
|
|
118
118
|
S3 URLs.
|
|
119
119
|
test_files: []
|