lsst-git-lfs-s3 0.2.6 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f6ba3c617f42a8dc43d5f39c4f2baa77acd8a084
-  data.tar.gz: c35b0e7293b2c9a343fa52c3cad6bebfbe48a94a
+  metadata.gz: 68caf4d6acf139d3cc4f120bc7715b7fc129a444
+  data.tar.gz: 78d03cd5f3d5428049b1f9f6918e78109bd97208
 SHA512:
-  metadata.gz: 84c16ef08fd5020073398b9c4ab50270f2154880d318dd8cbefdbc90030e3c90dde046d40089012041b8813398f51b255b0711ba93378bd66cf2867271972ddd
-  data.tar.gz: 6456b0f1741b306b5b6894159ce82da2cf5527114a966e8912fd7f6d8e4b459bdc6e410b9e5a71ba19a5f3c275c6f7665490cc86d20d11639021d62f2d274265
+  metadata.gz: 1d1d01d2052bda7b8a93b82f0f9c62e387225f3c46dbfb0eef75d6b581814af58c805364c67c7a1a74cbf91b476cca929a860c4b442214a614183b9793b8e4a6
+  data.tar.gz: ce0faca488560ff2f38fdaa6589c56d6c8c33d578af4d3be7a1f8a15c474523118392161ec2582b61ff81f32889845edeb9f61e34a1aa0ea96aafb7d82e506f7

data/README.md

@@ -29,6 +29,9 @@ All configuration is done via environment variables. All of these configuration
 * `AWS_SECRET_ACCESS_KEY` - your AWS secret key.
 * `S3_BUCKET` - the bucket you wish to use for LFS storage. While not required, I recommend using a dedicated bucket for this.
 * `LFS_SERVER_URL` - the URL where this server can be reached; needed to fetch download URLs.
+* `LFS_PUBLIC_SERVER` - (Optional) Support anonymous users for safe operations such as git-clone and git-pull.
+* `LFS_CEPH_S3` - (Optional) support [Ceph S3](http://ceph.com/) (Hammer version, through radosgw).
+* `LFS_CEPH_ENDPOINT` - (Optional) the Ceph S3 endpoint URL. Required when using `LFS_CEPH_S3`.
 
 You can (and should) also set authentication information. When you push for the first time from git, you will be prompted to enter a username and password when authentication is enabled. You can configure these with environment variables as well.
 
@@ -65,6 +68,7 @@ If you are new to Git LFS, make sure you read the [Getting Started](https://git-
 ``` git
 [lfs]
     url = "http://yourserver.com"
+    batch = false
 ```
 
 Once that is done, you can tell Git LFS to track files with `git lfs track "*.psd"`, for example.
@@ -83,6 +87,19 @@ However, because this is a Sinatra application, it can also be mounted within ot
 mount GitLfsS3::Application => '/lfs'
 ```
 
+
+
+Allow anonymous clones and pulls of the git-lfs git repository.
+
+## Ceph S3 Server
+
+Through the `LFS_CEPH_S3` and `LFS_CEPH_ENDPOINT` environment variables use a Ceph S3 storage service (Hammer version, through radosgw) instead of AWS S3.
+
+## More Complex Server Example
+
+* https://github.com/lsst-sqre/git-lfs-s3-server
+
 ## TODO
 
-* Cloudfront support
+* Cloudfront support.
+* Batch API support.

data/git-lfs-s3.gemspec

@@ -8,9 +8,9 @@ Gem::Specification.new do |gem|
   gem.version       = GitLfsS3::VERSION
   gem.authors       = ["Ryan LeFevre", "J. Matt Peterson"]
   gem.email         = ["meltingice8917@gmail.com", "jmatt@lsst.org"]
-  gem.description   = %q{LSST's Git LFS server that uses S3 for the storage backend.}
-  gem.summary       = %q{LSST's Git LFS server that uses S3 for the storage backend by providing presigned S3 URLs.}
-  gem.homepage      = "https://github.com/lsst-sqre/git-lfs-s3"
+  gem.description   = %q{A Git LFS server that uses S3 for the storage backend.}
+  gem.summary       = %q{A Git LFS server that uses S3 for the storage backend by providing presigned S3 URLs.}
+  gem.homepage      = "https://github.com/meltingice/git-lfs-s3"
   gem.license       = 'MIT'
 
   gem.files         = `git ls-files`.split($/)

data/lib/git-lfs-s3/application.rb

@@ -1,3 +1,5 @@
+require 'date'
+
 module GitLfsS3
   class Application < Sinatra::Application
     include AwsHelpers
@@ -47,6 +49,145 @@ module GitLfsS3
       "Git LFS S3 is online."
     end
 
+    def valid_object?(object)
+      begin
+        valid = object[:size] >= 0
+      rescue
+        valid = false
+      end
+      begin        
+        if valid
+          oid = object[:oid].hex
+          valid = oid.size == 32 && object[:oid].size == 64
+        end
+      rescue
+        valid = false
+      end
+      valid
+    end
+
+    def object_download(authenticated, object, object_json)
+      oid = object_json[:oid]
+      size = object_json[:size]
+      {
+        'oid' => oid,
+        'size' => size,
+        'authenticated' => authenticated,
+        'actions' => {
+          'download' => {
+            'href' => object.presigned_url(:get, :expires_in => 86400)
+          }
+        },
+        'expires_at' => DateTime.now.next_day.to_time.utc.iso8601
+      }
+    end
+
+    def object_upload(authenticated, object, object_json)
+      # Format a single upload object.
+      oid = object_json[:oid]
+      size = object_json[:size]
+      {
+        'oid' => oid,
+        'size' => size,
+        'authenticated' => authenticated,
+        'actions' => {
+          'upload' => {
+            'href' => object.presigned_url(:put, acl: 'public-read', :expires_in => 86400)
+          }
+        },
+        'expires_at' => DateTime.now.next_day.to_time.utc.iso8601
+      }
+    end
+
+    def object_error(error, message, object, object_json)
+      {
+        'oid' => object_json[:oid],
+        'size' => object_json[:size],
+        'error' => {
+          'code' => error,
+          'message' => message
+        }
+      }
+    end
+
+    def download(authenticated, params)
+      objects = Array.new
+      params[:objects].each do |object_json|
+        object_json = indifferent_params object_json
+        object = object_data object_json[:oid]
+        if valid_object? object_json
+          if object.exists?
+            objects.push object_download(authenticated, object, object_json)
+          else
+            objects.push object_error(404, 'Object does not exist', object, object_json)
+          end
+        else
+          objects.push object_error(422, 'Validation error', object, object_json)
+        end
+      end
+      objects
+    end
+
+    def upload(authenticated, params)
+      objects = Array.new
+      params[:objects].each do |object_json|
+        object_json = indifferent_params object_json
+        object = object_data object_json[:oid]
+        if valid_object? object_json
+          if object.exists?
+            objects.push object_download(authenticated, object, object_json)
+          else
+            objects.push object_upload(authenticated, object, object_json)
+          end
+        else
+          objects.push object_error(422, 'Validation error', object, object_json)
+        end
+      end
+      objects
+    end
+
+    def lfs_resp(objects)
+      status 200
+      resp = {
+        'transfer' => 'basic',
+        'objects' => objects
+      }
+      logger.debug resp
+      body MultiJson.dump(resp)
+    end
+    
+    def error_resp(status_code, message)
+      status status_code
+      resp = {
+        'message' => message,
+        'request_id' => SecureRandom::uuid
+      }
+      logger.debug resp
+      body MultiJson.dump(resp)
+    end
+    
+    post '/objects/batch', provides: 'application/vnd.git-lfs+json' do
+      # Git LFS Batch API
+      authenticated = authorized?
+      params = indifferent_params(JSON.parse(request.body.read))
+      
+      if params[:operation] == 'download'
+        objects = download(authenticated, params)
+      elsif params[:operation] == 'upload'
+        if authenticated
+          objects = upload(authenticated, params)
+        else
+          objects = nil
+        end
+      end
+      
+      if objects
+        lfs_resp(objects)
+      else
+        error_resp(401, 'Credentials needed')
+      end
+    end
+
     get "/objects/:oid", provides: 'application/vnd.git-lfs+json' do
       object = object_data(params[:oid])
 
@@ -94,6 +235,16 @@ module GitLfsS3
       body MultiJson.dump(service.response)
     end
 
+    post "/objects/batch", provides: 'application/vnd.git-lfs+json' do
+      logger.debug headers.inspect
+      service = UploadBatchService.service_for(request.body)
+      logger.debug service.response
+      
+      status service.status
+      body MultiJson.dump(service.response)
+    end
+
+
     post '/verify', provides: 'application/vnd.git-lfs+json' do
       data = MultiJson.load(request.body.tap { |b| b.rewind }.read)
       object = object_data(data['oid'])

data/lib/git-lfs-s3/services/upload/object_exists.rb

@@ -9,7 +9,7 @@ module GitLfsS3
         {
           '_links' => {
             'download' => {
-              'href' => object.presigned_url(:get)
+              'href' => object.presigned_url(:get, :expires_in => 86400)
             }
           }
         }

data/lib/git-lfs-s3/services/upload/upload_required.rb

@@ -32,9 +32,9 @@ module GitLfsS3
           GitLfsS3::CephPresignerService::signed_url(object)
         else
           if GitLfsS3::Application.settings.public_server
-            object.presigned_url(:put, acl: 'public-read')
+            object.presigned_url(:put, acl: 'public-read', :expires_in => 86400)
           else
-            object.presigned_url(:put)
+            object.presigned_url(:put, :expires_in => 86400)
           end
         end
       end

data/lib/git-lfs-s3/version.rb

@@ -1,3 +1,3 @@
 module GitLfsS3
-  VERSION = '0.2.6'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lsst-git-lfs-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors:
 - Ryan LeFevre
@@ -9,65 +9,65 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-04 00:00:00.000000000 Z
+date: 2017-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10'
-description: LSST's Git LFS server that uses S3 for the storage backend.
+description: A Git LFS server that uses S3 for the storage backend.
 email:
 - meltingice8917@gmail.com
 - jmatt@lsst.org
@@ -76,7 +76,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - README.md
 - Rakefile
@@ -91,7 +91,7 @@ files:
 - lib/git-lfs-s3/services/upload/object_exists.rb
 - lib/git-lfs-s3/services/upload/upload_required.rb
 - lib/git-lfs-s3/version.rb
-homepage: https://github.com/lsst-sqre/git-lfs-s3
+homepage: https://github.com/meltingice/git-lfs-s3
 licenses:
 - MIT
 metadata: {}
@@ -101,19 +101,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
-summary: LSST's Git LFS server that uses S3 for the storage backend by providing presigned
+summary: A Git LFS server that uses S3 for the storage backend by providing presigned
   S3 URLs.
 test_files: []