lsst-git-lfs-s3 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e8e53b53772ac904e3368fba37ed0bdddc509e8d
+  data.tar.gz: 10f464ba058e3290fc6b612b0bc736519ac96487
+SHA512:
+  metadata.gz: 97cba6586c2d3068e5c0fb60adb1a84d2a955a869c10611025a74388eecc9ee1d1057ca68d91cc99b1769c7fd782e9e2e3b32327250a83ab47be25dbbbfd4a38
+  data.tar.gz: 9b2a1afea4909ff49d2f0143e2f78e79201d70377ab7ab8cdd74ffd6a40dbcc92cb9394dd7b709ca1fd6963df7e7c5ec936a197281b93242ffd385e41f54fd9b

data/.gitignore

@@ -0,0 +1,7 @@
+.bundle
+.env
+.DS_Store
+logs
+pkg
+Gemfile.lock
+vendor

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -0,0 +1,88 @@
+# Git LFS S3
+
+A [Git LFS](https://git-lfs.github.com/) server that stores your large Git files on S3.
+
+It works by generating a presigned URL that the Git LFS client can use to upload directly to S3. It also provides download URLs that allow Git clients to download directly from S3. No data is proxied through the Git LFS server.
+
+## Installation
+
+Git LFS S3 is available on RubyGems.
+
+``` bash
+gem install git-lfs-s3
+```
+
+Or add it to your Gemfile if you wish to bundle it as a part of another application.
+
+``` ruby
+gem 'git-lfs-s3'
+```
+
+## Configuration
+
+### Standalone
+
+All configuration is done via environment variables. All of these configuration variables must be set.
+
+* `AWS_REGION` - the region where your S3 bucket is.
+* `AWS_ACCESS_KEY_ID` - your AWS access key.
+* `AWS_SECRET_ACCESS_KEY` - your AWS secret key.
+* `S3_BUCKET` - the bucket you wish to use for LFS storage. While not required, I recommend using a dedicated bucket for this.
+* `LFS_SERVER_URL` - the URL where this server can be reached; needed to fetch download URLs.
+
+You can (and should) also set authentication information. When you push for the first time from git, you will be prompted to enter a username and password when authentication is enabled. You can configure these with environment variables as well.
+
+* `USERNAME` - the login username.
+* `PASSWORD` - the login password.
+
+### Bundled
+
+If you are bundling the service inside of another application, such as Rails, or a different server of your choosing, you can set the configuration directly on `GitLfsS3::Application`. See [bin/git-lfs-s3](https://github.com/meltingice/git-lfs-s3/blob/master/bin/git-lfs-s3) for an example.
+
+You can also hook the authentication into your own service this way. For example:
+
+``` ruby
+GitLfsS3::Application.on_authenticate do |username, password, is_safe|
+  if is_safe
+    true
+  else
+    user = User.find(username: username)
+    user.verify_password(password)
+  end
+end
+```
+
+The logger can be configured as well. This is especially handy if you want to hook it into your Rails logger:
+
+``` ruby
+GitLfsS3::Application.set :logger, Rails.logger
+```
+
+### Git Setup
+
+If you are new to Git LFS, make sure you read the [Getting Started](https://git-lfs.github.com/) guide first. Once that's done, you can configure your Git client to use this server by creating a `.gitconfig` file in the root of your repository and adding this config, but with your server address:
+
+``` git
+[lfs]
+    url = "http://yourserver.com"
+```
+
+Once that is done, you can tell Git LFS to track files with `git lfs track "*.psd"`, for example.
+
+## Running
+
+This repository contains an executable that will run a basic WEBrick server. Since this service is so lightweight, that's likely all you'll need. The port will default to 8080, but can be configured with an environment variable.
+
+``` bash
+PORT=4000 git-lfs-s3
+```
+
+However, because this is a Sinatra application, it can also be mounted within other Rack based projects or other Rack-based servers such as Unicorn or Puma. For example, if you wanted to add this to your Rails project, configure `GitLfsS3` in an initializer, and add this your routes:
+
+``` ruby
+mount GitLfsS3::Application => '/lfs'
+```
+
+## TODO
+
+* Cloudfront support

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/git-lfs-s3

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "logger"
+require "git-lfs-s3"
+
+GitLfsS3::Application.set :aws_region, ENV['AWS_REGION']
+GitLfsS3::Application.set :aws_access_key_id, ENV['AWS_ACCESS_KEY_ID']
+GitLfsS3::Application.set :aws_secret_access_key, ENV['AWS_SECRET_ACCESS_KEY']
+GitLfsS3::Application.set :s3_bucket, ENV['S3_BUCKET']
+GitLfsS3::Application.set :server_url, ENV['LFS_SERVER_URL']
+GitLfsS3::Application.set :logger, Logger.new(STDOUT)
+
+GitLfsS3::Application.on_authenticate do |username, password, is_safe|
+  if is_safe # Whether the HTTP method is safe (GET, HEAD)
+    true
+  else
+    username == ENV['USERNAME'] && password == ENV['PASSWORD']
+  end
+end
+
+Rack::Handler::WEBrick.run(
+  GitLfsS3::Application.new,
+  Port: ENV['PORT'] || 8080
+)

data/git-lfs-s3.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'git-lfs-s3/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "lsst-git-lfs-s3"
+  gem.version       = GitLfsS3::VERSION
+  gem.authors       = ["Ryan LeFevre", "J. Matt Peterson"]
+  gem.email         = ["meltingice8917@gmail.com", "jmatt@lsst.org"]
+  gem.description   = %q{LSST's Git LFS server that uses S3 for the storage backend.}
+  gem.summary       = %q{LSST's Git LFS server that uses S3 for the storage backend by providing presigned S3 URLs.}
+  gem.homepage      = "https://github.com/lsst-sqre/git-lfs-s3"
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'aws-sdk', '~> 2'
+  gem.add_dependency 'sinatra', '~> 1'
+  gem.add_dependency 'multi_json', '~> 1'
+
+  gem.add_development_dependency 'rake', '~> 10'
+end

data/lib/git-lfs-s3.rb

@@ -0,0 +1,7 @@
+require 'sinatra/base'
+require 'aws-sdk'
+require 'multi_json'
+
+require "git-lfs-s3/aws"
+require "git-lfs-s3/services/upload"
+require "git-lfs-s3/application"

data/lib/git-lfs-s3/application.rb

@@ -0,0 +1,99 @@
+module GitLfsS3
+  class Application < Sinatra::Application
+    include AwsHelpers
+
+    class << self
+      attr_reader :auth_callback
+
+      def on_authenticate(&block)
+        @auth_callback = block
+      end
+
+      def authentication_enabled?
+        !auth_callback.nil?
+      end
+
+      def perform_authentication(username, password, is_safe)
+        auth_callback.call(username, password, is_safe)
+      end
+    end
+
+    configure do
+      disable :sessions
+      enable :logging
+    end
+
+    helpers do
+      def logger
+        settings.logger
+      end
+    end
+
+    def authorized?
+      @auth ||=  Rack::Auth::Basic::Request.new(request.env)
+      logger.debug request.safe?
+      @auth.provided? && @auth.basic? && @auth.credentials && self.class.auth_callback.call(
+        @auth.credentials[0], @auth.credentials[1], request.safe?
+      )
+    end
+
+    def protected!
+      unless authorized?
+        response['WWW-Authenticate'] = %(Basic realm="Restricted Area")
+        throw(:halt, [401, "Invalid username or password"])
+      end
+    end
+
+    before { protected! }
+
+    get '/' do
+      "Git LFS S3 is online."
+    end
+
+    get "/objects/:oid", provides: 'application/vnd.git-lfs+json' do
+      object = object_data(params[:oid])
+
+      if object.exists?
+        status 200
+        resp = {
+          'oid' => params[:oid],
+          'size' => object.size,
+          '_links' => {
+            'self' => {
+              'href' => File.join(settings.server_url, 'objects', params[:oid])
+            },
+            'download' => {
+              # TODO: cloudfront support
+              'href' => object_data(params[:oid]).presigned_url(:get)
+            }
+          }
+        }
+
+        body MultiJson.dump(resp)
+      else
+        status 404
+        body MultiJson.dump({message: 'Object not found'})
+      end
+    end
+
+    post "/objects", provides: 'application/vnd.git-lfs+json' do
+      logger.debug headers.inspect
+      service = UploadService.service_for(request.body)
+      logger.debug service.response
+      
+      status service.status
+      body MultiJson.dump(service.response)
+    end
+
+    post '/verify', provides: 'application/vnd.git-lfs+json' do
+      data = MultiJson.load(request.body.tap { |b| b.rewind }.read)
+      object = object_data(data['oid'])
+
+      if object.exists? && object.size == data['size']
+        status 200
+      else
+        status 404
+      end
+    end
+  end
+end

data/lib/git-lfs-s3/aws.rb

@@ -0,0 +1,35 @@
+module GitLfsS3
+  module AwsHelpers
+    def s3
+      @s3 ||= Aws::S3::Client.new({
+        region: aws_region,
+        access_key_id: aws_access_key_id,
+        secret_access_key: aws_secret_access_key
+      })
+    end
+
+    def bucket_name
+      GitLfsS3::Application.settings.s3_bucket
+    end
+
+    def bucket
+      @bucket ||= Aws::S3::Bucket.new(name: bucket_name, client: s3)
+    end
+
+    def object_data(oid)
+      bucket.object("data/#{oid}")
+    end
+
+    def aws_region
+      GitLfsS3::Application.settings.aws_region
+    end
+
+    def aws_access_key_id
+      GitLfsS3::Application.settings.aws_access_key_id
+    end
+
+    def aws_secret_access_key
+      GitLfsS3::Application.settings.aws_secret_access_key
+    end
+  end
+end

data/lib/git-lfs-s3/services/upload.rb

@@ -0,0 +1,26 @@
+require "git-lfs-s3/services/upload/base"
+require "git-lfs-s3/services/upload/object_exists"
+require "git-lfs-s3/services/upload/upload_required"
+
+module GitLfsS3
+  module UploadService
+    extend self
+    extend AwsHelpers
+
+    MODULES = [
+      ObjectExists,
+      UploadRequired
+    ]
+
+    def service_for(data)
+      req = MultiJson.load data.tap { |d| d.rewind }.read
+      object = object_data(req['oid'])
+
+      MODULES.each do |mod|
+        return mod.new(req, object) if mod.should_handle?(req, object)
+      end
+
+      nil
+    end
+  end
+end

data/lib/git-lfs-s3/services/upload/base.rb

@@ -0,0 +1,28 @@
+module GitLfsS3
+  module UploadService
+    class Base
+      include AwsHelpers
+      
+      attr_reader :req, :object
+
+      def initialize(req, object)
+        @req = req
+        @object = object
+      end
+
+      def response
+        raise "Override"
+      end
+
+      def status
+        raise "Override"
+      end
+
+      private
+
+      def server_url
+        GitLfsS3::Application.settings.server_url
+      end
+    end
+  end
+end

data/lib/git-lfs-s3/services/upload/object_exists.rb

@@ -0,0 +1,23 @@
+module GitLfsS3
+  module UploadService
+    class ObjectExists < Base
+      def self.should_handle?(req, object)
+        object.exists? && object.size == req['size']
+      end
+
+      def response
+        {
+          '_links' => {
+            'download' => {
+              'href' => object.presigned_url(:get)
+            }
+          }
+        }
+      end
+
+      def status
+        200
+      end
+    end
+  end
+end

data/lib/git-lfs-s3/services/upload/upload_required.rb

@@ -0,0 +1,37 @@
+module GitLfsS3
+  module UploadService
+    class UploadRequired < Base
+      def self.should_handle?(req, object)
+        !object.exists? || object.size != req['size']
+      end
+
+      def response
+        {
+          '_links' => {
+            'upload' => {
+              'href' => upload_destination,
+              'header' => upload_headers
+            },
+            'verify' => {
+              'href' => File.join(server_url, 'verify')
+            }
+          }
+        }
+      end
+
+      def status
+        202
+      end
+
+      private
+
+      def upload_destination
+        object.presigned_url(:put)
+      end
+
+      def upload_headers
+        {'content-type' => ''}
+      end
+    end
+  end
+end

data/lib/git-lfs-s3/version.rb

@@ -0,0 +1,3 @@
+module GitLfsS3
+  VERSION = '0.1.2'
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: lsst-git-lfs-s3
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Ryan LeFevre
+- J. Matt Peterson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+description: LSST's Git LFS server that uses S3 for the storage backend.
+email:
+- meltingice8917@gmail.com
+- jmatt@lsst.org
+executables:
+- git-lfs-s3
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/git-lfs-s3
+- git-lfs-s3.gemspec
+- lib/git-lfs-s3.rb
+- lib/git-lfs-s3/application.rb
+- lib/git-lfs-s3/aws.rb
+- lib/git-lfs-s3/services/upload.rb
+- lib/git-lfs-s3/services/upload/base.rb
+- lib/git-lfs-s3/services/upload/object_exists.rb
+- lib/git-lfs-s3/services/upload/upload_required.rb
+- lib/git-lfs-s3/version.rb
+homepage: https://github.com/lsst-sqre/git-lfs-s3
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: LSST's Git LFS server that uses S3 for the storage backend by providing presigned
+  S3 URLs.
+test_files: []