git-lfs-s3 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4548ca444d7fc60b4f00896823b11b6385e43243
+  data.tar.gz: 986bcbcc585dfda6dd70656bed1e005bf519ac77
+SHA512:
+  metadata.gz: 6d383ec8ef3ff7c0347e438650402e25db7141deb24e41de8b2684330ead5abd38d295a6784071d48a7a562ab9db7cff1e5544fbc446080da879783e907a4aee
+  data.tar.gz: 701233c012f4b11ff74494b8de0ffca43682eafed2f799bb38344939a42f1914219d5358423181410db2d60913be625fc28f12d57af985f45f9029f85be92cd6

data/.gitignore

@@ -0,0 +1,4 @@
+.env
+.DS_Store
+logs
+pkg

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,39 @@
+PATH
+  remote: .
+  specs:
+    git-lfs-s3 (0.0.1)
+      aws-sdk (~> 2)
+      multi_json
+      sinatra
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (2.0.38)
+      aws-sdk-resources (= 2.0.38)
+    aws-sdk-core (2.0.38)
+      builder (~> 3.0)
+      jmespath (~> 1.0)
+      multi_json (~> 1.0)
+    aws-sdk-resources (2.0.38)
+      aws-sdk-core (= 2.0.38)
+    builder (3.2.2)
+    jmespath (1.0.2)
+      multi_json (~> 1.0)
+    multi_json (1.11.0)
+    rack (1.6.0)
+    rack-protection (1.5.3)
+      rack
+    rake (10.4.2)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    tilt (2.0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  git-lfs-s3!
+  rake

data/README.md

@@ -0,0 +1,67 @@
+# Git LFS S3
+
+A [Git LFS](https://git-lfs.github.com/) server that stores your large Git files on S3.
+
+It works by generating a presigned URL that the Git LFS client can use to upload directly to S3. It also provides download URLs that allow Git clients to download directly from S3. No data is proxied through the Git LFS server.
+
+**Note:** the current version does not implement any authentication yet, so use with caution. Authentication will be added soon.
+
+## Installation
+
+Git LFS S3 is available on RubyGems.
+
+``` bash
+gem install git-lfs-s3
+```
+
+Or add it to your Gemfile if you wish to bundle it as a part of another application.
+
+``` ruby
+gem 'git-lfs-s3'
+```
+
+## Configuration
+
+### Standalone
+
+All configuration is done via environment variables. All of the configuration variables must be set.
+
+* `AWS_REGION` - the region where your S3 bucket is.
+* `AWS_ACCESS_KEY_ID` - your AWS access key.
+* `AWS_SECRET_ACCESS_KEY` - your AWS secret key.
+* `S3_BUCKET` - the bucket you wish to use for LFS storage. While not required, I recommend using a dedicated bucket for this.
+* `LFS_SERVER_URL` - the URL where this server can be reached; needed to fetch download URLs.
+
+### Bundled
+
+If you are bundling the server inside of another application, such as Rails, you can set the configuration directly on `GitLfsS3::Application`. See [bin/git-lfs-s3](https://github.com/meltingice/git-lfs-s3/blob/master/bin/git-lfs-s3) for an example.
+
+### Git Setup
+
+If you are new to Git LFS, make sure you read the [Getting Started](https://git-lfs.github.com/) guide first. Once that's done, you can configure your Git client to use this server by creating a `.gitconfig` file in the root of your repository and adding this config, but with your server address:
+
+``` git
+[lfs]
+    url = "http://yourserver.com"
+```
+
+Once that is done, you can tell Git LFS to track files with `git lfs track "*.psd"`, for example.
+
+## Running
+
+This repository contains an executable that will run a basic WEBrick server. Since this service is so lightweight, that's likely all you'll need. The port will default to 8080, but can be configured with an environment variable.
+
+``` bash
+PORT=4000 git-lfs-s3
+```
+
+However, because this is a Sinatra application, it can also be mounted within other Rack based projects or other Rack-based servers such as Unicorn or Puma. For example, if you wanted to add this to your Rails project, configure `GitLfsS3` in an initializer, and add this your routes:
+
+``` ruby
+mount GitLfsS3::Application => '/lfs'
+```
+
+## TODO
+
+* Authentication
+* Cloudfront support

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/git-lfs-s3

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "git-lfs-s3"
+
+GitLfsS3::Application.set :aws_region, ENV['AWS_REGION']
+GitLfsS3::Application.set :aws_access_key_id, ENV['AWS_ACCESS_KEY_ID']
+GitLfsS3::Application.set :aws_secret_access_key, ENV['AWS_SECRET_ACCESS_KEY']
+GitLfsS3::Application.set :s3_bucket, ENV['S3_BUCKET']
+GitLfsS3::Application.set :server_url, ENV['LFS_SERVER_URL']
+
+Rack::Handler::WEBrick.run(
+  GitLfsS3::Application.new,
+  Port: ENV['PORT'] || 8080
+)

data/git-lfs-s3.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'git-lfs-s3/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "git-lfs-s3"
+  gem.version       = GitLfsS3::VERSION
+  gem.authors       = ["Ryan LeFevre"]
+  gem.email         = ["meltingice8917@gmail.com"]
+  gem.description   = %q{A Git LFS server that uses S3 for the storage backend.}
+  gem.summary       = %q{A Git LFS server that uses S3 for the storage backend by providing presigned S3 URLs.}
+  gem.homepage      = "https://github.com/meltingice/git-lfs-s3"
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'aws-sdk', '~> 2'
+  gem.add_dependency 'sinatra'
+  gem.add_dependency 'multi_json'
+
+  gem.add_development_dependency 'rake'
+end

data/lib/git-lfs-s3.rb

@@ -0,0 +1,8 @@
+require 'logger'
+require 'sinatra/base'
+require 'aws-sdk'
+require 'multi_json'
+
+require "git-lfs-s3/aws"
+require "git-lfs-s3/services/upload"
+require "git-lfs-s3/application"

data/lib/git-lfs-s3/application.rb

@@ -0,0 +1,78 @@
+module GitLfsS3
+  class Application < Sinatra::Application
+    include AwsHelpers
+
+    configure do
+      disable :sessions
+      enable :logging
+
+      Dir.mkdir('logs') unless Dir.exists?('logs')
+      $logger = Logger.new("logs/#{settings.environment}.log", "weekly")
+      $logger.level = Logger::INFO
+    end
+
+    configure :development do
+      $logger.level = Logger::DEBUG
+    end
+
+    helpers do
+      def logger
+        $logger
+      end
+    end
+
+    # before do
+    #   raise headers['Accept'].inspect
+    #   if headers['Accept'] != 'application/vnd.git-lfs+json'
+    #     halt 406, {'Content-Type' => 'text/plain'}, 'Server only accepts application/vnd.git-lfs+json'
+    #   end
+    # end
+
+    get "/objects/:oid", provides: 'application/vnd.git-lfs+json' do
+      object = object_data(params[:oid])
+
+      if object.exists?
+        status 200
+        resp = {
+          'oid' => params[:oid],
+          'size' => object.size,
+          '_links' => {
+            'self' => {
+              'href' => File.join(settings.server_url, 'objects', params[:oid])
+            },
+            'download' => {
+              # TODO: cloudfront support
+              'href' => object_data(params[:oid]).presigned_url(:get)
+            }
+          }
+        }
+
+        body MultiJson.dump(resp)
+      else
+        status 404
+        body MultiJson.dump({message: 'Object not found'})
+      end
+    end
+
+    post "/objects", provides: 'application/vnd.git-lfs+json' do
+      logger.debug headers.inspect
+      service = UploadService.service_for(request.body)
+      logger.debug service.response
+      logger.debug service.to_curl
+      
+      status service.status
+      body MultiJson.dump(service.response)
+    end
+
+    post '/verify', provides: 'application/vnd.git-lfs+json' do
+      data = MultiJson.load(request.body.tap { |b| b.rewind }.read)
+      object = object_data(data['oid'])
+
+      if object.exists? && object.size == data['size']
+        status 200
+      else
+        status 404
+      end
+    end
+  end
+end

data/lib/git-lfs-s3/aws.rb

@@ -0,0 +1,35 @@
+module GitLfsS3
+  module AwsHelpers
+    def s3
+      @s3 ||= Aws::S3::Client.new({
+        region: aws_region,
+        access_key_id: aws_access_key_id,
+        secret_access_key: aws_secret_access_key
+      })
+    end
+
+    def bucket_name
+      GitLfsS3::Application.settings.s3_bucket
+    end
+
+    def bucket
+      @bucket ||= Aws::S3::Bucket.new(name: bucket_name, client: s3)
+    end
+
+    def object_data(oid)
+      bucket.object("data/#{oid}")
+    end
+
+    def aws_region
+      GitLfsS3::Application.settings.aws_region
+    end
+
+    def aws_access_key_id
+      GitLfsS3::Application.settings.aws_access_key_id
+    end
+
+    def aws_secret_access_key
+      GitLfsS3::Application.settings.aws_secret_access_key
+    end
+  end
+end

data/lib/git-lfs-s3/services/upload.rb

@@ -0,0 +1,26 @@
+require "git-lfs-s3/services/upload/base"
+require "git-lfs-s3/services/upload/object_exists"
+require "git-lfs-s3/services/upload/upload_required"
+
+module GitLfsS3
+  module UploadService
+    extend self
+    extend AwsHelpers
+
+    MODULES = [
+      ObjectExists,
+      UploadRequired
+    ]
+
+    def service_for(data)
+      req = MultiJson.load data.tap { |d| d.rewind }.read
+      object = object_data(req['oid'])
+
+      MODULES.each do |mod|
+        return mod.new(req, object) if mod.should_handle?(req, object)
+      end
+
+      nil
+    end
+  end
+end

data/lib/git-lfs-s3/services/upload/base.rb

@@ -0,0 +1,28 @@
+module GitLfsS3
+  module UploadService
+    class Base
+      include AwsHelpers
+      
+      attr_reader :req, :object
+
+      def initialize(req, object)
+        @req = req
+        @object = object
+      end
+
+      def response
+        raise "Override"
+      end
+
+      def status
+        raise "Override"
+      end
+
+      private
+
+      def server_url
+        GitLfsS3::Application.settings.server_url
+      end
+    end
+  end
+end

data/lib/git-lfs-s3/services/upload/object_exists.rb

@@ -0,0 +1,23 @@
+module GitLfsS3
+  module UploadService
+    class ObjectExists < Base
+      def self.should_handle?(req, object)
+        object.exists? && object.size == req['size']
+      end
+
+      def response
+        {
+          '_links' => {
+            'download' => {
+              'href' => object.presigned_url(:get)
+            }
+          }
+        }
+      end
+
+      def status
+        200
+      end
+    end
+  end
+end

data/lib/git-lfs-s3/services/upload/upload_required.rb

@@ -0,0 +1,48 @@
+module GitLfsS3
+  module UploadService
+    class UploadRequired < Base
+      def self.should_handle?(req, object)
+        !object.exists? || object.size != req['size']
+      end
+
+      def response
+        {
+          '_links' => {
+            'upload' => {
+              'href' => upload_destination,
+              'header' => upload_headers
+            },
+            'verify' => {
+              'href' => File.join(server_url, 'verify')
+            }
+          }
+        }
+      end
+
+      def status
+        202
+      end
+
+      def to_curl
+        curl = ["curl -XPUT"]
+        curl << "-T \"{{file}}\""
+        upload_headers.each do |k, v|
+          curl << "-H \"#{k}: #{v}\""
+        end
+        curl << upload_destination
+
+        curl.join(' ')
+      end
+
+      private
+
+      def upload_destination
+        object.presigned_url(:put)
+      end
+
+      def upload_headers
+        {'content-type' => ''}
+      end
+    end
+  end
+end

data/lib/git-lfs-s3/version.rb

@@ -0,0 +1,3 @@
+module GitLfsS3
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: git-lfs-s3
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ryan LeFevre
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Git LFS server that uses S3 for the storage backend.
+email:
+- meltingice8917@gmail.com
+executables:
+- git-lfs-s3
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/git-lfs-s3
+- git-lfs-s3.gemspec
+- lib/git-lfs-s3.rb
+- lib/git-lfs-s3/application.rb
+- lib/git-lfs-s3/aws.rb
+- lib/git-lfs-s3/services/upload.rb
+- lib/git-lfs-s3/services/upload/base.rb
+- lib/git-lfs-s3/services/upload/object_exists.rb
+- lib/git-lfs-s3/services/upload/upload_required.rb
+- lib/git-lfs-s3/version.rb
+homepage: https://github.com/meltingice/git-lfs-s3
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A Git LFS server that uses S3 for the storage backend by providing presigned
+  S3 URLs.
+test_files: []