git_dump 0.1.0

data/lib/git_dump/tree.rb

@@ -0,0 +1,31 @@
+require 'git_dump/path_object'
+require 'git_dump/tree/base'
+require 'git_dump/entry'
+
+class GitDump
+  # Interface to git tree
+  class Tree < PathObject
+    include Base
+
+    attr_reader :sha
+    def initialize(repo, dir, name, sha)
+      super(repo, dir, name)
+      @sha = sha
+      @entries = read_entries
+    end
+
+  private
+
+    def read_entries
+      entries = {}
+      repo.tree_entries(sha).each do |entry|
+        entries[entry[:name]] = if entry[:type] == :tree
+          self.class.new(repo, path, entry[:name], entry[:sha])
+        else
+          Entry.new(repo, path, entry[:name], entry[:sha], entry[:mode])
+        end
+      end
+      entries
+    end
+  end
+end

data/lib/git_dump/tree/base.rb

@@ -0,0 +1,54 @@
+require 'git_dump/path_object'
+require 'git_dump/entry'
+
+class GitDump
+  class Tree < PathObject
+    # Common methods in Tree and Builder
+    module Base
+      # Retrive tree or entry at path, return nil if there is nothing at path
+      def [](path)
+        get_at(parse_path(path))
+      end
+
+      # Iterate over every tree/entry of this tree, return enumerator if no
+      # block given
+      def each(&block)
+        return to_enum(:each) unless block
+        @entries.each do |_, entry|
+          block[entry]
+        end
+      end
+
+      # Iterate over all entries recursively, return enumerator if no block
+      # given
+      def each_recursive(&block)
+        return to_enum(:each_recursive) unless block
+        @entries.each do |_, entry|
+          if entry.is_a?(Entry)
+            block[entry]
+          else
+            entry.each_recursive(&block)
+          end
+        end
+      end
+
+    protected
+
+      def get_at(parts)
+        return unless (entry = @entries[parts.first])
+        if parts.length == 1
+          entry
+        elsif entry.is_a?(self.class)
+          entry.get_at(parts.drop(1))
+        end
+      end
+
+    private
+
+      def parse_path(path)
+        path = Array(path).join('/') unless path.is_a?(String)
+        path.scan(/[^\/]+/)
+      end
+    end
+  end
+end

data/lib/git_dump/tree/builder.rb

@@ -0,0 +1,62 @@
+require 'git_dump/path_object'
+require 'git_dump/entry'
+
+class GitDump
+  class Tree < PathObject
+    # Creating tree
+    class Builder < PathObject
+      include Base
+
+      def initialize(repo, dir, name)
+        super(repo, dir, name)
+        @entries = {}
+      end
+
+      # Store data `content` with mode `mode` at `path`
+      # Pass `nil` as content to remove
+      def store(path, content, mode = 0644)
+        put_at(parse_path(path), content && repo.data_sha(content), mode)
+      end
+      alias_method :[]=, :store
+
+      # Store data from `from` with mode `mode` (by default file mode) at `path`
+      def store_from(path, from, mode = nil)
+        mode ||= File.stat(from).mode
+        put_at(parse_path(path), repo.path_sha(from), mode)
+      end
+
+      def sha
+        repo.treeify(@entries.map do |name, entry|
+          attributes = {:name => name, :sha => entry.sha}
+          if entry.is_a?(self.class)
+            attributes.merge(:type => :tree)
+          else
+            attributes.merge(:type => :blob, :mode => entry.mode)
+          end
+        end)
+      end
+
+      def inspect
+        "#<#{self.class} #{@entries.inspect}>"
+      end
+
+    protected
+
+      def put_at(parts, sha, mode)
+        name = parts.shift
+        if parts.empty?
+          if sha.nil?
+            @entries.delete(name)
+          else
+            @entries[name] = Entry.new(repo, path, name, sha, mode)
+          end
+        else
+          unless @entries[name].is_a?(self.class)
+            @entries[name] = self.class.new(repo, path, name)
+          end
+          @entries[name].put_at(parts, sha, mode)
+        end
+      end
+    end
+  end
+end

data/lib/git_dump/version.rb

@@ -0,0 +1,55 @@
+require 'git_dump/version/base'
+require 'git_dump/tree'
+
+class GitDump
+  # Reading version
+  class Version
+    include Base
+
+    def self.list(repo)
+      repo.tag_entries.map do |entry|
+        Version.new(repo, entry[:name], entry[:sha], {
+          :time => entry[:author_time],
+          :commit_time => entry[:commit_time],
+          :annotation => entry[:tag_message],
+          :description => entry[:commit_message],
+        })
+      end
+    end
+
+    def self.by_id(repo, id)
+      list(repo).find{ |version| version.id == id }
+    end
+
+    attr_reader :repo, :id, :sha, :time
+    attr_reader :commit_time, :annotation, :description
+    def initialize(repo, id, sha, attributes = {})
+      @repo, @id, @sha = repo, id, sha
+      @time = attributes[:time]
+      @commit_time = attributes[:commit_time]
+      @annotation = attributes[:annotation]
+      @description = attributes[:description]
+    end
+
+    # Send this version to repo at url
+    # Use :progress => true to show progress
+    def push(url, options = {})
+      repo.push(url, id, options)
+    end
+
+    # Remove this version
+    def remove
+      repo.remove_tag(id)
+    end
+
+    def inspect
+      "#<#{self.class} id=#{@id} sha=#{@sha} tree=#{@tree.inspect}>"
+    end
+
+  private
+
+    def tree
+      @tree ||= Tree.new(repo, nil, nil, sha)
+    end
+  end
+end

data/lib/git_dump/version/base.rb

@@ -0,0 +1,21 @@
+class GitDump
+  class Version
+    # Common methods in Version and Builder
+    module Base
+      # Retrive tree or entry at path, return nil if there is nothing at path
+      def [](path)
+        tree[path]
+      end
+
+      # Iterate over every tree/entry at root level
+      def each(&block)
+        tree.each(&block)
+      end
+
+      # Iterate over all entries recursively
+      def each_recursive(&block)
+        tree.each_recursive(&block)
+      end
+    end
+  end
+end

data/lib/git_dump/version/builder.rb

@@ -0,0 +1,77 @@
+require 'git_dump/version'
+require 'git_dump/version/base'
+require 'git_dump/tree/builder'
+
+class GitDump
+  class Version
+    # Creating version
+    class Builder
+      include Base
+
+      attr_reader :repo
+      def initialize(repo)
+        @repo = repo
+        @tree = Tree::Builder.new(repo, nil, nil)
+      end
+
+      # Store data `content` with mode `mode` at `path`
+      # Pass `nil` as content to remove
+      def store(path, content, mode = 0644)
+        tree.store(path, content, mode)
+      end
+      alias_method :[]=, :store
+
+      # Store data from `from` with mode `mode` (by default file mode) at `path`
+      def store_from(path, from, mode = nil)
+        tree.store_from(path, from, mode)
+      end
+
+      # Create commit and tag it, returns Version instance
+      # Options:
+      #   :time - set version time (tag and commit)
+      #   :tags - list of strings to associate with this version
+      #   :annotation - tag message
+      #   :description - commit message
+      #   :keep_identity - don't override identity with "git_dump
+      # gitdump@hostname"
+      def commit(options = {})
+        options = {:time => Time.now}.merge(options)
+
+        base = {:time => options[:time]}
+        unless options[:keep_identity]
+          base[:name] = 'git_dump'
+          base[:email] = "git_dump@#{GitDump.hostname}"
+        end
+
+        commit_sha = repo.commit(tree.sha, base.merge({
+          :message => options[:description],
+        }))
+
+        tag_name = repo.tag(commit_sha, name_parts(options), base.merge({
+          :message => options[:annotation],
+        }))
+
+        repo.gc(:auto => true)
+
+        Version.by_id(repo, tag_name)
+      end
+
+      def inspect
+        "#<#{self.class} tree=#{tree.inspect}>"
+      end
+
+    private
+
+      attr_reader :tree
+
+      def name_parts(options)
+        [
+          options[:time].dup.utc.strftime('%Y-%m-%d_%H-%M-%S'),
+          GitDump.hostname,
+          Array(options[:tags]).join(','),
+          GitDump.uuid,
+        ]
+      end
+    end
+  end
+end

data/script/benchmark

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'git_dump'
+require 'benchmark'
+require 'tmpdir'
+
+def measure(label, number = nil, &block)
+  label = "#{label} x#{number}" if number
+  label = "#{label}:"
+
+  result = nil
+  print label.ljust(30)
+  times = Benchmark.measure do
+    (number || 1).times do |n|
+      result = block.call(n)
+    end
+  end
+  puts " #{times}"
+  result
+end
+
+puts "Using #{defined?(Rugged) ? 'libgit2' : 'git commands'}:"
+
+urandom = File.open('/dev/urandom')
+
+Dir.mktmpdir do |dir|
+  dump = measure 'init' do
+    GitDump.new(File.join(dir, 'bm.git'), :create => true)
+  end
+
+  builder = dump.new_version
+
+  [
+    ['1K', 1_000, 1024],
+    ['10M', 10, 10 * 1024 * 1024],
+  ].each do |label, times, size|
+    datas = Array.new(times) do
+      urandom.read(size)
+    end.each
+    measure "store #{label} strings", times do |n|
+      path = format('data%s%08d', label, n).scan(/.{1,2}/).join('/')
+      data = datas.next
+      builder.store(path, data)
+    end
+
+    files = Array.new(times) do |i|
+      path = File.join(dir, "file#{i}")
+      File.open(path, 'w') do |f|
+        f.write(urandom.read(size))
+      end
+      path
+    end.each
+    measure "store #{label} files", times do |n|
+      path = format('file%s%08d', label, n).scan(/.{1,2}/).join('/')
+      file = files.next
+      builder.store_from(path, file)
+    end
+  end
+
+  measure 'commit', 10 do
+    builder.commit
+  end
+
+  measure 'read versions', 100 do
+    dump.versions
+  end
+
+  entries = dump.versions.first.each_recursive.to_a
+
+  measure 'read entries', entries.length do |n|
+    entries[n].read
+  end
+
+  measure 'read entries to disk', entries.length do |n|
+    entries[n].write_to(File.join(dir, 'out'))
+  end
+end

data/spec/git_dump_spec.rb

@@ -0,0 +1,392 @@
+require 'spec_helper'
+require 'git_dump'
+require 'tmpdir'
+
+describe GitDump do
+  around do |example|
+    Dir.mktmpdir do |dir|
+      @tmp_dir = dir
+      example.run
+    end
+  end
+  let(:tmp_dir){ @tmp_dir }
+
+  DATAS = {
+    :text => "\r\n\r\nline\nline\rline\r\nline\n\rline\r\n\r\n",
+    :binary => 256.times.sort_by{ rand }.pack('C*'),
+  }
+
+  describe :new do
+    let(:path){ File.join(tmp_dir, 'dump') }
+
+    it 'initializes with bare git repo' do
+      system('git', 'init', '-q', '--bare', path)
+      expect(GitDump.new(path).path).to eq(path)
+    end
+
+    it 'initializes with full git repo' do
+      system('git', 'init', '-q', path)
+      expect(GitDump.new(path).path).to eq(path)
+    end
+
+    it 'creates bare git repo' do
+      dump = GitDump.new path, :create => true
+      Dir.chdir path do
+        expect(`git rev-parse --git-dir`.strip).to eq('.')
+        expect(`git rev-parse --is-bare-repository`.strip).to eq('true')
+      end
+      expect(dump.path).to eq(path)
+    end
+
+    it 'creates full git repo' do
+      dump = GitDump.new path, :create => :non_bare
+      Dir.chdir path do
+        expect(`git rev-parse --git-dir`.strip).to eq('.git')
+        expect(`git rev-parse --is-bare-repository`.strip).to eq('false')
+      end
+      expect(dump.path).to eq(path)
+    end
+
+    it 'raises if dump does not exist and not asked to create' do
+      expect{ GitDump.new path }.to raise_error GitDump::Repo::InitException
+    end
+
+    it 'raises if path is a file' do
+      File.open(path, 'w'){}
+      expect{ GitDump.new path }.to raise_error GitDump::Repo::InitException
+    end
+
+    it 'raises if dump is a directory but not a git repository' do
+      Dir.mkdir(path)
+      expect{ GitDump.new path }.to raise_error GitDump::Repo::InitException
+    end
+  end
+
+  context do
+    let(:dump){ GitDump.new File.join(tmp_dir, 'dump'), :create => true }
+
+    it 'returns empty list for empty repo versions' do
+      expect(dump.versions).to be_empty
+    end
+
+    it 'creates version for every commit' do
+      builder = dump.new_version
+      3.times{ builder.commit }
+
+      expect(dump.versions.length).to eq(3)
+    end
+
+    it 'builds id from time, hostname and uuid' do
+      allow(GitDump).to receive(:hostname).and_return('ivans')
+      time = Time.utc(2000, 10, 20, 12, 34, 56)
+
+      builder = dump.new_version
+      built = builder.commit(:time => time)
+
+      expect(built.id).to match(%r{
+        \A
+          2000-10-20_12-34-56
+        /
+          ivans
+        /
+          (?i:
+            [0-9a-f]{8}-
+            [0-9a-f]{4}-
+            4[0-9a-f]{3}-
+            [89ab][0-9a-f]{3}-
+            [0-9a-f]{12}
+          )
+        \z
+      }x)
+    end
+
+    [
+      'hello,world,foo,bar_',
+      %w[hello world foo bar_],
+      'hello,world,foo,bar!@#$%^&*()',
+    ].each do |tags|
+      it 'puts tags in name' do
+        builder = dump.new_version
+        built = builder.commit(:tags => tags)
+
+        expect(built.id.split('/')).to include('hello,world,foo,bar_')
+      end
+    end
+
+    it 'sets and reads version time' do
+      time = Time.parse('2000-10-20 12:34:56')
+
+      builder = dump.new_version
+      built = builder.commit(:time => time)
+
+      expect(built.time).to eq(time)
+      expect(dump.versions.first.time).to eq(time)
+    end
+
+    it 'reads commit time' do
+      builder = dump.new_version
+      from = Time.at(Time.now.to_i) # round down to second
+      built = builder.commit(:time => Time.parse('2000-10-20 12:34:56'))
+      to = Time.now
+
+      expect(built.commit_time).to be_between(from, to)
+      expect(dump.versions.first.commit_time).to be_between(from, to)
+    end
+
+    it 'sets and reads version annotation' do
+      message = DATAS[:text] + File.read(__FILE__)
+
+      builder = dump.new_version
+      built = builder.commit(:annotation => message)
+
+      expect(built.annotation).to eq(message)
+      expect(dump.versions.first.annotation).to eq(message)
+    end
+
+    it 'sets and reads version description' do
+      message = DATAS[:text] + File.read(__FILE__)
+
+      builder = dump.new_version
+      built = builder.commit(:description => message)
+
+      expect(built.description).to eq(message)
+      expect(dump.versions.first.description).to eq(message)
+    end
+
+    it 'creates and reads version' do
+      builder = dump.new_version
+      builder['string/x'] = 'test a'
+      builder.store('stringio/x', StringIO.new('test b'), 0644)
+      builder.store('io/x', File.open(__FILE__), 0755)
+      builder.store_from('path/x', __FILE__)
+      built = builder.commit
+
+      reinit_dump = GitDump.new(dump.path)
+
+      expect(reinit_dump.versions.length).to eq(1)
+
+      version = reinit_dump.versions.first
+
+      expect(version.id).to eq(built.id)
+
+      expect(version['string/x'].read).to eq('test a')
+      expect(version['stringio/x'].read).to eq('test b')
+      expect(version['io/x'].read).to eq(File.read(__FILE__))
+      expect(version['path/x'].read).to eq(File.read(__FILE__))
+      expect(version['should/not/be/there']).to be_nil
+    end
+
+    it 'removes version' do
+      builder = dump.new_version
+      builder['a'] = 'b'
+      built = builder.commit
+
+      expect(dump.versions.length).to eq(1)
+
+      built.remove
+
+      expect(dump.versions).to be_empty
+    end
+
+    it 'returns path and name for trees and entries' do
+      builder = dump.new_version
+      builder['a/b/c'] = 'test'
+
+      %w[
+        a
+        a/b
+        a/b/c
+      ].each do |path|
+        expect(builder[path].path).to eq(path)
+        expect(builder[path].name).to eq(path.split('/').last)
+      end
+    end
+
+    it 'cleans paths' do
+      builder = dump.new_version
+      builder['//aa//fa//'] = 'test a'
+
+      expect(builder['//aa//fa//'].read).to eq('test a')
+      expect(builder['aa/fa//'].read).to eq('test a')
+      expect(builder['aa/fa'].read).to eq('test a')
+    end
+
+    it 'replaces tree branches' do
+      builder = dump.new_version
+      builder['a/a/a/a'] = 'test a'
+      builder['a/a/a/b'] = 'test b'
+      builder['a/a'] = 'hello'
+
+      expect(builder['a/a/a/a']).to be_nil
+      expect(builder['a/a/a/b']).to be_nil
+      expect(builder['a/a'].read).to eq('hello')
+    end
+
+    it 'removes entries and trees' do
+      builder = dump.new_version
+      builder['a/a/a'] = 'test a'
+      builder['a/a/b'] = 'test b'
+      builder['b/a'] = 'test c'
+      builder['b/b'] = 'test d'
+
+      builder['a/a'] = nil
+      builder['b/a'] = nil
+
+      version = builder.commit
+
+      expect(version.each.map(&:path)).to match_array(%w[a b])
+
+      expect(version['a'].each.map(&:path)).to be_empty
+
+      expect(version['b'].each.map(&:path)).to match_array(%w[b/b])
+    end
+
+    DATAS.each do |type, data|
+      it "does not change #{type} data" do
+        builder = dump.new_version
+        builder['a'] = data
+
+        expect(builder['a'].read).to eq(data)
+      end
+
+      it "does not change #{type} data read from file" do
+        path = File.join(tmp_dir, 'file.txt')
+        File.open(path, 'wb') do |f|
+          f.write(data)
+        end
+
+        builder = dump.new_version
+        builder.store_from('a', path)
+
+        expect(builder['a'].read).to eq(data)
+      end
+    end
+
+    describe :traversing do
+      let(:version) do
+        builder = dump.new_version
+        builder['c/c/c'] = 'c\c\c'
+        builder['a'] = 'a'
+        builder['b/a'] = 'b\a'
+        builder['b/b'] = 'b\b'
+        builder
+      end
+
+      def recursive_path_n_read(o)
+        o.each_recursive.map do |entry|
+          [entry.path, entry.read]
+        end
+      end
+
+      it 'traverses entries recursively' do
+        expect(recursive_path_n_read(version)).to match_array([
+          %w[a a],
+          %w[b/a b\a],
+          %w[b/b b\b],
+          %w[c/c/c c\c\c],
+        ])
+
+        expect(recursive_path_n_read(version['b'])).to match_array([
+          %w[b/a b\a],
+          %w[b/b b\b],
+        ])
+
+        expect(recursive_path_n_read(version['c'])).to match_array([
+          %w[c/c/c c\c\c],
+        ])
+
+        expect(recursive_path_n_read(version['c/c'])).to match_array([
+          %w[c/c/c c\c\c],
+        ])
+      end
+
+      it 'traverses level' do
+        expect(version.each.map(&:path)).to match_array(%w[a b c])
+
+        expect(version['b'].each.map(&:path)).to match_array(%w[b/a b/b])
+
+        expect(version['c'].each.map(&:path)).to eq(%w[c/c])
+
+        expect(version['c/c'].each.map(&:path)).to eq(%w[c/c/c])
+      end
+    end
+
+    describe :write_to do
+      DATAS.each do |type, data|
+        it "writes back #{type} data" do
+          builder = dump.new_version
+          builder['a'] = data
+
+          path = File.join(tmp_dir, 'file')
+          builder['a'].write_to(path)
+
+          expect(File.open(path, 'rb', &:read)).to eq(data)
+        end
+      end
+
+      [0644, 0755].each do |mode|
+        it "sets mode to #{mode.to_s(8)}" do
+          builder = dump.new_version
+          builder.store('a', 'test', mode)
+
+          path = File.join(tmp_dir, 'file')
+          builder['a'].write_to(path)
+
+          expect(File.stat(path).mode & 0777).to eq(mode)
+        end
+
+        it "fixes mode to #{mode.to_s(8)}" do
+          builder = dump.new_version
+          builder.store('a', 'test', mode & 0100)
+
+          path = File.join(tmp_dir, 'file')
+          builder['a'].write_to(path)
+
+          expect(File.stat(path).mode & 0777).to eq(mode)
+        end
+      end
+    end
+
+    it 'gets remote version ids' do
+      builder = dump.new_version
+      3.times{ builder.commit }
+
+      expect(GitDump.remote_version_ids(dump.path)).
+        to eq(dump.versions.map(&:id))
+    end
+
+    describe :exchange do
+      let(:other_dump) do
+        GitDump.new File.join(tmp_dir, 'other'), :create => true
+      end
+
+      let(:built) do
+        builder = dump.new_version
+        builder['a'] = 'b'
+        builder.commit
+      end
+
+      def check_received_version
+        expect(other_dump.versions.length).to eq(1)
+
+        version = other_dump.versions.first
+        expect(version.id).to eq(built.id)
+        expect(version.each_recursive.map do |entry|
+          [entry.path, entry.read]
+        end).to eq([%w[a b]])
+      end
+
+      it 'pushes version' do
+        built.push(other_dump.path)
+
+        check_received_version
+      end
+
+      it 'fetches version' do
+        other_dump.fetch(dump.path, built.id)
+
+        check_received_version
+      end
+    end
+  end
+end