georgi-git_store 0.1.1 → 0.2

data/lib/git_store.rb

@@ -1,240 +1,320 @@
-require 'grit'
-
-# This fix ensures sorted yaml maps.
-class Hash
-	def to_yaml( opts = {} )
-		YAML::quick_emit( object_id, opts ) do |out|
-      out.map( taguri, to_yaml_style ) do |map|
-        sort_by { |k, v| k.to_s }.each do |k, v|
-          map.add( k, v )
-        end
-      end
-    end
-	end
-end
-
+require 'rubygems'
+require 'zlib'
+require 'digest/sha1'
+require 'yaml'
+
+require 'git_store/blob'
+require 'git_store/tree'
+require 'git_store/handlers'
+require 'git_store/pack'
+
+# GitStore implements a versioned data store based on the revision
+# management system git. You can store object hierarchies as nested
+# hashes, which will be mapped on the directory structure of a git
+# repository.
+#
+# GitStore supports transactions, so that updates to the store either
+# fail or succeed completely.
+#
+# GitStore manages concurrent access by a file locking scheme. So only
+# one process can start a transaction at one time. This is implemented
+# by locking the `refs/head/<branch>.lock` file, which is also respected
+# by the git binary.
+#
+# A regular commit should be atomic by the nature of git, as the only
+# critical part is writing the 40 bytes SHA1 hash of the commit object
+# to the file `refs/head/<branch>`, which is done atomically by the
+# operating system.
+#
+# So reading a repository should be always consistent in a git
+# repository. The head of a branch points to a commit object, which in
+# turn points to a tree object, which itself is a snapshot of the
+# GitStore at commit time. All involved objects are keyed by their
+# SHA1 value, so there is no chance for another process to write to
+# the same files.
+#
 class GitStore
+  include Enumerable
 
-  class DefaultHandler
-    def read(name, data)
-      data
-    end
+  attr_reader :path, :index, :root, :branch, :lock_file, :head  
 
-    def write(data)
-      data
-    end
+  # Initialize a store.
+  def initialize(path, branch = 'master')
+    @path   = path.chomp('/')
+    @branch = branch
+    @root   = Tree.new(self)
+    
+    load_packs("#{path}/.git/objects/pack")
+    load
   end
-  
-  class YAMLHandler    
-    def read(name, data)
-      YAML.load(data)
-    end
 
-    def write(data)
-      data.to_yaml
-    end    
+  # The path to the current head file.
+  def head_path
+    "#{path}/.git/refs/heads/#{branch}"
   end
 
-  class RubyHandler
-    def read(name, data)
-      Object.module_eval(data)
-    end
+  # The path to the object file for given id.
+  def object_path(id)
+    "#{path}/.git/objects/#{ id[0...2] }/#{ id[2..39] }"
   end
 
-  class ERBHandler
-    def read(name, data)
-      ERB.new(data)
-    end
+  # Read the id of the head commit.
+  #
+  # Returns the object id of the last commit.
+  def read_head
+    File.read(head_path).strip if File.exists?(head_path)
   end
 
-  Handler = {
-    'yml' => YAMLHandler.new,
-    'rhtml' => ERBHandler.new,
-    'rxml' => ERBHandler.new,
-    'rb' => RubyHandler.new
-  }
+  # Read an object for the specified path.
+  #
+  # Use multiple arguments or a string with slashes.
+  def [](*args)
+    root[*args]
+  end
 
-  Handler.default = DefaultHandler.new
+  # Write an object to the specified path.
+  #
+  # Use multiple arguments or a string with slashes.
+  def []=(*args)
+    value = args.pop
+    root[*args] = value
+  end
 
-  class Blob
+  # Delete the specified path.
+  #
+  # Use multiple arguments or a string with slashes.
+  def delete(*args)
+    root.delete(*args)
+  end
 
-    attr_reader :id
-    attr_accessor :name
+  # Returns the store as a hash tree.
+  def to_hash
+    root.to_hash
+  end
 
-    def initialize(*args)
-      if args.first.is_a?(Grit::Blob)
-        @blob = args.first
-        @name = @blob.name
-      else
-        @name = args[0]
-        self.data = args[1]
-      end
-    end
+  # Inspect the store.
+  def inspect
+    "#<GitStore #{path} #{branch} #{root.to_hash.inspect}>"
+  end
 
-    def extname
-      File.extname(name)[1..-1]
-    end
+  # Iterate over all values found in this store.
+  def each(&block)
+    root.each(&block)
+  end
 
-    def load(data)
-      @data = handler.read(name, data)
-    end
+  # Has our store been changed on disk?
+  def changed?
+    head != read_head
+  end
 
-    def handler
-      Handler[extname]
-    end
+  def refresh!
+    load if changed?
+  end
 
-    def data
-      @data or (@blob and load(@blob.data))
+  # Load the current head version from repository. 
+  def load
+    if @head = read_head
+      commit = get_object(head)[0]
+      root.id = commit.split(/[ \n]/, 3)[1].strip
+      root.data = get_object(root.id)[0]
+      root.load_from_store
     end
+  end
 
-    def data=(data)
-      @data = data
-    end
+  # Reload the store, if it has been changed on disk.
+  def refresh!
+    load if changed?
+  end
 
-    def to_s
-      if handler.respond_to?(:write)
-        handler.write(data)
-      else
-        @blob.data
-      end
-    end
+  # Do we have a current transacation?
+  def in_transaction?
+    Thread.current['git_store_lock']
+  end
+
+  # All changes made inside a transaction are atomic. If some
+  # exception occurs the transaction will be rolled back.
+  #
+  # Example:
+  #   store.transaction { store['a'] = 'b' }
+  #
+  def transaction(message = "")
+    start_transaction
+    result = yield
+    commit message
     
+    result
+  rescue
+    rollback
+    raise
+  ensure
+    finish_transaction
   end
 
-  class Tree
-    include Enumerable
+  # Start a transaction.
+  #
+  # Tries to get lock on lock file, reload the this store if
+  # has changed in the repository.
+  def start_transaction
+    file = open("#{head_path}.lock", "w")
+    file.flock(File::LOCK_EX)
+    
+    Thread.current['git_store_lock'] = file
+    
+    load if changed?
+  end
 
-    attr_reader :data
-    attr_accessor :name
+  # Restore the state of the store.
+  #
+  # Any changes made to the store are discarded.
+  def rollback
+    root.load_from_store
+    finish_transaction
+  end
+  
+  # Finish the transaction.
+  #
+  # Release the lock file.
+  def finish_transaction
+    Thread.current['git_store_lock'].close rescue nil
+    Thread.current['git_store_lock'] = nil
     
-    def initialize(name = nil)
-      @data = {}
-      @name = name
-    end
+    File.unlink("#{head_path}.lock") rescue nil    
+  end
 
-    def load(tree)
-      @name = tree.name
-      @data = tree.contents.inject({}) do |hash, file|
-        if file.is_a?(Grit::Tree)
-          hash[file.name] = (@data[file.name] || Tree.new).load(file)
-        else          
-          hash[file.name] = Blob.new(file)
-        end
-        hash
-      end
-      self
+  # Write the commit object to disk and set the head of the current branch.
+  #
+  # Returns the id of the commit object
+  def commit(message = '', author = 'ruby', committer = 'ruby')
+    time = "#{ Time.now.to_i } #{ Time.now.to_s.split[4] }"
+    tree = root.write_to_store
+
+    contents = [ "tree #{tree}", (head and "parent #{head}"),
+                 "author #{author} #{time}",
+                 "committer #{committer} #{time}", '', message
+                 ].compact.join("\n")
+
+    id = put_object(contents, 'commit')
+
+    open(head_path, "wb") do |file|
+      file.write(id)
     end
 
-    def inspect
-      "#<GitStore::Tree #{@data.inspect}>"
+    @head = id
+  end
+
+  # Read the raw object with the given id from the repository.
+  #
+  # Returns a pair of content and type of the object
+  def get_object(id)
+    path = object_path(id)
+    
+    if File.exists?(path)
+      buf = open(path, "rb") { |f| f.read }
+    else
+      get_object_from_pack(id)
     end
+    
+    raise if not legacy_loose_object?(buf)
+    
+    header, content = Zlib::Inflate.inflate(buf).split(/\0/, 2)
+    type, size = header.split(/ /, 2)
+    
+    raise if size.to_i != content.size
+    
+    return content, type
+  end
 
-    def fetch(name)
-      name = name.to_s
-      entry = @data[name]
-      case entry
-      when Blob then entry.data
-      when Tree then entry
-      end      
+  def get_object_from_pack(id)
+    packs.each do |pack|
+      data = pack[id] and return data
     end
+  end      
 
-    def store(name, value)
-      name = name.to_s
-      if value.is_a?(Tree)
-        value.name = name
-        @data[name] = value
-      else
-        @data[name] = Blob.new(name, value)
+  # Returns the hash value of an object string.
+  def sha(str)
+    Digest::SHA1.hexdigest(str)[0, 40]
+  end
+
+  # Write a raw object to the repository.
+  #
+  # Returns the object id.
+  def put_object(content, type)
+    size = content.length.to_s    
+    header = "#{type} #{size}\0"
+    data = header + content
+    
+    id = sha(data)
+    path = object_path(id)
+    
+    unless File.exists?(path)
+      FileUtils.mkpath(File.dirname(path))
+      open(path, 'wb') do |f|
+        f.write Zlib::Deflate.deflate(data)
       end
     end
+    
+    id
+  end
 
-    def has_key?(name)
-      @data.has_key?(name)
-    end
+  def legacy_loose_object?(buf)
+    word = (buf[0] << 8) + buf[1]
+    buf[0] == 0x78 && word % 31 == 0
+  end  
 
-    def [](*args)
-      args = args.first.to_s.split('/') if args.size == 1
-      args.inject(self) { |tree, key| tree.fetch(key) or return nil }
+  def load_packs(path)
+    if File.directory?(path)
+      Dir.open(path) do |dir|
+        entries = dir.select { |entry| entry =~ /\.pack$/i }
+        @packs = entries.map { |entry| PackStorage.new(File.join(path, entry)) }
+      end
+    end
+  end
+  
+  # FileStore reads a working copy out of a directory. Changes made to
+  # the store will not be written to a repository. This is useful, if
+  # you want to read a filesystem without having a git repository.
+  class FileStore < GitStore
+
+    def initialize(path)
+      @mtime = {}
+      super
     end
 
-    def []=(*args)
-      value = args.pop
-      args = args.first.to_s.split('/') if args.size == 1
-      tree = args[0..-2].to_a.inject(self) do |tree, key|
-        tree.has_key?(key) ? tree.fetch(key) : tree.store(key, Tree.new(key))
+    def load
+      root.load_from_disk
+      
+      each_blob_in(root) do |blob|
+        @mtime[blob.path] = File.mtime("#{path}/#{blob.path}")
       end
-      tree.store(args.last, value)
     end
 
-    def delete(name)
-      @data.delete(name)
-    end
-    
-    def each(&block)
-      @data.values.each do |entry|
+    def each_blob_in(tree, &blob)
+      tree.table.each do |name, entry|
         case entry
-        when Blob then yield entry.data
-        when Tree then entry.each(&block)
+        when Blob; yield entry
+        when Tree; each_blob_in(entry, &blob)
         end
       end
-    end
-
-    def each_with_path(path = [], &block)
-      @data.each do |name, entry|
-        child_path = path + [name]
-        case entry
-        when Blob then yield entry, child_path.join('/')
-        when Tree then entry.each_with_path(child_path, &block)
-        end        
+    end        
+
+    def refresh!
+      each_blob_in(root) do |blob|
+        path = "#{self.path}/#{blob.path}"
+        if File.exist?(path)
+          mtime = File.mtime(path)
+          if @mtime[blob.path] != mtime
+            @mtime[blob.path] = mtime
+            blob.load_from_disk
+          end
+        else
+          delete blob.path
+        end
       end
     end
-      
-    def to_hash
-      @data.inject({}) do |hash, (name, entry)|
-        hash[name] = entry.is_a?(Tree) ? entry.to_hash : entry.to_s
-        hash
-      end
+
+    def commit(message="")
     end
     
   end
 
-  attr_reader :repo, :index, :tree
-
-  def initialize(path, &block)
-    @repo = Grit::Repo.new(path)
-    @index = Grit::Index.new(@repo)
-    @tree = Tree.new
-  end
-
-  def commit(message="")
-    index.tree = tree.to_hash
-    head = repo.heads.first
-    index.commit(message, head ? head.commit.id : nil)
-  end
-
-  def [](*args)
-    tree[*args]
-  end
-
-  def []=(*args)
-    value = args.pop
-    tree[*args] = value
-  end
-
-  def delete(path)
-    tree.delete(path)
-  end
-
-  def load
-    tree.load(repo.tree)
-  end
-
-  def each(&block)
-    tree.each(&block)
-  end
-
-  def each_with_path(&block)
-    tree.each_with_path(&block)
-  end
-
 end

data/test/benchmark.rb

@@ -0,0 +1,30 @@
+require 'git_store'
+require 'grit'
+require 'benchmark'
+require 'fileutils'
+
+REPO = File.expand_path(File.dirname(__FILE__) + '/repo')
+
+FileUtils.rm_rf REPO
+FileUtils.mkpath REPO
+Dir.chdir REPO
+
+`git init`
+
+store = GitStore.new(REPO)
+
+Benchmark.bm 20 do |x|
+  x.report 'store 1000 objects' do
+    store.transaction { 'aaa'.upto('jjj') { |key| store[key] = rand.to_s } }
+  end
+  x.report 'commit one object' do
+    store.transaction { store['aa'] = rand.to_s }
+  end
+  x.report 'load 1000 objects' do
+    GitStore.new('.')
+  end
+  x.report 'load 1000 with grit' do
+    Grit::Repo.new('.').tree.contents.each { |e| e.data }
+  end  
+end
+