git-object-browser 0.0.6 → 0.0.7

data/lib/git-object-browser/dumper/packed_objects_dumper.rb

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+module GitObjectBrowser
+
+  module Dumper
+
+    class PackedObjectsDumper
+
+      def initialize(root, outdir)
+        @root   = root
+        @outdir = outdir
+      end
+
+      def dump(path, index)
+        File.open(File.join(@root, path)) do |input|
+          index.entries.each do |entry|
+            dump_packed_object(index, input, entry[:offset], path)
+          end
+        end
+      end
+
+      def dump_packed_object(index, input, offset, path)
+        obj = GitObjectBrowser::Models::PackedObject.new(index, input).parse(offset)
+        wrapped = GitObjectBrowser::Models::WrappedObject.new(nil, path, obj)
+
+        ostr = "0000#{ offset }"
+        outfile = File.join(@outdir, path, ostr[-2,2], ostr[-4,2], "#{ offset }.json")
+        FileUtils.mkdir_p(File.dirname(outfile))
+
+        File.open(outfile, 'w') do |output|
+          output << JSON.pretty_generate(wrapped.to_hash)
+        end
+      end
+    end
+  end
+end

data/lib/git-object-browser/dumper/packed_refs_dumper.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+
+module GitObjectBrowser
+
+  module Dumper
+
+    class PackedRefsDumper
+
+      def initialize(root, outdir)
+        @root   = root
+        @outdir = outdir
+      end
+
+      def dump
+        src_file = File.join(@root, "packed-refs")
+        dst_file = File.join(@outdir, "packed-refs.json")
+
+        return unless File.exist?(src_file)
+
+        puts "Write: packed-refs\n"
+        File.open(src_file) do |input|
+          File.open(dst_file, "w") do |output|
+            dump_object(input, output)
+          end
+        end
+      end
+
+      def dump_object(input, output)
+        obj =  GitObjectBrowser::Models::PackedRefs.new(input)
+        wrapped = GitObjectBrowser::Models::WrappedObject.new(nil, 'packed-refs', obj)
+        output << JSON.pretty_generate(wrapped.to_hash)
+      end
+
+    end
+  end
+end
+

data/lib/git-object-browser/dumper/plain_files_dumper.rb

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+
+module GitObjectBrowser
+
+  module Dumper
+
+    class PlainFilesDumper
+
+      def initialize(root, outdir)
+        @root   = root
+        @outdir = outdir
+      end
+
+      def dump
+        obj_files = []
+
+        # ! info/refs
+        # info
+        # logs
+        # objects/info
+
+        plain_files = []
+        subdirs = []
+
+        Dir.chdir(@root) do
+          Dir.glob('*') do |path|
+            next if %w{HEAD FETCH_HEAD ORIG_HEAD MERGE_HEAD CHERRY_PICK_HEAD}.include?(path)
+            next if %w{dump index objects refs packed-refs}.include?(path)
+
+            if File.directory?(path)
+              subdirs << path
+            else
+              plain_files << path
+            end
+          end
+        end
+        subdirs << 'objects/info'
+
+        subdirs.each do |dir|
+          next unless File.directory?(File.join(@root, dir))
+          Dir.chdir(File.join(@root, dir)) do
+            Dir.glob('**/*') do |path|
+              # skip info/refs (InfoRefs)
+              next if dir == 'info' && path == 'refs'
+              if File.file?(path)
+                plain_files << File.join(dir, path)
+              end
+            end
+          end
+        end
+
+        plain_files.each do |path|
+          outfile = File.join(@outdir, "#{ path }.json")
+          FileUtils.mkdir_p(File.dirname(outfile))
+
+          puts "Write: #{path}\n"
+          file = File.join(@root, path)
+          File.open(file) do |input|
+            File.open(outfile, "w") do |output|
+              dump_object(input, output, path)
+            end
+          end
+        end
+      end
+
+      def dump_object(input, output, path)
+        obj =  GitObjectBrowser::Models::PlainFile.new(input).parse
+        wrapped = GitObjectBrowser::Models::WrappedObject.new(nil, path, obj)
+        output << JSON.pretty_generate(wrapped.to_hash)
+      end
+    end
+  end
+end
+

data/lib/git-object-browser/dumper/refs_dumper.rb

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+module GitObjectBrowser
+
+  module Dumper
+
+    class RefsDumper
+
+      def initialize(root, outdir)
+        @root   = root
+        @outdir = outdir
+      end
+
+      def dump
+        ref_files = %w{HEAD FETCH_HEAD ORIG_HEAD MERGE_HEAD CHERRY_PICK_HEAD}
+        Dir.chdir(@root) do
+          Dir.glob("refs/**/*") do |path|
+            ref_files << path if File.file?(path)
+          end
+        end
+        return if ref_files.empty?
+
+        ref_files.each do |path|
+          next unless File.exist?(File.join(@root, path))
+          outfile = File.join(@outdir, "#{ path }.json")
+          FileUtils.mkdir_p(File.dirname(outfile))
+
+          puts "Write: #{path}\n"
+          ref_file = File.join(@root, path)
+          File.open(ref_file) do |input|
+            File.open(outfile, "w") do |output|
+              dump_object(input, output, path)
+            end
+          end
+        end
+      end
+
+      def dump_object(input, output, path)
+        obj =  GitObjectBrowser::Models::Ref.new(input)
+        wrapped = GitObjectBrowser::Models::WrappedObject.new(nil, path, obj)
+        output << JSON.pretty_generate(wrapped.to_hash)
+      end
+    end
+  end
+end
+

data/lib/git-object-browser/main.rb

@@ -4,7 +4,10 @@ module GitObjectBrowser
     def execute
       host = '127.0.0.1'
       port = 8080
-      dump = false
+      dump = nil
+      step = nil
+      diff_dir = nil
+      nextstep = false
       opts = OptionParser.new do |opts|
         opts.on('-p', '--port PORT', 'port number') do |value|
           port = value.to_i if 0 < value.to_i
@@ -12,8 +15,17 @@ module GitObjectBrowser
         opts.on('-b', '--bind HOST', 'address to bind') do |value|
           host = value
         end
-        opts.on('-d', '--dump', 'dump objects') do |value|
-          dump = true
+        opts.on('-d', '--dump PATH', 'dump .git as JSON and HTMLs') do |value|
+          dump = value
+        end
+        opts.on('--step STEP', 'dump JSONs into sub directory') do |value|
+          step = value
+        end
+        opts.on('--diff OLD_JSON_DIR', 'dump JSONs with diff data') do |value|
+          diff_dir = value
+        end
+        opts.on('--next', 'same with --step step${N} --diff step${N-1}') do |value|
+          nextstep = true
         end
         opts.on_tail("-h", "--help", "show this help.") do
           puts opts
@@ -26,10 +38,14 @@ module GitObjectBrowser
         opts.parse!(ARGV)
       end
       target = find_target(ARGV[0])
-      if dump
-        GitObjectBrowser::Dumper::Main.execute(target)
-      else
+      if dump.nil?
         GitObjectBrowser::Server::Main.execute(target, host, port)
+      else
+        if nextstep
+          step = :next
+          diff_dir = nil
+        end
+        GitObjectBrowser::Dumper::Main.execute(target, dump, step, diff_dir)
       end
     end
 

data/lib/git-object-browser/models/directory.rb

@@ -18,11 +18,13 @@ module GitObjectBrowser
             relpath = File.join(@path, file).gsub(%r{\A/}, '')
             entry = {}
             if File.directory?(file)
-              entry[:type] = "directory"
+              entry[:type] = 'directory'
             elsif File.symlink?(file)
-              entry[:type] = "symlink"
+              entry[:type] = 'symlink'
             elsif Ref::path?(relpath)
               entry[:type] = 'ref'
+            elsif Reflog::path?(relpath)
+              entry[:type] = 'reflog'
             elsif InfoRefs::path?(relpath)
               entry[:type] = 'info_refs'
             elsif PackedRefs::path?(relpath)
@@ -40,7 +42,7 @@ module GitObjectBrowser
             entries << entry
           end
         end
-        order = %w{directory ref info_refs packed_refs index object file symlink}
+        order = %w{directory ref reflog info_refs packed_refs index object file symlink}
         entries.sort do |a,b|
           (order.index(a[:type]) <=> order.index(b[:type])).nonzero? ||
             a[:basename] <=> b[:basename]

data/lib/git-object-browser/models/git_date.rb

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+module GitObjectBrowser
+
+  module Models
+
+    class GitDate
+      attr_reader :unixtime, :timezone, :date
+
+      def initialize(unixtime, timezone)
+        @unixtime = unixtime
+        @timezone = timezone
+        @date = DateTime.strptime(unixtime.to_s, '%s').new_offset(parse_timezone(timezone))
+      end
+
+      def parse_timezone(timezone)
+        timezone = '+00:00' if timezone == 'Z'
+        return Rational(0, 24) unless timezone =~ /(\+|-)?(\d\d):?(\d\d)/
+        Rational($2.to_i, 24) + Rational($3, 60) * (($1 == '-') ? -1 : 1)
+      end
+
+      def to_s
+        @date.iso8601
+      end
+    end
+  end
+end

data/lib/git-object-browser/models/git_object.rb

@@ -98,7 +98,7 @@ module GitObjectBrowser
             prop[:email] = force_utf8($2)
             prop[:unixtime] = $3
             prop[:timezone] = $4
-            prop[:date] = epoch($3.to_i, $4).iso8601
+            prop[:date] = GitDate.new($3, $4).to_s
 
             prop[:value] = force_utf8(prop[:value])
           else

data/lib/git-object-browser/models/pack_file.rb

@@ -17,7 +17,7 @@ module GitObjectBrowser
       end
 
       def self.path?(relpath)
-        return relpath =~ %r{\Aobjects/pack/pack-[0-9a-f]{40}.pack\z}
+        return relpath =~ %r{\Aobjects/pack/pack-[0-9a-f]{40}\.pack\z}
       end
 
       def parse

data/lib/git-object-browser/models/pack_index.rb

@@ -4,46 +4,142 @@ module GitObjectBrowser
   module Models
 
     # v2
-    #   signature  4bytes
-    #   version    4bytes
-    #   fanout     4bytes * 256
-    #   sha1      20bytes * fanout[255]
-    #   crc        4bytes * fanout[255]
-    #   offset     4bytes * fanout[255]
+    #   signature   4bytes
+    #   version     4bytes
+    #   fanout      4bytes * 256
+    #   sha1       20bytes * fanout[255]
+    #   crc32       4bytes * fanout[255]
+    #   offset      4bytes * fanout[255]
+    #   pack  sha1 20bytes
+    #   index sha1 20bytes
     class PackIndex < Bindata
+      attr_reader :entries
+
+      PER_PAGE = 200
 
       def initialize(input)
         super(input)
       end
 
-      def parse
+      def parse(order, page)
         parse_fanout
+        @page = page
+        @order = order
+
+        if order == 'digest'
+          parse_digest
+        elsif order == 'sha1'
+          parse_sha1_page
+          set_fanouts
+        else
+          @order = 'offset'
+          parse_offset_page
+          set_fanouts
+        end
+
+        # XXX Doesn't support packfile >= 2 GiB
+        # x.times do |i|
+        #   puts "offset[#{ i }] = #{ hex(8) }"
+        # end
+
+        seek(4 + 4 + 4 * 256 + (20 + 4 + 4) * @fanout[255])
+        @packfile_sha1 = hex(20)
+        @index_sha1 = hex(20)
+
+        self
+      end
 
+      def empty?
+        @entries.empty?
+      end
+
+      def set_fanouts
+        @fanout.each_with_index do |fo, i|
+          entry = @entries.select { |entry| entry[:index] == fo }
+          next if entry.empty?
+          entry = entry.first
+          entry[:fanout_min] = entry[:fanout_min] ? [entry[:fanout_min], i].min : i
+          entry[:fanout_max] = entry[:fanout_max] ? [entry[:fanout_max], i].max : i
+        end
+
+        @entries.each do |entry|
+          next unless entry[:fanout_min]
+          entry[:fanout_min] = '%02x' % entry[:fanout_min]
+          entry[:fanout_max] = '%02x' % entry[:fanout_max]
+          entry.delete(:fanout_max) if entry[:fanout_min] == entry[:fanout_max]
+        end
+      end
+      private :set_fanouts
+
+      def parse_digest
+        index = 0
         @entries = []
-        @fanout[255].times do |i|
-          entry = {}
-          entry[:sha1] = hex(20)
+        while index <= @fanout[255] - 1
+          @entries << hex(20)
+          index += PER_PAGE
+          skip(20 * (PER_PAGE - 1)) if index <= @fanout[255] - 1
+        end
+      end
+      private :parse_digest
+
+      def parse_sha1_page
+        @entries = []
+
+        index_start = PER_PAGE * (@page - 1)
+        index_end   = PER_PAGE * @page - 1
+        index_last  = @fanout[255] - 1
+        return if index_last < index_start
+        index_end = index_last if index_last <= index_end
+        entry_count = index_end - index_start + 1
+
+        skip(20 * index_start)
+        entry_count.times do |i|
+          entry = { :index => index_start + i, :sha1 => hex(20) }
           @entries << entry
         end
+        skip(20 * (index_last - index_end))
 
-        @fanout[255].times do |i|
+        skip(4 * index_start)
+        entry_count.times do |i|
           @entries[i][:crc32] = hex(4)
         end
+        skip(4 * (index_last - index_end))
 
-        @fanout[255].times do |i|
+        skip(4 * index_start)
+        entry_count.times do |i|
           @entries[i][:offset] = int
         end
+      end
+      private :parse_sha1_page
 
-        # XXX Doesn't support packfile >= 2 GiB
-        # x.times do |i|
-        #   puts "offset[#{ i }] = #{ hex(8) }"
-        # end
+      def parse_offset_page
+        @entries = []
 
-        @packfile_sha1 = hex(20)
-        @index_sha1 = hex(20)
+        index_start = PER_PAGE * (@page - 1)
+        index_end   = PER_PAGE * @page - 1
+        index_last  = @fanout[255] - 1
+        return if index_last < index_start
+        index_end = index_last if index_last <= index_end
+        entry_count = index_end - index_start + 1
 
-        self
+        # load all offsets
+        skip((20 + 4) * @fanout[255])
+        offsets = []
+        @fanout[255].times do |i|
+          offsets << [i, int]
+        end
+
+        offsets = offsets.sort { |a,b| a[1] <=> b[1] }[index_start, entry_count]
+        offsets.each do |offset|
+          @entries << { :index => offset[0], :offset => offset[1] }
+        end
+
+        @entries.each do |entry|
+          entry[:sha1] = get_sha1_hex(entry[:index])
+          entry[:crc32] = get_crc32_hex(entry[:index])
+        end
       end
+      private :parse_offset_page
 
       def parse_fanout
         return if @fanout
@@ -72,11 +168,11 @@ module GitObjectBrowser
 
         while lo < hi
           mid = (lo + hi) / 2
-          mid_sha1 = get_sha1(mid)
+          mid_sha1 = get_sha1_raw(mid)
           if mid_sha1 == sha1
             return {
               :sha1   => sha1_hex,
-              :crc    => get_crc_hex(mid),
+              :crc32  => get_crc32_hex(mid),
               :offset => get_offset(mid)
             }
           elsif sha1 < mid_sha1
@@ -88,7 +184,7 @@ module GitObjectBrowser
         nil
       end
 
-      def get_sha1(pos)
+      def get_sha1_raw(pos)
         if @version == 2
           seek(4 + 4 + 4 * 256 + 20 * pos)
         else
@@ -97,7 +193,16 @@ module GitObjectBrowser
         raw(20)
       end
 
-      def get_crc_hex(pos)
+      def get_sha1_hex(pos)
+        if @version == 2
+          seek(4 + 4 + 4 * 256 + 20 * pos)
+        else
+          raise "FIXME version 1"
+        end
+        hex(20)
+      end
+
+      def get_crc32_hex(pos)
         if @version == 2
           seek(4 + 4 + 4 * 256 + 20 * @fanout[255] + 4 * pos)
         else
@@ -116,18 +221,27 @@ module GitObjectBrowser
       end
 
       def self.path?(relpath)
-        return relpath =~ %r{\Aobjects/pack/pack-[0-9a-f]{40}.idx\z}
+        return relpath =~ %r{\Aobjects/pack/pack-[0-9a-f]{40}\.idx\z}
       end
 
       def to_hash
         return {
-          :fanout        => @fanout,
           :entries       => @entries,
           :packfile_sha1 => @packfile_sha1,
           :index_sha1    => @index_sha1,
         }
       end
 
+      # Extended pagenation data for Git Object Browser.
+      def page_data
+        return {
+          :per_page      => PER_PAGE,
+          :entry_count   => @fanout[255],
+          :page          => @page,
+          :order         => @order
+        }
+      end
+
       def load_object_types(input)
         obj = PackedObject.new(self, input)
         @entries.each do |entry|