gifts 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in gifts.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Sato Hiroyuki
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Gifts
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'gifts'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install gifts
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+Bundler.setup
+require 'rspec/core/rake_task'
+
+desc "run spec"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ["-c", "-fs"]
+end

data/gifts.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+vendor = File.expand_path('../vendor/lib', __FILE__)
+$LOAD_PATH.unshift(vendor) unless $LOAD_PATH.include?(vendor)
+require 'gifts/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "gifts"
+  gem.version       = Gifts::VERSION
+  gem.authors       = ["Sato Hiroyuki"]
+  gem.email         = ["sathiroyuki@gmail.com"]
+  gem.description   = %q{Git Fulltext Search library}
+  gem.summary       = %q{Git Fulltext Search library}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib", "vendor/lib"]
+
+  gem.add_dependency "posix-spawn", "~> 0.3.6"
+  gem.add_dependency "mime-types", "~> 1.15"
+  gem.add_dependency "diff-lcs", "~> 1.1"
+  gem.add_dependency "charlock_holmes", "~> 0.6.9"
+  gem.add_dependency "rroonga", "~> 2.1.3"
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "mocha"
+end

data/lib/gifts.rb

@@ -0,0 +1,22 @@
+require "groonga"
+require 'fileutils'
+require 'delegate'
+require 'forwardable'
+require 'charlock_holmes'
+
+require "gifts/grit"
+require "gifts/grit_ext"
+
+require "gifts/version"
+require "gifts/database"
+require "gifts/table_base"
+require "gifts/repo_table"
+require "gifts/commit_table"
+require "gifts/file_table"
+require "gifts/diff_table"
+require "gifts/term_table"
+require "gifts/user_table"
+
+module Gifts
+  # Your code goes here...
+end

data/lib/gifts/commit_table.rb

@@ -0,0 +1,70 @@
+module Gifts
+  class CommitTable < TableBase
+    StatusProcessing = 0
+    StatusCompleted = 1
+    StatusTimeout = 2
+
+    def table_name
+      "commit"
+    end
+
+    def define_schema
+      Groonga::Schema.define do |schema|
+        schema.create_table(table_name, type: :hash) do |table|
+          table.reference("author", "user")
+          table.time("authored_date")
+          table.reference("committer", "user")
+          table.time("committed_date")
+          table.string("message")
+          table.reference("repo")
+          table.string("rev")
+          table.int32("status")
+        end
+      end
+    end
+
+    def add(git_repo, db_repo)
+      result = []
+
+      offset = 0
+      count = 100
+      processed_prev_commit = false
+
+      begin
+        commits = Grit::Commit.find_all(git_repo, nil, { all: true, date_order: true, max_count: count, skip: offset })
+        commits.each do |git_commit|
+          key = db_repo.id.to_s + ":" + git_commit.id
+
+          if key == db_repo.last_commit_key
+            processed_prev_commit = true
+            break
+
+          else
+            author = @db.users.add(git_commit.author.name)
+            committer = @db.users.add(git_commit.committer.name)
+
+            db_commit =
+              table[key] ||
+              table.add(
+                key,
+                author: author,
+                authored_date: git_commit.authored_date,
+                committer: committer,
+                committed_date: git_commit.committed_date,
+                message: git_commit.message,
+                repo: db_repo,
+                rev: git_commit.id,
+                status: StatusProcessing
+              )
+
+              db_diffs = @db.diffs.add(git_commit, db_commit)
+              result << db_commit
+          end
+        end
+        offset += count
+      end until commits.count == 0 || processed_prev_commit
+
+      result
+    end
+  end
+end

data/lib/gifts/database.rb

@@ -0,0 +1,49 @@
+module Gifts
+  class Database
+    attr_reader :repos, :commits, :files, :diffs, :terms, :users
+
+    def self.open(filename)
+      if File.exist?(filename)
+        database = Groonga::Database.open(filename)
+      else
+        FileUtils.mkdir_p(File.dirname filename)
+        database = Groonga::Database.create(path: filename)
+      end
+
+      database = Database.new(database)
+
+      if block_given?
+        begin
+          yield database
+        ensure
+          database.close
+        end
+      else
+        database
+      end
+    end
+
+    def initialize(database)
+      @database = database
+
+      # Tables need initialized in the order corresponding to referenced.
+      @repos = RepoTable.new(self)
+      @users = UserTable.new(self)
+      @commits = CommitTable.new(self)
+      @files = FileTable.new(self)
+      @diffs = DiffTable.new(self)
+      @terms = TermTable.new(self)
+    end
+
+    def close
+      unless closed?
+        @database.close
+        @database = nil
+      end
+    end
+
+    def closed?
+      @database.nil? or @database.closed?
+    end
+  end
+end

data/lib/gifts/diff_table.rb

@@ -0,0 +1,55 @@
+module Gifts
+  class DiffTable < TableBase
+    def table_name
+      "diff"
+    end
+
+    def define_schema
+      Groonga::Schema.define do |schema|
+        schema.create_table(table_name, type: :hash) do |table|
+          table.reference("commit")
+          table.reference("file")
+          table.text("diff")
+        end
+      end
+    end
+
+    def add(git_commit, db_commit)
+      if db_commit.status == CommitTable::StatusProcessing
+        result = []
+
+        begin
+          git_commit.diffs.each do |git_diff|
+            next if git_diff.diff.nil?
+
+            db_file = @db.files.add(git_commit, git_diff, db_commit)
+
+            if db_file && db_file.type == FileTable::TypeText
+              key = db_commit.id.to_s + ":" + db_file.id.to_s
+              db_diff =
+                table[key] ||
+                table.add(
+                  key,
+                  commit: db_commit,
+                  file: db_file,
+                  diff: git_diff.diff
+                )
+              result << db_diff
+            end
+          end
+        rescue Grit::Git::GitTimeout => e
+          puts "Timeout commit:#{git_commit.id}"
+          db_commit.status = CommitTable::StatusTimeout
+        else
+          db_commit.status = CommitTable::StatusCompleted
+        end
+
+        result
+      else
+        records = table.select do |record|
+          record.commit == db_commit
+        end
+      end
+    end
+  end
+end

data/lib/gifts/file_table.rb

@@ -0,0 +1,51 @@
+module Gifts
+  class FileTable < TableBase
+    TypeText = 1
+    TypeBinary = 2
+
+    def table_name
+      "file"
+    end
+
+    def define_schema
+      Groonga::Schema.define do |schema|
+        schema.create_table(table_name, type: :hash) do |table|
+          table.string("encoding")
+          table.string("ext")
+          table.reference("last_commit", "commit")
+          table.string("path")
+          table.reference("repo")
+          table.int32("type")
+        end
+      end
+    end
+
+    def add(git_commit, git_diff, db_commit)
+      path = git_diff.new_path
+      file = (git_commit.tree / path)
+      if file.nil? && git_commit.parents.count > 0
+        path = git_diff.old_path
+        file = (git_commit.parents.first.tree / path)
+      end
+      return if file.nil?
+
+      key = db_commit.repo.id.to_s + ":" + path
+      return table[key] if table[key] && table[key].last_commit.committed_date > db_commit.committed_date
+
+      ext = File.extname(path).sub(".", "")
+
+      t =  table[key] || table.add(key, repo: db_commit.repo, path: path, ext: ext)
+      t.last_commit = db_commit
+
+      detection = CharlockHolmes::EncodingDetector.detect(file.data)
+      if detection[:type] == :binary
+        t.type = TypeBinary
+      else
+        t.type = TypeText
+        t.encoding = detection[:encoding]
+      end
+
+      t
+    end
+  end
+end

data/lib/gifts/repo_table.rb

@@ -0,0 +1,31 @@
+module Gifts
+  class RepoTable < TableBase
+    def table_name
+      "repo"
+    end
+
+    def define_schema
+      Groonga::Schema.define do |schema|
+        schema.create_table(table_name, type: :hash) do |table|
+          table.string("path")
+          table.string("last_commit_key")
+        end
+      end
+    end
+
+    def add(path)
+      path  = File.expand_path(path)
+
+      git_repo = Grit::Repo.new(path)
+      db_repo = table[path] || table.add(path, path: path)
+
+      db_commits = @db.commits.add(git_repo, db_repo)
+
+      # update last commit key
+      db_repo.last_commit_key = db_commits.first.key if db_commits.count > 0
+      pp "add", db_commits.count, db_repo.last_commit_key
+
+      db_repo
+    end
+  end
+end

data/lib/gifts/table_base.rb

@@ -0,0 +1,18 @@
+module Gifts
+  class TableBase
+    extend Forwardable
+
+    def_delegators :table, :count, :records, :select, :size
+
+    def initialize(database)
+      @db = database
+      define_schema
+    end
+
+    protected
+
+    def table
+      Groonga[table_name]
+    end
+  end
+end

data/lib/gifts/term_table.rb

@@ -0,0 +1,28 @@
+module Gifts
+  class TermTable < TableBase
+    def table_name
+      "term"
+    end
+
+    def define_schema
+      Groonga::Schema.define do |schema|
+        schema.create_table(
+          "term",
+          type: :patricia_trie,
+          normalizer: :NormalizerAuto,
+          default_tokenizer: "TokenBigram"
+        ) do |table|
+          table.index("commit.message", with_position: true)
+
+          table.index("diff.commit", with_position: true)
+          table.index("diff.diff", with_position: true)
+
+          table.index("file.ext", with_position: true)
+          table.index("file.path", with_position: true)
+
+          table.index("user.name", with_position: true)
+        end
+      end
+    end
+  end
+end