elasticsearch-git 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a7d0d063702f3bcc2f71f5a8ad9b55ec0377280
-  data.tar.gz: 656199334b4345491eddb15fd3f4e3ffd0a0b597
+  metadata.gz: 743ab16ca2c38bbe9ab23dc7447c5c41d578f415
+  data.tar.gz: 4fba8f5da92706100fa6b01e582cbcd16a83d5ad
 SHA512:
-  metadata.gz: bf49c347091bb8866356b74bed98b9652b44474d7e5e5341257f06bf759a0eaf48efdf46ccdafa4211a2a7d3e98d390f33ff3b59900a3f7bc737c1020b25be75
-  data.tar.gz: 99870c02c0532ed6f72b70ee7e962c778148d7bb794a73f20eebd0cafa74e859baf1fbc1d3443fcb90b87280d06018f40f43d8299651a7413985588b218f2c6e
+  metadata.gz: efe576c9de405aa4b22c89cfe21171d6de7ac5e686ed4207e73e085cf83d673b114dced51cc6ecb051b6f38aeba9227bf45619cd8f65b7a228101a703d250599
+  data.tar.gz: 2e18154c17713a627d3d4b9519f279348bcdccd0779819e7164f1d01e9c62092dc8c5c408b34782e2f46f6c8778a8c52fa75a3985537fcfb30baa36c5088cd66

data/Gemfile

@@ -9,7 +9,8 @@ gem 'rugged', git: 'git://github.com/libgit2/rugged.git', branch: 'development',
 gem 'bundler'
 gem 'rake'
 gem 'pry'
-gem 'gitlab_git'
+gem 'charlock_holmes', '~> 0.6.9'
+gem 'gitlab-linguist', '~> 3.0.0'
 gem 'minitest'
 gem 'activesupport', '> 4.0.0'
 gem 'activemodel', '> 4.0.0'

data/README.md

@@ -30,9 +30,11 @@ $ gem install elasticsearch-git
 class Repository
   include Elasticsearch::Git::Repository
 
-  set_repository_id       project.id
-  repository_for_indexing '/path/to/your/repo'
+  def repository_id
+    project.id
+  end
 
+  repository_for_indexing '/path/to/your/repo'
 end
 
 Repository.__elasticsearch__.create_index! force: true
@@ -41,6 +43,9 @@ repo = Repository.new
 repo.index_commits
 repo.index_blobs
 
+repo.index_commits(from_rev: "1802bafa70d3b1678cfa46a482fd396dd8a4bd40", to_rev: "8d4175e9f4a36065b52fa752c1fd3594c82c0f28")
+repo.index_blobs(from_rev: "1802bafa70d3b1678cfa46a482fd396dd8a4bd40", to_rev: "8d4175e9f4a36065b52fa752c1fd3594c82c0f28")
+
 Repository.search("query", type: 'blob')
 Repository.search("query", type: 'commit')
 
@@ -48,6 +53,257 @@ Repository.search("query", type: 'commit')
 Repository.search("query")
 ```
 
+## Integration with Gitlab
+
+``` ruby
+# app/elastic/repositories_search.rb
+module RepositoriesSearch
+  extend ActiveSupport::Concern
+
+  included do
+    include Elasticsearch::Git::Repository
+
+    def repository_id
+      project.id
+    end
+  end
+end
+
+
+# app/models/repository.rb
+class Repository
+  include RepositoriesSearch
+  #...
+  def project
+    @project ||= Project.find_with_namespace(@path_with_namespace)
+  end
+  #...
+end
+
+Project.last.repository.__elasticsearch__.create_index! force: true
+Project.last.repository.index_commits
+Project.last.repository.index_blobs
+
+Repository.search("some_query")
+# => {blobs: [{}, {}, {}], commits: [{}, {}, {}]}
+
+Repository.search("some_query", type: :blob)
+# => {blobs: [{}, {}, {}], commits: []}
+
+Repository.search("some_query", type: :commit)
+# => {blobs: [], commits: [{}, {}, {}]}
+
+Repository.search("some_query", type: :commit, page: 2, per: 50)
+# => ...
+
+Repository.search("some_query", options: { repository_id: Project.last.id })
+# => {blobs: [{}, {}, {}], commits: [{}, {}, {}]}
+
+Repository.search("some_query", options: { repository_id: current_user.authorized_projects.ids })
+# => {blobs: [{}, {}, {}], commits: [{}, {}, {}]}
+
+Project.last.repository.search("Copyright")[:blobs].first
+=> #<Elasticsearch::Model::Response::Result:0xbb84b3fc
+ @result=
+  {"_index"=>"repository-index-development",
+   "_type"=>"repository",
+   "_id"=>"4328_LICENSE.txt",
+   "_score"=>0.034848917,
+   "_source"=>
+    {"blob"=>
+      {"type"=>"blob",
+       "oid"=>"f99909cd4ecb6f2ad08f8e55aac3a9fcd86a2bd2",
+       "rid"=>4328,
+       "content"=>
+        "Copyright (c) 2014 Andrey Kumanyaev\n\nMIT
+License\n\nPermission is hereby granted, free of charge, to any person
+obtaining\na copy of this software and associated documentation files
+(the\n\"Software\"), to deal in the Software without restriction,
+including\nwithout limitation the rights to use, copy, modify, merge,
+publish,\ndistribute, sublicense, and/or sell copies of the Software,
+and to\npermit persons to whom the Software is furnished to do so,
+subject to\nthe following conditions:\n\nThe above copyright notice and
+this permission notice shall be\nincluded in all copies or substantial
+portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT
+WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE
+AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.\n",
+       "commit_sha"=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"}}}>
+```
+
+## Examples
+
+After integration this gem into [Gitlab](https://github.com/gitlabhq/gitlabhq)
+
+``` ruby
+Repository.search("too")[:commits].first
+=> #<Elasticsearch::Model::Response::Result:0xbb50dfdc
+ @result=
+  {"_index"=>"repository-index-development",
+   "_type"=>"repository",
+   "_id"=>"4328_1802bafa70d3b1678cfa46a482fd396dd8a4bd40",
+   "_score"=>0.15873253,
+   "_source"=>
+    {"commit"=>
+      {"type"=>"commit",
+       "rid"=>4328,
+       "sha"=>"1802bafa70d3b1678cfa46a482fd396dd8a4bd40",
+       "author"=>
+        {"name"=>"Andrey Kumanyaev",
+         "email"=>"me@zzet.org",
+         "time"=>"2014-02-16T02:24:23+04:00"},
+       "committer"=>
+        {"name"=>"Andrey Kumanyaev",
+         "email"=>"me@zzet.org",
+         "time"=>"2014-02-16T02:24:23+04:00"},
+       "message"=>"Save 2. Indexing work. Search too\n"}}}>
+
+
+Project.last.repository.as_indexed_json
+  Project Load (1.7ms)  SELECT "projects".* FROM "projects" ORDER BY "projects"."id" DESC LIMIT 1
+  Namespace Load (4.8ms)  SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" = $1 ORDER BY "namespaces"."id" ASC LIMIT 1  [["id", 3739]]
+  Namespace Load (0.9ms)  SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."path" = 'zzet' LIMIT 1
+  Project Load (0.7ms)  SELECT "projects".* FROM "projects" WHERE "projects"."namespace_id" = 3739 AND "projects"."path" = 'elasticsearch-git' LIMIT 1
+# Long lines are stripped manually
+=> {:blobs=>
+  [[{:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_.gitignore",
+     :rid=>4328,
+     :oid=>"d87d4be66f458acd52878902bbf1391732ad21e1",
+     :content=>
+      "*.gem\n*.rbc\n.bundle\n.config\n.yardoc\nGemfile.lock\nInstalledFiles\n_yardoc\ncoverage\ndoc/\nlib/bundler/man\npkg\nrdoc\nspec/reports\ntest/tmp\ntest/version_tmp\ntmp\n",
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_Gemfile",
+     :rid=>4328,
+     :oid=>"7322405f8f3ee5de24f7a727940ac52543e8954c",
+     :content=>
+      "source 'https://rubygems.org'\n\n# Specify your gem's dependencies in elasticsearch-git.gemspec\ngemspec\n\ngem 'elasticsearch-model', github: 'elasticsearch/elasticsearch-rails'\ngem 'elasticsearc....."
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_LICENSE.txt",
+     :rid=>4328,
+     :oid=>"f99909cd4ecb6f2ad08f8e55aac3a9fcd86a2bd2",
+     :content=>
+      "Copyright (c) 2014 Andrey Kumanyaev\n\nMIT License\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Softw...."
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_README.md",
+     :rid=>4328,
+     :oid=>"8258d574dfc8040a5d003f06c6493e0033527f36",
+     :content=>
+      "# Elasticsearch::Git\n\nAttention: Pre-pre-pre beta code. Not production.\n\n[Elasticsearch](https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-model) integrations for g...."
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_Rakefile",
+     :rid=>4328,
+     :oid=>"29955274e0d42e164337c411ad9144e8ffd7e46e",
+     :content=>"require \"bundler/gem_tasks\"\n",
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_elasticsearch-git.gemspec",
+     :rid=>4328,
+     :oid=>"67762437568dda1bb98ec5eca8be7e4a5c8115a9",
+     :content=>
+      "# coding: utf-8\nlib = File.expand_path('../lib', __FILE__)\n$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\nrequire 'elasticsearch/git/version'\n\nGem::Specification.new do |spec|\n  spec..."
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_elasticsearch/git.rb",
+     :rid=>4328,
+     :oid=>"d3817ec58af1f44dfd18856bf54ef2bf607901a8",
+     :content=>
+      "require \"elasticsearch/git/version\"\nrequire \"elasticsearch/git/model\"\nrequire \"elasticsearch/git/commit\"\n\nmodule Elasticsearch\n  module Git\n    class Test\n      include Elasticsearch::..."
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_git/model.rb",
+     :rid=>4328,
+     :oid=>"3dfbae747f25391779fbe012fe8cc4f38cc4651c",
+     :content=>
+      "require 'active_support/concern'\nrequire 'active_model'\nrequire 'elasticsearch/model'\n\nmodule Elasticsearch\n  module Git\n    module Model\n      extend ActiveSupport::Concern\n\n      include..."
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_git/repository.rb",
+     :rid=>4328,
+     :oid=>"70fe59c8391f6c27adb79c3e45824e6b4cf9566c",
+     :content=>
+      "require 'active_support/concern'\nrequire 'active_model'\nrequire 'elasticsearch'\nrequire 'elasticsearch/model'\nrequire 'rugged'\nrequire 'gitlab_git'\n\nmodule Elasticsearch\n  module Git\n    m..."
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_git/version.rb",
+     :rid=>4328,
+     :oid=>"79e8082b122492464732f1fb43e9f2bdc96ea146",
+     :content=>
+      "module Elasticsearch\n  module Git\n    VERSION = \"0.0.1\"\n  end\nend\n",
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"},
+    {:type=>"blob",
+     :id=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28_test/test_helper.rb",
+     :rid=>4328,
+     :oid=>"6acc0d2b7bf0f286557d3757c1140b41ab57e8f7",
+     :content=>
+      "require \"rubygems\"\nrequire 'bundler/setup'\nrequire 'pry'\n\nBundler.require\n\nrequire 'wrong/adapters/minitest'\n\nPROJECT_ROOT = File.join(Dir.pwd)\n\nWrong.config.color\n\nMinitest.autorun\n..."
+     :commit_sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28"}]],
+ :commits=>
+  [{:type=>"commit",
+    :sha=>"8d4175e9f4a36065b52fa752c1fd3594c82c0f28",
+    :author=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-16 13:50:32 +0400},
+    :committer=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-16 13:50:32 +0400},
+    :message=>"Improve readme\n"},
+   {:type=>"commit",
+    :sha=>"37f1b0710eb7f41254ae0c33db09794a25bbb246",
+    :author=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-16 13:49:25 +0400},
+    :committer=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-16 13:49:25 +0400},
+    :message=>"prepare first test release\n"},
+   {:type=>"commit",
+    :sha=>"1802bafa70d3b1678cfa46a482fd396dd8a4bd40",
+    :author=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-16 02:24:23 +0400},
+    :committer=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-16 02:24:23 +0400},
+    :message=>"Save 2. Indexing work. Search too\n"},
+   {:type=>"commit",
+    :sha=>"3ed383bfbf6cba611d191dbc3590779c0444b7f0",
+    :author=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-16 00:23:10 +0400},
+    :committer=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-16 00:23:10 +0400},
+    :message=>"Save commit\n"},
+   {:type=>"commit",
+    :sha=>"7021addf520a19bdeceef29947c8687965c132ff",
+    :author=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-15 14:28:43 +0400},
+    :committer=>
+     {:name=>"Andrey Kumanyaev",
+      :email=>"me@zzet.org",
+      :time=>2014-02-15 14:28:43 +0400},
+    :message=>"first commit\n"}]}
+```
+
 ## Contributing
 
 1. Fork it ( http://github.com/[my-github-username]/elasticsearch-git/fork )

data/elasticsearch-git.gemspec

@@ -21,7 +21,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'elasticsearch-model'
   spec.add_runtime_dependency 'elasticsearch-api'
   spec.add_runtime_dependency 'rugged'
-  spec.add_runtime_dependency 'gitlab_git'
+  spec.add_runtime_dependency 'charlock_holmes'
+  spec.add_runtime_dependency 'gitlab-linguist'
   spec.add_runtime_dependency 'activemodel'
   spec.add_runtime_dependency 'activesupport'
 end

data/lib/elasticsearch/git.rb

@@ -1,12 +1,9 @@
 require "elasticsearch/git/version"
 require "elasticsearch/git/model"
-require "elasticsearch/git/commit"
+require "elasticsearch/git/repository"
 
 module Elasticsearch
   module Git
-    class Test
-      include Elasticsearch::Git::Model
-    end
   end
 end
 

data/lib/elasticsearch/git/model.rb

@@ -12,7 +12,13 @@ module Elasticsearch
         include ActiveModel::Model
         include Elasticsearch::Model
 
-        index_name [self.name.downcase, 'index', Rails.env.to_s].join('-')
+        env = if defined?(::Rails)
+                ::Rails.env.to_s
+              else
+                "undefined"
+              end
+
+        index_name [self.name.downcase, 'index', env].join('-')
 
         settings \
           index: {

data/lib/elasticsearch/git/repository.rb

@@ -3,7 +3,7 @@ require 'active_model'
 require 'elasticsearch'
 require 'elasticsearch/model'
 require 'rugged'
-require 'gitlab_git'
+require 'linguist'
 
 module Elasticsearch
   module Git
@@ -14,14 +14,16 @@ module Elasticsearch
         include Elasticsearch::Git::Model
 
         mapping do
-          indexes :blobs do
+          indexes :blob do
             indexes :id,          type: :string, index_options: 'offsets', search_analyzer: :human_analyzer,  index_analyzer: :human_analyzer
+            indexes :rid,         type: :string, index: :not_analyzed
             indexes :oid,         type: :string, index_options: 'offsets', search_analyzer: :sha_analyzer,    index_analyzer: :sha_analyzer
             indexes :commit_sha,  type: :string, index_options: 'offsets', search_analyzer: :sha_analyzer,    index_analyzer: :sha_analyzer
             indexes :content,     type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,   index_analyzer: :human_analyzer
           end
-          indexes :commits do
+          indexes :commit do
             indexes :id,          type: :string, index_options: 'offsets', search_analyzer: :human_analyzer,  index_analyzer: :human_analyzer
+            indexes :rid,         type: :string, index: :not_analyzed
             indexes :sha,         type: :string, index_options: 'offsets', search_analyzer: :sha_analyzer,    index_analyzer: :sha_analyzer
             indexes :author do
               indexes :name,      type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,    index_analyzer: :human_analyzer
@@ -51,23 +53,89 @@ module Elasticsearch
         # }
         #
         # For search from blobs use type 'blob'
-        def index_blobs
-          target_sha = repository_for_indexing.head.target
-          repository_for_indexing.index.each do |blob|
+        def index_blobs(from_rev: nil, to_rev: nil)
+
+          if to_rev.present?
+            begin
+              raise unless repository_for_indexing.lookup(to_rev).type == :commit
+            rescue
+              raise ArgumentError, "'to_rev': '#{to_rev}' is a incorrect commit sha."
+            end
+          else
+            to_rev = repository_for_indexing.head.target
+          end
+
+          target_sha = to_rev
+
+          if from_rev.present?
+            begin
+              raise unless repository_for_indexing.lookup(from_rev).type == :commit
+            rescue
+              raise ArgumentError, "'from_rev': '#{from_rev}' is a incorrect commit sha."
+            end
+
+            diff = repository_for_indexing.diff(from_rev, to_rev)
+            diff.deltas.reverse.each do |delta|
+              if delta.status == :deleted
+                b = LiteBlob.new(repository_for_indexing, delta.old_file)
+                delete_from_index_blob(b)
+              else
+                b = LiteBlob.new(repository_for_indexing, delta.new_file)
+                index_blob(b, target_sha)
+              end
+            end
+          else
+            if repository_for_indexing.bare?
+              recurse_blobs_index(repository_for_indexing.lookup(target_sha).tree, target_sha)
+            else
+              repository_for_indexing.index.each do |blob|
+                b = LiteBlob.new(repository_for_indexing, blob)
+                index_blob(b, target_sha)
+              end
+            end
+          end
+        end
+
+        # Indexing bare repository via walking through tree
+        def recurse_blobs_index(tree, target_sha, path = "")
+          tree.each_blob do |blob|
+            blob[:path] = path + blob[:name]
             b = LiteBlob.new(repository_for_indexing, blob)
-            if b.text?
-              client_for_indexing.index \
-                index: "#{self.class.index_name}",
-                type: "blob",
-                id: "#{repository_id}_#{b.path}",
-                body: {
-                  blob: {
-                    oid: b.id,
-                    rid: repository_id,
-                    content: b.data,
-                    commit_sha: target_sha
-                  }
+            index_blob(b, target_sha)
+          end
+
+          tree.each_tree do |nested_tree|
+            recurse_blobs_index(repository_for_indexing.lookup(nested_tree[:oid]), target_sha, "#{path}#{nested_tree[:name]}/")
+          end
+        end
+
+        def index_blob(blob, target_sha)
+          if blob.text?
+            client_for_indexing.index \
+              index: "#{self.class.index_name}",
+              type: "repository",
+              id: "#{repository_id}_#{blob.path}",
+              body: {
+                blob: {
+                  type: "blob",
+                  oid: blob.id,
+                  rid: repository_id,
+                  content: blob.data,
+                  commit_sha: target_sha
                 }
+              }
+          end
+        end
+
+        def delete_from_index_blob(blob)
+          if blob.text?
+            begin
+              client_for_indexing.delete \
+                index: "#{self.class.index_name}",
+                type: "repository",
+                id: "#{repository_id}_#{blob.path}"
+            rescue Elasticsearch::Transport::Transport::Errors::NotFound
+              return true
             end
           end
         end
@@ -94,27 +162,45 @@ module Elasticsearch
         # }
         #
         # For search from commits use type 'commit'
-        def index_commits
-          repository_for_indexing.each_id do |oid|
-            obj = repository_for_indexing.lookup(oid)
-            if obj.type == :commit
-              client_for_indexing.index \
-                index: "#{self.class.index_name}",
-                type: "commit",
-                id: "#{repository_id}_#{obj.oid}",
-                body: {
-                  commit: {
-                    rid: repository_id,
-                    sha: obj.oid,
-                    author: obj.author,
-                    committer: obj.committer,
-                    message: obj.message
-                  }
-                }
+        def index_commits(from_rev: nil, to_rev: nil)
+          if from_rev.present? && to_rev.present?
+            begin
+              raise unless repository_for_indexing.lookup(from_rev).type == :commit
+              raise unless repository_for_indexing.lookup(from_rev).type == :commit
+            rescue
+              raise ArgumentError, "'from_rev': '#{from_rev}' is a incorrect commit sha."
+            end
+
+            repository_for_indexing.walk(from_rev, to_rev).each do |commit|
+              index_commit(commit)
+            end
+          else
+            repository_for_indexing.each_id do |oid|
+              obj = repository_for_indexing.lookup(oid)
+              if obj.type == :commit
+                index_commit(obj)
+              end
             end
           end
         end
 
+        def index_commit(commit)
+          client_for_indexing.index \
+            index: "#{self.class.index_name}",
+            type: "repository",
+            id: "#{repository_id}_#{commit.oid}",
+            body: {
+              commit: {
+                type: "commit",
+                rid: repository_id,
+                sha: commit.oid,
+                author: commit.author,
+                committer: commit.committer,
+                message: commit.message
+              }
+            }
+        end
+
         # Representation of repository as indexed json
         # Attention: It can be very very very huge hash
         def as_indexed_json(options = {})
@@ -129,19 +215,52 @@ module Elasticsearch
           result = []
 
           target_sha = repository_for_indexing.head.target
-          repository_for_indexing.index.each do |blob|
-            b = EasyBlob.new(repository_for_indexing, blob)
+
+          if repository_for_indexing.bare?
+            tree = repository_for_indexing.lookup(target_sha).tree
+            result.push(recurse_blobs_index_hash(tree))
+          else
+            repository_for_indexing.index.each do |blob|
+              b = EasyBlob.new(repository_for_indexing, blob)
+              result.push(
+                {
+                  type: 'blob',
+                  id: "#{target_sha}_#{b.path}",
+                  rid: repository_id,
+                  oid: b.id,
+                  content: b.data,
+                  commit_sha: target_sha
+                }
+              ) if b.text?
+            end
+          end
+
+          result
+        end
+
+        def recurse_blobs_index_hash(tree, path = "")
+          result = []
+
+          tree.each_blob do |blob|
+            blob[:path] = path + blob[:name]
+            b = LiteBlob.new(repository_for_indexing, blob)
             result.push(
               {
-                id: "#{target_sha}_#{b.path}",
+                type: 'blob',
+                id: "#{repository_for_indexing.head.target}_#{path}#{blob[:name]}",
+                rid: repository_id,
                 oid: b.id,
                 content: b.data,
-                commit_sha: target_sha
+                commit_sha: repository_for_indexing.head.target
               }
             ) if b.text?
           end
 
-          result
+          tree.each_tree do |nested_tree|
+            result.push(recurse_blobs_index_hash(repository_for_indexing.lookup(nested_tree[:oid]), "#{nested_tree[:name]}/"))
+          end
+
+          result.flatten
         end
 
         # Lookup all object ids for commit objects
@@ -153,6 +272,7 @@ module Elasticsearch
             if obj.type == :commit
               res.push(
                 {
+                  type: 'commit',
                   sha: obj.oid,
                   author: obj.author,
                   committer: obj.committer,
@@ -165,27 +285,141 @@ module Elasticsearch
           res
         end
 
+        def search(query, type: :all, page: 1, per: 20, options: {})
+          options[:repository_id] = repository_id if options[:repository_id].nil?
+          self.class.search(query, type: type, page: page, per: per, options: options)
+        end
+
         # Repository id used for identity data from different repositories
         # Update this value if need
-        def set_repository_id id
+        def set_repository_id id = nil
           @repository_id = id || path_to_repo
         end
 
+        # For Overwrite
+        def repository_id
+          @repository_id
+        end
+
+        unless defined?(path_to_repo)
+          def path_to_repo
+            if @path_to_repo.blank?
+              raise NotImplementedError, 'Please, define "path_to_repo" method, or set "path_to_repo" via "repository_for_indexing" method'
+            else
+              @path_to_repo
+            end
+          end
+        end
+
         def repository_for_indexing(repo_path = "")
           @path_to_repo ||= repo_path
+          set_repository_id
           Rugged::Repository.new(@path_to_repo)
         end
 
         def client_for_indexing
           @client_for_indexing ||= Elasticsearch::Client.new log: true
         end
+      end
+
+      module ClassMethods
+        def search(query, type: :all, page: 1, per: 20, options: {})
+          results = { blobs: [], commits: []}
+          case type.to_sym
+          when :all
+            results[:blobs] = search_blob(query, page: page, per: per, options: options)
+            results[:commits] = search_commit(query, page: page, per: per, options: options)
+          when :blob
+            results[:blobs] = search_blob(query, page: page, per: per, options: options)
+          when :commit
+            results[:commits] = search_commit(query, page: page, per: per, options: options)
+          end
+
+          results
+        end
+
+        def search_commit(query, page: 1, per: 20, options: {})
+          page ||= 1
+
+          fields = %w(message^10 sha^5 author.name^2 author.email^2 committer.name committer.email).map {|i| "commit.#{i}"}
+
+          query_hash = {
+            query: {
+              filtered: {
+                query: {
+                  multi_match: {
+                    fields: fields,
+                    query: "#{query}",
+                    operator: :and
+                  }
+                },
+              },
+            },
+            size: per,
+            from: per * (page - 1)
+          }
+
+          if query.blank?
+            query_hash[:query][:filtered][:query] = { match_all: {}}
+            query_hash[:track_scores] = true
+          end
+
+          if options[:repository_id]
+            query_hash[:query][:filtered][:filter] ||= { and: [] }
+            query_hash[:query][:filtered][:filter][:and] << {
+              terms: {
+                "commit.rid" => [options[:repository_id]].flatten
+              }
+            }
+          end
+
+          if options[:highlight]
+            query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
+          end
+
+          self.__elasticsearch__.search(query_hash).results
+        end
 
+        def search_blob(query, type: :all, page: 1, per: 20, options: {})
+          page ||= 1
+
+          query_hash = {
+            query: {
+              filtered: {
+                query: {
+                  match: {
+                    'blob.content' => {
+                      query: "#{query}",
+                      operator: :and
+                    }
+                  }
+                }
+              }
+            },
+            size: per,
+            from: per * (page - 1)
+          }
+
+          if options[:repository_id]
+            query_hash[:query][:filtered][:filter] ||= { and: [] }
+            query_hash[:query][:filtered][:filter][:and] << {
+              terms: {
+                "blob.rid" => [options[:repository_id]].flatten
+              }
+            }
+          end
+
+          if options[:highlight]
+            query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
+          end
+
+          self.__elasticsearch__.search(query_hash).results
+        end
       end
     end
 
     class LiteBlob
       include Linguist::BlobHelper
-      include EncodingHelper
 
       attr_accessor :id, :name, :path, :data, :commit_id
 
@@ -195,6 +429,37 @@ module Elasticsearch
         @name = @path.split("/").last
         @data = encode!(repo.lookup(@id).content)
       end
+
+      def encode!(message)
+        return nil unless message.respond_to? :force_encoding
+
+        # if message is utf-8 encoding, just return it
+        message.force_encoding("UTF-8")
+        return message if message.valid_encoding?
+
+        # return message if message type is binary
+        detect = CharlockHolmes::EncodingDetector.detect(message)
+        return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+
+        # encoding message to detect encoding
+        if detect && detect[:encoding]
+          message.force_encoding(detect[:encoding])
+        end
+
+        # encode and clean the bad chars
+        message.replace clean(message)
+      rescue
+        encoding = detect ? detect[:encoding] : "unknown"
+        "--broken encoding: #{encoding}"
+      end
+
+      private
+
+      def clean(message)
+        message.encode("UTF-16BE", :undef => :replace, :invalid => :replace, :replace => "")
+        .encode("UTF-8")
+        .gsub("\0".encode("UTF-8"), "")
+      end
     end
   end
 end

data/lib/elasticsearch/git/version.rb

@@ -1,5 +1,5 @@
 module Elasticsearch
   module Git
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-git
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Andrey Kumanyaev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-16 00:00:00.000000000 Z
+date: 2014-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch-model
@@ -53,7 +53,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: gitlab_git
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gitlab-linguist
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='