elasticsearch-git 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d48d7fd7bd1dc6d71ce6c24024a746cb75d0b60e
-  data.tar.gz: e4297f9c88cecef626495998bbb3838ee8829f61
+  metadata.gz: 55b0b79e7075658c15389e9fdd70fd50b0eb5af7
+  data.tar.gz: 46d05c192da782afe7303f0c85fc398ee13389ea
 SHA512:
-  metadata.gz: 0260393250e6bb3fdb4a757b363b5b48d5bd3851bbb923b53170ca88a8b40b7f6c7319a4f0ff6c4b14e4fa8ede81c70b3b6eb343617c624f8a9e961e366d039e
-  data.tar.gz: be0c4572531338cd2cfa595ce523b9bd16d65bc4139942319862f77fb00215bd85c63bdda4261f6da2b0142a373b7fb862a8f9426f47cff7a5b91481f29670ce
+  metadata.gz: 63d24b71254c1dce1e27c2f0a574aa49299eb451fd145524a6bc2e37ea06ab5f16b19cafd361a3d4a9f28149d88491dd0820bb52189cb6dd29d53f5d67ef4b70
+  data.tar.gz: 674709e3e2f99fca958a3854aebdebf3e8280126ea896ff07fb76be1176e9710b634f786aeb3a6010a3b361f1523cae07d0d360537d9a5d0697ae910c5914b33

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+support

data/CHANGELOG

@@ -0,0 +1,8 @@
+0.0.5
+
+Search by file name on master branch
+Migrate from elasticsearch 0.9.x -> 1.x
+
+0.0.4
+
+Stable version

data/Gemfile

@@ -5,13 +5,13 @@ gemspec
 
 gem 'elasticsearch-model', github: 'elasticsearch/elasticsearch-rails'
 gem 'elasticsearch-api', git: 'git://github.com/elasticsearch/elasticsearch-ruby.git'
-gem 'rugged', git: 'git://github.com/libgit2/rugged.git', branch: 'development', submodules: true
+gem 'rugged'
 gem 'bundler'
 gem 'rake'
 gem 'pry'
 gem 'charlock_holmes', '~> 0.6.9'
 gem 'gitlab-linguist', '~> 3.0.0'
 gem 'minitest'
-gem 'activesupport', '> 4.0.0'
-gem 'activemodel', '> 4.0.0'
+#gem 'activesupport', '> 4.0.0'
+#gem 'activemodel', '> 4.0.0'
 gem 'wrong'

data/README.md

@@ -1,13 +1,21 @@
 # Elasticsearch::Git
 
-[Elasticsearch](https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-model) integrations for git repositories
+[Elasticsearch](https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-model) integrations for git repositories.
+
+## NOTE
+
+Now indexing text-like documents mo nore 1 mb size
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ``` ruby
-gem 'elasticsearch-git'
+gem 'elasticsearch-git', '~> 0.0.5'
+
+# or
+
+gem 'elasticsearch-git', github: 'zzet/elasticsearch-git', ref: 'last_ref'
 ```
 
 And then execute:
@@ -53,6 +61,8 @@ Repository.search("query")
 
 ## Integration with Gitlab
 
+[Sample (In Undev Gitlab fork)](https://github.com/zzet/gitlabhq/blob/master/app/elastic/repositories_search.rb)
+
 ``` ruby
 # app/elastic/repositories_search.rb
 module RepositoriesSearch
@@ -61,6 +71,8 @@ module RepositoriesSearch
   included do
     include Elasticsearch::Git::Repository
 
+    self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
+
     def repository_id
       project.id
     end

data/Rakefile

@@ -1 +1,10 @@
 require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/lib/*_test.rb']
+end
+
+task :default => :test

data/elasticsearch-git.gemspec

@@ -6,8 +6,8 @@ require 'elasticsearch/git/version'
 Gem::Specification.new do |spec|
   spec.name          = "elasticsearch-git"
   spec.version       = Elasticsearch::Git::VERSION
-  spec.authors       = ["Andrey Kumanyaev"]
-  spec.email         = ["me@zzet.org"]
+  spec.authors       = ["Andrey Kumanyaev", "Evgeniy Sokovikov"]
+  spec.email         = ["me@zzet.org", "skv-headless@yandex.ru"]
   spec.summary       = %q{Elasticsearch integrations for git repositories.}
   spec.description   = %q{Elasticsearch integrations for indexing git repositories.}
   spec.homepage      = "https://github.com/zzet/elasticsearch-git"
@@ -20,9 +20,9 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'elasticsearch-model'
   spec.add_runtime_dependency 'elasticsearch-api', '> 0.4.0'
-  spec.add_runtime_dependency 'rugged', '~> 0.19.0'
+  spec.add_runtime_dependency 'rugged', '~> 0.21.0'
   spec.add_runtime_dependency 'charlock_holmes', '~> 0.6.9'
   spec.add_runtime_dependency 'gitlab-linguist', '> 2.9.0'
-  spec.add_runtime_dependency 'activemodel', '~> 4.0.0'
-  spec.add_runtime_dependency 'activesupport', '~> 4.0.0'
+  spec.add_runtime_dependency 'activemodel', '~> 4.1.0'
+  spec.add_runtime_dependency 'activesupport', '~> 4.1.0'
 end

data/lib/elasticsearch/git/lite_blob.rb

@@ -14,7 +14,7 @@ module Elasticsearch
 
         blob  = repo.lookup(@id)
 
-        @mode = '%06o' % raw_blob_hash[:filemode]
+        @mode = raw_blob_hash[:mode].to_s(8)
         @size = blob.size
         @path = encode!(raw_blob_hash[:path])
         @name = @path.split('/').last

data/lib/elasticsearch/git/model.rb

@@ -29,6 +29,11 @@ module Elasticsearch
                 tokenizer: 'human_tokenizer',
                 filter: %w(lowercase asciifolding human_ngrams)
               },
+              path_analyzer: {
+                type: 'custom',
+                tokenizer: 'path_tokenizer',
+                filter: %w(lowercase asciifolding path_ngrams)
+              },
               sha_analyzer: {
                 type: 'custom',
                 tokenizer: 'sha_tokenizer',
@@ -37,7 +42,7 @@ module Elasticsearch
               code_analyzer: {
                 type: 'custom',
                 tokenizer: 'standard',
-                filter: %w(lowercase asciifolding)
+                filter: %w(lowercase asciifolding code_stemmer)
               }
             },
             tokenizer: {
@@ -52,7 +57,11 @@ module Elasticsearch
                 min_gram: 1,
                 max_gram: 20,
                 token_chars: %w(letter digit)
-              }
+              },
+              path_tokenizer: {
+                type: 'path_hierarchy',
+                reverse: true
+              },
             },
             filter: {
               human_ngrams: {
@@ -64,6 +73,15 @@ module Elasticsearch
                 type: "edgeNGram",
                 min_gram: 8,
                 max_gram: 40
+              },
+              path_ngrams: {
+                type: "edgeNGram",
+                min_gram: 3,
+                max_gram: 15
+              },
+              code_stemmer: {
+                type: "stemmer",
+                name: "minimal_english"
               }
             }
           }

data/lib/elasticsearch/git/repository.rb

@@ -21,7 +21,7 @@ module Elasticsearch
             indexes :rid,         type: :string, index: :not_analyzed
             indexes :oid,         type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,   index_analyzer: :code_analyzer
             indexes :commit_sha,  type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,   index_analyzer: :code_analyzer
-            indexes :path,        type: :string, index_options: 'offsets', search_analyzer: :human_analyzer,  index_analyzer: :human_analyzer
+            indexes :path,        type: :string, search_analyzer: :path_analyzer,   index_analyzer: :path_analyzer
             indexes :content,     type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,   index_analyzer: :code_analyzer
             indexes :language,    type: :string, index: :not_analyzed
           end
@@ -61,73 +61,24 @@ module Elasticsearch
         # }
         #
         # For search from blobs use type 'blob'
-        def index_blobs(from_rev: nil, to_rev: nil)
+        def index_blobs(from_rev: nil, to_rev: repository_for_indexing.last_commit.oid)
+          from, to = parse_revs(from_rev, to_rev)
 
-          if to_rev.present?
-            begin
-              if to_rev != "0000000000000000000000000000000000000000"
-                raise unless repository_for_indexing.lookup(to_rev).type == :commit
-              end
-            rescue
-              raise ArgumentError, "'to_rev': '#{to_rev}' is a incorrect commit sha."
-            end
-          else
-            to_rev = repository_for_indexing.head.target
-          end
-
-          target_sha = to_rev
-
-          if from_rev.present?
-            begin
-              if from_rev != "0000000000000000000000000000000000000000"
-                raise unless repository_for_indexing.lookup(from_rev).type == :commit
-              end
-            rescue
-              raise ArgumentError, "'from_rev': '#{from_rev}' is a incorrect commit sha."
-            end
-
-            diff = repository_for_indexing.diff(from_rev, to_rev)
-
-            diff.deltas.reverse.each_with_index do |delta, step|
-              if delta.status == :deleted
-                b = LiteBlob.new(repository_for_indexing, delta.old_file)
-                delete_from_index_blob(b)
-              else
-                b = LiteBlob.new(repository_for_indexing, delta.new_file)
-                index_blob(b, target_sha)
-              end
+          diff = repository_for_indexing.diff(from, to)
 
-              # Run GC every 100 blobs
-              ObjectSpace.garbage_collect if step % 100 == 0
-            end
-          else
-            if repository_for_indexing.bare?
-              recurse_blobs_index(repository_for_indexing.lookup(target_sha).tree, target_sha)
+          diff.deltas.reverse.each_with_index do |delta, step|
+            if delta.status == :deleted
+              next if delta.old_file[:mode].to_s(8) == "160000"
+              b = LiteBlob.new(repository_for_indexing, delta.old_file)
+              delete_from_index_blob(b)
             else
-              repository_for_indexing.index.each_with_index do |blob, step|
-                b = LiteBlob.new(repository_for_indexing, blob)
-                index_blob(b, target_sha)
-
-                # Run GC every 100 blobs
-                ObjectSpace.garbage_collect if step % 100 == 0
-              end
+              next if delta.new_file[:mode].to_s(8) == "160000"
+              b = LiteBlob.new(repository_for_indexing, delta.new_file)
+              index_blob(b, to)
             end
-          end
-        end
 
-        # Indexing bare repository via walking through tree
-        def recurse_blobs_index(tree, target_sha, path = "")
-          tree.each_blob do |blob|
-            blob[:path] = path + blob[:name]
-            b = LiteBlob.new(repository_for_indexing, blob)
-            index_blob(b, target_sha)
-          end
-
-          # Run GC every recurse step
-          ObjectSpace.garbage_collect
-
-          tree.each_tree do |nested_tree|
-            recurse_blobs_index(repository_for_indexing.lookup(nested_tree[:oid]), target_sha, "#{path}#{nested_tree[:name]}/")
+            # Run GC every 100 blobs
+            ObjectSpace.garbage_collect if step % 100 == 0
           end
         end
 
@@ -205,46 +156,23 @@ module Elasticsearch
         # }
         #
         # For search from commits use type 'commit'
-        def index_commits(from_rev: nil, to_rev: nil)
-          to_rev = repository_for_indexing.head.target unless to_rev.present?
-
-          if to_rev != "0000000000000000000000000000000000000000"
-            # If to_rev correct
-            begin
-              raise unless repository_for_indexing.lookup(to_rev).type == :commit
-            rescue
-              raise ArgumentError, "'to_rev': '#{to_rev}' is a incorrect commit sha."
-            end
+        def index_commits(from_rev: nil, to_rev: repository_for_indexing.last_commit.oid)
+          from, to = parse_revs(from_rev, to_rev)
+          range = [from, to].reject(&:nil?).join('..')
+          out, err, status = Open3.capture3("git log #{range} --format=\"%H\"", chdir: repository_for_indexing.path)
 
-            begin
-              if from_rev.present? && from_rev != "0000000000000000000000000000000000000000"
-                raise unless repository_for_indexing.lookup(from_rev).type == :commit
-              end
-            rescue
-              raise ArgumentError, "'from_rev': '#{from_rev}' is a incorrect commit sha."
-            end
+          if status.success? && err.blank?
+            #TODO use rugged walker!!!
+            commit_oids = out.split("\n")
 
-            # If pushed new branch no need reindex all repository
-            # Find merge_base and reindex diff
-            if from_rev == "0000000000000000000000000000000000000000" && to_rev != repository_for_indexing.head.target
-              from_rev = repository_for_indexing.merge_base(to_rev, repository_for_indexing.head.target)
-            end
-
-            walker = Rugged::Walker.new(repository_for_indexing)
-            walker.push(to_rev)
-
-            if from_rev.present? && from_rev != "0000000000000000000000000000000000000000"
-              walker.hide(from_rev)
-            end
-
-            commits = walker.map { |c| c.oid }
-            walker.reset
-
-            commits.each_with_index do |commit, step|
+            commit_oids.each_with_index do |commit, step|
               index_commit(repository_for_indexing.lookup(commit))
               ObjectSpace.garbage_collect if step % 100 == 0
             end
+            return commit_oids.count
           end
+
+          0
         end
 
         def index_commit(commit)
@@ -276,6 +204,24 @@ module Elasticsearch
           end
         end
 
+        def parse_revs(from_rev, to_rev)
+          from = if index_new_branch?(from_rev)
+                   if to_rev == repository_for_indexing.last_commit.oid
+                     nil
+                   else
+                     merge_base(to_rev)
+                   end
+                 else
+                   from_rev
+                 end
+
+          return from, to_rev
+        end
+
+        def index_new_branch?(from)
+          from == '0000000000000000000000000000000000000000'
+        end
+
         # Representation of repository as indexed json
         # Attention: It can be very very very huge hash
         def as_indexed_json(options = {})
@@ -289,7 +235,7 @@ module Elasticsearch
         def index_blobs_array
           result = []
 
-          target_sha = repository_for_indexing.head.target
+          target_sha = repository_for_indexing.head.target.oid
 
           if repository_for_indexing.bare?
             tree = repository_for_indexing.lookup(target_sha).tree
@@ -322,11 +268,11 @@ module Elasticsearch
             result.push(
               {
                 type: 'blob',
-                id: "#{repository_for_indexing.head.target}_#{path}#{blob[:name]}",
+                id: "#{repository_for_indexing.head.target.oid}_#{path}#{blob[:name]}",
                 rid: repository_id,
                 oid: b.id,
                 content: b.data,
-                commit_sha: repository_for_indexing.head.target
+                commit_sha: repository_for_indexing.head.target.oid
               }
             ) if b.text?
           end
@@ -376,6 +322,11 @@ module Elasticsearch
           @repository_id
         end
 
+        # For Overwrite
+        def self.repositories_count
+          10
+        end
+
         unless defined?(path_to_repo)
           def path_to_repo
             if @path_to_repo.blank?
@@ -416,6 +367,13 @@ module Elasticsearch
         def logger
           @logger ||= Logger.new(STDOUT)
         end
+
+        private
+
+        def merge_base(to_rev)
+          head_sha = repository_for_indexing.last_commit.oid
+          repository_for_indexing.merge_base(to_rev, head_sha)
+        end
       end
 
       module ClassMethods
@@ -440,7 +398,8 @@ module Elasticsearch
               commitRepositoryFaset: {
                 terms: {
                   field: "commit.rid",
-                  all_term: true
+                  all_terms: true,
+                  size: repositories_count
                 }
               }
             },
@@ -462,6 +421,19 @@ module Elasticsearch
             }
           end
 
+          if options[:highlight]
+            es_fields = fields.map { |field| field.split('^').first }.inject({}) do |memo, field|
+              memo[field.to_sym] = {}
+              memo
+            end
+
+            query_hash[:highlight] = {
+                pre_tags: ["gitlabelasticsearch→"],
+                post_tags: ["←gitlabelasticsearch"],
+                fields: es_fields
+            }
+          end
+
           options[:order] = :default if options[:order].blank?
           order = case options[:order].to_sym
                   when :recently_indexed
@@ -474,14 +446,10 @@ module Elasticsearch
 
           query_hash[:sort] = order.blank? ? [:_score] : [order, :_score]
 
-          if options[:highlight]
-            #query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
-          end
-
           res = self.__elasticsearch__.search(query_hash)
           {
             results: res.results,
-            total_count: res.total_count,
+            total_count: res.size,
             repositories: res.response["facets"]["commitRepositoryFaset"]["terms"]
           }
         end
@@ -496,7 +464,7 @@ module Elasticsearch
                   match: {
                     'blob.content' => {
                       query: "#{query}",
-                      operator: :or
+                      operator: :and
                     }
                   }
                 }
@@ -506,13 +474,15 @@ module Elasticsearch
               languageFacet: {
                 terms: {
                   field: :language,
-                  all_term: true
+                  all_terms: true,
+                  size: 20
                 }
               },
               blobRepositoryFaset: {
                 terms: {
                   field: :rid,
-                  all_term: true
+                  all_terms: true,
+                  size: repositories_count
                 }
               }
             },
@@ -552,8 +522,8 @@ module Elasticsearch
 
           if options[:highlight]
             query_hash[:highlight] = {
-              pre_tags: [""],
-              post_tags: [""],
+              pre_tags: ["gitlabelasticsearch→"],
+              post_tags: ["←gitlabelasticsearch"],
               fields: {
                 "blob.content" => {},
                 "type" => "fvh",
@@ -566,11 +536,31 @@ module Elasticsearch
 
           {
             results: res.results,
-            total_count: res.total_count,
+            total_count: res.size,
             languages: res.response["facets"]["languageFacet"]["terms"],
             repositories: res.response["facets"]["blobRepositoryFaset"]["terms"]
           }
         end
+
+        def search_file_names(query, page: 1, per: 20, options: {})
+          query_hash = {
+              fields: ['blob.path'],
+              query: {
+                  fuzzy: {
+                      "repository.blob.path" => { value: query }
+                  },
+              },
+              filter: {
+                  term: {
+                      "repository.blob.rid" => [options[:repository_id]].flatten
+                  }
+              },
+              size: per,
+              from: per * (page - 1)
+          }
+
+          self.__elasticsearch__.search(query_hash)
+        end
       end
     end
   end

data/lib/elasticsearch/git/version.rb

@@ -1,5 +1,5 @@
 module Elasticsearch
   module Git
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

data/test/lib/repository_test.rb

@@ -0,0 +1,43 @@
+require 'test_helper'
+
+
+class RepositoryTest < TestCase
+  def setup
+    @repository = Repository.new
+    @repository.repository_for_indexing(TEST_REPO_PATH)
+    Repository.__elasticsearch__.create_index! force: true
+  end
+
+  def test_index_commits
+    commit_count = @repository.index_commits
+    assert { commit_count == RepoInfo::COMMIT_COUNT }
+  end
+
+  def test_index_commits_after_first_push
+    commit_count = @repository.index_commits(
+        from_rev: "0000000000000000000000000000000000000000",
+        to_rev: @repository.repository_for_indexing.head.target.oid)
+
+    assert { commit_count == RepoInfo::COMMIT_COUNT }
+  end
+
+  #TODO write better assertions
+  def test_index_all_blobs_from_head
+    blob_count = @repository.index_blobs
+    Repository.__elasticsearch__.refresh_index!
+
+    result = @repository.search('def project_name_regex')
+    assert { result[:blobs][:total_count]  == 1 }
+  end
+
+  #TODO write better assertions
+  def test_index_blobs_after_first_push
+    commit_count = @repository.index_blobs(
+        from_rev: "0000000000000000000000000000000000000000",
+        to_rev: @repository.repository_for_indexing.head.target.oid)
+    Repository.__elasticsearch__.refresh_index!
+
+    result = @repository.search('def project_name_regex')
+    assert { result[:blobs][:total_count]  == 1 }
+  end
+end

data/test/test_helper.rb

@@ -1,6 +1,14 @@
 require "rubygems"
 require 'bundler/setup'
 require 'pry'
+require 'elasticsearch/git'
+
+SUPPORT_PATH = File.join(File.expand_path(File.dirname(__FILE__)), '../support')
+TEST_REPO_PATH = File.join(SUPPORT_PATH, 'testme.git')
+
+require_relative 'support/seed_helper'
+require_relative 'support/repository'
+require_relative 'support/repo_info'
 
 Bundler.require
 

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-git
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Andrey Kumanyaev
+- Evgeniy Sokovikov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-10 00:00:00.000000000 Z
+date: 2014-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch-model
@@ -44,14 +45,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.19.0
+        version: 0.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.19.0
+        version: 0.21.0
 - !ruby/object:Gem::Dependency
   name: charlock_holmes
   requirement: !ruby/object:Gem::Requirement
@@ -86,36 +87,38 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 4.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 4.1.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 4.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 4.1.0
 description: Elasticsearch integrations for indexing git repositories.
 email:
 - me@zzet.org
+- skv-headless@yandex.ru
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -127,6 +130,9 @@ files:
 - lib/elasticsearch/git/model.rb
 - lib/elasticsearch/git/repository.rb
 - lib/elasticsearch/git/version.rb
+- test/lib/repository_test.rb
+- test/support/repository.rb
+- test/support/seed_helper.rb
 - test/test_helper.rb
 homepage: https://github.com/zzet/elasticsearch-git
 licenses:
@@ -148,9 +154,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Elasticsearch integrations for git repositories.
 test_files:
+- test/lib/repository_test.rb
+- test/support/repository.rb
+- test/support/seed_helper.rb
 - test/test_helper.rb