gitlab-elasticsearch-git 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a57ef9fc7d82e26ebcd5a4480c08828a1bb1bfb2
-  data.tar.gz: 25631d2855e4077afe625cbfef70bf920af38eeb
+  metadata.gz: 407b4f13d2f80ccce7365e649318b811fcf3d164
+  data.tar.gz: 80651810beb864805f008c8ecc7d51201eb6b4fc
 SHA512:
-  metadata.gz: 50924522805888f3991965ac6369f1ea8222cf8e87e8ae98110dcd49350c4f767ac03c953045b98e687a25412a831143cbaff55934dc83a879718a6dd952a967
-  data.tar.gz: 6b80258d5e1bad79e9f28066fcc8bd32e406d256d2b10c11f0437cf8b699de6e4a8237dba64f5289e86daa8b398aefd07b059f8e4be0ba6e06ebc6dff05fade9
+  metadata.gz: 276867d9624cccaba15a97c0847222fe57958394023d5d7afdb52f7c3e9f315770d5f8dac4393f899c8130d8e545ebdaf128f6064503a981722c0951a00a0c8e
+  data.tar.gz: 6ad0348184aa893e97b379e31e6b2b9dcfea7002ddd8bd80e6b2ec211b734a369ecee4149e73c27e536f3666aa6dfe54d3dd993c6f4d9b1ccdbe78a94c4d2d70

data/CHANGELOG

@@ -1,9 +1,17 @@
+0.0.8
+ - Using Elastic bulk API
+ - Optimisations of index mappings
+ - Performance optimization
+
+0.0.7
+ - Refactoring
+
 0.0.6
-  - Support elasticsearch-model gem version 0.1.8
+ - Support elasticsearch-model gem version 0.1.8
 
 0.0.5
-  - Search by file name on master branch
-  - Migrate from elasticsearch 0.9.x -> 1.x
+ - Search by file name on master branch
+ - Migrate from elasticsearch 0.9.x -> 1.x
 
 0.0.4
-- Stable version
+ - Stable version

data/gitlab-elasticsearch-git.gemspec

@@ -18,11 +18,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency 'elasticsearch-model'
-  spec.add_runtime_dependency 'elasticsearch-api', '~> 1.0.15'
+  spec.add_runtime_dependency 'elasticsearch-model', '~> 1.0'
+  spec.add_runtime_dependency 'elasticsearch-api', '~> 1.0'
   spec.add_runtime_dependency 'rugged', '~> 0.23.3'
-  spec.add_runtime_dependency 'charlock_holmes', '~> 0.7.3'
-  spec.add_runtime_dependency 'github-linguist', '~> 4.7.0'
-  spec.add_runtime_dependency 'activemodel', '~> 4.2.0'
-  spec.add_runtime_dependency 'activesupport', '~> 4.2.0'
+  spec.add_runtime_dependency 'charlock_holmes', '~> 0.7'
+  spec.add_runtime_dependency 'github-linguist', '~> 4.7'
+  spec.add_runtime_dependency 'activemodel', '~> 4.2'
+  spec.add_runtime_dependency 'activesupport', '~> 4.2'
 end

data/lib/elasticsearch/git/model.rb

@@ -27,17 +27,17 @@ module Elasticsearch
               human_analyzer: {
                 type: 'custom',
                 tokenizer: 'human_tokenizer',
-                filter: %w(lowercase asciifolding human_ngrams)
+                filter: %w(lowercase asciifolding)
               },
               path_analyzer: {
                 type: 'custom',
                 tokenizer: 'path_tokenizer',
-                filter: %w(lowercase asciifolding path_ngrams)
+                filter: %w(lowercase asciifolding)
               },
               sha_analyzer: {
                 type: 'custom',
                 tokenizer: 'sha_tokenizer',
-                filter: %w(lowercase asciifolding sha_ngrams)
+                filter: %w(lowercase asciifolding)
               },
               code_analyzer: {
                 type: 'custom',
@@ -64,21 +64,6 @@ module Elasticsearch
               },
             },
             filter: {
-              human_ngrams: {
-                type: "nGram",
-                min_gram: 1,
-                max_gram: 20
-              },
-              sha_ngrams: {
-                type: "edgeNGram",
-                min_gram: 8,
-                max_gram: 40
-              },
-              path_ngrams: {
-                type: "edgeNGram",
-                min_gram: 3,
-                max_gram: 15
-              },
               code_stemmer: {
                 type: "stemmer",
                 name: "minimal_english"

data/lib/elasticsearch/git/repository.rb

@@ -11,6 +11,9 @@ module Elasticsearch
     module Repository
       class CreateIndexException < StandardError; end
 
+      BLOBS_BATCH = 100
+      COMMMITS_BATCH = 500
+
       extend ActiveSupport::Concern
 
       included do
@@ -19,33 +22,33 @@ module Elasticsearch
 
         mapping _timestamp: { enabled: true } do
           indexes :blob do
-            indexes :id,          type: :string, index_options: 'offsets', search_analyzer: :human_analyzer,  analyzer: :human_analyzer
+            indexes :id,          type: :string, index_options: 'offsets', analyzer: :human_analyzer
             indexes :rid,         type: :string, index: :not_analyzed
-            indexes :oid,         type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,   analyzer: :code_analyzer
-            indexes :commit_sha,  type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,   analyzer: :code_analyzer
-            indexes :path,        type: :string, search_analyzer: :path_analyzer,   analyzer: :path_analyzer
-            indexes :content,     type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,   analyzer: :code_analyzer
+            indexes :oid,         type: :string, index_options: 'offsets', analyzer: :code_analyzer
+            indexes :commit_sha,  type: :string, index_options: 'offsets', analyzer: :sha_analyzer
+            indexes :path,        type: :string,                           analyzer: :path_analyzer
+            indexes :content,     type: :string, index_options: 'offsets', analyzer: :code_analyzer
             indexes :language,    type: :string, index: :not_analyzed
           end
 
           indexes :commit do
-            indexes :id,          type: :string, index_options: 'offsets', search_analyzer: :human_analyzer,   analyzer: :human_analyzer
+            indexes :id,          type: :string, index_options: 'offsets', analyzer: :human_analyzer
             indexes :rid,         type: :string, index: :not_analyzed
-            indexes :sha,         type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,    analyzer: :code_analyzer
+            indexes :sha,         type: :string, index_options: 'offsets', analyzer: :sha_analyzer
 
             indexes :author do
-              indexes :name,      type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,    analyzer: :code_analyzer
-              indexes :email,     type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,    analyzer: :code_analyzer
-              indexes :time,      type: :date
+              indexes :name,      type: :string, index_options: 'offsets', analyzer: :code_analyzer
+              indexes :email,     type: :string, index_options: 'offsets', analyzer: :code_analyzer
+              indexes :time,      type: :date, format: :basic_date_time_no_millis
             end
 
             indexes :commiter do
-              indexes :name,      type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,    analyzer: :code_analyzer
-              indexes :email,     type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,    analyzer: :code_analyzer
-              indexes :time,      type: :date
+              indexes :name,      type: :string, index_options: 'offsets', analyzer: :code_analyzer
+              indexes :email,     type: :string, index_options: 'offsets', analyzer: :code_analyzer
+              indexes :time,      type: :date, format: :basic_date_time_no_millis
             end
 
-            indexes :message,     type: :string, index_options: 'offsets', search_analyzer: :code_analyzer,    analyzer: :code_analyzer
+            indexes :message,     type: :string, index_options: 'offsets', analyzer: :code_analyzer
           end
         end
 
@@ -68,52 +71,52 @@ module Elasticsearch
 
           diff = repository_for_indexing.diff(from, to)
 
-          diff.deltas.reverse.each_with_index do |delta, step|
-            if delta.status == :deleted
-              next if delta.old_file[:mode].to_s(8) == "160000"
-              b = LiteBlob.new(repository_for_indexing, delta.old_file)
-              delete_from_index_blob(b)
-            else
-              next if delta.new_file[:mode].to_s(8) == "160000"
-              b = LiteBlob.new(repository_for_indexing, delta.new_file)
-              index_blob(b, to)
+          diff.deltas.reverse.each_slice(BLOBS_BATCH) do |slice|
+            bulk_operations = slice.map do |delta|
+              if delta.status == :deleted
+                next if delta.old_file[:mode].to_s(8) == "160000"
+                b = LiteBlob.new(repository_for_indexing, delta.old_file)
+                delete_blob(b)
+              else
+                next if delta.new_file[:mode].to_s(8) == "160000"
+                b = LiteBlob.new(repository_for_indexing, delta.new_file)
+                index_blob(b, to)
+              end
             end
 
-            # Run GC every 100 blobs
-            ObjectSpace.garbage_collect if step % 100 == 0
+            perform_bulk bulk_operations
           end
         end
 
+        def perform_bulk(bulk_operations)
+          client_for_indexing.bulk body: bulk_operations.compact
+        end
+
+
+        def delete_blob(blob)
+          return unless blob.text?
+          { delete: { _index: "#{self.class.index_name}", _type: self.class.name.underscore, _id: "#{repository_id}_#{blob.path}" } }
+        end
+
+
         def index_blob(blob, target_sha)
-          if can_index_blob?(blob)
-            tries = 0
-
-            begin
-              client_for_indexing.index \
-                index: "#{self.class.index_name}",
-                type: self.class.name.underscore,
-                id: "#{repository_id}_#{blob.path}",
-                body: {
-                  blob: {
-                    type: "blob",
-                    oid: blob.id,
-                    rid: repository_id,
-                    content: blob.data,
-                    commit_sha: target_sha,
-                    path: blob.path,
-                    language: blob.language ? blob.language.name : "Text"
-                  }
+          return unless can_index_blob?(blob)
+          {
+            index:  {
+              _index: "#{self.class.index_name}", _type: self.class.name.underscore, _id: "#{repository_id}_#{blob.path}",
+              data: {
+                blob: {
+                  type: "blob",
+                  oid: blob.id,
+                  rid: repository_id,
+                  content: blob.data,
+                  commit_sha: target_sha,
+                  path: blob.path,
+                  language: blob.language ? blob.language.name : "Text"
                 }
-            rescue Exception => ex
-              if tries < 2
-                tries += 1
-                sleep 1
-                retry
-              else
-                raise CreateIndexException, "Can't index #{repository_id}_#{blob.path}. Reason: #{ex.message}"
-              end
-            end
-          end
+              }
+            }
+          }
         end
 
         # Index text-like files which size less 1.mb
@@ -121,21 +124,6 @@ module Elasticsearch
           blob.text? && (blob.size && blob.size.to_i < 1048576)
         end
 
-        def delete_from_index_blob(blob)
-          if blob.text?
-            begin
-              client_for_indexing.delete \
-                index: "#{self.class.index_name}",
-                type: "repository",
-                id: "#{repository_id}_#{blob.path}"
-            rescue Elasticsearch::Transport::Transport::Errors::NotFound
-              return true
-            rescue Exception => ex
-              raise CreateIndexException, "Error with removing file from index #{repository_id}_#{blob.path}. Reason: #{ex.message}"
-            end
-          end
-        end
-
         # Indexing all commits in repository
         #
         # All data stored in global index
@@ -160,50 +148,65 @@ module Elasticsearch
         # For search from commits use type 'commit'
         def index_commits(from_rev: nil, to_rev: repository_for_indexing.last_commit.oid)
           from, to = parse_revs(from_rev, to_rev)
-          range = [from, to].reject(&:nil?).join('..')
+          range = [from, to].compact.join('..')
           out, err, status = Open3.capture3("git log #{range} --format=\"%H\"", chdir: repository_for_indexing.path)
 
           if status.success? && err.blank?
+            queue = Queue.new
+
+            workers = (0...3).map do
+              Thread.new do
+                while bulk_operations = queue.pop
+                  perform_bulk bulk_operations
+                end
+              end
+            end
+
             #TODO use rugged walker!!!
             commit_oids = out.split("\n")
 
-            commit_oids.each_with_index do |commit, step|
-              index_commit(repository_for_indexing.lookup(commit))
-              ObjectSpace.garbage_collect if step % 100 == 0
+            commit_oids.each_slice(COMMMITS_BATCH) do |batch|
+              bulk_operations = batch.map do |commit|
+                index_commit(repository_for_indexing.lookup(commit))
+              end
+
+              # perform_bulk bulk_operations
+              queue << bulk_operations
+            end
+
+            while queue.num_waiting < workers.count
+              sleep 0.1
             end
-            return commit_oids.count
-          end
 
-          0
+            # Kill off each thread now that they're idle and exit
+            workers.each(&:exit)
+          end
         end
 
         def index_commit(commit)
-          tries = 0
-
-          begin
-            client_for_indexing.index \
-              index: "#{self.class.index_name}",
-              type: self.class.name.underscore,
-              id: "#{repository_id}_#{commit.oid}",
-              body: {
+          {
+            index:  {
+              _index: "#{self.class.index_name}", _type: self.class.name.underscore, _id: "#{repository_id}_#{commit.oid}",
+              data: {
                 commit: {
                   type: "commit",
                   rid: repository_id,
                   sha: commit.oid,
-                  author: commit.author,
-                  committer: commit.committer,
+                  author: {
+                    name: commit.author[:name],
+                    email: commit.author[:email],
+                    time: commit.author[:time].strftime('%Y%m%dT%H%M%S%z'),
+                  },
+                  committer: {
+                    name: commit.committer[:name],
+                    email: commit.committer[:email],
+                    time: commit.committer[:time].strftime('%Y%m%dT%H%M%S%z'),
+                  },
                   message: encode!(commit.message)
                 }
               }
-          rescue Exception => ex
-            if tries < 2
-              tries += 1
-              sleep 1
-              retry
-            else
-              raise CreateIndexException, "Can't index #{repository_id}_#{commit.oid}. Reason: #{ex.message}"
-            end
-          end
+            }
+          }
         end
 
         def parse_revs(from_rev, to_rev)
@@ -350,7 +353,7 @@ module Elasticsearch
         end
 
         def client_for_indexing
-          @client_for_indexing ||= Elasticsearch::Client.new log: true
+          @client_for_indexing ||= Elasticsearch::Client.new retry_on_failure: 5
         end
 
         def self.search(query, type: :all, page: 1, per: 20, options: {})

data/lib/elasticsearch/git/version.rb

@@ -1,5 +1,5 @@
 module Elasticsearch
   module Git
-    VERSION = "0.0.7"
+    VERSION = "0.0.8"
   end
 end

data/test/test_helper.rb

@@ -4,7 +4,7 @@ require 'pry'
 require 'elasticsearch/git'
 
 SUPPORT_PATH = File.join(File.expand_path(File.dirname(__FILE__)), '../support')
-TEST_REPO_PATH = File.join(SUPPORT_PATH, 'testme.git')
+TEST_REPO_PATH = File.join(SUPPORT_PATH, 'gitlab-test.git')
 
 require_relative 'support/seed_helper'
 require_relative 'support/repository'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-elasticsearch-git
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Andrey Kumanyaev
@@ -10,36 +10,36 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-17 00:00:00.000000000 Z
+date: 2016-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch-model
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.15
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.15
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rugged
   requirement: !ruby/object:Gem::Requirement
@@ -60,56 +60,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.3
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.3
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: github-linguist
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.7.0
+        version: '4.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.7.0
+        version: '4.7'
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '4.2'
 description: Elasticsearch integrations for indexing git repositories.
 email:
 - me@zzet.org
@@ -157,7 +157,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Elasticsearch integrations for git repositories.