grumlin 0.21.1 → 0.22.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95963424bb38728bb21c2d3746a93be60daf303b2c2897b24d99684fd1fb7bff
-  data.tar.gz: 809b4b589b2bc5fd8687f0f63a6f73f17aab82060990b3a5942164c2ac92ef83
+  metadata.gz: 23a0ce8dacaf7c4add38674353e6aa69e51ebd9865ed83c3112f62ad96a5d213
+  data.tar.gz: 775c7c6009e6969d9a2b060772dc5d6caf9846267050509a5aeccf1bd5050e0c
 SHA512:
-  metadata.gz: b095f3958ebf4b70577e607a0b10e27fc073055fe08d6fe55787c71fd104e77cf78cdcd0a54d06e7881c9183cab479ef6e92e6117a1c1b418945d1ebb417655c
-  data.tar.gz: 0477a8ce8ccfc5ea1cc3b95b7b48501b05387207918478adc4eecb8e286e8495910b6bd718e1d5c00ab63e61e02ca6e89212913ec35d8df49b267cb75281e9eb
+  metadata.gz: 6b9a820912679a67ca1be3545130ee3affcf89bea1bd9085a47f18d4b0c2ed6583ebfe5e7c52c339d93efe672d7ec7d5e604a68681581a4c57f6b21d8dce9e21
+  data.tar.gz: a5d371995de61123f954361143f98427af0109831c189bb1ad60cdf722109c72749872478096aa60de22bc7d3139825fdf1ed07c8cce2330eca51cb5865fc6f9

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    grumlin (0.21.1)
+    grumlin (0.22.2)
       async-pool (~> 0.3)
       async-websocket (~> 0.19)
       oj (~> 3.13)
@@ -21,14 +21,14 @@ GEM
       console (~> 1.10)
       nio4r (~> 2.3)
       timers (~> 4.1)
-    async-http (0.57.0)
+    async-http (0.58.0)
       async (>= 1.25)
       async-io (>= 1.28)
       async-pool (>= 0.2)
       protocol-http (~> 0.23.1)
       protocol-http1 (~> 0.14.0)
       protocol-http2 (~> 0.14.0)
-      traces (~> 0.4.0)
+      traces (>= 0.4.0)
     async-io (1.33.0)
       async
     async-pool (0.3.10)
@@ -81,7 +81,7 @@ GEM
     parser (3.1.2.0)
       ast (~> 2.4.1)
     protocol-hpack (1.4.2)
-    protocol-http (0.23.1)
+    protocol-http (0.23.4)
     protocol-http1 (0.14.4)
       protocol-http (~> 0.22)
     protocol-http2 (0.14.2)
@@ -157,7 +157,7 @@ GEM
     thor (1.2.1)
     tilt (2.0.10)
     timers (4.3.3)
-    traces (0.4.1)
+    traces (0.6.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.1.0)

data/README.md

@@ -64,8 +64,7 @@ explicitly specify the provider you use.
 #### Provider features
 
 Every provider is described by a set of features. In the future `Grumlin` may decide to disable or enable 
-some parts of it's functionality to comply provider's supported features. Currently there is no difference
-in behaviour when working with different providers.
+some parts of it's functionality to comply provider's supported features.
 
 To check current providers supported features use
 
@@ -73,6 +72,12 @@ To check current providers supported features use
 Grumlin.features
 ```
 
+Current differences between providers:
+
+| Feature      | TinkerGraph                                                                                                                                               |AWS Neptune|
+|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
+| Transactions | Transaction semantic is ignoroed, data is always writen, `tx.rollback` does nothing, an info is printed every time transactions are used with TinkerGraph |Full support
+
 ### Traversing graphs
 
 **Warning**: Not all steps and expressions defined in the reference documentation are supported.
@@ -240,21 +245,29 @@ Each `return_mode` is mapped to a particular termination step:
 - `:traversal` - do not execute the query and return the traversal as is
 
 `Grumlin::Repository` also provides a set of generic CRUD operations:
-- `add_vertex(label, id = nil, **properties)`
-- `add_edge(label, id = nil, from:, to:, **properties)`
-- `drop_vertex(id)`
-- `drop_edge(id = nil, from: nil, to: nil, label: nil)`
+- `add_vertex(label, id = nil, start: g, **properties)`
+- `add_edge(label, id = nil, from:, to:, start: g, **properties)`
+- `drop_vertex(id, start: g)`
+- `drop_edge(id = nil, from: nil, to: nil, label: nil, start: g)`
+- `drop_in_batches(traversal, batch_size: 10_000)`
 
 and a few methods that emulate upserts:
-- `upsert_vertex(label, id, create_properties: {}, update_properties: {}, on_failure: :retry, **params)` 
-- `upsert_edge(label, from:, to:, create_properties: {}, update_properties: {}, on_failure: :retry, **params)`
-- `upsert_edges(edges, batch_size: 100, on_failure: :retry, **params)`
-- `upsert_vertices(edges, batch_size: 100, on_failure: :retry, **params)`
+- `upsert_vertex(label, id, create_properties: {}, update_properties: {}, on_failure: :retry, start: g, **params)` 
+- `upsert_edge(label, from:, to:, create_properties: {}, update_properties: {}, on_failure: :retry, start: g, **params)`
+- `upsert_edges(edges, batch_size: 100, on_failure: :retry, start: g, **params)`
+- `upsert_vertices(edges, batch_size: 100, on_failure: :retry, start: g, **params)`
 
 All of them support 3 different modes for error handling: `:retry`, `:ignore` and `:raise`. Retry mode is implemented
 with [retryable](https://github.com/nfedyashev/retryable). **params will be merged to the default config for upserts 
 and passed to `Retryable.retryable`. In case if you want to modify retryable behaviour you are to do so.
 
+If you want to use these methods inside a transaction simply pass your `gtx` as `start` parameter:
+```ruby
+g.tx do |gtx|
+  add_vertex(:vertex, start: gtx)
+end
+```
+
 If you don't want to define you own repository, simply use
 
 `Grumlin::Repository.new` returns an instance of an anonymous class extending `Grumlin::Repository`. 
@@ -282,6 +295,24 @@ it may be useful for debugging. Note that one needs to call a termination step m
 
 method will return profiling data of the results.
 
+#### Transactions
+
+Since 0.22.0 `Grumlin` supports transactions when working with providers that supports them:
+```ruby
+# Using Transaction directly
+tx = g.tx
+gtx = tx.begin
+gtx.addV(:vertex).iterate
+tx.commit # or tx.rollback
+
+# Using with a block
+g.tx do |gtx|
+  gtx.addV(:vertex).iterate
+  # raise Grumlin::Rollback to manually rollback
+  # any other exception will also rollback the transaction and will be reraised 
+end # commits automatically
+```
+
 #### IRB
 
 An example of how to start an IRB session with support for executing gremlin queries:

data/lib/grumlin/action.rb

@@ -16,7 +16,7 @@ module Grumlin
     end
 
     def configuration_step?
-      CONFIGURATION_STEPS.include?(@name)
+      CONFIGURATION_STEPS.include?(@name) || name.to_sym == :tx
     end
 
     def start_step?

data/lib/grumlin/client.rb

@@ -30,12 +30,6 @@ module Grumlin
         @count += 1
       end
 
-      def finalize_tx(action, session_id)
-        @client.finalize_tx(action, session_id)
-      ensure
-        @count += 1
-      end
-
       def viable?
         !closed?
       end
@@ -104,15 +98,15 @@ module Grumlin
       raise NotConnectedError unless connected?
 
       request = to_query(bytecode, session_id: session_id)
-      submit_request(request)
-    end
 
-    def finalize_tx(action, session_id)
-      raise NotConnectedError unless connected?
-      raise ArgumentError, "session_id cannot be nil" if session_id.nil?
-
-      request = finalize_tx_query(action, session_id)
-      submit_request(request)
+      channel = @request_dispatcher.add_request(request)
+      begin
+        @transport.write(request)
+        channel.dequeue.flat_map { |item| Typing.cast(item) }
+      rescue Async::Stop, Async::TimeoutError
+        close(check_requests: false)
+        raise
+      end
     end
 
     def inspect
@@ -125,18 +119,6 @@ module Grumlin
 
     private
 
-    def submit_request(request)
-      channel = @request_dispatcher.add_request(request)
-      @transport.write(request)
-
-      begin
-        channel.dequeue.flat_map { |item| Typing.cast(item) }
-      rescue Async::Stop, Async::TimeoutError
-        close(check_requests: false)
-        raise
-      end
-    end
-
     # This might be overridden in successors
     def build_transport
       Transport.new(@url, parent: @parent, **@client_options)
@@ -157,21 +139,5 @@ module Grumlin
         }.compact
       }
     end
-
-    def finalize_tx_query(action, session_id)
-      {
-        requestId: SecureRandom.uuid,
-        op: :bytecode,
-        processor: session_id ? :session : :traversal,
-        args: {
-          gremlin: {
-            :@type => "g:Bytecode",
-            :@value => { source: [[:tx, action]] }
-          },
-          aliases: { g: :g },
-          session: session_id
-        }.compact
-      }
-    end
   end
 end

data/lib/grumlin/dummy_transaction.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Grumlin
+  class DummyTransaction < Transaction
+    attr_reader :uuid
+
+    include Console
+
+    def initialize(traversal_start_class, pool: Grumlin.default_pool) # rubocop:disable Lint/MissingSuper, Lint/UnusedMethodArgument
+      @traversal_start_class = traversal_start_class
+
+      logger.info(self) do
+        "#{Grumlin.config.provider} does not support transactions. commit and rollback are ignored, data will be saved"
+      end
+    end
+
+    def commit
+      nil
+    end
+
+    def rollback
+      nil
+    end
+  end
+end

data/lib/grumlin/repository/instance_methods.rb

@@ -2,7 +2,7 @@
 
 module Grumlin
   module Repository
-    module InstanceMethods
+    module InstanceMethods # rubocop:disable Metrics/ModuleLength
       include Grumlin::Expressions
 
       extend Forwardable
@@ -22,51 +22,73 @@ module Grumlin
         self.class.shortcuts
       end
 
-      def drop_vertex(id)
-        g.V(id).drop.iterate
+      def drop_vertex(id, start: g)
+        start.V(id).drop.iterate
       end
 
-      def drop_edge(id = nil, from: nil, to: nil, label: nil) # rubocop:disable Metrics/AbcSize
+      def drop_in_batches(traversal, batch_size: 10_000) # rubocop:disable Metrics/AbcSize
+        total_count = traversal.count.next
+
+        batches = (total_count / batch_size) + 1
+
+        Console.logger.info(self) do
+          "drop_in_batches: total_count: #{total_count}, batch_size: #{batch_size}, batches: #{batches}"
+        end
+
+        batches.times do |batch|
+          Console.logger.info(self) { "drop_in_batches: deleting batch #{batch + 1}/#{batches}..." }
+          traversal.limit(batch_size).drop.iterate
+          Console.logger.info(self) { "drop_in_batches: batch #{batch + 1}/#{batches} deleted" }
+        end
+
+        return if traversal.count.next.zero?
+
+        drop_in_batches(traversal, batch_size: batch_size)
+
+        Console.logger.info(self) { "drop_in_batches: finished." }
+      end
+
+      def drop_edge(id = nil, from: nil, to: nil, label: nil, start: g) # rubocop:disable Metrics/AbcSize
         raise ArgumentError, "either id or from:, to: and label: must be passed" if [id, from, to, label].all?(&:nil?)
-        return g.E(id).drop.iterate unless id.nil?
+        return start.E(id).drop.iterate unless id.nil?
 
         raise ArgumentError, "from:, to: and label: must be passed" if [from, to, label].any?(&:nil?)
 
-        g.V(from).outE(label).where(__.inV.hasId(to)).limit(1).drop.iterate
+        start.V(from).outE(label).where(__.inV.hasId(to)).limit(1).drop.iterate
       end
 
-      def add_vertex(label, id = nil, **properties)
+      def add_vertex(label, id = nil, start: g, **properties)
         id ||= properties[T.id]
         properties = except(properties, T.id)
 
-        t = g.addV(label)
+        t = start.addV(label)
         t = t.props(T.id => id) unless id.nil?
         t.props(**properties).next
       end
 
-      def add_edge(label, id = nil, from:, to:, **properties)
+      def add_edge(label, id = nil, from:, to:, start: g, **properties)
         id ||= properties[T.id]
         properties = except(properties, T.label)
         properties[T.id] = id
 
-        g.addE(label).from(__.V(from)).to(__.V(to)).props(**properties).next
+        start.addE(label).from(__.V(from)).to(__.V(to)).props(**properties).next
       end
 
-      def upsert_vertex(label, id, create_properties: {}, update_properties: {}, on_failure: :retry, **params)
+      def upsert_vertex(label, id, create_properties: {}, update_properties: {}, on_failure: :retry, start: g, **params) # rubocop:disable Metrics/ParameterLists
         with_upsert_error_handling(on_failure, params) do
           create_properties, update_properties = cleanup_properties(create_properties, update_properties)
 
-          g.upsertV(label, id, create_properties, update_properties).id.next
+          start.upsertV(label, id, create_properties, update_properties).id.next
         end
       end
 
       # vertices:
       # [["label", "id", {create: :properties}, {update: properties}]]
       # params can override Retryable config from UPSERT_RETRY_PARAMS
-      def upsert_vertices(vertices, batch_size: 100, on_failure: :retry, **params)
+      def upsert_vertices(vertices, batch_size: 100, on_failure: :retry, start: g, **params)
         vertices.each_slice(batch_size) do |slice|
           with_upsert_error_handling(on_failure, params) do
-            slice.reduce(g) do |t, (label, id, create_properties, update_properties)|
+            slice.reduce(start) do |t, (label, id, create_properties, update_properties)|
               create_properties, update_properties = cleanup_properties(create_properties, update_properties)
 
               t.upsertV(label, id, create_properties, update_properties)
@@ -77,20 +99,21 @@ module Grumlin
 
       # Only from and to are used to find the existing edge, if one wants to assign an id to a created edge,
       # it must be passed as T.id in create_properties.
-      def upsert_edge(label, from:, to:, create_properties: {}, update_properties: {}, on_failure: :retry, **params) # rubocop:disable Metrics/ParameterLists
+      def upsert_edge(label, from:, to:, create_properties: {}, update_properties: {}, # rubocop:disable Metrics/ParameterLists
+                      on_failure: :retry, start: g, **params)
         with_upsert_error_handling(on_failure, params) do
           create_properties, update_properties = cleanup_properties(create_properties, update_properties, T.label)
-          g.upsertE(label, from, to, create_properties, update_properties).id.next
+          start.upsertE(label, from, to, create_properties, update_properties).id.next
         end
       end
 
       # edges:
       # [["label", "from", "to", {create: :properties}, {update: properties}]]
       # params can override Retryable config from UPSERT_RETRY_PARAMS
-      def upsert_edges(edges, batch_size: 100, on_failure: :retry, **params)
+      def upsert_edges(edges, batch_size: 100, on_failure: :retry, start: g, **params)
         edges.each_slice(batch_size) do |slice|
           with_upsert_error_handling(on_failure, params) do
-            slice.reduce(g) do |t, (label, from, to, create_properties, update_properties)|
+            slice.reduce(start) do |t, (label, from, to, create_properties, update_properties)|
               create_properties, update_properties = cleanup_properties(create_properties, update_properties, T.label)
 
               t.upsertE(label, from, to, create_properties, update_properties)

data/lib/grumlin/request_error_factory.rb

@@ -18,11 +18,19 @@ module Grumlin
     # {"detailedMessage":"",
     #  "requestId":"UUID",
     #  "code":"ConcurrentModificationException"}
-    # Currencly we simply search for substings to identify the exact error
+    # Currently we simply search for substrings to identify the exact error
     # TODO: parse json and use `code` instead
+
     VERTEX_ALREADY_EXISTS = "Vertex with id already exists:"
     EDGE_ALREADY_EXISTS = "Edge with id already exists:"
+
     CONCURRENT_VERTEX_INSERT_FAILED = "Failed to complete Insert operation for a Vertex due to conflicting concurrent"
+
+    CONCURRENT_VERTEX_PROPERTY_INSERT_FAILED =
+      "Failed to complete Insert operation for a VertexProperty due to conflicting concurrent"
+    CONCURRENT_EDGE_PROPERTY_INSERT_FAILED =
+      "Failed to complete Insert operation for a EdgeProperty due to conflicting concurrent"
+
     CONCURRENT_EDGE_INSERT_FAILED = "Failed to complete Insert operation for an Edge due to conflicting concurrent"
     CONCURRENCT_MODIFICATION_FAILED = "Failed to complete operation due to conflicting concurrent"
 
@@ -49,9 +57,15 @@ module Grumlin
         return EdgeAlreadyExistsError if status[:message]&.include?(EDGE_ALREADY_EXISTS)
       end
 
-      def concurrent_modification_error(status)
+      def concurrent_modification_error(status) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         return ConcurrentVertexInsertFailedError if status[:message]&.include?(CONCURRENT_VERTEX_INSERT_FAILED)
+        if status[:message]&.include?(CONCURRENT_VERTEX_PROPERTY_INSERT_FAILED)
+          return ConcurrentVertexPropertyInsertFailedError
+        end
         return ConcurrentEdgeInsertFailedError if status[:message]&.include?(CONCURRENT_EDGE_INSERT_FAILED)
+        if status[:message]&.include?(CONCURRENT_EDGE_PROPERTY_INSERT_FAILED)
+          return ConcurrentEdgePropertyInsertFailedError
+        end
         return ConcurrentModificationError if status[:message]&.include?(CONCURRENCT_MODIFICATION_FAILED)
       end
     end

data/lib/grumlin/steps.rb

@@ -32,7 +32,9 @@ module Grumlin
     end
 
     def add(name, args: [], params: {})
-      return add_configuration_step(name, args: args, params: params) if CONFIGURATION_STEPS.include?(name)
+      if CONFIGURATION_STEPS.include?(name) || name.to_sym == :tx
+        return add_configuration_step(name, args: args, params: params)
+      end
 
       StepData.new(name, args: cast_arguments(args), params: params).tap do |step|
         @steps << step

data/lib/grumlin/steps_serializers/bytecode.rb

@@ -10,17 +10,19 @@ module Grumlin
 
       def serialize
         steps = ShortcutsApplyer.call(@steps)
-        no_return = @params[:no_return] || false
-
-        {
-          step: (steps.steps + (no_return ? [NONE_STEP] : [])).map { |s| serialize_step(s) }
-        }.tap do |v|
-          v.merge!(source: steps.configuration_steps.map { |s| serialize_step(s) }) if steps.configuration_steps.any?
+        no_return = @params.fetch(:no_return, false)
+        {}.tap do |result|
+          result[:step] = serialize_steps(steps.steps + (no_return ? [NONE_STEP] : [])) if steps.steps.any?
+          result[:source] = serialize_steps(steps.configuration_steps) if steps.configuration_steps.any?
         end
       end
 
       private
 
+      def serialize_steps(steps)
+        steps.map { |s| serialize_step(s) }
+      end
+
       def serialize_step(step)
         [step.name].tap do |result|
           step.args.each do |arg|

data/lib/grumlin/transaction.rb

@@ -2,49 +2,37 @@
 
 module Grumlin
   class Transaction
-    attr_reader :uuid
+    attr_reader :session_id
 
     include Console
 
+    COMMIT = Grumlin::Repository.new.g.step(:tx, :commit).bytecode
+    ROLLBACK = Grumlin::Repository.new.g.step(:tx, :rollback).bytecode
+
     def initialize(traversal_start_class, pool: Grumlin.default_pool)
       @traversal_start_class = traversal_start_class
       @pool = pool
 
-      if supported?
-        @uuid = SecureRandom.uuid
-        return
-      end
-
-      logger.info(self) do
-        "#{Grumlin.config.provider} does not support transactions. commit and rollback are ignored, data will be saved"
-      end
-    end
-
-    def supported?
-      Grumlin.features.supports_transactions?
+      @session_id = SecureRandom.uuid
     end
 
     def begin
-      @traversal_start_class.new(session_id: @uuid)
+      @traversal_start_class.new(session_id: @session_id)
     end
 
     def commit
-      return unless supported?
-
-      finalize(:commit)
+      finalize(COMMIT)
     end
 
     def rollback
-      return unless supported?
-
-      finalize(:rollback)
+      finalize(ROLLBACK)
     end
 
     private
 
     def finalize(action)
       @pool.acquire do |client|
-        client.finalize_tx(action, @uuid)
+        client.write(action, session_id: @session_id)
       end
     end
   end

data/lib/grumlin/traversal_start.rb

@@ -5,12 +5,24 @@ module Grumlin
     include WithExtension
 
     class TraversalError < Grumlin::Error; end
-    class AlreadyBoundToTransationError < TraversalError; end
+    class AlreadyBoundToTransactionError < TraversalError; end
 
     def tx
-      raise AlreadyBoundToTransationError if @session_id
+      raise AlreadyBoundToTransactionError if @session_id
 
-      Transaction.new(self.class, pool: @pool)
+      transaction = tx_class.new(self.class, pool: @pool)
+      return transaction unless block_given?
+
+      begin
+        yield transaction.begin
+      rescue Grumlin::Rollback
+        transaction.rollback
+      rescue StandardError
+        transaction.rollback
+        raise
+      else
+        transaction.commit
+      end
     end
 
     def to_s(*)
@@ -20,5 +32,11 @@ module Grumlin
     def inspect
       self.class.inspect
     end
+
+    private
+
+    def tx_class
+      Grumlin.features.supports_transactions? ? Transaction : DummyTransaction
+    end
   end
 end

data/lib/grumlin/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Grumlin
-  VERSION = "0.21.1"
+  VERSION = "0.22.2"
 end

data/lib/grumlin.rb

@@ -33,6 +33,9 @@ loader.do_not_eager_load(test_helpers)
 module Grumlin
   class Error < StandardError; end
 
+  class TransactionError < Error; end
+  class Rollback < TransactionError; end
+
   class UnknownError < Error; end
 
   class ConnectionError < Error; end
@@ -104,6 +107,9 @@ module Grumlin
   class ConcurrentVertexInsertFailedError < ConcurrentInsertFailedError; end
   class ConcurrentEdgeInsertFailedError < ConcurrentInsertFailedError; end
 
+  class ConcurrentVertexPropertyInsertFailedError < ConcurrentInsertFailedError; end
+  class ConcurrentEdgePropertyInsertFailedError < ConcurrentInsertFailedError; end
+
   class ServerSerializationError < ServerSideError; end
 
   class ServerTimeoutError < ServerSideError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grumlin
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: 0.22.2
 platform: ruby
 authors:
 - Gleb Sinyavskiy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-11 00:00:00.000000000 Z
+date: 2022-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async-pool
@@ -117,6 +117,7 @@ files:
 - lib/grumlin/benchmark/repository.rb
 - lib/grumlin/client.rb
 - lib/grumlin/config.rb
+- lib/grumlin/dummy_transaction.rb
 - lib/grumlin/edge.rb
 - lib/grumlin/expressions/cardinality.rb
 - lib/grumlin/expressions/column.rb