odba 1.2.1 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea8626e332e7e9650d0221804e2cc6b6767e0fdedbabb507faaa2ad104b4199e
-  data.tar.gz: '06848ad3f7def177deec9c62385e1b63c56fcbf3e6d161f3a73e60a70a0c92a9'
+  metadata.gz: e275634934d76a06600ef71830600a6a6cd1e03c3f59069aa94f998f25d9a973
+  data.tar.gz: cf0ae8760286cafa23e4d1c1c37e304aa8a548daa8438c00e58c1f2492691743
 SHA512:
-  metadata.gz: 2d55bb2c6b46d92bd89da28489544b8fc68e8c4fd16a94f3deefef5970376fa2a607c9c9c811506c6f66147f02fcc9981148baf89acd54ae528c51c2b0a7dda6
-  data.tar.gz: 2337e2c7910c89653bdb6ddb90150aa38794096f1c5110bd94ae0f4bd54f84006d6ae8cf4bb08bc467e1d0c78d5d28cbfd5e7788611cf75bb83739432d8f3a75
+  metadata.gz: 9d7005efee025dddcf75ae2e80b50266482a322916b59a7bd25fa339149a5df762e87248d4351e80acf819550ab2738d2f3cb3d04046c759bddfb48418221b69
+  data.tar.gz: cd31af260988120f4ab9c4b73126896f8c0ce43d68b9b4da24ffd1e968fc8fd0c3564fec466ba5f1d2ecd36c5605afe2b3b124bcd5ef2eb990b7535bc29e3751

data/History.md

@@ -1,3 +1,29 @@
+## 1.2.2 / 31.08.2026
+
+Two processes on one database handed out the same odba_id. Whichever wrote
+last overwrote the other's row in `object`, and every reference to the lost
+object then resolved to a foreign one - an Array where a domain object
+belonged, or the reverse. The referring instance variable stays correct and
+points at the right number; a different object simply sits under it, so
+searching the application for the offending assignment finds nothing.
+
+* `Storage#next_id` takes the id from a Postgres sequence, `odba_id_seq`,
+  which `#setup` creates. It used to be `@next_id += 1` under a mutex, with
+  @next_id seeded once per process from the highest odba_id in the table -
+  sound for one process, wrong for every deployment running a web worker and
+  an import job against the same database. Measured with two processes side
+  by side, both answered `[61935067, 61935068, 61935069]`. Stores without the
+  sequence keep the old behaviour.
+* The sequence starts at `MAX(odba_id)` plus `ID_SEQUENCE_GAP`, not at 1: a
+  plain `CREATE SEQUENCE` would re-issue ids that already exist, and the gap
+  covers ids that processes still on the old counter hold but have not
+  written yet.
+* `Cache#next_id` no longer swallows `OdbaDuplicateIdError`. The guard was
+  there all along - a peer raises it when the id is taken and the method
+  retries - but `rescue` without a class caught it too, so the retry could
+  never run. Only `DRb::DRbError` is caught now, which is what the line was
+  for: an unreachable peer must not stop the allocation.
+
 ## 1.2.1 / 21.08.2026
 
 No library changes: lib/ is identical to 1.2.0. This release only ships a test

data/lib/odba/cache.rb

@@ -441,8 +441,14 @@ module ODBA
       end
       @peers.each do |peer|
         peer.reserve_next_id id
-      rescue
-        DRb::DRbError
+      rescue DRb::DRbError
+        # A peer we cannot reach must not stop the allocation. Note the
+        # explicit class: a bare rescue here also swallowed the
+        # OdbaDuplicateIdError a peer raises when the id is already taken,
+        # so the retry below could never run and both processes kept the
+        # same id. Whichever wrote last overwrote the other's row in
+        # `object`, and every reference to it then resolved to a foreign
+        # object.
       end
       id
     rescue OdbaDuplicateIdError

data/lib/odba/storage.rb

@@ -9,8 +9,17 @@ require "dbi"
 module ODBA
   class Storage # :nodoc: all
     include Singleton
-    attr_writer :dbi
+
+    # Not attr_writer: whether the store has an id sequence is memoized, and
+    # that answer belongs to the connection it was asked on.
+    def dbi=(dbi)
+      @id_sequence = nil
+      @dbi = dbi
+    end
     BULK_FETCH_STEP = 2500
+    # Distance between the highest id in use and the first the sequence
+    # hands out; see the odba_id_seq entry in TABLES.
+    ID_SEQUENCE_GAP = 100_000
     TABLES = [
       # in table 'object', the isolated dumps of all objects are stored
       ["object", <<~SQL],
@@ -37,12 +46,31 @@ module ODBA
         CREATE INDEX IF NOT EXISTS target_id_index ON object_connection(target_id);
       SQL
       # helper table 'collection'
-      ["collection", <<~SQL]
+      ["collection", <<~SQL],
         CREATE TABLE IF NOT EXISTS collection (
           odba_id integer NOT NULL, key text, value text,
           PRIMARY KEY(odba_id, key)
         );
       SQL
+      # The odba_id comes from this sequence, so that several processes on
+      # one database cannot hand out the same one. See #next_id.
+      #
+      # The start value is computed and must be: a plain CREATE SEQUENCE
+      # starts at 1 and would re-issue ids that already exist. The gap on
+      # top of MAX(odba_id) covers ids that processes still running with the
+      # old in-memory counter hold but have not written yet. Skipped numbers
+      # cost nothing - the odba_id is a surrogate key and carries no meaning.
+      ["odba_id_seq", <<~SQL]
+        DO $$
+        BEGIN
+          IF NOT EXISTS (SELECT 1 FROM pg_class
+                         WHERE relkind = 'S' AND relname = 'odba_id_seq') THEN
+            EXECUTE format('CREATE SEQUENCE odba_id_seq START WITH %s',
+                           (SELECT COALESCE(MAX(odba_id), 0) + #{ID_SEQUENCE_GAP}
+                            FROM object));
+          END IF;
+        END $$;
+      SQL
     ]
     def initialize
       @id_mutex = Mutex.new
@@ -425,13 +453,43 @@ module ODBA
       end
     end
 
+    # The id is allocated by the database, not by a counter in this process.
+    #
+    # It used to be `@next_id += 1` under this mutex, with @next_id seeded
+    # once per process from the highest odba_id in the table. That is sound
+    # for a single process and wrong for every deployment that runs more
+    # than one - web workers and import jobs on the same database each kept
+    # their own counter and handed out the same numbers, so one silently
+    # overwrote the other's row in `object`.
+    #
+    # Falls back to the old behaviour where no sequence exists, so a store
+    # that was never through #setup keeps working; #setup creates it.
     def next_id
-      @id_mutex.synchronize do
-        ensure_next_id_set
-        @next_id += 1
+      if id_sequence?
+        dbi.select_one("SELECT nextval('odba_id_seq')").first.to_i.tap { |id|
+          # max_id and reserve_next_id read @next_id, so keep it in step.
+          # Never backwards: a peer may already stand higher.
+          @id_mutex.synchronize {
+            @next_id = id if @next_id.nil? || @next_id < id
+          }
+        }
+      else
+        @id_mutex.synchronize do
+          ensure_next_id_set
+          @next_id += 1
+        end
       end
     end
 
+    def id_sequence?
+      return @id_sequence unless @id_sequence.nil?
+      @id_sequence = dbi.select_one(
+        "SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = 'odba_id_seq'"
+      ) ? true : false
+    rescue
+      @id_sequence = false
+    end
+
     def update_max_id(id)
       @id_mutex.synchronize do
         @next_id = id

data/lib/odba/version.rb

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
 class Odba
-  VERSION = '1.2.1'
+  VERSION = '1.2.2'
 end

data/test/test_cache.rb

@@ -19,6 +19,46 @@ module ODBA
     public :load_object
   end
 
+  # The peer conflict must reach the retry. Until 1.2.2 the loop read
+  # `peer.reserve_next_id id rescue DRb::DRbError` - a rescue without a
+  # class, which caught the OdbaDuplicateIdError a peer raises when the id
+  # is taken. Both processes then kept the same id and one overwrote the
+  # other's row in `object`.
+  class TestCacheNextId < Test::Unit::TestCase
+    include FlexMock::TestCase
+
+    def setup
+      @cache = ODBA::Cache.instance
+      @cache.instance_variable_set(:@file_lock, false)
+      @storage = flexmock("storage")
+      ODBA.storage = @storage
+    end
+
+    def test_a_peer_conflict_leads_to_a_new_id
+      @storage.should_receive(:next_id).and_return(100, 101)
+      seen = []
+      peer = flexmock("peer")
+      peer.should_receive(:reserve_next_id).and_return { |id|
+        seen << id
+        raise ODBA::OdbaDuplicateIdError, "taken" if seen.size == 1
+        true
+      }
+      @cache.instance_variable_set(:@peers, [peer])
+      assert_equal(101, @cache.next_id)
+      assert_equal([100, 101], seen)
+    end
+
+    # A peer we cannot reach must not stop the allocation - that is what the
+    # line was for, and it stays.
+    def test_an_unreachable_peer_does_not_stop_the_allocation
+      @storage.should_receive(:next_id).and_return(100)
+      peer = flexmock("peer")
+      peer.should_receive(:reserve_next_id).and_raise(DRb::DRbError, "gone")
+      @cache.instance_variable_set(:@peers, [peer])
+      assert_equal(100, @cache.next_id)
+    end
+  end
+
   class TestCache < Test::Unit::TestCase
     include FlexMock::TestCase
     class ODBAContainerInCache

data/test/test_storage.rb

@@ -15,6 +15,7 @@ module ODBA
       @storage = ODBA::Storage.instance
        @dbi = flexmock("DBI")
        @storage.dbi = @dbi
+       @storage.instance_variable_set(:@id_sequence, nil)
     end
 
     def teardown
@@ -106,12 +107,40 @@ module ODBA
        @storage.create_index("index_name")
     end
 
+    # Without a sequence the old counter still runs, so a store that was
+    # never through #setup keeps working.
     def test_next_id
+      @dbi.should_receive(:select_one).and_return(nil)
       @storage.next_id = 1
        assert_equal(2, @storage.next_id)
        assert_equal(3, @storage.next_id)
     end
 
+    # The point of 1.2.2: the id comes from the database, so two processes
+    # on one store cannot be handed the same one.
+    def test_next_id__from_the_sequence
+      @dbi.should_receive(:select_one)
+        .with("SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = 'odba_id_seq'")
+        .once.and_return([1])
+      @dbi.should_receive(:select_one)
+        .with("SELECT nextval('odba_id_seq')").twice.and_return([4711], [4712])
+      assert_equal(4711, @storage.next_id)
+      assert_equal(4712, @storage.next_id)
+    end
+
+    # max_id and reserve_next_id read @next_id, so it has to follow the
+    # sequence - and never move backwards, a peer may stand higher already.
+    def test_next_id__keeps_the_local_counter_in_step
+      @dbi.should_receive(:select_one)
+        .with(/pg_class/).and_return([1])
+      @dbi.should_receive(:select_one).with(/nextval/).and_return([90], [5])
+      @storage.next_id = 10
+      @storage.next_id
+      assert_equal(90, @storage.instance_variable_get(:@next_id))
+      @storage.next_id
+      assert_equal(90, @storage.instance_variable_get(:@next_id))
+    end
+
     def test_store__1
       dbi = flexmock("dbi")
        @storage.dbi = dbi

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: odba
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: ruby
 authors:
 - Masaomi Hatakeyama, Zeno R.R. Davatz