odba 1.2.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 384e9358b0cfe5735935d1b1d2e43706b01adaceb9a0c2cce51a357a2a8321eb
-  data.tar.gz: 79568f015a42389ee5abe8f159f6baefc0e300ac0c6d692b1856ab7c3613d08b
+  metadata.gz: e275634934d76a06600ef71830600a6a6cd1e03c3f59069aa94f998f25d9a973
+  data.tar.gz: cf0ae8760286cafa23e4d1c1c37e304aa8a548daa8438c00e58c1f2492691743
 SHA512:
-  metadata.gz: a2a5cac7871f9d46476e7f45edc3c2afc31dfc2517cfa67b5fdd97e740f331c8b39b018ce2c3928046215d5fcdc11e4dbe2e9b5b70bf6231c33e0690b6bc364f
-  data.tar.gz: 967b6811d2f8833145a9ee90ffe6c5b84c896ef23ee663e616b2a9f25efff33930cde98ea4d35b3d279ee6a74feed785badb249d5ca32cd1ba6eeb989fc794a9
+  metadata.gz: 9d7005efee025dddcf75ae2e80b50266482a322916b59a7bd25fa339149a5df762e87248d4351e80acf819550ab2738d2f3cb3d04046c759bddfb48418221b69
+  data.tar.gz: cd31af260988120f4ab9c4b73126896f8c0ce43d68b9b4da24ffd1e968fc8fd0c3564fec466ba5f1d2ecd36c5605afe2b3b124bcd5ef2eb990b7535bc29e3751

data/.github/workflows/devenv.yml

@@ -7,24 +7,28 @@ on:
 jobs:
   tests:
     strategy:
+      # without this a failure on one OS cancels the other, hiding its result
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        # macos-latest dropped: it costs ~8 minutes per run for a library that
+        # is deployed on Linux, and its Nix/devenv setup is the fragile part -
+        # not odba itself. Re-add it here if that ever becomes worth the wait.
+        os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4
-    - uses: cachix/install-nix-action@v26
-    - uses: cachix/cachix-action@v14
+    - uses: cachix/install-nix-action@v31
+    - uses: cachix/cachix-action@v16
       with:
         name: devenv
     - name: Install devenv.sh
       run: nix profile install nixpkgs#devenv
 
-    - name: Build the devenv shell and run any pre-commit hooks
+    # devenv test builds the shell, runs any pre-commit hooks, brings the
+    # postgres service up and then runs enterTest, which waits for the server
+    # and runs test/example.rb against it. A separate step re-running
+    # example.rb duplicated that work and raced initdb a second time without
+    # waiting, which is what kept macOS red.
+    - name: Build the devenv shell and run test/example.rb against a real postgres
       run: devenv test
-
-    - name: Run the example.rb which uses a real database
-      shell: devenv shell bash -- -e {0}
-      run: |
-        devenv up --detach
-        devenv shell bundle exec ruby test/example.rb

data/.github/workflows/ruby.yml

@@ -25,9 +25,8 @@ jobs:
       matrix:
         os: [ ubuntu]
         ruby: [3.2, 3.3, 3.4]
-    continue-on-error: ${{  matrix.ruby.to_s == '3.3' || matrix.ruby.to_s == '3.4' }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.gitignore

@@ -13,3 +13,7 @@ devenv.local.nix
 
 # pre-commit
 .pre-commit-config.yaml
+
+# built gems - a committed .gem ends up in spec.files (git ls-files) and
+# makes the gemspec invalid: "odba-x.y.z contains itself"
+*.gem

data/History.md

@@ -1,3 +1,49 @@
+## 1.2.2 / 31.08.2026
+
+Two processes on one database handed out the same odba_id. Whichever wrote
+last overwrote the other's row in `object`, and every reference to the lost
+object then resolved to a foreign one - an Array where a domain object
+belonged, or the reverse. The referring instance variable stays correct and
+points at the right number; a different object simply sits under it, so
+searching the application for the offending assignment finds nothing.
+
+* `Storage#next_id` takes the id from a Postgres sequence, `odba_id_seq`,
+  which `#setup` creates. It used to be `@next_id += 1` under a mutex, with
+  @next_id seeded once per process from the highest odba_id in the table -
+  sound for one process, wrong for every deployment running a web worker and
+  an import job against the same database. Measured with two processes side
+  by side, both answered `[61935067, 61935068, 61935069]`. Stores without the
+  sequence keep the old behaviour.
+* The sequence starts at `MAX(odba_id)` plus `ID_SEQUENCE_GAP`, not at 1: a
+  plain `CREATE SEQUENCE` would re-issue ids that already exist, and the gap
+  covers ids that processes still on the old counter hold but have not
+  written yet.
+* `Cache#next_id` no longer swallows `OdbaDuplicateIdError`. The guard was
+  there all along - a peer raises it when the id is taken and the method
+  retries - but `rescue` without a class caught it too, so the retry could
+  never run. Only `DRb::DRbError` is caught now, which is what the line was
+  for: an unreachable peer must not stop the allocation.
+
+## 1.2.1 / 21.08.2026
+
+No library changes: lib/ is identical to 1.2.0. This release only ships a test
+suite and CI that pass.
+
+* Cleared CacheEntry's @@id_table between cache tests. Its ObjectSpace
+  finalizer calls ODBA.cache.invalidate, so objects left by earlier tests were
+  collected at unpredictable moments and deleted ids the running test had just
+  stored - the long standing flakiness in test_clean, test_clean__prefetched
+  and test_transaction
+* Fixed test_to_obj for drb >= 2.2, which resolves only ids to_id handed out
+  rather than looking them up with ObjectSpace._id2ref
+* Bumped install-nix-action to v31: v26 cannot provision Nix build users on
+  current macOS images
+* Stopped starting the processes twice in enterTest and waited for postgres
+  before running test/example.rb
+* Declared the git-hooks input that current devenv no longer injects implicitly
+* Dropped a continue-on-error that used matrix.ruby.to_s, which GitHub Actions
+  expressions do not support, so it silently never applied
+
 ## 1.2.0 / 21.08.2026
 
 * Reconnect when the database connection was lost. ydbd-pg reports a lost

data/devenv.nix

@@ -47,7 +47,20 @@
   languages.ruby.version = "3.4";
   # See full reference at https://devenv.sh/reference/options/
   enterTest = ''
-    devenv up --detach
-    bundle exec test/example.rb
+    # No `devenv up` here: `devenv test` already starts the processes before
+    # running enterTest, so starting them again aborts the whole run with
+    # "Processes already running with PID ...".
+    #
+    # Those processes come up in parallel with this script. On slower runners
+    # initdb is still creating template0 when we arrive and the first connect
+    # is refused, so wait for postgres to accept connections. The bare
+    # pg_isready afterwards turns "never came up" into a clear error rather
+    # than a confusing PG::ConnectionBad from the test.
+    for _ in $(seq 1 60); do
+      if pg_isready --host 127.0.0.1 --port 5432 > /dev/null 2>&1; then break; fi
+      sleep 1
+    done
+    pg_isready --host 127.0.0.1 --port 5432
+    bundle exec ruby test/example.rb
   '';
 }

data/devenv.yaml

@@ -1,6 +1,14 @@
 inputs:
   nixpkgs:
     url: github:NixOS/nixpkgs/nixos-24.11
+  # Declared explicitly because devenv no longer injects it implicitly: its
+  # git-hooks module throws "git-hooks or pre-commit-hooks input required".
+  # This mirrors what devenv.lock already records, so the lock stays valid.
+  git-hooks:
+    url: github:cachix/git-hooks.nix
+    inputs:
+      nixpkgs:
+        follows: nixpkgs
   nixpkgs-ruby:
     url: github:bobvanderlinden/nixpkgs-ruby
     inputs:

data/lib/odba/cache.rb

@@ -441,8 +441,14 @@ module ODBA
       end
       @peers.each do |peer|
         peer.reserve_next_id id
-      rescue
-        DRb::DRbError
+      rescue DRb::DRbError
+        # A peer we cannot reach must not stop the allocation. Note the
+        # explicit class: a bare rescue here also swallowed the
+        # OdbaDuplicateIdError a peer raises when the id is already taken,
+        # so the retry below could never run and both processes kept the
+        # same id. Whichever wrote last overwrote the other's row in
+        # `object`, and every reference to it then resolved to a foreign
+        # object.
       end
       id
     rescue OdbaDuplicateIdError

data/lib/odba/storage.rb

@@ -9,8 +9,17 @@ require "dbi"
 module ODBA
   class Storage # :nodoc: all
     include Singleton
-    attr_writer :dbi
+
+    # Not attr_writer: whether the store has an id sequence is memoized, and
+    # that answer belongs to the connection it was asked on.
+    def dbi=(dbi)
+      @id_sequence = nil
+      @dbi = dbi
+    end
     BULK_FETCH_STEP = 2500
+    # Distance between the highest id in use and the first the sequence
+    # hands out; see the odba_id_seq entry in TABLES.
+    ID_SEQUENCE_GAP = 100_000
     TABLES = [
       # in table 'object', the isolated dumps of all objects are stored
       ["object", <<~SQL],
@@ -37,12 +46,31 @@ module ODBA
         CREATE INDEX IF NOT EXISTS target_id_index ON object_connection(target_id);
       SQL
       # helper table 'collection'
-      ["collection", <<~SQL]
+      ["collection", <<~SQL],
         CREATE TABLE IF NOT EXISTS collection (
           odba_id integer NOT NULL, key text, value text,
           PRIMARY KEY(odba_id, key)
         );
       SQL
+      # The odba_id comes from this sequence, so that several processes on
+      # one database cannot hand out the same one. See #next_id.
+      #
+      # The start value is computed and must be: a plain CREATE SEQUENCE
+      # starts at 1 and would re-issue ids that already exist. The gap on
+      # top of MAX(odba_id) covers ids that processes still running with the
+      # old in-memory counter hold but have not written yet. Skipped numbers
+      # cost nothing - the odba_id is a surrogate key and carries no meaning.
+      ["odba_id_seq", <<~SQL]
+        DO $$
+        BEGIN
+          IF NOT EXISTS (SELECT 1 FROM pg_class
+                         WHERE relkind = 'S' AND relname = 'odba_id_seq') THEN
+            EXECUTE format('CREATE SEQUENCE odba_id_seq START WITH %s',
+                           (SELECT COALESCE(MAX(odba_id), 0) + #{ID_SEQUENCE_GAP}
+                            FROM object));
+          END IF;
+        END $$;
+      SQL
     ]
     def initialize
       @id_mutex = Mutex.new
@@ -425,13 +453,43 @@ module ODBA
       end
     end
 
+    # The id is allocated by the database, not by a counter in this process.
+    #
+    # It used to be `@next_id += 1` under this mutex, with @next_id seeded
+    # once per process from the highest odba_id in the table. That is sound
+    # for a single process and wrong for every deployment that runs more
+    # than one - web workers and import jobs on the same database each kept
+    # their own counter and handed out the same numbers, so one silently
+    # overwrote the other's row in `object`.
+    #
+    # Falls back to the old behaviour where no sequence exists, so a store
+    # that was never through #setup keeps working; #setup creates it.
     def next_id
-      @id_mutex.synchronize do
-        ensure_next_id_set
-        @next_id += 1
+      if id_sequence?
+        dbi.select_one("SELECT nextval('odba_id_seq')").first.to_i.tap { |id|
+          # max_id and reserve_next_id read @next_id, so keep it in step.
+          # Never backwards: a peer may already stand higher.
+          @id_mutex.synchronize {
+            @next_id = id if @next_id.nil? || @next_id < id
+          }
+        }
+      else
+        @id_mutex.synchronize do
+          ensure_next_id_set
+          @next_id += 1
+        end
       end
     end
 
+    def id_sequence?
+      return @id_sequence unless @id_sequence.nil?
+      @id_sequence = dbi.select_one(
+        "SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = 'odba_id_seq'"
+      ) ? true : false
+    rescue
+      @id_sequence = false
+    end
+
     def update_max_id(id)
       @id_mutex.synchronize do
         @next_id = id

data/lib/odba/version.rb

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
 class Odba
-  VERSION = '1.2.0'
+  VERSION = '1.2.2'
 end

data/test/test_cache.rb

@@ -19,6 +19,46 @@ module ODBA
     public :load_object
   end
 
+  # The peer conflict must reach the retry. Until 1.2.2 the loop read
+  # `peer.reserve_next_id id rescue DRb::DRbError` - a rescue without a
+  # class, which caught the OdbaDuplicateIdError a peer raises when the id
+  # is taken. Both processes then kept the same id and one overwrote the
+  # other's row in `object`.
+  class TestCacheNextId < Test::Unit::TestCase
+    include FlexMock::TestCase
+
+    def setup
+      @cache = ODBA::Cache.instance
+      @cache.instance_variable_set(:@file_lock, false)
+      @storage = flexmock("storage")
+      ODBA.storage = @storage
+    end
+
+    def test_a_peer_conflict_leads_to_a_new_id
+      @storage.should_receive(:next_id).and_return(100, 101)
+      seen = []
+      peer = flexmock("peer")
+      peer.should_receive(:reserve_next_id).and_return { |id|
+        seen << id
+        raise ODBA::OdbaDuplicateIdError, "taken" if seen.size == 1
+        true
+      }
+      @cache.instance_variable_set(:@peers, [peer])
+      assert_equal(101, @cache.next_id)
+      assert_equal([100, 101], seen)
+    end
+
+    # A peer we cannot reach must not stop the allocation - that is what the
+    # line was for, and it stays.
+    def test_an_unreachable_peer_does_not_stop_the_allocation
+      @storage.should_receive(:next_id).and_return(100)
+      peer = flexmock("peer")
+      peer.should_receive(:reserve_next_id).and_raise(DRb::DRbError, "gone")
+      @cache.instance_variable_set(:@peers, [peer])
+      assert_equal(100, @cache.next_id)
+    end
+  end
+
   class TestCache < Test::Unit::TestCase
     include FlexMock::TestCase
     class ODBAContainerInCache
@@ -33,6 +73,15 @@ module ODBA
       @cache.fetched = {}
       @cache.prefetched = {}
       @cache.indices = {}
+      ## CacheEntry registers an ObjectSpace finalizer that calls
+      ## ODBA.cache.invalidate, which deletes the odba_id from the shared
+      ## singleton cache. Objects left behind by earlier tests are collected at
+      ## unpredictable moments, so without dropping their finalizer bookkeeping
+      ## a stray GC run deletes the very ids this test just stored. That is the
+      ## long standing flakiness in test_clean, test_clean__prefetched and
+      ## test_transaction: they failed on some Ruby versions and not others,
+      ## and a different test failed on each run of identical code.
+      ODBA::CacheEntry.class_variable_set(:@@id_table, {})
     end
 
     def teardown

data/test/test_drbwrapper.rb

@@ -78,7 +78,13 @@ module ODBA
     def test_to_obj
       ODBA.cache.should_receive(:store)
       o = Object.new
-      assert_equal(o, @idconv.to_obj(o.object_id))
+      ## drb >= 2.2 resolves only those ids it handed out itself: DRbIdConv
+      ## keeps a DRbObjectSpace WeakMap that to_id populates, and to_obj raises
+      ## RangeError for anything absent from it. Older drb looked the id up
+      ## with ObjectSpace._id2ref, which resolved any live object. Register the
+      ## object first, as real DRb traffic does - an id only reaches to_obj
+      ## after to_id produced it.
+      assert_equal(o, @idconv.to_obj(@idconv.to_id(o)))
       o.extend(ODBA::Persistable)
       o.instance_variable_set(:@odba_id, 4)
       o.odba_isolated_store

data/test/test_storage.rb

@@ -15,6 +15,7 @@ module ODBA
       @storage = ODBA::Storage.instance
        @dbi = flexmock("DBI")
        @storage.dbi = @dbi
+       @storage.instance_variable_set(:@id_sequence, nil)
     end
 
     def teardown
@@ -106,12 +107,40 @@ module ODBA
        @storage.create_index("index_name")
     end
 
+    # Without a sequence the old counter still runs, so a store that was
+    # never through #setup keeps working.
     def test_next_id
+      @dbi.should_receive(:select_one).and_return(nil)
       @storage.next_id = 1
        assert_equal(2, @storage.next_id)
        assert_equal(3, @storage.next_id)
     end
 
+    # The point of 1.2.2: the id comes from the database, so two processes
+    # on one store cannot be handed the same one.
+    def test_next_id__from_the_sequence
+      @dbi.should_receive(:select_one)
+        .with("SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = 'odba_id_seq'")
+        .once.and_return([1])
+      @dbi.should_receive(:select_one)
+        .with("SELECT nextval('odba_id_seq')").twice.and_return([4711], [4712])
+      assert_equal(4711, @storage.next_id)
+      assert_equal(4712, @storage.next_id)
+    end
+
+    # max_id and reserve_next_id read @next_id, so it has to follow the
+    # sequence - and never move backwards, a peer may stand higher already.
+    def test_next_id__keeps_the_local_counter_in_step
+      @dbi.should_receive(:select_one)
+        .with(/pg_class/).and_return([1])
+      @dbi.should_receive(:select_one).with(/nextval/).and_return([90], [5])
+      @storage.next_id = 10
+      @storage.next_id
+      assert_equal(90, @storage.instance_variable_get(:@next_id))
+      @storage.next_id
+      assert_equal(90, @storage.instance_variable_get(:@next_id))
+    end
+
     def test_store__1
       dbi = flexmock("dbi")
        @storage.dbi = dbi

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: odba
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.2
 platform: ruby
 authors:
 - Masaomi Hatakeyama, Zeno R.R. Davatz