honeymaker 0.11.9 → 0.11.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6f44f3c6ed35e460dcb6b6d53a73bebec433c34ad1109ab8b1531f799ad93b0
-  data.tar.gz: cd55a0b081c84f712e67ba00ede7633764592b0ff97d2b47fa50f29fc7c0cdd0
+  metadata.gz: 81ec4e430e403ef063e19f4c1a0827f9652bed081e7bdf845ba0f2c88b5c7a28
+  data.tar.gz: 779b9f7391e4259a711c478a2493445fecf88ef04bc46e64affdaf2b1267decb
 SHA512:
-  metadata.gz: 32e67280cabf17a5a7305660b92ad2ff2eda948fb947a50a771f6bbd102f7cd9cbbeefdda50b09c16d1087bdd637c4649ba25963046828c9dcbd73eec87c5a4b
-  data.tar.gz: 01b15492c985691a7fcde609a56f3f6e197febbf7fdaaace27186ecb66268a128f796e99ca2a28aef7700c7e56bedb753ceff557400c7f54e142b07a7cb36a20
+  metadata.gz: 2107a78f1bbec5167448470cfe8f26871c37fea3237be0ee1137d33cbe738ed9b23292ff99d35e1c9114f27a0634ba0aa315357cd126fd57f805919858a7c4eb
+  data.tar.gz: 1f694295c4ed6699728fd0fa500a5a081d1835b620a6b6d06e7ee3ab239804defe4bb8a9271ad638a0321e27aa5091dc6d5adbda5e69cfd97bdf89d9e310dd39

data/lib/honeymaker/exchange.rb

@@ -69,6 +69,15 @@ module Honeymaker
       3600
     end
 
+    # Venue symbols the last #get_tickers_info saw listed but could not describe, symbol => why. They
+    # are missing from the catalogue WITHOUT being delisted: a caller that reads absence as delisting
+    # must keep what it has for them. Empty for a venue whose catalogue is a single request.
+    # It describes the last get_tickers_info call only; the #tickers_info cache does not carry it, so a
+    # caller that needs both reads them from one get_tickers_info.
+    def unreadable_symbols
+      @unreadable_symbols || {}
+    end
+
     private
 
     # No HTTP answer means the transport failed, and its text rarely says how ("end of file
@@ -78,6 +87,17 @@ module Honeymaker
       status.nil? ? { status: nil, error_chain: Utils.error_chain(error) } : { status: status }
     end
 
+    # Records one instrument the venue lists but whose description could not be read, keyed as the
+    # venue emits its ticker. The reason is what a person needs to see: the venue's own answer when it
+    # gave one ("HTTP 400 {...InvalidSymbol...}"), otherwise the exception.
+    def set_aside(symbol, error)
+      status = error.respond_to?(:response_status) ? error.response_status : nil
+      body = error.respond_to?(:response_body) ? error.response_body.to_s : ""
+      why = body.empty? ? "#{error.class}: #{error.message}" : body
+      (@unreadable_symbols ||= {})[symbol.to_s.upcase] = (status ? "HTTP #{status} #{why}" : why)[0, 200]
+      nil
+    end
+
     def with_rescue
       Result::Success.new(yield)
     rescue Faraday::Error => e

data/lib/honeymaker/exchanges/gemini.rb

@@ -15,35 +15,39 @@ module Honeymaker
         Faraday::TooManyRequestsError, Faraday::ServerError
       ].freeze
 
+      # A symbol still failing after its retries is usually the venue or the connection, not the
+      # instrument, and each costs up to ~100 s. Past this many in one run the venue itself is down, so
+      # the whole catalogue fails, as any failure did before.
+      # ponytail: flat per-run budget; raise it if a rollout ever 5xxs a bigger family than this.
+      NO_ANSWER_LIMIT = 10
+
+      # /v1/symbols says what Gemini lists; the details only describe it. A listed symbol whose details
+      # cannot be read this run is SET ASIDE: left out of the catalogue and named in
+      # #unreadable_symbols, which callers must read as "unknown", never as delisted. Gemini lists new
+      # symbols before it can describe them (details/gramsgd answered 400 InvalidSymbol for hours on
+      # 2026-09-22), and failing the whole catalogue for that one left every Gemini pair without a
+      # verdict.
       def get_tickers_info
+        @unreadable_symbols = {}
         with_rescue do
           symbols = catalogue_get("/v1/symbols")
+          no_answers = 0
 
           tickers = symbols.filter_map do |symbol|
-            # A symbol that still fails raises, failing the WHOLE catalogue. Never skip it: callers
-            # read a pair missing from the catalogue as delisted, and data-api revokes it.
-            detail = catalogue_get("/v1/symbols/details/#{symbol}")
-            next unless spot?(symbol, detail)
-
-            tick_size = detail["tick_size"]&.to_s || "0.01"
-            quote_increment = detail["quote_increment"]&.to_s || "0.01"
-
-            {
-              ticker: symbol.upcase,
-              base: detail["base_currency"].upcase,
-              quote: detail["quote_currency"].upcase,
-              minimum_base_size: detail["min_order_size"],
-              minimum_quote_size: "0",
-              maximum_base_size: nil,
-              maximum_quote_size: nil,
-              base_decimals: Utils.decimals(tick_size),
-              quote_decimals: Utils.decimals(quote_increment),
-              price_decimals: Utils.decimals(quote_increment),
-              available: true,
-              trading_enabled: detail["status"] == "open"
-            }
+            describe(symbol, catalogue_get("/v1/symbols/details/#{symbol}"))
+          rescue *TRANSIENT_ERRORS => e
+            raise if (no_answers += 1) >= NO_ANSWER_LIMIT
+
+            set_aside(symbol, e)
+          rescue StandardError => e
+            set_aside(symbol, e)
+          end
+          tickers = set_aside_shared_pairs(tickers)
+          if tickers.empty? && @unreadable_symbols.any?
+            raise Error, "Gemini: no symbol could be read, e.g. #{@unreadable_symbols.first.join(': ')}"
           end
-          reject_shared_pairs!(tickers)
+
+          tickers
         end
       end
 
@@ -64,23 +68,52 @@ module Honeymaker
         @connection ||= build_connection(BASE_URL)
       end
 
+      def describe(symbol, detail)
+        return unless spot?(symbol, detail)
+
+        tick_size = detail["tick_size"]&.to_s || "0.01"
+        quote_increment = detail["quote_increment"]&.to_s || "0.01"
+
+        {
+          ticker: symbol.upcase,
+          base: detail["base_currency"].upcase,
+          quote: detail["quote_currency"].upcase,
+          minimum_base_size: detail["min_order_size"],
+          minimum_quote_size: "0",
+          maximum_base_size: nil,
+          maximum_quote_size: nil,
+          base_decimals: Utils.decimals(tick_size),
+          quote_decimals: Utils.decimals(quote_increment),
+          price_decimals: Utils.decimals(quote_increment),
+          available: true,
+          trading_enabled: detail["status"] == "open"
+        }
+      end
+
       # Gemini lists perpetual swaps beside spot, under the same base and quote (BTCGUSDPERP is
       # BTC/GUSD, product_type "swap"), so only product_type tells them apart. An instrument without
-      # one is not assumed to be spot: the catalogue fails and callers keep the one they have.
+      # one is not assumed to be spot: it is set aside. So is a body that is not an object -- JSON
+      # served as text/plain arrives as a String, and String#[] finds "product_type" inside it.
       def spot?(symbol, detail)
+        raise Error, "Gemini #{symbol}: details are not an object" unless detail.is_a?(Hash)
+
         product_type = detail["product_type"]
-        raise Error, "Gemini #{symbol}: no product_type in its details" unless product_type.is_a?(String)
+        raise Error, "Gemini #{symbol}: no product_type in its details" unless product_type.is_a?(String) && !product_type.empty?
+        # A missing status would otherwise read as "not open" and revoke the pair.
+        raise Error, "Gemini #{symbol}: no status in its details" unless detail["status"].is_a?(String)
 
         product_type == "spot"
       end
 
-      # One pair, one instrument. Two would leave callers to pick one by list order.
-      def reject_shared_pairs!(tickers)
-        shared = tickers.group_by { |t| "#{t[:base]}/#{t[:quote]}" }.find { |_, group| group.size > 1 }
-        return tickers unless shared
-
-        pair, group = shared
-        raise Error, "Gemini #{pair} is claimed by #{group.map { |t| t[:ticker] }.join(' and ')}"
+      # One pair, one instrument. Two would leave callers to pick one by list order, so neither is
+      # picked: both are set aside until Gemini lists only one.
+      def set_aside_shared_pairs(tickers)
+        shared = tickers.group_by { |t| [t[:base], t[:quote]] }.values.reject(&:one?)
+        shared.each do |group|
+          claim = "Gemini #{group.first[:base]}/#{group.first[:quote]} is claimed by #{group.map { |t| t[:ticker] }.join(' and ')}"
+          group.each { |t| @unreadable_symbols[t[:ticker]] = claim }
+        end
+        tickers - shared.flatten
       end
 
       def catalogue_get(path)

data/lib/honeymaker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Honeymaker
-  VERSION = "0.11.9"
+  VERSION = "0.11.10"
 end

data/test/honeymaker/exchanges/gemini_catalogue_test.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+# The set-aside rule, driven through honeymaker's real middleware stack: one symbol Gemini lists but
+# cannot describe is left out and named, and never fails the whole catalogue.
+class Honeymaker::Exchanges::GeminiCatalogueTest < Minitest::Test
+  JSON_CT = { "Content-Type" => "application/json" }.freeze
+
+  def detail(sym, base, quote, **over)
+    { "symbol" => sym.upcase, "base_currency" => base, "quote_currency" => quote, "tick_size" => "0.00000001",
+      "quote_increment" => "0.01", "min_order_size" => "0.00001", "status" => "open", "product_type" => "spot" }.merge(over.transform_keys(&:to_s))
+  end
+
+  def exchange_with(symbols, &details)
+    stubs = Faraday::Adapter::Test::Stubs.new do |s|
+      s.get("/v1/symbols") { [200, JSON_CT, JSON.dump(symbols)] }
+      symbols.each { |sym| s.get("/v1/symbols/details/#{sym}") { details.call(sym) } }
+    end
+    ex = Honeymaker::Exchanges::Gemini.new
+    @sleeps = sleeps = []
+    ex.define_singleton_method(:sleep) { |s| sleeps << s }
+    conn = Faraday.new(url: "https://api.gemini.com") do |c|
+      c.request :json; c.response :json; c.response :raise_error; c.adapter :test, stubs
+    end
+    ex.instance_variable_set(:@connection, conn)
+    ex
+  end
+
+  def ok(sym)
+    base = sym[0..-4].upcase
+    [200, JSON_CT, JSON.dump(detail(sym, base, "USD"))]
+  end
+
+  def test_incident_replay_sets_gramsgd_aside_and_reads_the_rest
+    body = '{"result":"error","reason":"InvalidSymbol","message":"Received unsupported symbol \'gramsgd\'"}'
+    ex = exchange_with(%w[btcusd gramsgd ethusd]) { |s| s == "gramsgd" ? [400, JSON_CT, body] : ok(s) }
+    r = ex.get_tickers_info
+    assert r.success?
+    assert_equal %w[BTCUSD ETHUSD], r.data.map { |t| t[:ticker] }
+    assert_equal({ "GRAMSGD" => "HTTP 400 #{body}" }, ex.unreadable_symbols)
+    assert_equal [0.5] * 4, @sleeps, "a 400 is not retried"
+  end
+
+  def test_text_plain_details_are_set_aside_not_read_as_non_spot
+    ex = exchange_with(%w[btcusd ethusd]) do |s|
+      s == "ethusd" ? [200, { "Content-Type" => "text/plain" }, JSON.dump(detail("ethusd", "ETH", "USD"))] : ok(s)
+    end
+    r = ex.get_tickers_info
+    assert r.success?
+    assert_equal %w[BTCUSD], r.data.map { |t| t[:ticker] }
+    assert_match(/not an object/, ex.unreadable_symbols["ETHUSD"])
+  end
+
+  def test_garbage_shapes_are_each_set_aside
+    shapes = {
+      "aaausd" => [200, JSON_CT, JSON.dump(detail("aaausd", "AAA", "USD").except("product_type"))],
+      "bbbusd" => [200, { "Content-Type" => "text/html" }, "<html>oops</html>"],
+      "cccusd" => [200, JSON_CT, "{not json"],
+      "dddusd" => [200, JSON_CT, JSON.dump([1, 2])],
+      "eeeusd" => [200, JSON_CT, JSON.dump(detail("eeeusd", nil, "USD"))],
+      "fffusd" => [403, { "Content-Type" => "text/html" }, "<html>blocked</html>"],
+      "gggusd" => [404, JSON_CT, "{}"],
+      "hhhusd" => [200, JSON_CT, JSON.dump(detail("hhhusd", "HHH", "USD", product_type: ""))],
+      "iiiusd" => [200, JSON_CT, JSON.dump(detail("iiiusd", "III", "USD").except("status"))]
+    }
+    ex = exchange_with(%w[btcusd] + shapes.keys) { |s| shapes[s] || ok(s) }
+    r = ex.get_tickers_info
+    assert r.success?, r.errors.inspect
+    assert_equal %w[BTCUSD], r.data.map { |t| t[:ticker] }
+    assert_equal shapes.keys.map(&:upcase).sort, ex.unreadable_symbols.keys.sort
+  end
+
+  def test_one_symbol_without_answer_is_set_aside_after_its_retries
+    ex = exchange_with(%w[btcusd ethusd solusd]) { |s| s == "ethusd" ? [503, JSON_CT, "{}"] : ok(s) }
+    r = ex.get_tickers_info
+    assert r.success?
+    assert_equal %w[BTCUSD SOLUSD], r.data.map { |t| t[:ticker] }
+    assert_equal [0.5, 0.5, 0.5, 2, 0.5, 4, 0.5, 0.5], @sleeps
+  end
+
+  def test_the_no_answer_budget_fails_the_whole_catalogue
+    syms = (1..12).map { |i| format("x%02dusd", i) }
+    requested = []
+    ex = exchange_with(syms) { |s| requested << s; [503, JSON_CT, "{}"] }
+    r = ex.get_tickers_info
+    assert r.failure?
+    assert_equal({ status: 503 }, r.data)
+    assert_equal 10, requested.uniq.size, "stops at the budget"
+  end
+
+  def test_nine_no_answers_among_readable_symbols_still_succeed
+    syms = (1..9).flat_map { |i| [format("x%02dusd", i), format("y%02dusd", i)] }
+    ex = exchange_with(syms) { |s| s.start_with?("x") ? [503, JSON_CT, "{}"] : ok(s) }
+    r = ex.get_tickers_info
+    assert r.success?, r.errors.inspect
+    assert_equal 9, r.data.size
+    assert_equal 9, ex.unreadable_symbols.size
+  end
+
+  def test_every_pair_claimed_twice_is_set_aside_and_the_rest_kept
+    twins = { "btcusd2" => %w[BTC USD], "ethusd2" => %w[ETH USD] }
+    ex = exchange_with(%w[btcusd btcusd2 ethusd ethusd2 solusd]) do |s|
+      twins[s] ? [200, JSON_CT, JSON.dump(detail(s, *twins[s]))] : ok(s)
+    end
+    r = ex.get_tickers_info
+    assert r.success?
+    assert_equal %w[SOLUSD], r.data.map { |t| t[:ticker] }
+    assert_equal %w[BTCUSD BTCUSD2 ETHUSD ETHUSD2], ex.unreadable_symbols.keys.sort
+  end
+
+  def test_nothing_readable_is_a_failure_and_state_resets_between_runs
+    ex = exchange_with(%w[btcusd]) { |_| [400, JSON_CT, "{}"] }
+    assert ex.get_tickers_info.failure?
+    assert_equal %w[BTCUSD], ex.unreadable_symbols.keys
+    assert_equal({}, Honeymaker::Exchanges::Binance.new.unreadable_symbols)
+  end
+end

data/test/honeymaker/exchanges/gemini_test.rb

@@ -76,28 +76,27 @@ class Honeymaker::Exchanges::GeminiTest < Minitest::Test
     end
   end
 
-  # All or nothing: data-api reads a pair missing from the catalogue as delisted, so a symbol that
-  # keeps failing must fail the whole catalogue, and nothing after it is fetched.
-  def test_get_tickers_info_fails_whole_catalogue_when_a_symbol_keeps_failing
+  # Set aside, not fatal: a symbol that keeps failing is left out and named, and the run goes on.
+  def test_get_tickers_info_sets_aside_a_symbol_that_keeps_failing
     connection = stub_catalogue(%w[btcusd ethusd solusd])
     connection.expects(:get).with("/v1/symbols/details/ethusd").times(3)
               .raises(Faraday::ConnectionFailed, "end of file reached")
-    connection.expects(:get).with("/v1/symbols/details/solusd").never
 
     result = @exchange.get_tickers_info
 
-    assert result.failure?
-    refute_kind_of Array, result.data, "no partial catalogue"
-    assert_equal ["end of file reached"], result.errors
-    assert_equal [0.5, 0.5, 0.5, 2, 0.5, 4, 0.5], @sleeps
+    assert result.success?
+    assert_equal %w[BTCUSD SOLUSD], result.data.map { |t| t[:ticker] }
+    assert_equal({ "ETHUSD" => "Faraday::ConnectionFailed: end of file reached" }, @exchange.unreadable_symbols)
+    assert_equal [0.5, 0.5, 0.5, 2, 0.5, 4, 0.5, 0.5], @sleeps
   end
 
   def test_get_tickers_info_does_not_retry_a_non_transient_error
-    connection = stub_catalogue(%w[btcusd])
+    connection = stub_catalogue(%w[btcusd ethusd])
     connection.expects(:get).with("/v1/symbols/details/btcusd").once
               .raises(Faraday::ResourceNotFound, "404")
 
-    assert @exchange.get_tickers_info.failure?
+    assert_equal %w[ETHUSD], @exchange.get_tickers_info.data.map { |t| t[:ticker] }
+    assert_equal %w[BTCUSD], @exchange.unreadable_symbols.keys
   end
 
   # Gemini lists perpetual swaps in /v1/symbols with the same base and quote as the spot pair
@@ -113,29 +112,29 @@ class Honeymaker::Exchanges::GeminiTest < Minitest::Test
     assert_equal %w[BTCUSD], result.data.map { |t| t[:ticker] }
   end
 
-  # An unclassified instrument is not assumed to be spot: the whole catalogue fails, and callers keep
-  # the one they have.
-  def test_get_tickers_info_fails_when_a_symbol_has_no_product_type
+  # An unclassified instrument is not assumed to be spot: it is set aside, and callers keep what
+  # they have for it.
+  def test_get_tickers_info_sets_aside_a_symbol_with_no_product_type
     connection = stub_catalogue(%w[btcusd ethusd])
     connection.stubs(:get).with("/v1/symbols/details/ethusd")
               .returns(stub(body: detail_body.except("product_type").merge("symbol" => "ETHUSD")))
 
     result = @exchange.get_tickers_info
 
-    assert result.failure?
-    assert_nil result.data
-    assert_match(/ethusd/, result.errors.first)
+    assert_equal %w[BTCUSD], result.data.map { |t| t[:ticker] }
+    assert_match(/ethusd: no product_type/, @exchange.unreadable_symbols["ETHUSD"])
   end
 
-  def test_get_tickers_info_fails_when_two_instruments_claim_one_pair
-    connection = stub_catalogue(%w[btcusd btcusd2])
+  def test_get_tickers_info_sets_aside_both_instruments_claiming_one_pair
+    connection = stub_catalogue(%w[btcusd btcusd2 ethusd])
     connection.stubs(:get).with("/v1/symbols/details/btcusd2")
               .returns(stub(body: detail_body.merge("symbol" => "BTCUSD2")))
 
     result = @exchange.get_tickers_info
 
-    assert result.failure?
-    assert_match(%r{BTC/USD}, result.errors.first)
+    assert_equal %w[ETHUSD], result.data.map { |t| t[:ticker] }
+    assert_equal %w[BTCUSD BTCUSD2], @exchange.unreadable_symbols.keys.sort
+    assert_match(%r{BTC/USD}, @exchange.unreadable_symbols["BTCUSD"])
   end
 
   def test_get_bid_ask_parses_response

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: honeymaker
 version: !ruby/object:Gem::Version
-  version: 0.11.9
+  version: 0.11.10
 platform: ruby
 authors:
 - Deltabadger
@@ -180,6 +180,7 @@ files:
 - test/honeymaker/exchanges/bitvavo_test.rb
 - test/honeymaker/exchanges/bybit_test.rb
 - test/honeymaker/exchanges/coinbase_test.rb
+- test/honeymaker/exchanges/gemini_catalogue_test.rb
 - test/honeymaker/exchanges/gemini_test.rb
 - test/honeymaker/exchanges/hyperliquid_test.rb
 - test/honeymaker/exchanges/kraken_test.rb