@bsv/sdk 1.8.12 → 1.9.0

package/src/overlay-tools/HostReputationTracker.ts

@@ -0,0 +1,232 @@
+interface HostReputationEntry {
+  host: string
+  totalSuccesses: number
+  totalFailures: number
+  consecutiveFailures: number
+  avgLatencyMs: number | null
+  lastLatencyMs: number | null
+  backoffUntil: number
+  lastUpdatedAt: number
+  lastError?: string
+}
+
+export interface RankedHost extends HostReputationEntry {
+  score: number
+}
+
+const DEFAULT_LATENCY_MS = 1500
+const LATENCY_SMOOTHING_FACTOR = 0.25
+const BASE_BACKOFF_MS = 1000
+const MAX_BACKOFF_MS = 60_000
+const FAILURE_PENALTY_MS = 400
+const SUCCESS_BONUS_MS = 30
+const FAILURE_BACKOFF_GRACE = 2
+const STORAGE_KEY = 'bsvsdk_overlay_host_reputation_v1'
+
+interface KeyValueStore {
+  get: (key: string) => string | null | undefined
+  set: (key: string, value: string) => void
+}
+
+export class HostReputationTracker {
+  private readonly stats: Map<string, HostReputationEntry>
+  private readonly store: KeyValueStore | undefined
+
+  constructor (store?: KeyValueStore) {
+    this.stats = new Map()
+    this.store = store ?? this.getLocalStorageAdapter()
+    this.loadFromStorage()
+  }
+
+  reset (): void {
+    this.stats.clear()
+  }
+
+  recordSuccess (host: string, latencyMs: number): void {
+    const entry = this.getOrCreate(host)
+    const now = Date.now()
+    const safeLatency = Number.isFinite(latencyMs) && latencyMs >= 0 ? latencyMs : DEFAULT_LATENCY_MS
+    if (entry.avgLatencyMs === null) {
+      entry.avgLatencyMs = safeLatency
+    } else {
+      entry.avgLatencyMs =
+        (1 - LATENCY_SMOOTHING_FACTOR) * entry.avgLatencyMs +
+        LATENCY_SMOOTHING_FACTOR * safeLatency
+    }
+    entry.lastLatencyMs = safeLatency
+    entry.totalSuccesses += 1
+    entry.consecutiveFailures = 0
+    entry.backoffUntil = 0
+    entry.lastUpdatedAt = now
+    entry.lastError = undefined
+    this.saveToStorage()
+  }
+
+  recordFailure (host: string, reason?: unknown): void {
+    const entry = this.getOrCreate(host)
+    const now = Date.now()
+    entry.totalFailures += 1
+    entry.consecutiveFailures += 1
+    const msg =
+      typeof reason === 'string'
+        ? reason
+        : reason instanceof Error
+          ? reason.message
+          : undefined
+    const immediate =
+      typeof msg === 'string' &&
+      (msg.includes('ERR_NAME_NOT_RESOLVED') ||
+        msg.includes('ENOTFOUND') ||
+        msg.includes('getaddrinfo') ||
+        msg.includes('Failed to fetch'))
+    if (immediate && entry.consecutiveFailures < FAILURE_BACKOFF_GRACE + 1) {
+      entry.consecutiveFailures = FAILURE_BACKOFF_GRACE + 1
+    }
+    const penaltyLevel = Math.max(entry.consecutiveFailures - FAILURE_BACKOFF_GRACE, 0)
+    if (penaltyLevel === 0) {
+      entry.backoffUntil = 0
+    } else {
+      const backoffDuration = Math.min(
+        MAX_BACKOFF_MS,
+        BASE_BACKOFF_MS * Math.pow(2, penaltyLevel - 1)
+      )
+      entry.backoffUntil = now + backoffDuration
+    }
+    entry.lastUpdatedAt = now
+    entry.lastError =
+      typeof reason === 'string'
+        ? reason
+        : reason instanceof Error
+          ? reason.message
+          : undefined
+    this.saveToStorage()
+  }
+
+  rankHosts (hosts: string[], now: number = Date.now()): RankedHost[] {
+    const seen = new Map<string, number>()
+    hosts.forEach((host, idx) => {
+      if (typeof host !== 'string' || host.length === 0) return
+      if (!seen.has(host)) seen.set(host, idx)
+    })
+
+    const orderedHosts = Array.from(seen.keys())
+    const ranked = orderedHosts.map((host) => {
+      const entry = this.getOrCreate(host)
+      return {
+        ...entry,
+        score: this.computeScore(entry, now),
+        originalOrder: seen.get(host) ?? 0
+      }
+    })
+
+    ranked.sort((a, b) => {
+      const aInBackoff = a.backoffUntil > now
+      const bInBackoff = b.backoffUntil > now
+      if (aInBackoff !== bInBackoff) return aInBackoff ? 1 : -1
+      if (a.score !== b.score) return a.score - b.score
+      if (a.totalSuccesses !== b.totalSuccesses) return b.totalSuccesses - a.totalSuccesses
+      return (a as any).originalOrder - (b as any).originalOrder
+    })
+
+    return ranked.map(({ originalOrder, ...rest }) => rest)
+  }
+
+  snapshot (host: string): HostReputationEntry | undefined {
+    const entry = this.stats.get(host)
+    return entry != null ? { ...entry } : undefined
+  }
+
+  private getStorage (): any {
+    try {
+      const g: any = typeof globalThis === 'object' ? globalThis : undefined
+      if (g == null || g.localStorage == null) return undefined
+      return g.localStorage
+    } catch {
+      return undefined
+    }
+  }
+
+  private getLocalStorageAdapter (): KeyValueStore | undefined {
+    const s = this.getStorage()
+    if (s == null) return undefined
+    return {
+      get: (key: string) => {
+        try { return s.getItem(key) } catch { return null }
+      },
+      set: (key: string, value: string) => {
+        try { s.setItem(key, value) } catch { }
+      }
+    }
+  }
+
+  private loadFromStorage (): void {
+    const s = this.store
+    if (s == null) return
+    try {
+      const raw = s.get(STORAGE_KEY)
+      if (typeof raw !== 'string' || raw.length === 0) return
+      const data = JSON.parse(raw)
+      if (typeof data !== 'object' || data === null) return
+      this.stats.clear()
+      for (const k of Object.keys(data)) {
+        const v: any = (data)[k]
+        if (v != null && typeof v === 'object') {
+          const entry: HostReputationEntry = {
+            host: String(v.host ?? k),
+            totalSuccesses: Number(v.totalSuccesses ?? 0),
+            totalFailures: Number(v.totalFailures ?? 0),
+            consecutiveFailures: Number(v.consecutiveFailures ?? 0),
+            avgLatencyMs: v.avgLatencyMs == null ? null : Number(v.avgLatencyMs),
+            lastLatencyMs: v.lastLatencyMs == null ? null : Number(v.lastLatencyMs),
+            backoffUntil: Number(v.backoffUntil ?? 0),
+            lastUpdatedAt: Number(v.lastUpdatedAt ?? 0),
+            lastError: typeof v.lastError === 'string' ? v.lastError : undefined
+          }
+          this.stats.set(entry.host, entry)
+        }
+      }
+    } catch {}
+  }
+
+  private saveToStorage (): void {
+    const s = this.store
+    if (s == null) return
+    try {
+      const obj: Record<string, any> = {}
+      for (const [host, entry] of this.stats.entries()) {
+        obj[host] = entry
+      }
+      s.set(STORAGE_KEY, JSON.stringify(obj))
+    } catch {}
+  }
+
+  private computeScore (entry: HostReputationEntry, now: number): number {
+    const latency = entry.avgLatencyMs ?? DEFAULT_LATENCY_MS
+    const failurePenalty = entry.consecutiveFailures * FAILURE_PENALTY_MS
+    const successBonus = Math.min(entry.totalSuccesses * SUCCESS_BONUS_MS, latency / 2)
+    const backoffPenalty = entry.backoffUntil > now ? entry.backoffUntil - now : 0
+    return latency + failurePenalty + backoffPenalty - successBonus
+  }
+
+  private getOrCreate (host: string): HostReputationEntry {
+    let entry = this.stats.get(host)
+    if (entry == null) {
+      entry = {
+        host,
+        totalSuccesses: 0,
+        totalFailures: 0,
+        consecutiveFailures: 0,
+        avgLatencyMs: null,
+        lastLatencyMs: null,
+        backoffUntil: 0,
+        lastUpdatedAt: 0
+      }
+      this.stats.set(host, entry)
+    }
+    return entry
+  }
+}
+
+const globalTracker = new HostReputationTracker()
+
+export const getOverlayHostReputationTracker = (): HostReputationTracker => globalTracker

package/src/overlay-tools/LookupResolver.ts

@@ -1,6 +1,7 @@
 import { Transaction } from '../transaction/index.js'
 import OverlayAdminTokenTemplate from './OverlayAdminTokenTemplate.js'
 import * as Utils from '../primitives/utils.js'
+import { getOverlayHostReputationTracker, HostReputationTracker } from './HostReputationTracker.js'
 
 const defaultFetch: typeof fetch =
   typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function'
@@ -92,6 +93,8 @@ export interface LookupResolverConfig {
   additionalHosts?: Record<string, string[]>
   /** Optional cache tuning. */
   cache?: CacheOptions
+  /** Optional storage for host reputation data. */
+  reputationStorage?: 'localStorage' | { get: (key: string) => string | null | undefined, set: (key: string, value: string) => void }
 }
 
 /** Facilitates lookups to URLs that return answers. */
@@ -204,6 +207,7 @@ export default class LookupResolver {
   private readonly hostOverrides: Record<string, string[]>
   private readonly additionalHosts: Record<string, string[]>
   private readonly networkPreset: 'mainnet' | 'testnet' | 'local'
+  private readonly hostReputation: HostReputationTracker
 
   // ---- Caches / memoization ----
   private readonly hostsCache: Map<string, { hosts: string[], expiresAt: number }>
@@ -223,6 +227,15 @@ export default class LookupResolver {
     this.hostOverrides = hostOverrides
     this.additionalHosts = config.additionalHosts ?? {}
 
+    const rs = config.reputationStorage
+    if (rs === 'localStorage') {
+      this.hostReputation = new HostReputationTracker()
+    } else if (typeof rs === 'object' && rs !== null && typeof rs.get === 'function' && typeof rs.set === 'function') {
+      this.hostReputation = new HostReputationTracker(rs)
+    } else {
+      this.hostReputation = getOverlayHostReputationTracker()
+    }
+
     // cache tuning
     this.hostsTtlMs = config.cache?.hostsTtlMs ?? 5 * 60 * 1000 // 5 min
     this.hostsMaxEntries = config.cache?.hostsMaxEntries ?? 128
@@ -262,10 +275,18 @@ export default class LookupResolver {
       )
     }
 
+    const rankedHosts = this.prepareHostsForQuery(
+      competentHosts,
+      `lookup service ${question.service}`
+    )
+    if (rankedHosts.length < 1) {
+      throw new Error(`All competent hosts for ${question.service} are temporarily unavailable due to backoff.`)
+    }
+
     // Fire all hosts with per-host timeout, harvest successful output-list responses
     const hostResponses = await Promise.allSettled(
-      competentHosts.map(async (host) => {
-        return await this.facilitator.lookup(host, question, timeout)
+      rankedHosts.map(async (host) => {
+        return await this.lookupHostWithTracking(host, question, timeout)
       })
     )
 
@@ -382,9 +403,15 @@ export default class LookupResolver {
     }
 
     // Query all SLAP trackers; tolerate failures.
+    const trackerHosts = this.prepareHostsForQuery(
+      this.slapTrackers,
+      'SLAP trackers'
+    )
+    if (trackerHosts.length === 0) return []
+
     const trackerResponses = await Promise.allSettled(
-      this.slapTrackers.map(async (tracker) =>
-        await this.facilitator.lookup(tracker, query, MAX_TRACKER_WAIT_TIME)
+      trackerHosts.map(async (tracker) =>
+        await this.lookupHostWithTracking(tracker, query, MAX_TRACKER_WAIT_TIME)
       )
     )
 
@@ -427,4 +454,46 @@ export default class LookupResolver {
       }
     }
   }
+
+  private prepareHostsForQuery (hosts: string[], context: string): string[] {
+    if (hosts.length === 0) return []
+    const now = Date.now()
+    const ranked = this.hostReputation.rankHosts(hosts, now)
+    const available = ranked.filter((h) => h.backoffUntil <= now).map((h) => h.host)
+    if (available.length > 0) return available
+
+    const soonest = Math.min(...ranked.map((h) => h.backoffUntil))
+    const waitMs = Math.max(soonest - now, 0)
+    throw new Error(
+      `All ${context} hosts are backing off for approximately ${waitMs}ms due to repeated failures.`
+    )
+  }
+
+  private async lookupHostWithTracking (
+    host: string,
+    question: LookupQuestion,
+    timeout?: number
+  ): Promise<LookupAnswer> {
+    const startedAt = Date.now()
+    try {
+      const answer = await this.facilitator.lookup(host, question, timeout)
+      const latency = Date.now() - startedAt
+      const isValid =
+        typeof answer === 'object' &&
+        answer !== null &&
+        answer.type === 'output-list' &&
+        Array.isArray((answer).outputs)
+
+      if (isValid) {
+        this.hostReputation.recordSuccess(host, latency)
+      } else {
+        this.hostReputation.recordFailure(host, 'Invalid lookup response')
+      }
+
+      return answer
+    } catch (err) {
+      this.hostReputation.recordFailure(host, err)
+      throw err
+    }
+  }
 }

package/src/overlay-tools/__tests/LookupResolver.test.ts

@@ -1,6 +1,7 @@
 import LookupResolver, {
   HTTPSOverlayLookupFacilitator
 } from '../LookupResolver'
+import { getOverlayHostReputationTracker } from '../HostReputationTracker'
 import OverlayAdminTokenTemplate from '../../overlay-tools/OverlayAdminTokenTemplate'
 import { CompletedProtoWallet } from '../../auth/certificates/__tests/CompletedProtoWallet'
 import { PrivateKey } from '../../primitives/index'
@@ -37,8 +38,10 @@ const sampleBeef4 = new Transaction(
 ).toBEEF()
 
 describe('LookupResolver', () => {
+  const hostTracker = getOverlayHostReputationTracker()
   beforeEach(() => {
     mockFacilitator.lookup.mockReset()
+    hostTracker.reset()
   })
 
   it('should query the host and return the response when a single host is found via SLAP', async () => {
@@ -967,6 +970,123 @@ describe('LookupResolver', () => {
       'HTTPS facilitator can only use URLs that start with "https:"'
     )
   })
+
+  describe('Host reputation tracking', () => {
+    it('shares performance learnings across resolver instances and prefers low latency hosts', async () => {
+      const fastHost = 'https://fast.host'
+      const slowHost = 'https://slow.host'
+      const hosts = [slowHost, fastHost]
+      let fakeNow = 0
+      const nowSpy = jest.spyOn(Date, 'now').mockImplementation(() => fakeNow)
+
+      try {
+        mockFacilitator.lookup.mockImplementation(async (url: string) => {
+          if (url === slowHost) {
+            fakeNow += 80
+          } else if (url === fastHost) {
+            fakeNow += 5
+          }
+          return {
+            type: 'output-list',
+            outputs: [
+              {
+                beef: url === fastHost ? sampleBeef2 : sampleBeef1,
+                outputIndex: url === fastHost ? 2 : 1
+              }
+            ]
+          }
+        })
+
+        const resolverA = new LookupResolver({
+          facilitator: mockFacilitator,
+          hostOverrides: {
+            ls_latency: hosts
+          }
+        })
+        await resolverA.query({
+          service: 'ls_latency',
+          query: { attempt: 1 }
+        })
+
+        mockFacilitator.lookup.mockClear()
+
+        const resolverB = new LookupResolver({
+          facilitator: mockFacilitator,
+          hostOverrides: {
+            ls_latency: hosts
+          }
+        })
+        await resolverB.query({
+          service: 'ls_latency',
+          query: { attempt: 2 }
+        })
+
+        const orderedHosts = mockFacilitator.lookup.mock.calls.map((call) => call[0])
+        expect(orderedHosts).toEqual([fastHost, slowHost])
+      } finally {
+        nowSpy.mockRestore()
+      }
+    })
+
+    it('exponentially backs off consistently failing hosts to avoid repeated work', async () => {
+      const failingHost = 'https://offline.host'
+      const healthyHost = 'https://healthy.host'
+      let fakeNow = 0
+      const nowSpy = jest.spyOn(Date, 'now').mockImplementation(() => fakeNow)
+      const callLog: string[] = []
+      let failingCalls = 0
+
+      try {
+        mockFacilitator.lookup.mockImplementation(async (url: string) => {
+          callLog.push(url)
+          fakeNow += 5
+          if (url === failingHost) {
+            failingCalls += 1
+            throw new Error('offline')
+          }
+          return {
+            type: 'output-list',
+            outputs: [
+              {
+                beef: sampleBeef3,
+                outputIndex: 0
+              }
+            ]
+          }
+        })
+
+        const resolver = new LookupResolver({
+          facilitator: mockFacilitator,
+          hostOverrides: {
+            ls_backoff: [failingHost, healthyHost]
+          }
+        })
+
+        // First three attempts should contact both hosts (grace period)
+        await resolver.query({ service: 'ls_backoff', query: { attempt: 1 } })
+        fakeNow += 20
+        await resolver.query({ service: 'ls_backoff', query: { attempt: 2 } })
+        fakeNow += 20
+        await resolver.query({ service: 'ls_backoff', query: { attempt: 3 } })
+
+        expect(failingCalls).toBe(3)
+
+        // Immediately try again; failing host should now be in backoff and skipped
+        fakeNow += 20
+        await resolver.query({ service: 'ls_backoff', query: { attempt: 4 } })
+        expect(failingCalls).toBe(3)
+        const lastCall = callLog[callLog.length - 1]
+        expect(lastCall).toBe(healthyHost)
+
+        // Advance beyond the backoff window so the failing host is retried
+        fakeNow = 2000
+        await resolver.query({ service: 'ls_backoff', query: { attempt: 5 } })
+        expect(failingCalls).toBe(4)
+      } finally {
+        nowSpy.mockRestore()
+      }
+    })
+  })
   describe('LookupResolver Resiliency', () => {
     beforeEach(() => {
       mockFacilitator.lookup.mockReset()

package/src/primitives/BigNumber.ts

@@ -1,6 +1,19 @@
 // @ts-nocheck
 import ReductionContext from './ReductionContext.js'
 
+const BufferCtor =
+  typeof globalThis !== 'undefined' ? (globalThis as any).Buffer : undefined
+const CAN_USE_BUFFER =
+  BufferCtor != null && typeof BufferCtor.from === 'function'
+const HEX_CHAR_TO_VALUE = new Int8Array(256).fill(-1)
+for (let i = 0; i < 10; i++) {
+  HEX_CHAR_TO_VALUE[48 + i] = i // '0'-'9'
+}
+for (let i = 0; i < 6; i++) {
+  HEX_CHAR_TO_VALUE[65 + i] = 10 + i // 'A'-'F'
+  HEX_CHAR_TO_VALUE[97 + i] = 10 + i // 'a'-'f'
+}
+
 /**
  * JavaScript numbers are only precise up to 53 bits. Since Bitcoin relies on
  * 256-bit cryptography, this BigNumber class enables operations on larger
@@ -1059,31 +1072,30 @@ export default class BigNumber {
   static fromSm (bytes: number[], endian: 'big' | 'little' = 'big'): BigNumber {
     if (bytes.length === 0) return new BigNumber(0n)
 
+    const beBytes = bytes.slice()
+    if (endian === 'little') {
+      beBytes.reverse()
+    }
     let sign: 0 | 1 = 0
-    let hex = ''
+    if (beBytes.length > 0 && (beBytes[0] & 0x80) !== 0) {
+      sign = 1
+      beBytes[0] &= 0x7f
+    }
 
-    if (endian === 'little') {
-      const last = bytes.length - 1
-      let firstByte = bytes[last]
-      if ((firstByte & 0x80) !== 0) { sign = 1; firstByte &= 0x7f }
-      hex += (firstByte < 16 ? '0' : '') + firstByte.toString(16)
-      for (let i = last - 1; i >= 0; i--) {
-        const b = bytes[i]
-        hex += (b < 16 ? '0' : '') + b.toString(16)
-      }
+    let magnitude = 0n
+    if (CAN_USE_BUFFER) {
+      const hex = BufferCtor.from(beBytes).toString('hex') as string
+      magnitude = hex.length === 0 ? 0n : BigInt('0x' + hex)
     } else {
-      let firstByte = bytes[0]
-      if ((firstByte & 0x80) !== 0) { sign = 1; firstByte &= 0x7f }
-      hex += (firstByte < 16 ? '0' : '') + firstByte.toString(16)
-      for (let i = 1; i < bytes.length; i++) {
-        const b = bytes[i]
-        hex += (b < 16 ? '0' : '') + b.toString(16)
+      let hex = ''
+      for (const byte of beBytes) {
+        hex += byte < 16 ? '0' + byte.toString(16) : byte.toString(16)
       }
+      magnitude = hex.length === 0 ? 0n : BigInt('0x' + hex)
     }
 
-    const mag = hex === '' ? 0n : BigInt('0x' + hex)
     const r = new BigNumber(0n)
-    r._initializeState(mag, sign)
+    r._initializeState(magnitude, sign)
     return r
   }
 
@@ -1105,17 +1117,26 @@ export default class BigNumber {
     const byteLen = hex.length / 2
     const bytes = new Array(byteLen)
     for (let i = 0, j = 0; i < hex.length; i += 2) {
-      bytes[j++] = parseInt(hex.slice(i, i + 2), 16)
+      const high = HEX_CHAR_TO_VALUE[hex.charCodeAt(i)]
+      const low = HEX_CHAR_TO_VALUE[hex.charCodeAt(i + 1)]
+      bytes[j++] = ((high & 0xf) << 4) | (low & 0xf)
     }
 
+    let result: number[]
     if (this._sign === 1) {
-      if ((bytes[0] & 0x80) !== 0) bytes.unshift(0x80)
-      else bytes[0] |= 0x80
+      if ((bytes[0] & 0x80) !== 0) {
+        result = [0x80, ...bytes]
+      } else {
+        result = bytes.slice()
+        result[0] |= 0x80
+      }
     } else if ((bytes[0] & 0x80) !== 0) {
-      bytes.unshift(0x00)
+      result = [0x00, ...bytes]
+    } else {
+      result = bytes.slice()
     }
 
-    return endian === 'little' ? bytes.reverse() : bytes
+    return endian === 'little' ? result.reverse() : result
   }
 
   /**