@goodandready/dsh-subscriptions 0.6.60 → 0.6.62

package/lib/stream-rotate.js

@@ -1,4 +1,5 @@
-import { pickAccount, markCooldown, isSwitchableError, isRegionError, modelFamily } from './rotate.js'
+import { defaultConcurrencyQueue } from './concurrency-queue.js'
+import { pickAccountWithTrace, markCooldown, isSwitchableError, isRegionError, modelFamily } from './rotate.js'
 import { putInQuarantine, REASON_RATE_LIMIT, REASON_HARD_LIMIT, REASON_REVOKED } from './quarantine.js'
 import { notifySessionExpired, notifyQuarantineEntered } from './alerts.js'
 import { withRunawayGuard } from './runaway-guard.js'
@@ -11,17 +12,39 @@ export async function* streamWithRotation({
   streamOnce,
   options,
   onCooldown,
+  onDecisionTrace,
   cascadeFallback, // #349: optional cross-vendor cascading fallback generator
   offlineFallback, // #174: optional fallback generator if all accounts exhausted
   webhookUrl,
   fetchImpl = fetch,
+  concurrencyQueue = defaultConcurrencyQueue,
 }) {
   const pool = (accounts || []).map((account) => ({ ...account }))
   let lastError = null
   const tried = new Set()
 
+  const decisionTrace = {
+    sessionId: options && options.sessionId,
+    provider: options && options.provider,
+    model: options && options.model,
+    attempts: [],
+    finalOutcome: null,
+    chosenRef: null,
+  }
+
+  const syncTrace = (outcome, extra = {}) => {
+    decisionTrace.finalOutcome = outcome
+    Object.assign(decisionTrace, extra)
+    if (options && options._decisionTrace) {
+      Object.assign(options._decisionTrace, decisionTrace)
+    }
+    if (typeof onDecisionTrace === 'function') {
+      try { onDecisionTrace(decisionTrace) } catch { /* ignore trace callback error */ }
+    }
+  }
+
   while (true) {
-    const account = pickAccount(pool, nowMs(), {
+    const pickRes = pickAccountWithTrace(pool, nowMs(), {
       switchAtRemaining,
       family: modelFamily(options && options.provider, options && options.model),
       sessionId: options && options.sessionId,
@@ -31,30 +54,84 @@ export async function* streamWithRotation({
       drainBeforeReset: options && (options.drainBeforeReset !== undefined ? options.drainBeforeReset : (options.config && options.config.drainBeforeReset)),
       drainSafetyPercent: options && (options.drainSafetyPercent !== undefined ? options.drainSafetyPercent : (options.config && options.config.drainSafetyPercent)),
     })
+    const account = pickRes.account
 
     if (!account || tried.has(account.ref || account.id)) {
       if (cascadeFallback) {
-        // #349 Cascading Cross-Vendor Fallback
+        // #349 Cascading Cross-Vendor Fallback (#517 trace)
+        decisionTrace.attempts.push({
+          attempt: decisionTrace.attempts.length + 1,
+          kind: 'cascade_fallback',
+          reason: 'primary_pool_exhausted',
+          lastError: lastError ? (lastError.code || lastError.message) : null,
+        })
+        syncTrace('cascade_fallback')
         yield* cascadeFallback(options, lastError)
         return
       }
       if (offlineFallback) {
-        // #174 Local Mock Server Offline Fallback
+        // #174 Local Mock Server Offline Fallback (#517 trace)
+        decisionTrace.attempts.push({
+          attempt: decisionTrace.attempts.length + 1,
+          kind: 'offline_fallback',
+          reason: 'all_accounts_exhausted',
+          lastError: lastError ? (lastError.code || lastError.message) : null,
+        })
+        syncTrace('offline_fallback')
         yield* offlineFallback(options, lastError)
         return
       }
+      syncTrace('exhausted')
       if (lastError) throw lastError
       const err = /** @type {Error & { code?: string }} */ (new Error('no usable subscription account for this provider'))
       err.code = 'RATE_LIMIT'
       throw err
     }
 
+    const currentAttempt = {
+      attempt: decisionTrace.attempts.length + 1,
+      ref: account.ref || account.id,
+      decisionReason: pickRes.trace.decisionReason,
+      tier: pickRes.trace.tier,
+      selectionMetrics: pickRes.trace.selectionMetrics,
+      evaluations: pickRes.trace.evaluations,
+      status: 'pending',
+    }
+    decisionTrace.attempts.push(currentAttempt)
+    decisionTrace.chosenRef = account.ref || account.id
+
     tried.add(account.ref || account.id)
 
+    const queueKey = account.ref || account.id || 'default'
+    const queueConfig = {
+      signal: options && options.signal,
+      timeoutMs: options && (options.queueTimeoutMs !== undefined ? options.queueTimeoutMs : (options.config && options.config.queueTimeoutMs)),
+      maxConcurrency: options && (options.accountMaxConcurrency !== undefined ? options.accountMaxConcurrency : (options.config && options.config.accountMaxConcurrency)),
+      maxQueueSize: options && (options.maxQueueSize !== undefined ? options.maxQueueSize : (options.config && options.config.maxQueueSize)),
+      enabled: options && (options.enableQueue !== undefined ? options.enableQueue : (options.config && options.config.enableQueue !== undefined ? options.config.enableQueue : true)),
+    }
+
+    let releaseSlot = null
+    try {
+      releaseSlot = await concurrencyQueue.acquire(queueKey, queueConfig)
+    } catch (queueErr) {
+      if (queueErr && (queueErr.code === 'ABORT_ERR' || queueErr.name === 'AbortError')) {
+        currentAttempt.status = 'aborted'
+        currentAttempt.error = queueErr.message
+        syncTrace('aborted')
+        throw queueErr
+      }
+      currentAttempt.status = queueErr && queueErr.code === 'QUEUE_OVERFLOW' ? 'queue_overflow' : 'queue_timeout'
+      currentAttempt.error = queueErr ? queueErr.message : String(queueErr)
+      lastError = queueErr
+      continue
+    }
+
     let firstChunkDelivered = false
     let regionRetries = 0
 
-    while (true) {
+    try {
+      while (true) {
       try {
         const runawayState = options && options._runawayState ? options._runawayState : { tripped: false, reason: null }
         const runawayOpts = {
@@ -72,6 +149,8 @@ export async function* streamWithRotation({
           firstChunkDelivered = true
           yield chunk
         }
+        currentAttempt.status = 'success'
+        syncTrace('success', { chosenRef: account.ref || account.id })
         return
       } catch (err) {
         lastError = err
@@ -82,12 +161,21 @@ export async function* streamWithRotation({
         // Transient region 400 error: retry same account up to 2 times before rotating (#391 / GH #9)
         if (isRegionError(err) && regionRetries < 2 && (!options || !options.signal || !options.signal.aborted)) {
           regionRetries++
+          currentAttempt.regionRetries = regionRetries
           await new Promise((r) => setTimeout(r, 400 * regionRetries))
           if (options && options.signal && options.signal.aborted) throw err
           continue
         }
 
-        if (!isSwitchableError(err)) throw err
+        if (!isSwitchableError(err)) {
+          currentAttempt.status = 'fatal_error'
+          currentAttempt.error = err ? (err.message || String(err)) : 'unknown'
+          syncTrace('fatal_error')
+          throw err
+        }
+
+        currentAttempt.status = 'switched'
+        currentAttempt.switchError = err ? (err.code || err.status || err.message) : 'switchable'
 
         // Move slot to cooldown and quarantine (#172)
         const cooled = markCooldown(account, nowMs(), cooldownMs, modelFamily(options && options.provider, options && options.model))
@@ -114,6 +202,9 @@ export async function* streamWithRotation({
           account.quarantineAttempts = (account.quarantineAttempts || 0) + 1
         }
 
+        currentAttempt.quarantineReason = account.quarantineReason || reason
+        currentAttempt.quarantineUntil = account.quarantineUntil
+
         try {
           notifyQuarantineEntered({
             ref: account.ref,
@@ -144,5 +235,10 @@ export async function* streamWithRotation({
         break
       }
     }
+    } finally {
+      if (typeof releaseSlot === 'function') {
+        releaseSlot()
+      }
+    }
   }
 }

package/lib/subscriptions.js

@@ -1,5 +1,5 @@
 import { isProvider } from "./refs.js"
-import { pickAccount, markCooldown, isSwitchableError, modelFamily } from "./rotate.js"
+import { pickAccountWithTrace, markCooldown, isSwitchableError, modelFamily } from "./rotate.js"
 import { quotaSnapshot } from "./ratelimit.js"
 import { pickFetch } from "./proxy.js"
 import { fetchWithTimeout } from "./http.js"
@@ -118,14 +118,16 @@ export function createSubscriptionsService(deps) {
       // copy pool for rotation
       const pool = list.map((a) => ({ ...a }))
       const tried = new Set()
+      const attempts = []
       let lastError = null
       while (true) {
-        const account = pickAccount(pool, Date.now(), {
+        const pickRes = pickAccountWithTrace(pool, Date.now(), {
           switchAtRemaining: thr,
           family: modelFamily(provider, body && body.model),
           drainBeforeReset: typeof deps.drainBeforeReset === 'function' ? deps.drainBeforeReset() : deps.drainBeforeReset,
           drainSafetyPercent: typeof deps.drainSafetyPercent === 'function' ? deps.drainSafetyPercent() : deps.drainSafetyPercent,
         })
+        const account = pickRes.account
         if (!account) {
           if (lastError) throw lastError
           const err = new Error("no usable subscription account for this provider")
@@ -139,6 +141,16 @@ export function createSubscriptionsService(deps) {
           throw err
         }
         tried.add(account.ref)
+        const currentAttempt = {
+          attempt: attempts.length + 1,
+          ref: account.ref,
+          decisionReason: pickRes.trace.decisionReason,
+          tier: pickRes.trace.tier,
+          selectionMetrics: pickRes.trace.selectionMetrics,
+          evaluations: pickRes.trace.evaluations,
+          status: 'pending',
+        }
+        attempts.push(currentAttempt)
         try {
           const blob = await deps.ensureFresh(provider, await deps.loadBlob(account.ref), account.ref)
           const cfg = deps.vendorConfig(provider)
@@ -235,7 +247,8 @@ export function createSubscriptionsService(deps) {
             else err.code = "VENDOR"
             throw err
           }
-          // #65: record in the request and cost history.
+          currentAttempt.status = 'success'
+          // #65: record in the request and cost history with decision trace (#517)
           if (typeof deps.recordHistory === "function") {
             try {
               deps.recordHistory({
@@ -247,11 +260,18 @@ export function createSubscriptionsService(deps) {
                 status: res.status || 200,
                 ms: Date.now() - t0,
                 kind: "request",
+                decisionTrace: {
+                  attempts,
+                  finalOutcome: 'success',
+                  chosenRef: account.ref,
+                },
               })
             } catch { /* ignore subscriptions error */ }
           }
           return res
         } catch (err) {
+          currentAttempt.status = 'failed'
+          currentAttempt.switchError = err ? (err.code || err.status || err.message) : 'switchable'
           lastError = err
           if (!isSwitchableError(err)) throw err
           const cooled = markCooldown(account, Date.now(), cooldownMs, modelFamily(provider, body && body.model))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-subscriptions",
-  "version": "0.6.60",
+  "version": "0.6.62",
   "description": "Use ChatGPT Codex, Claude, Grok, Antigravity, Kimi, GLM, Cursor, Kiro, Copilot, Qwen, ERNIE, Spark, JetBrains, Perplexity, Replit, and Cody subscriptions as DeepSeek Harness LLM providers via OAuth.",
   "license": "MIT",
   "type": "module",