@goodandready/dsh-subscriptions 0.6.70 → 0.6.72

package/lib/routes/status.js

@@ -1,3 +1,5 @@
+import { diagnoseVendorError } from '../doctor.js'
+import { INSTRUCTIONS } from '../instructions.js'
 import {
   getModelReasoningCapabilities,
   setSessionReasoning,
@@ -505,9 +507,13 @@ export function registerStatusRoutes(ctx, state) {
         blob = await store.loadBlob(ref)
         blob = await store.ensureFresh(provider, blob, ref)
       } catch (e) {
-        const latencyMs = Date.now() - t0
+        const latencyMs = Math.max(1, Date.now() - t0)
+        const latencyGrade = latencyMs < 250 ? 'good' : (latencyMs < 750 ? 'fair' : 'poor')
+        if (typeof store.rememberPing === "function") store.rememberPing(ref, { latencyMs, latencyGrade })
+        const docVerdict = diagnoseVendorError(e, provider, blob)
+        if (typeof store.rememberDoctorVerdict === "function") store.rememberDoctorVerdict(ref, docVerdict)
         history.add({ provider, ref, path: "/smoke", status: 401, ms: latencyMs })
-        writeJson(res, 502, { ok: false, provider, index, ref, latencyMs, error: { code: 'auth_failed', message: String(e && e.message || e) } })
+        writeJson(res, 502, { ok: false, provider, index, ref, latencyMs, latencyGrade, doctor: docVerdict, error: { code: 'auth_failed', message: String(e && e.message || e) } })
         return
       }
 
@@ -525,7 +531,10 @@ export function registerStatusRoutes(ctx, state) {
         }
         try {
           await vendor.check(blob, vendorConfig(provider, live()), probeFetch)
-          const latencyMs = Date.now() - t0
+          const latencyMs = Math.max(1, Date.now() - t0)
+          const latencyGrade = latencyMs < 250 ? 'good' : (latencyMs < 750 ? 'fair' : 'poor')
+          if (typeof store.rememberPing === "function") store.rememberPing(ref, { latencyMs, latencyGrade })
+          if (typeof store.rememberDoctorVerdict === "function") store.rememberDoctorVerdict(ref, null)
           history.add({ provider, ref, path: "/smoke", status: 200, ms: latencyMs })
           writeJson(res, 200, {
             ok: true,
@@ -533,6 +542,7 @@ export function registerStatusRoutes(ctx, state) {
             index,
             ref,
             latencyMs,
+            latencyGrade,
             label: pmL(blob.label),
             email: pmE(blob.email),
             quota: capturedQuota || info.quota || null,
@@ -540,7 +550,11 @@ export function registerStatusRoutes(ctx, state) {
           })
           return
         } catch (e) {
-          const latencyMs = Date.now() - t0
+          const latencyMs = Math.max(1, Date.now() - t0)
+          const latencyGrade = latencyMs < 250 ? 'good' : (latencyMs < 750 ? 'fair' : 'poor')
+          if (typeof store.rememberPing === "function") store.rememberPing(ref, { latencyMs, latencyGrade })
+          const docVerdict = diagnoseVendorError(e, provider, blob)
+          if (typeof store.rememberDoctorVerdict === "function") store.rememberDoctorVerdict(ref, docVerdict)
           history.add({ provider, ref, path: "/smoke", status: 502, ms: latencyMs })
           writeJson(res, 502, {
             ok: false,
@@ -548,13 +562,18 @@ export function registerStatusRoutes(ctx, state) {
             index,
             ref,
             latencyMs,
+            latencyGrade,
+            doctor: docVerdict,
             error: { code: e && e.code ? e.code : 'VENDOR', message: String(e && e.message || e).slice(0, 300) },
           })
           return
         }
       }
 
-      const latencyMs = Date.now() - t0
+      const latencyMs = Math.max(1, Date.now() - t0)
+      const latencyGrade = latencyMs < 250 ? 'good' : (latencyMs < 750 ? 'fair' : 'poor')
+      if (typeof store.rememberPing === "function") store.rememberPing(ref, { latencyMs, latencyGrade })
+      if (typeof store.rememberDoctorVerdict === "function") store.rememberDoctorVerdict(ref, null)
       history.add({ provider, ref, path: "/smoke", status: 200, ms: latencyMs })
       writeJson(res, 200, {
         ok: true,
@@ -562,6 +581,7 @@ export function registerStatusRoutes(ctx, state) {
         index,
         ref,
         latencyMs,
+        latencyGrade,
         label: pmL(blob.label),
         email: pmE(blob.email),
         quota: info.quota || null,
@@ -569,6 +589,80 @@ export function registerStatusRoutes(ctx, state) {
     }),
   }), 'dsh-subscriptions: /smoke')
 
+  // Issue #542: Latency RTT ping test endpoint
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: '/dsh-subscriptions/ping',
+    handler: safeJsonHandler(async (req, res) => {
+      if (req.method !== 'POST') {
+        writeJson(res, 405, { ok: false, error: { code: 'method', message: 'POST only' } })
+        return
+      }
+      if (!isTrustedSettingsRequest(req, ctx)) {
+        writeJson(res, 403, { ok: false, error: { code: 'forbidden', message: 'same-origin only' } })
+        return
+      }
+      let payload = {}
+      try { payload = JSON.parse((await readBody(req, 64 * 1024)).toString('utf8') || '{}') } catch { /* ignore ping payload */ }
+      const provider = payload.provider
+      const index = payload.index || 1
+      if (!provider || !isProvider(provider)) {
+        writeJson(res, 400, { ok: false, error: { code: 'invalid_provider', message: 'Provider required' } })
+        return
+      }
+      const ref = oauthRef(provider, index)
+      const info = await store.describeRef(ref)
+      if (!info.configured) {
+        writeJson(res, 400, { ok: false, error: { code: 'not_configured', message: 'Account is not configured' } })
+        return
+      }
+      const t0 = Date.now()
+      let latencyMs = 0
+      let latencyGrade = 'good'
+      try {
+        const blob = await store.loadBlob(ref)
+        const vendor = getVendor(provider)
+        const cfg = vendorConfig(provider, live())
+        const doFetch = (typeof fetchForRef === 'function' && fetchForRef(ref)) || fetch
+        if (typeof vendor.check === 'function') {
+          await vendor.check(blob, cfg, doFetch)
+        } else if (typeof vendor.listModels === 'function') {
+          await vendor.listModels(blob, cfg, doFetch)
+        }
+        latencyMs = Math.max(1, Date.now() - t0)
+        latencyGrade = latencyMs < 250 ? 'good' : (latencyMs < 750 ? 'fair' : 'poor')
+        if (typeof store.rememberPing === 'function') store.rememberPing(ref, { latencyMs, latencyGrade })
+        if (typeof store.rememberDoctorVerdict === 'function') store.rememberDoctorVerdict(ref, null)
+        writeJson(res, 200, { ok: true, provider, index, ref, latencyMs, latencyGrade })
+      } catch (err) {
+        latencyMs = Math.max(1, Date.now() - t0)
+        latencyGrade = 'poor'
+        const doc = diagnoseVendorError(err, provider)
+        if (typeof store.rememberPing === 'function') store.rememberPing(ref, { latencyMs, latencyGrade })
+        if (typeof store.rememberDoctorVerdict === 'function') store.rememberDoctorVerdict(ref, doc)
+        writeJson(res, 502, { ok: false, provider, index, ref, latencyMs, latencyGrade, doctor: doc, error: { message: String(err && err.message || err) } })
+      }
+    }),
+  }), 'dsh-subscriptions: /ping')
+
+  // Issue #540: Dynamic vendor instructions API
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: '/dsh-subscriptions/instructions',
+    handler: safeJsonHandler(async (req, res) => {
+      if (req.method !== 'GET') {
+        writeJson(res, 405, { ok: false, error: { code: 'method', message: 'GET only' } })
+        return
+      }
+      if (!isTrustedSettingsRequest(req, ctx)) {
+        writeJson(res, 403, { ok: false, error: { code: 'forbidden', message: 'same-origin only' } })
+        return
+      }
+      writeJson(res, 200, { ok: true, instructions: INSTRUCTIONS })
+    }),
+  }), 'dsh-subscriptions: /instructions')
+
+
   // #351: Quota and session alerts API
   ctx.effect(() => ctx.webServer.register({
     kind: 'exact',

package/lib/sentinel.js

@@ -0,0 +1,136 @@
+export const SOFT_FAILURE_PATTERNS = [
+  /currently (?:unable to process|unavailable|overloaded)/i,
+  /experiencing (?:exceptionally )?(?:high|heavy) (?:traffic|demand|load)/i,
+  /capacity (?:is )?temporarily (?:unavailable|limited|exceeded)/i,
+  /model is (?:currently )?(?:overloaded|busy)/i,
+  /due to high traffic/i,
+  /usage cap reached, switching to lower/i,
+  /you have reached your (?:current )?usage limit/i,
+  /server is currently under heavy load/i,
+  /i apologize, but (?:i am unable to assist|i cannot fulfill this request due to (?:high demand|capacity|server load))/i,
+  /our systems are experiencing heavy load/i,
+]
+
+const SUSPICIOUS_ROOTS = [
+  'i am',
+  'i apologize',
+  'we are',
+  'capacity',
+  'the model',
+  'due to',
+  'usage cap',
+  'you have reached',
+  'server is',
+  'our systems',
+]
+
+export function extractChunkText(chunk) {
+  if (!chunk) return ''
+  if (typeof chunk === 'string') return chunk
+  if (typeof chunk.delta === 'string') return chunk.delta
+  if (typeof chunk.content === 'string') return chunk.content
+  if (typeof chunk.text === 'string') return chunk.text
+  if (chunk.choices && Array.isArray(chunk.choices) && chunk.choices[0]) {
+    const c0 = chunk.choices[0]
+    if (c0.delta && typeof c0.delta.content === 'string') return c0.delta.content
+    if (typeof c0.text === 'string') return c0.text
+  }
+  if (chunk.candidates && Array.isArray(chunk.candidates) && chunk.candidates[0]) {
+    const cand = chunk.candidates[0]
+    if (cand.content && Array.isArray(cand.content.parts)) {
+      return cand.content.parts.map((p) => (p && typeof p.text === 'string' ? p.text : '')).join('')
+    }
+  }
+  return ''
+}
+
+export function detectSoftFailure(text) {
+  if (!text || typeof text !== 'string') return null
+  const cleaned = text.trim()
+  if (!cleaned) return null
+  for (const pattern of SOFT_FAILURE_PATTERNS) {
+    if (pattern.test(cleaned)) {
+      return pattern.toString()
+    }
+  }
+  return null
+}
+
+export function isCandidatePrefix(text) {
+  if (!text) return true
+  const low = text.toLowerCase().trim()
+  if (low.length < 3) return true
+  for (const root of SUSPICIOUS_ROOTS) {
+    if (root.startsWith(low) || low.startsWith(root)) {
+      return true
+    }
+  }
+  return false
+}
+
+export function createSoftFailureError(pattern, text) {
+  const preview = String(text || '').slice(0, 120).replace(/\s+/g, ' ')
+  const err = new Error(`Soft-failure detected: ${preview}`)
+  return Object.assign(err, {
+    name: 'SoftFailureError',
+    code: 'SOFT_FAILURE',
+    status: 429,
+    isSoftFailure: true,
+    degraded: true,
+    pattern,
+  })
+}
+
+export async function* withSoftFailureSentinel(stream, options = {}) {
+  const enabled = options.enabled !== false
+  if (!enabled || !stream) {
+    if (stream) {
+      for await (const chunk of stream) {
+        yield chunk
+      }
+    }
+    return
+  }
+
+  const bufferLimitChars = Number(options.bufferLimitChars) || 250
+  const maxBufferChunks = Number(options.maxBufferChunks) || 4
+
+  const bufferedChunks = []
+  let accumulatedText = ''
+  let flushed = false
+
+  for await (const chunk of stream) {
+    if (flushed) {
+      yield chunk
+      continue
+    }
+
+    bufferedChunks.push(chunk)
+    accumulatedText += extractChunkText(chunk)
+
+    const matched = detectSoftFailure(accumulatedText)
+    if (matched) {
+      throw createSoftFailureError(matched, accumulatedText)
+    }
+
+    const candidate = isCandidatePrefix(accumulatedText)
+    if (!candidate || accumulatedText.length >= bufferLimitChars || bufferedChunks.length >= maxBufferChunks) {
+      flushed = true
+      for (const b of bufferedChunks) {
+        yield b
+      }
+      bufferedChunks.length = 0
+    }
+  }
+
+  if (!flushed) {
+    const matched = detectSoftFailure(accumulatedText)
+    if (matched) {
+      throw createSoftFailureError(matched, accumulatedText)
+    }
+    for (const b of bufferedChunks) {
+      yield b
+    }
+    bufferedChunks.length = 0
+  }
+}

package/lib/stream-rotate.js

@@ -1,10 +1,12 @@
 import { defaultConcurrencyQueue } from './concurrency-queue.js'
 import { recordSessionCacheUsage } from './session-pin.js'
 import { pickAccountWithTrace, markCooldown, isSwitchableError, isRegionError, modelFamily } from './rotate.js'
-import { putInQuarantine, REASON_RATE_LIMIT, REASON_HARD_LIMIT, REASON_REVOKED } from './quarantine.js'
+import { putInQuarantine, REASON_RATE_LIMIT, REASON_HARD_LIMIT, REASON_REVOKED, REASON_DEGRADED } from './quarantine.js'
 import { notifySessionExpired, notifyQuarantineEntered } from './alerts.js'
 import { withRunawayGuard } from './runaway-guard.js'
 import { defaultErrorBuffer } from './error-buffer.js'
+import { withSoftFailureSentinel } from './sentinel.js'
+import { resolveAccurateReset } from './reset-clock.js'
 
 export async function* streamWithRotation({
   accounts,
@@ -55,6 +57,7 @@ export async function* streamWithRotation({
       autoPacing: options && (options.autoPacing !== undefined ? options.autoPacing : (options.config && options.config.autoPacing)),
       drainBeforeReset: options && (options.drainBeforeReset !== undefined ? options.drainBeforeReset : (options.config && options.config.drainBeforeReset)),
       drainSafetyPercent: options && (options.drainSafetyPercent !== undefined ? options.drainSafetyPercent : (options.config && options.config.drainSafetyPercent)),
+      rotationStrategy: options && (options.rotationStrategy !== undefined ? options.rotationStrategy : (options.config && options.config.rotationStrategy)),
     })
     const account = pickRes.account
 
@@ -147,7 +150,10 @@ export async function* streamWithRotation({
           streamOnce(account, { ...options, _runawayState: runawayState }),
           runawayOpts
         )
-        for await (const chunk of guardedStream) {
+        const sentineledStream = withSoftFailureSentinel(guardedStream, {
+          enabled: (options?.enableSoftFailureSentinel !== false) && (options?.config?.enableSoftFailureSentinel !== false),
+        })
+        for await (const chunk of sentineledStream) {
           firstChunkDelivered = true
           const cachedTokens = Number(
             (chunk && chunk.usage && (chunk.usage.cacheReadTokens || chunk.usage.cachedTokens)) ||
@@ -195,18 +201,24 @@ export async function* streamWithRotation({
         currentAttempt.status = 'switched'
         currentAttempt.switchError = err ? (err.code || err.status || err.message) : 'switchable'
 
-        // Move slot to cooldown and quarantine (#172)
-        const cooled = markCooldown(account, nowMs(), cooldownMs, modelFamily(options && options.provider, options && options.model))
+        const accurateUntil = resolveAccurateReset(err, account, options && options.provider, nowMs())
+        const isSoft = Boolean(err && (err.code === 'SOFT_FAILURE' || err.isSoftFailure))
+
+        // Move slot to cooldown and quarantine (#172, #538)
+        const effectiveCooldownMs = isSoft ? Math.min(cooldownMs, 5 * 60 * 1000) : cooldownMs
+        const cooled = markCooldown(account, nowMs(), effectiveCooldownMs, modelFamily(options && options.provider, options && options.model), accurateUntil)
         account.cooldownUntil = cooled.cooldownUntil
         if (cooled.cooldownFamilies) account.cooldownFamilies = cooled.cooldownFamilies
 
         const status = Number(err && (err.status || err.statusCode) || 0)
         const code = String(err && err.code || '')
-        const reason = (status === 401 || code === 'AUTH' || code === 'TOKEN_REVOKED')
-          ? REASON_REVOKED
-          : (status === 403 || code === 'HARD_LIMIT' || code === 'LICENSE_REQUIRED')
-            ? REASON_HARD_LIMIT
-            : REASON_RATE_LIMIT
+        const reason = isSoft
+          ? REASON_DEGRADED
+          : (status === 401 || code === 'AUTH' || code === 'TOKEN_REVOKED')
+            ? REASON_REVOKED
+            : (status === 403 || code === 'HARD_LIMIT' || code === 'LICENSE_REQUIRED')
+              ? REASON_HARD_LIMIT
+              : REASON_RATE_LIMIT
 
         if (err && err.isWarmupProbeFailure && err.warmupAccount) {
           account.quarantineUntil = err.warmupAccount.quarantineUntil
@@ -214,7 +226,7 @@ export async function* streamWithRotation({
           account.quarantineAttempts = err.warmupAccount.quarantineAttempts
           account.status = err.warmupAccount.status
         } else {
-          const quarantined = putInQuarantine(account, reason, nowMs())
+          const quarantined = putInQuarantine(account, reason, nowMs(), accurateUntil)
           account.quarantineUntil = quarantined.quarantineUntil
           account.quarantineReason = quarantined.quarantineReason
           account.quarantineAttempts = (account.quarantineAttempts || 0) + 1
@@ -222,6 +234,10 @@ export async function* streamWithRotation({
 
         currentAttempt.quarantineReason = account.quarantineReason || reason
         currentAttempt.quarantineUntil = account.quarantineUntil
+        if (accurateUntil) {
+          currentAttempt.accurateReset = true
+          currentAttempt.resetAt = accurateUntil
+        }
 
         try {
           notifyQuarantineEntered({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-subscriptions",
-  "version": "0.6.70",
+  "version": "0.6.72",
   "description": "Use ChatGPT Codex, Claude, Grok, Antigravity, Kimi, GLM, Cursor, Kiro, Copilot, Qwen, ERNIE, Spark, JetBrains, Perplexity, Replit, and Cody subscriptions as DeepSeek Harness LLM providers via OAuth.",
   "license": "MIT",
   "type": "module",