@ljwei-stak/dsh-model-router 0.16.0 → 0.16.1

package/.dsh-plugin/shared/adaptive-feedback.mjs

@@ -6,11 +6,15 @@
  */
 export const FEEDBACK_POLICY_VERSION = 'subjective-utility-v1'
 export const FEEDBACK_RUN_WINDOW = 200
+/** Cache invalidation cadence; weights themselves still use exact evaluatedAt. */
+export const FEEDBACK_DECAY_REFRESH_MS = 3_600_000
 
 const DAY_MS = 86_400_000
 const finite = value => typeof value === 'number' && Number.isFinite(value)
 const clean = value => typeof value === 'string' ? value.trim() : ''
 const routeKey = (provider, model) => `${provider}\u0000${model}`
+const record = value => value !== null && typeof value === 'object' && !Array.isArray(value)
+const own = (value, key) => record(value) && Object.hasOwn(value, key) ? value[key] : undefined
 // Code-unit ordering avoids machine/browser locale changing cache identity.
 const compare = (left, right) => left === right ? 0 : left < right ? -1 : 1
 
@@ -50,7 +54,7 @@ function completedAt(run, item) {
   // Stored packages have their own completion time. A historical run may have
   // only a run-level completion timestamp, then its creation timestamp.
   // Invalid explicitly supplied timestamps are not repaired by a fallback.
-  for (const candidate of [item.finishedAt, run.finishedAt, run.createdAt]) {
+  for (const candidate of [own(item, 'finishedAt'), own(run, 'finishedAt'), own(run, 'createdAt')]) {
     if (candidate === undefined || candidate === null) continue
     return finite(candidate) && candidate >= 0 ? candidate : null
   }
@@ -58,29 +62,37 @@ function completedAt(run, item) {
 }
 
 function evidenceOf(run, item, options) {
-  const provider = clean(item.provider)
-  const model = clean(item.model)
-  if (!provider || !model || provider.includes('\u0000') || model.includes('\u0000') || item.ran !== true || item.ok !== true
+  const reject = reason => ({ event: null, reason })
+  const provider = clean(own(item, 'provider'))
+  const model = clean(own(item, 'model'))
+  if (!provider || !model || provider.includes('\u0000') || model.includes('\u0000')) return reject('invalid-route')
+  if (own(item, 'ran') !== true || own(item, 'ok') !== true
     || item.blocked === true || item.waiting === true || item.paused === true
-    || item.cancelled === true || (item.status !== undefined && !['succeeded', 'fallback'].includes(item.status))) return null
-  if (item.rating !== 1 && item.rating !== -1) return null
+    || item.cancelled === true || (item.status !== undefined && !['succeeded', 'fallback'].includes(item.status))) return reject('not-successful')
+  const rating = own(item, 'rating')
+  if (rating !== 1 && rating !== -1) return reject(rating === undefined || rating === null || rating === 0 ? 'unrated' : 'invalid-rating')
   // A CLI default or a different actual model must not train the recommended
   // model. Aliases are not silently equated; a future verified adapter may do so.
-  if (item.channel === 'official-cli' && clean(item.actualModel) !== model) return null
+  if (item.channel === 'official-cli' && clean(own(item, 'actualModel')) !== model) return reject('cli-model-unverified')
   const finishedAt = completedAt(run, item)
-  if (finishedAt === null || finishedAt > options.now || finishedAt <= options.resetAt) return null
-  if (item.ratedAt !== undefined && item.ratedAt !== null
-    && (!finite(item.ratedAt) || item.ratedAt < 0 || item.ratedAt > options.now)) return null
-  return {
-    runId: run.id,
-    packageId: item.id,
+  if (finishedAt === null) return reject('invalid-time')
+  if (finishedAt > options.now) return reject('future-time')
+  if (finishedAt <= options.resetAt) return reject('before-reset')
+  const suppliedRatedAt = own(item, 'ratedAt')
+  if (suppliedRatedAt !== undefined && suppliedRatedAt !== null) {
+    if (!finite(suppliedRatedAt) || suppliedRatedAt < finishedAt) return reject('invalid-time')
+    if (suppliedRatedAt > options.now) return reject('future-time')
+  }
+  return { reason: null, event: {
+    runId: own(run, 'id'),
+    packageId: own(item, 'id'),
     provider,
     model,
-    domain: clean(item.type) || 'general',
-    rating: item.rating,
+    domain: clean(own(item, 'type')) || 'general',
+    rating,
     finishedAt,
-    ratedAt: item.ratedAt ?? finishedAt,
-  }
+    ratedAt: suppliedRatedAt ?? finishedAt,
+  } }
 }
 
 /**
@@ -112,35 +124,72 @@ export function buildFeedbackProfile(runs, options = {}) {
     resetAt: settings.resetAt,
     window: 'last-200-runs',
     adjustments: {},
+    cellStats: {},
+    exclusionReasons: {},
+    evaluatedAt: settings.now,
+    decayRefreshMs: FEEDBACK_DECAY_REFRESH_MS,
+    decayEpoch: null,
+    evidenceRevision: '',
+    revisionSemantics: 'evidence-settings-and-hourly-decay-cache-identity-not-security-hash',
     revision: '',
   }
+  const exclude = reason => {
+    profile.ignoredCount += 1
+    profile.exclusionReasons[reason] = (profile.exclusionReasons[reason] ?? 0) + 1
+  }
   const latest = new Map()
   if (settings.enabled) {
     for (const run of (Array.isArray(runs) ? runs : []).slice(-FEEDBACK_RUN_WINDOW)) {
-      for (const item of Array.isArray(run?.packages) ? run.packages : []) {
-        if (!item || typeof item !== 'object' || !clean(run?.id) || !clean(item.id)) {
-          profile.ignoredCount += 1
+      if (!record(run) || (run.packages !== undefined && !Array.isArray(own(run, 'packages')))) {
+        exclude('invalid-record')
+        continue
+      }
+      for (const item of Array.isArray(own(run, 'packages')) ? run.packages : []) {
+        if (!record(item)) {
+          exclude('invalid-record')
+          continue
+        }
+        if (!clean(own(run, 'id')) || !clean(own(item, 'id'))) {
+          exclude('missing-identity')
           continue
         }
         const identity = JSON.stringify([run.id, item.id])
-        if (latest.has(identity)) profile.ignoredCount += 1
+        if (latest.has(identity)) exclude('superseded')
         latest.set(identity, { run, item })
       }
     }
   }
-  const events = [...latest.values()].map(({ run, item }) => evidenceOf(run, item, settings))
-    .filter(event => {
-      if (!event) profile.ignoredCount += 1
-      return event !== null
-    }).sort((left, right) => compare(JSON.stringify([left.runId, left.packageId]), JSON.stringify([right.runId, right.packageId])))
+  const events = []
+  for (const { run, item } of latest.values()) {
+    const result = evidenceOf(run, item, settings)
+    if (result.event) events.push(result.event)
+    else exclude(result.reason)
+  }
+  events.sort((left, right) => compare(JSON.stringify([left.runId, left.packageId]), JSON.stringify([right.runId, right.packageId])))
   const cells = new Map()
   for (const event of events) {
     const weight = 2 ** (-(settings.now - event.finishedAt) / (settings.halfLifeDays * DAY_MS))
     const key = routeKey(event.provider, event.model)
     const domains = cells.get(key) ?? new Map()
-    const cell = domains.get(event.domain) ?? { sum: 0, weight: 0 }
+    const cell = domains.get(event.domain) ?? { sum: 0, weight: 0, count: 0, positive: 0, negative: 0,
+      weightSquaredSum: 0, maxWeight: 0, scaledWeightSum: 0, scaledWeightSquares: 0 }
     cell.sum += weight * event.rating
     cell.weight += weight
+    cell.count += 1
+    if (event.rating === 1) cell.positive += 1
+    else cell.negative += 1
+    cell.weightSquaredSum += weight * weight
+    // Rescaling prevents old tiny weights squared from underflowing Kish ESS.
+    if (weight > cell.maxWeight) {
+      const scale = cell.maxWeight / weight
+      cell.scaledWeightSum = cell.scaledWeightSum * scale + 1
+      cell.scaledWeightSquares = cell.scaledWeightSquares * scale * scale + 1
+      cell.maxWeight = weight
+    } else if (weight > 0) {
+      const scaled = weight / cell.maxWeight
+      cell.scaledWeightSum += scaled
+      cell.scaledWeightSquares += scaled * scaled
+    }
     domains.set(event.domain, cell)
     cells.set(key, domains)
     profile.feedbackCount += 1
@@ -149,11 +198,31 @@ export function buildFeedbackProfile(runs, options = {}) {
   }
   // Object.fromEntries safely creates own keys even for opaque identifiers
   // such as '__proto__'; no task text or answer is copied into this profile.
-  profile.adjustments = Object.fromEntries([...cells.entries()].sort(([left], [right]) => compare(left, right))
-    .map(([key, domains]) => [key, Object.fromEntries([...domains.entries()].sort(([left], [right]) => compare(left, right))
-      .map(([domain, cell]) => [domain, Math.max(-settings.maxAdjustment, Math.min(settings.maxAdjustment,
-        settings.maxAdjustment * cell.sum / (settings.priorWeight + cell.weight)))]))]))
-  profile.revision = cacheRevision({ settings, events })
+  const sortedCells = [...cells.entries()].sort(([left], [right]) => compare(left, right))
+    .map(([key, domains]) => [key, [...domains.entries()].sort(([left], [right]) => compare(left, right))])
+  const adjustment = cell => Math.max(-settings.maxAdjustment, Math.min(settings.maxAdjustment,
+    settings.maxAdjustment * cell.sum / (settings.priorWeight + cell.weight)))
+  profile.adjustments = Object.fromEntries(sortedCells.map(([key, domains]) => [key,
+    Object.fromEntries(domains.map(([domain, cell]) => [domain, adjustment(cell)]))]))
+  profile.cellStats = Object.fromEntries(sortedCells.map(([key, domains]) => [key,
+    Object.fromEntries(domains.map(([domain, cell]) => [domain, {
+      feedbackCount: cell.count, positiveCount: cell.positive, negativeCount: cell.negative,
+      weightedRatingSum: cell.sum, effectiveWeight: cell.weight, weightSquaredSum: cell.weightSquaredSum,
+      // Keep machine output within the mathematical bound despite ULP rounding.
+      effectiveSampleSize: cell.scaledWeightSquares > 0
+        ? Math.min(cell.count, Math.max(0, cell.scaledWeightSum ** 2 / cell.scaledWeightSquares)) : 0,
+      shrinkage: cell.weight / (settings.priorWeight + cell.weight), adjustment: adjustment(cell),
+      semantics: 'observed-feedback-weight-diagnostics-not-confidence-or-correctness',
+    }]))]))
+  const { now: evaluatedAt, ...evidenceSettings } = settings
+  const identityEvents = events.map(({ ratedAt, ...event }) => event)
+  profile.evidenceRevision = cacheRevision({ settings: evidenceSettings, events: identityEvents })
+  profile.decayEpoch = events.length > 0 && settings.maxAdjustment > 0
+    ? Math.floor(evaluatedAt / FEEDBACK_DECAY_REFRESH_MS) : null
+  // A decision cache identity, intentionally not the hash of every continuously
+  // changing weight. Empty evidence and audit-only re-rating do not erase plans;
+  // active preferences invalidate once/hour so long-term decay keeps updating.
+  profile.revision = cacheRevision({ evidenceRevision: profile.evidenceRevision, decayEpoch: profile.decayEpoch })
   return profile
 }
 
@@ -163,18 +232,25 @@ export function buildFeedbackProfile(runs, options = {}) {
  * profile. The planner must consume the domain-specific adjustment explicitly.
  */
 export function applyFeedbackProfile(routes, profile) {
-  if (!Array.isArray(routes) || profile?.enabled !== true || !profile.adjustments
-    || typeof profile.adjustments !== 'object' || Array.isArray(profile.adjustments)) return routes
+  if (!Array.isArray(routes) || !record(profile) || typeof profile.enabled !== 'boolean') return routes
   return routes.map(route => {
-    if (!route || typeof route !== 'object') return route
+    if (!record(route)) return route
+    let base = route
+    if (Object.hasOwn(route, 'preferenceAdjustments') || Object.hasOwn(route, 'preferenceAdjustmentMeta')) {
+      base = { ...route }
+      delete base.preferenceAdjustments
+      delete base.preferenceAdjustmentMeta
+    }
+    if (profile.enabled !== true || !record(profile.adjustments)) return base
     const key = routeKey(clean(route.provider), clean(route.model))
-    if (!Object.hasOwn(profile.adjustments, key)) return route
+    if (!Object.hasOwn(profile.adjustments, key)) return base
     const value = profile.adjustments[key]
-    if (!value || typeof value !== 'object' || Array.isArray(value)) return route
-    const adjustments = Object.fromEntries(Object.entries(value).filter(([, bias]) => finite(bias) && Math.abs(bias) <= 0.1))
-    if (!Object.keys(adjustments).length) return route
+    if (!record(value)) return base
+    const limit = finite(profile.maxAdjustment) && profile.maxAdjustment >= 0 && profile.maxAdjustment <= 0.1 ? profile.maxAdjustment : 0.1
+    const adjustments = Object.fromEntries(Object.entries(value).filter(([, bias]) => finite(bias) && Math.abs(bias) <= limit))
+    if (!Object.keys(adjustments).length) return base
     return {
-      ...route,
+      ...base,
       preferenceAdjustments: adjustments,
       preferenceAdjustmentMeta: {
         policyVersion: profile.policyVersion,
@@ -186,6 +262,10 @@ export function applyFeedbackProfile(routes, profile) {
         halfLifeDays: profile.halfLifeDays,
         maxAdjustment: profile.maxAdjustment,
         window: profile.window,
+        cellStats: Object.hasOwn(profile.cellStats ?? {}, key) ? profile.cellStats[key] : {},
+        evaluatedAt: profile.evaluatedAt,
+        decayRefreshMs: profile.decayRefreshMs,
+        revisionSemantics: profile.revisionSemantics,
       },
     }
   })

package/.dsh-plugin/shared/dynamic-data-view.mjs

@@ -2,7 +2,53 @@
 const DEFAULT_TTL_MS = 86_400_000
 const currentTime = value => typeof value === 'function' ? value() : value ?? Date.now()
 const exactRouteKey = route => `${String(route?.provider ?? '')}\0${String(route?.model ?? '')}`
-const publicError = value => value === 'snapshot-rollback-rejected' ? value : value ? 'refresh-failed' : ''
+const publicErrors = new Set(['snapshot-rollback-rejected', 'refresh-failed', 'storage-failed', 'refresh-interrupted'])
+const publicError = value => publicErrors.has(value) ? value : value ? 'refresh-failed' : ''
+
+/** Same public URL contract on Host and Client; validates syntax, not DNS routing. */
+export function publicDynamicEndpoint(value) {
+  if (typeof value !== 'string' || value.length > 2048) throw new Error('invalid-public-url')
+  const url = new URL(value)
+  const host = url.hostname.toLowerCase()
+  if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash
+    || (url.port && url.port !== '443') || host.startsWith('[')
+    || !/^(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}$/u.test(host)
+    || /(?:^|\.)(?:localhost|local|internal|lan|home|test|invalid)$/u.test(host)) throw new Error('invalid-public-url')
+  return url.toString()
+}
+
+const publicEndpointMetadata = value => { try { return publicDynamicEndpoint(value) } catch { return null } }
+
+/** Mask stopped or unmatched sources; an old A snapshot must never impersonate new B. */
+export function projectDynamicData(data, { enabled = false, liveBenchEndpoint = 'https://livebench.ai', pricingSnapshotEndpoint = '' } = {}) {
+  if (enabled !== true) return { liveBench: null, pricing: null, status: {}, revision: '' }
+  const result = { liveBench: data?.liveBench ?? null, pricing: data?.pricing ?? null,
+    status: { ...(data?.status ?? {}) }, revision: String(data?.revision ?? '') }
+  for (const [kind, raw] of [['liveBench', liveBenchEndpoint], ['pricing', pricingSnapshotEndpoint]]) {
+    if (typeof raw === 'string' && raw.trim() === '') {
+      result[kind] = null
+      result.status[kind] = { disabled: true, endpoint: '', error: '' }
+      continue
+    }
+    let endpoint
+    try { endpoint = publicDynamicEndpoint(raw) }
+    catch {
+      result[kind] = null
+      result.status[kind] = { endpoint: '', error: 'refresh-failed' }
+      continue
+    }
+    const snapshot = result[kind]
+    const status = result.status[kind]
+    if (snapshot && snapshot.endpoint !== endpoint) {
+      result[kind] = null
+      result.status[kind] = { ...(status?.endpoint === endpoint ? status : {}), endpoint,
+        lastGoodEndpoint: publicEndpointMetadata(snapshot.endpoint), error: status?.endpoint === endpoint ? publicError(status?.error) : '' }
+    } else if (status?.endpoint !== endpoint) {
+      result.status[kind] = { endpoint, error: '' }
+    }
+  }
+  return result
+}
 
 /** Explicit user prices override the curated public feed, including stale last-good data. */
 export function applyPricingSnapshot(routes, snapshot) {
@@ -25,15 +71,24 @@ function sourceSummary(data, kind, enabled, ttlMs, at) {
   const snapshot = enabled ? data?.[kind] : null
   const status = enabled ? data?.status?.[kind] : null
   const verifiedAt = Number.isFinite(snapshot?.verifiedAt) ? snapshot.verifiedAt : null
-  const expired = verifiedAt !== null && at - verifiedAt > ttlMs
-  const error = publicError(status?.error)
+  const expired = verifiedAt !== null && (verifiedAt > at + 60_000 || at - verifiedAt > ttlMs)
+  const deadline = Number.isFinite(status?.requestDeadlineAt) ? status.requestDeadlineAt
+    : Number.isFinite(status?.lastAttemptAt) ? status.lastAttemptAt + 13_000 : null
+  const interrupted = Boolean(status?.requestId) && deadline !== null && at >= deadline
+  const pending = Boolean(status?.requestId) && !interrupted
+  const error = interrupted ? 'refresh-interrupted' : publicError(status?.error)
   return {
     status: !enabled || status?.disabled ? 'disabled' : !snapshot ? (error ? 'error' : 'missing')
       : expired || error ? 'stale' : 'fresh',
     verifiedAt, publishedAt: snapshot?.publishedAt ?? null,
     version: snapshot?.version ?? null, source: snapshot?.source ?? null,
-    endpoint: snapshot?.endpoint ?? null, configuredEndpoint: status?.endpoint ?? null,
+    endpoint: publicEndpointMetadata(snapshot?.endpoint), configuredEndpoint: publicEndpointMetadata(status?.endpoint),
     error: enabled ? error : '',
+    refreshing: enabled && pending, pending: enabled && pending,
+    lastAttemptAt: Number.isFinite(status?.lastAttemptAt) ? status.lastAttemptAt : null,
+    lastSuccessAt: Number.isFinite(status?.lastSuccessAt) ? status.lastSuccessAt : null,
+    requestDeadlineAt: pending ? deadline : null,
+    lastGoodEndpoint: publicEndpointMetadata(status?.lastGoodEndpoint),
     ...(kind === 'liveBench' ? { modelCount: Object.keys(snapshot?.models ?? {}).length }
       : { rowCount: Object.keys(snapshot?.prices ?? {}).length }),
   }