dsh-context-compression-improved 0.1.1 → 0.2.0

package/packages/selector/src/client/preset-options.ts

@@ -0,0 +1,70 @@
+/**
+ * Field-wise writes for the `presetOptions` section of the
+ * context-compression settings namespace.
+ *
+ * `settingsScope.set(field, value)` replaces the value AT that field, so
+ * writing the section root with a patch deleted every sibling: choosing a
+ * provider after choosing a channel erased `estimatorMode` and silently turned
+ * the estimator back off, and the stored document kept exactly the last field
+ * the user touched. The client reads the complete section (this namespace
+ * declares no secret fields), so a patch is applied over the current document
+ * and the merged result is what gets written.
+ *
+ * A patch member set to `undefined` means "clear this field" (it then
+ * re-inherits the preset default); absent members stay untouched.
+ */
+
+import type { PresetOptionsSettings } from '../profiles.ts'
+
+/** Every field a patch may address, in the schema's own order. */
+const PRESET_OPTION_KEYS = [
+  'dedupeToolResults', 'summaryLocator', 'prefixStabilizer', 'readState', 'estimatorMode',
+  'estimatorProvider', 'estimatorModel', 'estimatorBaseUrl', 'estimatorApiKey', 'estimatorTimeoutMs',
+] as const
+
+/** One partial edit of `presetOptions`; `undefined` clears the named field. */
+export type PresetOptionsPatch = {
+  readonly [K in keyof PresetOptionsSettings]?: PresetOptionsSettings[K] | undefined
+}
+
+/**
+ * Apply one patch over the stored section.
+ *
+ * @param current - the decoded `presetOptions` section, when one is stored.
+ * @param patch - the fields to write or clear.
+ * @returns the complete section to store.
+ */
+export function mergePresetOptionsPatch(
+  current: PresetOptionsSettings | undefined,
+  patch: PresetOptionsPatch,
+): PresetOptionsSettings {
+  const source = current as Record<string, unknown> | undefined
+  const merged: Record<string, unknown> = {}
+  for (const key of PRESET_OPTION_KEYS) {
+    const stored = source?.[key]
+    if (stored !== undefined) merged[key] = stored
+  }
+  for (const key of Object.keys(patch)) {
+    const value = (patch as Record<string, unknown>)[key]
+    if (value === undefined) delete merged[key]
+    else merged[key] = value
+  }
+  return merged as PresetOptionsSettings
+}
+
+/**
+ * Compare two sections field by field, so an unchanged patch neither rewrites
+ * the document nor reports a save the Host never committed.
+ *
+ * @param left - one section (or none).
+ * @param right - the other section.
+ * @returns whether every known field holds the same value.
+ */
+export function presetOptionsEqual(
+  left: PresetOptionsSettings | undefined,
+  right: PresetOptionsSettings | undefined,
+): boolean {
+  const a = left as Record<string, unknown> | undefined
+  const b = right as Record<string, unknown> | undefined
+  return PRESET_OPTION_KEYS.every(key => a?.[key] === b?.[key])
+}

package/packages/selector/src/client/settings-section.tsx

@@ -0,0 +1,126 @@
+/**
+ * Full-page Settings surface backed by the same durable selector state.
+ *
+ * Extracted from CompressionProfileSelector.tsx to reduce god-module size.
+ *
+ * @module dsh-context-compression-improved/client/settings-section
+ */
+
+import { useEffect, useState } from 'react'
+import css from './CompressionProfileSelector.module.css'
+import {
+  AUTO_COMPACT_THRESHOLD_LIMITS,
+  COMPRESSION_PROFILES,
+  type CompressionProfile,
+  type CustomCompressionPolicyV3,
+} from '../profiles.ts'
+import type { CompressionProfileSelectorProps } from './CompressionProfileSelector.tsx'
+import { AutoCompactThresholdControls, CodeSkeletonControls } from './CompressionProfileControls.tsx'
+import { EstimatorControls, EstimatorInactiveNotice } from './EstimatorControls.tsx'
+import { CustomPolicyEditor, editableCustom } from './CustomPolicyEditor.tsx'
+
+/** Full-page Settings surface backed by the same durable selector state. */
+export function ContextCompressionSettingsSection(props: CompressionProfileSelectorProps) {
+  return <SettingsCompressionProfileControls {...props} />
+}
+
+export function SettingsCompressionProfileControls({
+  useCompression, useSessions, select, saveCustom, resetCustom, saveAutoCompact, saveCodeSkeleton,
+  savePresetOptions, t,
+}: CompressionProfileSelectorProps) {
+  const state = useCompression(snapshot => snapshot)
+  const currentPreset = useSessions((sessions) => {
+    const current = sessions.current
+    return current === undefined ? undefined : sessions.byId[current]?.agentPreset
+  })
+  const selectorAvailable = currentPreset !== 'minimal'
+  const [saving, setSaving] = useState(false)
+  const [saveError, setSaveError] = useState<string | null>(null)
+  const [draft, setDraft] = useState<CustomCompressionPolicyV3 | null>(null)
+  const current = state.value?.profile ?? 'balanced'
+  useEffect(() => {
+    const custom = state.value?.custom
+    setDraft(current === 'custom' && custom !== undefined ? editableCustom(custom) : null)
+  }, [current, state.value?.custom])
+  if (state.status === 'unavailable') return null
+
+  const busy = state.status === 'loading' || saving
+  const selectProfile = (profile: CompressionProfile): void => {
+    if (!selectorAvailable || !state.writable || profile === current) return
+    setSaveError(null)
+    setSaving(true)
+    void select(profile).then(
+      () => { setSaving(false) },
+      (error: unknown) => {
+        setSaving(false)
+        setSaveError(error instanceof Error && error.message !== '' ? error.message : t('status.saveFailed'))
+      },
+    )
+  }
+  const settle = (operation: () => Promise<void>): void => {
+    setSaveError(null)
+    setSaving(true)
+    void operation().then(
+      () => { setSaving(false) },
+      (error: unknown) => {
+        setSaving(false)
+        setSaveError(error instanceof Error && error.message !== '' ? error.message : t('status.saveFailed'))
+      },
+    )
+  }
+
+  return (
+    <section className={css.settingsSection}>
+      <h2 className={css.settingsTitle}>{t('settings.title')}</h2>
+      <p className={css.settingsDescription}>{t('settings.description')}</p>
+      {selectorAvailable ? (
+        <div className={css.profileGrid} aria-label={t('label')}>
+          {COMPRESSION_PROFILES.map((profile) => {
+            const selected = profile === current
+            return (
+              <button key={profile} type="button" className={css.profileCard} aria-pressed={selected}
+                disabled={busy || !state.writable} onClick={() => { selectProfile(profile) }}>
+                <span className={css.profileCardTop}>
+                  <span className={css.profileCardTitle}>{t(`profile.${profile}`)}</span>
+                  {selected ? <span className={css.profileCurrent}>{t('profile.current')}</span> : null}
+                </span>
+                <span className={css.profileCardDetail}>{t(`detail.${profile}`)}</span>
+              </button>
+            )
+          })}
+        </div>
+      ) : <div className={css.unavailable} role="status">{t('status.minimalUnavailable')}</div>}
+      <AutoCompactThresholdControls
+        value={state.value?.autoCompact?.thresholdPercent ?? AUTO_COMPACT_THRESHOLD_LIMITS.default}
+        disabled={busy || !state.writable || !selectorAvailable}
+        save={saveAutoCompact}
+        settle={settle}
+        t={t}
+      />
+      <CodeSkeletonControls
+        value={state.value?.codeSkeleton?.enabled ?? false}
+        disabled={busy || !state.writable || !selectorAvailable}
+        save={saveCodeSkeleton}
+        settle={settle}
+        t={t}
+      />
+      {current !== 'tokenpilot-inspired' ? (
+        <EstimatorInactiveNotice profile={t(`profile.${current}`)} t={t} />
+      ) : (
+        <EstimatorControls
+          options={state.value?.presetOptions ?? {}}
+          disabled={busy || !state.writable || !selectorAvailable}
+          save={savePresetOptions}
+          settle={settle}
+          t={t}
+        />
+      )}
+      <div className={css.pricing}>{t('pricing.disclosure')}</div>
+      {current !== 'custom' || draft === null || !selectorAvailable ? null : (
+        <CustomPolicyEditor value={draft} disabled={busy || !state.writable} setValue={setDraft}
+          save={() => saveCustom(structuredClone(draft))} reset={resetCustom} settle={settle} t={t} />
+      )}
+      {saveError === null ? null : <div className={css.error} role="alert">{saveError}</div>}
+    </section>
+  )
+}

package/packages/selector/src/css-modules.d.ts

@@ -0,0 +1,6 @@
+declare module '*.module.css' {
+  const classes: Record<string, string>
+  export default classes
+}
+
+declare module '*.css'

package/packages/selector/src/deepseek-v4-tokenizer.ts

@@ -0,0 +1,210 @@
+/** Offline DeepSeek tokenizers backed by pinned official Hugging Face assets. */
+
+import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+import { Tokenizer } from '@huggingface/tokenizers'
+import type { ExactTokenizerTokenCount } from './runtime/token-count.ts'
+
+const TOKENIZER_ID = 'deepseek-ai/DeepSeek-V4-Pro'
+const TOKENIZER_REVISION = '0e1a0e5e52aea73055f50fef6f2423db370265b6'
+const TOKENIZER_SHA256 = '8f9f37ca37fdc4f5fd36d5cf4d3b0e8392edb4e894fd10cc0d70b4957c8633cf'
+const CONFIG_SHA256 = '6ac8c8dc065ed118161d02dd532749ae3f52c243deac27872134fae2f50d8547'
+const VISION_TOKENIZER_ID = 'deepseek-ai/DeepSeek-V4-Flash-Vision-Exp'
+const VISION_TOKENIZER_REVISION = '6821d6ad3681a4b137b066b76094fa82ebd0a380'
+const VISION_TOKENIZER_SHA256 = 'c90dfa01249db1be4245780a052ede752e1361c612ac6d08e2bdada7d599476b'
+const VISION_CONFIG_SHA256 = '6ac8c8dc065ed118161d02dd532749ae3f52c243deac27872134fae2f50d8547'
+
+/** Auditable origin and compatibility mapping for one bundled tokenizer. */
+export interface DeepSeekTokenizerArtifactOrigin {
+  /** Hugging Face repository the assets were pinned from. */
+  readonly repository: string
+  /** Immutable commit revision the assets were pinned from. */
+  readonly revision: string
+  /** Upstream license of the repository. */
+  readonly license: string
+  /** Exact API wire model ids served by this artifact. */
+  readonly modelIds: readonly string[]
+  readonly tokenizerSha256: string
+  readonly tokenizerConfigSha256: string
+}
+
+/** Auditable origin and compatibility mapping for the bundled V4 Pro tokenizer. */
+export const DEEPSEEK_V4_TOKENIZER_ARTIFACT: DeepSeekTokenizerArtifactOrigin = Object.freeze({
+  repository: TOKENIZER_ID,
+  revision: TOKENIZER_REVISION,
+  license: 'MIT',
+  tokenizerSha256: TOKENIZER_SHA256,
+  tokenizerConfigSha256: CONFIG_SHA256,
+  modelIds: Object.freeze(['deepseek-v4-flash', 'deepseek-v4-pro']),
+})
+
+/**
+ * Auditable origin and compatibility mapping for the bundled V4 Flash Vision
+ * tokenizer. The vision repository ships a distinct `tokenizer.json` (it adds
+ * the `<｜deepseek_image｜>` special token), so the vision model must never be
+ * mapped onto the V4 Pro tokenizer as an alias.
+ */
+export const DEEPSEEK_VISION_TOKENIZER_ARTIFACT: DeepSeekTokenizerArtifactOrigin = Object.freeze({
+  repository: VISION_TOKENIZER_ID,
+  revision: VISION_TOKENIZER_REVISION,
+  license: 'MIT',
+  tokenizerSha256: VISION_TOKENIZER_SHA256,
+  tokenizerConfigSha256: VISION_CONFIG_SHA256,
+  modelIds: Object.freeze(['deepseek-v4-flash-vision-exp']),
+})
+
+/** Synchronous exact counter backed only by the verified local V4 artifacts. */
+export interface DeepSeekV4TextTokenizer {
+  /** Count one fully determined text value with the official tokenizer. */
+  countText(text: string): ExactTokenizerTokenCount
+}
+
+/** Integrity tuple for one tokenizer JSON asset. @internal */
+export interface DeepSeekV4TokenizerAssetDescriptor {
+  readonly bytes: number
+  readonly sha256: string
+}
+
+/** Injectable integrity manifest used by negative loader tests. @internal */
+export interface DeepSeekV4TokenizerAssetIntegrity {
+  readonly tokenizer: DeepSeekV4TokenizerAssetDescriptor
+  readonly config: DeepSeekV4TokenizerAssetDescriptor
+}
+
+interface TokenizerArtifact {
+  readonly origin: DeepSeekTokenizerArtifactOrigin
+  readonly assetRoot: URL
+  readonly integrity: DeepSeekV4TokenizerAssetIntegrity
+}
+
+/**
+ * Every bundled artifact is registered with independent asset roots, integrity
+ * manifests, and cache entries: one corrupted artifact must never disable the
+ * tokenizer serving the other model family.
+ *
+ * This module deliberately sits at the package's `src/` root rather than in
+ * `src/runtime/`: the build flattens the runtime into `lib/*.js`, so only a
+ * module one level below the package root resolves `../assets/` identically
+ * from source and from the published artifact.
+ */
+const ARTIFACTS: readonly TokenizerArtifact[] = Object.freeze([
+  Object.freeze({
+    origin: DEEPSEEK_V4_TOKENIZER_ARTIFACT,
+    assetRoot: new URL('../assets/deepseek-v4/', import.meta.url),
+    integrity: Object.freeze({
+      tokenizer: Object.freeze({ bytes: 6_367_146, sha256: TOKENIZER_SHA256 }),
+      config: Object.freeze({ bytes: 801, sha256: CONFIG_SHA256 }),
+    }),
+  }),
+  Object.freeze({
+    origin: DEEPSEEK_VISION_TOKENIZER_ARTIFACT,
+    assetRoot: new URL('../assets/deepseek-v4-vision-exp/', import.meta.url),
+    integrity: Object.freeze({
+      tokenizer: Object.freeze({ bytes: 6_367_257, sha256: VISION_TOKENIZER_SHA256 }),
+      config: Object.freeze({ bytes: 801, sha256: VISION_CONFIG_SHA256 }),
+    }),
+  }),
+])
+
+interface ArtifactCacheEntry {
+  readonly tokenizer?: DeepSeekV4TextTokenizer
+  readonly failure?: string
+}
+
+const registryCache = new Map<DeepSeekTokenizerArtifactOrigin, ArtifactCacheEntry>()
+
+function artifactForModel(modelId: string): TokenizerArtifact | undefined {
+  return ARTIFACTS.find(artifact => (artifact.origin.modelIds as readonly string[]).includes(modelId))
+}
+
+/**
+ * Resolve the shared offline tokenizer for one compatible API model.
+ * Unknown models and a cached asset/runtime failure return `undefined`; callers
+ * must report unavailable instead of manufacturing a character estimate.
+ * @param modelId - exact DeepSeek API wire model id.
+ * @returns the shared verified tokenizer, or undefined when unsupported/unavailable.
+ */
+export function deepSeekV4TokenizerForModel(modelId: string): DeepSeekV4TextTokenizer | undefined {
+  const artifact = artifactForModel(modelId)
+  if (artifact === undefined) return undefined
+  const cached = registryCache.get(artifact.origin)
+  if (cached !== undefined) return cached.tokenizer
+  let entry: ArtifactCacheEntry
+  try {
+    const tokenizer = createDeepSeekV4TokenizerFromAssets(artifact.assetRoot, artifact.integrity, artifact.origin)
+    entry = { tokenizer }
+  } catch (error: unknown) {
+    entry = { failure: error instanceof Error ? error.message : String(error) }
+  }
+  registryCache.set(artifact.origin, entry)
+  return entry.tokenizer
+}
+
+/**
+ * Read the cached initialization diagnostic for provider registration.
+ * @param modelId - optional exact model id; defaults to the V4 Pro artifact.
+ * @returns the first initialization failure for that artifact, or undefined.
+ */
+export function deepSeekV4TokenizerFailureReason(modelId?: string): string | undefined {
+  const artifact = modelId === undefined ? ARTIFACTS[0] : artifactForModel(modelId)
+  if (artifact === undefined) return undefined
+  return registryCache.get(artifact.origin)?.failure
+}
+
+/** Complete pinned artifact inventory, in stable registration order. */
+export function deepSeekTokenizerArtifacts(): readonly DeepSeekTokenizerArtifactOrigin[] {
+  return ARTIFACTS.map(artifact => artifact.origin)
+}
+
+/**
+ * Build a tokenizer from one local asset directory after byte/hash validation.
+ * This provider-private seam exists so tests can prove every failure branch
+ * without mutating the committed artifact.
+ * @param assetRoot - local URL containing tokenizer.json and tokenizer_config.json.
+ * @param integrity - expected byte length and SHA-256 for both files.
+ * @param origin - auditable identity recorded on every returned count.
+ * @returns a synchronous exact text counter.
+ * @internal
+ */
+export function createDeepSeekV4TokenizerFromAssets(
+  assetRoot: URL,
+  integrity: DeepSeekV4TokenizerAssetIntegrity,
+  origin: DeepSeekTokenizerArtifactOrigin = DEEPSEEK_V4_TOKENIZER_ARTIFACT,
+): DeepSeekV4TextTokenizer {
+  const tokenizerJson = readVerifiedJson(assetRoot, 'tokenizer.json', integrity.tokenizer)
+  const tokenizerConfig = readVerifiedJson(assetRoot, 'tokenizer_config.json', integrity.config)
+  const runtime = new Tokenizer(tokenizerJson, tokenizerConfig)
+  return Object.freeze({
+    countText(text: string): ExactTokenizerTokenCount {
+      const tokens = runtime.encode(text, { add_special_tokens: false }).ids.length
+      return Object.freeze({
+        kind: 'exact-tokenizer',
+        tokens,
+        tokenizerId: origin.repository,
+        tokenizerRevision: origin.revision,
+      })
+    },
+  })
+}
+
+function readVerifiedJson(
+  assetRoot: URL,
+  name: string,
+  descriptor: DeepSeekV4TokenizerAssetDescriptor,
+): Record<string, unknown> {
+  const bytes = readFileSync(new URL(name, assetRoot))
+  if (bytes.byteLength !== descriptor.bytes) {
+    throw new Error(
+      `DeepSeek tokenizer asset ${name} has ${String(bytes.byteLength)} bytes; expected ${String(descriptor.bytes)}`,
+    )
+  }
+  const actualSha256 = createHash('sha256').update(bytes).digest('hex')
+  if (actualSha256 !== descriptor.sha256) {
+    throw new Error(`DeepSeek tokenizer asset ${name} failed SHA-256 verification`)
+  }
+  const parsed: unknown = JSON.parse(bytes.toString('utf8'))
+  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+    throw new Error(`DeepSeek tokenizer asset ${name} must contain a JSON object`)
+  }
+  return parsed as Record<string, unknown>
+}

package/packages/selector/src/estimator-catalog.ts

@@ -0,0 +1,104 @@
+/**
+ * Estimator model-catalog projection for the settings card: the live
+ * provider/model-group catalog from the DSH `llm` service — including
+ * user-configured custom groups — plus the effective host selection
+ * (override → agentDefaultModel.currentSelection). The same projection seam
+ * dsh-perm-gate's receiver-info and dsh-prime-memory's llm-providers view use.
+ *
+ * Pure projection: never throws; per-provider failures are reported inline so
+ * one broken group never blanks the whole list.
+ */
+
+/** One model entry as offered by a provider group. */
+export interface CatalogModel {
+  readonly id: string
+  readonly name: string
+}
+
+/** One provider group and its models (or the reason it failed to enumerate). */
+export interface CatalogProvider {
+  readonly id: string
+  readonly name: string
+  readonly models: readonly CatalogModel[]
+  readonly error?: string
+}
+
+/** What the settings card needs to render the host-route dropdowns. */
+export interface EstimatorCatalog {
+  /** Live provider groups; empty when the `llm` service is unavailable. */
+  readonly providers: readonly CatalogProvider[]
+  /** The provider/model the host channel would use right now (undefined when undetermined). */
+  readonly selection?: { readonly provider: string, readonly model: string }
+}
+
+/** The minimal faces of the host services this projection reads. */
+export interface EstimatorCatalogDeps {
+  llm?: {
+    listProviders?: () => readonly { id: string, name: string }[]
+    listModels?: (providerId: string) => Promise<readonly { id: string, name: string }[]>
+    /** Fallback enumeration source for builds whose `listModels` is absent or dormant. */
+    listConfigurableProviders?: () => readonly { provider: string, settingsNs: string }[]
+    /** Discovery for a configured route answers from the adapter's own knowledge — no network call. */
+    discoverModels?: (settingsNs: string, request: { provider?: string }) => Promise<readonly { id: string, name?: string }[]>
+  }
+  /** Live default-model selection (optional `agentDefaultModel` service). */
+  currentSelection?: () => { provider?: unknown, model?: unknown } | undefined
+  /** Explicit estimator overrides (empty string = follow the host selection). */
+  overrideProvider?: string
+  overrideModel?: string
+}
+
+function str(value: unknown): string {
+  return typeof value === 'string' ? value : ''
+}
+
+/** Resolve the effective host model route (override → session default). */
+export function resolveHostRoute(deps: EstimatorCatalogDeps): { provider: string, model: string } | undefined {
+  const overrideProvider = str(deps.overrideProvider)
+  const overrideModel = str(deps.overrideModel)
+  const selected = deps.currentSelection?.()
+  const selectedProvider = str(selected?.provider)
+  const selectedModel = str(selected?.model)
+  const provider = overrideProvider !== '' ? overrideProvider : selectedProvider
+  const model = overrideModel !== '' ? overrideModel : selectedModel
+  if (provider === '' || model === '') return undefined
+  return { provider, model }
+}
+
+/** Build the catalog projection. Never throws. */
+export async function buildEstimatorCatalog(deps: EstimatorCatalogDeps): Promise<EstimatorCatalog> {
+  if (deps.llm?.listProviders === undefined) {
+    const selection = resolveHostRoute(deps)
+    return selection === undefined ? { providers: [] } : { providers: [], selection }
+  }
+  const selection = resolveHostRoute(deps)
+  const raw = deps.llm.listProviders()
+  const providers = await Promise.all(raw.map(async (p): Promise<CatalogProvider> => {
+    let models: CatalogModel[] = []
+    let error: string | undefined
+    try {
+      models = (await deps.llm?.listModels?.(p.id) ?? []).map(m => ({ id: m.id, name: m.name }))
+    } catch (e) {
+      error = String((e as Error)?.message ?? e)
+    }
+    if (models.length === 0) {
+      // Fallback: route discovery. For an already-configured route the
+      // adapter answers from its own stored knowledge (no network call).
+      try {
+        const entry = deps.llm?.listConfigurableProviders?.().find(c => c.provider === p.id)
+        if (entry !== undefined && deps.llm?.discoverModels !== undefined) {
+          const discovered = await deps.llm.discoverModels(entry.settingsNs, { provider: p.id })
+          if (discovered.length > 0) {
+            models = discovered.map(m => ({ id: m.id, name: m.name ?? m.id }))
+            error = undefined
+          }
+        }
+      } catch {
+        // keep the primary error / empty state
+      }
+    }
+    if (models.length === 0 && error === undefined) error = 'no models advertised'
+    return { id: p.id, name: p.name, models, ...(error === undefined ? {} : { error }) }
+  }))
+  return { providers, ...(selection === undefined ? {} : { selection }) }
+}