dsh-cait 1.0.4 → 1.1.3

package/lib/store-utils.js

@@ -1,22 +1,56 @@
 import { createHash, randomUUID } from 'node:crypto'
 import { CatError } from './errors.js'
 import { clamp, clampInt } from './similarity.js'
+import { DEFAULT_PREPROCESS, normalizePreprocessOptions } from './preprocess.js'
 
 export { clamp, clampInt }
 
 export const MAX_SEGMENTS = 20000
-export const CAIT_VERSION = '1.0.4'
+export const CAIT_VERSION = '1.1.3'
 export const MAX_SEGMENT_CHARS = 20000
 export const MAX_IMPORT_CHARS = 8_000_000
 export const MAX_DOCX_BASE64 = 28_000_000
 export const MAX_STORE_BYTES = 64 * 1024 * 1024
-export const STORE_VERSION = 2
+export const STORE_VERSION = 4
 export const STORE_FILENAME = 'store.json'
 export const DEFAULT_TM_ID = 'tm_default'
 export const DEFAULT_GLOSSARY_ID = 'glossary_default'
 export const DEFAULT_TM_NAME = 'Default TM'
 export const DEFAULT_GLOSSARY_NAME = 'Default glossary'
 
+// Segment lifecycle: nothing translated yet → a draft or a proposal exists →
+// a human confirmed it. `origin` records where the current target/suggestion
+// came from, which is what makes "MT filled it, I only reviewed" auditable.
+export const SEGMENT_STATUSES = ['pending', 'translated', 'confirmed']
+export const SEGMENT_ORIGINS = ['tm', 'mt', 'agent', 'manual', 'auto']
+
+export function segmentStatus(segment) {
+  const target = String(segment?.target ?? '')
+  const suggestion = String(segment?.suggestion ?? '')
+  if (target && segment?.confirmedAt) return 'confirmed'
+  if (target || suggestion) return 'translated'
+  return 'pending'
+}
+
+/**
+ * Re-derive the stored fields from the facts (target/suggestion/confirmedAt)
+ * so the three-state model cannot drift out of sync with the text.
+ */
+export function normalizeSegmentState(segment) {
+  if (!String(segment.target ?? '')) segment.confirmedAt = null
+  if (!String(segment.suggestion ?? '')) {
+    segment.suggestion = ''
+    segment.suggestionReason = ''
+  }
+  segment.status = segmentStatus(segment)
+  if (segment.status === 'pending') {
+    segment.origin = null
+  } else if (!SEGMENT_ORIGINS.includes(segment.origin)) {
+    segment.origin = 'manual'
+  }
+  return segment
+}
+
 // UI languages the workbench can switch between; 'auto' follows the DSH locale.
 export const UI_LANG_CODES = ['auto', 'zh', 'en', 'ja']
 
@@ -26,6 +60,7 @@ export const DEFAULT_SETTINGS = {
   fuzzyThreshold: 0.55,
   tmLimit: 5,
   uiLang: 'auto',
+  preprocess: { ...DEFAULT_PREPROCESS },
   mt: {
     provider: 'mock',
     model: 'cait-mock',
@@ -161,11 +196,11 @@ function cleanInlineTokens(value) {
 export function cleanSegmentMeta(value) {
   if (!value || typeof value !== 'object' || Array.isArray(value)) return null
   const meta = {}
-  for (const key of ['start', 'end', 'xlsxRow', 'pptxParagraph']) {
+  for (const key of ['start', 'end', 'xlsxRow', 'pptxParagraph', 'epubBlock']) {
     const number = Number(value[key])
     if (Number.isFinite(number) && number >= 0) meta[key] = Math.round(number)
   }
-  for (const key of ['settings', 'identifier', 'context', 'msgidPlural', 'xliffId', 'format', 'jsonPath', 'pptxSlide']) {
+  for (const key of ['settings', 'identifier', 'context', 'msgidPlural', 'xliffId', 'format', 'jsonPath', 'pptxSlide', 'epubHref']) {
     if (typeof value[key] === 'string' && value[key].trim()) {
       meta[key] = value[key].slice(0, 500)
     }
@@ -212,6 +247,7 @@ export function restoreSettings(raw) {
     if (Number.isFinite(limit)) settings.tmLimit = clampInt(limit, 1, 20, settings.tmLimit)
     const uiLang = String(raw.uiLang ?? '').trim().toLowerCase()
     if (UI_LANG_CODES.includes(uiLang)) settings.uiLang = uiLang
+    settings.preprocess = normalizePreprocessOptions(raw.preprocess)
 
     if (raw.mt && typeof raw.mt === 'object' && !Array.isArray(raw.mt)) {
       const provider = String(raw.mt.provider ?? '').trim()
@@ -330,10 +366,16 @@ export function parseGlossaryCsv(text) {
 
   const entries = []
   for (const cells of rows) {
-    const [source, target, note, lang] = cells.map((cell) => cell.trim())
+    const [source, target, note, lang, caseFlag] = cells.map((cell) => cell.trim())
     if (!source || !target) continue
     if (/^(source|term|原文|术语)$/i.test(source) && /^(target|translation|译文)$/i.test(target)) continue
-    entries.push({ source, target, note: note || '', sourceLang: lang || '' })
+    entries.push({
+      source,
+      target,
+      note: note || '',
+      sourceLang: lang || '',
+      caseSensitive: /^(true|1|yes|y)$/i.test(caseFlag || ''),
+    })
   }
   return entries
 }

package/lib/store.js

@@ -2,16 +2,24 @@ import { CatError } from './errors.js'
 import { createRequire } from 'node:module'
 import { join, resolve } from 'node:path'
 import { splitIntoSegments } from './segmenter.js'
-import { canonicalLanguage, cleanFormatMeta, cleanLang, cleanName, cleanSegmentMeta, cleanText, clamp, clampInt, countWords, documentFingerprint, DEFAULT_SETTINGS, DEFAULT_GLOSSARY_ID, DEFAULT_GLOSSARY_NAME, DEFAULT_TM_ID, DEFAULT_TM_NAME, escapeRegExp, MAX_IMPORT_CHARS, MAX_SEGMENTS, MAX_SEGMENT_CHARS, now, makeId, parseGlossaryCsv, restoredDate, restoredLang, restoredText, restoreSettings, safeBaseName, STORE_FILENAME, UI_LANG_CODES } from './store-utils.js'
+import { canonicalLanguage, cleanFormatMeta, cleanLang, cleanName, cleanSegmentMeta, cleanText, clamp, clampInt, countWords, documentFingerprint, DEFAULT_SETTINGS, DEFAULT_GLOSSARY_ID, DEFAULT_GLOSSARY_NAME, DEFAULT_TM_ID, DEFAULT_TM_NAME, escapeRegExp, MAX_IMPORT_CHARS, MAX_SEGMENTS, MAX_SEGMENT_CHARS, normalizeSegmentState, now, makeId, parseGlossaryCsv, restoredDate, restoredLang, restoredText, restoreSettings, safeBaseName, SEGMENT_ORIGINS, STORE_FILENAME, UI_LANG_CODES } from './store-utils.js'
 import { libraryMethods } from './libraries.js'
 import { tmMethods } from './tm.js'
 import { glossaryMethods } from './glossary.js'
 import { persistenceMethods } from './persistence.js'
 import { importExportMethods } from './import-export.js'
+import { preTranslateMethods } from './pre-translate.js'
+import { qaMethods } from './qa.js'
+import { libraryViewMethods } from './library-view.js'
+import { propagateMethods } from './propagate.js'
+import { alignMethods } from './aligner.js'
+import { termMethods } from './terms.js'
 import { classifyMatchType, normalizeSource, normalizeText, similarity } from './similarity.js'
+import { contextOf } from './context.js'
 import { MOCK_MODEL, MOCK_PROVIDER, NO_MODEL_MESSAGE, NO_ROUTE_MESSAGE, collectStreamText } from './mt.js'
 import { buildTmx } from './tmx.js'
 import { missingInlineTokens } from './formats.js'
+import { normalizePreprocessOptions, preprocessSegments } from './preprocess.js'
 
 export { CatError } from './errors.js'
 export { canonicalLanguage } from './store-utils.js'
@@ -31,6 +39,7 @@ export class CatStore {
     this.resetLibraryState()
     this.dataDir = String(dataDir ?? '').trim() ? resolve(String(dataDir)) : ''
     this.storageFile = this.dataDir ? join(this.dataDir, STORE_FILENAME) : ''
+    this.saveTimer = null
     this.persistence = {
       enabled: Boolean(this.storageFile),
       file: this.storageFile || null,
@@ -84,10 +93,14 @@ export class CatStore {
   progress(doc) {
     const total = doc.segments.length
     const confirmed = doc.segments.filter((segment) => segment.status === 'confirmed').length
+    const translated = doc.segments.filter((segment) => segment.status === 'translated').length
     return {
       total,
       confirmed,
-      pending: total - confirmed,
+      // "translated" means a draft or a proposal exists and only review is left;
+      // "pending" is everything with no text at all.
+      translated,
+      pending: total - confirmed - translated,
       percent: total ? Math.round((confirmed / total) * 100) : 0,
     }
   }
@@ -112,6 +125,9 @@ export class CatStore {
       source: segment.source,
       target: segment.target,
       status: segment.status,
+      origin: segment.origin ?? null,
+      suggestion: segment.suggestion ?? '',
+      suggestionReason: segment.suggestionReason ?? '',
       paragraph: segment.paragraph,
       confirmedAt: segment.confirmedAt,
     }
@@ -127,6 +143,8 @@ export class CatStore {
         query: segment.source,
         sourceLang: doc.sourceLang,
         targetLang: doc.targetLang,
+        documentId: doc.id,
+        index,
       }).matches,
       glossaryHits: this.findGlossaryHits(segment.source, doc.sourceLang),
     }
@@ -177,6 +195,10 @@ export class CatStore {
           (sum, item) => sum + item.segments.filter((segment) => segment.status === 'confirmed').length,
           0,
         ),
+        translated: [...this.docs.values()].reduce(
+          (sum, item) => sum + item.segments.filter((segment) => segment.status === 'translated').length,
+          0,
+        ),
         tm: this.tm.length,
         glossary: this.glossary.length,
       },
@@ -216,6 +238,7 @@ export class CatStore {
     }
 
     if (!rawSegments.length) throw new CatError('EMPTY_SOURCE', '没有可翻译的句段。')
+    rawSegments = preprocessSegments(rawSegments, this.settings.preprocess)
     if (rawSegments.length > MAX_SEGMENTS) {
       throw new CatError('TOO_MANY_SEGMENTS', `句段数量超过 ${MAX_SEGMENTS} 上限`)
     }
@@ -232,16 +255,19 @@ export class CatStore {
           ? raw.paragraph
           : index
       const meta = cleanSegmentMeta(raw.meta)
-      return {
+      const segment = {
         id: `s${index + 1}`,
         source,
         target,
         status: target ? 'confirmed' : 'pending',
+        origin: target ? 'manual' : null,
+        suggestion: '',
         paragraph,
         ...(meta ? { meta } : {}),
         confirmedAt: target ? createdAt : null,
         updatedAt: createdAt,
       }
+      return normalizeSegmentState(segment)
     })
 
     const format = typeof input.format === 'string' ? input.format.slice(0, 24) : ''
@@ -251,6 +277,7 @@ export class CatStore {
     if (existing) {
       if (input.docxSource) this.attachDocxSource(existing, input.docxSource, input.docxName)
       if (input.pptxSource) this.attachPptxSource(existing, input.pptxSource, input.pptxName)
+      if (input.epubSource) this.attachEpubSource(existing, input.epubSource, input.epubName)
       if (input.format) existing.format = String(input.format).slice(0, 24)
       if (formatMeta) existing.formatMeta = cleanFormatMeta({ ...(existing.formatMeta || {}), ...formatMeta })
       this.activeDocId = existing.id
@@ -278,6 +305,7 @@ export class CatStore {
     if (doc.cursor < 0) doc.cursor = 0
     if (input.docxSource) this.attachDocxSource(doc, input.docxSource, input.docxName)
     if (input.pptxSource) this.attachPptxSource(doc, input.pptxSource, input.pptxName)
+    if (input.epubSource) this.attachEpubSource(doc, input.epubSource, input.epubName)
 
     this.docs.set(doc.id, doc)
     this.activeDocId = doc.id
@@ -299,6 +327,7 @@ export class CatStore {
     const tabIndex = this.openDocIds.indexOf(doc.id)
     this.removeDocxSource(doc)
     this.removePptxSource(doc)
+    this.removeEpubSource(doc)
     this.docs.delete(doc.id)
     this.openDocIds = this.openDocIds.filter((id) => id !== doc.id)
     if (this.activeDocId === doc.id) {
@@ -341,20 +370,98 @@ export class CatStore {
     doc.cursor = target
     doc.updatedAt = now()
     this.touch()
+    this.schedulePersist()
     return { cursor: doc.cursor, current: this.segmentDetail(doc, doc.cursor) }
   }
 
   saveDraft({ documentId, index, target } = {}) {
     const doc = this.requireDocument(documentId)
-    const segment = this.findSegment(doc, Number(index ?? doc.cursor))
+    const segmentIndex = Number(index ?? doc.cursor)
+    const segment = this.findSegment(doc, segmentIndex)
     const value = cleanText(target, '译文')
-    segment.target = value
+    if (value !== segment.target) {
+      segment.target = value
+      segment.updatedAt = now()
+      doc.updatedAt = segment.updatedAt
+      // Typed by a human. A model proposal stays visible next to it for
+      // reference until it is adopted or replaced.
+      if (value) segment.origin = 'manual'
+      // Duplicates of this source follow the human's wording instead of
+      // keeping a stale copy. Confirmed siblings are never touched.
+      if (value) this.propagateEdit(doc, segmentIndex)
+    }
+    normalizeSegmentState(segment)
+    this.touch()
+    this.schedulePersist()
+    return {
+      segment: this.segmentView(segment, segmentIndex),
+      cursor: doc.cursor,
+    }
+  }
+
+  adoptSuggestion({ documentId, index } = {}) {
+    const doc = this.requireDocument(documentId)
+    const segmentIndex = Number(index ?? doc.cursor)
+    const segment = this.findSegment(doc, segmentIndex)
+    if (!segment.suggestion) {
+      throw new CatError('NO_SUGGESTION', '当前句段没有可供采纳的建议。')
+    }
+    if (segment.status === 'confirmed') {
+      throw new CatError('ALREADY_CONFIRMED', '已确认的句段不能采纳建议，请先撤销确认。')
+    }
+    segment.target = segment.suggestion
     segment.updatedAt = now()
     doc.updatedAt = segment.updatedAt
+    // Adoption must keep the proposal's provenance (mt/agent/tm), not mark it
+    // as human-authored.
+    normalizeSegmentState(segment)
     this.touch()
+    this.persist()
     return {
-      segment: this.segmentView(segment, doc.segments.indexOf(segment)),
+      segment: this.segmentView(segment, segmentIndex),
       cursor: doc.cursor,
+      progress: this.progress(doc),
+    }
+  }
+
+  proposeTranslation({ documentId, index, target, reason = '' } = {}) {
+    const doc = this.requireDocument(documentId)
+    const segmentIndex = Number(index ?? doc.cursor)
+    const segment = this.findSegment(doc, segmentIndex)
+    if (segment.status === 'confirmed') {
+      throw new CatError('ALREADY_CONFIRMED', '已确认的句段不能提建议，请先撤销确认。')
+    }
+    const text = cleanText(target, '建议译文')
+    if (!text) throw new CatError('EMPTY_TARGET', '建议译文为空。')
+    const before = {
+      revision: this.revision,
+      updatedAt: doc.updatedAt,
+      segment: {
+        origin: segment.origin,
+        suggestion: segment.suggestion,
+        suggestionReason: segment.suggestionReason,
+        updatedAt: segment.updatedAt,
+      },
+    }
+    segment.suggestion = text
+    segment.suggestionReason = String(reason ?? '').slice(0, 500)
+    // A proposal never touches `target`: the human keeps the decision.
+    if (!segment.target) segment.origin = 'agent'
+    segment.updatedAt = now()
+    doc.updatedAt = segment.updatedAt
+    normalizeSegmentState(segment)
+    this.touch()
+    try {
+      this.persist({ strict: true })
+    } catch (error) {
+      this.revision = before.revision
+      doc.updatedAt = before.updatedAt
+      Object.assign(segment, before.segment)
+      throw error
+    }
+    return {
+      segment: this.segmentView(segment, segmentIndex),
+      progress: this.progress(doc),
     }
   }
 
@@ -370,6 +477,8 @@ export class CatStore {
       segment: {
         target: segment.target,
         status: segment.status,
+        origin: segment.origin,
+        suggestion: segment.suggestion,
         confirmedAt: segment.confirmedAt,
         updatedAt: segment.updatedAt,
       },
@@ -377,7 +486,11 @@ export class CatStore {
     }
 
     if (target !== undefined) {
-      segment.target = cleanText(target, '译文')
+      const value = cleanText(target, '译文')
+      // Confirming with the proposal's own text is adoption: keep where it came
+      // from. Any other text is authored right here.
+      if (value !== segment.target && value !== segment.suggestion) segment.origin = 'manual'
+      segment.target = value
     }
     if (!segment.target) {
       throw new CatError('EMPTY_TARGET', '译文为空，无法确认句段。')
@@ -391,17 +504,25 @@ export class CatStore {
       )
     }
 
-    segment.status = 'confirmed'
     segment.confirmedAt = now()
     segment.updatedAt = segment.confirmedAt
+    normalizeSegmentState(segment)
     doc.updatedAt = segment.confirmedAt
 
     const tmResult = addToTm
-      ? this.upsertTM(segment.source, segment.target, doc.sourceLang, doc.targetLang, writeTmId)
+      ? this.upsertTM(
+          segment.source,
+          segment.target,
+          doc.sourceLang,
+          doc.targetLang,
+          writeTmId,
+          contextOf(doc, segmentIndex),
+        )
       : null
 
     const nextIndex = this.findNextUnconfirmed(doc, segmentIndex)
     doc.cursor = nextIndex
+    this.propagateFromSegment(doc, segmentIndex)
     this.touch()
     try {
       this.persist({ strict: true })
@@ -415,12 +536,15 @@ export class CatStore {
     }
 
     const progress = this.progress(doc)
+    // Translated-but-unreviewed segments still have to be visited, so the
+    // remaining count is driven by confirmed only.
+    const remaining = progress.total - progress.confirmed
     return {
       confirmed: this.segmentView(segment, segmentIndex),
-      next: progress.pending ? this.segmentDetail(doc, doc.cursor) : null,
+      next: remaining ? this.segmentDetail(doc, doc.cursor) : null,
       cursor: doc.cursor,
       progress,
-      done: progress.pending === 0,
+      done: remaining === 0,
       tm: tmResult
         ? {
             added: tmResult.created ? 1 : 0,
@@ -473,6 +597,13 @@ export class CatStore {
       this.settings.uiLang = uiLang
     }
 
+    if (patch.preprocess && typeof patch.preprocess === 'object') {
+      this.settings.preprocess = normalizePreprocessOptions({
+        ...this.settings.preprocess,
+        ...patch.preprocess,
+      })
+    }
+
     if (patch.mt && typeof patch.mt === 'object') {
       const mt = patch.mt
       if (mt.provider !== undefined) {
@@ -731,4 +862,4 @@ export class CatStore {
   }
 }
 
-Object.assign(CatStore.prototype, libraryMethods, tmMethods, glossaryMethods, persistenceMethods, importExportMethods)
+Object.assign(CatStore.prototype, libraryMethods, tmMethods, glossaryMethods, persistenceMethods, importExportMethods, preTranslateMethods, qaMethods, libraryViewMethods, propagateMethods, alignMethods, termMethods)

package/lib/term-extract.js

@@ -0,0 +1,190 @@
+/**
+ * Terminology candidates, found by counting rather than by asking a model to
+ * read a document. Models are good at judging a short list ("is this a domain
+ * term? what does it translate to?") and bad at enumerating one: without
+ * frequency evidence a long list cannot be reviewed and gets abandoned, which
+ * is exactly what happened the first time.
+ *
+ * So this module only proposes candidates with evidence — occurrences, length,
+ * boundary entropy, containment — and the model (or the human) picks.
+ */
+
+const STOPWORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'if', 'then', 'than', 'that', 'this', 'these', 'those',
+  'is', 'are', 'was', 'were', 'be', 'been', 'being', 'to', 'of', 'in', 'on', 'at', 'by', 'for',
+  'with', 'from', 'as', 'it', 'its', 'we', 'you', 'they', 'he', 'she', 'i', 'not', 'no', 'do',
+  'does', 'did', 'can', 'could', 'will', 'would', 'should', 'may', 'might', 'must', 'have', 'has',
+  'had', 'there', 'here', 'when', 'where', 'which', 'who', 'what', 'how', 'all', 'any', 'some',
+  'more', 'most', 'other', 'into', 'over', 'under', 'about', 'after', 'before', 'also', 'just',
+  'only', 'very', 'so', 'such', 'use', 'used', 'using', 'one', 'two', 'new', 'get', 'set', 'see',
+])
+
+const CJK = /[\u3400-\u9fff\u3040-\u30ff\uac00-\ud7af]/
+
+const WORD = /[A-Za-z][A-Za-z0-9'’.-]*/g
+
+export function splitSentencesForTerms(text) {
+  return String(text ?? '')
+    .split(/(?<=[.!?。！？])\s+|\n+/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+}
+
+function wordsOf(sentence) {
+  return (sentence.match(WORD) || []).map((word) => word.replace(/[.'’-]+$/, ''))
+}
+
+function normalizedTerm(value) {
+  return String(value ?? '')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .toLocaleLowerCase()
+}
+
+/** Latin candidates are word n-grams (1–4) without a stopword at either edge. */
+export function latinCandidates(sentences, options = {}) {
+  const maxN = options.maxN ?? 4
+  const counts = new Map()
+  const examples = new Map()
+  const display = new Map()
+  for (const sentence of sentences) {
+    const words = wordsOf(sentence)
+    for (let n = 1; n <= maxN; n += 1) {
+      for (let start = 0; start + n <= words.length; start += 1) {
+        const slice = words.slice(start, start + n)
+        if (STOPWORDS.has(slice[0].toLocaleLowerCase())) continue
+        if (STOPWORDS.has(slice[n - 1].toLocaleLowerCase())) continue
+        const term = slice.join(' ')
+        // A single token has to look like a name or a compound; plain everyday
+        // words are noise.
+        if (n === 1 && !/^[A-Z][A-Za-z0-9]*$|[-_]|[a-z][A-Z]/.test(slice[0])) continue
+        const key = normalizedTerm(term)
+        counts.set(key, (counts.get(key) || 0) + 1)
+        if (!display.has(key)) display.set(key, term)
+        if (!examples.has(key)) examples.set(key, [])
+        const bucket = examples.get(key)
+        if (bucket.length < 2 && !bucket.includes(sentence)) bucket.push(sentence)
+      }
+    }
+  }
+  return [...counts.entries()].map(([key, count]) => ({
+    term: display.get(key) ?? key,
+    key,
+    count,
+    kind: 'latin',
+    examples: examples.get(key) ?? [],
+  }))
+}
+
+function entropy(values) {
+  if (!values.length) return 0
+  const counts = new Map()
+  for (const value of values) counts.set(value, (counts.get(value) || 0) + 1)
+  let value = 0
+  for (const amount of counts.values()) {
+    const p = amount / values.length
+    value -= p * Math.log2(p)
+  }
+  return value
+}
+
+/**
+ * Chinese, Japanese and Korean have no spaces, so candidates are repeated
+ * substrings. Left/right neighbour entropy filters out fragments that are
+ * merely the middle of a longer word: a real term appears after many different
+ * characters, a fragment appears after the same one.
+ */
+export function cjkCandidates(text, options = {}) {
+  const min = options.min ?? 2
+  const max = options.max ?? 6
+  const limit = options.limit ?? 400
+  const body = String(text ?? '').replace(/\s+/g, ' ')
+  const counts = new Map()
+  const left = new Map()
+  const right = new Map()
+  for (let start = 0; start < body.length; start += 1) {
+    if (!CJK.test(body[start])) continue
+    for (let size = min; size <= max; size += 1) {
+      const end = start + size
+      if (end > body.length) break
+      const piece = body.slice(start, end)
+      if (!CJK.test(piece[0]) || !CJK.test(piece[size - 1])) continue
+      counts.set(piece, (counts.get(piece) || 0) + 1)
+      if (!left.has(piece)) left.set(piece, [])
+      if (!right.has(piece)) right.set(piece, [])
+      left.get(piece).push(body[start - 1] ?? '')
+      right.get(piece).push(body[end] ?? '')
+    }
+  }
+  const candidates = []
+  for (const [term, count] of counts) {
+    if (count < 2) continue
+    const boundary = Math.min(entropy(left.get(term) || []), entropy(right.get(term) || []))
+    if (boundary < 0.5) continue
+    candidates.push({ term, count, kind: 'cjk', boundary, examples: [] })
+  }
+  return candidates.sort((a, b) => b.count - a.count).slice(0, limit)
+}
+
+/**
+ * C-value in spirit: frequent, long candidates win, and a string that is only
+ * ever contained in a longer candidate loses to that longer one.
+ */
+export function rankCandidates(candidates) {
+  for (const item of candidates) {
+    let contained = 0
+    let longerCount = 0
+    for (const other of candidates) {
+      if (other === item) continue
+      if (other.term.length > item.term.length && other.term.includes(item.term)) {
+        contained += 1
+        longerCount += other.count
+      }
+    }
+    const lengthBonus = Math.log2(2 + Array.from(item.term).length)
+    const cvalue = item.count * lengthBonus - (contained ? longerCount / contained : 0)
+    item.score = Math.round(Math.max(0, cvalue) * 1000) / 1000
+  }
+  // A fragment that only ever shows up inside a longer term is part of that
+  // term, not a term of its own ("design" next to "Design System").
+  const covered = new Set()
+  for (const item of candidates) {
+    for (const other of candidates) {
+      if (other === item) continue
+      if (other.term.length > item.term.length && other.term.includes(item.term) && other.count >= item.count) {
+        covered.add(item)
+        break
+      }
+    }
+  }
+  const seen = new Set()
+  const unique = []
+  for (const item of candidates.filter((candidate) => !covered.has(candidate)).sort((a, b) => b.score - a.score || b.count - a.count)) {
+    // Fold case and a trailing plural: "Mode"/"modes" are one concept.
+    const fold = normalizedTerm(item.term).replace(/(\w)s$/, '$1')
+    if (seen.has(fold)) continue
+    seen.add(fold)
+    unique.push(item)
+  }
+  return unique
+}
+
+/**
+ * Candidates for one text, annotated against the terms already in memory.
+ */
+export function extractTerms(text, options = {}) {
+  const sentences = splitSentencesForTerms(text)
+  const latin = latinCandidates(sentences, options)
+  const cjk = cjkCandidates(text, options)
+  const ranked = rankCandidates([...latin, ...cjk])
+  const known = new Set((options.knownTerms ?? []).map(normalizedTerm))
+  const limit = options.limit ?? 60
+  const minimumCount = options.minCount ?? 2
+  return ranked
+    .filter((item) => item.count >= minimumCount)
+    .map((item) => ({
+      ...item,
+      inLibrary: known.has(normalizedTerm(item.term)),
+    }))
+    .slice(0, limit)
+}