dsh-cait 1.0.4 → 1.1.3

package/lib/pre-translate.js

@@ -0,0 +1,263 @@
+import { CatError } from './errors.js'
+import { DEFAULT_TM_ID, clampInt, normalizeSegmentState, now } from './store-utils.js'
+import { normalizeText } from './similarity.js'
+import { contextHashOf, contextOf } from './context.js'
+
+export const PRETRANSLATE_BLOCK_MAX = 32
+export const PRETRANSLATE_DEFAULT_COUNT = 8
+export const PRETRANSLATE_CONCURRENCY = 2
+
+/**
+ * Run `worker` over `items` with a small pool. Machine translation is network
+ * bound, so a little concurrency buys most of the speed without hammering the
+ * provider or the event loop.
+ */
+async function runPool(items, limit, worker) {
+  const results = new Array(items.length)
+  let cursor = 0
+  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
+    while (cursor < items.length) {
+      const position = cursor
+      cursor += 1
+      results[position] = await worker(items[position], position)
+    }
+  })
+  await Promise.all(runners)
+  return results
+}
+
+export const preTranslateMethods = {
+  /**
+   * Exact TM lookup by key, for pre-filling a block without scoring every entry.
+   * Mounted libraries only, matching the document's language pair. Each key gets
+   * one generic entry (no context) plus its context variants, so a segment can
+   * be resolved as ICE first and as a plain 100% match second.
+   */
+  tmExactIndex(doc) {
+    const mounted = new Set(this.mounts.tmIds)
+    const index = new Map()
+    for (const entry of this.tm) {
+      if (!mounted.has(entry.tmId || DEFAULT_TM_ID)) continue
+      if (entry.sourceLang !== doc.sourceLang || entry.targetLang !== doc.targetLang) continue
+      if (!entry.key) continue
+      let slot = index.get(entry.key)
+      if (!slot) {
+        slot = { generic: null, byContext: new Map(), latest: null }
+        index.set(entry.key, slot)
+      }
+      if (!slot.latest || entry.updatedAt > slot.latest.updatedAt) slot.latest = entry
+      if (!entry.contextHash) {
+        if (!slot.generic) slot.generic = entry
+      } else if (!slot.byContext.has(entry.contextHash)) {
+        slot.byContext.set(entry.contextHash, entry)
+      }
+    }
+    return index
+  }
+,
+  /**
+   * Resolve one segment against the exact index: context match first, then the
+   * generic entry, then any variant of the same text. A text match with a
+   * different context is still worth 100%, so it must not be dropped.
+   */
+  tmEntryForSegment(index, doc, segmentIndex, segment) {
+    const slot = index.get(normalizeText(segment.source))
+    if (!slot) return { entry: null, ice: false }
+    const hash = contextHashOf(contextOf(doc, segmentIndex))
+    if (hash) {
+      const ice = slot.byContext.get(hash)
+      if (ice) return { entry: ice, ice: true }
+    }
+    return { entry: slot.generic ?? slot.latest, ice: false }
+  }
+,
+  /**
+   * Pre-translate one block of segments: TM exact matches first, then machine
+   * translation. Never overwrites a segment that already has a target, so
+   * human work and earlier passes survive. Returns the next block offset so the
+   * client can drive progress and cancellation by looping.
+   */
+  async preTranslate({ documentId, from = 0, count = PRETRANSLATE_DEFAULT_COUNT, mode = 'mt', overwrite = false } = {}) {
+    const doc = this.requireDocument(documentId)
+    const total = doc.segments.length
+    if (!total) throw new CatError('NO_SEGMENTS', '当前文档没有句段。', 409)
+
+    const translationMode = String(mode ?? 'mt').trim() || 'mt'
+    if (!['mt', 'tm'].includes(translationMode)) {
+      throw new CatError('INVALID_PRETRANSLATE_MODE', '预翻译模式必须是 mt 或 tm。')
+    }
+
+    const start = clampInt(from, 0, Math.max(0, total - 1), 0)
+    const size = clampInt(count, 1, PRETRANSLATE_BLOCK_MAX, PRETRANSLATE_DEFAULT_COUNT)
+    const end = Math.min(total, start + size)
+    const tmIndex = this.tmExactIndex(doc)
+    const skipped = []
+    const work = []
+
+    for (let index = start; index < end; index += 1) {
+      const segment = doc.segments[index]
+      if (segment.status === 'confirmed') {
+        skipped.push({ index, reason: 'confirmed' })
+        continue
+      }
+      if (segment.target && !overwrite) {
+        skipped.push({ index, reason: 'translated' })
+        continue
+      }
+      work.push({ index, segment })
+    }
+
+    const failed = []
+    const filled = await runPool(work, PRETRANSLATE_CONCURRENCY, async ({ index, segment }) => {
+      const match = this.tmEntryForSegment(tmIndex, doc, index, segment)
+      try {
+        let text = ''
+        let origin = 'mt'
+        if (match.entry && match.entry.target.trim()) {
+          text = match.entry.target
+          origin = 'tm'
+        } else if (translationMode === 'tm') {
+          return { index, reason: 'no-tm-match' }
+        } else {
+          const result = await this.mtSuggest({
+            text: segment.source,
+            documentId: doc.id,
+            index,
+            sourceLang: doc.sourceLang,
+            targetLang: doc.targetLang,
+          })
+          text = String(result?.text ?? '').trim()
+          if (!text) throw new CatError('MT_EMPTY', '机器翻译没有返回译文。', 502)
+        }
+        segment.target = text
+        // Keep the baseline the machine produced: the review pane shows it in
+        // the suggestion column so the human can compare against their edit.
+        segment.suggestion = text
+        segment.origin = origin
+        segment.updatedAt = now()
+        normalizeSegmentState(segment)
+        return { index, status: segment.status, origin, ice: origin === 'tm' && match.ice }
+      } catch (error) {
+        failed.push({ index, code: error?.code || 'PRETRANSLATE_FAILED', message: error?.message || String(error) })
+        return null
+      }
+    })
+
+    const segments = filled.filter(Boolean)
+    const unchanged = segments.filter((item) => item.reason)
+    if (segments.length || unchanged.length) {
+      this.touch()
+      this.persist()
+    }
+
+    return {
+      documentId: doc.id,
+      from: start,
+      to: end,
+      count: end - start,
+      mode: translationMode,
+      filled: segments.filter((item) => !item.reason).length,
+      noMatch: unchanged.map((item) => item.index),
+      skipped,
+      failed,
+      progress: this.progress(doc),
+      next: end < total ? end : null,
+      segments: segments.filter((item) => !item.reason).map((item) => ({
+        index: item.index,
+        status: item.status,
+        origin: item.origin,
+        ice: Boolean(item.ice),
+      })),
+    }
+  }
+,
+  /** Segment listing for agents: bounded, status-filterable, cursor-free. */
+  segmentsForTool({ documentId, from = 0, count = 20, status } = {}) {
+    const doc = this.requireDocument(documentId)
+    const total = doc.segments.length
+    const start = clampInt(from, 0, Math.max(0, total - 1), 0)
+    const size = clampInt(count, 1, 50, 20)
+    const wanted = String(status ?? '').trim()
+    const rows = []
+    for (let index = start; index < total && rows.length < size; index += 1) {
+      const segment = doc.segments[index]
+      if (wanted && segment.status !== wanted) continue
+      rows.push({
+        index,
+        id: segment.id,
+        source: segment.source,
+        target: segment.target,
+        status: segment.status,
+        origin: segment.origin,
+        suggestion: segment.suggestion || '',
+      })
+    }
+    return {
+      document: {
+        id: doc.id,
+        name: doc.name,
+        sourceLang: doc.sourceLang,
+        targetLang: doc.targetLang,
+        total,
+        cursor: doc.cursor,
+      },
+      from: start,
+      segments: rows,
+      next: start + size < total ? start + size : null,
+      progress: this.progress(doc),
+    }
+  }
+,
+  /**
+   * Batch confirm. Segments without a target are reported, not invented; each
+   * confirmed pair goes through the normal confirmSegment path so the TM write
+   * and the strict persistence rollback keep working per segment.
+   */
+  confirmSegmentsForTool({ documentId, indices, from = 0, count = 20, status } = {}) {
+    const doc = this.requireDocument(documentId)
+    const total = doc.segments.length
+    let targets
+    if (Array.isArray(indices) && indices.length) {
+      targets = [...new Set(indices.map((index) => Math.trunc(Number(index))))].filter(
+        (index) => Number.isInteger(index) && index >= 0 && index < total,
+      )
+    } else if (status === 'translated') {
+      // "Confirm everything awaiting review" — the bulk action behind the
+      // review pane. Bounded so one click cannot confirm an enormous document
+      // without the user seeing a count first.
+      targets = []
+      for (let index = 0; index < total && targets.length < 500; index += 1) {
+        if (doc.segments[index].status === 'translated') targets.push(index)
+      }
+    } else {
+      const start = clampInt(from, 0, Math.max(0, total - 1), 0)
+      const size = clampInt(count, 1, 50, 20)
+      targets = []
+      for (let index = start; index < total && targets.length < size; index += 1) targets.push(index)
+    }
+
+    const confirmed = []
+    const skipped = []
+    for (const index of targets) {
+      const segment = doc.segments[index]
+      if (segment.status === 'confirmed') {
+        skipped.push({ index, reason: 'confirmed' })
+        continue
+      }
+      if (!segment.target) {
+        skipped.push({ index, reason: 'no-target' })
+        continue
+      }
+      this.confirmSegment({ documentId: doc.id, index, target: segment.target })
+      confirmed.push(index)
+    }
+
+    return {
+      documentId: doc.id,
+      confirmed,
+      skipped,
+      progress: this.progress(doc),
+      done: this.progress(doc).pending === 0 && this.progress(doc).translated === 0,
+    }
+  }
+}

package/lib/preprocess.js

@@ -0,0 +1,65 @@
+/**
+ * Import-time preprocessing stage.
+ *
+ * Parsers keep the source roughly as authored: pasted documents often contain
+ * non-breaking spaces, zero-width characters, CRLF line endings and stray
+ * indentation. None of that is meaningful for translation memory or MT, but it
+ * does leak into segments and makes matches fail. The stage is deliberately
+ * conservative: it only touches characters that carry no linguistic meaning and
+ * never rewrites punctuation, casing or inline placeholder tokens.
+ */
+export const DEFAULT_PREPROCESS = {
+  normalizeWhitespace: true,
+  stripInvisible: true,
+  collapseBlankLines: true,
+}
+
+const INVISIBLE = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u200B-\u200F\u202A-\u202E\u2060-\u2064\uFEFF]/g
+
+export function normalizePreprocessOptions(value) {
+  const source = value && typeof value === 'object' && !Array.isArray(value) ? value : {}
+  return {
+    normalizeWhitespace:
+      typeof source.normalizeWhitespace === 'boolean'
+        ? source.normalizeWhitespace
+        : DEFAULT_PREPROCESS.normalizeWhitespace,
+    stripInvisible:
+      typeof source.stripInvisible === 'boolean' ? source.stripInvisible : DEFAULT_PREPROCESS.stripInvisible,
+    collapseBlankLines:
+      typeof source.collapseBlankLines === 'boolean'
+        ? source.collapseBlankLines
+        : DEFAULT_PREPROCESS.collapseBlankLines,
+  }
+}
+
+export function preprocessText(value, options = DEFAULT_PREPROCESS) {
+  const config = normalizePreprocessOptions(options)
+  let text = String(value ?? '')
+  // Line endings carry no linguistic meaning and would otherwise split CRLF
+  // imports into different segment fingerprints than the same text pasted in.
+  text = text.replace(/\r\n?/g, '\n')
+  if (config.stripInvisible) text = text.replace(INVISIBLE, '')
+  if (config.normalizeWhitespace) {
+    text = text
+      .replace(/\u00A0/g, ' ')
+      .replace(/[ \t]+\n/g, '\n')
+      .replace(/[ \t]{2,}/g, ' ')
+      .replace(/\n[ \t]+/g, '\n')
+  }
+  if (config.collapseBlankLines) text = text.replace(/\n{3,}/g, '\n\n')
+  return config.normalizeWhitespace || config.collapseBlankLines ? text.trim() : text
+}
+
+/**
+ * Apply the configured stage to parsed import segments. Inline tokens such as
+ * `{1}` are untouched because they contain no whitespace or control characters.
+ */
+export function preprocessSegments(segments, options) {
+  const config = normalizePreprocessOptions(options)
+  return segments.map((segment) => {
+    if (!segment || typeof segment !== 'object') return segment
+    const source = preprocessText(segment.source, config)
+    if (source === segment.source) return segment
+    return { ...segment, source }
+  })
+}

package/lib/propagate.js

@@ -0,0 +1,133 @@
+import { normalizeText } from './similarity.js'
+import { normalizeSegmentState } from './store-utils.js'
+
+/**
+ * Segments that repeat the same source text inside one document. The map is
+ * rebuilt on demand: documents are small (MAX_SEGMENTS 20000) and keeping an
+ * index in every document would have to be migrated forever.
+ */
+function sourceGroups(doc) {
+  const groups = new Map()
+  doc.segments.forEach((segment, index) => {
+    const key = normalizeText(segment.source)
+    if (!key) return
+    const bucket = groups.get(key)
+    if (bucket) bucket.push(index)
+    else groups.set(key, [index])
+  })
+  return groups
+}
+
+function groupOf(doc, index) {
+  const key = normalizeText(doc.segments[index]?.source)
+  if (!key) return []
+  return sourceGroups(doc).get(key) ?? []
+}
+
+/**
+ * The text a repetition should copy: a confirmed segment wins, then a human
+ * edit, then anything with text at all.
+ */
+function referenceOf(doc, indices) {
+  let fallback = null
+  for (const index of indices) {
+    const segment = doc.segments[index]
+    if (!segment?.target) continue
+    if (segment.status === 'confirmed') return { index, segment }
+    if (segment.origin === 'manual') return { index, segment }
+    if (!fallback) fallback = { index, segment }
+  }
+  return fallback
+}
+
+export const propagateMethods = {
+  sourceGroups(doc) {
+    return sourceGroups(doc)
+  },
+
+  /**
+   * Fill unconfirmed repetitions of `index`'s source with its target. Called
+   * right after a confirmation, so one decision settles every duplicate.
+   */
+  propagateFromSegment(doc, index) {
+    const group = groupOf(doc, index)
+    if (group.length < 2) return 0
+    const reference = referenceOf(doc, group.filter((item) => item !== index)) || {
+      index,
+      segment: doc.segments[index],
+    }
+    const source = reference.segment?.target
+    if (!source) return 0
+    let filled = 0
+    for (const item of group) {
+      if (item === index) continue
+      const segment = doc.segments[item]
+      if (segment.status === 'confirmed') continue
+      if (segment.target === source) continue
+      segment.target = source
+      segment.origin = 'auto'
+      segment.updatedAt = nowIso(reference.segment.updatedAt)
+      normalizeSegmentState(segment)
+      filled += 1
+    }
+    if (filled) doc.updatedAt = doc.segments[index].updatedAt
+    return filled
+  },
+
+  /**
+   * A human rewrote one copy: siblings that nobody has confirmed yet follow,
+   * otherwise the document would keep a pile of stale duplicates. Confirmed
+   * siblings are never touched.
+   */
+  propagateEdit(doc, index) {
+    const segment = doc.segments[index]
+    if (!segment?.target || segment.status === 'confirmed') return 0
+    const group = groupOf(doc, index)
+    if (group.length < 2) return 0
+    let filled = 0
+    for (const item of group) {
+      if (item === index) continue
+      const sibling = doc.segments[item]
+      if (sibling.status === 'confirmed') continue
+      if (sibling.target === segment.target) continue
+      sibling.target = segment.target
+      sibling.origin = 'auto'
+      sibling.updatedAt = segment.updatedAt
+      normalizeSegmentState(sibling)
+      filled += 1
+    }
+    return filled
+  },
+
+  propagateDocument({ documentId } = {}) {
+    const doc = this.requireDocument(documentId)
+    const groups = sourceGroups(doc)
+    let filled = 0
+    for (const indices of groups.values()) {
+      if (indices.length < 2) continue
+      const reference = referenceOf(doc, indices)
+      if (!reference?.segment?.target) continue
+      for (const index of indices) {
+        if (index === reference.index) continue
+        const segment = doc.segments[index]
+        if (segment.status === 'confirmed') continue
+        if (segment.target === reference.segment.target) continue
+        segment.target = reference.segment.target
+        segment.origin = 'auto'
+        segment.updatedAt = reference.segment.updatedAt
+        normalizeSegmentState(segment)
+        filled += 1
+      }
+    }
+    if (filled) {
+      doc.updatedAt = new Date().toISOString()
+      this.touch()
+      this.persist()
+    }
+    return { filled, groups: groups.size, progress: this.progress(doc) }
+  },
+}
+
+function nowIso(value) {
+  return typeof value === 'string' && value ? value : new Date().toISOString()
+}

package/lib/qa.js

@@ -0,0 +1,167 @@
+import { missingInlineTokens } from './formats.js'
+import { normalizeText } from './similarity.js'
+import { clampInt, DEFAULT_GLOSSARY_ID } from './store-utils.js'
+
+const TOKEN_PATTERN = /\{\d+\}/g
+const NUMBER_PATTERN = /\d+(?:[.,]\d+)*/g
+
+/** Inline tokens are `{0}`, `{1}`… — the only markup CAIT keeps verbatim. */
+function tokensIn(text) {
+  return String(text ?? '').match(TOKEN_PATTERN) || []
+}
+
+/**
+ * Numbers are compared as a multiset: "1,200" and "1200" are the same value,
+ * while "2024" and "2025" are not. Thousands separators are the only thing we
+ * normalise, because that is the one difference translators introduce on purpose.
+ */
+function numbersIn(text) {
+  // `{0}` is a placeholder, not the number zero — strip tokens before scanning.
+  const raw = String(text ?? '').replace(/\{\d+\}/g, ' ').match(NUMBER_PATTERN) || []
+  return raw.map((value) => value.replace(/(\d),(?=\d{3}(?:\D|$))/g, '$1'))
+}
+
+function sameMultiset(left, right) {
+  if (left.length !== right.length) return false
+  const a = [...left].sort()
+  const b = [...right].sort()
+  return a.every((value, index) => value === b[index])
+}
+
+/**
+ * Lightweight QA. Everything here is advisory: it never blocks an export and
+ * nothing is stored, so the checks can evolve without a migration.
+ */
+export const qaMethods = {
+  /**
+   * Groups of segments that share one source but disagree on the target. The
+   * mechanical part of a consistency review; an agent can then judge which
+   * variant is right and propose a fix.
+   */
+  consistencyReport({ documentId, limit = 50 } = {}) {
+    const doc = this.requireDocument(documentId)
+    const groups = new Map()
+    doc.segments.forEach((segment, index) => {
+      const key = normalizeText(segment.source)
+      if (!key) return
+      const bucket = groups.get(key)
+      const item = { index, target: String(segment.target ?? '') }
+      if (bucket) bucket.push(item)
+      else groups.set(key, [item])
+    })
+    const conflicts = []
+    for (const [key, items] of groups) {
+      const variants = new Map()
+      for (const item of items) {
+        const value = normalizeText(item.target)
+        if (!value) continue
+        if (!variants.has(value)) variants.set(value, [])
+        variants.get(value).push(item.index)
+      }
+      if (variants.size < 2) continue
+      conflicts.push({
+        source: items[0] ? doc.segments[items[0].index].source : key,
+        variants: [...variants.entries()].map(([target, indices]) => ({ target, indices })),
+        indices: items.map((item) => item.index),
+      })
+    }
+    return {
+      documentId: doc.id,
+      conflicts: conflicts.slice(0, clampInt(limit, 1, 200, 50)),
+      total: conflicts.length,
+    }
+  },
+
+  getQaReport({ documentId } = {}) {
+    const doc = this.requireDocument(documentId)
+    const mounted = new Set(this.mounts.glossaryIds)
+    const terms = this.glossary.filter(
+      (entry) =>
+        mounted.has(entry.glossaryId || DEFAULT_GLOSSARY_ID) &&
+        (!entry.sourceLang || entry.sourceLang === doc.sourceLang),
+    )
+    const issues = []
+    const push = (index, code, message) => issues.push({ index, code, message })
+    let checked = 0
+    let withText = 0
+    // Group by source so a repeated sentence with two different renderings is
+    // visible: propagation makes this the most likely mistake after an import.
+    const bySource = new Map()
+
+    for (let index = 0; index < doc.segments.length; index += 1) {
+      const segment = doc.segments[index]
+      const target = String(segment.target ?? '')
+      checked += 1
+
+      if (segment.status !== 'pending' && !target.trim()) {
+        push(index, 'EMPTY_TARGET', '句段标记为已翻译/已确认，但译文为空。')
+        continue
+      }
+      if (!target.trim()) continue
+      withText += 1
+
+      const missing = missingInlineTokens(segment.meta?.inline, target)
+      for (const token of missing) {
+        push(index, 'PLACEHOLDER_MISSING', `译文缺少标签占位符 ${token}。`)
+      }
+      const sourceTokens = new Set(tokensIn(segment.source))
+      for (const token of sourceTokens) {
+        if (!missing.includes(token) && !target.includes(token)) {
+          push(index, 'PLACEHOLDER_MISSING', `译文缺少占位符 ${token}。`)
+        }
+      }
+
+      if (!sameMultiset(numbersIn(segment.source), numbersIn(target))) {
+        push(index, 'NUMBER_MISMATCH', '源文与译文中的数字不一致。')
+      }
+
+      for (const entry of terms) {
+        const present = entry.caseSensitive
+          ? segment.source.includes(entry.source)
+          : segment.source.toLocaleLowerCase().includes(entry.source.toLocaleLowerCase())
+        if (!present) continue
+        const used = entry.caseSensitive
+          ? target.includes(entry.target)
+          : target.toLocaleLowerCase().includes(entry.target.toLocaleLowerCase())
+        if (!used) push(index, 'TERM_NOT_USED', `术语「${entry.source} → ${entry.target}」未在译文中出现。`)
+      }
+
+      // Copying the source into the target is the most common disguise for a
+      // missed translation. Pure digits or punctuation are identical by nature.
+      if (/\p{L}/u.test(segment.source) && normalizeText(target) === normalizeText(segment.source)) {
+        push(index, 'UNTRANSLATED_RESIDUE', '译文与源文完全相同，可能是漏译。')
+      }
+
+      const sourceKey = normalizeText(segment.source)
+      if (sourceKey) {
+        const bucket = bySource.get(sourceKey)
+        if (bucket) bucket.push({ index, target: normalizeText(target) })
+        else bySource.set(sourceKey, [{ index, target: normalizeText(target) }])
+      }
+    }
+
+    for (const group of bySource.values()) {
+      if (group.length < 2) continue
+      const variants = new Set(group.map((item) => item.target))
+      if (variants.size < 2) continue
+      for (const item of group) {
+        push(item.index, 'INCONSISTENT_TARGET', `同一源文出现了 ${variants.size} 种不同译文。`)
+      }
+    }
+
+    const counts = issues.reduce(
+      (acc, issue) => ({ ...acc, [issue.code]: (acc[issue.code] || 0) + 1 }),
+      { PLACEHOLDER_MISSING: 0, NUMBER_MISMATCH: 0, TERM_NOT_USED: 0, EMPTY_TARGET: 0, UNTRANSLATED_RESIDUE: 0, INCONSISTENT_TARGET: 0 },
+    )
+
+    return {
+      documentId: doc.id,
+      total: doc.segments.length,
+      checked,
+      withText,
+      issueCount: issues.length,
+      counts,
+      issues,
+    }
+  },
+}