dsh-cait 1.0.4 → 1.0.5

package/lib/html.js

@@ -0,0 +1,271 @@
+/**
+ * Dependency-free HTML/XHTML block scanner shared by the HTML and EPUB
+ * adapters.
+ *
+ * A real DOM is unnecessary for CAT import: what matters is a stable mapping
+ * between translatable inline runs and exact byte ranges of the source file.
+ * The scanner records contiguous runs between block-level elements, so export
+ * can replace exactly those ranges while leaving structure, attributes,
+ * scripts and styles untouched.
+ */
+import { decodeXml, escapeXml } from './xml.js'
+
+const VOID_TAGS = new Set([
+  'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link',
+  'meta', 'param', 'source', 'track', 'wbr',
+])
+
+const BLOCK_TAGS = new Set([
+  'address', 'article', 'aside', 'blockquote', 'body', 'caption', 'dd', 'details',
+  'dialog', 'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer',
+  'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'html',
+  'li', 'main', 'nav', 'ol', 'p', 'pre', 'section', 'summary', 'table',
+  'tbody', 'td', 'tfoot', 'th', 'thead', 'tr', 'ul',
+])
+
+const SKIP_TAGS = new Set([
+  'canvas', 'head', 'iframe', 'math', 'noscript', 'object', 'rp', 'rt',
+  'script', 'style', 'svg', 'template',
+])
+
+const TAG_PATTERN = /<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<![^>]*>|<\?[\s\S]*?\?>|<(\/?)([A-Za-z][\w:.-]*)((?:"[^"]*"|'[^']*'|[^>"'])*?)(\/?)>/g
+
+function parseAttributes(source) {
+  const attrs = {}
+  const pattern = /([^\s=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g
+  let match
+  while ((match = pattern.exec(String(source ?? '')))) {
+    const name = match[1].toLowerCase()
+    attrs[name] = decodeXml(match[2] ?? match[3] ?? match[4] ?? '')
+  }
+  return attrs
+}
+
+function escapeRegExp(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+function findClosingTag(html, tag, from) {
+  const pattern = new RegExp(`</${escapeRegExp(tag)}\\s*>`, 'ig')
+  pattern.lastIndex = from
+  const match = pattern.exec(html)
+  return match ? { start: match.index, end: match.index + match[0].length } : null
+}
+
+function isHidden(node) {
+  const attrs = node.attrs || {}
+  if (Object.prototype.hasOwnProperty.call(attrs, 'hidden')) return true
+  if (String(attrs['aria-hidden'] || '').toLowerCase() === 'true') return true
+  if (/\bdisplay\s*:\s*none\b/i.test(attrs.style || '')) return true
+  return false
+}
+
+/**
+ * Tokenise inline markup into stable {n} placeholders so tags survive
+ * translation and export can restore them verbatim.
+ */
+export function tokenizeHtml(raw, { preformatted = false } = {}) {
+  const tokens = []
+  const stripped = String(raw ?? '').replace(
+    /<!--[\s\S]*?-->|<\/?[A-Za-z][\w:.-]*(?:"[^"]*"|'[^']*'|[^>"'])*?\/?>/g,
+    (xml) => {
+      if (xml.startsWith('<!--')) return ''
+      const token = `{${tokens.length + 1}}`
+      tokens.push({ token, xml })
+      return token
+    },
+  )
+  let text = decodeXml(stripped).replace(/\r\n?/g, '\n')
+  if (preformatted) {
+    text = text.replace(/[ \t]+\n/g, '\n').replace(/[ \t]{2,}/g, ' ').replace(/^\n+|\n+$/g, '')
+  } else {
+    text = text.replace(/\s+/g, ' ')
+  }
+  return { text: text.trim(), tokens }
+}
+
+/** Render a tokenised string back to HTML, escaping only text portions. */
+export function renderHtmlTokens(value, tokens) {
+  const text = String(value ?? '')
+  const map = new Map((Array.isArray(tokens) ? tokens : []).map((item) => [item.token, item.xml]))
+  const tokenPattern = /\{(\d+)\}/g
+  let output = ''
+  let last = 0
+  for (const match of text.matchAll(tokenPattern)) {
+    output += escapeXml(text.slice(last, match.index))
+    output += map.get(match[0]) ?? escapeXml(match[0])
+    last = match.index + match[0].length
+  }
+  output += escapeXml(text.slice(last))
+  return output
+}
+
+/** Plain-text view of an HTML fragment (used for metadata and fallbacks). */
+export function htmlFragmentToText(raw) {
+  return decodeXml(
+    String(raw ?? '')
+      .replace(/<!--[\s\S]*?-->/g, '')
+      .replace(/<(script|style|noscript|head)\b[\s\S]*?<\/\1>/gi, '')
+      .replace(/<br\s*\/?>/gi, '\n')
+      .replace(/<\/(?:p|div|li|h[1-6]|section|article|blockquote|pre|tr)>/gi, '\n\n')
+      .replace(/<\/(?:td|th)>/gi, '\t')
+      .replace(/<[^>]+>/g, ''),
+  )
+    .replace(/[ \t]+\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim()
+}
+
+function parseTree(html) {
+  const root = { type: 'element', tag: '#document', attrs: {}, children: [] }
+  const stack = [root]
+  let cursor = 0
+
+  const closeTo = (depth, closeStart, end) => {
+    while (stack.length > depth) {
+      const node = stack.pop()
+      node.closeStart = closeStart
+      node.end = end
+    }
+  }
+
+  TAG_PATTERN.lastIndex = 0
+  let match
+  while ((match = TAG_PATTERN.exec(html))) {
+    if (match.index > cursor) {
+      stack[stack.length - 1].children.push({
+        type: 'text',
+        start: cursor,
+        end: match.index,
+      })
+    }
+    cursor = match.index + match[0].length
+    if (!match[2]) continue
+
+    const rawTag = match[2]
+    const tag = rawTag.toLowerCase()
+    if (match[1] === '/') {
+      let depth = stack.length - 1
+      while (depth > 0 && stack[depth].tag !== tag) depth -= 1
+      if (depth > 0) closeTo(depth, match.index, cursor)
+      continue
+    }
+
+    const node = {
+      type: 'element',
+      tag,
+      attrs: parseAttributes(match[3]),
+      children: [],
+      start: match.index,
+      openEnd: cursor,
+      closeStart: null,
+      end: null,
+    }
+    stack[stack.length - 1].children.push(node)
+
+    if (match[4] === '/' || VOID_TAGS.has(tag)) {
+      node.closeStart = cursor
+      node.end = cursor
+      continue
+    }
+
+    if (SKIP_TAGS.has(tag)) {
+      const close = findClosingTag(html, rawTag, cursor)
+      node.closeStart = close ? close.start : html.length
+      node.end = close ? close.end : html.length
+      cursor = node.end
+      TAG_PATTERN.lastIndex = cursor
+      continue
+    }
+
+    stack.push(node)
+  }
+
+  if (cursor < html.length) {
+    stack[stack.length - 1].children.push({ type: 'text', start: cursor, end: html.length })
+  }
+  closeTo(1, html.length, html.length)
+  return root
+}
+
+function collectRuns(html, root) {
+  const runs = []
+
+  const walk = (node) => {
+    let runStart = null
+    let runEnd = null
+    const preformatted = node.tag === 'pre'
+
+    const flush = () => {
+      if (runStart === null) return
+      const { text, tokens } = tokenizeHtml(html.slice(runStart, runEnd), { preformatted })
+      if (text) {
+        runs.push({ index: runs.length, start: runStart, end: runEnd, text, tokens })
+      }
+      runStart = null
+      runEnd = null
+    }
+
+    for (const child of node.children) {
+      if (child.type === 'text') {
+        if (runStart !== null && child.start > runEnd) flush()
+        if (runStart === null) runStart = child.start
+        runEnd = child.end
+        continue
+      }
+      if (SKIP_TAGS.has(child.tag) || isHidden(child)) {
+        // Do not let a run span across skipped markup: patching the range
+        // would otherwise delete the skipped element.
+        flush()
+        continue
+      }
+      if (BLOCK_TAGS.has(child.tag)) {
+        flush()
+        walk(child)
+        continue
+      }
+      if (runStart !== null && child.start > runEnd) flush()
+      if (runStart === null) runStart = child.start
+      runEnd = child.end
+    }
+    flush()
+  }
+
+  walk(root)
+  return runs
+}
+
+/**
+ * Parse an HTML/XHTML document into ordered inline runs. `runs[n].start` and
+ * `runs[n].end` index into the original string, which is what EPUB export
+ * needs to patch translated text back without reserialising the document.
+ */
+export function parseHtmlBlocks(input) {
+  // Keep the original string untouched: run offsets must index into the exact
+  // bytes that EPUB export will patch later.
+  const html = String(input ?? '')
+  const titleMatch = html.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i)
+  const root = parseTree(html)
+  const runs = collectRuns(html, root)
+  return {
+    runs,
+    title: titleMatch ? htmlFragmentToText(titleMatch[1]) : '',
+    warnings: [],
+  }
+}
+
+/** Replace run ranges with already-rendered HTML. */
+export function patchHtmlRuns(html, replacements) {
+  const source = String(html ?? '')
+  const { runs } = parseHtmlBlocks(source)
+  let output = ''
+  let last = 0
+  for (const run of runs) {
+    if (!replacements.has(run.index)) continue
+    if (run.start < last) continue
+    output += source.slice(last, run.start)
+    output += replacements.get(run.index)
+    last = run.end
+  }
+  return output + source.slice(last)
+}

package/lib/import-export.js

@@ -4,120 +4,117 @@ import { CatError } from './errors.js'
 import { MAX_DOCX_BASE64, MAX_IMPORT_CHARS, MAX_SEGMENTS, MAX_SEGMENT_CHARS, canonicalLanguage, cleanLang, cleanName, cleanText, csvCell, makeId, mdCell, now, safeBaseName } from './store-utils.js'
 import { stripMarkdown } from './markdown.js'
 import { DOCX_MIME, buildDocx, buildDocxFromText, parseDocxDetailed } from './docx.js'
+import { EPUB_MIME, buildEpub, buildEpubFromText, parseEpub } from './epub.js'
 import { PPTX_MIME, buildPptx, parsePptxDetailed } from './pptx.js'
 import { XLSX_MIME, buildXlsx, parseXlsx } from './xlsx.js'
 import { buildTmx, parseTmx } from './tmx.js'
 import { buildHtml, buildJson, buildPo, buildRtf, buildSrt, buildVtt, buildXliff, buildXliff2, parseCsv, parseHtml, parseJson, parsePo, parseRtf, parseSrt, parseVtt, parseXliff } from './formats.js'
 import { decodeXml } from './xml.js'
 
-export const importExportMethods = {
-  /**
-   * Keep the original DOCX package so exporting can patch translated text into
-   * the source layout. Persisted stores use a small sidecar file; transient
-   * in-memory stores keep the bytes in the document object only.
-   */
-  attachDocxSource(doc, buffer, name = 'source.docx') {
-    if (!Buffer.isBuffer(buffer) || !buffer.length) return
-    const fileName = cleanName(name, 'source.docx')
-    if (this.dataDir) {
-      const asset = `sources/${doc.id}.docx`
-      try {
-        mkdirSync(join(this.dataDir, 'sources'), { recursive: true })
-        writeFileSync(join(this.dataDir, asset), buffer)
-      } catch (error) {
-        throw new CatError(
-          'DOCX_SOURCE_WRITE_FAILED',
-          `无法保存 DOCX 源文件，无法保留源格式: ${error.message}`,
-          500,
-        )
+const BINARY_SOURCES = {
+  docx: { label: 'DOCX', fallback: 'source.docx' },
+  pptx: { label: 'PPTX', fallback: 'source.pptx' },
+  epub: { label: 'EPUB', fallback: 'source.epub' },
+}
+
+function sourceField(kind) {
+  return `source${kind[0].toUpperCase()}${kind.slice(1)}`
+}
+
+/**
+ * Keep the original binary package so exporting can patch translated text into
+ * the source layout. Persisted stores use a small sidecar file; transient
+ * in-memory stores keep the bytes in the document object only.
+ */
+function attachSource(store, doc, kind, buffer, name) {
+  if (!Buffer.isBuffer(buffer) || !buffer.length) return
+  const config = BINARY_SOURCES[kind]
+  const fileName = cleanName(name, config.fallback)
+  if (store.dataDir) {
+    const asset = `sources/${doc.id}.${kind}`
+    try {
+      mkdirSync(join(store.dataDir, 'sources'), { recursive: true })
+      writeFileSync(join(store.dataDir, asset), buffer)
+    } catch (error) {
+      throw new CatError(
+        `${kind.toUpperCase()}_SOURCE_WRITE_FAILED`,
+        `无法保存 ${config.label} 源文件，无法保留源格式: ${error.message}`,
+        500,
+      )
+    }
+    doc[sourceField(kind)] = { kind: 'file', asset, name: fileName }
+    return
+  }
+  doc[sourceField(kind)] = { kind: 'base64', data: buffer.toString('base64'), name: fileName }
+}
+
+function removeSource(store, doc, kind) {
+  const field = sourceField(kind)
+  const source = doc?.[field]
+  if (!source) return
+  if (source.kind === 'file' && store.dataDir) {
+    try {
+      unlinkSync(join(store.dataDir, source.asset))
+    } catch (error) {
+      if (error?.code !== 'ENOENT') {
+        store.logger?.warn?.(`CAIT: 删除 ${BINARY_SOURCES[kind].label} 源文件失败: ${error.message}`)
       }
-      doc.sourceDocx = { kind: 'file', asset, name: fileName }
-      return
     }
-    doc.sourceDocx = { kind: 'base64', data: buffer.toString('base64'), name: fileName }
+  }
+  delete doc[field]
+}
+
+function readSource(store, doc, kind) {
+  const source = doc?.[sourceField(kind)]
+  if (!source) return null
+  try {
+    if (source.kind === 'base64' && source.data) {
+      return Buffer.from(source.data, 'base64')
+    }
+    if (source.kind === 'file' && store.dataDir && source.asset) {
+      return readFileSync(join(store.dataDir, source.asset))
+    }
+  } catch (error) {
+    store.logger?.warn?.(`CAIT: 读取 ${BINARY_SOURCES[kind].label} 源文件失败: ${error.message}`)
+  }
+  return null
+}
+
+export const importExportMethods = {
+  attachDocxSource(doc, buffer, name = 'source.docx') {
+    attachSource(this, doc, 'docx', buffer, name)
   },
 
   removeDocxSource(doc) {
-    const source = doc?.sourceDocx
-    if (!source) return
-    if (source.kind === 'file' && this.dataDir) {
-      try {
-        unlinkSync(join(this.dataDir, source.asset))
-      } catch (error) {
-        if (error?.code !== 'ENOENT') {
-          this.logger?.warn?.(`CAIT: 删除 DOCX 源文件失败: ${error.message}`)
-        }
-      }
-    }
-    delete doc.sourceDocx
+    removeSource(this, doc, 'docx')
   },
 
   readDocxSource(doc) {
-    const source = doc?.sourceDocx
-    if (!source) return null
-    try {
-      if (source.kind === 'base64' && source.data) {
-        return Buffer.from(source.data, 'base64')
-      }
-      if (source.kind === 'file' && this.dataDir && source.asset) {
-        return readFileSync(join(this.dataDir, source.asset))
-      }
-    } catch (error) {
-      this.logger?.warn?.(`CAIT: 读取 DOCX 源文件失败: ${error.message}`)
-    }
-    return null
+    return readSource(this, doc, 'docx')
   },
 
   attachPptxSource(doc, buffer, name = 'source.pptx') {
-    if (!Buffer.isBuffer(buffer) || !buffer.length) return
-    const fileName = cleanName(name, 'source.pptx')
-    if (this.dataDir) {
-      const asset = `sources/${doc.id}.pptx`
-      try {
-        mkdirSync(join(this.dataDir, 'sources'), { recursive: true })
-        writeFileSync(join(this.dataDir, asset), buffer)
-      } catch (error) {
-        throw new CatError(
-          'PPTX_SOURCE_WRITE_FAILED',
-          `无法保存 PPTX 源文件，无法保留源格式: ${error.message}`,
-          500,
-        )
-      }
-      doc.sourcePptx = { kind: 'file', asset, name: fileName }
-      return
-    }
-    doc.sourcePptx = { kind: 'base64', data: buffer.toString('base64'), name: fileName }
+    attachSource(this, doc, 'pptx', buffer, name)
   },
 
   removePptxSource(doc) {
-    const source = doc?.sourcePptx
-    if (!source) return
-    if (source.kind === 'file' && this.dataDir) {
-      try {
-        unlinkSync(join(this.dataDir, source.asset))
-      } catch (error) {
-        if (error?.code !== 'ENOENT') {
-          this.logger?.warn?.(`CAIT: 删除 PPTX 源文件失败: ${error.message}`)
-        }
-      }
-    }
-    delete doc.sourcePptx
+    removeSource(this, doc, 'pptx')
   },
 
   readPptxSource(doc) {
-    const source = doc?.sourcePptx
-    if (!source) return null
-    try {
-      if (source.kind === 'base64' && source.data) {
-        return Buffer.from(source.data, 'base64')
-      }
-      if (source.kind === 'file' && this.dataDir && source.asset) {
-        return readFileSync(join(this.dataDir, source.asset))
-      }
-    } catch (error) {
-      this.logger?.warn?.(`CAIT: 读取 PPTX 源文件失败: ${error.message}`)
-    }
-    return null
+    return readSource(this, doc, 'pptx')
+  },
+
+  attachEpubSource(doc, buffer, name = 'source.epub') {
+    attachSource(this, doc, 'epub', buffer, name)
+  },
+
+  removeEpubSource(doc) {
+    removeSource(this, doc, 'epub')
+  },
+
+  readEpubSource(doc) {
+    return readSource(this, doc, 'epub')
   },
 
   importTmx({ text, name, sourceLang, targetLang, importAs = 'tm', tmId } = {}) {
@@ -204,6 +201,35 @@ export const importExportMethods = {
       })
     }
 
+    if (extension === 'epub') {
+      if (!base64) throw new CatError('EMPTY_FILE', 'EPUB 文件内容为空。')
+      if (base64.length > MAX_DOCX_BASE64) throw new CatError('FILE_TOO_LARGE', 'EPUB 文件过大。')
+      const buffer = Buffer.from(base64, 'base64')
+      let parsed
+      try {
+        parsed = parseEpub(buffer)
+      } catch (error) {
+        if (error instanceof CatError) throw error
+        throw new CatError('EPUB_PARSE', `EPUB 解析失败: ${error?.message ?? error}`)
+      }
+      const created = this.createDocument({
+        name: fileName,
+        segments: parsed.segments,
+        sourceLang: sourceLang || parsed.language,
+        targetLang,
+        format: 'epub',
+        epubSource: buffer,
+        epubName: fileName,
+      })
+      return {
+        ...created,
+        sourceLang: created.document.sourceLang,
+        targetLang: created.document.targetLang,
+        units: parsed.segments.length,
+        warnings: parsed.warnings ?? [],
+      }
+    }
+
     if (extension === 'xlsx' || extension === 'pptx') {
       if (!base64) throw new CatError('EMPTY_FILE', `${extension.toUpperCase()} 文件内容为空。`)
       if (base64.length > MAX_DOCX_BASE64) throw new CatError('FILE_TOO_LARGE', `${extension.toUpperCase()} 文件过大。`)
@@ -385,6 +411,25 @@ export const importExportMethods = {
       }
     }
 
+    if (format === 'epub') {
+      let content = null
+      const template = this.readEpubSource(doc)
+      if (template) {
+        try {
+          content = buildEpub(doc, template)
+        } catch (error) {
+          this.logger?.warn?.(`CAIT: EPUB 模板套用失败，改用纯文本 EPUB: ${error.message}`)
+        }
+      }
+      if (!content) content = buildEpubFromText(doc)
+      return {
+        filename: `${base}.${doc.targetLang}.epub`,
+        mime: EPUB_MIME,
+        content,
+        count: doc.segments.length,
+      }
+    }
+
     if (format === 'srt') {
       return {
         filename: `${base}.${doc.targetLang}.srt`,

package/lib/persistence.js

@@ -144,6 +144,20 @@ export const persistenceMethods = {
           name: cleanName(sourcePptx.name, 'source.pptx'),
         }
       }
+      const sourceEpub = rawDoc.sourceEpub
+      if (
+        sourceEpub &&
+        typeof sourceEpub === 'object' &&
+        sourceEpub.kind === 'file' &&
+        typeof sourceEpub.asset === 'string' &&
+        /^sources\/[A-Za-z0-9_.-]+\.epub$/.test(sourceEpub.asset)
+      ) {
+        doc.sourceEpub = {
+          kind: 'file',
+          asset: sourceEpub.asset,
+          name: cleanName(sourceEpub.name, 'source.epub'),
+        }
+      }
       if (docs.has(doc.id)) throw new Error(`文档 id 重复: ${doc.id}`)
       docs.set(doc.id, doc)
     }
@@ -259,6 +273,12 @@ export const persistenceMethods = {
       targetLang: this.settings.targetLang,
       fuzzyThreshold: this.settings.fuzzyThreshold,
       tmLimit: this.settings.tmLimit,
+      uiLang: this.settings.uiLang,
+      preprocess: {
+        normalizeWhitespace: this.settings.preprocess.normalizeWhitespace,
+        stripInvisible: this.settings.preprocess.stripInvisible,
+        collapseBlankLines: this.settings.preprocess.collapseBlankLines,
+      },
       mt: {
         provider: this.settings.mt.provider,
         model: this.settings.mt.model,

package/lib/preprocess.js

@@ -0,0 +1,65 @@
+/**
+ * Import-time preprocessing stage.
+ *
+ * Parsers keep the source roughly as authored: pasted documents often contain
+ * non-breaking spaces, zero-width characters, CRLF line endings and stray
+ * indentation. None of that is meaningful for translation memory or MT, but it
+ * does leak into segments and makes matches fail. The stage is deliberately
+ * conservative: it only touches characters that carry no linguistic meaning and
+ * never rewrites punctuation, casing or inline placeholder tokens.
+ */
+export const DEFAULT_PREPROCESS = {
+  normalizeWhitespace: true,
+  stripInvisible: true,
+  collapseBlankLines: true,
+}
+
+const INVISIBLE = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u200B-\u200F\u202A-\u202E\u2060-\u2064\uFEFF]/g
+
+export function normalizePreprocessOptions(value) {
+  const source = value && typeof value === 'object' && !Array.isArray(value) ? value : {}
+  return {
+    normalizeWhitespace:
+      typeof source.normalizeWhitespace === 'boolean'
+        ? source.normalizeWhitespace
+        : DEFAULT_PREPROCESS.normalizeWhitespace,
+    stripInvisible:
+      typeof source.stripInvisible === 'boolean' ? source.stripInvisible : DEFAULT_PREPROCESS.stripInvisible,
+    collapseBlankLines:
+      typeof source.collapseBlankLines === 'boolean'
+        ? source.collapseBlankLines
+        : DEFAULT_PREPROCESS.collapseBlankLines,
+  }
+}
+
+export function preprocessText(value, options = DEFAULT_PREPROCESS) {
+  const config = normalizePreprocessOptions(options)
+  let text = String(value ?? '')
+  // Line endings carry no linguistic meaning and would otherwise split CRLF
+  // imports into different segment fingerprints than the same text pasted in.
+  text = text.replace(/\r\n?/g, '\n')
+  if (config.stripInvisible) text = text.replace(INVISIBLE, '')
+  if (config.normalizeWhitespace) {
+    text = text
+      .replace(/\u00A0/g, ' ')
+      .replace(/[ \t]+\n/g, '\n')
+      .replace(/[ \t]{2,}/g, ' ')
+      .replace(/\n[ \t]+/g, '\n')
+  }
+  if (config.collapseBlankLines) text = text.replace(/\n{3,}/g, '\n\n')
+  return config.normalizeWhitespace || config.collapseBlankLines ? text.trim() : text
+}
+
+/**
+ * Apply the configured stage to parsed import segments. Inline tokens such as
+ * `{1}` are untouched because they contain no whitespace or control characters.
+ */
+export function preprocessSegments(segments, options) {
+  const config = normalizePreprocessOptions(options)
+  return segments.map((segment) => {
+    if (!segment || typeof segment !== 'object') return segment
+    const source = preprocessText(segment.source, config)
+    if (source === segment.source) return segment
+    return { ...segment, source }
+  })
+}

package/lib/store-utils.js

@@ -1,11 +1,12 @@
 import { createHash, randomUUID } from 'node:crypto'
 import { CatError } from './errors.js'
 import { clamp, clampInt } from './similarity.js'
+import { DEFAULT_PREPROCESS, normalizePreprocessOptions } from './preprocess.js'
 
 export { clamp, clampInt }
 
 export const MAX_SEGMENTS = 20000
-export const CAIT_VERSION = '1.0.4'
+export const CAIT_VERSION = '1.0.5'
 export const MAX_SEGMENT_CHARS = 20000
 export const MAX_IMPORT_CHARS = 8_000_000
 export const MAX_DOCX_BASE64 = 28_000_000
@@ -26,6 +27,7 @@ export const DEFAULT_SETTINGS = {
   fuzzyThreshold: 0.55,
   tmLimit: 5,
   uiLang: 'auto',
+  preprocess: { ...DEFAULT_PREPROCESS },
   mt: {
     provider: 'mock',
     model: 'cait-mock',
@@ -161,11 +163,11 @@ function cleanInlineTokens(value) {
 export function cleanSegmentMeta(value) {
   if (!value || typeof value !== 'object' || Array.isArray(value)) return null
   const meta = {}
-  for (const key of ['start', 'end', 'xlsxRow', 'pptxParagraph']) {
+  for (const key of ['start', 'end', 'xlsxRow', 'pptxParagraph', 'epubBlock']) {
     const number = Number(value[key])
     if (Number.isFinite(number) && number >= 0) meta[key] = Math.round(number)
   }
-  for (const key of ['settings', 'identifier', 'context', 'msgidPlural', 'xliffId', 'format', 'jsonPath', 'pptxSlide']) {
+  for (const key of ['settings', 'identifier', 'context', 'msgidPlural', 'xliffId', 'format', 'jsonPath', 'pptxSlide', 'epubHref']) {
     if (typeof value[key] === 'string' && value[key].trim()) {
       meta[key] = value[key].slice(0, 500)
     }
@@ -212,6 +214,7 @@ export function restoreSettings(raw) {
     if (Number.isFinite(limit)) settings.tmLimit = clampInt(limit, 1, 20, settings.tmLimit)
     const uiLang = String(raw.uiLang ?? '').trim().toLowerCase()
     if (UI_LANG_CODES.includes(uiLang)) settings.uiLang = uiLang
+    settings.preprocess = normalizePreprocessOptions(raw.preprocess)
 
     if (raw.mt && typeof raw.mt === 'object' && !Array.isArray(raw.mt)) {
       const provider = String(raw.mt.provider ?? '').trim()