dsh-cait 1.0.5 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +159 -0
- package/README.ja.md +100 -18
- package/README.md +126 -19
- package/README.zh-CN.md +90 -17
- package/client/components.js +2023 -347
- package/client/entry.js +13 -0
- package/client/helpers.js +110 -7
- package/client/locales.js +408 -0
- package/client/styles.js +137 -3
- package/client.js +2690 -356
- package/index.js +3 -0
- package/lib/aligner.js +411 -0
- package/lib/api.js +31 -0
- package/lib/context.js +71 -0
- package/lib/glossary.js +4 -4
- package/lib/import-export.js +72 -4
- package/lib/libraries.js +55 -0
- package/lib/library-export.js +75 -0
- package/lib/library-view.js +215 -0
- package/lib/mt.js +17 -7
- package/lib/persistence.js +72 -7
- package/lib/pre-translate.js +263 -0
- package/lib/propagate.js +133 -0
- package/lib/qa.js +167 -0
- package/lib/store-utils.js +43 -4
- package/lib/store.js +131 -12
- package/lib/term-extract.js +190 -0
- package/lib/terms.js +102 -0
- package/lib/tm.js +123 -11
- package/lib/tmx.js +47 -2
- package/lib/tools.js +472 -0
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -42,4 +42,7 @@ export function apply(ctx, config = {}) {
|
|
|
42
42
|
|
|
43
43
|
ctx.effect(() => registerCatTools(ctx, store))
|
|
44
44
|
ctx.effect(() => registerCatApi(ctx, store))
|
|
45
|
+
// Cursor and draft edits are debounced; flush them when the plugin unloads so
|
|
46
|
+
// a restart lands on the same segment with the same draft.
|
|
47
|
+
ctx.effect(() => () => store.flushPersist())
|
|
45
48
|
}
|
package/lib/aligner.js
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
import { CatError } from './errors.js'
|
|
2
|
+
import { splitIntoSegments } from './segmenter.js'
|
|
3
|
+
import { normalizeText } from './similarity.js'
|
|
4
|
+
import { cleanLang, cleanText } from './store-utils.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Cross-language alignment of two separate files.
|
|
8
|
+
*
|
|
9
|
+
* Literal similarity is useless here: "3 modes" and "3 种模式" share no
|
|
10
|
+
* characters. What survives translation are structural signals — numbers,
|
|
11
|
+
* placeholders, URLs, all-caps names, relative length and paragraph position —
|
|
12
|
+
* so the score is built from those instead.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export const ALIGN_WEIGHTS = { anchor: 0.45, length: 0.35, struct: 0.2 }
|
|
16
|
+
export const ALIGN_HIGH = 0.6
|
|
17
|
+
export const ALIGN_MEDIUM = 0.35
|
|
18
|
+
export const ALIGN_SKIP_COST = 1
|
|
19
|
+
export const ALIGN_MERGE_PENALTY = 0.15
|
|
20
|
+
// A merge glues two sentences of one side onto one of the other. Without hard
|
|
21
|
+
// evidence (a shared number, placeholder or name) that is almost always a
|
|
22
|
+
// wrong pairing invented to make the totals look neat, so merges need a real
|
|
23
|
+
// score before they are even considered.
|
|
24
|
+
export const ALIGN_MERGE_MIN_SCORE = 0.55
|
|
25
|
+
export const ALIGN_MAX_CELLS = 4_000_000
|
|
26
|
+
export const ALIGN_BLOCK_SIZE = 500
|
|
27
|
+
|
|
28
|
+
const ANCHOR_PATTERNS = [
|
|
29
|
+
/\{\d+\}/g,
|
|
30
|
+
/https?:\/\/[^\s)]+/gi,
|
|
31
|
+
/\b\d[\d.,]*%?/g,
|
|
32
|
+
/\b[A-Z][A-Z0-9]{1,}\b/g,
|
|
33
|
+
/\b[a-z]+[A-Z][A-Za-z0-9]*\b/g,
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
function anchorsIn(text) {
|
|
37
|
+
const found = []
|
|
38
|
+
const value = String(text ?? '')
|
|
39
|
+
for (const pattern of ANCHOR_PATTERNS) {
|
|
40
|
+
for (const match of value.match(pattern) || []) {
|
|
41
|
+
found.push(match.replace(/[.,]$/, '').toLocaleLowerCase())
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return found
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Multiset Dice over anchors. No anchors at all is neutral, not a failure. */
|
|
48
|
+
export function anchorOverlap(left, right) {
|
|
49
|
+
const a = anchorsIn(left)
|
|
50
|
+
const b = anchorsIn(right)
|
|
51
|
+
if (!a.length && !b.length) return 0.5
|
|
52
|
+
if (!a.length || !b.length) return 0
|
|
53
|
+
const counts = new Map()
|
|
54
|
+
for (const item of a) counts.set(item, (counts.get(item) || 0) + 1)
|
|
55
|
+
let shared = 0
|
|
56
|
+
for (const item of b) {
|
|
57
|
+
const remaining = counts.get(item) || 0
|
|
58
|
+
if (remaining > 0) {
|
|
59
|
+
shared += 1
|
|
60
|
+
counts.set(item, remaining - 1)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return (2 * shared) / (a.length + b.length)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function median(values) {
|
|
67
|
+
const clean = values.filter((value) => value > 0).sort((a, b) => a - b)
|
|
68
|
+
if (!clean.length) return 1
|
|
69
|
+
return clean[Math.floor(clean.length / 2)]
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Gale-Church style length hypothesis: compare each length against the median
|
|
74
|
+
* of its own language, so a Chinese sentence can match an English sentence
|
|
75
|
+
* that is 1.8× longer without any language-specific table.
|
|
76
|
+
*/
|
|
77
|
+
export function lengthFit(leftLength, rightLength, medianLeft, medianRight) {
|
|
78
|
+
const ratioLeft = leftLength / (medianLeft || 1)
|
|
79
|
+
const ratioRight = rightLength / (medianRight || 1)
|
|
80
|
+
if (!ratioLeft && !ratioRight) return 0.5
|
|
81
|
+
if (!ratioLeft || !ratioRight) return 0
|
|
82
|
+
const spread = Math.max(ratioLeft, ratioRight)
|
|
83
|
+
return Math.max(0, 1 - Math.abs(ratioLeft - ratioRight) / (spread || 1))
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function isHeadingLike(text) {
|
|
87
|
+
const value = String(text ?? '').trim()
|
|
88
|
+
if (!value) return false
|
|
89
|
+
if (value.length > 60) return false
|
|
90
|
+
return !/[.!?。!?::;;]$/.test(value)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Structure signal: same paragraph number, both sides headings, or adjacent
|
|
95
|
+
* paragraphs (a merge or split shifts numbers by one).
|
|
96
|
+
*/
|
|
97
|
+
export function structMatch(left, right) {
|
|
98
|
+
let score = 0
|
|
99
|
+
if (left.paragraph === right.paragraph) score += 0.5
|
|
100
|
+
if (isHeadingLike(left.source) && isHeadingLike(right.source)) score += 0.5
|
|
101
|
+
else if (Math.abs((left.paragraph ?? 0) - (right.paragraph ?? 0)) === 1) score += 0.25
|
|
102
|
+
return Math.min(1, score)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function scorePair(left, right, medians) {
|
|
106
|
+
const anchor = anchorOverlap(left.source, right.source)
|
|
107
|
+
const length = lengthFit(
|
|
108
|
+
Array.from(left.source).length,
|
|
109
|
+
Array.from(right.source).length,
|
|
110
|
+
medians.left,
|
|
111
|
+
medians.right,
|
|
112
|
+
)
|
|
113
|
+
const struct = structMatch(left, right)
|
|
114
|
+
const score =
|
|
115
|
+
ALIGN_WEIGHTS.anchor * anchor + ALIGN_WEIGHTS.length * length + ALIGN_WEIGHTS.struct * struct
|
|
116
|
+
return { score: Math.min(1, Math.max(0, score)), anchor, length, struct }
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function confidenceOf(score) {
|
|
120
|
+
if (score >= ALIGN_HIGH) return 'high'
|
|
121
|
+
if (score >= ALIGN_MEDIUM) return 'medium'
|
|
122
|
+
return 'low'
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function merged(leftSegments, rightSegments) {
|
|
126
|
+
const text = `${leftSegments.map((item) => item.source).join(' ')}`
|
|
127
|
+
const other = `${rightSegments.map((item) => item.source).join(' ')}`
|
|
128
|
+
return {
|
|
129
|
+
left: { source: text, paragraph: leftSegments[0]?.paragraph ?? 0 },
|
|
130
|
+
right: { source: other, paragraph: rightSegments[0]?.paragraph ?? 0 },
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function pairCost(leftSegment, rightSegment, medians) {
|
|
135
|
+
return 1 - scorePair(leftSegment, rightSegment, medians).score
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Dynamic programming over prefixes. Moves: 1:1, 1:0, 0:1, 2:1 and 1:2. Skips
|
|
140
|
+
* cost a full 1.0 so the algorithm prefers a bad pair over dropping both
|
|
141
|
+
* sides, and merges carry a penalty so it cannot cheat its way to a cheap
|
|
142
|
+
* total by gluing everything together.
|
|
143
|
+
*/
|
|
144
|
+
export function alignBlock(source, target, medians) {
|
|
145
|
+
const n = source.length
|
|
146
|
+
const m = target.length
|
|
147
|
+
const width = m + 1
|
|
148
|
+
const cost = new Float64Array((n + 1) * width).fill(Number.POSITIVE_INFINITY)
|
|
149
|
+
const move = new Int8Array((n + 1) * width)
|
|
150
|
+
cost[0] = 0
|
|
151
|
+
for (let i = 0; i <= n; i += 1) {
|
|
152
|
+
for (let j = 0; j <= m; j += 1) {
|
|
153
|
+
if (!i && !j) continue
|
|
154
|
+
const here = i * width + j
|
|
155
|
+
let best = Number.POSITIVE_INFINITY
|
|
156
|
+
let kind = 0
|
|
157
|
+
if (i && j) {
|
|
158
|
+
const value = cost[(i - 1) * width + (j - 1)] + pairCost(source[i - 1], target[j - 1], medians)
|
|
159
|
+
if (value < best) {
|
|
160
|
+
best = value
|
|
161
|
+
kind = 3
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (i && j >= 1 && i >= 2) {
|
|
165
|
+
const { left, right } = merged([source[i - 2], source[i - 1]], [target[j - 1]])
|
|
166
|
+
const scored = scorePair(left, right, medians).score
|
|
167
|
+
if (scored >= ALIGN_MERGE_MIN_SCORE) {
|
|
168
|
+
const value =
|
|
169
|
+
cost[(i - 2) * width + (j - 1)] + (1 - scored) + ALIGN_MERGE_PENALTY
|
|
170
|
+
if (value < best) {
|
|
171
|
+
best = value
|
|
172
|
+
kind = 4
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
if (i && j >= 2) {
|
|
177
|
+
const { left, right } = merged([source[i - 1]], [target[j - 2], target[j - 1]])
|
|
178
|
+
const scored = scorePair(left, right, medians).score
|
|
179
|
+
if (scored >= ALIGN_MERGE_MIN_SCORE) {
|
|
180
|
+
const value =
|
|
181
|
+
cost[(i - 1) * width + (j - 2)] + (1 - scored) + ALIGN_MERGE_PENALTY
|
|
182
|
+
if (value < best) {
|
|
183
|
+
best = value
|
|
184
|
+
kind = 5
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (i) {
|
|
189
|
+
const value = cost[(i - 1) * width + j] + ALIGN_SKIP_COST
|
|
190
|
+
if (value < best) {
|
|
191
|
+
best = value
|
|
192
|
+
kind = 1
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (j) {
|
|
196
|
+
const value = cost[i * width + (j - 1)] + ALIGN_SKIP_COST
|
|
197
|
+
if (value < best) {
|
|
198
|
+
best = value
|
|
199
|
+
kind = 2
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
cost[here] = best
|
|
203
|
+
move[here] = kind
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const rows = []
|
|
208
|
+
let i = n
|
|
209
|
+
let j = m
|
|
210
|
+
while (i > 0 || j > 0) {
|
|
211
|
+
const kind = move[i * width + j]
|
|
212
|
+
if (kind === 3) {
|
|
213
|
+
rows.push({ source: source[i - 1], target: target[j - 1], left: 1, right: 1 })
|
|
214
|
+
i -= 1
|
|
215
|
+
j -= 1
|
|
216
|
+
} else if (kind === 4) {
|
|
217
|
+
const { left, right } = merged([source[i - 2], source[i - 1]], [target[j - 1]])
|
|
218
|
+
rows.push({ source: left, target: right, left: 2, right: 1, merged: true })
|
|
219
|
+
i -= 2
|
|
220
|
+
j -= 1
|
|
221
|
+
} else if (kind === 5) {
|
|
222
|
+
const { left, right } = merged([source[i - 1]], [target[j - 2], target[j - 1]])
|
|
223
|
+
rows.push({ source: left, target: right, left: 1, right: 2, merged: true })
|
|
224
|
+
i -= 1
|
|
225
|
+
j -= 2
|
|
226
|
+
} else if (kind === 1) {
|
|
227
|
+
rows.push({ source: source[i - 1], target: null, left: 1, right: 0 })
|
|
228
|
+
i -= 1
|
|
229
|
+
} else {
|
|
230
|
+
rows.push({ source: null, target: target[j - 1], left: 0, right: 1 })
|
|
231
|
+
j -= 1
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
rows.reverse()
|
|
235
|
+
return rows
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function decorate(row, sourceIndex, targetIndex, medians) {
|
|
239
|
+
if (!row.source || !row.target) {
|
|
240
|
+
return {
|
|
241
|
+
sourceIndex,
|
|
242
|
+
targetIndex,
|
|
243
|
+
source: row.source ? row.source.source : '',
|
|
244
|
+
target: row.target ? row.target.source : '',
|
|
245
|
+
sourceParagraph: row.source?.paragraph ?? null,
|
|
246
|
+
targetParagraph: row.target?.paragraph ?? null,
|
|
247
|
+
score: 0,
|
|
248
|
+
confidence: 'missing',
|
|
249
|
+
merged: Boolean(row.merged),
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
const { score } = scorePair(row.source, row.target, medians)
|
|
253
|
+
return {
|
|
254
|
+
sourceIndex,
|
|
255
|
+
targetIndex,
|
|
256
|
+
source: row.source.source,
|
|
257
|
+
target: row.target.source,
|
|
258
|
+
sourceParagraph: row.source.paragraph ?? null,
|
|
259
|
+
targetParagraph: row.target.paragraph ?? null,
|
|
260
|
+
score: Math.round(score * 1000) / 1000,
|
|
261
|
+
confidence: confidenceOf(score),
|
|
262
|
+
merged: Boolean(row.merged),
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Align two plain texts. Returns every row (including unpaired ones) so the UI
|
|
268
|
+
* can let a human repair them; `pairs` is only the two-sided rows.
|
|
269
|
+
*/
|
|
270
|
+
export function alignTexts(sourceText, targetText, options = {}) {
|
|
271
|
+
const source = splitIntoSegments(sourceText)
|
|
272
|
+
const target = splitIntoSegments(targetText)
|
|
273
|
+
if (!source.length || !target.length) {
|
|
274
|
+
throw new CatError('ALIGN_EMPTY', '两侧都需要有文本才能对齐。')
|
|
275
|
+
}
|
|
276
|
+
const medians = {
|
|
277
|
+
left: median(source.map((item) => Array.from(item.source).length)),
|
|
278
|
+
right: median(target.map((item) => Array.from(item.source).length)),
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const blocked = source.length * target.length > ALIGN_MAX_CELLS
|
|
282
|
+
const blocks = []
|
|
283
|
+
if (blocked) {
|
|
284
|
+
for (let start = 0; start < Math.max(source.length, target.length); start += ALIGN_BLOCK_SIZE) {
|
|
285
|
+
blocks.push({
|
|
286
|
+
source: source.slice(start, start + ALIGN_BLOCK_SIZE),
|
|
287
|
+
target: target.slice(start, start + ALIGN_BLOCK_SIZE),
|
|
288
|
+
offset: start,
|
|
289
|
+
})
|
|
290
|
+
}
|
|
291
|
+
} else {
|
|
292
|
+
blocks.push({ source, target, offset: 0 })
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const rows = []
|
|
296
|
+
for (const block of blocks) {
|
|
297
|
+
const aligned = alignBlock(block.source, block.target, medians)
|
|
298
|
+
let cursorSource = block.offset
|
|
299
|
+
let cursorTarget = block.offset
|
|
300
|
+
for (const row of aligned) {
|
|
301
|
+
const hasSource = Boolean(row.source)
|
|
302
|
+
const hasTarget = Boolean(row.target)
|
|
303
|
+
rows.push(
|
|
304
|
+
decorate(
|
|
305
|
+
row,
|
|
306
|
+
hasSource ? cursorSource : null,
|
|
307
|
+
hasTarget ? cursorTarget : null,
|
|
308
|
+
medians,
|
|
309
|
+
),
|
|
310
|
+
)
|
|
311
|
+
if (hasSource) cursorSource += row.left
|
|
312
|
+
if (hasTarget) cursorTarget += row.right
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
const pairs = rows.filter((row) => row.source && row.target)
|
|
317
|
+
const high = pairs.filter((row) => row.confidence === 'high')
|
|
318
|
+
const medium = pairs.filter((row) => row.confidence === 'medium')
|
|
319
|
+
const low = pairs.filter((row) => row.confidence === 'low')
|
|
320
|
+
return {
|
|
321
|
+
sourceName: options.sourceName ?? '',
|
|
322
|
+
targetName: options.targetName ?? '',
|
|
323
|
+
sourceSegments: source.length,
|
|
324
|
+
targetSegments: target.length,
|
|
325
|
+
rows,
|
|
326
|
+
pairs,
|
|
327
|
+
high: high.length,
|
|
328
|
+
medium: medium.length,
|
|
329
|
+
low: low.length,
|
|
330
|
+
unpairedSource: rows.filter((row) => row.source && !row.target).length,
|
|
331
|
+
unpairedTarget: rows.filter((row) => row.target && !row.source).length,
|
|
332
|
+
blocked,
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
export const alignMethods = {
|
|
337
|
+
/** Read two uploads down to segments without creating a document. */
|
|
338
|
+
prepareAlignment({ sourceFile, targetFile } = {}) {
|
|
339
|
+
if (!sourceFile?.name || !targetFile?.name) {
|
|
340
|
+
throw new CatError('ALIGN_INPUT', '需要同时提供源文文件与译文文件。')
|
|
341
|
+
}
|
|
342
|
+
const source = this.readUploadText(sourceFile)
|
|
343
|
+
const target = this.readUploadText(targetFile)
|
|
344
|
+
return { source, target }
|
|
345
|
+
},
|
|
346
|
+
|
|
347
|
+
alignUploads({ sourceFile, targetFile, sourceLang, targetLang } = {}) {
|
|
348
|
+
const { source, target } = this.prepareAlignment({ sourceFile, targetFile })
|
|
349
|
+
const result = alignTexts(source.text, target.text, {
|
|
350
|
+
sourceName: source.name,
|
|
351
|
+
targetName: target.name,
|
|
352
|
+
})
|
|
353
|
+
return {
|
|
354
|
+
...result,
|
|
355
|
+
sourceLang: cleanLang(sourceLang || this.settings.sourceLang, this.settings.sourceLang),
|
|
356
|
+
targetLang: cleanLang(targetLang || this.settings.targetLang, this.settings.targetLang),
|
|
357
|
+
}
|
|
358
|
+
},
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Write reviewed pairs into one memory. Pairs identical to what is already
|
|
362
|
+
* stored are skipped, and nothing else in the store is touched: no document,
|
|
363
|
+
* no segment status.
|
|
364
|
+
*/
|
|
365
|
+
writeAlignment({ pairs, tmId } = {}) {
|
|
366
|
+
const list = Array.isArray(pairs) ? pairs : []
|
|
367
|
+
if (!list.length) throw new CatError('ALIGN_EMPTY', '没有可写入的配对。')
|
|
368
|
+
const writeTmId = this.requireMountedTmLibrary(tmId)
|
|
369
|
+
const sourceLang = cleanLang(this.settings.sourceLang, this.settings.sourceLang)
|
|
370
|
+
const targetLang = cleanLang(this.settings.targetLang, this.settings.targetLang)
|
|
371
|
+
let written = 0
|
|
372
|
+
let updated = 0
|
|
373
|
+
let skipped = 0
|
|
374
|
+
for (const pair of list) {
|
|
375
|
+
const sourceText = cleanText(pair?.source, '源文')
|
|
376
|
+
const targetText = cleanText(pair?.target, '译文')
|
|
377
|
+
if (!sourceText || !targetText) {
|
|
378
|
+
skipped += 1
|
|
379
|
+
continue
|
|
380
|
+
}
|
|
381
|
+
const key = normalizeText(sourceText)
|
|
382
|
+
const existing = this.tm.find(
|
|
383
|
+
(entry) =>
|
|
384
|
+
entry.tmId === writeTmId &&
|
|
385
|
+
entry.sourceLang === sourceLang &&
|
|
386
|
+
entry.targetLang === targetLang &&
|
|
387
|
+
(entry.contextHash ?? null) === null &&
|
|
388
|
+
normalizeText(entry.source) === key,
|
|
389
|
+
)
|
|
390
|
+
if (existing && existing.target === targetText) {
|
|
391
|
+
skipped += 1
|
|
392
|
+
continue
|
|
393
|
+
}
|
|
394
|
+
this.upsertTM(sourceText, targetText, sourceLang, targetLang, writeTmId, null)
|
|
395
|
+
if (existing) updated += 1
|
|
396
|
+
else written += 1
|
|
397
|
+
}
|
|
398
|
+
if (written || updated) {
|
|
399
|
+
this.touch()
|
|
400
|
+
this.persist()
|
|
401
|
+
}
|
|
402
|
+
return {
|
|
403
|
+
written,
|
|
404
|
+
updated,
|
|
405
|
+
skipped,
|
|
406
|
+
total: list.length,
|
|
407
|
+
tmId: writeTmId,
|
|
408
|
+
count: this.mountedTmCount(),
|
|
409
|
+
}
|
|
410
|
+
},
|
|
411
|
+
}
|
package/lib/api.js
CHANGED
|
@@ -14,9 +14,18 @@ const MUTATIONS = new Set([
|
|
|
14
14
|
'loadDemo',
|
|
15
15
|
'selectSegment',
|
|
16
16
|
'saveDraft',
|
|
17
|
+
'adoptSuggestion',
|
|
18
|
+
'proposeTranslation',
|
|
19
|
+
'writeAlignment',
|
|
20
|
+
'propagateDocument',
|
|
21
|
+
'addGlossaryEntries',
|
|
22
|
+
'preTranslate',
|
|
23
|
+
'confirmSegments',
|
|
17
24
|
'confirmSegment',
|
|
18
25
|
'moveSegment',
|
|
19
26
|
'addTM',
|
|
27
|
+
'updateTM',
|
|
28
|
+
'deleteTM',
|
|
20
29
|
'importTM',
|
|
21
30
|
'clearTM',
|
|
22
31
|
'addGlossary',
|
|
@@ -25,6 +34,8 @@ const MUTATIONS = new Set([
|
|
|
25
34
|
'importGlossary',
|
|
26
35
|
'clearGlossary',
|
|
27
36
|
'createLibrary',
|
|
37
|
+
'renameLibrary',
|
|
38
|
+
'deleteLibrary',
|
|
28
39
|
'setMounts',
|
|
29
40
|
'saveSettings',
|
|
30
41
|
])
|
|
@@ -104,6 +115,18 @@ function createHandlers(store) {
|
|
|
104
115
|
loadDemo: () => store.loadDemo(),
|
|
105
116
|
selectSegment: (params) => store.selectSegment(params),
|
|
106
117
|
saveDraft: (params) => store.saveDraft(params),
|
|
118
|
+
adoptSuggestion: (params) => store.adoptSuggestion(params),
|
|
119
|
+
proposeTranslation: (params) => store.proposeTranslation(params),
|
|
120
|
+
writeAlignment: (params) => store.writeAlignment(params),
|
|
121
|
+
readUploadText: (params) => store.readUploadText(params),
|
|
122
|
+
alignUploads: (params) => store.alignUploads(params),
|
|
123
|
+
consistencyReport: (params) => store.consistencyReport(params),
|
|
124
|
+
termCandidates: (params) => store.termCandidates(params),
|
|
125
|
+
addGlossaryEntries: (params) => store.addGlossaryEntries(params),
|
|
126
|
+
propagateDocument: (params) => store.propagateDocument(params),
|
|
127
|
+
preTranslate: (params) => store.preTranslate(params),
|
|
128
|
+
confirmSegments: (params) => store.confirmSegmentsForTool(params),
|
|
129
|
+
getQaReport: (params) => store.getQaReport(params),
|
|
107
130
|
confirmSegment: (params) => store.confirmSegment(params),
|
|
108
131
|
moveSegment: (params) => store.moveSegment(params),
|
|
109
132
|
|
|
@@ -111,7 +134,13 @@ function createHandlers(store) {
|
|
|
111
134
|
const result = store.lookupTM(params)
|
|
112
135
|
return { ...result, count: result.matches.length }
|
|
113
136
|
},
|
|
137
|
+
listTmEntries: (params) => store.listTmEntries(params),
|
|
138
|
+
listGlossaryEntries: (params) => store.listGlossaryEntries(params),
|
|
139
|
+
exportTM: (params) => store.exportTM(params),
|
|
140
|
+
exportGlossary: (params) => store.exportGlossary(params),
|
|
114
141
|
addTM: (params) => store.addTM(params),
|
|
142
|
+
updateTM: (params) => store.updateTM(params),
|
|
143
|
+
deleteTM: (params) => store.deleteTM(params),
|
|
115
144
|
importTM: (params) => store.importTM(params),
|
|
116
145
|
clearTM: confirmDangerous((params) => store.clearTM(params)),
|
|
117
146
|
|
|
@@ -122,6 +151,8 @@ function createHandlers(store) {
|
|
|
122
151
|
clearGlossary: confirmDangerous((params) => store.clearGlossary(params)),
|
|
123
152
|
|
|
124
153
|
createLibrary: (params) => store.createLibrary(params),
|
|
154
|
+
renameLibrary: (params) => store.renameLibrary(params),
|
|
155
|
+
deleteLibrary: (params) => store.deleteLibrary(params),
|
|
125
156
|
setMounts: (params) => store.setMounts(params),
|
|
126
157
|
saveSettings: (params) => store.saveSettings(params),
|
|
127
158
|
mtSuggest: (params) => store.mtSuggest(params),
|
package/lib/context.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context fingerprints for translation-memory entries.
|
|
3
|
+
*
|
|
4
|
+
* A translation unit is often ambiguous without its neighbours: "Close" is a
|
|
5
|
+
* button in one document and a menu entry in another. Mainstream CAT tools solve
|
|
6
|
+
* this by storing the surrounding segments with every unit and giving an exact
|
|
7
|
+
* context match a score above 100% (ICE).
|
|
8
|
+
*
|
|
9
|
+
* CAIT keeps the two neighbouring *source* segments, normalises them, and
|
|
10
|
+
* reduces them to a short FNV-1a hash used for indexing. The plain text is kept
|
|
11
|
+
* next to the hash so the review pane can show what the context was.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { normalizeText } from './similarity.js'
|
|
15
|
+
|
|
16
|
+
export const CONTEXT_SEPARATOR = '\u0001'
|
|
17
|
+
|
|
18
|
+
/** 32-bit FNV-1a, hex. Not a security primitive — only an index key. */
|
|
19
|
+
export function fnv1a(value) {
|
|
20
|
+
let hash = 0x811c9dc5
|
|
21
|
+
const text = String(value ?? '')
|
|
22
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
23
|
+
hash ^= text.charCodeAt(index)
|
|
24
|
+
hash = Math.imul(hash, 0x01000193) >>> 0
|
|
25
|
+
}
|
|
26
|
+
return (hash >>> 0).toString(16).padStart(8, '0')
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Normalise a `{prev, next}` pair; an empty pair is not a context at all. */
|
|
30
|
+
export function normalizeContext(context) {
|
|
31
|
+
if (!context || typeof context !== 'object') return null
|
|
32
|
+
const prev = normalizeText(context.prev)
|
|
33
|
+
const next = normalizeText(context.next)
|
|
34
|
+
if (!prev && !next) return null
|
|
35
|
+
return { prev, next }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function contextHashOf(context) {
|
|
39
|
+
const normalized = normalizeContext(context)
|
|
40
|
+
if (!normalized) return null
|
|
41
|
+
return fnv1a(`${normalized.prev}${CONTEXT_SEPARATOR}${normalized.next}`)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Context of one segment: adjacent source text only. Translations are
|
|
46
|
+
* deliberately ignored — editing a neighbour must not invalidate a match.
|
|
47
|
+
* A single-segment document has no context (`null`), so it can never score 101%.
|
|
48
|
+
*/
|
|
49
|
+
export function contextOf(doc, index) {
|
|
50
|
+
const segments = doc?.segments ?? []
|
|
51
|
+
const position = Math.trunc(Number(index))
|
|
52
|
+
if (!Number.isInteger(position) || position < 0 || position >= segments.length) return null
|
|
53
|
+
const prev = position > 0 ? segments[position - 1]?.source ?? '' : ''
|
|
54
|
+
const next = position < segments.length - 1 ? segments[position + 1]?.source ?? '' : ''
|
|
55
|
+
return normalizeContext({ prev, next })
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** `<prop type="x-context">` payload: prev, separator, next. */
|
|
59
|
+
export function contextToProp(context) {
|
|
60
|
+
const normalized = normalizeContext(context)
|
|
61
|
+
if (!normalized) return ''
|
|
62
|
+
return `${normalized.prev}${CONTEXT_SEPARATOR}${normalized.next}`
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function contextFromProp(value) {
|
|
66
|
+
const text = String(value ?? '')
|
|
67
|
+
if (!text.trim()) return null
|
|
68
|
+
const cut = text.indexOf(CONTEXT_SEPARATOR)
|
|
69
|
+
if (cut < 0) return normalizeContext({ prev: text, next: '' })
|
|
70
|
+
return normalizeContext({ prev: text.slice(0, cut), next: text.slice(cut + CONTEXT_SEPARATOR.length) })
|
|
71
|
+
}
|
package/lib/glossary.js
CHANGED
|
@@ -78,7 +78,9 @@ export const glossaryMethods = {
|
|
|
78
78
|
updateGlossary({ id, source, target, note, caseSensitive } = {}) {
|
|
79
79
|
const entry = this.glossary.find((item) => item.id === id)
|
|
80
80
|
if (!entry) throw new CatError('TERM_NOT_FOUND', '术语不存在。', 404)
|
|
81
|
-
|
|
81
|
+
// The library page manages every library, mounted or not; only the target
|
|
82
|
+
// library has to exist.
|
|
83
|
+
this.requireGlossaryLibrary(entry.glossaryId || DEFAULT_GLOSSARY_ID)
|
|
82
84
|
if (source !== undefined) {
|
|
83
85
|
const value = cleanText(source, '术语')
|
|
84
86
|
if (!value) throw new CatError('EMPTY_TERM', '术语不能为空。')
|
|
@@ -100,9 +102,7 @@ export const glossaryMethods = {
|
|
|
100
102
|
deleteGlossary({ id } = {}) {
|
|
101
103
|
const index = this.glossary.findIndex((item) => item.id === id)
|
|
102
104
|
if (index < 0) throw new CatError('TERM_NOT_FOUND', '术语不存在。', 404)
|
|
103
|
-
this.
|
|
104
|
-
this.glossary[index].glossaryId || DEFAULT_GLOSSARY_ID,
|
|
105
|
-
)
|
|
105
|
+
this.requireGlossaryLibrary(this.glossary[index].glossaryId || DEFAULT_GLOSSARY_ID)
|
|
106
106
|
const [removed] = this.glossary.splice(index, 1)
|
|
107
107
|
this.touch()
|
|
108
108
|
this.persist()
|
package/lib/import-export.js
CHANGED
|
@@ -17,6 +17,44 @@ const BINARY_SOURCES = {
|
|
|
17
17
|
epub: { label: 'EPUB', fallback: 'source.epub' },
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
+
const ALIGN_TEXT_PARSERS = {
|
|
21
|
+
csv: parseCsv,
|
|
22
|
+
srt: parseSrt,
|
|
23
|
+
vtt: parseVtt,
|
|
24
|
+
po: parsePo,
|
|
25
|
+
pot: parsePo,
|
|
26
|
+
json: parseJson,
|
|
27
|
+
rtf: parseRtf,
|
|
28
|
+
xliff: parseXliff,
|
|
29
|
+
xlf: parseXliff,
|
|
30
|
+
sdlxliff: parseXliff,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Text of an uploaded file, without creating a document. The aligner works on
|
|
35
|
+
* two files and only needs their paragraphs, so this path must not consume the
|
|
36
|
+
* segment budget or appear in the document list.
|
|
37
|
+
*/
|
|
38
|
+
function readUploadBody(fileName, extension, text, base64) {
|
|
39
|
+
if (extension === 'docx' || extension === 'epub' || extension === 'xlsx' || extension === 'pptx') {
|
|
40
|
+
if (!base64) throw new CatError('EMPTY_FILE', `${fileName} 内容为空。`)
|
|
41
|
+
if (base64.length > MAX_DOCX_BASE64) throw new CatError('FILE_TOO_LARGE', `${fileName} 过大。`)
|
|
42
|
+
const buffer = Buffer.from(base64, 'base64')
|
|
43
|
+
if (extension === 'docx') return parseDocxDetailed(buffer).text
|
|
44
|
+
if (extension === 'epub') return parseEpub(buffer).segments.map((segment) => segment.source).join('\n')
|
|
45
|
+
const parsed = extension === 'xlsx' ? parseXlsx(buffer) : parsePptxDetailed(buffer)
|
|
46
|
+
return parsed.segments.map((segment) => segment.source).join('\n')
|
|
47
|
+
}
|
|
48
|
+
const raw = String(text ?? '')
|
|
49
|
+
if (!raw.trim()) throw new CatError('EMPTY_FILE', `${fileName} 内容为空。`)
|
|
50
|
+
if (raw.length > MAX_IMPORT_CHARS) throw new CatError('FILE_TOO_LARGE', `${fileName} 过大。`)
|
|
51
|
+
if (extension === 'md' || extension === 'markdown') return stripMarkdown(raw)
|
|
52
|
+
if (ALIGN_TEXT_PARSERS[extension]) {
|
|
53
|
+
return ALIGN_TEXT_PARSERS[extension](raw).segments.map((segment) => segment.source).join('\n')
|
|
54
|
+
}
|
|
55
|
+
return raw
|
|
56
|
+
}
|
|
57
|
+
|
|
20
58
|
function sourceField(kind) {
|
|
21
59
|
return `source${kind[0].toUpperCase()}${kind.slice(1)}`
|
|
22
60
|
}
|
|
@@ -135,16 +173,22 @@ export const importExportMethods = {
|
|
|
135
173
|
const pairs = []
|
|
136
174
|
const segments = []
|
|
137
175
|
|
|
138
|
-
|
|
176
|
+
parsed.units.forEach((variants, index) => {
|
|
139
177
|
const source =
|
|
140
178
|
variants.find((variant) => variant.lang === src) ??
|
|
141
179
|
variants.find((variant) => variant.lang !== tgt) ??
|
|
142
180
|
variants[0]
|
|
143
|
-
if (!source?.text)
|
|
181
|
+
if (!source?.text) return
|
|
144
182
|
const target = variants.find((variant) => variant.lang === tgt)
|
|
145
|
-
pairs.push({
|
|
183
|
+
pairs.push({
|
|
184
|
+
source: source.text,
|
|
185
|
+
target: target?.text || '',
|
|
186
|
+
sourceLang: src,
|
|
187
|
+
targetLang: tgt,
|
|
188
|
+
context: parsed.contexts?.[index] ?? null,
|
|
189
|
+
})
|
|
146
190
|
segments.push({ source: source.text, target: target?.text || '', paragraph: segments.length })
|
|
147
|
-
}
|
|
191
|
+
})
|
|
148
192
|
|
|
149
193
|
if (!pairs.length) throw new CatError('TMX_NO_UNITS', 'TMX 中没有可用的翻译单元。')
|
|
150
194
|
|
|
@@ -175,6 +219,30 @@ export const importExportMethods = {
|
|
|
175
219
|
}
|
|
176
220
|
}
|
|
177
221
|
,
|
|
222
|
+
readUploadText({ name, text, base64 } = {}) {
|
|
223
|
+
const fileName = cleanName(name, '文件')
|
|
224
|
+
const extension = (fileName.match(/\.([a-z0-9]+)$/i)?.[1] ?? '').toLowerCase()
|
|
225
|
+
let body
|
|
226
|
+
try {
|
|
227
|
+
body = readUploadBody(fileName, extension, text, base64)
|
|
228
|
+
} catch (error) {
|
|
229
|
+
if (error instanceof CatError) throw error
|
|
230
|
+
throw new CatError(
|
|
231
|
+
'ALIGN_PARSE',
|
|
232
|
+
`${fileName} 解析失败: ${error?.message ?? error}`,
|
|
233
|
+
400,
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
const content = String(body ?? '').trim()
|
|
237
|
+
if (!content) throw new CatError('EMPTY_FILE', `${fileName} 没有可用文本。`)
|
|
238
|
+
return {
|
|
239
|
+
name: fileName,
|
|
240
|
+
format: extension || 'txt',
|
|
241
|
+
text: content,
|
|
242
|
+
chars: Array.from(content).length,
|
|
243
|
+
}
|
|
244
|
+
},
|
|
245
|
+
|
|
178
246
|
importFile({ name, text, base64, sourceLang, targetLang, importAs, csvSourceColumn, csvTargetColumn } = {}) {
|
|
179
247
|
const fileName = cleanName(name, '导入文档')
|
|
180
248
|
const extension = (fileName.match(/\.([a-z0-9]+)$/i)?.[1] ?? '').toLowerCase()
|