dsh-cait 1.0.4 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +202 -0
- package/README.ja.md +147 -25
- package/README.md +176 -27
- package/README.zh-CN.md +137 -26
- package/client/components.js +2047 -340
- package/client/entry.js +13 -0
- package/client/helpers.js +110 -7
- package/client/locales.js +435 -6
- package/client/rpc.js +1 -1
- package/client/styles.js +137 -3
- package/client.js +2742 -356
- package/index.js +3 -0
- package/lib/aligner.js +411 -0
- package/lib/api.js +31 -0
- package/lib/context.js +71 -0
- package/lib/docx.js +12 -9
- package/lib/epub.js +355 -0
- package/lib/formats.js +88 -33
- package/lib/glossary.js +4 -4
- package/lib/html.js +271 -0
- package/lib/import-export.js +208 -95
- package/lib/libraries.js +55 -0
- package/lib/library-export.js +75 -0
- package/lib/library-view.js +215 -0
- package/lib/mt.js +17 -7
- package/lib/persistence.js +92 -7
- package/lib/pre-translate.js +263 -0
- package/lib/preprocess.js +65 -0
- package/lib/propagate.js +133 -0
- package/lib/qa.js +167 -0
- package/lib/store-utils.js +48 -6
- package/lib/store.js +143 -12
- package/lib/term-extract.js +190 -0
- package/lib/terms.js +102 -0
- package/lib/tm.js +123 -11
- package/lib/tmx.js +47 -2
- package/lib/tools.js +472 -0
- package/package.json +2 -2
package/index.js
CHANGED
|
@@ -42,4 +42,7 @@ export function apply(ctx, config = {}) {
|
|
|
42
42
|
|
|
43
43
|
ctx.effect(() => registerCatTools(ctx, store))
|
|
44
44
|
ctx.effect(() => registerCatApi(ctx, store))
|
|
45
|
+
// Cursor and draft edits are debounced; flush them when the plugin unloads so
|
|
46
|
+
// a restart lands on the same segment with the same draft.
|
|
47
|
+
ctx.effect(() => () => store.flushPersist())
|
|
45
48
|
}
|
package/lib/aligner.js
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
import { CatError } from './errors.js'
|
|
2
|
+
import { splitIntoSegments } from './segmenter.js'
|
|
3
|
+
import { normalizeText } from './similarity.js'
|
|
4
|
+
import { cleanLang, cleanText } from './store-utils.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Cross-language alignment of two separate files.
|
|
8
|
+
*
|
|
9
|
+
* Literal similarity is useless here: "3 modes" and "3 种模式" share no
|
|
10
|
+
* characters. What survives translation are structural signals — numbers,
|
|
11
|
+
* placeholders, URLs, all-caps names, relative length and paragraph position —
|
|
12
|
+
* so the score is built from those instead.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export const ALIGN_WEIGHTS = { anchor: 0.45, length: 0.35, struct: 0.2 }
|
|
16
|
+
export const ALIGN_HIGH = 0.6
|
|
17
|
+
export const ALIGN_MEDIUM = 0.35
|
|
18
|
+
export const ALIGN_SKIP_COST = 1
|
|
19
|
+
export const ALIGN_MERGE_PENALTY = 0.15
|
|
20
|
+
// A merge glues two sentences of one side onto one of the other. Without hard
|
|
21
|
+
// evidence (a shared number, placeholder or name) that is almost always a
|
|
22
|
+
// wrong pairing invented to make the totals look neat, so merges need a real
|
|
23
|
+
// score before they are even considered.
|
|
24
|
+
export const ALIGN_MERGE_MIN_SCORE = 0.55
|
|
25
|
+
export const ALIGN_MAX_CELLS = 4_000_000
|
|
26
|
+
export const ALIGN_BLOCK_SIZE = 500
|
|
27
|
+
|
|
28
|
+
const ANCHOR_PATTERNS = [
|
|
29
|
+
/\{\d+\}/g,
|
|
30
|
+
/https?:\/\/[^\s)]+/gi,
|
|
31
|
+
/\b\d[\d.,]*%?/g,
|
|
32
|
+
/\b[A-Z][A-Z0-9]{1,}\b/g,
|
|
33
|
+
/\b[a-z]+[A-Z][A-Za-z0-9]*\b/g,
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
function anchorsIn(text) {
|
|
37
|
+
const found = []
|
|
38
|
+
const value = String(text ?? '')
|
|
39
|
+
for (const pattern of ANCHOR_PATTERNS) {
|
|
40
|
+
for (const match of value.match(pattern) || []) {
|
|
41
|
+
found.push(match.replace(/[.,]$/, '').toLocaleLowerCase())
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return found
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Multiset Dice over anchors. No anchors at all is neutral, not a failure. */
|
|
48
|
+
export function anchorOverlap(left, right) {
|
|
49
|
+
const a = anchorsIn(left)
|
|
50
|
+
const b = anchorsIn(right)
|
|
51
|
+
if (!a.length && !b.length) return 0.5
|
|
52
|
+
if (!a.length || !b.length) return 0
|
|
53
|
+
const counts = new Map()
|
|
54
|
+
for (const item of a) counts.set(item, (counts.get(item) || 0) + 1)
|
|
55
|
+
let shared = 0
|
|
56
|
+
for (const item of b) {
|
|
57
|
+
const remaining = counts.get(item) || 0
|
|
58
|
+
if (remaining > 0) {
|
|
59
|
+
shared += 1
|
|
60
|
+
counts.set(item, remaining - 1)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return (2 * shared) / (a.length + b.length)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function median(values) {
|
|
67
|
+
const clean = values.filter((value) => value > 0).sort((a, b) => a - b)
|
|
68
|
+
if (!clean.length) return 1
|
|
69
|
+
return clean[Math.floor(clean.length / 2)]
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Gale-Church style length hypothesis: compare each length against the median
|
|
74
|
+
* of its own language, so a Chinese sentence can match an English sentence
|
|
75
|
+
* that is 1.8× longer without any language-specific table.
|
|
76
|
+
*/
|
|
77
|
+
export function lengthFit(leftLength, rightLength, medianLeft, medianRight) {
|
|
78
|
+
const ratioLeft = leftLength / (medianLeft || 1)
|
|
79
|
+
const ratioRight = rightLength / (medianRight || 1)
|
|
80
|
+
if (!ratioLeft && !ratioRight) return 0.5
|
|
81
|
+
if (!ratioLeft || !ratioRight) return 0
|
|
82
|
+
const spread = Math.max(ratioLeft, ratioRight)
|
|
83
|
+
return Math.max(0, 1 - Math.abs(ratioLeft - ratioRight) / (spread || 1))
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function isHeadingLike(text) {
|
|
87
|
+
const value = String(text ?? '').trim()
|
|
88
|
+
if (!value) return false
|
|
89
|
+
if (value.length > 60) return false
|
|
90
|
+
return !/[.!?。!?::;;]$/.test(value)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Structure signal: same paragraph number, both sides headings, or adjacent
|
|
95
|
+
* paragraphs (a merge or split shifts numbers by one).
|
|
96
|
+
*/
|
|
97
|
+
export function structMatch(left, right) {
|
|
98
|
+
let score = 0
|
|
99
|
+
if (left.paragraph === right.paragraph) score += 0.5
|
|
100
|
+
if (isHeadingLike(left.source) && isHeadingLike(right.source)) score += 0.5
|
|
101
|
+
else if (Math.abs((left.paragraph ?? 0) - (right.paragraph ?? 0)) === 1) score += 0.25
|
|
102
|
+
return Math.min(1, score)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function scorePair(left, right, medians) {
|
|
106
|
+
const anchor = anchorOverlap(left.source, right.source)
|
|
107
|
+
const length = lengthFit(
|
|
108
|
+
Array.from(left.source).length,
|
|
109
|
+
Array.from(right.source).length,
|
|
110
|
+
medians.left,
|
|
111
|
+
medians.right,
|
|
112
|
+
)
|
|
113
|
+
const struct = structMatch(left, right)
|
|
114
|
+
const score =
|
|
115
|
+
ALIGN_WEIGHTS.anchor * anchor + ALIGN_WEIGHTS.length * length + ALIGN_WEIGHTS.struct * struct
|
|
116
|
+
return { score: Math.min(1, Math.max(0, score)), anchor, length, struct }
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function confidenceOf(score) {
|
|
120
|
+
if (score >= ALIGN_HIGH) return 'high'
|
|
121
|
+
if (score >= ALIGN_MEDIUM) return 'medium'
|
|
122
|
+
return 'low'
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function merged(leftSegments, rightSegments) {
|
|
126
|
+
const text = `${leftSegments.map((item) => item.source).join(' ')}`
|
|
127
|
+
const other = `${rightSegments.map((item) => item.source).join(' ')}`
|
|
128
|
+
return {
|
|
129
|
+
left: { source: text, paragraph: leftSegments[0]?.paragraph ?? 0 },
|
|
130
|
+
right: { source: other, paragraph: rightSegments[0]?.paragraph ?? 0 },
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function pairCost(leftSegment, rightSegment, medians) {
|
|
135
|
+
return 1 - scorePair(leftSegment, rightSegment, medians).score
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Dynamic programming over prefixes. Moves: 1:1, 1:0, 0:1, 2:1 and 1:2. Skips
|
|
140
|
+
* cost a full 1.0 so the algorithm prefers a bad pair over dropping both
|
|
141
|
+
* sides, and merges carry a penalty so it cannot cheat its way to a cheap
|
|
142
|
+
* total by gluing everything together.
|
|
143
|
+
*/
|
|
144
|
+
export function alignBlock(source, target, medians) {
|
|
145
|
+
const n = source.length
|
|
146
|
+
const m = target.length
|
|
147
|
+
const width = m + 1
|
|
148
|
+
const cost = new Float64Array((n + 1) * width).fill(Number.POSITIVE_INFINITY)
|
|
149
|
+
const move = new Int8Array((n + 1) * width)
|
|
150
|
+
cost[0] = 0
|
|
151
|
+
for (let i = 0; i <= n; i += 1) {
|
|
152
|
+
for (let j = 0; j <= m; j += 1) {
|
|
153
|
+
if (!i && !j) continue
|
|
154
|
+
const here = i * width + j
|
|
155
|
+
let best = Number.POSITIVE_INFINITY
|
|
156
|
+
let kind = 0
|
|
157
|
+
if (i && j) {
|
|
158
|
+
const value = cost[(i - 1) * width + (j - 1)] + pairCost(source[i - 1], target[j - 1], medians)
|
|
159
|
+
if (value < best) {
|
|
160
|
+
best = value
|
|
161
|
+
kind = 3
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (i && j >= 1 && i >= 2) {
|
|
165
|
+
const { left, right } = merged([source[i - 2], source[i - 1]], [target[j - 1]])
|
|
166
|
+
const scored = scorePair(left, right, medians).score
|
|
167
|
+
if (scored >= ALIGN_MERGE_MIN_SCORE) {
|
|
168
|
+
const value =
|
|
169
|
+
cost[(i - 2) * width + (j - 1)] + (1 - scored) + ALIGN_MERGE_PENALTY
|
|
170
|
+
if (value < best) {
|
|
171
|
+
best = value
|
|
172
|
+
kind = 4
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
if (i && j >= 2) {
|
|
177
|
+
const { left, right } = merged([source[i - 1]], [target[j - 2], target[j - 1]])
|
|
178
|
+
const scored = scorePair(left, right, medians).score
|
|
179
|
+
if (scored >= ALIGN_MERGE_MIN_SCORE) {
|
|
180
|
+
const value =
|
|
181
|
+
cost[(i - 1) * width + (j - 2)] + (1 - scored) + ALIGN_MERGE_PENALTY
|
|
182
|
+
if (value < best) {
|
|
183
|
+
best = value
|
|
184
|
+
kind = 5
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (i) {
|
|
189
|
+
const value = cost[(i - 1) * width + j] + ALIGN_SKIP_COST
|
|
190
|
+
if (value < best) {
|
|
191
|
+
best = value
|
|
192
|
+
kind = 1
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (j) {
|
|
196
|
+
const value = cost[i * width + (j - 1)] + ALIGN_SKIP_COST
|
|
197
|
+
if (value < best) {
|
|
198
|
+
best = value
|
|
199
|
+
kind = 2
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
cost[here] = best
|
|
203
|
+
move[here] = kind
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const rows = []
|
|
208
|
+
let i = n
|
|
209
|
+
let j = m
|
|
210
|
+
while (i > 0 || j > 0) {
|
|
211
|
+
const kind = move[i * width + j]
|
|
212
|
+
if (kind === 3) {
|
|
213
|
+
rows.push({ source: source[i - 1], target: target[j - 1], left: 1, right: 1 })
|
|
214
|
+
i -= 1
|
|
215
|
+
j -= 1
|
|
216
|
+
} else if (kind === 4) {
|
|
217
|
+
const { left, right } = merged([source[i - 2], source[i - 1]], [target[j - 1]])
|
|
218
|
+
rows.push({ source: left, target: right, left: 2, right: 1, merged: true })
|
|
219
|
+
i -= 2
|
|
220
|
+
j -= 1
|
|
221
|
+
} else if (kind === 5) {
|
|
222
|
+
const { left, right } = merged([source[i - 1]], [target[j - 2], target[j - 1]])
|
|
223
|
+
rows.push({ source: left, target: right, left: 1, right: 2, merged: true })
|
|
224
|
+
i -= 1
|
|
225
|
+
j -= 2
|
|
226
|
+
} else if (kind === 1) {
|
|
227
|
+
rows.push({ source: source[i - 1], target: null, left: 1, right: 0 })
|
|
228
|
+
i -= 1
|
|
229
|
+
} else {
|
|
230
|
+
rows.push({ source: null, target: target[j - 1], left: 0, right: 1 })
|
|
231
|
+
j -= 1
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
rows.reverse()
|
|
235
|
+
return rows
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function decorate(row, sourceIndex, targetIndex, medians) {
|
|
239
|
+
if (!row.source || !row.target) {
|
|
240
|
+
return {
|
|
241
|
+
sourceIndex,
|
|
242
|
+
targetIndex,
|
|
243
|
+
source: row.source ? row.source.source : '',
|
|
244
|
+
target: row.target ? row.target.source : '',
|
|
245
|
+
sourceParagraph: row.source?.paragraph ?? null,
|
|
246
|
+
targetParagraph: row.target?.paragraph ?? null,
|
|
247
|
+
score: 0,
|
|
248
|
+
confidence: 'missing',
|
|
249
|
+
merged: Boolean(row.merged),
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
const { score } = scorePair(row.source, row.target, medians)
|
|
253
|
+
return {
|
|
254
|
+
sourceIndex,
|
|
255
|
+
targetIndex,
|
|
256
|
+
source: row.source.source,
|
|
257
|
+
target: row.target.source,
|
|
258
|
+
sourceParagraph: row.source.paragraph ?? null,
|
|
259
|
+
targetParagraph: row.target.paragraph ?? null,
|
|
260
|
+
score: Math.round(score * 1000) / 1000,
|
|
261
|
+
confidence: confidenceOf(score),
|
|
262
|
+
merged: Boolean(row.merged),
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Align two plain texts. Returns every row (including unpaired ones) so the UI
|
|
268
|
+
* can let a human repair them; `pairs` is only the two-sided rows.
|
|
269
|
+
*/
|
|
270
|
+
export function alignTexts(sourceText, targetText, options = {}) {
|
|
271
|
+
const source = splitIntoSegments(sourceText)
|
|
272
|
+
const target = splitIntoSegments(targetText)
|
|
273
|
+
if (!source.length || !target.length) {
|
|
274
|
+
throw new CatError('ALIGN_EMPTY', '两侧都需要有文本才能对齐。')
|
|
275
|
+
}
|
|
276
|
+
const medians = {
|
|
277
|
+
left: median(source.map((item) => Array.from(item.source).length)),
|
|
278
|
+
right: median(target.map((item) => Array.from(item.source).length)),
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const blocked = source.length * target.length > ALIGN_MAX_CELLS
|
|
282
|
+
const blocks = []
|
|
283
|
+
if (blocked) {
|
|
284
|
+
for (let start = 0; start < Math.max(source.length, target.length); start += ALIGN_BLOCK_SIZE) {
|
|
285
|
+
blocks.push({
|
|
286
|
+
source: source.slice(start, start + ALIGN_BLOCK_SIZE),
|
|
287
|
+
target: target.slice(start, start + ALIGN_BLOCK_SIZE),
|
|
288
|
+
offset: start,
|
|
289
|
+
})
|
|
290
|
+
}
|
|
291
|
+
} else {
|
|
292
|
+
blocks.push({ source, target, offset: 0 })
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const rows = []
|
|
296
|
+
for (const block of blocks) {
|
|
297
|
+
const aligned = alignBlock(block.source, block.target, medians)
|
|
298
|
+
let cursorSource = block.offset
|
|
299
|
+
let cursorTarget = block.offset
|
|
300
|
+
for (const row of aligned) {
|
|
301
|
+
const hasSource = Boolean(row.source)
|
|
302
|
+
const hasTarget = Boolean(row.target)
|
|
303
|
+
rows.push(
|
|
304
|
+
decorate(
|
|
305
|
+
row,
|
|
306
|
+
hasSource ? cursorSource : null,
|
|
307
|
+
hasTarget ? cursorTarget : null,
|
|
308
|
+
medians,
|
|
309
|
+
),
|
|
310
|
+
)
|
|
311
|
+
if (hasSource) cursorSource += row.left
|
|
312
|
+
if (hasTarget) cursorTarget += row.right
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
const pairs = rows.filter((row) => row.source && row.target)
|
|
317
|
+
const high = pairs.filter((row) => row.confidence === 'high')
|
|
318
|
+
const medium = pairs.filter((row) => row.confidence === 'medium')
|
|
319
|
+
const low = pairs.filter((row) => row.confidence === 'low')
|
|
320
|
+
return {
|
|
321
|
+
sourceName: options.sourceName ?? '',
|
|
322
|
+
targetName: options.targetName ?? '',
|
|
323
|
+
sourceSegments: source.length,
|
|
324
|
+
targetSegments: target.length,
|
|
325
|
+
rows,
|
|
326
|
+
pairs,
|
|
327
|
+
high: high.length,
|
|
328
|
+
medium: medium.length,
|
|
329
|
+
low: low.length,
|
|
330
|
+
unpairedSource: rows.filter((row) => row.source && !row.target).length,
|
|
331
|
+
unpairedTarget: rows.filter((row) => row.target && !row.source).length,
|
|
332
|
+
blocked,
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
export const alignMethods = {
|
|
337
|
+
/** Read two uploads down to segments without creating a document. */
|
|
338
|
+
prepareAlignment({ sourceFile, targetFile } = {}) {
|
|
339
|
+
if (!sourceFile?.name || !targetFile?.name) {
|
|
340
|
+
throw new CatError('ALIGN_INPUT', '需要同时提供源文文件与译文文件。')
|
|
341
|
+
}
|
|
342
|
+
const source = this.readUploadText(sourceFile)
|
|
343
|
+
const target = this.readUploadText(targetFile)
|
|
344
|
+
return { source, target }
|
|
345
|
+
},
|
|
346
|
+
|
|
347
|
+
alignUploads({ sourceFile, targetFile, sourceLang, targetLang } = {}) {
|
|
348
|
+
const { source, target } = this.prepareAlignment({ sourceFile, targetFile })
|
|
349
|
+
const result = alignTexts(source.text, target.text, {
|
|
350
|
+
sourceName: source.name,
|
|
351
|
+
targetName: target.name,
|
|
352
|
+
})
|
|
353
|
+
return {
|
|
354
|
+
...result,
|
|
355
|
+
sourceLang: cleanLang(sourceLang || this.settings.sourceLang, this.settings.sourceLang),
|
|
356
|
+
targetLang: cleanLang(targetLang || this.settings.targetLang, this.settings.targetLang),
|
|
357
|
+
}
|
|
358
|
+
},
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Write reviewed pairs into one memory. Pairs identical to what is already
|
|
362
|
+
* stored are skipped, and nothing else in the store is touched: no document,
|
|
363
|
+
* no segment status.
|
|
364
|
+
*/
|
|
365
|
+
writeAlignment({ pairs, tmId } = {}) {
|
|
366
|
+
const list = Array.isArray(pairs) ? pairs : []
|
|
367
|
+
if (!list.length) throw new CatError('ALIGN_EMPTY', '没有可写入的配对。')
|
|
368
|
+
const writeTmId = this.requireMountedTmLibrary(tmId)
|
|
369
|
+
const sourceLang = cleanLang(this.settings.sourceLang, this.settings.sourceLang)
|
|
370
|
+
const targetLang = cleanLang(this.settings.targetLang, this.settings.targetLang)
|
|
371
|
+
let written = 0
|
|
372
|
+
let updated = 0
|
|
373
|
+
let skipped = 0
|
|
374
|
+
for (const pair of list) {
|
|
375
|
+
const sourceText = cleanText(pair?.source, '源文')
|
|
376
|
+
const targetText = cleanText(pair?.target, '译文')
|
|
377
|
+
if (!sourceText || !targetText) {
|
|
378
|
+
skipped += 1
|
|
379
|
+
continue
|
|
380
|
+
}
|
|
381
|
+
const key = normalizeText(sourceText)
|
|
382
|
+
const existing = this.tm.find(
|
|
383
|
+
(entry) =>
|
|
384
|
+
entry.tmId === writeTmId &&
|
|
385
|
+
entry.sourceLang === sourceLang &&
|
|
386
|
+
entry.targetLang === targetLang &&
|
|
387
|
+
(entry.contextHash ?? null) === null &&
|
|
388
|
+
normalizeText(entry.source) === key,
|
|
389
|
+
)
|
|
390
|
+
if (existing && existing.target === targetText) {
|
|
391
|
+
skipped += 1
|
|
392
|
+
continue
|
|
393
|
+
}
|
|
394
|
+
this.upsertTM(sourceText, targetText, sourceLang, targetLang, writeTmId, null)
|
|
395
|
+
if (existing) updated += 1
|
|
396
|
+
else written += 1
|
|
397
|
+
}
|
|
398
|
+
if (written || updated) {
|
|
399
|
+
this.touch()
|
|
400
|
+
this.persist()
|
|
401
|
+
}
|
|
402
|
+
return {
|
|
403
|
+
written,
|
|
404
|
+
updated,
|
|
405
|
+
skipped,
|
|
406
|
+
total: list.length,
|
|
407
|
+
tmId: writeTmId,
|
|
408
|
+
count: this.mountedTmCount(),
|
|
409
|
+
}
|
|
410
|
+
},
|
|
411
|
+
}
|
package/lib/api.js
CHANGED
|
@@ -14,9 +14,18 @@ const MUTATIONS = new Set([
|
|
|
14
14
|
'loadDemo',
|
|
15
15
|
'selectSegment',
|
|
16
16
|
'saveDraft',
|
|
17
|
+
'adoptSuggestion',
|
|
18
|
+
'proposeTranslation',
|
|
19
|
+
'writeAlignment',
|
|
20
|
+
'propagateDocument',
|
|
21
|
+
'addGlossaryEntries',
|
|
22
|
+
'preTranslate',
|
|
23
|
+
'confirmSegments',
|
|
17
24
|
'confirmSegment',
|
|
18
25
|
'moveSegment',
|
|
19
26
|
'addTM',
|
|
27
|
+
'updateTM',
|
|
28
|
+
'deleteTM',
|
|
20
29
|
'importTM',
|
|
21
30
|
'clearTM',
|
|
22
31
|
'addGlossary',
|
|
@@ -25,6 +34,8 @@ const MUTATIONS = new Set([
|
|
|
25
34
|
'importGlossary',
|
|
26
35
|
'clearGlossary',
|
|
27
36
|
'createLibrary',
|
|
37
|
+
'renameLibrary',
|
|
38
|
+
'deleteLibrary',
|
|
28
39
|
'setMounts',
|
|
29
40
|
'saveSettings',
|
|
30
41
|
])
|
|
@@ -104,6 +115,18 @@ function createHandlers(store) {
|
|
|
104
115
|
loadDemo: () => store.loadDemo(),
|
|
105
116
|
selectSegment: (params) => store.selectSegment(params),
|
|
106
117
|
saveDraft: (params) => store.saveDraft(params),
|
|
118
|
+
adoptSuggestion: (params) => store.adoptSuggestion(params),
|
|
119
|
+
proposeTranslation: (params) => store.proposeTranslation(params),
|
|
120
|
+
writeAlignment: (params) => store.writeAlignment(params),
|
|
121
|
+
readUploadText: (params) => store.readUploadText(params),
|
|
122
|
+
alignUploads: (params) => store.alignUploads(params),
|
|
123
|
+
consistencyReport: (params) => store.consistencyReport(params),
|
|
124
|
+
termCandidates: (params) => store.termCandidates(params),
|
|
125
|
+
addGlossaryEntries: (params) => store.addGlossaryEntries(params),
|
|
126
|
+
propagateDocument: (params) => store.propagateDocument(params),
|
|
127
|
+
preTranslate: (params) => store.preTranslate(params),
|
|
128
|
+
confirmSegments: (params) => store.confirmSegmentsForTool(params),
|
|
129
|
+
getQaReport: (params) => store.getQaReport(params),
|
|
107
130
|
confirmSegment: (params) => store.confirmSegment(params),
|
|
108
131
|
moveSegment: (params) => store.moveSegment(params),
|
|
109
132
|
|
|
@@ -111,7 +134,13 @@ function createHandlers(store) {
|
|
|
111
134
|
const result = store.lookupTM(params)
|
|
112
135
|
return { ...result, count: result.matches.length }
|
|
113
136
|
},
|
|
137
|
+
listTmEntries: (params) => store.listTmEntries(params),
|
|
138
|
+
listGlossaryEntries: (params) => store.listGlossaryEntries(params),
|
|
139
|
+
exportTM: (params) => store.exportTM(params),
|
|
140
|
+
exportGlossary: (params) => store.exportGlossary(params),
|
|
114
141
|
addTM: (params) => store.addTM(params),
|
|
142
|
+
updateTM: (params) => store.updateTM(params),
|
|
143
|
+
deleteTM: (params) => store.deleteTM(params),
|
|
115
144
|
importTM: (params) => store.importTM(params),
|
|
116
145
|
clearTM: confirmDangerous((params) => store.clearTM(params)),
|
|
117
146
|
|
|
@@ -122,6 +151,8 @@ function createHandlers(store) {
|
|
|
122
151
|
clearGlossary: confirmDangerous((params) => store.clearGlossary(params)),
|
|
123
152
|
|
|
124
153
|
createLibrary: (params) => store.createLibrary(params),
|
|
154
|
+
renameLibrary: (params) => store.renameLibrary(params),
|
|
155
|
+
deleteLibrary: (params) => store.deleteLibrary(params),
|
|
125
156
|
setMounts: (params) => store.setMounts(params),
|
|
126
157
|
saveSettings: (params) => store.saveSettings(params),
|
|
127
158
|
mtSuggest: (params) => store.mtSuggest(params),
|
package/lib/context.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context fingerprints for translation-memory entries.
|
|
3
|
+
*
|
|
4
|
+
* A translation unit is often ambiguous without its neighbours: "Close" is a
|
|
5
|
+
* button in one document and a menu entry in another. Mainstream CAT tools solve
|
|
6
|
+
* this by storing the surrounding segments with every unit and giving an exact
|
|
7
|
+
* context match a score above 100% (ICE).
|
|
8
|
+
*
|
|
9
|
+
* CAIT keeps the two neighbouring *source* segments, normalises them, and
|
|
10
|
+
* reduces them to a short FNV-1a hash used for indexing. The plain text is kept
|
|
11
|
+
* next to the hash so the review pane can show what the context was.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { normalizeText } from './similarity.js'
|
|
15
|
+
|
|
16
|
+
export const CONTEXT_SEPARATOR = '\u0001'
|
|
17
|
+
|
|
18
|
+
/** 32-bit FNV-1a, hex. Not a security primitive — only an index key. */
|
|
19
|
+
export function fnv1a(value) {
|
|
20
|
+
let hash = 0x811c9dc5
|
|
21
|
+
const text = String(value ?? '')
|
|
22
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
23
|
+
hash ^= text.charCodeAt(index)
|
|
24
|
+
hash = Math.imul(hash, 0x01000193) >>> 0
|
|
25
|
+
}
|
|
26
|
+
return (hash >>> 0).toString(16).padStart(8, '0')
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Normalise a `{prev, next}` pair; an empty pair is not a context at all. */
|
|
30
|
+
export function normalizeContext(context) {
|
|
31
|
+
if (!context || typeof context !== 'object') return null
|
|
32
|
+
const prev = normalizeText(context.prev)
|
|
33
|
+
const next = normalizeText(context.next)
|
|
34
|
+
if (!prev && !next) return null
|
|
35
|
+
return { prev, next }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function contextHashOf(context) {
|
|
39
|
+
const normalized = normalizeContext(context)
|
|
40
|
+
if (!normalized) return null
|
|
41
|
+
return fnv1a(`${normalized.prev}${CONTEXT_SEPARATOR}${normalized.next}`)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Context of one segment: adjacent source text only. Translations are
|
|
46
|
+
* deliberately ignored — editing a neighbour must not invalidate a match.
|
|
47
|
+
* A single-segment document has no context (`null`), so it can never score 101%.
|
|
48
|
+
*/
|
|
49
|
+
export function contextOf(doc, index) {
|
|
50
|
+
const segments = doc?.segments ?? []
|
|
51
|
+
const position = Math.trunc(Number(index))
|
|
52
|
+
if (!Number.isInteger(position) || position < 0 || position >= segments.length) return null
|
|
53
|
+
const prev = position > 0 ? segments[position - 1]?.source ?? '' : ''
|
|
54
|
+
const next = position < segments.length - 1 ? segments[position + 1]?.source ?? '' : ''
|
|
55
|
+
return normalizeContext({ prev, next })
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** `<prop type="x-context">` payload: prev, separator, next. */
|
|
59
|
+
export function contextToProp(context) {
|
|
60
|
+
const normalized = normalizeContext(context)
|
|
61
|
+
if (!normalized) return ''
|
|
62
|
+
return `${normalized.prev}${CONTEXT_SEPARATOR}${normalized.next}`
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function contextFromProp(value) {
|
|
66
|
+
const text = String(value ?? '')
|
|
67
|
+
if (!text.trim()) return null
|
|
68
|
+
const cut = text.indexOf(CONTEXT_SEPARATOR)
|
|
69
|
+
if (cut < 0) return normalizeContext({ prev: text, next: '' })
|
|
70
|
+
return normalizeContext({ prev: text.slice(0, cut), next: text.slice(cut + CONTEXT_SEPARATOR.length) })
|
|
71
|
+
}
|
package/lib/docx.js
CHANGED
|
@@ -59,16 +59,16 @@ function readZipEntry(buffer, wanted) {
|
|
|
59
59
|
return null
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
-
function readZipEntries(buffer) {
|
|
62
|
+
function readZipEntries(buffer, label = 'DOCX') {
|
|
63
63
|
const eocd = findEndOfCentralDirectory(buffer)
|
|
64
|
-
if (eocd < 0) throw new Error(
|
|
64
|
+
if (eocd < 0) throw new Error(`${label} 文件不是有效的 ZIP 容器`)
|
|
65
65
|
const count = buffer.readUInt16LE(eocd + 10)
|
|
66
66
|
let offset = buffer.readUInt32LE(eocd + 16)
|
|
67
67
|
const entries = []
|
|
68
68
|
|
|
69
69
|
for (let index = 0; index < count; index += 1) {
|
|
70
70
|
if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== CENTRAL) {
|
|
71
|
-
throw new Error(
|
|
71
|
+
throw new Error(`${label} ZIP 中央目录损坏`)
|
|
72
72
|
}
|
|
73
73
|
const method = buffer.readUInt16LE(offset + 10)
|
|
74
74
|
const compressedSize = buffer.readUInt32LE(offset + 20)
|
|
@@ -78,7 +78,7 @@ function readZipEntries(buffer) {
|
|
|
78
78
|
const localOffset = buffer.readUInt32LE(offset + 42)
|
|
79
79
|
const name = buffer.subarray(offset + 46, offset + 46 + nameLength).toString('utf8')
|
|
80
80
|
|
|
81
|
-
if (buffer.readUInt32LE(localOffset) !== LOCAL) throw new Error(
|
|
81
|
+
if (buffer.readUInt32LE(localOffset) !== LOCAL) throw new Error(`${label} ZIP 本地头损坏`)
|
|
82
82
|
const localNameLength = buffer.readUInt16LE(localOffset + 26)
|
|
83
83
|
const localExtraLength = buffer.readUInt16LE(localOffset + 28)
|
|
84
84
|
const start = localOffset + 30 + localNameLength + localExtraLength
|
|
@@ -86,9 +86,9 @@ function readZipEntries(buffer) {
|
|
|
86
86
|
let data
|
|
87
87
|
if (method === 0) data = Buffer.from(compressed)
|
|
88
88
|
else if (method === 8) data = inflateRawSync(compressed)
|
|
89
|
-
else throw new Error(
|
|
89
|
+
else throw new Error(`${label} 使用了不支持的压缩方式 (${method})`)
|
|
90
90
|
|
|
91
|
-
entries.push({ name, data })
|
|
91
|
+
entries.push({ name, data, method })
|
|
92
92
|
offset += 46 + nameLength + extraLength + commentLength
|
|
93
93
|
}
|
|
94
94
|
|
|
@@ -134,14 +134,17 @@ function writeZip(entries) {
|
|
|
134
134
|
for (const entry of entries) {
|
|
135
135
|
const name = Buffer.from(entry.name, 'utf8')
|
|
136
136
|
const data = Buffer.isBuffer(entry.data) ? entry.data : Buffer.from(entry.data)
|
|
137
|
-
|
|
137
|
+
// EPUB requires the `mimetype` entry to be stored (method 0) and first.
|
|
138
|
+
const stored = entry.store === true
|
|
139
|
+
const method = stored ? 0 : 8
|
|
140
|
+
const compressed = stored ? data : deflateRawSync(data)
|
|
138
141
|
const crc = crc32(data)
|
|
139
142
|
|
|
140
143
|
const local = Buffer.alloc(30)
|
|
141
144
|
local.writeUInt32LE(LOCAL, 0)
|
|
142
145
|
local.writeUInt16LE(20, 4)
|
|
143
146
|
local.writeUInt16LE(0x0800, 6)
|
|
144
|
-
local.writeUInt16LE(
|
|
147
|
+
local.writeUInt16LE(method, 8)
|
|
145
148
|
local.writeUInt16LE(time, 10)
|
|
146
149
|
local.writeUInt16LE(date, 12)
|
|
147
150
|
local.writeUInt32LE(crc, 14)
|
|
@@ -156,7 +159,7 @@ function writeZip(entries) {
|
|
|
156
159
|
central.writeUInt16LE(20, 4)
|
|
157
160
|
central.writeUInt16LE(20, 6)
|
|
158
161
|
central.writeUInt16LE(0x0800, 8)
|
|
159
|
-
central.writeUInt16LE(
|
|
162
|
+
central.writeUInt16LE(method, 10)
|
|
160
163
|
central.writeUInt16LE(time, 12)
|
|
161
164
|
central.writeUInt16LE(date, 14)
|
|
162
165
|
central.writeUInt32LE(crc, 16)
|