@langlink/dsh-l10n 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -4,7 +4,16 @@ LangLink localization toolsets for DeepSeek Harness (dsh).
4
4
 
5
5
  ## Tools
6
6
 
7
- `l10n_inspect`, `l10n_match_tm`, `l10n_match_terms`, `l10n_check`, `l10n_translate`, `l10n_convert`, `l10n_validate`, `l10n_extract_doc`, `l10n_backfill`, `l10n_polish`, `l10n_semantic_check`, `l10n_assets`.
7
+ `l10n_inspect`, `l10n_match_tm`, `l10n_match_terms`, `l10n_check`, `l10n_translate`, `l10n_convert`,
8
+ `l10n_validate`, `l10n_extract_doc`, `l10n_backfill`, `l10n_polish`, `l10n_semantic_check`, `l10n_assets`.
9
+
10
+ `l10n_check` supports `sourceWhitelist` / `shortSegmentMaxLength` / `targetLocale` for UNTRANSLATED_COPY
11
+ exemptions and locale-aware language-variant rules.
12
+
13
+ ## Skills
14
+
15
+ `l10n-inspect`, `l10n-project`, `l10n-translate`, `l10n-polish`, `l10n-check`, `l10n-assets`, `l10n-backfill`,
16
+ `l10n-cat-formats`.
8
17
 
9
18
  ## Install
10
19
 
package/dist/index.js CHANGED
@@ -218,7 +218,7 @@ function sdlState(segDefs) {
218
218
  }
219
219
  return Object.keys(state).length > 0 ? state : void 0;
220
220
  }
221
- function makeSegment(fileIndex, unitId, segmentIndex, sourceContainer, targetContainer, mid, sdl, sdlNode, xml, warnings) {
221
+ function makeSegment(fileIndex, unitId, segmentIndex, ordinal, sourceContainer, targetContainer, mid, sdl, sdlNode, xml, warnings) {
222
222
  const sourceTags = sourceContainer ? elementSignature(sourceContainer) : [];
223
223
  const targetTags = targetContainer ? elementSignature(targetContainer) : [];
224
224
  const id = `${fileIndex}:${unitId}:${mid ?? segmentIndex}`;
@@ -228,7 +228,7 @@ function makeSegment(fileIndex, unitId, segmentIndex, sourceContainer, targetCon
228
228
  }
229
229
  return {
230
230
  id,
231
- index: segmentIndex,
231
+ index: ordinal,
232
232
  location: { fileIndex, unitId, ...mid === void 0 ? {} : { mid }, segmentIndex },
233
233
  sourceInner: sourceContainer ? xml.slice(sourceContainer.innerStart, sourceContainer.innerEnd) : "",
234
234
  targetInner: targetContainer ? xml.slice(targetContainer.innerStart, targetContainer.innerEnd) : "",
@@ -237,6 +237,7 @@ function makeSegment(fileIndex, unitId, segmentIndex, sourceContainer, targetCon
237
237
  sourceTags,
238
238
  targetTags,
239
239
  targetRange: targetContainer ? { start: targetContainer.innerStart, end: targetContainer.innerEnd } : null,
240
+ ...sourceContainer === void 0 ? {} : { sourceNode: sourceContainer },
240
241
  ...targetContainer === void 0 ? {} : { targetNode: targetContainer },
241
242
  ...sdlNode === void 0 ? {} : { sdlNode },
242
243
  ...sdl === void 0 ? {} : { sdl }
@@ -295,6 +296,7 @@ function parseXliff(xml, options = {}) {
295
296
  fileIndex,
296
297
  unitId,
297
298
  segmentIndex,
299
+ segments.length,
298
300
  source,
299
301
  target,
300
302
  segmentId,
@@ -328,6 +330,7 @@ function parseXliff(xml, options = {}) {
328
330
  fileIndex,
329
331
  unitId,
330
332
  segmentIndex,
333
+ segments.length,
331
334
  sourceMrk,
332
335
  targetMrk ?? (targetMrks.length === 0 ? targetElement : void 0),
333
336
  mid,
@@ -342,7 +345,7 @@ function parseXliff(xml, options = {}) {
342
345
  }
343
346
  if (sourceElement || targetElement) {
344
347
  segments.push(
345
- makeSegment(fileIndex, unitId, 0, sourceElement, targetElement, void 0, sdl, sdlNode, xml, warnings)
348
+ makeSegment(fileIndex, unitId, 0, segments.length, sourceElement, targetElement, void 0, sdl, sdlNode, xml, warnings)
346
349
  );
347
350
  } else {
348
351
  warnings.push(`${fileIndex}:${unitId}: no source/target`);
@@ -2161,18 +2164,23 @@ function checkWhitespace(context) {
2161
2164
  function checkFullwidthHalfwidth(context) {
2162
2165
  const target = context.target;
2163
2166
  if (target.trim().length === 0) return;
2164
- const fullwidth = /[\uff10-\uff19\uff21-\uff3a\uff41-\uff5a]/u.exec(target);
2165
- if (fullwidth && fullwidth.index !== void 0) {
2167
+ const exempt = new Set(context.locale?.fullwidthExemptChars ?? []);
2168
+ const fullwidthPattern = /[\uff10-\uff19\uff21-\uff3a\uff41-\uff5a]/gu;
2169
+ for (const match of target.matchAll(fullwidthPattern)) {
2170
+ const char = match[0];
2171
+ if (exempt.has(char)) continue;
2172
+ const index = match.index ?? 0;
2166
2173
  draft(context, {
2167
2174
  ruleId: "FULLWIDTH_HALFWIDTH",
2168
2175
  message: "\u8BD1\u6587\u4E2D\u5B57\u6BCD/\u6570\u5B57\u4F7F\u7528\u5168\u89D2\u5F62\u5F0F",
2169
- evidence: `found '${fullwidth[0]}'`,
2176
+ evidence: `found '${char}'`,
2170
2177
  suggestion: "\u5B57\u6BCD\u4E0E\u6570\u5B57\u901A\u5E38\u4F7F\u7528\u534A\u89D2\u5F62\u5F0F",
2171
- targetRange: { start: fullwidth.index, end: fullwidth.index + 1 }
2178
+ targetRange: { start: index, end: index + 1 }
2172
2179
  });
2173
2180
  return;
2174
2181
  }
2175
- if (!context.targetIsCjk) return;
2182
+ const expectFullwidthPunctuation = context.locale?.fullwidthPunctuation ?? context.targetIsCjk;
2183
+ if (!expectFullwidthPunctuation) return;
2176
2184
  const mixed = /[\u4e00-\u9fff][,;:!?]|[,;:!?][\u4e00-\u9fff]/u.exec(target);
2177
2185
  if (mixed === null || mixed.index === void 0) return;
2178
2186
  draft(context, {
@@ -2310,10 +2318,17 @@ function checkEmptyTargetGroup(context) {
2310
2318
  function checkUntranslatedCopyGroup(context) {
2311
2319
  const { source, target } = context;
2312
2320
  if (source.trim().length === 0 || target.trim().length === 0) return;
2313
- if (normalizeForCompare(source) !== normalizeForCompare(target)) return;
2321
+ const normalizedSource = normalizeForCompare(source);
2322
+ if (normalizedSource !== normalizeForCompare(target)) return;
2323
+ const policy = context.untranslatedCopy;
2324
+ if (policy?.whitelist?.some((entry) => normalizeForCompare(entry) === normalizedSource)) return;
2314
2325
  const tokens = wordTokens(source);
2315
2326
  const cjkChars = Array.from(source).filter((char) => CJK_RE.test(char)).length;
2316
- if (tokens.length <= 1 && cjkChars < 2 && source.trim().length <= 4) return;
2327
+ const maxShort = policy?.shortSegmentMaxLength ?? 4;
2328
+ if (tokens.length <= 1 && cjkChars < 2 && source.trim().length <= maxShort) return;
2329
+ if (tokens.length <= 1 && policy?.singleTokenExemptions?.some((entry) => normalizeForCompare(entry) === normalizedSource)) {
2330
+ return;
2331
+ }
2317
2332
  draft(context, {
2318
2333
  ruleId: "UNTRANSLATED_COPY",
2319
2334
  message: "\u8BD1\u6587\u4E0E\u6E90\u6587\u5B8C\u5168\u76F8\u540C\uFF08\u672A\u7FFB\u8BD1\uFF09",
@@ -2391,6 +2406,8 @@ function buildContext(segment, options, glossary) {
2391
2406
  targetIsCjk: isCjkHeavy(segment.targetText),
2392
2407
  glossary,
2393
2408
  drafts: [],
2409
+ ...options.locale === void 0 ? {} : { locale: options.locale },
2410
+ ...options.untranslatedCopy === void 0 ? {} : { untranslatedCopy: options.untranslatedCopy },
2394
2411
  ...termHits === void 0 ? {} : { termHits }
2395
2412
  };
2396
2413
  }
@@ -2532,7 +2549,9 @@ function checkDocument(document, options = {}) {
2532
2549
  const qaOptions = {
2533
2550
  ...options.glossary === void 0 ? {} : { glossary: options.glossary },
2534
2551
  ...segmentTermHits.size === 0 ? {} : { segmentTermHits },
2535
- ...options.enabledRules === void 0 ? {} : { enabledRules: options.enabledRules }
2552
+ ...options.enabledRules === void 0 ? {} : { enabledRules: options.enabledRules },
2553
+ ...options.untranslatedCopy === void 0 ? {} : { untranslatedCopy: options.untranslatedCopy },
2554
+ ...options.locale === void 0 ? {} : { locale: options.locale }
2536
2555
  };
2537
2556
  const qa = runDeterministicChecks(document.segments, qaOptions);
2538
2557
  return { qa, segmentTermHits };
@@ -3431,7 +3450,20 @@ function createCheckTool() {
3431
3450
  path: { type: "string", required: true, description: "XLIFF/SDLXLIFF file path." },
3432
3451
  tm: { type: "string", description: "Optional TMX 1.4b file path." },
3433
3452
  glossary: { type: "string", description: "Optional CSV glossary path." },
3434
- maxIssues: { type: "integer", description: "Max issues returned; default 100." }
3453
+ maxIssues: { type: "integer", description: "Max issues returned; default 100." },
3454
+ sourceWhitelist: {
3455
+ type: "array",
3456
+ items: { type: "string" },
3457
+ description: "Source texts intentionally identical to the target (brand names, codes); exempts UNTRANSLATED_COPY."
3458
+ },
3459
+ shortSegmentMaxLength: {
3460
+ type: "integer",
3461
+ description: "Source length at or below which single-token segments are exempt from UNTRANSLATED_COPY; default 4."
3462
+ },
3463
+ targetLocale: {
3464
+ type: "string",
3465
+ description: "Target locale hint (e.g. zh-CN) used by language-variant rules."
3466
+ }
3435
3467
  },
3436
3468
  output: {
3437
3469
  schema: { type: "string" },
@@ -3442,7 +3474,17 @@ function createCheckTool() {
3442
3474
  const document = documentFromXml(xml, args.path);
3443
3475
  const tmEntries = args.tm ? parseTmx(await readUtf8(args.tm)).entries : [];
3444
3476
  const glossary = args.glossary ? parseGlossaryCsv(await readUtf8(args.glossary)).entries : [];
3445
- const checked = checkDocument(document, { tmEntries, glossary });
3477
+ const checked = checkDocument(document, {
3478
+ tmEntries,
3479
+ glossary,
3480
+ ...args.sourceWhitelist === void 0 && args.shortSegmentMaxLength === void 0 ? {} : {
3481
+ untranslatedCopy: {
3482
+ ...args.sourceWhitelist === void 0 ? {} : { whitelist: args.sourceWhitelist },
3483
+ ...args.shortSegmentMaxLength === void 0 ? {} : { shortSegmentMaxLength: args.shortSegmentMaxLength }
3484
+ }
3485
+ },
3486
+ ...args.targetLocale === void 0 ? {} : { locale: { targetLocale: args.targetLocale } }
3487
+ });
3446
3488
  const bySeverity = {};
3447
3489
  const byRule = {};
3448
3490
  for (const issue of checked.qa.issues) {
@@ -3606,9 +3648,9 @@ async function readText(path) {
3606
3648
  function createExtractDocTool() {
3607
3649
  return defineTool2({
3608
3650
  name: "l10n_extract_doc",
3609
- description: "Extract local DOCX paragraph text or XLSX text cells into a bilingual table plus a map.json. Writes new files; the original is never modified.",
3651
+ description: "Extract local DOCX paragraph text, XLSX text cells or PPTX slide text into a bilingual table plus a map.json. Writes new files; the original is never modified.",
3610
3652
  parameters: {
3611
- path: { type: "string", required: true, description: "DOCX or XLSX file path." },
3653
+ path: { type: "string", required: true, description: "DOCX, XLSX or PPTX file path." },
3612
3654
  rowsOutput: { type: "string", required: true, description: "Bilingual CSV/XLSX output path." },
3613
3655
  mapOutput: { type: "string", required: true, description: "map.json output path." },
3614
3656
  format: { type: "string", enum: ["csv", "xlsx"], description: "Rows output format; default by extension." }
@@ -3653,12 +3695,12 @@ function createExtractDocTool() {
3653
3695
  function createBackfillTool() {
3654
3696
  return defineTool2({
3655
3697
  name: "l10n_backfill",
3656
- description: "Backfill translated bilingual rows into a NEW DOCX/XLSX file using a previously generated map.json. The original is never modified.",
3698
+ description: "Backfill translated bilingual rows into a NEW DOCX/XLSX/PPTX file using a previously generated map.json. The original is never modified.",
3657
3699
  parameters: {
3658
- original: { type: "string", required: true, description: "Original DOCX/XLSX path." },
3700
+ original: { type: "string", required: true, description: "Original DOCX/XLSX/PPTX path." },
3659
3701
  map: { type: "string", required: true, description: "map.json path generated by l10n_extract_doc." },
3660
3702
  bilingual: { type: "string", required: true, description: "Translated bilingual CSV/XLSX path." },
3661
- output: { type: "string", required: true, description: "New output DOCX/XLSX path." }
3703
+ output: { type: "string", required: true, description: "New output DOCX/XLSX/PPTX path." }
3662
3704
  },
3663
3705
  output: {
3664
3706
  schema: {
@@ -3678,7 +3720,7 @@ function createBackfillTool() {
3678
3720
  const output = resolve2(args.output);
3679
3721
  if (original === output) throw new Error("refusing to overwrite the input file");
3680
3722
  const map = JSON.parse(await readText(args.map));
3681
- if (map.format !== "docx" && map.format !== "xlsx") throw new Error("invalid map.json");
3723
+ if (map.format !== "docx" && map.format !== "xlsx" && map.format !== "pptx") throw new Error("invalid map.json");
3682
3724
  const table = await parseBilingualFileAsync(args.bilingual, await readBytes2(args.bilingual));
3683
3725
  const translations = new Map(
3684
3726
  table.rows.filter((row) => row.target.trim() !== "").map((row) => [row.id, row.target])
@@ -3765,10 +3807,61 @@ function createPolishTool() {
3765
3807
 
3766
3808
  // packages/dsh-l10n/src/tools-assets.ts
3767
3809
  import { defineTool as defineTool3 } from "@deepseek-ai/dsh-tools";
3810
+ function isPrivateOrReservedHost(hostname) {
3811
+ const host = hostname.toLowerCase().replace(/^\[/u, "").replace(/\]$/u, "");
3812
+ if (host === "localhost" || host.endsWith(".localhost") || host === "::1" || host === "::") return true;
3813
+ if (host.includes(":")) {
3814
+ return /^(fc|fd|fe[89ab])/u.test(host);
3815
+ }
3816
+ const match = host.match(/^\d{1,3}(\.\d{1,3}){3}$/u);
3817
+ if (!match) return false;
3818
+ const octets = match[0].split(".").map(Number);
3819
+ if (octets.some((n) => n > 255)) return true;
3820
+ const a = octets[0];
3821
+ const b = octets[1];
3822
+ const c = octets[2];
3823
+ return a === 0 || a === 10 || a === 127 || a === 100 && b >= 64 && b <= 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 192 && b === 0 && c === 2 || a === 198 && (b === 18 || b === 19) || a >= 224;
3824
+ }
3825
+ function assertSafeAssetBaseUrl(raw, options = {}) {
3826
+ let url;
3827
+ try {
3828
+ url = new URL(raw);
3829
+ } catch {
3830
+ throw new Error("asset base URL is not a valid URL");
3831
+ }
3832
+ if (url.protocol !== "http:" && url.protocol !== "https:") {
3833
+ throw new Error("asset base URL must use http or https");
3834
+ }
3835
+ if (url.pathname !== "/" || url.search !== "" || url.hash !== "") {
3836
+ throw new Error("asset base URL must be a bare origin (no path, query, or fragment)");
3837
+ }
3838
+ if (url.username !== "" || url.password !== "") {
3839
+ throw new Error("asset base URL must not embed credentials");
3840
+ }
3841
+ if (!options.allowPrivateHosts && isPrivateOrReservedHost(url.hostname)) {
3842
+ throw new Error(
3843
+ "asset base URL points to a loopback/private/reserved host; pass allowPrivateHosts (L10N_ASSET_ALLOW_PRIVATE_HOSTS=1) if this is intended"
3844
+ );
3845
+ }
3846
+ return url;
3847
+ }
3848
+ var ASSET_ENDPOINTS = {
3849
+ prod: "https://assets.langlink.tech",
3850
+ staging: "https://assets-staging.langlink.tech",
3851
+ local: "http://127.0.0.1:47160"
3852
+ };
3853
+ function resolveAssetEndpoint(name2) {
3854
+ if (name2 === void 0 || name2 === "") return void 0;
3855
+ const raw = ASSET_ENDPOINTS[name2];
3856
+ if (raw === void 0) {
3857
+ throw new Error(`unknown asset endpoint "${name2}"; expected one of: ${Object.keys(ASSET_ENDPOINTS).join(", ")}`);
3858
+ }
3859
+ return new URL(raw);
3860
+ }
3768
3861
  function createAssetInfoTool(baseUrl) {
3769
3862
  return defineTool3({
3770
3863
  name: "l10n_assets",
3771
- description: "Read-only company localization asset service: health, glossary entries, or TM entries. Requires L10N_ASSET_BASE_URL.",
3864
+ description: "Read-only company localization asset service: health, glossary entries, or TM entries. Targets the prod endpoint by default.",
3772
3865
  parameters: {
3773
3866
  view: { type: "string", required: true, enum: ["health", "glossary", "tm"], description: "Which asset view to read." },
3774
3867
  maxEntries: { type: "integer", description: "Cap entries returned; default 100." }
@@ -3778,10 +3871,9 @@ function createAssetInfoTool(baseUrl) {
3778
3871
  render: (_args, value) => [{ type: "text", text: value }]
3779
3872
  },
3780
3873
  async execute(args) {
3781
- if (!baseUrl) throw new Error("L10N_ASSET_BASE_URL is not configured");
3782
- const root = baseUrl.replace(/\/$/u, "");
3783
- const path = args.view === "health" ? "/health" : args.view === "glossary" ? "/glossary" : "/tm";
3784
- const response = await fetch(`${root}${path}`, { signal: AbortSignal.timeout(15e3) });
3874
+ if (!baseUrl) throw new Error("asset endpoint is not configured");
3875
+ const path = args.view === "health" ? "health" : args.view === "glossary" ? "glossary" : "tm";
3876
+ const response = await fetch(new URL(path, baseUrl), { signal: AbortSignal.timeout(15e3) });
3785
3877
  if (!response.ok) throw new Error(`asset service returned ${response.status}`);
3786
3878
  const value = await response.json();
3787
3879
  if (Array.isArray(value.entries) && args.view !== "health") {
@@ -3935,6 +4027,7 @@ var name = "langlink-l10n";
3935
4027
  var inject = ["tools"];
3936
4028
  function apply(ctx) {
3937
4029
  const llm = ctx.get?.("llm");
4030
+ const assetRoot = () => resolveAssetEndpoint("prod");
3938
4031
  for (const tool of [
3939
4032
  createInspectTool(),
3940
4033
  createMatchTmTool(),
@@ -3947,13 +4040,14 @@ function apply(ctx) {
3947
4040
  createBackfillTool(),
3948
4041
  createPolishTool(),
3949
4042
  createSemanticCheckTool(llm),
3950
- createAssetInfoTool(process.env.L10N_ASSET_BASE_URL)
4043
+ createAssetInfoTool(assetRoot())
3951
4044
  ]) {
3952
4045
  ctx.tools.register(tool);
3953
4046
  }
3954
4047
  }
3955
4048
  export {
3956
4049
  apply,
4050
+ assertSafeAssetBaseUrl,
3957
4051
  createAssetInfoTool,
3958
4052
  createBackfillTool,
3959
4053
  createCheckTool,
@@ -3969,6 +4063,7 @@ export {
3969
4063
  createValidateTool,
3970
4064
  inject,
3971
4065
  inspectFile,
3972
- name
4066
+ name,
4067
+ resolveAssetEndpoint
3973
4068
  };
3974
4069
  //# sourceMappingURL=index.js.map