@intlayer/engine 9.1.3 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/initConfig/templates/cjs.txt +4 -1
- package/dist/assets/initConfig/templates/json.txt +4 -1
- package/dist/assets/initConfig/templates/mjs.txt +4 -1
- package/dist/assets/initConfig/templates/ts.txt +4 -1
- package/dist/cjs/buildIntlayerDictionary/buildIntlayerDictionary.cjs +14 -2
- package/dist/cjs/buildIntlayerDictionary/buildIntlayerDictionary.cjs.map +1 -1
- package/dist/cjs/buildIntlayerDictionary/writeDynamicDictionary.cjs +24 -4
- package/dist/cjs/buildIntlayerDictionary/writeDynamicDictionary.cjs.map +1 -1
- package/dist/cjs/buildIntlayerDictionary/writeFetchDictionary.cjs +43 -6
- package/dist/cjs/buildIntlayerDictionary/writeFetchDictionary.cjs.map +1 -1
- package/dist/cjs/buildIntlayerDictionary/writeNestedDictionary.cjs +96 -0
- package/dist/cjs/buildIntlayerDictionary/writeNestedDictionary.cjs.map +1 -0
- package/dist/cjs/docReview/alignBlocks.cjs +57 -8
- package/dist/cjs/docReview/alignBlocks.cjs.map +1 -1
- package/dist/cjs/docReview/rebuildDocument.cjs +37 -9
- package/dist/cjs/docReview/rebuildDocument.cjs.map +1 -1
- package/dist/cjs/docReview/segmentDocument.cjs +19 -2
- package/dist/cjs/docReview/segmentDocument.cjs.map +1 -1
- package/dist/cjs/fetchDistantDictionaries.cjs +107 -13
- package/dist/cjs/fetchDistantDictionaries.cjs.map +1 -1
- package/dist/cjs/init/frameworkSetup/tanstackStart/transforms.cjs +1 -1
- package/dist/cjs/init/frameworkSetup/tanstackStart/transforms.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/loadDictionaries.cjs +4 -2
- package/dist/cjs/loadDictionaries/loadDictionaries.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/loadRemoteDictionaries.cjs +3 -2
- package/dist/cjs/loadDictionaries/loadRemoteDictionaries.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/log.cjs +28 -5
- package/dist/cjs/loadDictionaries/log.cjs.map +1 -1
- package/dist/cjs/prepareIntlayer.cjs +7 -3
- package/dist/cjs/prepareIntlayer.cjs.map +1 -1
- package/dist/cjs/utils/chunkArray.cjs +24 -0
- package/dist/cjs/utils/chunkArray.cjs.map +1 -0
- package/dist/cjs/utils/index.cjs +2 -0
- package/dist/cjs/utils/runOnce.cjs +129 -50
- package/dist/cjs/utils/runOnce.cjs.map +1 -1
- package/dist/cjs/writeContentDeclaration/transformJSFile.cjs +153 -92
- package/dist/cjs/writeContentDeclaration/transformJSFile.cjs.map +1 -1
- package/dist/cjs/writeFileIfChanged.cjs +14 -7
- package/dist/cjs/writeFileIfChanged.cjs.map +1 -1
- package/dist/esm/buildIntlayerDictionary/buildIntlayerDictionary.mjs +15 -3
- package/dist/esm/buildIntlayerDictionary/buildIntlayerDictionary.mjs.map +1 -1
- package/dist/esm/buildIntlayerDictionary/writeDynamicDictionary.mjs +24 -4
- package/dist/esm/buildIntlayerDictionary/writeDynamicDictionary.mjs.map +1 -1
- package/dist/esm/buildIntlayerDictionary/writeFetchDictionary.mjs +43 -6
- package/dist/esm/buildIntlayerDictionary/writeFetchDictionary.mjs.map +1 -1
- package/dist/esm/buildIntlayerDictionary/writeNestedDictionary.mjs +92 -0
- package/dist/esm/buildIntlayerDictionary/writeNestedDictionary.mjs.map +1 -0
- package/dist/esm/docReview/alignBlocks.mjs +57 -8
- package/dist/esm/docReview/alignBlocks.mjs.map +1 -1
- package/dist/esm/docReview/rebuildDocument.mjs +37 -9
- package/dist/esm/docReview/rebuildDocument.mjs.map +1 -1
- package/dist/esm/docReview/segmentDocument.mjs +19 -2
- package/dist/esm/docReview/segmentDocument.mjs.map +1 -1
- package/dist/esm/fetchDistantDictionaries.mjs +107 -13
- package/dist/esm/fetchDistantDictionaries.mjs.map +1 -1
- package/dist/esm/init/frameworkSetup/tanstackStart/transforms.mjs +1 -1
- package/dist/esm/init/frameworkSetup/tanstackStart/transforms.mjs.map +1 -1
- package/dist/esm/loadDictionaries/loadDictionaries.mjs +4 -2
- package/dist/esm/loadDictionaries/loadDictionaries.mjs.map +1 -1
- package/dist/esm/loadDictionaries/loadRemoteDictionaries.mjs +3 -2
- package/dist/esm/loadDictionaries/loadRemoteDictionaries.mjs.map +1 -1
- package/dist/esm/loadDictionaries/log.mjs +28 -5
- package/dist/esm/loadDictionaries/log.mjs.map +1 -1
- package/dist/esm/prepareIntlayer.mjs +7 -3
- package/dist/esm/prepareIntlayer.mjs.map +1 -1
- package/dist/esm/utils/chunkArray.mjs +22 -0
- package/dist/esm/utils/chunkArray.mjs.map +1 -0
- package/dist/esm/utils/index.mjs +2 -1
- package/dist/esm/utils/runOnce.mjs +129 -50
- package/dist/esm/utils/runOnce.mjs.map +1 -1
- package/dist/esm/writeContentDeclaration/transformJSFile.mjs +153 -92
- package/dist/esm/writeContentDeclaration/transformJSFile.mjs.map +1 -1
- package/dist/esm/writeFileIfChanged.mjs +15 -8
- package/dist/esm/writeFileIfChanged.mjs.map +1 -1
- package/dist/types/buildIntlayerDictionary/buildIntlayerDictionary.d.ts.map +1 -1
- package/dist/types/buildIntlayerDictionary/writeDynamicDictionary.d.ts +14 -2
- package/dist/types/buildIntlayerDictionary/writeDynamicDictionary.d.ts.map +1 -1
- package/dist/types/buildIntlayerDictionary/writeFetchDictionary.d.ts +16 -2
- package/dist/types/buildIntlayerDictionary/writeFetchDictionary.d.ts.map +1 -1
- package/dist/types/buildIntlayerDictionary/writeNestedDictionary.d.ts +54 -0
- package/dist/types/buildIntlayerDictionary/writeNestedDictionary.d.ts.map +1 -0
- package/dist/types/docReview/alignBlocks.d.ts +4 -2
- package/dist/types/docReview/alignBlocks.d.ts.map +1 -1
- package/dist/types/docReview/rebuildDocument.d.ts.map +1 -1
- package/dist/types/docReview/segmentDocument.d.ts.map +1 -1
- package/dist/types/docReview/types.d.ts +9 -0
- package/dist/types/docReview/types.d.ts.map +1 -1
- package/dist/types/fetchDistantDictionaries.d.ts +26 -0
- package/dist/types/fetchDistantDictionaries.d.ts.map +1 -1
- package/dist/types/formatDictionary.d.ts +2 -0
- package/dist/types/formatDictionary.d.ts.map +1 -1
- package/dist/types/loadDictionaries/log.d.ts +16 -1
- package/dist/types/loadDictionaries/log.d.ts.map +1 -1
- package/dist/types/utils/chunkArray.d.ts +16 -0
- package/dist/types/utils/chunkArray.d.ts.map +1 -0
- package/dist/types/utils/index.d.ts +3 -2
- package/dist/types/utils/runOnce.d.ts +30 -6
- package/dist/types/utils/runOnce.d.ts.map +1 -1
- package/dist/types/writeContentDeclaration/transformJSFile.d.ts.map +1 -1
- package/package.json +11 -11
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alignBlocks.mjs","names":[],"sources":["../../../src/docReview/alignBlocks.ts"],"sourcesContent":["import { computeJaccardSimilarity } from './computeSimilarity';\nimport type { AlignmentPair, FingerprintedBlock } from './types';\n\n/**\n * Align the blocks of a base document with the blocks of its translation using a\n * Needleman–Wunsch global alignment over anchor similarity and block type.\n *\n * Because prose differs across languages, the score is weighted toward the\n * structural anchor (digits and symbols) rather than the words themselves.\n *\n * @param baseBlocks - Blocks of the base (source) document.\n * @param targetBlocks - Blocks of the target (translated) document.\n * @returns The ordered list of alignment pairs, including insertions and deletions.\n */\nexport const alignBaseAndTargetBlocks = (\n baseBlocks: FingerprintedBlock[],\n targetBlocks: FingerprintedBlock[]\n): AlignmentPair[] => {\n const baseLength = baseBlocks.length;\n const targetLength = targetBlocks.length;\n\n const scoreMatrix: number[][] = Array.from({ length: baseLength + 1 }, () =>\n Array.from({ length: targetLength + 1 }, () => 0)\n );\n const traceMatrix: ('diagonal' | 'up' | 'left')[][] = Array.from(\n { length: baseLength + 1 },\n () => Array.from({ length: targetLength + 1 }, () => 'diagonal')\n );\n\n const
|
|
1
|
+
{"version":3,"file":"alignBlocks.mjs","names":[],"sources":["../../../src/docReview/alignBlocks.ts"],"sourcesContent":["import { computeJaccardSimilarity } from './computeSimilarity';\nimport type { AlignmentPair, FingerprintedBlock } from './types';\n\n/** Cost of leaving a block unaligned (an insertion or a deletion). */\nconst GAP_PENALTY = -2;\n\n/**\n * Score of a pair that structural evidence rules out.\n *\n * Strictly below the cost of two gaps (`2 * GAP_PENALTY`) so the aligner always\n * prefers reporting an insertion plus a deletion over such a pair. Every other\n * score is positive, which means that without an explicit veto the aligner would\n * rather pair two unrelated blocks than leave them unaligned — and a bogus pair\n * is planned as `reuse`, which keeps the stale translation verbatim *and*\n * inserts the freshly translated base block next to it, producing the duplicated\n * headings this penalty exists to prevent.\n */\nconst STRUCTURAL_MISMATCH_PENALTY = GAP_PENALTY * 2 - 1;\n\n/** Reward for two blocks opened by a heading of the very same depth. */\nconst HEADING_DEPTH_MATCH_BONUS = 3;\n\n/** Minimum length ratio for the (small) \"comparable size\" reward. */\nconst COMPARABLE_LENGTH_RATIO = 0.75;\n\n/**\n * Body length from which a section is considered to carry content of its own.\n *\n * Kept above a single sentence so a translator's short lead-in paragraph is not\n * mistaken for a whole section body.\n */\nconst SUBSTANTIAL_BODY_LENGTH = 80;\n\n/**\n * Length of the body a block carries below its opening heading.\n *\n * A section that only holds its heading (because subsections carry all of its\n * content) is structurally different from one that holds a full body, and that\n * difference survives translation.\n *\n * @param block - The block to measure.\n * @returns The trimmed length of everything below the opening heading line.\n */\nconst measureBodyLength = (block: FingerprintedBlock): number => {\n if (block.headingDepth === null) return block.content.trim().length;\n\n const [, ...bodyLines] = block.content.split('\\n');\n\n return bodyLines.join('\\n').trim().length;\n};\n\n/**\n * Align the blocks of a base document with the blocks of its translation using a\n * Needleman–Wunsch global alignment over heading depth, anchor similarity and\n * block type.\n *\n * Because prose differs across languages, the score is weighted toward the\n * structural signals — heading depth first, then the anchor (digits and symbols)\n * — rather than the words themselves.\n *\n * @param baseBlocks - Blocks of the base (source) document.\n * @param targetBlocks - Blocks of the target (translated) document.\n * @returns The ordered list of alignment pairs, including insertions and deletions.\n */\nexport const alignBaseAndTargetBlocks = (\n baseBlocks: FingerprintedBlock[],\n targetBlocks: FingerprintedBlock[]\n): AlignmentPair[] => {\n const baseLength = baseBlocks.length;\n const targetLength = targetBlocks.length;\n\n const scoreMatrix: number[][] = Array.from({ length: baseLength + 1 }, () =>\n Array.from({ length: targetLength + 1 }, () => 0)\n );\n const traceMatrix: ('diagonal' | 'up' | 'left')[][] = Array.from(\n { length: baseLength + 1 },\n () => Array.from({ length: targetLength + 1 }, () => 'diagonal')\n );\n\n const computeMatchScore = (\n baseIndex: number,\n targetIndex: number\n ): number => {\n const baseBlock = baseBlocks[baseIndex]!;\n const targetBlock = targetBlocks[targetIndex]!;\n\n // Translating a document never changes its heading depths, so two headings\n // of different depths cannot be counterparts, however similar their content.\n const hasComparableHeadingDepths =\n baseBlock.headingDepth !== null && targetBlock.headingDepth !== null;\n\n if (\n hasComparableHeadingDepths &&\n baseBlock.headingDepth !== targetBlock.headingDepth\n ) {\n return STRUCTURAL_MISMATCH_PENALTY;\n }\n\n // A section reduced to its bare heading (its content living in subsections)\n // is not the translation of a section holding a full body. Pairing them\n // reuses that whole body while the base heading is translated again right\n // next to it — which is exactly how a duplicated heading appears.\n const baseBodyLength = measureBodyLength(baseBlock);\n const targetBodyLength = measureBodyLength(targetBlock);\n const isBodyPresenceMismatched =\n (baseBodyLength === 0 && targetBodyLength >= SUBSTANTIAL_BODY_LENGTH) ||\n (targetBodyLength === 0 && baseBodyLength >= SUBSTANTIAL_BODY_LENGTH);\n\n if (isBodyPresenceMismatched) return STRUCTURAL_MISMATCH_PENALTY;\n\n const lengthRatio =\n Math.min(baseBlock.content.length, targetBlock.content.length) /\n Math.max(baseBlock.content.length, targetBlock.content.length);\n\n const headingDepthBonus = hasComparableHeadingDepths\n ? HEADING_DEPTH_MATCH_BONUS\n : 0;\n const typeBonus = baseBlock.type === targetBlock.type ? 2 : 0;\n const anchorSimilarity = computeJaccardSimilarity(\n baseBlock.anchorText,\n targetBlock.anchorText,\n 3\n );\n const lengthBonus = lengthRatio > COMPARABLE_LENGTH_RATIO ? 1 : 0;\n\n // weighted toward the structural signals (heading depth, then anchor)\n return headingDepthBonus + typeBonus + lengthBonus + anchorSimilarity * 8;\n };\n\n // initialize first row and column\n for (let i = 1; i <= baseLength; i += 1) {\n scoreMatrix[i][0] = scoreMatrix[i - 1][0] + GAP_PENALTY;\n traceMatrix[i][0] = 'up';\n }\n for (let j = 1; j <= targetLength; j += 1) {\n scoreMatrix[0][j] = scoreMatrix[0][j - 1] + GAP_PENALTY;\n traceMatrix[0][j] = 'left';\n }\n\n // fill\n for (let i = 1; i <= baseLength; i += 1) {\n for (let j = 1; j <= targetLength; j += 1) {\n const match = scoreMatrix[i - 1][j - 1] + computeMatchScore(i - 1, j - 1);\n const deleteGap = scoreMatrix[i - 1][j] + GAP_PENALTY;\n const insertGap = scoreMatrix[i][j - 1] + GAP_PENALTY;\n\n const best = Math.max(match, deleteGap, insertGap);\n scoreMatrix[i][j] = best;\n traceMatrix[i][j] =\n best === match ? 'diagonal' : best === deleteGap ? 'up' : 'left';\n }\n }\n\n // traceback\n const result: AlignmentPair[] = [];\n let i = baseLength;\n let j = targetLength;\n while (i > 0 || j > 0) {\n if (i > 0 && j > 0 && traceMatrix[i][j] === 'diagonal') {\n const baseIndex = i - 1;\n const targetIndex = j - 1;\n const similarityScore = computeJaccardSimilarity(\n baseBlocks[baseIndex].anchorText,\n targetBlocks[targetIndex].anchorText,\n 3\n );\n result.unshift({ baseIndex, targetIndex, similarityScore });\n i -= 1;\n j -= 1;\n } else if (i > 0 && (j === 0 || traceMatrix[i][j] === 'up')) {\n result.unshift({\n baseIndex: i - 1,\n targetIndex: null,\n similarityScore: 0,\n });\n i -= 1;\n } else if (j > 0 && (i === 0 || traceMatrix[i][j] === 'left')) {\n // target block has no corresponding base block (deleted)\n result.unshift({\n baseIndex: -1,\n targetIndex: j - 1,\n similarityScore: 0,\n });\n j -= 1;\n }\n }\n return result;\n};\n"],"mappings":";;;;AAIA,MAAM,cAAc;;;;;;;;;;;;AAapB,MAAM,8BAA8B,cAAc,IAAI;;AAGtD,MAAM,4BAA4B;;AAGlC,MAAM,0BAA0B;;;;;;;AAQhC,MAAM,0BAA0B;;;;;;;;;;;AAYhC,MAAM,qBAAqB,UAAsC;CAC/D,IAAI,MAAM,iBAAiB,MAAM,OAAO,MAAM,QAAQ,KAAK,CAAC,CAAC;CAE7D,MAAM,GAAG,GAAG,aAAa,MAAM,QAAQ,MAAM,IAAI;CAEjD,OAAO,UAAU,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;AACrC;;;;;;;;;;;;;;AAeA,MAAa,4BACX,YACA,iBACoB;CACpB,MAAM,aAAa,WAAW;CAC9B,MAAM,eAAe,aAAa;CAElC,MAAM,cAA0B,MAAM,KAAK,EAAE,QAAQ,aAAa,EAAE,SAClE,MAAM,KAAK,EAAE,QAAQ,eAAe,EAAE,SAAS,CAAC,CAClD;CACA,MAAM,cAAgD,MAAM,KAC1D,EAAE,QAAQ,aAAa,EAAE,SACnB,MAAM,KAAK,EAAE,QAAQ,eAAe,EAAE,SAAS,UAAU,CACjE;CAEA,MAAM,qBACJ,WACA,gBACW;EACX,MAAM,YAAY,WAAW;EAC7B,MAAM,cAAc,aAAa;EAIjC,MAAM,6BACJ,UAAU,iBAAiB,QAAQ,YAAY,iBAAiB;EAElE,IACE,8BACA,UAAU,iBAAiB,YAAY,cAEvC,OAAO;EAOT,MAAM,iBAAiB,kBAAkB,SAAS;EAClD,MAAM,mBAAmB,kBAAkB,WAAW;EAKtD,IAHG,mBAAmB,KAAK,oBAAoB,2BAC5C,qBAAqB,KAAK,kBAAkB,yBAEjB,OAAO;EAErC,MAAM,cACJ,KAAK,IAAI,UAAU,QAAQ,QAAQ,YAAY,QAAQ,MAAM,IAC7D,KAAK,IAAI,UAAU,QAAQ,QAAQ,YAAY,QAAQ,MAAM;EAE/D,MAAM,oBAAoB,6BACtB,4BACA;EACJ,MAAM,YAAY,UAAU,SAAS,YAAY,OAAO,IAAI;EAC5D,MAAM,mBAAmB,yBACvB,UAAU,YACV,YAAY,YACZ,CACF;EACA,MAAM,cAAc,cAAc,0BAA0B,IAAI;EAGhE,OAAO,oBAAoB,YAAY,cAAc,mBAAmB;CAC1E;CAGA,KAAK,IAAI,IAAI,GAAG,KAAK,YAAY,KAAK,GAAG;EACvC,YAAY,EAAE,CAAC,KAAK,YAAY,IAAI,EAAE,CAAC,KAAK;EAC5C,YAAY,EAAE,CAAC,KAAK;CACtB;CACA,KAAK,IAAI,IAAI,GAAG,KAAK,cAAc,KAAK,GAAG;EACzC,YAAY,EAAE,CAAC,KAAK,YAAY,EAAE,CAAC,IAAI,KAAK;EAC5C,YAAY,EAAE,CAAC,KAAK;CACtB;CAGA,KAAK,IAAI,IAAI,GAAG,KAAK,YAAY,KAAK,GACpC,KAAK,IAAI,IAAI,GAAG,KAAK,cAAc,KAAK,GAAG;EACzC,MAAM,QAAQ,YAAY,IAAI,EAAE,CAAC,IAAI,KAAK,kBAAkB,IAAI,GAAG,IAAI,CAAC;EACxE,MAAM,YAAY,YAAY,IAAI,EAAE,CAAC,KAAK;EAC1C,MAAM,YAAY,YAAY,EAAE,CAAC,IAAI,KAAK;EAE1C,MAAM,OAAO,KAAK,IAAI,OAAO,WAAW,SAAS;EACjD,YAAY,EAAE,CAAC,KAAK;EACpB,YAAY,EAAE,CAAC,KACb,SAAS,QAAQ,aAAa,SAAS,YAAY,OAAO;CAC9D;CAIF,MAAM,SAA0B,CAAC;CACjC,IAAI,IAAI;CACR,IAAI,IAAI;CACR,OAAO,IAAI,KAAK,IAAI,GAClB,IAAI,IAAI,KAAK,IAAI,KAAK,YAAY,EAAE,CAAC,OAAO,YAAY;EACtD,MAAM,YAAY,IAAI;EACtB,MAAM,cAAc,IAAI;EACxB,MAAM,kBAAkB,yBACtB,WAAW,UAAU,CAAC,YACtB,aAAa,YAAY,CAAC,YAC1B,CACF;EACA,OAAO,QAAQ;GAAE;GAAW;GAAa;EAAgB,CAAC;EAC1D,KAAK;EACL,KAAK;CACP,OAAO,IAAI,IAAI,MAAM,MAAM,KAAK,YAAY,EAAE,CAAC,OAAO,OAAO;EAC3D,OAAO,QAAQ;GACb,WAAW,IAAI;GACf,aAAa;GACb,iBAAiB;EACnB,CAAC;EACD,KAAK;CACP,OAAO,IAAI,IAAI,MAAM,MAAM,KAAK,YAAY,EAAE,CAAC,OAAO,SAAS;EAE7D,OAAO,QAAQ;GACb,WAAW;GACX,aAAa,IAAI;GACjB,iBAAiB;EACnB,CAAC;EACD,KAAK;CACP;CAEF,OAAO;AACT"}
|
|
@@ -29,6 +29,18 @@ const identifySegmentsToReview = ({ baseBlocks, targetBlocks, plan }) => {
|
|
|
29
29
|
});
|
|
30
30
|
return { segmentsToReview };
|
|
31
31
|
};
|
|
32
|
+
/** Markdown separates two blocks with a blank line. */
|
|
33
|
+
const endsWithBlockBoundary = (text) => /\n[ \t]*\n$/.test(text);
|
|
34
|
+
/**
|
|
35
|
+
* The newlines missing at the end of `text` for it to close a markdown block.
|
|
36
|
+
*
|
|
37
|
+
* @param text - The document built so far.
|
|
38
|
+
* @returns `''`, `'\n'` or `'\n\n'` depending on how the text already ends.
|
|
39
|
+
*/
|
|
40
|
+
const buildBlockSeparator = (text) => {
|
|
41
|
+
if (text.length === 0 || endsWithBlockBoundary(text)) return "";
|
|
42
|
+
return text.endsWith("\n") ? "\n" : "\n\n";
|
|
43
|
+
};
|
|
32
44
|
/**
|
|
33
45
|
* Merge reviewed translations back into the final document following the
|
|
34
46
|
* alignment plan, reusing untouched target blocks as-is.
|
|
@@ -39,22 +51,38 @@ const identifySegmentsToReview = ({ baseBlocks, targetBlocks, plan }) => {
|
|
|
39
51
|
* @returns The rebuilt target document.
|
|
40
52
|
*/
|
|
41
53
|
const mergeReviewedSegments = (plan, targetBlocks, reviewedSegments) => {
|
|
42
|
-
|
|
54
|
+
let mergedText = "";
|
|
55
|
+
let previousPartWasGenerated = false;
|
|
56
|
+
/**
|
|
57
|
+
* Append one part, keeping a blank line around the content this run produced.
|
|
58
|
+
*
|
|
59
|
+
* Blocks own the blank lines that trail them, so two verbatim blocks already
|
|
60
|
+
* separate themselves and are appended untouched — a document with nothing to
|
|
61
|
+
* change merges back byte for byte. Generated content is the exception: the
|
|
62
|
+
* block it lands after may be the one that used to end the document, which
|
|
63
|
+
* carries no trailing blank line, and the two would be glued into a single
|
|
64
|
+
* markdown block (a heading swallowed by the paragraph above it).
|
|
65
|
+
*/
|
|
66
|
+
const appendPart = (content, isGenerated) => {
|
|
67
|
+
if (content.length === 0) return;
|
|
68
|
+
if (isGenerated || previousPartWasGenerated) mergedText += buildBlockSeparator(mergedText);
|
|
69
|
+
mergedText += content;
|
|
70
|
+
previousPartWasGenerated = isGenerated;
|
|
71
|
+
};
|
|
43
72
|
plan.actions.forEach((action, actionIndex) => {
|
|
44
|
-
if (action.kind === "reuse")
|
|
73
|
+
if (action.kind === "reuse") appendPart(targetBlocks[action.targetIndex].content, false);
|
|
45
74
|
else if (action.kind === "review" || action.kind === "insert_new") {
|
|
46
75
|
const reviewedContent = reviewedSegments.get(actionIndex);
|
|
47
|
-
if (reviewedContent !== void 0)
|
|
48
|
-
else if (action.kind === "review" && action.targetIndex !== null)
|
|
49
|
-
else
|
|
76
|
+
if (reviewedContent !== void 0) appendPart(reviewedContent, true);
|
|
77
|
+
else if (action.kind === "review" && action.targetIndex !== null) appendPart(targetBlocks[action.targetIndex].content, false);
|
|
78
|
+
else appendPart("\n", false);
|
|
50
79
|
} else if (action.kind === "delete") {
|
|
51
80
|
const reviewedContent = reviewedSegments.get(actionIndex);
|
|
52
|
-
if (reviewedContent !== void 0)
|
|
53
|
-
|
|
54
|
-
} else outputParts.push(targetBlocks[action.targetIndex].content);
|
|
81
|
+
if (reviewedContent !== void 0) appendPart(reviewedContent, true);
|
|
82
|
+
else appendPart(targetBlocks[action.targetIndex].content, false);
|
|
55
83
|
}
|
|
56
84
|
});
|
|
57
|
-
return
|
|
85
|
+
return mergedText;
|
|
58
86
|
};
|
|
59
87
|
|
|
60
88
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rebuildDocument.mjs","names":[],"sources":["../../../src/docReview/rebuildDocument.ts"],"sourcesContent":["import type { AlignmentPlan, FingerprintedBlock } from './types';\n\n/**\n * A block that needs to be translated or re-translated by an external consumer\n * (an AI client, a human, or an agent).\n */\nexport type SegmentToReview = {\n /** The base block to translate. */\n baseBlock: FingerprintedBlock;\n /** Existing target translation, or `null` when the block is new. */\n targetBlockText: string | null;\n /** Index of the originating action within {@link AlignmentPlan.actions}. */\n actionIndex: number;\n};\n\nexport type RebuildInput = {\n baseBlocks: FingerprintedBlock[];\n targetBlocks: FingerprintedBlock[];\n plan: AlignmentPlan;\n};\n\nexport type RebuildResult = {\n segmentsToReview: SegmentToReview[];\n};\n\n/**\n * Analyze the alignment plan and return only the segments that need\n * review/translation. Does not generate output text - that is done by\n * {@link mergeReviewedSegments} once the translations are available.\n *\n * @param input - The base/target blocks and the alignment plan.\n * @returns The list of segments that require translation.\n */\nexport const identifySegmentsToReview = ({\n baseBlocks,\n targetBlocks,\n plan,\n}: RebuildInput): RebuildResult => {\n const segmentsToReview: SegmentToReview[] = [];\n\n plan.actions.forEach((action, actionIndex) => {\n if (action.kind === 'review') {\n const baseBlock = baseBlocks[action.baseIndex];\n const targetBlockText =\n action.targetIndex !== null\n ? targetBlocks[action.targetIndex].content\n : null;\n\n segmentsToReview.push({ baseBlock, targetBlockText, actionIndex });\n } else if (action.kind === 'insert_new') {\n const baseBlock = baseBlocks[action.baseIndex];\n\n segmentsToReview.push({\n baseBlock,\n targetBlockText: null,\n actionIndex,\n });\n }\n });\n\n return { segmentsToReview };\n};\n\n/**\n * Merge reviewed translations back into the final document following the\n * alignment plan, reusing untouched target blocks as-is.\n *\n * @param plan - The alignment plan.\n * @param targetBlocks - Blocks of the existing target document.\n * @param reviewedSegments - Map of action index to its reviewed translation.\n * @returns The rebuilt target document.\n */\nexport const mergeReviewedSegments = (\n plan: AlignmentPlan,\n targetBlocks: FingerprintedBlock[],\n reviewedSegments: Map<number, string>\n): string => {\n const
|
|
1
|
+
{"version":3,"file":"rebuildDocument.mjs","names":[],"sources":["../../../src/docReview/rebuildDocument.ts"],"sourcesContent":["import type { AlignmentPlan, FingerprintedBlock } from './types';\n\n/**\n * A block that needs to be translated or re-translated by an external consumer\n * (an AI client, a human, or an agent).\n */\nexport type SegmentToReview = {\n /** The base block to translate. */\n baseBlock: FingerprintedBlock;\n /** Existing target translation, or `null` when the block is new. */\n targetBlockText: string | null;\n /** Index of the originating action within {@link AlignmentPlan.actions}. */\n actionIndex: number;\n};\n\nexport type RebuildInput = {\n baseBlocks: FingerprintedBlock[];\n targetBlocks: FingerprintedBlock[];\n plan: AlignmentPlan;\n};\n\nexport type RebuildResult = {\n segmentsToReview: SegmentToReview[];\n};\n\n/**\n * Analyze the alignment plan and return only the segments that need\n * review/translation. Does not generate output text - that is done by\n * {@link mergeReviewedSegments} once the translations are available.\n *\n * @param input - The base/target blocks and the alignment plan.\n * @returns The list of segments that require translation.\n */\nexport const identifySegmentsToReview = ({\n baseBlocks,\n targetBlocks,\n plan,\n}: RebuildInput): RebuildResult => {\n const segmentsToReview: SegmentToReview[] = [];\n\n plan.actions.forEach((action, actionIndex) => {\n if (action.kind === 'review') {\n const baseBlock = baseBlocks[action.baseIndex];\n const targetBlockText =\n action.targetIndex !== null\n ? targetBlocks[action.targetIndex].content\n : null;\n\n segmentsToReview.push({ baseBlock, targetBlockText, actionIndex });\n } else if (action.kind === 'insert_new') {\n const baseBlock = baseBlocks[action.baseIndex];\n\n segmentsToReview.push({\n baseBlock,\n targetBlockText: null,\n actionIndex,\n });\n }\n });\n\n return { segmentsToReview };\n};\n\n/** Markdown separates two blocks with a blank line. */\nconst endsWithBlockBoundary = (text: string): boolean =>\n /\\n[ \\t]*\\n$/.test(text);\n\n/**\n * The newlines missing at the end of `text` for it to close a markdown block.\n *\n * @param text - The document built so far.\n * @returns `''`, `'\\n'` or `'\\n\\n'` depending on how the text already ends.\n */\nconst buildBlockSeparator = (text: string): string => {\n if (text.length === 0 || endsWithBlockBoundary(text)) return '';\n\n return text.endsWith('\\n') ? '\\n' : '\\n\\n';\n};\n\n/**\n * Merge reviewed translations back into the final document following the\n * alignment plan, reusing untouched target blocks as-is.\n *\n * @param plan - The alignment plan.\n * @param targetBlocks - Blocks of the existing target document.\n * @param reviewedSegments - Map of action index to its reviewed translation.\n * @returns The rebuilt target document.\n */\nexport const mergeReviewedSegments = (\n plan: AlignmentPlan,\n targetBlocks: FingerprintedBlock[],\n reviewedSegments: Map<number, string>\n): string => {\n let mergedText = '';\n let previousPartWasGenerated = false;\n\n /**\n * Append one part, keeping a blank line around the content this run produced.\n *\n * Blocks own the blank lines that trail them, so two verbatim blocks already\n * separate themselves and are appended untouched — a document with nothing to\n * change merges back byte for byte. Generated content is the exception: the\n * block it lands after may be the one that used to end the document, which\n * carries no trailing blank line, and the two would be glued into a single\n * markdown block (a heading swallowed by the paragraph above it).\n */\n const appendPart = (content: string, isGenerated: boolean): void => {\n if (content.length === 0) return;\n\n if (isGenerated || previousPartWasGenerated) {\n mergedText += buildBlockSeparator(mergedText);\n }\n\n mergedText += content;\n previousPartWasGenerated = isGenerated;\n };\n\n plan.actions.forEach((action, actionIndex) => {\n if (action.kind === 'reuse') {\n appendPart(targetBlocks[action.targetIndex]!.content, false);\n } else if (action.kind === 'review' || action.kind === 'insert_new') {\n const reviewedContent = reviewedSegments.get(actionIndex);\n\n if (reviewedContent !== undefined) {\n appendPart(reviewedContent, true);\n } else {\n // Fallback: if review failed, use existing or blank\n if (action.kind === 'review' && action.targetIndex !== null) {\n appendPart(targetBlocks[action.targetIndex]!.content, false);\n } else {\n appendPart('\\n', false);\n }\n }\n } else if (action.kind === 'delete') {\n const reviewedContent = reviewedSegments.get(actionIndex);\n if (reviewedContent !== undefined) {\n // Caller explicitly resolved this block: empty string = actually delete,\n // non-empty string = replacement content.\n appendPart(reviewedContent, true);\n } else {\n // Default: keep verbatim. A target block with no base counterpart may\n // just be a section the aligner could not follow (reordering, split\n // prose) — keeping it prevents accidental data loss in log/read-only mode.\n appendPart(targetBlocks[action.targetIndex]!.content, false);\n }\n }\n });\n\n return mergedText;\n};\n"],"mappings":";;;;;;;;;AAiCA,MAAa,4BAA4B,EACvC,YACA,cACA,WACiC;CACjC,MAAM,mBAAsC,CAAC;CAE7C,KAAK,QAAQ,SAAS,QAAQ,gBAAgB;EAC5C,IAAI,OAAO,SAAS,UAAU;GAC5B,MAAM,YAAY,WAAW,OAAO;GACpC,MAAM,kBACJ,OAAO,gBAAgB,OACnB,aAAa,OAAO,YAAY,CAAC,UACjC;GAEN,iBAAiB,KAAK;IAAE;IAAW;IAAiB;GAAY,CAAC;EACnE,OAAO,IAAI,OAAO,SAAS,cAAc;GACvC,MAAM,YAAY,WAAW,OAAO;GAEpC,iBAAiB,KAAK;IACpB;IACA,iBAAiB;IACjB;GACF,CAAC;EACH;CACF,CAAC;CAED,OAAO,EAAE,iBAAiB;AAC5B;;AAGA,MAAM,yBAAyB,SAC7B,cAAc,KAAK,IAAI;;;;;;;AAQzB,MAAM,uBAAuB,SAAyB;CACpD,IAAI,KAAK,WAAW,KAAK,sBAAsB,IAAI,GAAG,OAAO;CAE7D,OAAO,KAAK,SAAS,IAAI,IAAI,OAAO;AACtC;;;;;;;;;;AAWA,MAAa,yBACX,MACA,cACA,qBACW;CACX,IAAI,aAAa;CACjB,IAAI,2BAA2B;;;;;;;;;;;CAY/B,MAAM,cAAc,SAAiB,gBAA+B;EAClE,IAAI,QAAQ,WAAW,GAAG;EAE1B,IAAI,eAAe,0BACjB,cAAc,oBAAoB,UAAU;EAG9C,cAAc;EACd,2BAA2B;CAC7B;CAEA,KAAK,QAAQ,SAAS,QAAQ,gBAAgB;EAC5C,IAAI,OAAO,SAAS,SAClB,WAAW,aAAa,OAAO,YAAY,CAAE,SAAS,KAAK;OACtD,IAAI,OAAO,SAAS,YAAY,OAAO,SAAS,cAAc;GACnE,MAAM,kBAAkB,iBAAiB,IAAI,WAAW;GAExD,IAAI,oBAAoB,QACtB,WAAW,iBAAiB,IAAI;QAGhC,IAAI,OAAO,SAAS,YAAY,OAAO,gBAAgB,MACrD,WAAW,aAAa,OAAO,YAAY,CAAE,SAAS,KAAK;QAE3D,WAAW,MAAM,KAAK;EAG5B,OAAO,IAAI,OAAO,SAAS,UAAU;GACnC,MAAM,kBAAkB,iBAAiB,IAAI,WAAW;GACxD,IAAI,oBAAoB,QAGtB,WAAW,iBAAiB,IAAI;QAKhC,WAAW,aAAa,OAAO,YAAY,CAAE,SAAS,KAAK;EAE/D;CACF,CAAC;CAED,OAAO;AACT"}
|
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
//#region src/docReview/segmentDocument.ts
|
|
2
|
+
const HEADING_PATTERN = /^\s*(#{1,6})\s+/;
|
|
2
3
|
const isBlankLine = (line) => line.trim().length === 0;
|
|
3
4
|
const isFencedCodeDelimiter = (line) => /^\s*```/.test(line);
|
|
4
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Read the depth of an ATX markdown heading (`#` → 1, `######` → 6).
|
|
7
|
+
*
|
|
8
|
+
* @param line - The line to inspect.
|
|
9
|
+
* @returns The heading depth, or `null` when the line is not a heading.
|
|
10
|
+
*/
|
|
11
|
+
const parseHeadingDepth = (line) => {
|
|
12
|
+
return HEADING_PATTERN.exec(line)?.[1]?.length ?? null;
|
|
13
|
+
};
|
|
14
|
+
const isHeading = (line) => parseHeadingDepth(line) !== null;
|
|
5
15
|
const isFrontmatterDelimiter = (line) => /^\s*---\s*$/.test(line);
|
|
6
16
|
/**
|
|
7
17
|
* Split a markdown document into fine-grained blocks.
|
|
@@ -35,6 +45,7 @@ const segmentDocument = (text) => {
|
|
|
35
45
|
if (index < lineCount) index += 1;
|
|
36
46
|
units.push({
|
|
37
47
|
type: "unknown",
|
|
48
|
+
headingDepth: null,
|
|
38
49
|
startIndex,
|
|
39
50
|
endIndex: index - 1
|
|
40
51
|
});
|
|
@@ -47,14 +58,17 @@ const segmentDocument = (text) => {
|
|
|
47
58
|
if (index < lineCount) index += 1;
|
|
48
59
|
units.push({
|
|
49
60
|
type: "code_block",
|
|
61
|
+
headingDepth: null,
|
|
50
62
|
startIndex,
|
|
51
63
|
endIndex: index - 1
|
|
52
64
|
});
|
|
53
65
|
continue;
|
|
54
66
|
}
|
|
55
|
-
|
|
67
|
+
const headingDepth = parseHeadingDepth(currentLine);
|
|
68
|
+
if (headingDepth !== null) {
|
|
56
69
|
units.push({
|
|
57
70
|
type: "heading",
|
|
71
|
+
headingDepth,
|
|
58
72
|
startIndex: index,
|
|
59
73
|
endIndex: index
|
|
60
74
|
});
|
|
@@ -65,6 +79,7 @@ const segmentDocument = (text) => {
|
|
|
65
79
|
while (index < lineCount && !isBlankLine(lines[index]) && !isHeading(lines[index]) && !isFencedCodeDelimiter(lines[index])) index += 1;
|
|
66
80
|
units.push({
|
|
67
81
|
type: "paragraph",
|
|
82
|
+
headingDepth: null,
|
|
68
83
|
startIndex,
|
|
69
84
|
endIndex: index - 1
|
|
70
85
|
});
|
|
@@ -78,6 +93,7 @@ const segmentDocument = (text) => {
|
|
|
78
93
|
return {
|
|
79
94
|
type: unit.type,
|
|
80
95
|
content,
|
|
96
|
+
headingDepth: unit.headingDepth,
|
|
81
97
|
lineStart: blockStartIndex + 1,
|
|
82
98
|
lineEnd: blockEndIndex + 1
|
|
83
99
|
};
|
|
@@ -112,6 +128,7 @@ const segmentSections = (text) => {
|
|
|
112
128
|
sections.push({
|
|
113
129
|
type: firstBlock.type,
|
|
114
130
|
content: currentBlocks.map((block) => block.content).join(""),
|
|
131
|
+
headingDepth: firstBlock.headingDepth,
|
|
115
132
|
lineStart: firstBlock.lineStart,
|
|
116
133
|
lineEnd: lastBlock.lineEnd
|
|
117
134
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"segmentDocument.mjs","names":[],"sources":["../../../src/docReview/segmentDocument.ts"],"sourcesContent":["import type { Block, BlockType } from './types';\n\nconst isBlankLine = (line: string): boolean => line.trim().length === 0;\nconst isFencedCodeDelimiter = (line: string): boolean => /^\\s*```/.test(line);\nconst isHeading = (line: string): boolean => /^\\s*#{1,6}\\s+/.test(line);\nconst isFrontmatterDelimiter = (line: string): boolean =>\n /^\\s*---\\s*$/.test(line);\n\n/**\n * A content unit (heading, paragraph, code block or frontmatter) spanning a\n * 0-based, inclusive line range. Blank-line runs are not units of their own;\n * they are folded into the preceding unit as trailing separators (see\n * {@link segmentDocument}) so that the concatenation of every block's content\n * reproduces the document byte-for-byte.\n */\ntype ContentUnit = {\n type: BlockType;\n startIndex: number;\n endIndex: number;\n};\n\n/**\n * Split a markdown document into fine-grained blocks.\n *\n * Boundaries are drawn at headings, blank lines (paragraph breaks) and fenced\n * code blocks, while frontmatter and the inside of code fences are kept intact.\n * Each blank-line run is appended to the block that precedes it, so the blocks\n * form an exact partition of the document: concatenating every `content` (in\n * order) yields the original text unchanged. This is what lets the block-aware\n * review re-translate only the paragraphs/snippets that actually changed instead\n * of the whole heading section.\n *\n * @param text - The full markdown document.\n * @returns The ordered list of blocks with their 1-based line ranges.\n */\nexport const segmentDocument = (text: string): Block[] => {\n const lines = text.split('\\n');\n const lineCount = lines.length;\n\n // 1. Tokenize into content units, skipping blank-line runs (folded in below).\n const units: ContentUnit[] = [];\n let index = 0;\n\n while (index < lineCount) {\n const currentLine = lines[index];\n\n if (isBlankLine(currentLine)) {\n index += 1;\n continue;\n }\n\n // Frontmatter: only when it opens the document.\n if (units.length === 0 && isFrontmatterDelimiter(currentLine)) {\n const startIndex = index;\n index += 1;\n while (index < lineCount && !isFrontmatterDelimiter(lines[index])) {\n index += 1;\n }\n // Include the closing delimiter when present.\n if (index < lineCount) index += 1;\n units.push({ type: 'unknown', startIndex, endIndex: index - 1 });\n continue;\n }\n\n // Fenced code block: consumed whole so inner blank lines and `#` lines are\n // never treated as boundaries.\n if (isFencedCodeDelimiter(currentLine)) {\n const startIndex = index;\n index += 1;\n while (index < lineCount && !isFencedCodeDelimiter(lines[index])) {\n index += 1;\n }\n // Include the closing fence when present.\n if (index < lineCount) index += 1;\n units.push({ type: 'code_block', startIndex, endIndex: index - 1 });\n continue;\n }\n\n // Heading: a single self-contained line.\n if (isHeading(currentLine)) {\n units.push({ type: 'heading', startIndex: index, endIndex: index });\n index += 1;\n continue;\n }\n\n // Paragraph: a run of consecutive lines until a blank line, a heading or a\n // code fence. Tables and tight lists stay together (no blank line between\n // their rows/items).\n const startIndex = index;\n while (\n index < lineCount &&\n !isBlankLine(lines[index]) &&\n !isHeading(lines[index]) &&\n !isFencedCodeDelimiter(lines[index])\n ) {\n index += 1;\n }\n units.push({ type: 'paragraph', startIndex, endIndex: index - 1 });\n }\n\n if (units.length === 0) return [];\n\n // 2. Turn each unit into a block whose line range extends to just before the\n // next unit, so the trailing blank-line run is owned by it. The first block\n // also absorbs any leading blank lines, and the last block runs to EOF.\n return units.map((unit, unitIndex): Block => {\n const blockStartIndex = unitIndex === 0 ? 0 : unit.startIndex;\n const blockEndIndex =\n unitIndex === units.length - 1\n ? lineCount - 1\n : units[unitIndex + 1].startIndex - 1;\n\n const blockLines = lines.slice(blockStartIndex, blockEndIndex + 1);\n // Re-append the boundary newline dropped by `split` for every block but the\n // one ending at EOF, so concatenating all blocks rebuilds the document.\n const content =\n blockEndIndex < lineCount - 1\n ? `${blockLines.join('\\n')}\\n`\n : blockLines.join('\\n');\n\n return {\n type: unit.type,\n content,\n lineStart: blockStartIndex + 1,\n lineEnd: blockEndIndex + 1,\n };\n });\n};\n\n/**\n * Split a markdown document into coarse, heading-anchored sections.\n *\n * Built by grouping the fine blocks of {@link segmentDocument}: frontmatter and\n * each heading open a new section, and the following non-heading blocks are\n * folded into it. Because it only concatenates adjacent fine blocks, the result\n * is still an exact partition of the document (sections concatenate back to the\n * source unchanged).\n *\n * Sections are the robust alignment unit between a base document and its\n * translation — both share the same heading structure, so they align almost\n * perfectly and a translation that splits its prose into a different number of\n * paragraphs never causes a section to be dropped. Fine-grained review happens\n * within a section once it is known to have changed.\n *\n * @param text - The full markdown document.\n * @returns The ordered list of sections with their 1-based line ranges.\n */\nexport const segmentSections = (text: string): Block[] => {\n const fineBlocks = segmentDocument(text);\n const sections: Block[] = [];\n let currentBlocks: Block[] = [];\n\n const flushSection = (): void => {\n if (currentBlocks.length === 0) return;\n\n const [firstBlock] = currentBlocks;\n const lastBlock = currentBlocks[currentBlocks.length - 1];\n\n sections.push({\n type: firstBlock.type,\n content: currentBlocks.map((block) => block.content).join(''),\n lineStart: firstBlock.lineStart,\n lineEnd: lastBlock.lineEnd,\n });\n currentBlocks = [];\n };\n\n for (const block of fineBlocks) {\n // Frontmatter (a leading `unknown` block) and every heading open a section.\n const opensSection =\n block.type === 'heading' ||\n (block.type === 'unknown' && sections.length === 0);\n\n if (opensSection) flushSection();\n currentBlocks.push(block);\n if (block.type === 'unknown' && sections.length === 0) flushSection();\n }\n\n flushSection();\n\n return sections;\n};\n"],"mappings":";AAEA,MAAM,eAAe,SAA0B,KAAK,KAAK,CAAC,CAAC,WAAW;AACtE,MAAM,yBAAyB,SAA0B,UAAU,KAAK,IAAI;AAC5E,MAAM,aAAa,SAA0B,gBAAgB,KAAK,IAAI;AACtE,MAAM,0BAA0B,SAC9B,cAAc,KAAK,IAAI;;;;;;;;;;;;;;;AA6BzB,MAAa,mBAAmB,SAA0B;CACxD,MAAM,QAAQ,KAAK,MAAM,IAAI;CAC7B,MAAM,YAAY,MAAM;CAGxB,MAAM,QAAuB,CAAC;CAC9B,IAAI,QAAQ;CAEZ,OAAO,QAAQ,WAAW;EACxB,MAAM,cAAc,MAAM;EAE1B,IAAI,YAAY,WAAW,GAAG;GAC5B,SAAS;GACT;EACF;EAGA,IAAI,MAAM,WAAW,KAAK,uBAAuB,WAAW,GAAG;GAC7D,MAAM,aAAa;GACnB,SAAS;GACT,OAAO,QAAQ,aAAa,CAAC,uBAAuB,MAAM,MAAM,GAC9D,SAAS;GAGX,IAAI,QAAQ,WAAW,SAAS;GAChC,MAAM,KAAK;IAAE,MAAM;IAAW;IAAY,UAAU,QAAQ;GAAE,CAAC;GAC/D;EACF;EAIA,IAAI,sBAAsB,WAAW,GAAG;GACtC,MAAM,aAAa;GACnB,SAAS;GACT,OAAO,QAAQ,aAAa,CAAC,sBAAsB,MAAM,MAAM,GAC7D,SAAS;GAGX,IAAI,QAAQ,WAAW,SAAS;GAChC,MAAM,KAAK;IAAE,MAAM;IAAc;IAAY,UAAU,QAAQ;GAAE,CAAC;GAClE;EACF;EAGA,IAAI,UAAU,WAAW,GAAG;GAC1B,MAAM,KAAK;IAAE,MAAM;IAAW,YAAY;IAAO,UAAU;GAAM,CAAC;GAClE,SAAS;GACT;EACF;EAKA,MAAM,aAAa;EACnB,OACE,QAAQ,aACR,CAAC,YAAY,MAAM,MAAM,KACzB,CAAC,UAAU,MAAM,MAAM,KACvB,CAAC,sBAAsB,MAAM,MAAM,GAEnC,SAAS;EAEX,MAAM,KAAK;GAAE,MAAM;GAAa;GAAY,UAAU,QAAQ;EAAE,CAAC;CACnE;CAEA,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC;CAKhC,OAAO,MAAM,KAAK,MAAM,cAAqB;EAC3C,MAAM,kBAAkB,cAAc,IAAI,IAAI,KAAK;EACnD,MAAM,gBACJ,cAAc,MAAM,SAAS,IACzB,YAAY,IACZ,MAAM,YAAY,EAAE,CAAC,aAAa;EAExC,MAAM,aAAa,MAAM,MAAM,iBAAiB,gBAAgB,CAAC;EAGjE,MAAM,UACJ,gBAAgB,YAAY,IACxB,GAAG,WAAW,KAAK,IAAI,EAAE,MACzB,WAAW,KAAK,IAAI;EAE1B,OAAO;GACL,MAAM,KAAK;GACX;GACA,WAAW,kBAAkB;GAC7B,SAAS,gBAAgB;EAC3B;CACF,CAAC;AACH;;;;;;;;;;;;;;;;;;;AAoBA,MAAa,mBAAmB,SAA0B;CACxD,MAAM,aAAa,gBAAgB,IAAI;CACvC,MAAM,WAAoB,CAAC;CAC3B,IAAI,gBAAyB,CAAC;CAE9B,MAAM,qBAA2B;EAC/B,IAAI,cAAc,WAAW,GAAG;EAEhC,MAAM,CAAC,cAAc;EACrB,MAAM,YAAY,cAAc,cAAc,SAAS;EAEvD,SAAS,KAAK;GACZ,MAAM,WAAW;GACjB,SAAS,cAAc,KAAK,UAAU,MAAM,OAAO,CAAC,CAAC,KAAK,EAAE;GAC5D,WAAW,WAAW;GACtB,SAAS,UAAU;EACrB,CAAC;EACD,gBAAgB,CAAC;CACnB;CAEA,KAAK,MAAM,SAAS,YAAY;EAM9B,IAHE,MAAM,SAAS,aACd,MAAM,SAAS,aAAa,SAAS,WAAW,GAEjC,aAAa;EAC/B,cAAc,KAAK,KAAK;EACxB,IAAI,MAAM,SAAS,aAAa,SAAS,WAAW,GAAG,aAAa;CACtE;CAEA,aAAa;CAEb,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"segmentDocument.mjs","names":[],"sources":["../../../src/docReview/segmentDocument.ts"],"sourcesContent":["import type { Block, BlockType } from './types';\n\nconst HEADING_PATTERN = /^\\s*(#{1,6})\\s+/;\n\nconst isBlankLine = (line: string): boolean => line.trim().length === 0;\nconst isFencedCodeDelimiter = (line: string): boolean => /^\\s*```/.test(line);\n\n/**\n * Read the depth of an ATX markdown heading (`#` → 1, `######` → 6).\n *\n * @param line - The line to inspect.\n * @returns The heading depth, or `null` when the line is not a heading.\n */\nconst parseHeadingDepth = (line: string): number | null => {\n const match = HEADING_PATTERN.exec(line);\n\n return match?.[1]?.length ?? null;\n};\n\nconst isHeading = (line: string): boolean => parseHeadingDepth(line) !== null;\nconst isFrontmatterDelimiter = (line: string): boolean =>\n /^\\s*---\\s*$/.test(line);\n\n/**\n * A content unit (heading, paragraph, code block or frontmatter) spanning a\n * 0-based, inclusive line range. Blank-line runs are not units of their own;\n * they are folded into the preceding unit as trailing separators (see\n * {@link segmentDocument}) so that the concatenation of every block's content\n * reproduces the document byte-for-byte.\n */\ntype ContentUnit = {\n type: BlockType;\n /** Depth of the ATX heading opening the unit, `null` when it is not a heading. */\n headingDepth: number | null;\n startIndex: number;\n endIndex: number;\n};\n\n/**\n * Split a markdown document into fine-grained blocks.\n *\n * Boundaries are drawn at headings, blank lines (paragraph breaks) and fenced\n * code blocks, while frontmatter and the inside of code fences are kept intact.\n * Each blank-line run is appended to the block that precedes it, so the blocks\n * form an exact partition of the document: concatenating every `content` (in\n * order) yields the original text unchanged. This is what lets the block-aware\n * review re-translate only the paragraphs/snippets that actually changed instead\n * of the whole heading section.\n *\n * @param text - The full markdown document.\n * @returns The ordered list of blocks with their 1-based line ranges.\n */\nexport const segmentDocument = (text: string): Block[] => {\n const lines = text.split('\\n');\n const lineCount = lines.length;\n\n // 1. Tokenize into content units, skipping blank-line runs (folded in below).\n const units: ContentUnit[] = [];\n let index = 0;\n\n while (index < lineCount) {\n const currentLine = lines[index];\n\n if (isBlankLine(currentLine)) {\n index += 1;\n continue;\n }\n\n // Frontmatter: only when it opens the document.\n if (units.length === 0 && isFrontmatterDelimiter(currentLine)) {\n const startIndex = index;\n index += 1;\n while (index < lineCount && !isFrontmatterDelimiter(lines[index])) {\n index += 1;\n }\n // Include the closing delimiter when present.\n if (index < lineCount) index += 1;\n units.push({\n type: 'unknown',\n headingDepth: null,\n startIndex,\n endIndex: index - 1,\n });\n continue;\n }\n\n // Fenced code block: consumed whole so inner blank lines and `#` lines are\n // never treated as boundaries.\n if (isFencedCodeDelimiter(currentLine)) {\n const startIndex = index;\n index += 1;\n while (index < lineCount && !isFencedCodeDelimiter(lines[index])) {\n index += 1;\n }\n // Include the closing fence when present.\n if (index < lineCount) index += 1;\n units.push({\n type: 'code_block',\n headingDepth: null,\n startIndex,\n endIndex: index - 1,\n });\n continue;\n }\n\n // Heading: a single self-contained line.\n const headingDepth = parseHeadingDepth(currentLine);\n\n if (headingDepth !== null) {\n units.push({\n type: 'heading',\n headingDepth,\n startIndex: index,\n endIndex: index,\n });\n index += 1;\n continue;\n }\n\n // Paragraph: a run of consecutive lines until a blank line, a heading or a\n // code fence. Tables and tight lists stay together (no blank line between\n // their rows/items).\n const startIndex = index;\n while (\n index < lineCount &&\n !isBlankLine(lines[index]) &&\n !isHeading(lines[index]) &&\n !isFencedCodeDelimiter(lines[index])\n ) {\n index += 1;\n }\n units.push({\n type: 'paragraph',\n headingDepth: null,\n startIndex,\n endIndex: index - 1,\n });\n }\n\n if (units.length === 0) return [];\n\n // 2. Turn each unit into a block whose line range extends to just before the\n // next unit, so the trailing blank-line run is owned by it. The first block\n // also absorbs any leading blank lines, and the last block runs to EOF.\n return units.map((unit, unitIndex): Block => {\n const blockStartIndex = unitIndex === 0 ? 0 : unit.startIndex;\n const blockEndIndex =\n unitIndex === units.length - 1\n ? lineCount - 1\n : units[unitIndex + 1].startIndex - 1;\n\n const blockLines = lines.slice(blockStartIndex, blockEndIndex + 1);\n // Re-append the boundary newline dropped by `split` for every block but the\n // one ending at EOF, so concatenating all blocks rebuilds the document.\n const content =\n blockEndIndex < lineCount - 1\n ? `${blockLines.join('\\n')}\\n`\n : blockLines.join('\\n');\n\n return {\n type: unit.type,\n content,\n headingDepth: unit.headingDepth,\n lineStart: blockStartIndex + 1,\n lineEnd: blockEndIndex + 1,\n };\n });\n};\n\n/**\n * Split a markdown document into coarse, heading-anchored sections.\n *\n * Built by grouping the fine blocks of {@link segmentDocument}: frontmatter and\n * each heading open a new section, and the following non-heading blocks are\n * folded into it. Because it only concatenates adjacent fine blocks, the result\n * is still an exact partition of the document (sections concatenate back to the\n * source unchanged).\n *\n * Sections are the robust alignment unit between a base document and its\n * translation — both share the same heading structure, so they align almost\n * perfectly and a translation that splits its prose into a different number of\n * paragraphs never causes a section to be dropped. Fine-grained review happens\n * within a section once it is known to have changed.\n *\n * @param text - The full markdown document.\n * @returns The ordered list of sections with their 1-based line ranges.\n */\nexport const segmentSections = (text: string): Block[] => {\n const fineBlocks = segmentDocument(text);\n const sections: Block[] = [];\n let currentBlocks: Block[] = [];\n\n const flushSection = (): void => {\n if (currentBlocks.length === 0) return;\n\n const [firstBlock] = currentBlocks;\n const lastBlock = currentBlocks[currentBlocks.length - 1];\n\n sections.push({\n type: firstBlock.type,\n content: currentBlocks.map((block) => block.content).join(''),\n // A section is identified by the heading that opens it, so it inherits its\n // depth — this is what keeps a `##` section from aligning with a `###` one.\n headingDepth: firstBlock.headingDepth,\n lineStart: firstBlock.lineStart,\n lineEnd: lastBlock.lineEnd,\n });\n currentBlocks = [];\n };\n\n for (const block of fineBlocks) {\n // Frontmatter (a leading `unknown` block) and every heading open a section.\n const opensSection =\n block.type === 'heading' ||\n (block.type === 'unknown' && sections.length === 0);\n\n if (opensSection) flushSection();\n currentBlocks.push(block);\n if (block.type === 'unknown' && sections.length === 0) flushSection();\n }\n\n flushSection();\n\n return sections;\n};\n"],"mappings":";AAEA,MAAM,kBAAkB;AAExB,MAAM,eAAe,SAA0B,KAAK,KAAK,CAAC,CAAC,WAAW;AACtE,MAAM,yBAAyB,SAA0B,UAAU,KAAK,IAAI;;;;;;;AAQ5E,MAAM,qBAAqB,SAAgC;CAGzD,OAFc,gBAAgB,KAAK,IAExB,CAAC,GAAG,EAAE,EAAE,UAAU;AAC/B;AAEA,MAAM,aAAa,SAA0B,kBAAkB,IAAI,MAAM;AACzE,MAAM,0BAA0B,SAC9B,cAAc,KAAK,IAAI;;;;;;;;;;;;;;;AA+BzB,MAAa,mBAAmB,SAA0B;CACxD,MAAM,QAAQ,KAAK,MAAM,IAAI;CAC7B,MAAM,YAAY,MAAM;CAGxB,MAAM,QAAuB,CAAC;CAC9B,IAAI,QAAQ;CAEZ,OAAO,QAAQ,WAAW;EACxB,MAAM,cAAc,MAAM;EAE1B,IAAI,YAAY,WAAW,GAAG;GAC5B,SAAS;GACT;EACF;EAGA,IAAI,MAAM,WAAW,KAAK,uBAAuB,WAAW,GAAG;GAC7D,MAAM,aAAa;GACnB,SAAS;GACT,OAAO,QAAQ,aAAa,CAAC,uBAAuB,MAAM,MAAM,GAC9D,SAAS;GAGX,IAAI,QAAQ,WAAW,SAAS;GAChC,MAAM,KAAK;IACT,MAAM;IACN,cAAc;IACd;IACA,UAAU,QAAQ;GACpB,CAAC;GACD;EACF;EAIA,IAAI,sBAAsB,WAAW,GAAG;GACtC,MAAM,aAAa;GACnB,SAAS;GACT,OAAO,QAAQ,aAAa,CAAC,sBAAsB,MAAM,MAAM,GAC7D,SAAS;GAGX,IAAI,QAAQ,WAAW,SAAS;GAChC,MAAM,KAAK;IACT,MAAM;IACN,cAAc;IACd;IACA,UAAU,QAAQ;GACpB,CAAC;GACD;EACF;EAGA,MAAM,eAAe,kBAAkB,WAAW;EAElD,IAAI,iBAAiB,MAAM;GACzB,MAAM,KAAK;IACT,MAAM;IACN;IACA,YAAY;IACZ,UAAU;GACZ,CAAC;GACD,SAAS;GACT;EACF;EAKA,MAAM,aAAa;EACnB,OACE,QAAQ,aACR,CAAC,YAAY,MAAM,MAAM,KACzB,CAAC,UAAU,MAAM,MAAM,KACvB,CAAC,sBAAsB,MAAM,MAAM,GAEnC,SAAS;EAEX,MAAM,KAAK;GACT,MAAM;GACN,cAAc;GACd;GACA,UAAU,QAAQ;EACpB,CAAC;CACH;CAEA,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC;CAKhC,OAAO,MAAM,KAAK,MAAM,cAAqB;EAC3C,MAAM,kBAAkB,cAAc,IAAI,IAAI,KAAK;EACnD,MAAM,gBACJ,cAAc,MAAM,SAAS,IACzB,YAAY,IACZ,MAAM,YAAY,EAAE,CAAC,aAAa;EAExC,MAAM,aAAa,MAAM,MAAM,iBAAiB,gBAAgB,CAAC;EAGjE,MAAM,UACJ,gBAAgB,YAAY,IACxB,GAAG,WAAW,KAAK,IAAI,EAAE,MACzB,WAAW,KAAK,IAAI;EAE1B,OAAO;GACL,MAAM,KAAK;GACX;GACA,cAAc,KAAK;GACnB,WAAW,kBAAkB;GAC7B,SAAS,gBAAgB;EAC3B;CACF,CAAC;AACH;;;;;;;;;;;;;;;;;;;AAoBA,MAAa,mBAAmB,SAA0B;CACxD,MAAM,aAAa,gBAAgB,IAAI;CACvC,MAAM,WAAoB,CAAC;CAC3B,IAAI,gBAAyB,CAAC;CAE9B,MAAM,qBAA2B;EAC/B,IAAI,cAAc,WAAW,GAAG;EAEhC,MAAM,CAAC,cAAc;EACrB,MAAM,YAAY,cAAc,cAAc,SAAS;EAEvD,SAAS,KAAK;GACZ,MAAM,WAAW;GACjB,SAAS,cAAc,KAAK,UAAU,MAAM,OAAO,CAAC,CAAC,KAAK,EAAE;GAG5D,cAAc,WAAW;GACzB,WAAW,WAAW;GACtB,SAAS,UAAU;EACrB,CAAC;EACD,gBAAgB,CAAC;CACnB;CAEA,KAAK,MAAM,SAAS,YAAY;EAM9B,IAHE,MAAM,SAAS,aACd,MAAM,SAAS,aAAa,SAAS,WAAW,GAEjC,aAAa;EAC/B,cAAc,KAAK,KAAK;EACxB,IAAI,MAAM,SAAS,aAAa,SAAS,WAAW,GAAG,aAAa;CACtE;CAEA,aAAa;CAEb,OAAO;AACT"}
|
|
@@ -1,45 +1,139 @@
|
|
|
1
1
|
import { parallelize } from "./utils/parallelize.mjs";
|
|
2
|
+
import { chunkArray } from "./utils/chunkArray.mjs";
|
|
2
3
|
import { getAppLogger, x } from "@intlayer/config/logger";
|
|
3
4
|
import { getConfiguration } from "@intlayer/config/node";
|
|
5
|
+
import { retryManager } from "@intlayer/config/utils";
|
|
4
6
|
import { createIntlayerCMS } from "@intlayer/api";
|
|
5
7
|
import { dictionaryEndpoint } from "@intlayer/api/dictionary";
|
|
6
8
|
|
|
7
9
|
//#region src/fetchDistantDictionaries.ts
|
|
8
10
|
/**
|
|
11
|
+
* Number of dictionaries requested per HTTP call. Must stay under the batch
|
|
12
|
+
* limit enforced by the backend (`MAX_DICTIONARY_KEYS_PER_REQUEST`).
|
|
13
|
+
*/
|
|
14
|
+
const DEFAULT_DICTIONARIES_PER_REQUEST = 3;
|
|
15
|
+
/** Number of batch requests kept in flight at the same time. */
|
|
16
|
+
const DEFAULT_PARALLEL_REQUESTS = 5;
|
|
17
|
+
/** Number of retries performed on a failing batch request. */
|
|
18
|
+
const DEFAULT_MAX_RETRY = 2;
|
|
19
|
+
/** Delay between two attempts of a failing batch request, in milliseconds. */
|
|
20
|
+
const DEFAULT_RETRY_DELAY = 250;
|
|
21
|
+
/** HTTP statuses worth another attempt despite being client errors. */
|
|
22
|
+
const RETRYABLE_CLIENT_ERROR_STATUSES = /* @__PURE__ */ new Set([
|
|
23
|
+
408,
|
|
24
|
+
425,
|
|
25
|
+
429
|
|
26
|
+
]);
|
|
27
|
+
/**
|
|
28
|
+
* A request is worth retrying when it failed for a reason another attempt could
|
|
29
|
+
* fix: a network error (no status at all) or a server-side / throttling status.
|
|
30
|
+
* Replaying a `404` or a `401` only multiplies the noise.
|
|
31
|
+
*/
|
|
32
|
+
const isRetryableError = (error) => {
|
|
33
|
+
const status = error?.status;
|
|
34
|
+
if (typeof status !== "number") return true;
|
|
35
|
+
if (status >= 500) return true;
|
|
36
|
+
return RETRYABLE_CLIENT_ERROR_STATUSES.has(status);
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* A backend that predates the batch endpoint routes `/dictionary/by-keys` to
|
|
40
|
+
* its `/dictionary/:dictionaryKey` handler, which answers `404` - either as an
|
|
41
|
+
* unknown dictionary or as an unknown route. Both mean the same thing here:
|
|
42
|
+
* this backend cannot serve batches, so fall back to one request per key.
|
|
43
|
+
*/
|
|
44
|
+
const isBatchEndpointUnsupported = (error) => error?.status === 404;
|
|
45
|
+
/**
|
|
9
46
|
* Fetch distant dictionaries and update the logger with their statuses.
|
|
47
|
+
*
|
|
48
|
+
* Keys are grouped into batches fetched by a single request each, and those
|
|
49
|
+
* requests are kept in flight together — by default 5 concurrent requests of 3
|
|
50
|
+
* dictionaries, so 15 dictionaries travel at any given time. A slot is refilled
|
|
51
|
+
* as soon as its request settles, so no batch waits for the whole wave.
|
|
52
|
+
* Failing requests are retried before their keys are reported as errors.
|
|
10
53
|
*/
|
|
11
54
|
const fetchDistantDictionaries = async (options, onStatusUpdate) => {
|
|
12
55
|
const config = getConfiguration();
|
|
13
56
|
const appLogger = getAppLogger(config);
|
|
14
57
|
try {
|
|
15
58
|
const dictionary = dictionaryEndpoint(createIntlayerCMS(config));
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
|
|
59
|
+
const { dictionariesPerRequest = DEFAULT_DICTIONARIES_PER_REQUEST, parallelRequests = DEFAULT_PARALLEL_REQUESTS, maxRetry = DEFAULT_MAX_RETRY, retryDelay = DEFAULT_RETRY_DELAY } = options;
|
|
60
|
+
const distantDictionariesKeys = [...new Set(options.dictionaryKeys)];
|
|
61
|
+
if (distantDictionariesKeys.length === 0) return [];
|
|
62
|
+
const batches = chunkArray(distantDictionariesKeys, dictionariesPerRequest);
|
|
63
|
+
let isBatchEndpointSupported = true;
|
|
64
|
+
let hasLoggedBatchFallback = false;
|
|
65
|
+
/**
|
|
66
|
+
* Fetch each key of a batch with its own request. Used against backends
|
|
67
|
+
* without the batch endpoint. A key missing on the remote only discards
|
|
68
|
+
* itself, never its neighbours.
|
|
69
|
+
*/
|
|
70
|
+
const requestDictionariesOneByOne = async (batchKeys) => {
|
|
71
|
+
return (await parallelize(batchKeys, async (dictionaryKey) => {
|
|
72
|
+
try {
|
|
73
|
+
return (await dictionary.getDictionary(dictionaryKey)).data;
|
|
74
|
+
} catch (_error) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
}, batchKeys.length)).filter((distantDictionary) => distantDictionary !== void 0);
|
|
78
|
+
};
|
|
79
|
+
/**
|
|
80
|
+
* Request one batch of dictionaries, retrying the whole request when it
|
|
81
|
+
* fails for a transient reason - a batch carries several dictionaries, so
|
|
82
|
+
* a network hiccup would otherwise discard all of them at once.
|
|
83
|
+
*/
|
|
84
|
+
const requestBatch = retryManager(async (batchKeys) => {
|
|
85
|
+
if (isBatchEndpointSupported) try {
|
|
86
|
+
return (await dictionary.getDictionariesByKeys(batchKeys)).data ?? [];
|
|
87
|
+
} catch (error) {
|
|
88
|
+
if (!isBatchEndpointUnsupported(error)) throw error;
|
|
89
|
+
isBatchEndpointSupported = false;
|
|
90
|
+
if (!hasLoggedBatchFallback) {
|
|
91
|
+
hasLoggedBatchFallback = true;
|
|
92
|
+
appLogger("This backend does not provide the batch dictionary endpoint - falling back to one request per dictionary.", { level: "warn" });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return await requestDictionariesOneByOne(batchKeys);
|
|
96
|
+
}, {
|
|
97
|
+
maxRetry,
|
|
98
|
+
delay: retryDelay,
|
|
99
|
+
shouldRetry: isRetryableError,
|
|
100
|
+
onError: ({ error, attempt }) => appLogger(`Retrying distant dictionaries request (attempt ${attempt + 1}/${maxRetry}): ${error}`, { level: "warn" })
|
|
101
|
+
});
|
|
102
|
+
/**
|
|
103
|
+
* Fetch one batch of dictionaries within a single request, and report the
|
|
104
|
+
* resulting status of every key it contains.
|
|
105
|
+
*/
|
|
106
|
+
const processBatch = async (batchKeys) => {
|
|
107
|
+
onStatusUpdate?.(batchKeys.map((dictionaryKey) => ({
|
|
19
108
|
dictionaryKey,
|
|
20
109
|
type: "remote",
|
|
21
110
|
status: "fetching"
|
|
22
|
-
}
|
|
111
|
+
})));
|
|
23
112
|
try {
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
onStatusUpdate?.(
|
|
113
|
+
const distantDictionaries = await requestBatch(batchKeys);
|
|
114
|
+
const fetchedKeys = new Set(distantDictionaries.map((distantDictionary) => distantDictionary.key));
|
|
115
|
+
onStatusUpdate?.(batchKeys.map((dictionaryKey) => fetchedKeys.has(dictionaryKey) ? {
|
|
27
116
|
dictionaryKey,
|
|
28
117
|
type: "remote",
|
|
29
118
|
status: "fetched"
|
|
30
|
-
}
|
|
31
|
-
|
|
119
|
+
} : {
|
|
120
|
+
dictionaryKey,
|
|
121
|
+
type: "remote",
|
|
122
|
+
status: "error",
|
|
123
|
+
error: `Dictionary ${dictionaryKey} not found on remote`
|
|
124
|
+
}));
|
|
125
|
+
return distantDictionaries;
|
|
32
126
|
} catch (error) {
|
|
33
|
-
onStatusUpdate?.(
|
|
127
|
+
onStatusUpdate?.(batchKeys.map((dictionaryKey) => ({
|
|
34
128
|
dictionaryKey,
|
|
35
129
|
type: "remote",
|
|
36
130
|
status: "error",
|
|
37
131
|
error: `Error fetching dictionary ${dictionaryKey}: ${error}`
|
|
38
|
-
}
|
|
39
|
-
return;
|
|
132
|
+
})));
|
|
133
|
+
return [];
|
|
40
134
|
}
|
|
41
135
|
};
|
|
42
|
-
return (await parallelize(
|
|
136
|
+
return (await parallelize(batches, processBatch, parallelRequests)).flat();
|
|
43
137
|
} catch (_error) {
|
|
44
138
|
appLogger(`${x} Failed to fetch distant dictionaries`, { level: "error" });
|
|
45
139
|
return [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetchDistantDictionaries.mjs","names":[],"sources":["../../src/fetchDistantDictionaries.ts"],"sourcesContent":["import { createIntlayerCMS } from '@intlayer/api';\nimport { dictionaryEndpoint } from '@intlayer/api/dictionary';\n// @ts-ignore @intlayer/backend is not build yet\nimport type { DictionaryAPI } from '@intlayer/backend';\nimport { getAppLogger, x } from '@intlayer/config/logger';\nimport { getConfiguration } from '@intlayer/config/node';\nimport type { DictionariesStatus } from './loadDictionaries';\nimport { parallelize } from './utils/parallelize';\n\ntype FetchDistantDictionariesOptions = {\n dictionaryKeys: string[];\n newDictionariesPath?: string;\n logPrefix?: string;\n};\n\n/**\n * Fetch distant dictionaries and update the logger with their statuses.\n */\nexport const fetchDistantDictionaries = async (\n options: FetchDistantDictionariesOptions,\n onStatusUpdate?: (status: DictionariesStatus[]) => void\n): Promise<DictionaryAPI[]> => {\n const config = getConfiguration();\n const appLogger = getAppLogger(config);\n try {\n const dictionary = dictionaryEndpoint(createIntlayerCMS(config));\n\n const distantDictionariesKeys = options.dictionaryKeys;\n // Process dictionaries in parallel with a concurrency limit\n const processDictionary = async (\n dictionaryKey: string\n ): Promise<DictionaryAPI | undefined> => {\n onStatusUpdate?.([\n {\n dictionaryKey,\n type: 'remote',\n status: 'fetching',\n },\n ]);\n\n try {\n // Fetch the dictionary\n const getDictionaryResult =\n await dictionary.getDictionary(dictionaryKey);\n\n const distantDictionary = getDictionaryResult.data;\n\n if (!distantDictionary) {\n throw new Error(`Dictionary ${dictionaryKey} not found on remote`);\n }\n\n onStatusUpdate?.([\n { dictionaryKey, type: 'remote', status: 'fetched' },\n ]);\n\n return distantDictionary;\n } catch (error) {\n onStatusUpdate?.([\n {\n dictionaryKey,\n type: 'remote',\n status: 'error',\n error: `Error fetching dictionary ${dictionaryKey}: ${error}`,\n },\n ]);\n return undefined;\n }\n };\n\n const result = await parallelize(\n distantDictionariesKeys,\n async (dictionaryKey) => await processDictionary(dictionaryKey)\n );\n\n // Remove undefined values\n const filteredResult = result.filter(\n (dict: DictionaryAPI | undefined): dict is DictionaryAPI =>\n dict !== undefined\n );\n\n return filteredResult;\n } catch (_error) {\n appLogger(`${x} Failed to fetch distant dictionaries`, { level: 'error' });\n return [];\n }\n};\n"],"mappings":";;;;;;;;;;AAkBA,MAAa,2BAA2B,OACtC,SACA,mBAC6B;CAC7B,MAAM,SAAS,iBAAiB;CAChC,MAAM,YAAY,aAAa,MAAM;CACrC,IAAI;EACF,MAAM,aAAa,mBAAmB,kBAAkB,MAAM,CAAC;EAE/D,MAAM,0BAA0B,QAAQ;EAExC,MAAM,oBAAoB,OACxB,kBACuC;GACvC,iBAAiB,CACf;IACE;IACA,MAAM;IACN,QAAQ;GACV,CACF,CAAC;GAED,IAAI;IAKF,MAAM,qBAAoB,MAFlB,WAAW,cAAc,aAAa,EAED,CAAC;IAE9C,IAAI,CAAC,mBACH,MAAM,IAAI,MAAM,cAAc,cAAc,qBAAqB;IAGnE,iBAAiB,CACf;KAAE;KAAe,MAAM;KAAU,QAAQ;IAAU,CACrD,CAAC;IAED,OAAO;GACT,SAAS,OAAO;IACd,iBAAiB,CACf;KACE;KACA,MAAM;KACN,QAAQ;KACR,OAAO,6BAA6B,cAAc,IAAI;IACxD,CACF,CAAC;IACD;GACF;EACF;EAaA,QALuB,MANF,YACnB,yBACA,OAAO,kBAAkB,MAAM,kBAAkB,aAAa,CAChE,EAG6B,CAAC,QAC3B,SACC,SAAS,MAGO;CACtB,SAAS,QAAQ;EACf,UAAU,GAAG,EAAE,wCAAwC,EAAE,OAAO,QAAQ,CAAC;EACzE,OAAO,CAAC;CACV;AACF"}
|
|
1
|
+
{"version":3,"file":"fetchDistantDictionaries.mjs","names":[],"sources":["../../src/fetchDistantDictionaries.ts"],"sourcesContent":["import { createIntlayerCMS } from '@intlayer/api';\nimport { dictionaryEndpoint } from '@intlayer/api/dictionary';\n// @ts-ignore @intlayer/backend is not build yet\nimport type { DictionaryAPI } from '@intlayer/backend';\nimport { getAppLogger, x } from '@intlayer/config/logger';\nimport { getConfiguration } from '@intlayer/config/node';\nimport { retryManager } from '@intlayer/config/utils';\nimport type { DictionariesStatus } from './loadDictionaries';\nimport { chunkArray } from './utils/chunkArray';\nimport { parallelize } from './utils/parallelize';\n\n/**\n * Number of dictionaries requested per HTTP call. Must stay under the batch\n * limit enforced by the backend (`MAX_DICTIONARY_KEYS_PER_REQUEST`).\n */\nconst DEFAULT_DICTIONARIES_PER_REQUEST = 3;\n\n/** Number of batch requests kept in flight at the same time. */\nconst DEFAULT_PARALLEL_REQUESTS = 5;\n\n/** Number of retries performed on a failing batch request. */\nconst DEFAULT_MAX_RETRY = 2;\n\n/** Delay between two attempts of a failing batch request, in milliseconds. */\nconst DEFAULT_RETRY_DELAY = 250;\n\n/** HTTP statuses worth another attempt despite being client errors. */\nconst RETRYABLE_CLIENT_ERROR_STATUSES = new Set([408, 425, 429]);\n\n/**\n * A request is worth retrying when it failed for a reason another attempt could\n * fix: a network error (no status at all) or a server-side / throttling status.\n * Replaying a `404` or a `401` only multiplies the noise.\n */\nconst isRetryableError = (error: unknown): boolean => {\n const status = (error as { status?: number })?.status;\n\n if (typeof status !== 'number') return true;\n\n if (status >= 500) return true;\n\n return RETRYABLE_CLIENT_ERROR_STATUSES.has(status);\n};\n\n/**\n * A backend that predates the batch endpoint routes `/dictionary/by-keys` to\n * its `/dictionary/:dictionaryKey` handler, which answers `404` - either as an\n * unknown dictionary or as an unknown route. Both mean the same thing here:\n * this backend cannot serve batches, so fall back to one request per key.\n */\nconst isBatchEndpointUnsupported = (error: unknown): boolean =>\n (error as { status?: number })?.status === 404;\n\ntype FetchDistantDictionariesOptions = {\n dictionaryKeys: string[];\n newDictionariesPath?: string;\n logPrefix?: string;\n /**\n * Number of dictionaries fetched by a single request.\n * @default 3\n */\n dictionariesPerRequest?: number;\n /**\n * Number of requests kept in flight at the same time.\n * @default 5\n */\n parallelRequests?: number;\n /**\n * Number of retries performed when a batch request fails.\n * @default 2\n */\n maxRetry?: number;\n /**\n * Delay between two attempts of a failing batch request, in milliseconds.\n * @default 250\n */\n retryDelay?: number;\n};\n\n/**\n * Fetch distant dictionaries and update the logger with their statuses.\n *\n * Keys are grouped into batches fetched by a single request each, and those\n * requests are kept in flight together — by default 5 concurrent requests of 3\n * dictionaries, so 15 dictionaries travel at any given time. A slot is refilled\n * as soon as its request settles, so no batch waits for the whole wave.\n * Failing requests are retried before their keys are reported as errors.\n */\nexport const fetchDistantDictionaries = async (\n options: FetchDistantDictionariesOptions,\n onStatusUpdate?: (status: DictionariesStatus[]) => void\n): Promise<DictionaryAPI[]> => {\n const config = getConfiguration();\n const appLogger = getAppLogger(config);\n try {\n const dictionary = dictionaryEndpoint(createIntlayerCMS(config));\n\n const {\n dictionariesPerRequest = DEFAULT_DICTIONARIES_PER_REQUEST,\n parallelRequests = DEFAULT_PARALLEL_REQUESTS,\n maxRetry = DEFAULT_MAX_RETRY,\n retryDelay = DEFAULT_RETRY_DELAY,\n } = options;\n\n // A same key can be listed twice when several qualified dictionaries share\n // it - one request per key is enough to retrieve all of its siblings.\n const distantDictionariesKeys = [...new Set(options.dictionaryKeys)];\n\n if (distantDictionariesKeys.length === 0) return [];\n\n const batches = chunkArray(distantDictionariesKeys, dictionariesPerRequest);\n\n // Flipped once for the whole run as soon as a backend proves unable to\n // serve batches, so the remaining batches skip the doomed request.\n let isBatchEndpointSupported = true;\n let hasLoggedBatchFallback = false;\n\n /**\n * Fetch each key of a batch with its own request. Used against backends\n * without the batch endpoint. A key missing on the remote only discards\n * itself, never its neighbours.\n */\n const requestDictionariesOneByOne = async (\n batchKeys: string[]\n ): Promise<DictionaryAPI[]> => {\n const results = await parallelize(\n batchKeys,\n async (dictionaryKey) => {\n try {\n const getDictionaryResult =\n await dictionary.getDictionary(dictionaryKey);\n\n return getDictionaryResult.data;\n } catch (_error) {\n return undefined;\n }\n },\n batchKeys.length\n );\n\n return results.filter(\n (distantDictionary): distantDictionary is DictionaryAPI =>\n distantDictionary !== undefined\n );\n };\n\n /**\n * Request one batch of dictionaries, retrying the whole request when it\n * fails for a transient reason - a batch carries several dictionaries, so\n * a network hiccup would otherwise discard all of them at once.\n */\n const requestBatch = retryManager(\n async (batchKeys: string[]): Promise<DictionaryAPI[]> => {\n if (isBatchEndpointSupported) {\n try {\n const getDictionariesResult =\n await dictionary.getDictionariesByKeys(batchKeys);\n\n return getDictionariesResult.data ?? [];\n } catch (error) {\n if (!isBatchEndpointUnsupported(error)) throw error;\n\n isBatchEndpointSupported = false;\n\n if (!hasLoggedBatchFallback) {\n hasLoggedBatchFallback = true;\n appLogger(\n 'This backend does not provide the batch dictionary endpoint - falling back to one request per dictionary.',\n { level: 'warn' }\n );\n }\n }\n }\n\n return await requestDictionariesOneByOne(batchKeys);\n },\n {\n maxRetry,\n delay: retryDelay,\n shouldRetry: isRetryableError,\n onError: ({ error, attempt }) =>\n appLogger(\n `Retrying distant dictionaries request (attempt ${attempt + 1}/${maxRetry}): ${error}`,\n { level: 'warn' }\n ),\n }\n );\n\n /**\n * Fetch one batch of dictionaries within a single request, and report the\n * resulting status of every key it contains.\n */\n const processBatch = async (\n batchKeys: string[]\n ): Promise<DictionaryAPI[]> => {\n onStatusUpdate?.(\n batchKeys.map((dictionaryKey) => ({\n dictionaryKey,\n type: 'remote',\n status: 'fetching',\n }))\n );\n\n try {\n const distantDictionaries = await requestBatch(batchKeys);\n\n const fetchedKeys = new Set(\n distantDictionaries.map(\n (distantDictionary: DictionaryAPI) => distantDictionary.key\n )\n );\n\n onStatusUpdate?.(\n batchKeys.map(\n (dictionaryKey): DictionariesStatus =>\n fetchedKeys.has(dictionaryKey)\n ? {\n dictionaryKey,\n type: 'remote',\n status: 'fetched',\n }\n : {\n dictionaryKey,\n type: 'remote',\n status: 'error',\n error: `Dictionary ${dictionaryKey} not found on remote`,\n }\n )\n );\n\n return distantDictionaries;\n } catch (error) {\n onStatusUpdate?.(\n batchKeys.map((dictionaryKey) => ({\n dictionaryKey,\n type: 'remote',\n status: 'error',\n error: `Error fetching dictionary ${dictionaryKey}: ${error}`,\n }))\n );\n return [];\n }\n };\n\n const batchResults = await parallelize(\n batches,\n processBatch,\n parallelRequests\n );\n\n return batchResults.flat();\n } catch (_error) {\n appLogger(`${x} Failed to fetch distant dictionaries`, { level: 'error' });\n return [];\n }\n};\n"],"mappings":";;;;;;;;;;;;;AAeA,MAAM,mCAAmC;;AAGzC,MAAM,4BAA4B;;AAGlC,MAAM,oBAAoB;;AAG1B,MAAM,sBAAsB;;AAG5B,MAAM,kDAAkC,IAAI,IAAI;CAAC;CAAK;CAAK;AAAG,CAAC;;;;;;AAO/D,MAAM,oBAAoB,UAA4B;CACpD,MAAM,SAAU,OAA+B;CAE/C,IAAI,OAAO,WAAW,UAAU,OAAO;CAEvC,IAAI,UAAU,KAAK,OAAO;CAE1B,OAAO,gCAAgC,IAAI,MAAM;AACnD;;;;;;;AAQA,MAAM,8BAA8B,UACjC,OAA+B,WAAW;;;;;;;;;;AAqC7C,MAAa,2BAA2B,OACtC,SACA,mBAC6B;CAC7B,MAAM,SAAS,iBAAiB;CAChC,MAAM,YAAY,aAAa,MAAM;CACrC,IAAI;EACF,MAAM,aAAa,mBAAmB,kBAAkB,MAAM,CAAC;EAE/D,MAAM,EACJ,yBAAyB,kCACzB,mBAAmB,2BACnB,WAAW,mBACX,aAAa,wBACX;EAIJ,MAAM,0BAA0B,CAAC,GAAG,IAAI,IAAI,QAAQ,cAAc,CAAC;EAEnE,IAAI,wBAAwB,WAAW,GAAG,OAAO,CAAC;EAElD,MAAM,UAAU,WAAW,yBAAyB,sBAAsB;EAI1E,IAAI,2BAA2B;EAC/B,IAAI,yBAAyB;;;;;;EAO7B,MAAM,8BAA8B,OAClC,cAC6B;GAgB7B,QAAO,MAfe,YACpB,WACA,OAAO,kBAAkB;IACvB,IAAI;KAIF,QAAO,MAFC,WAAW,cAAc,aAAa,EAEpB,CAAC;IAC7B,SAAS,QAAQ;KACf;IACF;GACF,GACA,UAAU,MACZ,EAEc,CAAC,QACZ,sBACC,sBAAsB,MAC1B;EACF;;;;;;EAOA,MAAM,eAAe,aACnB,OAAO,cAAkD;GACvD,IAAI,0BACF,IAAI;IAIF,QAAO,MAFC,WAAW,sBAAsB,SAAS,EAEtB,CAAC,QAAQ,CAAC;GACxC,SAAS,OAAO;IACd,IAAI,CAAC,2BAA2B,KAAK,GAAG,MAAM;IAE9C,2BAA2B;IAE3B,IAAI,CAAC,wBAAwB;KAC3B,yBAAyB;KACzB,UACE,6GACA,EAAE,OAAO,OAAO,CAClB;IACF;GACF;GAGF,OAAO,MAAM,4BAA4B,SAAS;EACpD,GACA;GACE;GACA,OAAO;GACP,aAAa;GACb,UAAU,EAAE,OAAO,cACjB,UACE,kDAAkD,UAAU,EAAE,GAAG,SAAS,KAAK,SAC/E,EAAE,OAAO,OAAO,CAClB;EACJ,CACF;;;;;EAMA,MAAM,eAAe,OACnB,cAC6B;GAC7B,iBACE,UAAU,KAAK,mBAAmB;IAChC;IACA,MAAM;IACN,QAAQ;GACV,EAAE,CACJ;GAEA,IAAI;IACF,MAAM,sBAAsB,MAAM,aAAa,SAAS;IAExD,MAAM,cAAc,IAAI,IACtB,oBAAoB,KACjB,sBAAqC,kBAAkB,GAC1D,CACF;IAEA,iBACE,UAAU,KACP,kBACC,YAAY,IAAI,aAAa,IACzB;KACE;KACA,MAAM;KACN,QAAQ;IACV,IACA;KACE;KACA,MAAM;KACN,QAAQ;KACR,OAAO,cAAc,cAAc;IACrC,CACR,CACF;IAEA,OAAO;GACT,SAAS,OAAO;IACd,iBACE,UAAU,KAAK,mBAAmB;KAChC;KACA,MAAM;KACN,QAAQ;KACR,OAAO,6BAA6B,cAAc,IAAI;IACxD,EAAE,CACJ;IACA,OAAO,CAAC;GACV;EACF;EAQA,QAAO,MANoB,YACzB,SACA,cACA,gBACF,EAEmB,CAAC,KAAK;CAC3B,SAAS,QAAQ;EACf,UAAU,GAAG,EAAE,wCAAwC,EAAE,OAAO,QAAQ,CAAC;EACzE,OAAO,CAAC;CACV;AACF"}
|
|
@@ -80,7 +80,7 @@ const setHtmlLangAndDir = (ast) => {
|
|
|
80
80
|
const upsertAttribute = (name, value) => {
|
|
81
81
|
const existing = node.attributes?.find((attr) => attr.type === "JSXAttribute" && attr.name?.name === name);
|
|
82
82
|
if (existing) existing.value = value;
|
|
83
|
-
else node.attributes
|
|
83
|
+
else node.attributes?.push(b.jsxAttribute(b.jsxIdentifier(name), value));
|
|
84
84
|
};
|
|
85
85
|
upsertAttribute("lang", b.jsxExpressionContainer(b.identifier("locale")));
|
|
86
86
|
upsertAttribute("dir", b.jsxExpressionContainer(b.callExpression(b.identifier("getHTMLTextDir"), [b.identifier("locale")])));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transforms.mjs","names":[],"sources":["../../../../../src/init/frameworkSetup/tanstackStart/transforms.ts"],"sourcesContent":["import * as recast from 'recast';\nimport { babelTsParser } from '../../../utils/babelParser';\nimport { ensureNamedImport } from '../../utils/astImports';\n\nconst { builders: b, namedTypes: n } = recast.types;\n\n/** babel-ts parser handles TypeScript *and* JSX. */\nconst parseTsx = (code: string): any =>\n recast.parse(code, { parser: babelTsParser });\n\n/** Result of a source transform. `code` is unchanged for any non-`wrapped` status. */\nexport type TransformResult = {\n code: string;\n status: 'wrapped' | 'already' | 'skipped';\n};\n\n/**\n * Finds the single function that returns the `<html>` document element. Returns\n * `null` when there is no such function, or more than one (ambiguous).\n */\nconst findHtmlDocumentFunction = (ast: any): any => {\n const found = new Set<any>();\n\n recast.visit(ast, {\n visitJSXOpeningElement(path) {\n const node = path.node;\n if (node.name?.type === 'JSXIdentifier' && node.name.name === 'html') {\n // Walk up to the nearest enclosing function node.\n let current = path.parentPath;\n while (current) {\n const value = current.value;\n if (\n value &&\n (n.FunctionDeclaration.check(value) ||\n n.FunctionExpression.check(value) ||\n n.ArrowFunctionExpression.check(value))\n ) {\n found.add(value);\n break;\n }\n current = current.parentPath;\n }\n }\n this.traverse(path);\n },\n });\n\n return found.size === 1 ? [...found][0] : null;\n};\n\n/**\n * Whether the binding pattern `node` introduces a variable named `name`.\n * Recurses through destructuring patterns so all of `const locale`,\n * `const { locale } = …`, `const { locale = fallback } = …`,\n * `const { foo: locale } = …`, `const [locale] = …` and `...locale` are\n * detected — not just a bare identifier.\n */\nconst patternBindsName = (node: any, name: string): boolean => {\n if (!node) return false;\n switch (node.type) {\n case 'Identifier':\n return node.name === name;\n case 'AssignmentPattern':\n // `locale = fallback`\n return patternBindsName(node.left, name);\n case 'RestElement':\n // `...locale`\n return patternBindsName(node.argument, name);\n case 'ArrayPattern':\n return node.elements.some((element: any) =>\n patternBindsName(element, name)\n );\n case 'ObjectPattern':\n return node.properties.some((property: any) =>\n property.type === 'RestElement'\n ? patternBindsName(property.argument, name)\n : // The *value* is the binding target (`{ foo: locale }` binds `locale`).\n patternBindsName(property.value, name)\n );\n default:\n return false;\n }\n};\n\n/**\n * Inserts `const locale = …` once, at the top of the function body. A single\n * self-contained statement is used (rather than an intermediate `params` const)\n * so it can never collide with a `params` variable the document already\n * declares. Expression-bodied arrow functions are first converted to a block so\n * the declaration has somewhere to live.\n *\n * No-ops when `locale` is already bound in the function — whether as a simple\n * identifier, through a destructuring pattern (e.g.\n * `const { locale = defaultLocale } = useLoaderData()`), or as a function\n * parameter — so the injected declaration never collides with one the document\n * already provides (which would produce an\n * `Identifier 'locale' has already been declared` parse error).\n */\nconst ensureLocaleFromParams = (funcNode: any): void => {\n if (funcNode.body?.type !== 'BlockStatement') {\n funcNode.body = b.blockStatement([b.returnStatement(funcNode.body)]);\n }\n\n const declaredInBody = funcNode.body.body.some(\n (stmt: any) =>\n stmt.type === 'VariableDeclaration' &&\n stmt.declarations.some((d: any) => patternBindsName(d.id, 'locale'))\n );\n const declaredInParams = (funcNode.params ?? []).some((param: any) =>\n patternBindsName(param, 'locale')\n );\n if (declaredInBody || declaredInParams) return;\n\n const statement = parseTsx(\n 'const locale = useParams({ strict: false }).locale ?? defaultLocale;'\n ).program.body[0];\n funcNode.body.body.unshift(statement);\n};\n\n/** Sets `lang={locale}` and `dir={getHTMLTextDir(locale)}` on the first `<html>` element. */\nconst setHtmlLangAndDir = (ast: any): void => {\n recast.visit(ast, {\n visitJSXOpeningElement(path) {\n const node = path.node;\n if (node.name?.type !== 'JSXIdentifier' || node.name.name !== 'html') {\n this.traverse(path);\n return;\n }\n\n const upsertAttribute = (name: string, value: any): void => {\n const existing = node.attributes?.find(\n (attr: any) =>\n attr.type === 'JSXAttribute' && attr.name?.name === name\n ) as any;\n if (existing) {\n existing.value = value;\n } else {\n node.attributes.push(b.jsxAttribute(b.jsxIdentifier(name), value));\n }\n };\n\n upsertAttribute('lang', b.jsxExpressionContainer(b.identifier('locale')));\n upsertAttribute(\n 'dir',\n b.jsxExpressionContainer(\n b.callExpression(b.identifier('getHTMLTextDir'), [\n b.identifier('locale'),\n ])\n )\n );\n return false;\n },\n });\n};\n\n/**\n * Wraps the `{children}` of a TanStack Start root document with\n * `IntlayerProvider`, deriving the locale from the `{-$locale}` route param.\n * Safe and idempotent: bails (returns the original code) when the provider is\n * already present, when no `<html>` document function is found, or when there is\n * no `{children}` placeholder to wrap.\n */\nexport const wrapRootWithProvider = (code: string): TransformResult => {\n const ast = parseTsx(code);\n\n if (code.includes('IntlayerProvider')) return { code, status: 'already' };\n\n const funcNode = findHtmlDocumentFunction(ast);\n if (!funcNode) return { code, status: 'skipped' };\n\n let wrapped = false;\n recast.visit(funcNode, {\n visitJSXExpressionContainer(path) {\n if (wrapped) return false;\n const expression = path.node.expression;\n if (expression?.type === 'Identifier' && expression.name === 'children') {\n const template = parseTsx(\n 'const __wrap = <IntlayerProvider locale={locale}>{__child__}</IntlayerProvider>;'\n );\n const providerElement = template.program.body[0].declarations[0].init;\n providerElement.children = [path.node];\n path.replace(providerElement);\n wrapped = true;\n return false;\n }\n this.traverse(path);\n },\n });\n\n if (!wrapped) return { code, status: 'skipped' };\n\n ensureNamedImport(ast, 'IntlayerProvider', 'react-intlayer');\n ensureNamedImport(ast, 'useParams', '@tanstack/react-router');\n ensureNamedImport(ast, 'defaultLocale', 'intlayer');\n ensureNamedImport(ast, 'getHTMLTextDir', 'intlayer');\n ensureLocaleFromParams(funcNode);\n setHtmlLangAndDir(ast);\n\n return { code: recast.print(ast).code, status: 'wrapped' };\n};\n"],"mappings":";;;;;AAIA,MAAM,EAAE,UAAU,GAAG,YAAY,MAAM,OAAO;;AAG9C,MAAM,YAAY,SAChB,OAAO,MAAM,MAAM,EAAE,QAAQ,cAAc,CAAC;;;;;AAY9C,MAAM,4BAA4B,QAAkB;CAClD,MAAM,wBAAQ,IAAI,IAAS;CAE3B,OAAO,MAAM,KAAK,EAChB,uBAAuB,MAAM;EAC3B,MAAM,OAAO,KAAK;EAClB,IAAI,KAAK,MAAM,SAAS,mBAAmB,KAAK,KAAK,SAAS,QAAQ;GAEpE,IAAI,UAAU,KAAK;GACnB,OAAO,SAAS;IACd,MAAM,QAAQ,QAAQ;IACtB,IACE,UACC,EAAE,oBAAoB,MAAM,KAAK,KAChC,EAAE,mBAAmB,MAAM,KAAK,KAChC,EAAE,wBAAwB,MAAM,KAAK,IACvC;KACA,MAAM,IAAI,KAAK;KACf;IACF;IACA,UAAU,QAAQ;GACpB;EACF;EACA,KAAK,SAAS,IAAI;CACpB,EACF,CAAC;CAED,OAAO,MAAM,SAAS,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK;AAC5C;;;;;;;;AASA,MAAM,oBAAoB,MAAW,SAA0B;CAC7D,IAAI,CAAC,MAAM,OAAO;CAClB,QAAQ,KAAK,MAAb;EACE,KAAK,cACH,OAAO,KAAK,SAAS;EACvB,KAAK,qBAEH,OAAO,iBAAiB,KAAK,MAAM,IAAI;EACzC,KAAK,eAEH,OAAO,iBAAiB,KAAK,UAAU,IAAI;EAC7C,KAAK,gBACH,OAAO,KAAK,SAAS,MAAM,YACzB,iBAAiB,SAAS,IAAI,CAChC;EACF,KAAK,iBACH,OAAO,KAAK,WAAW,MAAM,aAC3B,SAAS,SAAS,gBACd,iBAAiB,SAAS,UAAU,IAAI,IAExC,iBAAiB,SAAS,OAAO,IAAI,CAC3C;EACF,SACE,OAAO;CACX;AACF;;;;;;;;;;;;;;;AAgBA,MAAM,0BAA0B,aAAwB;CACtD,IAAI,SAAS,MAAM,SAAS,kBAC1B,SAAS,OAAO,EAAE,eAAe,CAAC,EAAE,gBAAgB,SAAS,IAAI,CAAC,CAAC;CAGrE,MAAM,iBAAiB,SAAS,KAAK,KAAK,MACvC,SACC,KAAK,SAAS,yBACd,KAAK,aAAa,MAAM,MAAW,iBAAiB,EAAE,IAAI,QAAQ,CAAC,CACvE;CACA,MAAM,oBAAoB,SAAS,UAAU,CAAC,EAAC,CAAE,MAAM,UACrD,iBAAiB,OAAO,QAAQ,CAClC;CACA,IAAI,kBAAkB,kBAAkB;CAExC,MAAM,YAAY,SAChB,sEACF,CAAC,CAAC,QAAQ,KAAK;CACf,SAAS,KAAK,KAAK,QAAQ,SAAS;AACtC;;AAGA,MAAM,qBAAqB,QAAmB;CAC5C,OAAO,MAAM,KAAK,EAChB,uBAAuB,MAAM;EAC3B,MAAM,OAAO,KAAK;EAClB,IAAI,KAAK,MAAM,SAAS,mBAAmB,KAAK,KAAK,SAAS,QAAQ;GACpE,KAAK,SAAS,IAAI;GAClB;EACF;EAEA,MAAM,mBAAmB,MAAc,UAAqB;GAC1D,MAAM,WAAW,KAAK,YAAY,MAC/B,SACC,KAAK,SAAS,kBAAkB,KAAK,MAAM,SAAS,IACxD;GACA,IAAI,UACF,SAAS,QAAQ;QAEjB,KAAK,WAAW,KAAK,EAAE,aAAa,EAAE,cAAc,IAAI,GAAG,KAAK,CAAC;EAErE;EAEA,gBAAgB,QAAQ,EAAE,uBAAuB,EAAE,WAAW,QAAQ,CAAC,CAAC;EACxE,gBACE,OACA,EAAE,uBACA,EAAE,eAAe,EAAE,WAAW,gBAAgB,GAAG,CAC/C,EAAE,WAAW,QAAQ,CACvB,CAAC,CACH,CACF;EACA,OAAO;CACT,EACF,CAAC;AACH;;;;;;;;AASA,MAAa,wBAAwB,SAAkC;CACrE,MAAM,MAAM,SAAS,IAAI;CAEzB,IAAI,KAAK,SAAS,kBAAkB,GAAG,OAAO;EAAE;EAAM,QAAQ;CAAU;CAExE,MAAM,WAAW,yBAAyB,GAAG;CAC7C,IAAI,CAAC,UAAU,OAAO;EAAE;EAAM,QAAQ;CAAU;CAEhD,IAAI,UAAU;CACd,OAAO,MAAM,UAAU,EACrB,4BAA4B,MAAM;EAChC,IAAI,SAAS,OAAO;EACpB,MAAM,aAAa,KAAK,KAAK;EAC7B,IAAI,YAAY,SAAS,gBAAgB,WAAW,SAAS,YAAY;GAIvE,MAAM,kBAHW,SACf,kFAE6B,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,aAAa,EAAE,CAAC;GACjE,gBAAgB,WAAW,CAAC,KAAK,IAAI;GACrC,KAAK,QAAQ,eAAe;GAC5B,UAAU;GACV,OAAO;EACT;EACA,KAAK,SAAS,IAAI;CACpB,EACF,CAAC;CAED,IAAI,CAAC,SAAS,OAAO;EAAE;EAAM,QAAQ;CAAU;CAE/C,kBAAkB,KAAK,oBAAoB,gBAAgB;CAC3D,kBAAkB,KAAK,aAAa,wBAAwB;CAC5D,kBAAkB,KAAK,iBAAiB,UAAU;CAClD,kBAAkB,KAAK,kBAAkB,UAAU;CACnD,uBAAuB,QAAQ;CAC/B,kBAAkB,GAAG;CAErB,OAAO;EAAE,MAAM,OAAO,MAAM,GAAG,CAAC,CAAC;EAAM,QAAQ;CAAU;AAC3D"}
|
|
1
|
+
{"version":3,"file":"transforms.mjs","names":[],"sources":["../../../../../src/init/frameworkSetup/tanstackStart/transforms.ts"],"sourcesContent":["import * as recast from 'recast';\nimport { babelTsParser } from '../../../utils/babelParser';\nimport { ensureNamedImport } from '../../utils/astImports';\n\nconst { builders: b, namedTypes: n } = recast.types;\n\n/** babel-ts parser handles TypeScript *and* JSX. */\nconst parseTsx = (code: string): any =>\n recast.parse(code, { parser: babelTsParser });\n\n/** Result of a source transform. `code` is unchanged for any non-`wrapped` status. */\nexport type TransformResult = {\n code: string;\n status: 'wrapped' | 'already' | 'skipped';\n};\n\n/**\n * Finds the single function that returns the `<html>` document element. Returns\n * `null` when there is no such function, or more than one (ambiguous).\n */\nconst findHtmlDocumentFunction = (ast: any): any => {\n const found = new Set<any>();\n\n recast.visit(ast, {\n visitJSXOpeningElement(path) {\n const node = path.node;\n if (node.name?.type === 'JSXIdentifier' && node.name.name === 'html') {\n // Walk up to the nearest enclosing function node.\n let current = path.parentPath;\n while (current) {\n const value = current.value;\n if (\n value &&\n (n.FunctionDeclaration.check(value) ||\n n.FunctionExpression.check(value) ||\n n.ArrowFunctionExpression.check(value))\n ) {\n found.add(value);\n break;\n }\n current = current.parentPath;\n }\n }\n this.traverse(path);\n },\n });\n\n return found.size === 1 ? [...found][0] : null;\n};\n\n/**\n * Whether the binding pattern `node` introduces a variable named `name`.\n * Recurses through destructuring patterns so all of `const locale`,\n * `const { locale } = …`, `const { locale = fallback } = …`,\n * `const { foo: locale } = …`, `const [locale] = …` and `...locale` are\n * detected — not just a bare identifier.\n */\nconst patternBindsName = (node: any, name: string): boolean => {\n if (!node) return false;\n switch (node.type) {\n case 'Identifier':\n return node.name === name;\n case 'AssignmentPattern':\n // `locale = fallback`\n return patternBindsName(node.left, name);\n case 'RestElement':\n // `...locale`\n return patternBindsName(node.argument, name);\n case 'ArrayPattern':\n return node.elements.some((element: any) =>\n patternBindsName(element, name)\n );\n case 'ObjectPattern':\n return node.properties.some((property: any) =>\n property.type === 'RestElement'\n ? patternBindsName(property.argument, name)\n : // The *value* is the binding target (`{ foo: locale }` binds `locale`).\n patternBindsName(property.value, name)\n );\n default:\n return false;\n }\n};\n\n/**\n * Inserts `const locale = …` once, at the top of the function body. A single\n * self-contained statement is used (rather than an intermediate `params` const)\n * so it can never collide with a `params` variable the document already\n * declares. Expression-bodied arrow functions are first converted to a block so\n * the declaration has somewhere to live.\n *\n * No-ops when `locale` is already bound in the function — whether as a simple\n * identifier, through a destructuring pattern (e.g.\n * `const { locale = defaultLocale } = useLoaderData()`), or as a function\n * parameter — so the injected declaration never collides with one the document\n * already provides (which would produce an\n * `Identifier 'locale' has already been declared` parse error).\n */\nconst ensureLocaleFromParams = (funcNode: any): void => {\n if (funcNode.body?.type !== 'BlockStatement') {\n funcNode.body = b.blockStatement([b.returnStatement(funcNode.body)]);\n }\n\n const declaredInBody = funcNode.body.body.some(\n (stmt: any) =>\n stmt.type === 'VariableDeclaration' &&\n stmt.declarations.some((d: any) => patternBindsName(d.id, 'locale'))\n );\n const declaredInParams = (funcNode.params ?? []).some((param: any) =>\n patternBindsName(param, 'locale')\n );\n if (declaredInBody || declaredInParams) return;\n\n const statement = parseTsx(\n 'const locale = useParams({ strict: false }).locale ?? defaultLocale;'\n ).program.body[0];\n funcNode.body.body.unshift(statement);\n};\n\n/** Sets `lang={locale}` and `dir={getHTMLTextDir(locale)}` on the first `<html>` element. */\nconst setHtmlLangAndDir = (ast: any): void => {\n recast.visit(ast, {\n visitJSXOpeningElement(path) {\n const node = path.node;\n if (node.name?.type !== 'JSXIdentifier' || node.name.name !== 'html') {\n this.traverse(path);\n return;\n }\n\n const upsertAttribute = (name: string, value: any): void => {\n const existing = node.attributes?.find(\n (attr: any) =>\n attr.type === 'JSXAttribute' && attr.name?.name === name\n ) as any;\n if (existing) {\n existing.value = value;\n } else {\n node.attributes?.push(b.jsxAttribute(b.jsxIdentifier(name), value));\n }\n };\n\n upsertAttribute('lang', b.jsxExpressionContainer(b.identifier('locale')));\n upsertAttribute(\n 'dir',\n b.jsxExpressionContainer(\n b.callExpression(b.identifier('getHTMLTextDir'), [\n b.identifier('locale'),\n ])\n )\n );\n return false;\n },\n });\n};\n\n/**\n * Wraps the `{children}` of a TanStack Start root document with\n * `IntlayerProvider`, deriving the locale from the `{-$locale}` route param.\n * Safe and idempotent: bails (returns the original code) when the provider is\n * already present, when no `<html>` document function is found, or when there is\n * no `{children}` placeholder to wrap.\n */\nexport const wrapRootWithProvider = (code: string): TransformResult => {\n const ast = parseTsx(code);\n\n if (code.includes('IntlayerProvider')) return { code, status: 'already' };\n\n const funcNode = findHtmlDocumentFunction(ast);\n if (!funcNode) return { code, status: 'skipped' };\n\n let wrapped = false;\n recast.visit(funcNode, {\n visitJSXExpressionContainer(path) {\n if (wrapped) return false;\n const expression = path.node.expression;\n if (expression?.type === 'Identifier' && expression.name === 'children') {\n const template = parseTsx(\n 'const __wrap = <IntlayerProvider locale={locale}>{__child__}</IntlayerProvider>;'\n );\n const providerElement = template.program.body[0].declarations[0].init;\n providerElement.children = [path.node];\n path.replace(providerElement);\n wrapped = true;\n return false;\n }\n this.traverse(path);\n },\n });\n\n if (!wrapped) return { code, status: 'skipped' };\n\n ensureNamedImport(ast, 'IntlayerProvider', 'react-intlayer');\n ensureNamedImport(ast, 'useParams', '@tanstack/react-router');\n ensureNamedImport(ast, 'defaultLocale', 'intlayer');\n ensureNamedImport(ast, 'getHTMLTextDir', 'intlayer');\n ensureLocaleFromParams(funcNode);\n setHtmlLangAndDir(ast);\n\n return { code: recast.print(ast).code, status: 'wrapped' };\n};\n"],"mappings":";;;;;AAIA,MAAM,EAAE,UAAU,GAAG,YAAY,MAAM,OAAO;;AAG9C,MAAM,YAAY,SAChB,OAAO,MAAM,MAAM,EAAE,QAAQ,cAAc,CAAC;;;;;AAY9C,MAAM,4BAA4B,QAAkB;CAClD,MAAM,wBAAQ,IAAI,IAAS;CAE3B,OAAO,MAAM,KAAK,EAChB,uBAAuB,MAAM;EAC3B,MAAM,OAAO,KAAK;EAClB,IAAI,KAAK,MAAM,SAAS,mBAAmB,KAAK,KAAK,SAAS,QAAQ;GAEpE,IAAI,UAAU,KAAK;GACnB,OAAO,SAAS;IACd,MAAM,QAAQ,QAAQ;IACtB,IACE,UACC,EAAE,oBAAoB,MAAM,KAAK,KAChC,EAAE,mBAAmB,MAAM,KAAK,KAChC,EAAE,wBAAwB,MAAM,KAAK,IACvC;KACA,MAAM,IAAI,KAAK;KACf;IACF;IACA,UAAU,QAAQ;GACpB;EACF;EACA,KAAK,SAAS,IAAI;CACpB,EACF,CAAC;CAED,OAAO,MAAM,SAAS,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK;AAC5C;;;;;;;;AASA,MAAM,oBAAoB,MAAW,SAA0B;CAC7D,IAAI,CAAC,MAAM,OAAO;CAClB,QAAQ,KAAK,MAAb;EACE,KAAK,cACH,OAAO,KAAK,SAAS;EACvB,KAAK,qBAEH,OAAO,iBAAiB,KAAK,MAAM,IAAI;EACzC,KAAK,eAEH,OAAO,iBAAiB,KAAK,UAAU,IAAI;EAC7C,KAAK,gBACH,OAAO,KAAK,SAAS,MAAM,YACzB,iBAAiB,SAAS,IAAI,CAChC;EACF,KAAK,iBACH,OAAO,KAAK,WAAW,MAAM,aAC3B,SAAS,SAAS,gBACd,iBAAiB,SAAS,UAAU,IAAI,IAExC,iBAAiB,SAAS,OAAO,IAAI,CAC3C;EACF,SACE,OAAO;CACX;AACF;;;;;;;;;;;;;;;AAgBA,MAAM,0BAA0B,aAAwB;CACtD,IAAI,SAAS,MAAM,SAAS,kBAC1B,SAAS,OAAO,EAAE,eAAe,CAAC,EAAE,gBAAgB,SAAS,IAAI,CAAC,CAAC;CAGrE,MAAM,iBAAiB,SAAS,KAAK,KAAK,MACvC,SACC,KAAK,SAAS,yBACd,KAAK,aAAa,MAAM,MAAW,iBAAiB,EAAE,IAAI,QAAQ,CAAC,CACvE;CACA,MAAM,oBAAoB,SAAS,UAAU,CAAC,EAAC,CAAE,MAAM,UACrD,iBAAiB,OAAO,QAAQ,CAClC;CACA,IAAI,kBAAkB,kBAAkB;CAExC,MAAM,YAAY,SAChB,sEACF,CAAC,CAAC,QAAQ,KAAK;CACf,SAAS,KAAK,KAAK,QAAQ,SAAS;AACtC;;AAGA,MAAM,qBAAqB,QAAmB;CAC5C,OAAO,MAAM,KAAK,EAChB,uBAAuB,MAAM;EAC3B,MAAM,OAAO,KAAK;EAClB,IAAI,KAAK,MAAM,SAAS,mBAAmB,KAAK,KAAK,SAAS,QAAQ;GACpE,KAAK,SAAS,IAAI;GAClB;EACF;EAEA,MAAM,mBAAmB,MAAc,UAAqB;GAC1D,MAAM,WAAW,KAAK,YAAY,MAC/B,SACC,KAAK,SAAS,kBAAkB,KAAK,MAAM,SAAS,IACxD;GACA,IAAI,UACF,SAAS,QAAQ;QAEjB,KAAK,YAAY,KAAK,EAAE,aAAa,EAAE,cAAc,IAAI,GAAG,KAAK,CAAC;EAEtE;EAEA,gBAAgB,QAAQ,EAAE,uBAAuB,EAAE,WAAW,QAAQ,CAAC,CAAC;EACxE,gBACE,OACA,EAAE,uBACA,EAAE,eAAe,EAAE,WAAW,gBAAgB,GAAG,CAC/C,EAAE,WAAW,QAAQ,CACvB,CAAC,CACH,CACF;EACA,OAAO;CACT,EACF,CAAC;AACH;;;;;;;;AASA,MAAa,wBAAwB,SAAkC;CACrE,MAAM,MAAM,SAAS,IAAI;CAEzB,IAAI,KAAK,SAAS,kBAAkB,GAAG,OAAO;EAAE;EAAM,QAAQ;CAAU;CAExE,MAAM,WAAW,yBAAyB,GAAG;CAC7C,IAAI,CAAC,UAAU,OAAO;EAAE;EAAM,QAAQ;CAAU;CAEhD,IAAI,UAAU;CACd,OAAO,MAAM,UAAU,EACrB,4BAA4B,MAAM;EAChC,IAAI,SAAS,OAAO;EACpB,MAAM,aAAa,KAAK,KAAK;EAC7B,IAAI,YAAY,SAAS,gBAAgB,WAAW,SAAS,YAAY;GAIvE,MAAM,kBAHW,SACf,kFAE6B,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,aAAa,EAAE,CAAC;GACjE,gBAAgB,WAAW,CAAC,KAAK,IAAI;GACrC,KAAK,QAAQ,eAAe;GAC5B,UAAU;GACV,OAAO;EACT;EACA,KAAK,SAAS,IAAI;CACpB,EACF,CAAC;CAED,IAAI,CAAC,SAAS,OAAO;EAAE;EAAM,QAAQ;CAAU;CAE/C,kBAAkB,KAAK,oBAAoB,gBAAgB;CAC3D,kBAAkB,KAAK,aAAa,wBAAwB;CAC5D,kBAAkB,KAAK,iBAAiB,UAAU;CAClD,kBAAkB,KAAK,kBAAkB,UAAU;CACnD,uBAAuB,QAAQ;CAC/B,kBAAkB,GAAG;CAErB,OAAO;EAAE,MAAM,OAAO,MAAM,GAAG,CAAC,CAAC;EAAM,QAAQ;CAAU;AAC3D"}
|
|
@@ -78,11 +78,13 @@ const loadDictionaries = async (contentDeclarationsPaths, configuration) => {
|
|
|
78
78
|
const { plugins } = configuration;
|
|
79
79
|
const loadDictionariesStartTime = Date.now();
|
|
80
80
|
getAppLogger(configuration)("Dictionaries:", { isVerbose: true });
|
|
81
|
-
const
|
|
81
|
+
const pluginsWithLoadDictionaries = (await Promise.all(plugins ?? [])).filter((plugin) => plugin.loadDictionaries);
|
|
82
|
+
logger$1.setPluginNames(pluginsWithLoadDictionaries.map(({ name }) => name));
|
|
83
|
+
const loadPluginDictionariesPromise = pluginsWithLoadDictionaries.map(async (plugin) => {
|
|
82
84
|
try {
|
|
83
85
|
return await plugin.loadDictionaries?.({ configuration }) ?? [];
|
|
84
86
|
} catch (error) {
|
|
85
|
-
logger$1.setPluginError(error);
|
|
87
|
+
logger$1.setPluginError(error, plugin.name);
|
|
86
88
|
return [];
|
|
87
89
|
}
|
|
88
90
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loadDictionaries.mjs","names":["logger"],"sources":["../../../src/loadDictionaries/loadDictionaries.ts"],"sourcesContent":["import * as ANSIColors from '@intlayer/config/colors';\nimport {\n colon,\n colorize,\n colorizeKey,\n getAppLogger,\n} from '@intlayer/config/logger';\nimport type { IntlayerConfig } from '@intlayer/types/config';\nimport type { Dictionary } from '@intlayer/types/dictionary';\nimport { filterInvalidDictionaries } from '../filterInvalidDictionaries';\nimport { formatDictionaries } from '../formatDictionary';\nimport { loadContentDeclarations } from './loadContentDeclaration';\nimport { loadRemoteDictionaries } from './loadRemoteDictionaries';\nimport { DictionariesLogger } from './log';\n\nexport type DictionariesStatus = {\n dictionaryKey: string;\n type: 'local' | 'remote';\n status:\n | 'pending' // Key found but not fetched yet\n | 'fetching' // If dictionary fetch is in progress\n | 'fetched' // If dictionary fetch succeeded\n | 'error' // If dictionary fetch failed\n | 'imported' // If dictionary already fetched and still up to date\n | 'found' // If dictionary key is found but promise is not resolved yet (ex: fetching distant content)\n | 'building' // If dictionary is being built\n | 'built'; // If dictionary is built;\n error?: string;\n};\n\nlet loadDictionariesStatus: DictionariesStatus[] = [];\nconst logger = new DictionariesLogger();\n\nconst setLoadDictionariesStatus = (statuses: DictionariesStatus[]) => {\n const updated: DictionariesStatus[] = [...loadDictionariesStatus];\n\n for (const incoming of statuses) {\n const index = updated.findIndex(\n (s) =>\n s.dictionaryKey === incoming.dictionaryKey && s.type === incoming.type\n );\n if (index >= 0) {\n updated[index] = incoming;\n } else {\n updated.push(incoming);\n }\n }\n\n loadDictionariesStatus = updated;\n logger.update(statuses);\n\n return updated;\n};\n\ntype StatusRecord = {\n local?: DictionariesStatus['status'];\n remote?: DictionariesStatus['status'];\n};\n\nconst iconFor = (status: DictionariesStatus['status']) => {\n switch (status) {\n case 'built':\n case 'imported':\n case 'fetched':\n return '✔';\n case 'error':\n return '✖';\n default:\n return '⏲';\n }\n};\n\nconst colorFor = (status: DictionariesStatus['status']) => {\n switch (status) {\n case 'built':\n case 'imported':\n case 'fetched':\n return ANSIColors.GREEN;\n case 'error':\n return ANSIColors.RED;\n default:\n return ANSIColors.BLUE;\n }\n};\n\nconst printSummary = (configuration: IntlayerConfig) => {\n if (configuration.log.mode !== 'verbose') return;\n\n const appLogger = getAppLogger(configuration);\n\n // Aggregate by dictionary key\n const byKey = new Map<string, StatusRecord>();\n for (const status of loadDictionariesStatus) {\n const rec = byKey.get(status.dictionaryKey) ?? {};\n if (status.type === 'local') rec.local = status.status;\n if (status.type === 'remote') rec.remote = status.status;\n byKey.set(status.dictionaryKey, rec);\n }\n\n const keys = Array.from(byKey.keys()).sort((a, b) => a.localeCompare(b));\n\n // Compute the max visible length of the local label to align distant labels\n let maxLocalLabelLen = 0;\n for (const key of keys) {\n const rec = byKey.get(key)!;\n if (rec.local) {\n const visibleLocal = `[local: ${iconFor(rec.local)} ${rec.local}]`;\n if (visibleLocal.length > maxLocalLabelLen) {\n maxLocalLabelLen = visibleLocal.length;\n }\n }\n }\n\n for (const key of keys) {\n const rec = byKey.get(key)!;\n const labels: string[] = [];\n\n if (rec.local) {\n const inner = colorize(\n `${iconFor(rec.local)} ${rec.local}`,\n colorFor(rec.local)\n );\n const coloredLocal =\n `${ANSIColors.GREY}[` +\n colorize('local: ', ANSIColors.GREY) +\n inner +\n `${ANSIColors.GREY}]${ANSIColors.RESET}`;\n\n // Pad to align distant label across rows\n const visibleLocal = `[local: ${iconFor(rec.local)} ${rec.local}]`;\n const pad = Math.max(0, maxLocalLabelLen - visibleLocal.length);\n labels.push(coloredLocal + ' '.repeat(pad));\n } else {\n // If no local label, insert spaces to keep distant aligned\n labels.push(' '.repeat(maxLocalLabelLen));\n }\n\n if (rec.remote) {\n const inner = colorize(\n `${iconFor(rec.remote)} ${rec.remote}`,\n colorFor(rec.remote)\n );\n labels.push(\n `${ANSIColors.GREY}[` +\n colorize('distant: ', ANSIColors.GREY) +\n inner +\n `${ANSIColors.GREY}]${ANSIColors.RESET}`\n );\n }\n\n appLogger(\n ` - ${colon(colorizeKey(key), { colSize: keys })} ${labels.join(' ')}`\n );\n }\n};\n\nexport const loadDictionaries = async (\n contentDeclarationsPaths: string[] | string,\n configuration: IntlayerConfig\n): Promise<{\n localDictionaries: Dictionary[];\n remoteDictionaries: Dictionary[];\n pluginDictionaries: Dictionary[];\n time: {\n localDictionaries: number;\n remoteDictionaries: number;\n pluginDictionaries: number;\n };\n}> => {\n const { plugins } = configuration;\n const loadDictionariesStartTime = Date.now();\n const appLogger = getAppLogger(configuration);\n\n appLogger('Dictionaries:', { isVerbose: true });\n\n // Load additional dictionaries via plugins (e.g., ICU JSON ingestion)\n const resolvedPlugins = await Promise.all(plugins ?? []);\n\n const pluginsWithLoadDictionaries = resolvedPlugins.filter(\n (plugin) => plugin.loadDictionaries\n );\n\n const loadPluginDictionariesPromise = pluginsWithLoadDictionaries.map(\n async (plugin) => {\n try {\n const res = await plugin.loadDictionaries?.({\n configuration,\n });\n\n return (res as Dictionary[] | undefined) ?? [];\n } catch (error) {\n logger.setPluginError(error as Error);\n\n return [];\n }\n }\n );\n\n const pluginDictionaries: Dictionary[] = await Promise.all(\n loadPluginDictionariesPromise as Promise<Dictionary[]>[]\n )\n .then((dictionaries) => dictionaries.flat())\n .then((dictionaries) =>\n filterInvalidDictionaries(dictionaries, configuration)\n )\n .then((dictionaries) => formatDictionaries(dictionaries));\n\n logger.setPluginTotal(pluginDictionaries.length);\n logger.setPluginDone(pluginDictionaries.length);\n\n const pluginDictionariesTime = Date.now();\n\n const files = Array.isArray(contentDeclarationsPaths)\n ? contentDeclarationsPaths\n : [contentDeclarationsPaths];\n\n const localDictionaries: Dictionary[] = await loadContentDeclarations(\n files,\n configuration,\n setLoadDictionariesStatus\n )\n .then((dictionaries) =>\n filterInvalidDictionaries(dictionaries, configuration)\n )\n .then((dictionaries) => formatDictionaries(dictionaries));\n\n const localDictionariesTime = Date.now();\n\n const localDictionariesStatus = localDictionaries.map(\n (dictionary) =>\n ({\n dictionaryKey: dictionary.key,\n type: 'local',\n status: 'built',\n }) as const\n );\n\n setLoadDictionariesStatus(localDictionariesStatus);\n\n const hasRemoteDictionaries = Boolean(\n configuration.editor.clientId && configuration.editor.clientSecret\n );\n\n if (hasRemoteDictionaries) {\n // We expect to fetch remote dictionaries soon; suppress a transient local-only render\n logger.setExpectRemote(true);\n }\n\n let remoteDictionaries: Dictionary[] = [];\n\n if (hasRemoteDictionaries) {\n remoteDictionaries = await loadRemoteDictionaries(\n configuration,\n setLoadDictionariesStatus,\n {\n onStartRemoteCheck: () => logger.startRemoteCheck(),\n onStopRemoteCheck: () => logger.stopRemoteCheck(),\n onError: (e) => logger.setRemoteError(e),\n }\n )\n .then((dictionaries) =>\n filterInvalidDictionaries(dictionaries, configuration)\n )\n .then((dictionaries) => formatDictionaries(dictionaries));\n }\n\n const remoteDictionariesTime = Date.now();\n\n // Stop spinner and show final progress line(s)\n logger.finish();\n\n printSummary(configuration);\n\n return {\n localDictionaries,\n remoteDictionaries,\n pluginDictionaries,\n time: {\n localDictionaries: localDictionariesTime - pluginDictionariesTime,\n remoteDictionaries: remoteDictionariesTime - localDictionariesTime,\n pluginDictionaries: pluginDictionariesTime - loadDictionariesStartTime,\n },\n };\n};\n"],"mappings":";;;;;;;;;AA8BA,IAAI,yBAA+C,CAAC;AACpD,MAAMA,WAAS,IAAI,mBAAmB;AAEtC,MAAM,6BAA6B,aAAmC;CACpE,MAAM,UAAgC,CAAC,GAAG,sBAAsB;CAEhE,KAAK,MAAM,YAAY,UAAU;EAC/B,MAAM,QAAQ,QAAQ,WACnB,MACC,EAAE,kBAAkB,SAAS,iBAAiB,EAAE,SAAS,SAAS,IACtE;EACA,IAAI,SAAS,GACX,QAAQ,SAAS;OAEjB,QAAQ,KAAK,QAAQ;CAEzB;CAEA,yBAAyB;CACzB,SAAO,OAAO,QAAQ;CAEtB,OAAO;AACT;AAOA,MAAM,WAAW,WAAyC;CACxD,QAAQ,QAAR;EACE,KAAK;EACL,KAAK;EACL,KAAK,WACH,OAAO;EACT,KAAK,SACH,OAAO;EACT,SACE,OAAO;CACX;AACF;AAEA,MAAM,YAAY,WAAyC;CACzD,QAAQ,QAAR;EACE,KAAK;EACL,KAAK;EACL,KAAK,WACH,OAAO,WAAW;EACpB,KAAK,SACH,OAAO,WAAW;EACpB,SACE,OAAO,WAAW;CACtB;AACF;AAEA,MAAM,gBAAgB,kBAAkC;CACtD,IAAI,cAAc,IAAI,SAAS,WAAW;CAE1C,MAAM,YAAY,aAAa,aAAa;CAG5C,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,UAAU,wBAAwB;EAC3C,MAAM,MAAM,MAAM,IAAI,OAAO,aAAa,KAAK,CAAC;EAChD,IAAI,OAAO,SAAS,SAAS,IAAI,QAAQ,OAAO;EAChD,IAAI,OAAO,SAAS,UAAU,IAAI,SAAS,OAAO;EAClD,MAAM,IAAI,OAAO,eAAe,GAAG;CACrC;CAEA,MAAM,OAAO,MAAM,KAAK,MAAM,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,cAAc,CAAC,CAAC;CAGvE,IAAI,mBAAmB;CACvB,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,MAAM,MAAM,IAAI,GAAG;EACzB,IAAI,IAAI,OAAO;GACb,MAAM,eAAe,WAAW,QAAQ,IAAI,KAAK,EAAE,GAAG,IAAI,MAAM;GAChE,IAAI,aAAa,SAAS,kBACxB,mBAAmB,aAAa;EAEpC;CACF;CAEA,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,MAAM,MAAM,IAAI,GAAG;EACzB,MAAM,SAAmB,CAAC;EAE1B,IAAI,IAAI,OAAO;GACb,MAAM,QAAQ,SACZ,GAAG,QAAQ,IAAI,KAAK,EAAE,GAAG,IAAI,SAC7B,SAAS,IAAI,KAAK,CACpB;GACA,MAAM,eACJ,GAAG,WAAW,KAAK,KACnB,SAAS,WAAW,WAAW,IAAI,IACnC,QACA,GAAG,WAAW,KAAK,GAAG,WAAW;GAGnC,MAAM,eAAe,WAAW,QAAQ,IAAI,KAAK,EAAE,GAAG,IAAI,MAAM;GAChE,MAAM,MAAM,KAAK,IAAI,GAAG,mBAAmB,aAAa,MAAM;GAC9D,OAAO,KAAK,eAAe,IAAI,OAAO,GAAG,CAAC;EAC5C,OAEE,OAAO,KAAK,IAAI,OAAO,gBAAgB,CAAC;EAG1C,IAAI,IAAI,QAAQ;GACd,MAAM,QAAQ,SACZ,GAAG,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI,UAC9B,SAAS,IAAI,MAAM,CACrB;GACA,OAAO,KACL,GAAG,WAAW,KAAK,KACjB,SAAS,aAAa,WAAW,IAAI,IACrC,QACA,GAAG,WAAW,KAAK,GAAG,WAAW,OACrC;EACF;EAEA,UACE,MAAM,MAAM,YAAY,GAAG,GAAG,EAAE,SAAS,KAAK,CAAC,EAAE,GAAG,OAAO,KAAK,GAAG,GACrE;CACF;AACF;AAEA,MAAa,mBAAmB,OAC9B,0BACA,kBAUI;CACJ,MAAM,EAAE,YAAY;CACpB,MAAM,4BAA4B,KAAK,IAAI;CAG3C,AAFkB,aAAa,aAEvB,CAAC,CAAC,iBAAiB,EAAE,WAAW,KAAK,CAAC;CAS9C,MAAM,iCAJ8B,MAFN,QAAQ,IAAI,WAAW,CAAC,CAAC,EAEJ,CAAC,QACjD,WAAW,OAAO,gBAG2C,CAAC,CAAC,IAChE,OAAO,WAAW;EAChB,IAAI;GAKF,OAAQ,MAJU,OAAO,mBAAmB,EAC1C,cACF,CAAC,KAE2C,CAAC;EAC/C,SAAS,OAAO;GACd,SAAO,eAAe,KAAc;GAEpC,OAAO,CAAC;EACV;CACF,CACF;CAEA,MAAM,qBAAmC,MAAM,QAAQ,IACrD,6BACF,CAAC,CACE,MAAM,iBAAiB,aAAa,KAAK,CAAC,CAAC,CAC3C,MAAM,iBACL,0BAA0B,cAAc,aAAa,CACvD,CAAC,CACA,MAAM,iBAAiB,mBAAmB,YAAY,CAAC;CAE1D,SAAO,eAAe,mBAAmB,MAAM;CAC/C,SAAO,cAAc,mBAAmB,MAAM;CAE9C,MAAM,yBAAyB,KAAK,IAAI;CAMxC,MAAM,oBAAkC,MAAM,wBAJhC,MAAM,QAAQ,wBAAwB,IAChD,2BACA,CAAC,wBAAwB,GAI3B,eACA,yBACF,CAAC,CACE,MAAM,iBACL,0BAA0B,cAAc,aAAa,CACvD,CAAC,CACA,MAAM,iBAAiB,mBAAmB,YAAY,CAAC;CAE1D,MAAM,wBAAwB,KAAK,IAAI;CAEvC,MAAM,0BAA0B,kBAAkB,KAC/C,gBACE;EACC,eAAe,WAAW;EAC1B,MAAM;EACN,QAAQ;CACV,EACJ;CAEA,0BAA0B,uBAAuB;CAEjD,MAAM,wBAAwB,QAC5B,cAAc,OAAO,YAAY,cAAc,OAAO,YACxD;CAEA,IAAI,uBAEF,SAAO,gBAAgB,IAAI;CAG7B,IAAI,qBAAmC,CAAC;CAExC,IAAI,uBACF,qBAAqB,MAAM,uBACzB,eACA,2BACA;EACE,0BAA0BA,SAAO,iBAAiB;EAClD,yBAAyBA,SAAO,gBAAgB;EAChD,UAAU,MAAMA,SAAO,eAAe,CAAC;CACzC,CACF,CAAC,CACE,MAAM,iBACL,0BAA0B,cAAc,aAAa,CACvD,CAAC,CACA,MAAM,iBAAiB,mBAAmB,YAAY,CAAC;CAG5D,MAAM,yBAAyB,KAAK,IAAI;CAGxC,SAAO,OAAO;CAEd,aAAa,aAAa;CAE1B,OAAO;EACL;EACA;EACA;EACA,MAAM;GACJ,mBAAmB,wBAAwB;GAC3C,oBAAoB,yBAAyB;GAC7C,oBAAoB,yBAAyB;EAC/C;CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"loadDictionaries.mjs","names":["logger"],"sources":["../../../src/loadDictionaries/loadDictionaries.ts"],"sourcesContent":["import * as ANSIColors from '@intlayer/config/colors';\nimport {\n colon,\n colorize,\n colorizeKey,\n getAppLogger,\n} from '@intlayer/config/logger';\nimport type { IntlayerConfig } from '@intlayer/types/config';\nimport type { Dictionary } from '@intlayer/types/dictionary';\nimport { filterInvalidDictionaries } from '../filterInvalidDictionaries';\nimport { formatDictionaries } from '../formatDictionary';\nimport { loadContentDeclarations } from './loadContentDeclaration';\nimport { loadRemoteDictionaries } from './loadRemoteDictionaries';\nimport { DictionariesLogger } from './log';\n\nexport type DictionariesStatus = {\n dictionaryKey: string;\n type: 'local' | 'remote';\n status:\n | 'pending' // Key found but not fetched yet\n | 'fetching' // If dictionary fetch is in progress\n | 'fetched' // If dictionary fetch succeeded\n | 'error' // If dictionary fetch failed\n | 'imported' // If dictionary already fetched and still up to date\n | 'found' // If dictionary key is found but promise is not resolved yet (ex: fetching distant content)\n | 'building' // If dictionary is being built\n | 'built'; // If dictionary is built;\n error?: string;\n};\n\nlet loadDictionariesStatus: DictionariesStatus[] = [];\nconst logger = new DictionariesLogger();\n\nconst setLoadDictionariesStatus = (statuses: DictionariesStatus[]) => {\n const updated: DictionariesStatus[] = [...loadDictionariesStatus];\n\n for (const incoming of statuses) {\n const index = updated.findIndex(\n (s) =>\n s.dictionaryKey === incoming.dictionaryKey && s.type === incoming.type\n );\n if (index >= 0) {\n updated[index] = incoming;\n } else {\n updated.push(incoming);\n }\n }\n\n loadDictionariesStatus = updated;\n logger.update(statuses);\n\n return updated;\n};\n\ntype StatusRecord = {\n local?: DictionariesStatus['status'];\n remote?: DictionariesStatus['status'];\n};\n\nconst iconFor = (status: DictionariesStatus['status']) => {\n switch (status) {\n case 'built':\n case 'imported':\n case 'fetched':\n return '✔';\n case 'error':\n return '✖';\n default:\n return '⏲';\n }\n};\n\nconst colorFor = (status: DictionariesStatus['status']) => {\n switch (status) {\n case 'built':\n case 'imported':\n case 'fetched':\n return ANSIColors.GREEN;\n case 'error':\n return ANSIColors.RED;\n default:\n return ANSIColors.BLUE;\n }\n};\n\nconst printSummary = (configuration: IntlayerConfig) => {\n if (configuration.log.mode !== 'verbose') return;\n\n const appLogger = getAppLogger(configuration);\n\n // Aggregate by dictionary key\n const byKey = new Map<string, StatusRecord>();\n for (const status of loadDictionariesStatus) {\n const rec = byKey.get(status.dictionaryKey) ?? {};\n if (status.type === 'local') rec.local = status.status;\n if (status.type === 'remote') rec.remote = status.status;\n byKey.set(status.dictionaryKey, rec);\n }\n\n const keys = Array.from(byKey.keys()).sort((a, b) => a.localeCompare(b));\n\n // Compute the max visible length of the local label to align distant labels\n let maxLocalLabelLen = 0;\n for (const key of keys) {\n const rec = byKey.get(key)!;\n if (rec.local) {\n const visibleLocal = `[local: ${iconFor(rec.local)} ${rec.local}]`;\n if (visibleLocal.length > maxLocalLabelLen) {\n maxLocalLabelLen = visibleLocal.length;\n }\n }\n }\n\n for (const key of keys) {\n const rec = byKey.get(key)!;\n const labels: string[] = [];\n\n if (rec.local) {\n const inner = colorize(\n `${iconFor(rec.local)} ${rec.local}`,\n colorFor(rec.local)\n );\n const coloredLocal =\n `${ANSIColors.GREY}[` +\n colorize('local: ', ANSIColors.GREY) +\n inner +\n `${ANSIColors.GREY}]${ANSIColors.RESET}`;\n\n // Pad to align distant label across rows\n const visibleLocal = `[local: ${iconFor(rec.local)} ${rec.local}]`;\n const pad = Math.max(0, maxLocalLabelLen - visibleLocal.length);\n labels.push(coloredLocal + ' '.repeat(pad));\n } else {\n // If no local label, insert spaces to keep distant aligned\n labels.push(' '.repeat(maxLocalLabelLen));\n }\n\n if (rec.remote) {\n const inner = colorize(\n `${iconFor(rec.remote)} ${rec.remote}`,\n colorFor(rec.remote)\n );\n labels.push(\n `${ANSIColors.GREY}[` +\n colorize('distant: ', ANSIColors.GREY) +\n inner +\n `${ANSIColors.GREY}]${ANSIColors.RESET}`\n );\n }\n\n appLogger(\n ` - ${colon(colorizeKey(key), { colSize: keys })} ${labels.join(' ')}`\n );\n }\n};\n\nexport const loadDictionaries = async (\n contentDeclarationsPaths: string[] | string,\n configuration: IntlayerConfig\n): Promise<{\n localDictionaries: Dictionary[];\n remoteDictionaries: Dictionary[];\n pluginDictionaries: Dictionary[];\n time: {\n localDictionaries: number;\n remoteDictionaries: number;\n pluginDictionaries: number;\n };\n}> => {\n const { plugins } = configuration;\n const loadDictionariesStartTime = Date.now();\n const appLogger = getAppLogger(configuration);\n\n appLogger('Dictionaries:', { isVerbose: true });\n\n // Load additional dictionaries via plugins (e.g., ICU JSON ingestion)\n const resolvedPlugins = await Promise.all(plugins ?? []);\n\n const pluginsWithLoadDictionaries = resolvedPlugins.filter(\n (plugin) => plugin.loadDictionaries\n );\n\n logger.setPluginNames(pluginsWithLoadDictionaries.map(({ name }) => name));\n\n const loadPluginDictionariesPromise = pluginsWithLoadDictionaries.map(\n async (plugin) => {\n try {\n const res = await plugin.loadDictionaries?.({\n configuration,\n });\n\n return (res as Dictionary[] | undefined) ?? [];\n } catch (error) {\n logger.setPluginError(error as Error, plugin.name);\n\n return [];\n }\n }\n );\n\n const pluginDictionaries: Dictionary[] = await Promise.all(\n loadPluginDictionariesPromise as Promise<Dictionary[]>[]\n )\n .then((dictionaries) => dictionaries.flat())\n .then((dictionaries) =>\n filterInvalidDictionaries(dictionaries, configuration)\n )\n .then((dictionaries) => formatDictionaries(dictionaries));\n\n logger.setPluginTotal(pluginDictionaries.length);\n logger.setPluginDone(pluginDictionaries.length);\n\n const pluginDictionariesTime = Date.now();\n\n const files = Array.isArray(contentDeclarationsPaths)\n ? contentDeclarationsPaths\n : [contentDeclarationsPaths];\n\n const localDictionaries: Dictionary[] = await loadContentDeclarations(\n files,\n configuration,\n setLoadDictionariesStatus\n )\n .then((dictionaries) =>\n filterInvalidDictionaries(dictionaries, configuration)\n )\n .then((dictionaries) => formatDictionaries(dictionaries));\n\n const localDictionariesTime = Date.now();\n\n const localDictionariesStatus = localDictionaries.map(\n (dictionary) =>\n ({\n dictionaryKey: dictionary.key,\n type: 'local',\n status: 'built',\n }) as const\n );\n\n setLoadDictionariesStatus(localDictionariesStatus);\n\n const hasRemoteDictionaries = Boolean(\n configuration.editor.clientId && configuration.editor.clientSecret\n );\n\n if (hasRemoteDictionaries) {\n // We expect to fetch remote dictionaries soon; suppress a transient local-only render\n logger.setExpectRemote(true);\n }\n\n let remoteDictionaries: Dictionary[] = [];\n\n if (hasRemoteDictionaries) {\n remoteDictionaries = await loadRemoteDictionaries(\n configuration,\n setLoadDictionariesStatus,\n {\n onStartRemoteCheck: () => logger.startRemoteCheck(),\n onStopRemoteCheck: () => logger.stopRemoteCheck(),\n onError: (e) => logger.setRemoteError(e),\n }\n )\n .then((dictionaries) =>\n filterInvalidDictionaries(dictionaries, configuration)\n )\n .then((dictionaries) => formatDictionaries(dictionaries));\n }\n\n const remoteDictionariesTime = Date.now();\n\n // Stop spinner and show final progress line(s)\n logger.finish();\n\n printSummary(configuration);\n\n return {\n localDictionaries,\n remoteDictionaries,\n pluginDictionaries,\n time: {\n localDictionaries: localDictionariesTime - pluginDictionariesTime,\n remoteDictionaries: remoteDictionariesTime - localDictionariesTime,\n pluginDictionaries: pluginDictionariesTime - loadDictionariesStartTime,\n },\n };\n};\n"],"mappings":";;;;;;;;;AA8BA,IAAI,yBAA+C,CAAC;AACpD,MAAMA,WAAS,IAAI,mBAAmB;AAEtC,MAAM,6BAA6B,aAAmC;CACpE,MAAM,UAAgC,CAAC,GAAG,sBAAsB;CAEhE,KAAK,MAAM,YAAY,UAAU;EAC/B,MAAM,QAAQ,QAAQ,WACnB,MACC,EAAE,kBAAkB,SAAS,iBAAiB,EAAE,SAAS,SAAS,IACtE;EACA,IAAI,SAAS,GACX,QAAQ,SAAS;OAEjB,QAAQ,KAAK,QAAQ;CAEzB;CAEA,yBAAyB;CACzB,SAAO,OAAO,QAAQ;CAEtB,OAAO;AACT;AAOA,MAAM,WAAW,WAAyC;CACxD,QAAQ,QAAR;EACE,KAAK;EACL,KAAK;EACL,KAAK,WACH,OAAO;EACT,KAAK,SACH,OAAO;EACT,SACE,OAAO;CACX;AACF;AAEA,MAAM,YAAY,WAAyC;CACzD,QAAQ,QAAR;EACE,KAAK;EACL,KAAK;EACL,KAAK,WACH,OAAO,WAAW;EACpB,KAAK,SACH,OAAO,WAAW;EACpB,SACE,OAAO,WAAW;CACtB;AACF;AAEA,MAAM,gBAAgB,kBAAkC;CACtD,IAAI,cAAc,IAAI,SAAS,WAAW;CAE1C,MAAM,YAAY,aAAa,aAAa;CAG5C,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,UAAU,wBAAwB;EAC3C,MAAM,MAAM,MAAM,IAAI,OAAO,aAAa,KAAK,CAAC;EAChD,IAAI,OAAO,SAAS,SAAS,IAAI,QAAQ,OAAO;EAChD,IAAI,OAAO,SAAS,UAAU,IAAI,SAAS,OAAO;EAClD,MAAM,IAAI,OAAO,eAAe,GAAG;CACrC;CAEA,MAAM,OAAO,MAAM,KAAK,MAAM,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,cAAc,CAAC,CAAC;CAGvE,IAAI,mBAAmB;CACvB,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,MAAM,MAAM,IAAI,GAAG;EACzB,IAAI,IAAI,OAAO;GACb,MAAM,eAAe,WAAW,QAAQ,IAAI,KAAK,EAAE,GAAG,IAAI,MAAM;GAChE,IAAI,aAAa,SAAS,kBACxB,mBAAmB,aAAa;EAEpC;CACF;CAEA,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,MAAM,MAAM,IAAI,GAAG;EACzB,MAAM,SAAmB,CAAC;EAE1B,IAAI,IAAI,OAAO;GACb,MAAM,QAAQ,SACZ,GAAG,QAAQ,IAAI,KAAK,EAAE,GAAG,IAAI,SAC7B,SAAS,IAAI,KAAK,CACpB;GACA,MAAM,eACJ,GAAG,WAAW,KAAK,KACnB,SAAS,WAAW,WAAW,IAAI,IACnC,QACA,GAAG,WAAW,KAAK,GAAG,WAAW;GAGnC,MAAM,eAAe,WAAW,QAAQ,IAAI,KAAK,EAAE,GAAG,IAAI,MAAM;GAChE,MAAM,MAAM,KAAK,IAAI,GAAG,mBAAmB,aAAa,MAAM;GAC9D,OAAO,KAAK,eAAe,IAAI,OAAO,GAAG,CAAC;EAC5C,OAEE,OAAO,KAAK,IAAI,OAAO,gBAAgB,CAAC;EAG1C,IAAI,IAAI,QAAQ;GACd,MAAM,QAAQ,SACZ,GAAG,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI,UAC9B,SAAS,IAAI,MAAM,CACrB;GACA,OAAO,KACL,GAAG,WAAW,KAAK,KACjB,SAAS,aAAa,WAAW,IAAI,IACrC,QACA,GAAG,WAAW,KAAK,GAAG,WAAW,OACrC;EACF;EAEA,UACE,MAAM,MAAM,YAAY,GAAG,GAAG,EAAE,SAAS,KAAK,CAAC,EAAE,GAAG,OAAO,KAAK,GAAG,GACrE;CACF;AACF;AAEA,MAAa,mBAAmB,OAC9B,0BACA,kBAUI;CACJ,MAAM,EAAE,YAAY;CACpB,MAAM,4BAA4B,KAAK,IAAI;CAG3C,AAFkB,aAAa,aAEvB,CAAC,CAAC,iBAAiB,EAAE,WAAW,KAAK,CAAC;CAK9C,MAAM,+BAA8B,MAFN,QAAQ,IAAI,WAAW,CAAC,CAAC,EAEJ,CAAC,QACjD,WAAW,OAAO,gBACrB;CAEA,SAAO,eAAe,4BAA4B,KAAK,EAAE,WAAW,IAAI,CAAC;CAEzE,MAAM,gCAAgC,4BAA4B,IAChE,OAAO,WAAW;EAChB,IAAI;GAKF,OAAQ,MAJU,OAAO,mBAAmB,EAC1C,cACF,CAAC,KAE2C,CAAC;EAC/C,SAAS,OAAO;GACd,SAAO,eAAe,OAAgB,OAAO,IAAI;GAEjD,OAAO,CAAC;EACV;CACF,CACF;CAEA,MAAM,qBAAmC,MAAM,QAAQ,IACrD,6BACF,CAAC,CACE,MAAM,iBAAiB,aAAa,KAAK,CAAC,CAAC,CAC3C,MAAM,iBACL,0BAA0B,cAAc,aAAa,CACvD,CAAC,CACA,MAAM,iBAAiB,mBAAmB,YAAY,CAAC;CAE1D,SAAO,eAAe,mBAAmB,MAAM;CAC/C,SAAO,cAAc,mBAAmB,MAAM;CAE9C,MAAM,yBAAyB,KAAK,IAAI;CAMxC,MAAM,oBAAkC,MAAM,wBAJhC,MAAM,QAAQ,wBAAwB,IAChD,2BACA,CAAC,wBAAwB,GAI3B,eACA,yBACF,CAAC,CACE,MAAM,iBACL,0BAA0B,cAAc,aAAa,CACvD,CAAC,CACA,MAAM,iBAAiB,mBAAmB,YAAY,CAAC;CAE1D,MAAM,wBAAwB,KAAK,IAAI;CAEvC,MAAM,0BAA0B,kBAAkB,KAC/C,gBACE;EACC,eAAe,WAAW;EAC1B,MAAM;EACN,QAAQ;CACV,EACJ;CAEA,0BAA0B,uBAAuB;CAEjD,MAAM,wBAAwB,QAC5B,cAAc,OAAO,YAAY,cAAc,OAAO,YACxD;CAEA,IAAI,uBAEF,SAAO,gBAAgB,IAAI;CAG7B,IAAI,qBAAmC,CAAC;CAExC,IAAI,uBACF,qBAAqB,MAAM,uBACzB,eACA,2BACA;EACE,0BAA0BA,SAAO,iBAAiB;EAClD,yBAAyBA,SAAO,gBAAgB;EAChD,UAAU,MAAMA,SAAO,eAAe,CAAC;CACzC,CACF,CAAC,CACE,MAAM,iBACL,0BAA0B,cAAc,aAAa,CACvD,CAAC,CACA,MAAM,iBAAiB,mBAAmB,YAAY,CAAC;CAG5D,MAAM,yBAAyB,KAAK,IAAI;CAGxC,SAAO,OAAO;CAEd,aAAa,aAAa;CAE1B,OAAO;EACL;EACA;EACA;EACA,MAAM;GACJ,mBAAmB,wBAAwB;GAC3C,oBAAoB,yBAAyB;GAC7C,oBAAoB,yBAAyB;EAC/C;CACF;AACF"}
|
|
@@ -39,7 +39,7 @@ const loadRemoteDictionaries = async (configuration = getConfiguration(), onStat
|
|
|
39
39
|
type: "remote",
|
|
40
40
|
status: "imported"
|
|
41
41
|
})));
|
|
42
|
-
const orderedDistantDictionaryKeys = dictionariesIdToFetch.map(([, data]) => data.key).sort(sortAlphabetically);
|
|
42
|
+
const orderedDistantDictionaryKeys = [...new Set(dictionariesIdToFetch.map(([, data]) => data.key))].sort(sortAlphabetically);
|
|
43
43
|
if (orderedDistantDictionaryKeys.length > 0) onStatusUpdate?.(orderedDistantDictionaryKeys.map((key) => ({
|
|
44
44
|
dictionaryKey: key,
|
|
45
45
|
type: "remote",
|
|
@@ -47,7 +47,8 @@ const loadRemoteDictionaries = async (configuration = getConfiguration(), onStat
|
|
|
47
47
|
})));
|
|
48
48
|
const distantDictionariesData = await fetchDistantDictionaries({ dictionaryKeys: orderedDistantDictionaryKeys }, onStatusUpdate);
|
|
49
49
|
const distantDictionaries = formatDistantDictionaries(distantDictionariesData);
|
|
50
|
-
|
|
50
|
+
const fetchedDictionaryIds = new Set(distantDictionaries.map((dictionary) => dictionary.id));
|
|
51
|
+
return [...cachedDictionaries.filter((dictionary) => !fetchedDictionaryIds.has(dictionary.id)), ...distantDictionaries];
|
|
51
52
|
} catch (error) {
|
|
52
53
|
options?.onError?.(error);
|
|
53
54
|
return [];
|