@intlayer/cli 7.0.7 → 7.0.8

package/dist/assets/translation-alignment/INTEGRATION_EXAMPLE.md

@@ -0,0 +1,682 @@
+# Integration Example
+
+This guide shows how to integrate the block-aware translation system into your existing workflow.
+
+## Step 1: Update Your Review Function
+
+Replace the existing `reviewDoc.ts` call to `reviewFile` with `reviewFileBlockAware` for locales that should use block-aware alignment.
+
+### Before
+
+```typescript
+// In reviewDoc.ts
+import { reviewFile } from './reviewDoc';
+
+await reviewFile(
+  absoluteBaseFilePath,
+  outputFilePath,
+  locale as Locale,
+  baseLocale,
+  aiOptions,
+  configOptions,
+  customInstructions,
+  changedLines
+);
+```
+
+### After
+
+```typescript
+// In reviewDoc.ts
+import { reviewFile } from './reviewDoc';
+import { reviewFileBlockAware } from './reviewDocBlockAware';
+
+// Use block-aware for markdown files
+const useBlockAware = absoluteBaseFilePath.endsWith('.md');
+
+if (useBlockAware) {
+  await reviewFileBlockAware(
+    absoluteBaseFilePath,
+    outputFilePath,
+    locale as Locale,
+    baseLocale,
+    aiOptions,
+    configOptions,
+    customInstructions,
+    changedLines
+  );
+} else {
+  // Fallback to original for non-markdown files
+  await reviewFile(
+    absoluteBaseFilePath,
+    outputFilePath,
+    locale as Locale,
+    baseLocale,
+    aiOptions,
+    configOptions,
+    customInstructions,
+    changedLines
+  );
+}
+```
+
+## Step 2: Modify reviewDoc.ts
+
+Here's the complete modified section of `reviewDoc.ts`:
+
+```typescript
+// Around line 330-386 in reviewDoc.ts
+const allTasks = docList.flatMap((docPath) =>
+  locales.map((locale) => async () => {
+    appLogger(
+      `Reviewing file: ${formatPath(docPath)} to ${formatLocale(locale)}`
+    );
+
+    const absoluteBaseFilePath = join(configuration.content.baseDir, docPath);
+    const outputFilePath = getOutputFilePath(
+      absoluteBaseFilePath,
+      locale,
+      baseLocale
+    );
+
+    // Skip if file exists and skipIfExists option is enabled
+    if (skipIfExists && existsSync(outputFilePath)) {
+      const relativePath = relative(
+        configuration.content.baseDir,
+        outputFilePath
+      );
+      appLogger(
+        `${colorize('⊘', ANSIColors.YELLOW)} File ${formatPath(relativePath)} already exists, skipping.`
+      );
+      return;
+    }
+
+    const fileModificationData = checkFileModifiedRange(outputFilePath, {
+      skipIfModifiedBefore,
+      skipIfModifiedAfter,
+    });
+
+    if (fileModificationData.isSkipped) {
+      appLogger(fileModificationData.message);
+      return;
+    }
+
+    let changedLines: number[] | undefined;
+    if (gitOptions) {
+      const gitChangedLines = await listGitLines(
+        absoluteBaseFilePath,
+        gitOptions
+      );
+
+      appLogger(`Git changed lines: ${gitChangedLines.join(', ')}`);
+      changedLines = gitChangedLines;
+    }
+
+    // NEW: Use block-aware translation for markdown files
+    const useBlockAware = absoluteBaseFilePath.endsWith('.md');
+    
+    if (useBlockAware) {
+      await reviewFileBlockAware(
+        absoluteBaseFilePath,
+        outputFilePath,
+        locale as Locale,
+        baseLocale,
+        aiOptions,
+        configOptions,
+        customInstructions,
+        changedLines
+      );
+    } else {
+      await reviewFile(
+        absoluteBaseFilePath,
+        outputFilePath,
+        locale as Locale,
+        baseLocale,
+        aiOptions,
+        configOptions,
+        customInstructions,
+        changedLines
+      );
+    }
+  })
+);
+```
+
+## Step 3: Add Import Statement
+
+At the top of `reviewDoc.ts`, add the import for the new function:
+
+```typescript
+// Around line 1-35 in reviewDoc.ts
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { dirname, join, relative } from 'node:path';
+import { readAsset } from 'utils:asset';
+import type { AIOptions } from '@intlayer/api';
+import {
+  formatLocale,
+  formatPath,
+  getChunk,
+  type ListGitFilesOptions,
+  listGitFiles,
+  listGitLines,
+  parallelize,
+} from '@intlayer/chokidar';
+// ... other imports ...
+
+// ADD THIS LINE:
+import { reviewFileBlockAware } from './reviewDocBlockAware';
+```
+
+## Step 4: Test with a Sample Document
+
+Create a test script to verify the integration:
+
+```typescript
+// test-block-aware.ts
+import { reviewFileBlockAware } from './reviewDocBlockAware';
+import { Locales } from '@intlayer/types';
+
+const testTranslation = async () => {
+  await reviewFileBlockAware(
+    'docs/en/test.md',
+    'docs/fr/test.md',
+    Locales.FRENCH,
+    Locales.ENGLISH,
+    {
+      // Your AI options
+      provider: 'openai',
+      // ... other options
+    },
+    {
+      // Your config options
+    },
+    'Translate technical documentation accurately',
+    [10, 15, 20] // Changed lines from git
+  );
+};
+
+testTranslation().catch(console.error);
+```
+
+## Step 5: Monitor Performance
+
+Add logging to track the efficiency of the block-aware system:
+
+```typescript
+// In reviewDocBlockAware.ts, after buildAlignmentPlan:
+
+const reusedCount = plan.actions.filter((a) => a.kind === "reuse").length;
+const reviewCount = plan.actions.filter((a) => a.kind === "review").length;
+const newCount = plan.actions.filter((a) => a.kind === "insert_new").length;
+const deleteCount = plan.actions.filter((a) => a.kind === "delete").length;
+const totalCount = plan.actions.length;
+
+const efficiencyRate = totalCount > 0 
+  ? ((reusedCount / totalCount) * 100).toFixed(1)
+  : '0.0';
+
+applicationLogger(
+  `${filePrefix}Efficiency: ${colorizeNumber(efficiencyRate)}% reused (${colorizeNumber(reusedCount)}/${colorizeNumber(totalCount)} blocks)`
+);
+```
+
+## Step 6: Configure for Your Locales
+
+You might want to enable block-aware translation only for specific locales:
+
+```typescript
+// Configuration approach
+const BLOCK_AWARE_LOCALES = [
+  Locales.FRENCH,
+  Locales.SPANISH,
+  Locales.GERMAN,
+  // Add more locales as needed
+];
+
+const shouldUseBlockAware = (filePath: string, locale: Locale): boolean => {
+  return (
+    filePath.endsWith('.md') && 
+    BLOCK_AWARE_LOCALES.includes(locale)
+  );
+};
+
+// In your review task:
+if (shouldUseBlockAware(absoluteBaseFilePath, locale)) {
+  await reviewFileBlockAware(/* ... */);
+} else {
+  await reviewFile(/* ... */);
+}
+```
+
+## Step 7: Gradual Rollout
+
+For safety, you can roll out gradually:
+
+### Phase 1: Parallel Testing (No Impact)
+
+```typescript
+// Run both systems and compare results
+const blockAwareOutput = await reviewFileBlockAware(/* ... */);
+const originalOutput = await reviewFile(/* ... */);
+
+// Log differences for analysis
+logDifferences(blockAwareOutput, originalOutput);
+
+// Use original output for now
+writeFileSync(outputFilePath, originalOutput);
+```
+
+### Phase 2: Selective Files
+
+```typescript
+const TEST_FILES = [
+  'docs/en/getting-started.md',
+  'docs/en/api-reference.md',
+];
+
+const useBlockAware = TEST_FILES.includes(docPath);
+```
+
+### Phase 3: Full Rollout
+
+```typescript
+// All markdown files use block-aware
+const useBlockAware = absoluteBaseFilePath.endsWith('.md');
+```
+
+## Step 8: Adjust Configuration Based on Results
+
+After analyzing real-world performance, tune the similarity thresholds:
+
+```typescript
+// In reviewDocBlockAware.ts
+const { plan, segmentsToReview } = buildAlignmentPlan({
+  englishText,
+  frenchText,
+  changedLines,
+  similarityOptions: {
+    // Adjust these based on your needs:
+    minimumMatchForReuse: 0.92,        // Higher = more conservative (more reviews)
+    minimumMatchForNearDuplicate: 0.82, // For future near-duplicate detection
+  },
+});
+```
+
+### Finding Optimal Thresholds
+
+Run analysis on your corpus:
+
+```typescript
+// analyze-thresholds.ts
+const thresholds = [0.85, 0.87, 0.90, 0.92, 0.95];
+const results = new Map();
+
+for (const threshold of thresholds) {
+  const { plan } = buildAlignmentPlan({
+    englishText,
+    frenchText,
+    changedLines,
+    similarityOptions: { minimumMatchForReuse: threshold },
+  });
+  
+  const reusedCount = plan.actions.filter(a => a.kind === "reuse").length;
+  results.set(threshold, reusedCount);
+}
+
+console.table(results);
+// Choose threshold with best balance of accuracy and efficiency
+```
+
+## Step 9: Handle Edge Cases
+
+Add error handling for edge cases:
+
+```typescript
+export const reviewFileBlockAware = async (
+  baseFilePath: string,
+  outputFilePath: string,
+  locale: Locale,
+  baseLocale: Locale,
+  aiOptions?: AIOptions,
+  configOptions?: GetConfigurationOptions,
+  customInstructions?: string,
+  changedLines?: number[]
+) => {
+  try {
+    const configuration = getConfiguration(configOptions);
+    const applicationLogger = getAppLogger(configuration);
+
+    // Read files with fallbacks
+    const englishText = await readFile(baseFilePath, "utf-8").catch((error) => {
+      applicationLogger(`Error reading base file: ${error.message}`);
+      throw error;
+    });
+    
+    const frenchText = await readFile(outputFilePath, "utf-8").catch(() => {
+      // If French file doesn't exist, treat as new translation
+      applicationLogger(`French file doesn't exist, creating new translation`);
+      return "";
+    });
+
+    // Validate inputs
+    if (!englishText.trim()) {
+      applicationLogger(`Base file is empty, skipping`);
+      return;
+    }
+
+    // Rest of implementation...
+
+  } catch (error) {
+    const configuration = getConfiguration(configOptions);
+    const applicationLogger = getAppLogger(configuration);
+    applicationLogger(
+      `${colorize('✖', ANSIColors.RED)} Error processing ${formatPath(baseFilePath)}: ${error.message}`
+    );
+    throw error;
+  }
+};
+```
+
+## Step 10: Create Migration Script
+
+For existing translations, create a one-time migration script:
+
+```typescript
+// migrate-to-block-aware.ts
+import fg from 'fast-glob';
+import { reviewFileBlockAware } from './reviewDocBlockAware';
+
+const migrateAllTranslations = async () => {
+  const baseFiles = await fg('docs/en/**/*.md');
+  const locales = [Locales.FRENCH, Locales.SPANISH, Locales.GERMAN];
+
+  for (const baseFile of baseFiles) {
+    for (const locale of locales) {
+      const outputFile = baseFile.replace('/en/', `/${locale}/`);
+      
+      console.log(`Migrating: ${baseFile} → ${outputFile}`);
+      
+      try {
+        await reviewFileBlockAware(
+          baseFile,
+          outputFile,
+          locale,
+          Locales.ENGLISH,
+          aiOptions,
+          configOptions,
+          undefined,
+          undefined // No changed lines = full review (but with block awareness)
+        );
+      } catch (error) {
+        console.error(`Failed to migrate ${baseFile}:`, error);
+      }
+    }
+  }
+};
+
+migrateAllTranslations().catch(console.error);
+```
+
+## Complete Integration Example
+
+Here's the complete modified `reviewDoc.ts` with block-aware integration:
+
+```typescript
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { dirname, join, relative } from 'node:path';
+import { readAsset } from 'utils:asset';
+import type { AIOptions } from '@intlayer/api';
+import {
+  formatLocale,
+  formatPath,
+  getChunk,
+  type ListGitFilesOptions,
+  listGitFiles,
+  listGitLines,
+  parallelize,
+} from '@intlayer/chokidar';
+import {
+  ANSIColors,
+  colon,
+  colorize,
+  colorizeNumber,
+  type GetConfigurationOptions,
+  getAppLogger,
+  getConfiguration,
+  retryManager,
+} from '@intlayer/config';
+import { getLocaleName } from '@intlayer/core';
+import { type Locale, Locales } from '@intlayer/types';
+import fg from 'fast-glob';
+import { chunkText } from './utils/calculateChunks';
+import { checkAIAccess } from './utils/checkAccess';
+import { checkFileModifiedRange } from './utils/checkFileModifiedRange';
+import { chunkInference } from './utils/chunkInference';
+import { fixChunkStartEndChars } from './utils/fixChunkStartEndChars';
+import { getOutputFilePath } from './utils/getOutputFilePath';
+import { mapChunksBetweenFiles } from './utils/mapChunksBetweenFiles';
+import { reviewFileBlockAware } from './reviewDocBlockAware'; // NEW IMPORT
+
+// Keep existing reviewFile function...
+export const reviewFile = async (/* ... */) => {
+  // Existing implementation
+};
+
+// Keep existing ReviewDocOptions type...
+type ReviewDocOptions = {
+  docPattern: string[];
+  locales: Locale[];
+  excludedGlobPattern: string[];
+  baseLocale: Locale;
+  aiOptions?: AIOptions;
+  nbSimultaneousFileProcessed?: number;
+  configOptions?: GetConfigurationOptions;
+  customInstructions?: string;
+  skipIfModifiedBefore?: number | string | Date;
+  skipIfModifiedAfter?: number | string | Date;
+  skipIfExists?: boolean;
+  gitOptions?: ListGitFilesOptions;
+  useBlockAware?: boolean; // NEW OPTION
+};
+
+export const reviewDoc = async ({
+  docPattern,
+  locales,
+  excludedGlobPattern,
+  baseLocale,
+  aiOptions,
+  nbSimultaneousFileProcessed,
+  configOptions,
+  customInstructions,
+  skipIfModifiedBefore,
+  skipIfModifiedAfter,
+  skipIfExists,
+  gitOptions,
+  useBlockAware = true, // NEW: Default to true
+}: ReviewDocOptions) => {
+  const configuration = getConfiguration(configOptions);
+  const appLogger = getAppLogger(configuration);
+
+  const hasCMSAuth = await checkAIAccess(configuration, aiOptions);
+  if (!hasCMSAuth) return;
+
+  if (nbSimultaneousFileProcessed && nbSimultaneousFileProcessed > 10) {
+    appLogger(
+      `Warning: nbSimultaneousFileProcessed is set to ${nbSimultaneousFileProcessed}, which is greater than 10. Setting it to 10.`
+    );
+    nbSimultaneousFileProcessed = 10;
+  }
+
+  let docList: string[] = await fg(docPattern, {
+    ignore: excludedGlobPattern,
+  });
+
+  if (gitOptions) {
+    const gitChangedFiles = await listGitFiles(gitOptions);
+    if (gitChangedFiles) {
+      docList = docList.filter((path) =>
+        gitChangedFiles.some((gitFile) => join(process.cwd(), path) === gitFile)
+      );
+    }
+  }
+
+  appLogger(`Base locale is ${formatLocale(baseLocale)}`);
+  appLogger(
+    `Reviewing ${colorizeNumber(locales.length)} locales: [ ${formatLocale(locales)} ]`
+  );
+  appLogger(`Reviewing ${colorizeNumber(docList.length)} files:`);
+  appLogger(docList.map((path) => ` - ${formatPath(path)}\n`));
+
+  const allTasks = docList.flatMap((docPath) =>
+    locales.map((locale) => async () => {
+      appLogger(
+        `Reviewing file: ${formatPath(docPath)} to ${formatLocale(locale)}`
+      );
+
+      const absoluteBaseFilePath = join(configuration.content.baseDir, docPath);
+      const outputFilePath = getOutputFilePath(
+        absoluteBaseFilePath,
+        locale,
+        baseLocale
+      );
+
+      if (skipIfExists && existsSync(outputFilePath)) {
+        const relativePath = relative(
+          configuration.content.baseDir,
+          outputFilePath
+        );
+        appLogger(
+          `${colorize('⊘', ANSIColors.YELLOW)} File ${formatPath(relativePath)} already exists, skipping.`
+        );
+        return;
+      }
+
+      const fileModificationData = checkFileModifiedRange(outputFilePath, {
+        skipIfModifiedBefore,
+        skipIfModifiedAfter,
+      });
+
+      if (fileModificationData.isSkipped) {
+        appLogger(fileModificationData.message);
+        return;
+      }
+
+      let changedLines: number[] | undefined;
+      if (gitOptions) {
+        const gitChangedLines = await listGitLines(
+          absoluteBaseFilePath,
+          gitOptions
+        );
+        appLogger(`Git changed lines: ${gitChangedLines.join(', ')}`);
+        changedLines = gitChangedLines;
+      }
+
+      // NEW: Decide whether to use block-aware translation
+      const shouldUseBlockAware = 
+        useBlockAware && absoluteBaseFilePath.endsWith('.md');
+
+      if (shouldUseBlockAware) {
+        await reviewFileBlockAware(
+          absoluteBaseFilePath,
+          outputFilePath,
+          locale as Locale,
+          baseLocale,
+          aiOptions,
+          configOptions,
+          customInstructions,
+          changedLines
+        );
+      } else {
+        await reviewFile(
+          absoluteBaseFilePath,
+          outputFilePath,
+          locale as Locale,
+          baseLocale,
+          aiOptions,
+          configOptions,
+          customInstructions,
+          changedLines
+        );
+      }
+    })
+  );
+
+  await parallelize(
+    allTasks,
+    (task) => task(),
+    nbSimultaneousFileProcessed ?? 3
+  );
+};
+```
+
+## Testing Checklist
+
+Before deploying to production:
+
+- [ ] Test with empty French file (new translation)
+- [ ] Test with identical files (should reuse 100%)
+- [ ] Test with minor changes (should have high reuse rate)
+- [ ] Test with reordered paragraphs (should detect reordering)
+- [ ] Test with deleted paragraphs
+- [ ] Test with added paragraphs
+- [ ] Test with Git changed lines integration
+- [ ] Test without Git changed lines (fallback)
+- [ ] Test error handling (missing files, corrupt files)
+- [ ] Verify output formatting matches original
+- [ ] Compare AI token usage (should be lower)
+- [ ] Verify translation quality matches or exceeds original system
+
+## Monitoring in Production
+
+Add metrics collection:
+
+```typescript
+// metrics.ts
+export interface TranslationMetrics {
+  fileName: string;
+  locale: string;
+  totalBlocks: number;
+  reusedBlocks: number;
+  reviewedBlocks: number;
+  newBlocks: number;
+  deletedBlocks: number;
+  tokensUsed: number;
+  processingTimeMs: number;
+  efficiencyRate: number;
+}
+
+const metrics: TranslationMetrics[] = [];
+
+export const collectMetrics = (metric: TranslationMetrics) => {
+  metrics.push(metric);
+};
+
+export const getAverageEfficiency = (): number => {
+  if (metrics.length === 0) return 0;
+  const total = metrics.reduce((sum, m) => sum + m.efficiencyRate, 0);
+  return total / metrics.length;
+};
+```
+
+## Support
+
+If you encounter issues:
+
+1. Check the logs for alignment details
+2. Verify input files are valid markdown
+3. Ensure Git integration is working if using changed lines
+4. Review similarity thresholds for your content type
+5. Open an issue with sample input/output files
+
+## Next Steps
+
+After successful integration:
+
+1. Monitor efficiency rates and adjust thresholds
+2. Gather feedback on translation quality
+3. Consider extending to other file types (HTML, RST, etc.)
+4. Explore parallel translation for multiple segments
+5. Add visual tools for reviewing alignment decisions
+