@llm-translate/cli 1.0.0-next.1

package/tests/unit/glossary.test.ts

@@ -0,0 +1,146 @@
+import { describe, it, expect } from 'vitest';
+import {
+  resolveGlossary,
+  createGlossaryLookup,
+  checkGlossaryCompliance,
+} from '../../src/services/glossary.js';
+import type { Glossary } from '../../src/types/index.js';
+
+const testGlossary: Glossary = {
+  metadata: {
+    name: 'Test Glossary',
+    sourceLang: 'en',
+    targetLangs: ['ko', 'ja'],
+    version: '1.0.0',
+  },
+  terms: [
+    {
+      source: 'machine learning',
+      targets: { ko: '머신러닝', ja: '機械学習' },
+    },
+    {
+      source: 'API',
+      targets: {},
+      doNotTranslate: true,
+    },
+    {
+      source: 'neural network',
+      targets: { ko: '신경망', ja: 'ニューラルネットワーク' },
+      caseSensitive: false,
+    },
+    {
+      source: 'SDK',
+      targets: { ko: 'SDK', ja: 'SDK' },
+      doNotTranslateFor: ['ko', 'ja'],
+    },
+  ],
+};
+
+describe('resolveGlossary', () => {
+  it('should resolve glossary for Korean target language', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+
+    expect(resolved.metadata.targetLang).toBe('ko');
+    expect(resolved.terms).toHaveLength(4);
+
+    const mlTerm = resolved.terms.find((t) => t.source === 'machine learning');
+    expect(mlTerm?.target).toBe('머신러닝');
+  });
+
+  it('should handle doNotTranslate terms', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+
+    const apiTerm = resolved.terms.find((t) => t.source === 'API');
+    expect(apiTerm?.target).toBe('API');
+    expect(apiTerm?.doNotTranslate).toBe(true);
+  });
+
+  it('should handle doNotTranslateFor specific languages', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+
+    const sdkTerm = resolved.terms.find((t) => t.source === 'SDK');
+    expect(sdkTerm?.target).toBe('SDK');
+    expect(sdkTerm?.doNotTranslate).toBe(true);
+  });
+});
+
+describe('createGlossaryLookup', () => {
+  it('should find exact term matches', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+    const lookup = createGlossaryLookup(resolved);
+
+    const term = lookup.find('machine learning');
+    expect(term?.source).toBe('machine learning');
+    expect(term?.target).toBe('머신러닝');
+  });
+
+  it('should handle case-insensitive lookup', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+    const lookup = createGlossaryLookup(resolved);
+
+    const term = lookup.find('Neural Network');
+    expect(term?.source).toBe('neural network');
+    expect(term?.target).toBe('신경망');
+  });
+
+  it('should find all matching terms in text', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+    const lookup = createGlossaryLookup(resolved);
+
+    const text = 'Learn about machine learning and neural network using our API.';
+    const matches = lookup.findAll(text);
+
+    expect(matches).toHaveLength(3);
+    expect(matches.map((m) => m.source)).toContain('machine learning');
+    expect(matches.map((m) => m.source)).toContain('neural network');
+    expect(matches.map((m) => m.source)).toContain('API');
+  });
+
+  it('should format glossary for prompt injection', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+    const lookup = createGlossaryLookup(resolved);
+
+    const formatted = lookup.formatForPrompt();
+
+    expect(formatted).toContain('"machine learning" → "머신러닝"');
+    expect(formatted).toContain('[DO NOT TRANSLATE, keep as-is]');
+  });
+});
+
+describe('checkGlossaryCompliance', () => {
+  it('should detect applied glossary terms', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+
+    const source = 'Machine learning is a field of artificial intelligence.';
+    const translation = '머신러닝은 인공지능의 한 분야입니다.';
+
+    const result = checkGlossaryCompliance(source, translation, resolved);
+
+    expect(result.applied).toContain('machine learning');
+    expect(result.score).toBe(100);
+  });
+
+  it('should detect missed glossary terms', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+
+    const source = 'Machine learning uses neural networks.';
+    const translation = '기계 학습은 뉴럴 네트워크를 사용합니다.'; // Wrong translations
+
+    const result = checkGlossaryCompliance(source, translation, resolved);
+
+    expect(result.missed).toContain('machine learning');
+    expect(result.missed).toContain('neural network');
+    expect(result.score).toBe(0);
+  });
+
+  it('should return 100% for text without glossary terms', () => {
+    const resolved = resolveGlossary(testGlossary, 'ko');
+
+    const source = 'Hello world!';
+    const translation = '안녕하세요!';
+
+    const result = checkGlossaryCompliance(source, translation, resolved);
+
+    expect(result.score).toBe(100);
+  });
+});

package/tests/unit/markdown.test.ts

@@ -0,0 +1,205 @@
+import { describe, it, expect } from 'vitest';
+import {
+  parseMarkdown,
+  applyTranslations,
+  getTranslatableText,
+  createTranslationMap,
+  extractTextForTranslation,
+  restorePreservedSections,
+} from '../../src/parsers/markdown.js';
+
+describe('parseMarkdown', () => {
+  it('should parse simple markdown content', async () => {
+    const content = '# Hello World\n\nThis is a paragraph.';
+    const doc = await parseMarkdown(content);
+
+    expect(doc.original).toBe(content);
+    expect(doc.ast).toBeDefined();
+    expect(doc.textNodes.length).toBeGreaterThan(0);
+  });
+
+  it('should identify text nodes as translatable', async () => {
+    const content = 'This is translatable text.';
+    const doc = await parseMarkdown(content);
+
+    const translatableNodes = doc.textNodes.filter(n => n.translatable);
+    expect(translatableNodes.length).toBeGreaterThan(0);
+  });
+
+  it('should mark code blocks as non-translatable', async () => {
+    const content = '```javascript\nconst x = 1;\n```';
+    const doc = await parseMarkdown(content);
+
+    const codeNodes = doc.textNodes.filter(n => n.type === 'code');
+    expect(codeNodes.every(n => !n.translatable)).toBe(true);
+  });
+
+  it('should mark inline code as non-translatable', async () => {
+    const content = 'Use the `console.log()` function.';
+    const doc = await parseMarkdown(content);
+
+    const inlineCodeNodes = doc.textNodes.filter(n => n.type === 'inlineCode');
+    expect(inlineCodeNodes.every(n => !n.translatable)).toBe(true);
+  });
+});
+
+describe('getTranslatableText', () => {
+  it('should return only translatable text', async () => {
+    const content = '# Title\n\nSome text.\n\n```code\nblock\n```\n\nMore text.';
+    const doc = await parseMarkdown(content);
+    const texts = getTranslatableText(doc);
+
+    expect(texts.length).toBeGreaterThan(0);
+    expect(texts.some(t => t.includes('Title'))).toBe(true);
+    expect(texts.some(t => t.includes('Some text'))).toBe(true);
+  });
+});
+
+describe('createTranslationMap', () => {
+  it('should create map from translations array', async () => {
+    const content = 'Hello. World.';
+    const doc = await parseMarkdown(content);
+    const translatableTexts = getTranslatableText(doc);
+    const translations = translatableTexts.map(t => `[TRANSLATED] ${t}`);
+
+    const map = createTranslationMap(doc, translations);
+
+    expect(Object.keys(map).length).toBe(translatableTexts.length);
+  });
+});
+
+describe('applyTranslations', () => {
+  it('should apply translations to document', async () => {
+    const content = 'Hello World';
+    const doc = await parseMarkdown(content);
+
+    const translatableNodes = doc.textNodes.filter(n => n.translatable);
+    const map: Record<string, string> = {};
+    for (const node of translatableNodes) {
+      map[node.id] = '안녕하세요 세계';
+    }
+
+    const result = await applyTranslations(doc, map);
+    expect(result).toContain('안녕하세요');
+  });
+});
+
+describe('extractTextForTranslation', () => {
+  it('should preserve code blocks with placeholders', () => {
+    const content = 'Before code\n\n```js\nconst x = 1;\n```\n\nAfter code';
+    const { text, preservedSections } = extractTextForTranslation(content);
+
+    expect(text).toContain('Before code');
+    expect(text).toContain('After code');
+    expect(text).toContain('__CODE_BLOCK_');
+    expect(text).not.toContain('const x = 1');
+    expect(preservedSections.size).toBeGreaterThan(0);
+  });
+
+  it('should preserve inline code with placeholders', () => {
+    const content = 'Use `npm install` to install.';
+    const { text, preservedSections } = extractTextForTranslation(content);
+
+    expect(text).toContain('Use');
+    expect(text).toContain('to install');
+    expect(text).toContain('__INLINE_CODE_');
+    expect(text).not.toContain('`npm install`');
+    expect(preservedSections.size).toBe(1);
+  });
+
+  it('should preserve URLs in links with placeholders', () => {
+    const content = 'Visit [our site](https://example.com) for more.';
+    const { text, preservedSections } = extractTextForTranslation(content);
+
+    expect(text).toContain('[our site]');
+    expect(text).toContain('__LINK_URL_');
+    expect(text).not.toContain('https://example.com');
+
+    // URL should be in preserved sections
+    const urls = Array.from(preservedSections.values());
+    expect(urls.some(u => u.includes('example.com'))).toBe(true);
+  });
+
+  it('should handle multiple code blocks', () => {
+    const content = `\`\`\`python
+print("hello")
+\`\`\`
+
+Some text
+
+\`\`\`javascript
+console.log("world");
+\`\`\``;
+
+    const { text, preservedSections } = extractTextForTranslation(content);
+
+    expect(text).toContain('Some text');
+    expect(preservedSections.size).toBe(2);
+  });
+});
+
+describe('restorePreservedSections', () => {
+  it('should restore code blocks from placeholders', () => {
+    const content = 'Text with ```js\ncode\n```';
+    const { text, preservedSections } = extractTextForTranslation(content);
+
+    // Simulate translation (keep placeholders)
+    const translated = text.replace('Text with', '텍스트와 함께');
+    const restored = restorePreservedSections(translated, preservedSections);
+
+    expect(restored).toContain('텍스트와 함께');
+    expect(restored).toContain('```js');
+    expect(restored).toContain('code');
+  });
+
+  it('should restore URLs in links', () => {
+    const content = 'Check [this link](https://test.com).';
+    const { text, preservedSections } = extractTextForTranslation(content);
+
+    const translated = text.replace('Check', '확인하세요').replace('this link', '이 링크');
+    const restored = restorePreservedSections(translated, preservedSections);
+
+    expect(restored).toContain('확인하세요');
+    expect(restored).toContain('[이 링크]');
+    expect(restored).toContain('https://test.com');
+  });
+
+  it('should handle empty preserved sections', () => {
+    const text = 'Simple text without code';
+    const preserved = new Map<string, string>();
+
+    const result = restorePreservedSections(text, preserved);
+    expect(result).toBe(text);
+  });
+
+  it('should ensure proper spacing around inline code after restoration', () => {
+    // Simulate a case where LLM removed spaces around placeholder
+    const content = '1. Scan your `./docs` directory';
+    const { text, preservedSections } = extractTextForTranslation(content);
+
+    // Simulate LLM translation that removes spaces around placeholder
+    // Original: "1. Scan your __INLINE_CODE_0__ directory"
+    // LLM might produce: "1.__INLINE_CODE_0__디렉토리를 스캔합니다"
+    const translatedWithoutSpaces = text
+      .replace('1. Scan your ', '1.')
+      .replace(' directory', '디렉토리를 스캔합니다');
+
+    const restored = restorePreservedSections(translatedWithoutSpaces, preservedSections);
+
+    // Should have proper spacing around inline code
+    expect(restored).toContain('1. `./docs`');  // Space before backtick
+    expect(restored).toContain('`./docs` 디렉토리');  // Space after backtick
+  });
+
+  it('should add space between Korean text and inline code', () => {
+    const preserved = new Map<string, string>();
+    preserved.set('__INLINE_CODE_0__', '`code`');
+
+    // Korean text directly touching placeholder
+    const translated = '스캔합니다__INLINE_CODE_0__디렉토리';
+    const restored = restorePreservedSections(translated, preserved);
+
+    // Should have spaces around inline code
+    expect(restored).toBe('스캔합니다 `code` 디렉토리');
+  });
+});

package/tests/unit/tokens.test.ts

@@ -0,0 +1,81 @@
+import { describe, it, expect } from 'vitest';
+import {
+  estimateTokens,
+  exceedsTokenLimit,
+  truncateToTokenLimit,
+} from '../../src/utils/tokens.js';
+
+describe('estimateTokens', () => {
+  it('should return 0 for empty string', () => {
+    expect(estimateTokens('')).toBe(0);
+  });
+
+  it('should estimate tokens for English text', () => {
+    const text = 'Hello world, this is a test sentence for token estimation.';
+    const tokens = estimateTokens(text);
+
+    // ~60 characters, ~4 chars/token = ~15 tokens
+    expect(tokens).toBeGreaterThan(10);
+    expect(tokens).toBeLessThan(25);
+  });
+
+  it('should estimate tokens for Korean text', () => {
+    const text = '안녕하세요. 이것은 토큰 추정을 위한 테스트 문장입니다.';
+    const tokens = estimateTokens(text);
+
+    // CJK characters are ~1.5 chars/token, so higher token count
+    expect(tokens).toBeGreaterThan(15);
+  });
+
+  it('should estimate tokens for mixed text', () => {
+    const text = 'Hello 세계! This is 테스트입니다.';
+    const tokens = estimateTokens(text);
+
+    expect(tokens).toBeGreaterThan(5);
+  });
+
+  it('should handle code snippets', () => {
+    const code = `function hello() {
+  console.log("Hello, World!");
+}`;
+    const tokens = estimateTokens(code);
+
+    expect(tokens).toBeGreaterThan(10);
+    expect(tokens).toBeLessThan(50);
+  });
+});
+
+describe('exceedsTokenLimit', () => {
+  it('should return false when under limit', () => {
+    const text = 'Short text';
+    expect(exceedsTokenLimit(text, 100)).toBe(false);
+  });
+
+  it('should return true when over limit', () => {
+    const text = 'a'.repeat(1000); // ~250 tokens
+    expect(exceedsTokenLimit(text, 100)).toBe(true);
+  });
+});
+
+describe('truncateToTokenLimit', () => {
+  it('should return original text when under limit', () => {
+    const text = 'Short text';
+    expect(truncateToTokenLimit(text, 100)).toBe(text);
+  });
+
+  it('should truncate text when over limit', () => {
+    const text = 'a'.repeat(1000); // ~250 tokens
+    const truncated = truncateToTokenLimit(text, 50);
+
+    expect(truncated.length).toBeLessThan(text.length);
+    expect(truncated.endsWith('...')).toBe(true);
+  });
+
+  it('should preserve approximately the requested token count', () => {
+    const text = 'This is a longer piece of text that should be truncated to fit within the token limit.';
+    const truncated = truncateToTokenLimit(text, 10);
+
+    const estimatedTokens = estimateTokens(truncated);
+    expect(estimatedTokens).toBeLessThanOrEqual(15); // Some margin for the ellipsis
+  });
+});

package/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "lib": ["ES2022"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "noUncheckedIndexedAccess": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "exactOptionalPropertyTypes": false,
+    "isolatedModules": true,
+    "verbatimModuleSyntax": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "tests"]
+}

package/tsup.config.ts

@@ -0,0 +1,34 @@
+import { defineConfig } from 'tsup';
+
+export default defineConfig([
+  // Library entry point
+  {
+    entry: {
+      index: 'src/index.ts',
+    },
+    format: ['esm'],
+    dts: true,
+    sourcemap: true,
+    clean: true,
+    target: 'node20',
+    shims: false,
+    splitting: false,
+    treeshake: true,
+  },
+  // CLI entry point
+  {
+    entry: {
+      'cli/index': 'src/cli/index.ts',
+    },
+    format: ['esm'],
+    dts: true,
+    sourcemap: true,
+    target: 'node20',
+    shims: false,
+    splitting: false,
+    treeshake: true,
+    banner: {
+      js: '#!/usr/bin/env node',
+    },
+  },
+]);

package/vitest.config.ts

@@ -0,0 +1,16 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['tests/**/*.test.ts'],
+    coverage: {
+      provider: 'v8',
+      reporter: ['text', 'json', 'html'],
+      include: ['src/**/*.ts'],
+      exclude: ['src/**/*.d.ts', 'src/cli/index.ts'],
+    },
+    testTimeout: 30000,
+  },
+});