ai-localize-scanner 1.0.0

package/src/ast-scanner.ts

@@ -0,0 +1,225 @@
+import * as parser from '@babel/parser';
+import traverse from '@babel/traverse';
+import * as t from '@babel/types';
+
+import type { DetectedText, TextContext } from '@ai-localize/shared';
+import {
+  isHumanReadableText,
+  normalizeText,
+  TEXT_ATTRIBUTE_NAMES,
+  generateLocaleKey,
+} from '@ai-localize/shared';
+
+export interface AstScanOptions {
+  filePath: string;
+  content: string;
+  sourceRoot?: string;
+}
+
+const TRANSLATION_IMPORT_SOURCES = new Set([
+  'react-i18next',
+  'i18next',
+  'vue-i18n',
+  '@ngx-translate/core',
+]);
+
+/**
+ * Scans a JS/TS/JSX/TSX file using Babel AST to find hardcoded text.
+ */
+export class AstScanner {
+  private options: AstScanOptions;
+  private detectedTexts: DetectedText[] = [];
+  private translationFunctionNames = new Set<string>(['t', '$t', 'i18n', 'translate']);
+
+  constructor(options: AstScanOptions) {
+    this.options = options;
+  }
+
+  scan(): DetectedText[] {
+    const { content } = this.options;
+
+    let ast: t.File;
+    try {
+ast = parser.parse(content, {
+        sourceType: 'module',
+        plugins: [
+  'jsx',
+      'typescript',
+       'decorators-legacy',
+          'classProperties',
+     'optionalChaining',
+      'nullishCoalescingOperator',
+    'dynamicImport',
+          'exportDefaultFrom',
+        ],
+        errorRecovery: true,
+      });
+    } catch {
+      return this.regexFallbackScan();
+    }
+
+    this.collectTranslationImports(ast);
+
+    traverse(ast, {
+      JSXText: (nodePath) => {
+        const text = normalizeText(nodePath.node.value);
+        if (!isHumanReadableText(text)) return;
+        if (this.isInsideTranslationCall(nodePath)) return;
+        this.addDetected(
+          text,
+          nodePath.node.loc?.start.line ?? 0,
+          nodePath.node.loc?.start.column ?? 0,
+          'jsx-text',
+'JSXText'
+        );
+      },
+
+      JSXAttribute: (nodePath) => {
+        const attrName = t.isJSXIdentifier(nodePath.node.name)
+          ? nodePath.node.name.name
+        : '';
+    if (!TEXT_ATTRIBUTE_NAMES.has(attrName.toLowerCase())) return;
+   const valueNode = nodePath.node.value;
+if (!t.isStringLiteral(valueNode)) return;
+      const text = normalizeText(valueNode.value);
+        if (!isHumanReadableText(text)) return;
+    if (this.isInsideTranslationCall(nodePath)) return;
+      const context = this.mapAttrToContext(attrName);
+        this.addDetected(
+  text,
+          valueNode.loc?.start.line ?? 0,
+        valueNode.loc?.start.column ?? 0,
+          context,
+          'JSXAttribute'
+        );
+      },
+
+      StringLiteral: (nodePath) => {
+        if (t.isImportDeclaration(nodePath.parent)) return;
+        if (t.isObjectProperty(nodePath.parent) && nodePath.parent.key === nodePath.node) return;
+     if (t.isJSXAttribute(nodePath.parent)) return;
+        if (this.isInsideTranslationCall(nodePath)) return;
+    if (/^[a-z][a-z0-9_.]+$/.test(nodePath.node.value)) return;
+        const text = normalizeText(nodePath.node.value);
+        if (!isHumanReadableText(text)) return;
+     this.addDetected(
+          text,
+          nodePath.node.loc?.start.line ?? 0,
+          nodePath.node.loc?.start.column ?? 0,
+          'string-literal',
+     'StringLiteral'
+        );
+      },
+
+ TemplateLiteral: (nodePath) => {
+        if (nodePath.node.expressions.length > 0) return;
+        if (this.isInsideTranslationCall(nodePath)) return;
+     const text = normalizeText(nodePath.node.quasis[0]?.value.cooked ?? '');
+ if (!isHumanReadableText(text)) return;
+     this.addDetected(
+text,
+          nodePath.node.loc?.start.line ?? 0,
+          nodePath.node.loc?.start.column ?? 0,
+       'template-literal',
+          'TemplateLiteral'
+  );
+   },
+    });
+
+    return this.detectedTexts;
+  }
+
+  private collectTranslationImports(ast: t.File): void {
+    for (const node of ast.program.body) {
+      if (!t.isImportDeclaration(node)) continue;
+      if (!TRANSLATION_IMPORT_SOURCES.has(node.source.value)) continue;
+ for (const specifier of node.specifiers) {
+     if (t.isImportSpecifier(specifier) && t.isIdentifier(specifier.local)) {
+          this.translationFunctionNames.add(specifier.local.name);
+        }
+      }
+    }
+  }
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private isInsideTranslationCall(nodePath: any): boolean {
+    let current = nodePath.parentPath;
+    while (current) {
+      const node = current.node;
+      if (t.isCallExpression(node)) {
+        const callee = node.callee;
+        if (t.isIdentifier(callee) && this.translationFunctionNames.has(callee.name)) {
+          return true;
+        }
+ if (
+          t.isMemberExpression(callee) &&
+          t.isIdentifier(callee.property) &&
+        this.translationFunctionNames.has(callee.property.name)
+        ) {
+          return true;
+        }
+    }
+      current = current.parentPath;
+    }
+    return false;
+  }
+
+  private addDetected(
+    text: string,
+    line: number,
+    column: number,
+    context: TextContext,
+    nodeType: string
+  ): void {
+    const key = generateLocaleKey(
+    this.options.filePath,
+      text,
+      this.options.sourceRoot || 'src'
+    );
+    this.detectedTexts.push({
+      filePath: this.options.filePath,
+      line,
+      column,
+text,
+      suggestedKey: key,
+    context,
+  nodeType,
+      alreadyTranslated: false,
+    });
+  }
+
+  private mapAttrToContext(attrName: string): TextContext {
+    const lower = attrName.toLowerCase();
+    if (lower === 'placeholder') return 'placeholder';
+    if (lower === 'aria-label' || lower === 'aria-placeholder') return 'aria-label';
+    if (lower === 'title') return 'title';
+    if (lower === 'alt') return 'alt';
+    return 'jsx-attribute';
+  }
+
+  private regexFallbackScan(): DetectedText[] {
+    const results: DetectedText[] = [];
+    const jsxTextRegex = />([^<>{}\n]+)</g;
+    const lines = this.options.content.split('\n');
+    lines.forEach((line, idx) => {
+      let m: RegExpExecArray | null;
+ jsxTextRegex.lastIndex = 0;
+      while ((m = jsxTextRegex.exec(line)) !== null) {
+        const text = normalizeText(m[1]);
+        if (!isHumanReadableText(text)) continue;
+  const key = generateLocaleKey(this.options.filePath, text, this.options.sourceRoot || 'src');
+        results.push({
+          filePath: this.options.filePath,
+     line: idx + 1,
+          column: m.index,
+     text,
+ suggestedKey: key,
+        context: 'jsx-text',
+          nodeType: 'regex-fallback',
+   alreadyTranslated: false,
+        });
+ }
+    });
+    return results;
+  }
+}

package/src/git-scanner.ts

@@ -0,0 +1,52 @@
+import { execSync } from 'child_process';
+import * as path from 'path';
+
+export class GitScanner {
+  private cwd: string;
+
+  constructor(cwd = process.cwd()) {
+    this.cwd = cwd;
+  }
+
+  getStagedFiles(extensions = ['ts', 'tsx', 'js', 'jsx', 'vue']): string[] {
+    try {
+    const out = execSync('git diff --cached --name-only --diff-filter=ACM', {
+        cwd: this.cwd,
+        encoding: 'utf-8',
+   });
+      return this.filter(out.trim().split('\n'), extensions);
+    } catch {
+      return [];
+    }
+  }
+
+  getChangedFiles(base = 'main', extensions = ['ts', 'tsx', 'js', 'jsx', 'vue']): string[] {
+    try {
+      const out = execSync(`git diff --name-only --diff-filter=ACM ${base}...HEAD`, {
+        cwd: this.cwd,
+     encoding: 'utf-8',
+      });
+      return this.filter(out.trim().split('\n'), extensions);
+    } catch {
+   return [];
+    }
+  }
+
+  getRecentlyChangedFiles(commits = 1, extensions = ['ts', 'tsx', 'js', 'jsx', 'vue']): string[] {
+    try {
+      const out = execSync(
+      `git diff --name-only --diff-filter=ACM HEAD~${commits}...HEAD`,
+        { cwd: this.cwd, encoding: 'utf-8' }
+      );
+      return this.filter(out.trim().split('\n'), extensions);
+ } catch {
+      return [];
+    }
+  }
+
+  private filter(files: string[], extensions: string[]): string[] {
+    return files
+      .filter((f) => f && extensions.some((e) => f.endsWith(`.${e}`)))
+      .map((f) => path.join(this.cwd, f));
+  }
+}

package/src/incremental-scanner.ts

@@ -0,0 +1,58 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+
+import type { IncrementalCache, DetectedText } from '@ai-localize/shared';
+import { readJsonSafe, writeJson, ensureDir } from '@ai-localize/shared';
+
+export class IncrementalScanCache {
+  private cachePath: string;
+  private cache: IncrementalCache;
+
+  constructor(cacheDir: string) {
+    ensureDir(cacheDir);
+    this.cachePath = path.join(cacheDir, 'scan-cache.json');
+    this.cache = this.load();
+}
+
+  private load(): IncrementalCache {
+    const existing = readJsonSafe<IncrementalCache>(this.cachePath);
+    if (existing?.version === '1') return existing;
+    return { version: '1', lastRun: new Date().toISOString(), fileHashes: {}, processedFiles: {} };
+  }
+
+  isFileChanged(filePath: string): boolean {
+    return this.hashFile(filePath) !== this.cache.fileHashes[filePath];
+  }
+
+  getCachedResult(filePath: string): DetectedText[] | null {
+    const entry = this.cache.processedFiles[filePath];
+    if (!entry) return null;
+    if (entry.hash !== this.hashFile(filePath)) return null;
+return entry.detectedTexts;
+  }
+
+  setCachedResult(filePath: string, texts: DetectedText[]): void {
+    const hash = this.hashFile(filePath);
+    this.cache.fileHashes[filePath] = hash;
+    this.cache.processedFiles[filePath] = { hash, detectedTexts: texts, lastModified: Date.now() };
+  }
+
+  persist(): void {
+    this.cache.lastRun = new Date().toISOString();
+    writeJson(this.cachePath, this.cache);
+  }
+
+  private hashFile(filePath: string): string {
+    try {
+      return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
+    } catch {
+ return '';
+    }
+  }
+
+  clear(): void {
+    this.cache = { version: '1', lastRun: new Date().toISOString(), fileHashes: {}, processedFiles: {} };
+    this.persist();
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './ast-scanner.js';
+export * from './asset-scanner.js';
+export * from './incremental-scanner.js';
+export * from './project-scanner.js';
+export * from './git-scanner.js';

package/src/project-scanner.ts

@@ -0,0 +1,114 @@
+import * as path from 'path';
+import * as os from 'os';
+
+import type {
+  DetectedText,
+  AssetReference,
+  LegacyCdnUrl,
+  ScanResult,
+  LocalizationConfig,
+} from '@ai-localize/shared';
+import { collectFiles, DEFAULT_IGNORE_DIRS, SOURCE_EXTENSIONS } from '@ai-localize/shared';
+
+import { AstScanner } from './ast-scanner.js';
+import { AssetScanner } from './asset-scanner.js';
+import { IncrementalScanCache } from './incremental-scanner.js';
+
+export interface ScanOptions {
+  files?: string[];
+  incremental?: boolean;
+}
+
+export class ProjectScanner {
+  private config: LocalizationConfig;
+  private sourceRoot: string;
+  private cache?: IncrementalScanCache;
+  private assetScanner: AssetScanner;
+
+  constructor(config: LocalizationConfig) {
+    this.config = config;
+    this.sourceRoot = path.join(process.cwd(), config.sourceDir);
+    this.assetScanner = new AssetScanner(config.aws?.legacyCdnPattern);
+    if (config.incrementalCache) {
+      this.cache = new IncrementalScanCache(
+      path.join(process.cwd(), config.cacheDir || '.ai-localize-cache')
+      );
+    }
+  }
+
+  async scan(options: ScanOptions = {}): Promise<ScanResult> {
+    const startTime = Date.now();
+    const filesToScan = options.files?.length
+      ? options.files
+      : collectFiles(this.sourceRoot, SOURCE_EXTENSIONS, [
+ ...DEFAULT_IGNORE_DIRS,
+          ...(this.config.ignorePatterns || []),
+   ]);
+
+    const allTexts: DetectedText[] = [];
+    const allAssets: AssetReference[] = [];
+    const allLegacyUrls: LegacyCdnUrl[] = [];
+
+  const chunkSize = Math.max(
+  1,
+  Math.min(50, Math.ceil(filesToScan.length / (os.cpus().length || 4)))
+    );
+    const chunks = this.chunkArray(filesToScan, chunkSize);
+
+    for (const chunk of chunks) {
+const results = await Promise.all(chunk.map((f) => this.scanFile(f)));
+      for (const r of results) {
+        allTexts.push(...r.texts);
+        allAssets.push(...r.assets);
+        allLegacyUrls.push(...r.legacyUrls);
+      }
+    }
+
+    this.cache?.persist();
+
+    return {
+  framework: this.config.framework,
+      scannedFiles: filesToScan.length,
+      detectedTexts: allTexts,
+    assets: allAssets,
+      legacyCdnUrls: allLegacyUrls,
+    duration: Date.now() - startTime,
+      timestamp: new Date().toISOString(),
+    };
+  }
+
+  private async scanFile(filePath: string): Promise<{
+    texts: DetectedText[];
+    assets: AssetReference[];
+    legacyUrls: LegacyCdnUrl[];
+  }> {
+    if (this.cache && !this.cache.isFileChanged(filePath)) {
+      const cached = this.cache.getCachedResult(filePath);
+      if (cached) return { texts: cached, assets: [], legacyUrls: [] };
+    }
+
+    let content: string;
+    try {
+      const { readFileSync } = await import('fs');
+      content = readFileSync(filePath, 'utf-8');
+    } catch {
+      return { texts: [], assets: [], legacyUrls: [] };
+ }
+
+    const scanner = new AstScanner({ filePath, content, sourceRoot: this.config.sourceDir });
+    const texts = scanner.scan();
+    const { assets, legacyCdnUrls } = this.assetScanner.scanFile(filePath);
+
+    this.cache?.setCachedResult(filePath, texts);
+
+    return { texts, assets, legacyUrls: legacyCdnUrls };
+  }
+
+  private chunkArray<T>(array: T[], size: number): T[][] {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += size) {
+      chunks.push(array.slice(i, i + size));
+    }
+    return chunks;
+  }
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}