ai-localize-scanner 1.0.0

package/dist/index.d.mts

@@ -0,0 +1,73 @@
+import { DetectedText, AssetReference, LegacyCdnUrl, LocalizationConfig, ScanResult } from '@ai-localize/shared';
+
+interface AstScanOptions {
+    filePath: string;
+    content: string;
+    sourceRoot?: string;
+}
+/**
+ * Scans a JS/TS/JSX/TSX file using Babel AST to find hardcoded text.
+ */
+declare class AstScanner {
+    private options;
+    private detectedTexts;
+    private translationFunctionNames;
+    constructor(options: AstScanOptions);
+    scan(): DetectedText[];
+    private collectTranslationImports;
+    private isInsideTranslationCall;
+    private addDetected;
+    private mapAttrToContext;
+    private regexFallbackScan;
+}
+
+declare class AssetScanner {
+    private legacyCdnPattern;
+    constructor(legacyCdnPattern?: string);
+    scanFile(filePath: string): {
+        assets: AssetReference[];
+        legacyCdnUrls: LegacyCdnUrl[];
+    };
+    private getLineNumber;
+    private getAssetType;
+    private extractPathFromUrl;
+}
+
+declare class IncrementalScanCache {
+    private cachePath;
+    private cache;
+    constructor(cacheDir: string);
+    private load;
+    isFileChanged(filePath: string): boolean;
+    getCachedResult(filePath: string): DetectedText[] | null;
+    setCachedResult(filePath: string, texts: DetectedText[]): void;
+    persist(): void;
+    private hashFile;
+    clear(): void;
+}
+
+interface ScanOptions {
+    files?: string[];
+    incremental?: boolean;
+}
+declare class ProjectScanner {
+    private config;
+    private sourceRoot;
+    private cache?;
+    private assetScanner;
+    constructor(config: LocalizationConfig);
+    scan(options?: ScanOptions): Promise<ScanResult>;
+    private scanFile;
+    private chunkArray;
+}
+
+declare class GitScanner {
+    private cwd;
+    constructor(cwd?: string);
+    getStagedFiles(extensions?: string[]): string[];
+    getChangedFiles(base?: string, extensions?: string[]): string[];
+    getRecentlyChangedFiles(commits?: number, extensions?: string[]): string[];
+    private filter;
+}
+
+export { AssetScanner, type AstScanOptions, AstScanner, GitScanner, IncrementalScanCache, ProjectScanner, type ScanOptions };

package/dist/index.d.ts

@@ -0,0 +1,73 @@
+import { DetectedText, AssetReference, LegacyCdnUrl, LocalizationConfig, ScanResult } from '@ai-localize/shared';
+
+interface AstScanOptions {
+    filePath: string;
+    content: string;
+    sourceRoot?: string;
+}
+/**
+ * Scans a JS/TS/JSX/TSX file using Babel AST to find hardcoded text.
+ */
+declare class AstScanner {
+    private options;
+    private detectedTexts;
+    private translationFunctionNames;
+    constructor(options: AstScanOptions);
+    scan(): DetectedText[];
+    private collectTranslationImports;
+    private isInsideTranslationCall;
+    private addDetected;
+    private mapAttrToContext;
+    private regexFallbackScan;
+}
+
+declare class AssetScanner {
+    private legacyCdnPattern;
+    constructor(legacyCdnPattern?: string);
+    scanFile(filePath: string): {
+        assets: AssetReference[];
+        legacyCdnUrls: LegacyCdnUrl[];
+    };
+    private getLineNumber;
+    private getAssetType;
+    private extractPathFromUrl;
+}
+
+declare class IncrementalScanCache {
+    private cachePath;
+    private cache;
+    constructor(cacheDir: string);
+    private load;
+    isFileChanged(filePath: string): boolean;
+    getCachedResult(filePath: string): DetectedText[] | null;
+    setCachedResult(filePath: string, texts: DetectedText[]): void;
+    persist(): void;
+    private hashFile;
+    clear(): void;
+}
+
+interface ScanOptions {
+    files?: string[];
+    incremental?: boolean;
+}
+declare class ProjectScanner {
+    private config;
+    private sourceRoot;
+    private cache?;
+    private assetScanner;
+    constructor(config: LocalizationConfig);
+    scan(options?: ScanOptions): Promise<ScanResult>;
+    private scanFile;
+    private chunkArray;
+}
+
+declare class GitScanner {
+    private cwd;
+    constructor(cwd?: string);
+    getStagedFiles(extensions?: string[]): string[];
+    getChangedFiles(base?: string, extensions?: string[]): string[];
+    getRecentlyChangedFiles(commits?: number, extensions?: string[]): string[];
+    private filter;
+}
+
+export { AssetScanner, type AstScanOptions, AstScanner, GitScanner, IncrementalScanCache, ProjectScanner, type ScanOptions };

package/dist/index.js

@@ -0,0 +1,504 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  AssetScanner: () => AssetScanner,
+  AstScanner: () => AstScanner,
+  GitScanner: () => GitScanner,
+  IncrementalScanCache: () => IncrementalScanCache,
+  ProjectScanner: () => ProjectScanner
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/ast-scanner.ts
+var parser = __toESM(require("@babel/parser"));
+var import_traverse = __toESM(require("@babel/traverse"));
+var t = __toESM(require("@babel/types"));
+var import_shared = require("@ai-localize/shared");
+var TRANSLATION_IMPORT_SOURCES = /* @__PURE__ */ new Set([
+  "react-i18next",
+  "i18next",
+  "vue-i18n",
+  "@ngx-translate/core"
+]);
+var AstScanner = class {
+  options;
+  detectedTexts = [];
+  translationFunctionNames = /* @__PURE__ */ new Set(["t", "$t", "i18n", "translate"]);
+  constructor(options) {
+    this.options = options;
+  }
+  scan() {
+    const { content } = this.options;
+    let ast;
+    try {
+      ast = parser.parse(content, {
+        sourceType: "module",
+        plugins: [
+          "jsx",
+          "typescript",
+          "decorators-legacy",
+          "classProperties",
+          "optionalChaining",
+          "nullishCoalescingOperator",
+          "dynamicImport",
+          "exportDefaultFrom"
+        ],
+        errorRecovery: true
+      });
+    } catch {
+      return this.regexFallbackScan();
+    }
+    this.collectTranslationImports(ast);
+    (0, import_traverse.default)(ast, {
+      JSXText: (nodePath) => {
+        const text = (0, import_shared.normalizeText)(nodePath.node.value);
+        if (!(0, import_shared.isHumanReadableText)(text)) return;
+        if (this.isInsideTranslationCall(nodePath)) return;
+        this.addDetected(
+          text,
+          nodePath.node.loc?.start.line ?? 0,
+          nodePath.node.loc?.start.column ?? 0,
+          "jsx-text",
+          "JSXText"
+        );
+      },
+      JSXAttribute: (nodePath) => {
+        const attrName = t.isJSXIdentifier(nodePath.node.name) ? nodePath.node.name.name : "";
+        if (!import_shared.TEXT_ATTRIBUTE_NAMES.has(attrName.toLowerCase())) return;
+        const valueNode = nodePath.node.value;
+        if (!t.isStringLiteral(valueNode)) return;
+        const text = (0, import_shared.normalizeText)(valueNode.value);
+        if (!(0, import_shared.isHumanReadableText)(text)) return;
+        if (this.isInsideTranslationCall(nodePath)) return;
+        const context = this.mapAttrToContext(attrName);
+        this.addDetected(
+          text,
+          valueNode.loc?.start.line ?? 0,
+          valueNode.loc?.start.column ?? 0,
+          context,
+          "JSXAttribute"
+        );
+      },
+      StringLiteral: (nodePath) => {
+        if (t.isImportDeclaration(nodePath.parent)) return;
+        if (t.isObjectProperty(nodePath.parent) && nodePath.parent.key === nodePath.node) return;
+        if (t.isJSXAttribute(nodePath.parent)) return;
+        if (this.isInsideTranslationCall(nodePath)) return;
+        if (/^[a-z][a-z0-9_.]+$/.test(nodePath.node.value)) return;
+        const text = (0, import_shared.normalizeText)(nodePath.node.value);
+        if (!(0, import_shared.isHumanReadableText)(text)) return;
+        this.addDetected(
+          text,
+          nodePath.node.loc?.start.line ?? 0,
+          nodePath.node.loc?.start.column ?? 0,
+          "string-literal",
+          "StringLiteral"
+        );
+      },
+      TemplateLiteral: (nodePath) => {
+        if (nodePath.node.expressions.length > 0) return;
+        if (this.isInsideTranslationCall(nodePath)) return;
+        const text = (0, import_shared.normalizeText)(nodePath.node.quasis[0]?.value.cooked ?? "");
+        if (!(0, import_shared.isHumanReadableText)(text)) return;
+        this.addDetected(
+          text,
+          nodePath.node.loc?.start.line ?? 0,
+          nodePath.node.loc?.start.column ?? 0,
+          "template-literal",
+          "TemplateLiteral"
+        );
+      }
+    });
+    return this.detectedTexts;
+  }
+  collectTranslationImports(ast) {
+    for (const node of ast.program.body) {
+      if (!t.isImportDeclaration(node)) continue;
+      if (!TRANSLATION_IMPORT_SOURCES.has(node.source.value)) continue;
+      for (const specifier of node.specifiers) {
+        if (t.isImportSpecifier(specifier) && t.isIdentifier(specifier.local)) {
+          this.translationFunctionNames.add(specifier.local.name);
+        }
+      }
+    }
+  }
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  isInsideTranslationCall(nodePath) {
+    let current = nodePath.parentPath;
+    while (current) {
+      const node = current.node;
+      if (t.isCallExpression(node)) {
+        const callee = node.callee;
+        if (t.isIdentifier(callee) && this.translationFunctionNames.has(callee.name)) {
+          return true;
+        }
+        if (t.isMemberExpression(callee) && t.isIdentifier(callee.property) && this.translationFunctionNames.has(callee.property.name)) {
+          return true;
+        }
+      }
+      current = current.parentPath;
+    }
+    return false;
+  }
+  addDetected(text, line, column, context, nodeType) {
+    const key = (0, import_shared.generateLocaleKey)(
+      this.options.filePath,
+      text,
+      this.options.sourceRoot || "src"
+    );
+    this.detectedTexts.push({
+      filePath: this.options.filePath,
+      line,
+      column,
+      text,
+      suggestedKey: key,
+      context,
+      nodeType,
+      alreadyTranslated: false
+    });
+  }
+  mapAttrToContext(attrName) {
+    const lower = attrName.toLowerCase();
+    if (lower === "placeholder") return "placeholder";
+    if (lower === "aria-label" || lower === "aria-placeholder") return "aria-label";
+    if (lower === "title") return "title";
+    if (lower === "alt") return "alt";
+    return "jsx-attribute";
+  }
+  regexFallbackScan() {
+    const results = [];
+    const jsxTextRegex = />([^<>{}\n]+)</g;
+    const lines = this.options.content.split("\n");
+    lines.forEach((line, idx) => {
+      let m;
+      jsxTextRegex.lastIndex = 0;
+      while ((m = jsxTextRegex.exec(line)) !== null) {
+        const text = (0, import_shared.normalizeText)(m[1]);
+        if (!(0, import_shared.isHumanReadableText)(text)) continue;
+        const key = (0, import_shared.generateLocaleKey)(this.options.filePath, text, this.options.sourceRoot || "src");
+        results.push({
+          filePath: this.options.filePath,
+          line: idx + 1,
+          column: m.index,
+          text,
+          suggestedKey: key,
+          context: "jsx-text",
+          nodeType: "regex-fallback",
+          alreadyTranslated: false
+        });
+      }
+    });
+    return results;
+  }
+};
+
+// src/asset-scanner.ts
+var fs = __toESM(require("fs"));
+var path = __toESM(require("path"));
+var import_shared2 = require("@ai-localize/shared");
+var CDN_URL_PATTERN = /https?:\/\/[a-zA-Z0-9\-.]+\.[a-zA-Z]{2,}\/[^\s"'`\)\]>]+/g;
+var CSS_URL_PATTERN = /url\(['"\s]?([^'")]+)['"\s]?\)/g;
+var IMPORT_ASSET_PATTERN = /import\s+\w+\s+from\s+['"]([^'"]+\.(png|jpg|jpeg|svg|webp|gif|ico|woff|woff2|ttf|eot|mp4))['"];?/gi;
+var SRC_ATTR_PATTERN = /(?:src|href)=["']([^"']+\.(png|jpg|jpeg|svg|webp|gif|ico|mp4))["']/gi;
+var AssetScanner = class {
+  legacyCdnPattern = null;
+  constructor(legacyCdnPattern) {
+    if (legacyCdnPattern) {
+      try {
+        this.legacyCdnPattern = new RegExp(legacyCdnPattern, "g");
+      } catch {
+      }
+    }
+  }
+  scanFile(filePath) {
+    const assets = [];
+    const legacyCdnUrls = [];
+    let content;
+    try {
+      content = fs.readFileSync(filePath, "utf-8");
+    } catch {
+      return { assets, legacyCdnUrls };
+    }
+    let m;
+    IMPORT_ASSET_PATTERN.lastIndex = 0;
+    while ((m = IMPORT_ASSET_PATTERN.exec(content)) !== null) {
+      const assetPath = m[1];
+      assets.push({
+        filePath,
+        line: this.getLineNumber(content, m.index),
+        assetPath,
+        assetType: this.getAssetType(assetPath),
+        referenceType: "import"
+      });
+    }
+    CSS_URL_PATTERN.lastIndex = 0;
+    while ((m = CSS_URL_PATTERN.exec(content)) !== null) {
+      const assetPath = m[1];
+      if (assetPath.startsWith("data:")) continue;
+      assets.push({
+        filePath,
+        line: this.getLineNumber(content, m.index),
+        assetPath,
+        assetType: this.getAssetType(assetPath),
+        referenceType: "css-url"
+      });
+    }
+    SRC_ATTR_PATTERN.lastIndex = 0;
+    while ((m = SRC_ATTR_PATTERN.exec(content)) !== null) {
+      assets.push({
+        filePath,
+        line: this.getLineNumber(content, m.index),
+        assetPath: m[1],
+        assetType: this.getAssetType(m[1]),
+        referenceType: "src-attr"
+      });
+    }
+    if (this.legacyCdnPattern) {
+      this.legacyCdnPattern.lastIndex = 0;
+      while ((m = this.legacyCdnPattern.exec(content)) !== null) {
+        const url = m[0];
+        legacyCdnUrls.push({
+          filePath,
+          line: this.getLineNumber(content, m.index),
+          url,
+          assetPath: this.extractPathFromUrl(url)
+        });
+      }
+    }
+    CDN_URL_PATTERN.lastIndex = 0;
+    while ((m = CDN_URL_PATTERN.exec(content)) !== null) {
+      const url = m[0];
+      if (!import_shared2.ASSET_EXTENSIONS.some((ext) => url.includes(`.${ext}`))) continue;
+      const line = this.getLineNumber(content, m.index);
+      if (!legacyCdnUrls.find((u) => u.url === url && u.line === line)) {
+        legacyCdnUrls.push({ filePath, line, url, assetPath: this.extractPathFromUrl(url) });
+      }
+    }
+    return { assets, legacyCdnUrls };
+  }
+  getLineNumber(content, index) {
+    return content.slice(0, index).split("\n").length;
+  }
+  getAssetType(assetPath) {
+    const ext = path.extname(assetPath).toLowerCase().replace(".", "");
+    return import_shared2.ASSET_EXTENSIONS.includes(ext) ? ext : "other";
+  }
+  extractPathFromUrl(url) {
+    try {
+      return new URL(url).pathname;
+    } catch {
+      return url;
+    }
+  }
+};
+
+// src/incremental-scanner.ts
+var fs2 = __toESM(require("fs"));
+var path2 = __toESM(require("path"));
+var crypto = __toESM(require("crypto"));
+var import_shared3 = require("@ai-localize/shared");
+var IncrementalScanCache = class {
+  cachePath;
+  cache;
+  constructor(cacheDir) {
+    (0, import_shared3.ensureDir)(cacheDir);
+    this.cachePath = path2.join(cacheDir, "scan-cache.json");
+    this.cache = this.load();
+  }
+  load() {
+    const existing = (0, import_shared3.readJsonSafe)(this.cachePath);
+    if (existing?.version === "1") return existing;
+    return { version: "1", lastRun: (/* @__PURE__ */ new Date()).toISOString(), fileHashes: {}, processedFiles: {} };
+  }
+  isFileChanged(filePath) {
+    return this.hashFile(filePath) !== this.cache.fileHashes[filePath];
+  }
+  getCachedResult(filePath) {
+    const entry = this.cache.processedFiles[filePath];
+    if (!entry) return null;
+    if (entry.hash !== this.hashFile(filePath)) return null;
+    return entry.detectedTexts;
+  }
+  setCachedResult(filePath, texts) {
+    const hash = this.hashFile(filePath);
+    this.cache.fileHashes[filePath] = hash;
+    this.cache.processedFiles[filePath] = { hash, detectedTexts: texts, lastModified: Date.now() };
+  }
+  persist() {
+    this.cache.lastRun = (/* @__PURE__ */ new Date()).toISOString();
+    (0, import_shared3.writeJson)(this.cachePath, this.cache);
+  }
+  hashFile(filePath) {
+    try {
+      return crypto.createHash("sha256").update(fs2.readFileSync(filePath)).digest("hex");
+    } catch {
+      return "";
+    }
+  }
+  clear() {
+    this.cache = { version: "1", lastRun: (/* @__PURE__ */ new Date()).toISOString(), fileHashes: {}, processedFiles: {} };
+    this.persist();
+  }
+};
+
+// src/project-scanner.ts
+var path3 = __toESM(require("path"));
+var os = __toESM(require("os"));
+var import_shared4 = require("@ai-localize/shared");
+var ProjectScanner = class {
+  config;
+  sourceRoot;
+  cache;
+  assetScanner;
+  constructor(config) {
+    this.config = config;
+    this.sourceRoot = path3.join(process.cwd(), config.sourceDir);
+    this.assetScanner = new AssetScanner(config.aws?.legacyCdnPattern);
+    if (config.incrementalCache) {
+      this.cache = new IncrementalScanCache(
+        path3.join(process.cwd(), config.cacheDir || ".ai-localize-cache")
+      );
+    }
+  }
+  async scan(options = {}) {
+    const startTime = Date.now();
+    const filesToScan = options.files?.length ? options.files : (0, import_shared4.collectFiles)(this.sourceRoot, import_shared4.SOURCE_EXTENSIONS, [
+      ...import_shared4.DEFAULT_IGNORE_DIRS,
+      ...this.config.ignorePatterns || []
+    ]);
+    const allTexts = [];
+    const allAssets = [];
+    const allLegacyUrls = [];
+    const chunkSize = Math.max(
+      1,
+      Math.min(50, Math.ceil(filesToScan.length / (os.cpus().length || 4)))
+    );
+    const chunks = this.chunkArray(filesToScan, chunkSize);
+    for (const chunk of chunks) {
+      const results = await Promise.all(chunk.map((f) => this.scanFile(f)));
+      for (const r of results) {
+        allTexts.push(...r.texts);
+        allAssets.push(...r.assets);
+        allLegacyUrls.push(...r.legacyUrls);
+      }
+    }
+    this.cache?.persist();
+    return {
+      framework: this.config.framework,
+      scannedFiles: filesToScan.length,
+      detectedTexts: allTexts,
+      assets: allAssets,
+      legacyCdnUrls: allLegacyUrls,
+      duration: Date.now() - startTime,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString()
+    };
+  }
+  async scanFile(filePath) {
+    if (this.cache && !this.cache.isFileChanged(filePath)) {
+      const cached = this.cache.getCachedResult(filePath);
+      if (cached) return { texts: cached, assets: [], legacyUrls: [] };
+    }
+    let content;
+    try {
+      const { readFileSync: readFileSync3 } = await import("fs");
+      content = readFileSync3(filePath, "utf-8");
+    } catch {
+      return { texts: [], assets: [], legacyUrls: [] };
+    }
+    const scanner = new AstScanner({ filePath, content, sourceRoot: this.config.sourceDir });
+    const texts = scanner.scan();
+    const { assets, legacyCdnUrls } = this.assetScanner.scanFile(filePath);
+    this.cache?.setCachedResult(filePath, texts);
+    return { texts, assets, legacyUrls: legacyCdnUrls };
+  }
+  chunkArray(array, size) {
+    const chunks = [];
+    for (let i = 0; i < array.length; i += size) {
+      chunks.push(array.slice(i, i + size));
+    }
+    return chunks;
+  }
+};
+
+// src/git-scanner.ts
+var import_child_process = require("child_process");
+var path4 = __toESM(require("path"));
+var GitScanner = class {
+  cwd;
+  constructor(cwd = process.cwd()) {
+    this.cwd = cwd;
+  }
+  getStagedFiles(extensions = ["ts", "tsx", "js", "jsx", "vue"]) {
+    try {
+      const out = (0, import_child_process.execSync)("git diff --cached --name-only --diff-filter=ACM", {
+        cwd: this.cwd,
+        encoding: "utf-8"
+      });
+      return this.filter(out.trim().split("\n"), extensions);
+    } catch {
+      return [];
+    }
+  }
+  getChangedFiles(base = "main", extensions = ["ts", "tsx", "js", "jsx", "vue"]) {
+    try {
+      const out = (0, import_child_process.execSync)(`git diff --name-only --diff-filter=ACM ${base}...HEAD`, {
+        cwd: this.cwd,
+        encoding: "utf-8"
+      });
+      return this.filter(out.trim().split("\n"), extensions);
+    } catch {
+      return [];
+    }
+  }
+  getRecentlyChangedFiles(commits = 1, extensions = ["ts", "tsx", "js", "jsx", "vue"]) {
+    try {
+      const out = (0, import_child_process.execSync)(
+        `git diff --name-only --diff-filter=ACM HEAD~${commits}...HEAD`,
+        { cwd: this.cwd, encoding: "utf-8" }
+      );
+      return this.filter(out.trim().split("\n"), extensions);
+    } catch {
+      return [];
+    }
+  }
+  filter(files, extensions) {
+    return files.filter((f) => f && extensions.some((e) => f.endsWith(`.${e}`))).map((f) => path4.join(this.cwd, f));
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AssetScanner,
+  AstScanner,
+  GitScanner,
+  IncrementalScanCache,
+  ProjectScanner
+});