@scoutello/i18n-magic 0.64.0 → 0.66.0

package/src/commands/scan.ts

@@ -1,8 +1,7 @@
 import type { Configuration } from "../lib/types.js"
 import {
-  checkAllKeysExist,
-  findExistingTranslations,
-  getMissingKeys,
+  getKeysWithNamespaces,
+  getPureKey,
   getTextInput,
   loadLocalesFile,
   translateKey,
@@ -10,11 +9,15 @@ import {
 } from "../lib/utils.js"
 import { findUnusedKeys, removeUnusedKeys } from "./clean.js"
 
+const fileKey = (locale: string, namespace: string) =>
+  JSON.stringify([locale, namespace])
+
 export const translateMissing = async (config: Configuration) => {
   const {
     loadPath,
     savePath,
     defaultLocale,
+    defaultNamespace,
     namespaces,
     locales,
     context,
@@ -22,138 +25,157 @@ export const translateMissing = async (config: Configuration) => {
     disableTranslationDuringScan,
     autoClear,
   } = config
-
-  if (autoClear) {
-    console.log("🧹 Checking for unused translations before scanning...")
-    const report = await findUnusedKeys(config)
-
-    if (report.unusedCount > 0) {
-      await removeUnusedKeys(config)
-      console.log("")
-    } else {
-      console.log("✅ No unused keys found.\n")
+  const activeLocales = disableTranslationDuringScan
+    ? [defaultLocale]
+    : locales
+  const keysWithNamespaces = await getKeysWithNamespaces({
+    globPatterns: config.globPatterns,
+    defaultNamespace,
+  })
+  const expectedByNamespace = Object.fromEntries(
+    namespaces.map((namespace) => [namespace, new Set<string>()]),
+  ) as Record<string, Set<string>>
+
+  for (const { key, namespaces: keyNamespaces } of keysWithNamespaces) {
+    for (const namespace of keyNamespaces) {
+      if (!expectedByNamespace[namespace]) continue
+      const pureKey = getPureKey(
+        key,
+        namespace,
+        namespace === defaultNamespace,
+      )
+      const expectedKey = pureKey || (!key.includes(":") ? key : null)
+      if (expectedKey !== null) expectedByNamespace[namespace].add(expectedKey)
     }
   }
 
-  const newKeys = await getMissingKeys(config)
-
-  if (newKeys.length === 0) {
-    console.log("No new keys found.")
-
-    await checkAllKeysExist(config)
-
-    return
-  }
-
-  console.log(
-    `${newKeys.length} keys are missing. Please provide the values for the following keys in ${defaultLocale}:`,
-  )
-
-  const newKeysWithDefaultLocale = []
-
-  // Check for existing translations in parallel
-  const keysList = newKeys.map((k) => k.key)
-  const existingTranslationResults = await findExistingTranslations(
-    keysList,
-    namespaces,
-    defaultLocale,
-    loadPath,
+  const stagedFiles = new Map<string, Record<string, string>>()
+  await Promise.all(
+    activeLocales.flatMap((locale) =>
+      namespaces.map(async (namespace) => {
+        const translations = await loadLocalesFile(
+          loadPath,
+          locale,
+          namespace,
+          { silent: true },
+        )
+        stagedFiles.set(fileKey(locale, namespace), { ...translations })
+      }),
+    ),
   )
 
-  const reusedKeys: string[] = []
-  for (const newKey of newKeys) {
-    const existingValue = existingTranslationResults[newKey.key]
-
-    let answer: string
-    // Use explicit null check instead of truthy check to handle empty string values
-    if (existingValue !== null) {
-      reusedKeys.push(newKey.key)
-      answer = existingValue
-    } else {
-      answer = await getTextInput(newKey.key, newKey.namespaces)
+  const changedFiles = new Set<string>()
+  const missingDefaultKeys = new Map<string, Set<string>>()
+  for (const namespace of namespaces) {
+    const translations = stagedFiles.get(fileKey(defaultLocale, namespace))!
+    for (const key of expectedByNamespace[namespace]) {
+      if (!Object.hasOwn(translations, key)) {
+        missingDefaultKeys.get(key)?.add(namespace) ??
+          missingDefaultKeys.set(key, new Set([namespace]))
+      }
     }
-
-    newKeysWithDefaultLocale.push({
-      key: newKey.key,
-      namespace: newKey.namespace,
-      namespaces: newKey.namespaces,
-      value: answer,
-    })
   }
 
-  // Batch log reused keys
-  if (reusedKeys.length > 0) {
+  if (missingDefaultKeys.size > 0) {
     console.log(
-      `🔄 Auto-reused ${reusedKeys.length} existing values from other namespaces`,
+      `${missingDefaultKeys.size} keys are missing. Please provide the values for the following keys in ${defaultLocale}:`,
     )
   }
 
-  const newKeysObject = newKeysWithDefaultLocale.reduce((prev, next) => {
-    prev[next.key] = next.value
+  for (const [key, keyNamespaces] of missingDefaultKeys) {
+    let value: string | undefined
+    for (const namespace of namespaces) {
+      const translations = stagedFiles.get(fileKey(defaultLocale, namespace))!
+      if (Object.hasOwn(translations, key)) {
+        value = translations[key]
+        break
+      }
+    }
 
-    return prev
-  }, {})
+    value ??= await getTextInput(key, Array.from(keyNamespaces))
+    for (const namespace of keyNamespaces) {
+      stagedFiles.get(fileKey(defaultLocale, namespace))![key] = value
+      changedFiles.add(fileKey(defaultLocale, namespace))
+    }
+  }
 
-  const allLocales = disableTranslationDuringScan ? [defaultLocale] : locales
+  await Promise.all(
+    activeLocales
+      .filter((locale) => locale !== defaultLocale)
+      .map(async (locale) => {
+        const missingByKey = new Map<string, Set<string>>()
+        for (const namespace of namespaces) {
+          const translations = stagedFiles.get(fileKey(locale, namespace))!
+          for (const key of expectedByNamespace[namespace]) {
+            if (!Object.hasOwn(translations, key)) {
+              missingByKey.get(key)?.add(namespace) ??
+                missingByKey.set(key, new Set([namespace]))
+            }
+          }
+        }
 
-  // Batch translate for all non-default locales in parallel
-  const translationCache: Record<string, Record<string, string>> = {
-    [defaultLocale]: newKeysObject,
-  }
+        const reused: Record<string, string> = {}
+        const keysToTranslate: Record<string, string> = {}
+        for (const key of missingByKey.keys()) {
+          let existingValue: string | undefined
+          for (const namespace of namespaces) {
+            const translations = stagedFiles.get(fileKey(locale, namespace))!
+            if (Object.hasOwn(translations, key)) {
+              existingValue = translations[key]
+              break
+            }
+          }
+
+          if (existingValue !== undefined) {
+            reused[key] = existingValue
+            continue
+          }
+
+          const sourceNamespace = Array.from(missingByKey.get(key)!)[0]
+          const sourceTranslations = stagedFiles.get(
+            fileKey(defaultLocale, sourceNamespace),
+          )!
+          keysToTranslate[key] = sourceTranslations[key]
+        }
 
-  const nonDefaultLocales = allLocales.filter((l) => l !== defaultLocale)
-  if (nonDefaultLocales.length > 0) {
-    await Promise.all(
-      nonDefaultLocales.map(async (locale) => {
-        const translatedValues = await translateKey({
+        const translated = await translateKey({
           inputLanguage: defaultLocale,
           outputLanguage: locale,
           context,
-          object: newKeysObject,
+          object: keysToTranslate,
           openai,
           model: config.model,
         })
-        translationCache[locale] = translatedValues
-      }),
-    )
-  }
-
-  // Process all locale/namespace combinations in parallel
-  const writeResults: Array<{
-    locale: string
-    namespace: string
-    keyCount: number
-  }> = []
-  await Promise.all(
-    allLocales.flatMap((locale) =>
-      namespaces.map(async (namespace) => {
-        const existingKeys = await loadLocalesFile(loadPath, locale, namespace)
-
-        const relevantKeys = newKeysWithDefaultLocale.filter((key) =>
-          key.namespaces?.includes(namespace),
-        )
-
-        if (relevantKeys.length === 0) {
-          return
-        }
-
-        const translatedValues = translationCache[locale]
-        for (const key of relevantKeys) {
-          existingKeys[key.key] = translatedValues[key.key]
+        const resolvedValues = { ...reused, ...translated }
+
+        for (const [key, keyNamespaces] of missingByKey) {
+          for (const namespace of keyNamespaces) {
+            stagedFiles.get(fileKey(locale, namespace))![key] =
+              resolvedValues[key]
+            changedFiles.add(fileKey(locale, namespace))
+          }
         }
-
-        await writeLocalesFile(savePath, locale, namespace, existingKeys)
-        writeResults.push({ locale, namespace, keyCount: relevantKeys.length })
       }),
-    ),
   )
 
-  // Log where keys were written
-  for (const { locale, namespace, keyCount } of writeResults) {
-    console.log(`   📝 Wrote ${keyCount} key(s) to ${locale}/${namespace}.json`)
+  for (const locale of activeLocales) {
+    for (const namespace of namespaces) {
+      const key = fileKey(locale, namespace)
+      if (!changedFiles.has(key)) continue
+      await writeLocalesFile(savePath, locale, namespace, stagedFiles.get(key)!)
+      console.log(`   📝 Updated ${locale}/${namespace}.json`)
+    }
   }
 
-  await checkAllKeysExist(config)
+  if (autoClear) {
+    console.log("🧹 Checking for unused translations after scanning...")
+    const report = await findUnusedKeys(config)
+    if (report.unusedCount > 0) await removeUnusedKeys(config)
+  }
 
-  console.log(`Successfully translated ${newKeys.length} keys.`)
+  if (changedFiles.size === 0) {
+    console.log("No missing translations found.")
+  } else {
+    console.log(`Successfully updated ${changedFiles.size} locale file(s).`)
+  }
 }

package/src/commands/sync-locales.ts

@@ -1,10 +1,8 @@
-import fs from "node:fs"
 import chalk from "chalk"
 import cliProgress from "cli-progress"
 import { languages } from "../lib/languges.js"
 import type { Configuration } from "../lib/types.js"
 import {
-  findExistingTranslations,
   loadLocalesFile,
   TranslationError,
   translateKey,
@@ -51,34 +49,52 @@ export const syncLocales = async (config: Configuration) => {
   const localeResults: Record<
     string,
     {
-      status: "pending" | "processing" | "done" | "error"
+      status: "pending" | "processing" | "translating" | "done" | "error"
       translated: number
+      translationCompleted: number
+      translationTotal: number
       reused: number
       error?: string
     }
   > = {}
 
   for (const locale of localesToProcess) {
-    localeResults[locale] = { status: "pending", translated: 0, reused: 0 }
+    localeResults[locale] = {
+      status: "pending",
+      translated: 0,
+      translationCompleted: 0,
+      translationTotal: 0,
+      reused: 0,
+    }
   }
 
-  // Create a single progress bar for overall progress
-  const progressBar = new cliProgress.SingleBar(
+  const stagedFiles = new Map<
+    string,
+    { locale: string; namespace: string; translations: Record<string, string> }
+  >()
+
+  const progressBars = new cliProgress.MultiBar(
     {
-      format: `   {bar} {percentage}% | {current}/{total} | {lang} {status}`,
+      format: `   {label} {bar} {percentage}% | {value}/{total} | {duration_formatted} | {status}`,
       barCompleteChar: chalk.green("█"),
       barIncompleteChar: chalk.gray("░"),
       hideCursor: true,
       clearOnComplete: false,
+      emptyOnZero: true,
+      etaAsynchronousUpdate: true,
       barsize: 30,
     },
     cliProgress.Presets.shades_classic,
   )
 
-  progressBar.start(localesToProcess.length, 0, {
+  const localeProgressBar = progressBars.create(localesToProcess.length, 0, {
+    label: "Languages   ",
     status: "",
-    current: 0,
-    lang: "",
+  })
+
+  const translationProgressBar = progressBars.create(0, 0, {
+    label: "Translations",
+    status: "finding keys to translate...",
   })
 
   // Suppress console.log during progress bar display
@@ -86,63 +102,72 @@ export const syncLocales = async (config: Configuration) => {
   const suppressedLog = () => {}
 
   try {
-    // Helper to ensure a locale/namespace file exists before we add keys
-    const ensureLocaleNamespaceFile = async (
-      locale: string,
-      namespace: string,
-    ) => {
-      if (typeof savePath === "string") {
-        const filePath = savePath
-          .replace("{{lng}}", locale)
-          .replace("{{ns}}", namespace)
-        if (!fs.existsSync(filePath)) {
-          await writeLocalesFile(savePath, locale, namespace, {})
-        }
-      }
-    }
-
-    // Ensure all default-locale namespace files exist first
-    await Promise.all(
-      namespaces.map((namespace) =>
-        ensureLocaleNamespaceFile(defaultLocale, namespace),
-      ),
-    )
-
     // Process locales in parallel batches
     let completedCount = 0
     const currentlyProcessing: Set<string> = new Set()
 
-    // Helper to update progress bar with currently processing languages
+    // Helper to update progress bars with currently processing languages
     const updateProgressDisplay = () => {
       const processingArray = Array.from(currentlyProcessing)
-      const langs = processingArray
-        .slice(0, 3)
-        .map((l) => getLanguageLabel(l).substring(0, 8))
-        .join(", ")
-      const status = processingArray.length > 0 ? `processing ${langs}...` : ""
-      progressBar.update(completedCount, {
-        status,
-        current: completedCount,
-        lang: "",
+      const localeStatus =
+        processingArray.length > 0
+          ? `processing ${processingArray.join(", ")}`
+          : ""
+      localeProgressBar.update(completedCount, {
+        label: "Languages   ",
+        status: localeStatus,
+      })
+
+      const translationTotals = localesToProcess.reduce(
+        (totals, locale) => {
+          const result = localeResults[locale]
+          totals.completed += result.translationCompleted
+          totals.total += result.translationTotal
+          return totals
+        },
+        { completed: 0, total: 0 },
+      )
+      const translationTotal = translationTotals.total
+      const translationCurrent =
+        translationTotal > 0
+          ? Math.min(translationTotals.completed, translationTotal)
+          : 0
+      const activeTranslations = localesToProcess.filter((locale) => {
+        const result = localeResults[locale]
+        return (
+          currentlyProcessing.has(locale) &&
+          result.translationTotal > 0 &&
+          result.translationCompleted < result.translationTotal
+        )
+      })
+      const translationStatus =
+        activeTranslations.length > 0
+          ? `translating ${activeTranslations
+              .map((locale) => {
+                const result = localeResults[locale]
+                return `${locale} ${result.translationCompleted}/${result.translationTotal}`
+              })
+              .join(", ")}`
+          : processingArray.length > 0
+            ? "waiting for translation work..."
+            : translationTotal > 0
+              ? "translations complete"
+              : "finding keys to translate..."
+
+      translationProgressBar.setTotal(translationTotal)
+      translationProgressBar.update(translationCurrent, {
+        label: "Translations",
+        status: translationStatus,
       })
     }
 
     // Process a single locale
     const processLocale = async (locale: string) => {
-      const langLabel = getLanguageLabel(locale)
-      const shortLang = langLabel.substring(0, 10).padEnd(10)
       localeResults[locale].status = "processing"
       currentlyProcessing.add(locale)
       updateProgressDisplay()
 
       try {
-        // Ensure all namespace files for this locale exist
-        await Promise.all(
-          namespaces.map((namespace) =>
-            ensureLocaleNamespaceFile(locale, namespace),
-          ),
-        )
-
         // Collect all missing keys for this locale across all namespaces
         const allMissingKeys: Record<
           string,
@@ -181,7 +206,7 @@ export const syncLocales = async (config: Configuration) => {
           namespaceKeys[namespace] = localeKeys
 
           for (const [key, value] of Object.entries(defaultLocaleKeys)) {
-            if (!localeKeys[key]) {
+            if (!Object.hasOwn(localeKeys, key)) {
               if (allMissingKeys[key]) {
                 allMissingKeys[key].namespaces.push(namespace)
               } else {
@@ -208,18 +233,12 @@ export const syncLocales = async (config: Configuration) => {
         const keysToTranslate: Record<string, string> = {}
         const existingTranslations: Record<string, string> = {}
 
-        const existingTranslationResults = await findExistingTranslations(
-          missingKeysList,
-          namespaces,
-          locale,
-          loadPath,
-          { silent: true },
-        )
-
         for (const key of missingKeysList) {
-          const existingValue = existingTranslationResults[key]
-          if (existingValue !== null) {
-            existingTranslations[key] = existingValue
+          const existingNamespace = namespaces.find((namespace) =>
+            Object.hasOwn(namespaceKeys[namespace], key),
+          )
+          if (existingNamespace !== undefined) {
+            existingTranslations[key] = namespaceKeys[existingNamespace][key]
             localeResults[locale].reused++
           } else {
             keysToTranslate[key] = allMissingKeys[key].value
@@ -231,6 +250,12 @@ export const syncLocales = async (config: Configuration) => {
         // Translate if needed
         if (Object.keys(keysToTranslate).length > 0) {
           try {
+            localeResults[locale].status = "translating"
+            localeResults[locale].translationCompleted = 0
+            localeResults[locale].translationTotal =
+              Object.keys(keysToTranslate).length
+            updateProgressDisplay()
+
             translatedValues = await translateKey({
               inputLanguage: defaultLocale,
               outputLanguage: locale,
@@ -239,9 +264,13 @@ export const syncLocales = async (config: Configuration) => {
               openai,
               model: config.model,
               onProgress: (completed, total) => {
-                // Progress callback still works but doesn't update UI during parallel processing
+                localeResults[locale].translationCompleted = completed
+                localeResults[locale].translationTotal = total
+                updateProgressDisplay()
               },
             })
+            localeResults[locale].translationCompleted =
+              Object.keys(keysToTranslate).length
             localeResults[locale].translated =
               Object.keys(translatedValues).length
           } catch (error) {
@@ -258,24 +287,25 @@ export const syncLocales = async (config: Configuration) => {
         // Combine and save
         const allTranslations = { ...existingTranslations, ...translatedValues }
 
-        await Promise.all(
-          namespaces.map(async (namespace) => {
-            let hasChanges = false
-            const updatedKeys = { ...namespaceKeys[namespace] }
-
-            for (const key of missingKeysList) {
-              if (allMissingKeys[key].namespaces.includes(namespace)) {
-                const translation = allTranslations[key] || ""
-                updatedKeys[key] = translation
-                hasChanges = true
-              }
-            }
+        for (const namespace of namespaces) {
+          let hasChanges = false
+          const updatedKeys = { ...namespaceKeys[namespace] }
 
-            if (hasChanges) {
-              await writeLocalesFile(savePath, locale, namespace, updatedKeys)
+          for (const key of missingKeysList) {
+            if (allMissingKeys[key].namespaces.includes(namespace)) {
+              updatedKeys[key] = allTranslations[key]
+              hasChanges = true
             }
-          }),
-        )
+          }
+
+          if (hasChanges) {
+            stagedFiles.set(JSON.stringify([locale, namespace]), {
+              locale,
+              namespace,
+              translations: updatedKeys,
+            })
+          }
+        }
 
         localeResults[locale].status = "done"
         currentlyProcessing.delete(locale)
@@ -303,7 +333,20 @@ export const syncLocales = async (config: Configuration) => {
     // Restore console.log
     console.log = originalConsoleLog
 
-    progressBar.stop()
+    progressBars.stop()
+
+    const errorLocales = localesToProcess.filter(
+      (locale) => localeResults[locale].status === "error",
+    )
+    if (errorLocales.length > 0) {
+      throw new TranslationError(
+        `Sync failed for ${errorLocales.length} locale(s); no translations were written.`,
+      )
+    }
+
+    for (const { locale, namespace, translations } of stagedFiles.values()) {
+      await writeLocalesFile(savePath, locale, namespace, translations)
+    }
 
     // Print summary
     console.log("")
@@ -328,10 +371,6 @@ export const syncLocales = async (config: Configuration) => {
     const doneLocales = localesToProcess.filter(
       (l) => localeResults[l].status === "done",
     )
-    const errorLocales = localesToProcess.filter(
-      (l) => localeResults[l].status === "error",
-    )
-
     // Show successful languages in a compact grid
     if (doneLocales.length > 0) {
       const columns = 3
@@ -381,7 +420,7 @@ export const syncLocales = async (config: Configuration) => {
   } catch (error) {
     // Restore console.log
     console.log = originalConsoleLog
-    progressBar.stop()
+    progressBars.stop()
     if (error instanceof TranslationError) {
       throw error
     }

package/src/index.ts

@@ -1,5 +1,10 @@
 // Export command functions for programmatic usage
-export { checkMissing } from "./commands/check-missing.js"
+export {
+  checkMissing,
+  collectMissingTranslations,
+  MissingTranslationsError,
+} from "./commands/check-missing.js"
+export type { MissingTranslationsReport } from "./commands/check-missing.js"
 export { removeUnusedKeys } from "./commands/clean.js"
 
 export { replaceTranslation } from "./commands/replace.js"

package/src/lib/languges.ts

@@ -19,11 +19,21 @@ export const languages = [
     name: "French",
     value: "fr",
   },
+  {
+    label: "Dansk",
+    name: "Danish",
+    value: "da",
+  },
   {
     label: "Dansk",
     name: "Danish",
     value: "dk",
   },
+  {
+    label: "中文",
+    name: "Chinese",
+    value: "zh",
+  },
   {
     label: "中文",
     name: "Chinese",
@@ -144,4 +154,14 @@ export const languages = [
     name: "Thai",
     value: "th",
   },
+  {
+    label: "العربية",
+    name: "Arabic",
+    value: "ar",
+  },
+  {
+    label: "فارسی",
+    name: "Persian",
+    value: "fa",
+  },
 ]