@brickflow/cli 0.0.6 → 0.0.8

package/src/translate/ai.js

@@ -0,0 +1,168 @@
+import { generateContentWithLimits } from './gemini.js'
+import { getTranslateRuntimeConfig } from './runtime-config.js'
+
+const MODEL = process.env.TRANSLATE_AI_MODEL || 'gemini-3.1-flash-lite-preview'
+
+export async function translateBatch(strings, options) {
+  const { componentContext = '', sourceLocale = 'en', targetLocales = [] } = options ?? {}
+
+  if (!Array.isArray(strings) || strings.length === 0) {
+    return Object.fromEntries(targetLocales.map((locale) => [locale, {}]))
+  }
+
+  const contents = buildContents(strings, sourceLocale, targetLocales, componentContext)
+  const systemInstruction = buildSystemInstruction()
+  const maxRetries = 3
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      const raw = await requestTranslation(contents, systemInstruction)
+      const parsed = safeParse(raw)
+      return normalizeTranslations(parsed, strings, targetLocales)
+    } catch (error) {
+      console.warn(`AI translate attempt ${attempt}/${maxRetries} failed`)
+
+      if (attempt === maxRetries) {
+        throw new Error('Translation failed after retries', { cause: error })
+      }
+    }
+  }
+
+  return Object.fromEntries(targetLocales.map((locale) => [locale, {}]))
+}
+
+function buildContents(strings, sourceLocale, targetLocales, componentContext) {
+  const payload = strings.map((item) => ({
+    component: item.component || extractComponent(item.filePath),
+    key: item.key,
+    text: item.text,
+    type: detectType(item.text),
+  }))
+
+  return [
+    `Source locale: ${sourceLocale}`,
+    `Target locales: ${targetLocales.join(', ')}`,
+    '',
+    'Component context for translators:',
+    componentContext || 'No extra component context available.',
+    '',
+    'Return this exact schema:',
+    '{',
+    '  "pl": { "some.key": "..." },',
+    '  "ru": { "some.key": "..." }',
+    '}',
+    '',
+    'Input strings:',
+    JSON.stringify(payload, null, 2),
+  ].join('\n')
+}
+
+function buildSystemInstruction() {
+  const { productContext, terminology, tone } = getTranslateRuntimeConfig()
+
+  return [
+    'You are a professional localization engine for a paid adult content platform.',
+    '',
+    'PRODUCT CONTEXT:',
+    productContext,
+    '',
+    'TERMINOLOGY:',
+    terminology,
+    '',
+    'TONE:',
+    tone,
+    '',
+    'RULES:',
+    '- Return only raw JSON.',
+    '- Preserve JSON shape exactly.',
+    '- Do not omit keys or locales.',
+    '- Never remove words, qualifiers, examples, slang, or awkward fragments from the source.',
+    '- Preserve the full meaning of the source even if the text is clumsy, ungrammatical, explicit, or repetitive.',
+    '- If the source contains an unusual word or phrase, translate it or preserve it, but do not silently drop it.',
+    '- Preserve placeholders like {count}, {price}, %s, :name, \\n and HTML tags.',
+    '- Do not translate product or brand names unless source text clearly localizes them.',
+    '- Keep short labels concise only when the source itself is short. Do not compress longer messages.',
+    '- Example rule: "Username can contain from %s to %s characters only with dildo" must keep the final phrase in translation and must not be shortened.',
+  ].join('\n')
+}
+
+function cleanJson(text) {
+  return text
+    .replace(/```json/gi, '')
+    .replace(/```/g, '')
+    .trim()
+}
+
+function detectType(text) {
+  if (text.length <= 12) {
+    return 'short_ui'
+  }
+  if (text.includes('?')) {
+    return 'question'
+  }
+  if (text.includes('{') || text.includes('%')) {
+    return 'template'
+  }
+  return 'text'
+}
+
+function extractComponent(filePath) {
+  if (!filePath) {
+    return 'Unknown'
+  }
+
+  const parts = filePath.split('/').filter(Boolean)
+  const file = parts[parts.length - 2] || parts[parts.length - 1]
+
+  return file || 'Unknown'
+}
+
+function extractJson(text) {
+  const match = text.match(/\{[\s\S]*\}/)
+  return match ? match[0] : text
+}
+
+function normalizeTranslations(parsed, strings, targetLocales) {
+  const keys = strings.map((item) => item.key)
+  const result = {}
+
+  for (const locale of targetLocales) {
+    const localeValues = parsed?.[locale]
+
+    if (!localeValues || typeof localeValues !== 'object' || Array.isArray(localeValues)) {
+      throw new Error(`Invalid locale block: ${locale}`)
+    }
+
+    result[locale] = {}
+
+    for (const key of keys) {
+      const value = localeValues[key]
+
+      if (typeof value !== 'string') {
+        throw new Error(`Missing translation for ${locale}.${key}`)
+      }
+
+      result[locale][key] = value.trim()
+    }
+  }
+
+  return result
+}
+
+async function requestTranslation(contents, systemInstruction) {
+  const { apiKey } = getTranslateRuntimeConfig()
+
+  return await generateContentWithLimits({
+    apiKey,
+    config: {
+      systemInstruction,
+    },
+    contents,
+    model: MODEL,
+  })
+}
+
+function safeParse(text) {
+  const cleaned = extractJson(cleanJson(text))
+  return JSON.parse(cleaned)
+}

package/src/translate/gemini.js

@@ -0,0 +1,142 @@
+import { GoogleGenAI } from '@google/genai'
+
+import { getModelRpm } from './models.js'
+
+const clientsByApiKey = new Map()
+const modelState = new Map()
+
+export async function generateContentWithLimits({ apiKey, config, contents, maxRetries = 5, model }) {
+  const client = getClient(apiKey)
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    await waitForModelSlot(model)
+
+    try {
+      const response = await client.models.generateContent({
+        config,
+        contents,
+        model,
+      })
+
+      markModelRequest(model)
+      return response.text ?? ''
+    } catch (error) {
+      markModelRequest(model)
+
+      if (!isRateLimitError(error)) {
+        throw error
+      }
+
+      const delayMs = getRetryDelayMs(error, model)
+      console.warn(
+        `Rate limited for ${model}, retrying in ${Math.ceil(delayMs / 1000)}s (${attempt}/${maxRetries})`,
+      )
+      await sleep(delayMs)
+
+      if (attempt === maxRetries) {
+        throw error
+      }
+    }
+  }
+
+  return ''
+}
+
+function getClient(apiKey) {
+  if (!apiKey) {
+    throw new Error('Gemini API key is required')
+  }
+
+  const cached = clientsByApiKey.get(apiKey)
+
+  if (cached) {
+    return cached
+  }
+
+  const client = new GoogleGenAI({ apiKey })
+  clientsByApiKey.set(apiKey, client)
+  return client
+}
+
+function getModelState(model) {
+  const current = modelState.get(model) ?? {
+    nextAllowedAt: 0,
+  }
+
+  modelState.set(model, current)
+  return current
+}
+
+function getRetryDelayMs(error, model) {
+  const retryFromDetails = parseRetryDelayFromDetails(error)
+
+  if (retryFromDetails !== null) {
+    return retryFromDetails
+  }
+
+  const rpm = Math.max(getModelRpm(model), 1)
+  return Math.ceil(60000 / rpm)
+}
+
+function isRateLimitError(error) {
+  return error?.status === 429
+}
+
+function markModelRequest(model) {
+  const state = getModelState(model)
+  const rpm = Math.max(getModelRpm(model), 1)
+  const intervalMs = Math.ceil(60000 / rpm)
+
+  state.nextAllowedAt = Math.max(state.nextAllowedAt, Date.now()) + intervalMs
+}
+
+function parseDurationMs(value) {
+  const match = String(value).match(/^([\d.]+)s$/i)
+
+  if (!match) {
+    return null
+  }
+
+  return Math.ceil(Number.parseFloat(match[1]) * 1000)
+}
+
+function parseRetryDelayFromDetails(error) {
+  const details = error?.errorInfo?.details ?? error?.details ?? error?.message
+
+  if (Array.isArray(details)) {
+    for (const detail of details) {
+      const retryDelay = detail?.retryDelay
+
+      if (typeof retryDelay === 'string') {
+        const parsed = parseDurationMs(retryDelay)
+
+        if (parsed !== null) {
+          return parsed
+        }
+      }
+    }
+  }
+
+  if (typeof details === 'string') {
+    const match = details.match(/retry in ([\d.]+)s/i) || details.match(/"retryDelay":"(\d+)s"/i)
+
+    if (match) {
+      return Math.ceil(Number.parseFloat(match[1]) * 1000)
+    }
+  }
+
+  return null
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+async function waitForModelSlot(model) {
+  const state = getModelState(model)
+  const delayMs = state.nextAllowedAt - Date.now()
+
+  if (delayMs > 0) {
+    await sleep(delayMs)
+  }
+}

package/src/translate/index.js

@@ -0,0 +1,331 @@
+import fs from 'fs'
+import { globSync } from 'glob'
+import { dirname, join, relative, resolve } from 'path'
+import { fileURLToPath } from 'url'
+
+import { getAiContextState } from './ai-context.js'
+import { translateBatch } from './ai.js'
+import { buildTranslateHelp, parseTranslateRuntimeArgs, setTranslateRuntimeConfig } from './runtime-config.js'
+import { listTranslationTargets, sortObjectKeys, stringifySortedJson } from './utils.js'
+
+const currentDir = dirname(fileURLToPath(import.meta.url))
+const workspaceRoot = resolve(currentDir, '../../../..')
+const rawArgs = process.argv.slice(3)
+
+if (rawArgs.includes('--help') || rawArgs.includes('-h')) {
+  console.log(buildTranslateHelp())
+  process.exit(0)
+}
+
+const { options } = parseTranslateRuntimeArgs(rawArgs)
+
+try {
+  setTranslateRuntimeConfig(options)
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error))
+  console.error('')
+  console.error(buildTranslateHelp())
+  process.exit(1)
+}
+
+const DEFAULT_LANGUAGE_CODES = [
+  'bn',
+  'cz',
+  'dk',
+  'de',
+  'en',
+  'es',
+  'fi',
+  'fr',
+  'hi',
+  'hu',
+  'it',
+  'ja',
+  'nl',
+  'no',
+  'pl',
+  'pt',
+  'ru',
+  'si',
+  'se',
+  'sk',
+]
+const BATCH_MAX_ITEMS = readPositiveInt('TRANSLATE_BATCH_MAX_ITEMS', 30)
+const BATCH_MAX_CHARS = readPositiveInt('TRANSLATE_BATCH_MAX_CHARS', 3500)
+const languagePaths = globSync(
+  '{packages/brick,apps/*}/{components/**/translate,pages-translate/*,layouts/**,global/*}/generated/*.json',
+  {
+    absolute: true,
+    cwd: workspaceRoot,
+    ignore: ['**/node_modules/**', '**/.nuxt/**', '**/dist/**', '**/.output/**', '**/coverage/**', '**/public/**'],
+  },
+).sort()
+
+const languageCodes = [
+  ...new Set([
+    ...languagePaths.map((filePath) => filePath.replace(/.*\/([^/]+)\.json$/, '$1')),
+    ...DEFAULT_LANGUAGE_CODES,
+  ]),
+].sort()
+
+const tasks = listTranslationTargets(workspaceRoot).map(({ samplePath, sourceFilePath }) => ({
+  sample: readJson(samplePath),
+  samplePath,
+  sourceFilePath,
+}))
+
+const total = tasks.reduce((count, task) => count + Object.keys(task.sample).length * languageCodes.length, 0)
+const progress = createProgress(total)
+
+if (tasks.length === 0) {
+  console.log('✅ No translation sample folders found')
+  process.exit(0)
+}
+
+for (const task of tasks) {
+  await processSample(task)
+}
+
+process.stdout.write('\n')
+console.log(
+  `✅ Done: ${tasks.length} sample folders, ${languageCodes.length} languages, batch=${BATCH_MAX_ITEMS}/${BATCH_MAX_CHARS}`,
+)
+
+function createProgress(totalCount) {
+  let done = 0
+  let lastRenderedAt = 0
+  const start = Date.now()
+
+  return function update(languageCode, samplePath) {
+    done += 1
+
+    const now = Date.now()
+    if (done !== totalCount && now - lastRenderedAt < 80) {
+      return
+    }
+
+    lastRenderedAt = now
+
+    const percent = totalCount === 0 ? 100 : Math.round((done * 100) / totalCount)
+    const filled = Math.round(percent / 5)
+    const empty = 20 - filled
+    const elapsed = ((now - start) / 1000).toFixed(1)
+    const shortName = shortenPath(relative(workspaceRoot, samplePath))
+
+    process.stdout.write(
+      `\r🌍 Translation: [${'█'.repeat(filled)}${' '.repeat(empty)}] ` +
+        `${percent}% (${done}/${totalCount}) ` +
+        `⏱ ${elapsed}s ` +
+        `\x1b[90m${languageCode} ${shortName}\x1b[0m\x1b[K`,
+    )
+  }
+}
+
+function detectEol(filePath) {
+  if (!fs.existsSync(filePath)) {
+    return '\n'
+  }
+
+  const text = fs.readFileSync(filePath, 'utf-8')
+  return text.includes('\r\n') ? '\r\n' : '\n'
+}
+
+function estimateEntryChars(entry) {
+  return String(entry.key).length + String(entry.text).length + String(entry.filePath ?? '').length + 32
+}
+
+function existsJson(filePath) {
+  return fs.existsSync(filePath)
+}
+
+function normalizeEol(content) {
+  return String(content).replace(/\r\n/g, '\n')
+}
+
+async function processSample({ sample, samplePath, sourceFilePath }) {
+  writeSortedJsonIfNeeded(samplePath, sample)
+
+  const generatedDir = join(dirname(samplePath), 'generated')
+  const enPath = join(generatedDir, 'en.json')
+  const currentEn = existsJson(enPath) ? readJson(enPath) : {}
+  const currentByLanguage = new Map(
+    languageCodes.map((languageCode) => [
+      languageCode,
+      existsJson(join(generatedDir, `${languageCode}.json`))
+        ? readJson(join(generatedDir, `${languageCode}.json`))
+        : {},
+    ]),
+  )
+  const resultByLanguage = new Map(languageCodes.map((languageCode) => [languageCode, {}]))
+  const pendingEntriesByLocales = new Map()
+
+  fs.mkdirSync(generatedDir, { recursive: true })
+
+  for (const [key, sampleValue] of Object.entries(sample)) {
+    const missingLocales = []
+
+    for (const languageCode of languageCodes) {
+      const languageResult = resultByLanguage.get(languageCode)
+
+      if (!languageResult) {
+        continue
+      }
+
+      if (languageCode === 'en' || typeof sampleValue !== 'string') {
+        languageResult[key] = sampleValue
+        progress(languageCode, samplePath)
+        continue
+      }
+
+      const currentLang = currentByLanguage.get(languageCode) ?? {}
+
+      if (typeof currentLang[key] === 'string' && currentEn[key] === sampleValue) {
+        languageResult[key] = currentLang[key]
+        progress(languageCode, samplePath)
+        continue
+      }
+
+      missingLocales.push(languageCode)
+    }
+
+    if (typeof sampleValue === 'string' && missingLocales.length > 0) {
+      const localeKey = missingLocales.join(',')
+      const entries = pendingEntriesByLocales.get(localeKey) ?? []
+
+      entries.push({
+        filePath: relative(workspaceRoot, samplePath),
+        key,
+        text: sampleValue,
+      })
+
+      pendingEntriesByLocales.set(localeKey, entries)
+    }
+  }
+
+  let componentContext = null
+
+  if (pendingEntriesByLocales.size > 0) {
+    componentContext = getAiContextState({
+      samplePath,
+      sourceFilePath,
+    }).description
+  }
+
+  for (const [localeKey, entries] of pendingEntriesByLocales) {
+    const targetLocales = localeKey.split(',').filter(Boolean)
+    const chunks = splitIntoBatches(entries, BATCH_MAX_ITEMS, BATCH_MAX_CHARS)
+
+    for (const chunk of chunks) {
+      const translations = await translateChunk(chunk, samplePath, targetLocales, componentContext)
+
+      for (const languageCode of targetLocales) {
+        const languageResult = resultByLanguage.get(languageCode)
+        const localizedValues = translations[languageCode]
+
+        if (!languageResult || !localizedValues) {
+          throw new Error(
+            `Missing locale "${languageCode}" in AI response for ${relative(workspaceRoot, samplePath)}`,
+          )
+        }
+
+        for (const entry of chunk) {
+          const translatedValue = localizedValues[entry.key]
+
+          if (typeof translatedValue !== 'string' || translatedValue.length === 0) {
+            throw new Error(
+              `Missing translation for ${languageCode} ${relative(workspaceRoot, samplePath)} :: ${entry.key}`,
+            )
+          }
+
+          languageResult[entry.key] = translatedValue
+          progress(languageCode, samplePath)
+        }
+      }
+    }
+  }
+
+  for (const languageCode of languageCodes) {
+    const generatedPath = join(generatedDir, `${languageCode}.json`)
+    const languageResult = sortObjectKeys(resultByLanguage.get(languageCode) ?? {})
+    writeTextPreservingEol(generatedPath, stringifySortedJson(languageResult))
+  }
+}
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf-8'))
+}
+
+function readPositiveInt(name, fallback) {
+  const raw = process.env[name]
+
+  if (!raw) {
+    return fallback
+  }
+
+  const value = Number.parseInt(raw, 10)
+  return Number.isFinite(value) && value > 0 ? value : fallback
+}
+
+function shortenPath(filePath, maxLength = 72) {
+  if (filePath.length <= maxLength) {
+    return filePath
+  }
+
+  return `...${filePath.slice(-(maxLength - 3))}`
+}
+
+function splitIntoBatches(entries, maxItems, maxChars) {
+  const batches = []
+  let current = []
+  let currentChars = 0
+
+  for (const entry of entries) {
+    const entryChars = estimateEntryChars(entry)
+    const shouldFlush = current.length > 0 && (current.length >= maxItems || currentChars + entryChars > maxChars)
+
+    if (shouldFlush) {
+      batches.push(current)
+      current = []
+      currentChars = 0
+    }
+
+    current.push(entry)
+    currentChars += entryChars
+  }
+
+  if (current.length > 0) {
+    batches.push(current)
+  }
+
+  return batches
+}
+
+async function translateChunk(entries, samplePath, targetLocales, componentContext) {
+  try {
+    return await translateBatch(entries, {
+      componentContext,
+      sourceLocale: 'en',
+      targetLocales,
+    })
+  } catch (error) {
+    throw new Error(
+      `Translation failed for ${relative(workspaceRoot, samplePath)} (${entries.length} strings, ${targetLocales.join(', ')})`,
+      { cause: error },
+    )
+  }
+}
+
+function writeSortedJsonIfNeeded(filePath, value) {
+  const nextText = stringifySortedJson(value)
+  const prevText = fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf-8') : null
+
+  if (prevText === null || normalizeEol(prevText) !== normalizeEol(nextText)) {
+    writeTextPreservingEol(filePath, nextText)
+  }
+}
+
+function writeTextPreservingEol(filePath, content) {
+  const eol = detectEol(filePath)
+  const normalized = String(content).replace(/\r?\n/g, eol)
+  fs.writeFileSync(filePath, normalized, 'utf-8')
+}

package/src/translate/models.js

@@ -0,0 +1,12 @@
+export const GEMINI_MODEL_LIMITS = {
+  'gemini-3-flash-preview': {
+    rpm: 5,
+  },
+  'gemini-3.1-flash-lite-preview': {
+    rpm: 15,
+  },
+}
+
+export function getModelRpm(model) {
+  return GEMINI_MODEL_LIMITS[model]?.rpm ?? 5
+}