locize-cli 11.0.0 → 12.0.0

package/dist/cjs/lngs.js

@@ -0,0 +1,215 @@
+'use strict';
+
+const lngCodes = [
+  'dev',
+  'ach',
+  'arn',
+  'ast',
+  'cgg',
+  'csb',
+  'dz',
+  'es_ar',
+  'es-ar',
+  'fil',
+  'fur',
+  'gun',
+  'jbo',
+  'mai',
+  'mfe',
+  'mnk',
+  'nah',
+  'nap',
+  'nso',
+  'pap',
+  'pms',
+  'pt_br',
+  'pt-br',
+  'sah',
+  'sco',
+  'son',
+  'ab',
+  'aa',
+  'af',
+  'ak',
+  'sq',
+  'am',
+  'ar',
+  'an',
+  'hy',
+  'as',
+  'av',
+  'ae',
+  'ay',
+  'az',
+  'bm',
+  'ba',
+  'eu',
+  'be',
+  'bn',
+  'bh',
+  'bi',
+  'bs',
+  'br',
+  'bg',
+  'my',
+  'ca',
+  'ch',
+  'ce',
+  'ny',
+  'zh',
+  'cv',
+  'kw',
+  'co',
+  'cr',
+  'hr',
+  'cs',
+  'da',
+  'dv',
+  'nl',
+  'en',
+  'eo',
+  'et',
+  'ee',
+  'fo',
+  'fj',
+  'fi',
+  'fr',
+  'ff',
+  'gl',
+  'ka',
+  'de',
+  'el',
+  'gn',
+  'gu',
+  'ht',
+  'ha',
+  'he',
+  'hz',
+  'hi',
+  'ho',
+  'hu',
+  'ia',
+  'id',
+  'ie',
+  'ga',
+  'ig',
+  'ik',
+  'io',
+  'is',
+  'it',
+  'iu',
+  'ja',
+  'jv',
+  'kl',
+  'kn',
+  'kr',
+  'ks',
+  'kk',
+  'km',
+  'ki',
+  'rw',
+  'ky',
+  'kv',
+  'kg',
+  'ko',
+  'ku',
+  'ckb',
+  'kj',
+  'la',
+  'lb',
+  'lg',
+  'li',
+  'ln',
+  'lo',
+  'lt',
+  'lu',
+  'lv',
+  'gv',
+  'mk',
+  'mg',
+  'ms',
+  'ml',
+  'mt',
+  'mi',
+  'mr',
+  'mh',
+  'mn',
+  'na',
+  'nv',
+  'nb',
+  'nd',
+  'ne',
+  'ng',
+  'nn',
+  'no',
+  'ii',
+  'nr',
+  'oc',
+  'oj',
+  'cu',
+  'om',
+  'or',
+  'os',
+  'pa',
+  'pi',
+  'fa',
+  'pl',
+  'ps',
+  'pt',
+  'qu',
+  'rm',
+  'rn',
+  'ro',
+  'ru',
+  'sa',
+  'sc',
+  'sd',
+  'se',
+  'sm',
+  'sg',
+  'sr',
+  'gd',
+  'sn',
+  'si',
+  'sk',
+  'sl',
+  'so',
+  'st',
+  'es',
+  'su',
+  'sw',
+  'ss',
+  'sv',
+  'ta',
+  'te',
+  'tg',
+  'th',
+  'ti',
+  'bo',
+  'tk',
+  'tl',
+  'tn',
+  'to',
+  'tr',
+  'ts',
+  'tt',
+  'tw',
+  'ty',
+  'ug',
+  'uk',
+  'ur',
+  'uz',
+  've',
+  'vi',
+  'vo',
+  'wa',
+  'cy',
+  'wo',
+  'fy',
+  'xh',
+  'yi',
+  'yo',
+  'za'
+];
+
+module.exports = lngCodes;

package/dist/cjs/mapLimit.js

@@ -0,0 +1,22 @@
+'use strict';
+
+async function mapLimit (arr, limit, asyncFn) {
+  const ret = [];
+  let i = 0;
+  let active = 0;
+  return new Promise((resolve, reject) => {
+    function next () {
+      if (i === arr.length && active === 0) return resolve(ret)
+      while (active < limit && i < arr.length) {
+        const cur = i++;
+        active++;
+        Promise.resolve(asyncFn(arr[cur], cur, arr))
+          .then((res) => { ret[cur] = res; active--; next(); })
+          .catch(reject);
+      }
+    }
+    next();
+  })
+}
+
+module.exports = mapLimit;

package/dist/cjs/mergeBranch.js

@@ -0,0 +1,80 @@
+'use strict';
+
+var colors = require('colors');
+var request = require('./request.js');
+var getBranches = require('./getBranches.js');
+var isValidUuid = require('./isValidUuid.js');
+var getJob = require('./getJob.js');
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+const merge = async (opt) => {
+  const queryParams = new URLSearchParams();
+  if (opt.delete) {
+    queryParams.append('delete', 'true');
+  }
+  const queryString = queryParams.size > 0 ? '?' + queryParams.toString() : '';
+  const { res, obj, err } = await request(opt.apiEndpoint + '/branch/merge/' + opt.branch + queryString, {
+    method: 'post',
+    headers: {
+      Authorization: opt.apiKey
+    }
+  });
+  if (err || (obj && (obj.errorMessage || obj.message))) {
+    console.log(colors.red('merging branch failed...'));
+    if (err) {
+      console.error(colors.red(err.message));
+      throw err
+    }
+    if (obj && (obj.errorMessage || obj.message)) {
+      console.error(colors.red((obj.errorMessage || obj.message)));
+      throw new Error((obj.errorMessage || obj.message))
+    }
+  }
+  if (res.status === 404) {
+    console.error(colors.yellow(res.statusText + ' (' + res.status + ')'));
+    return null
+  }
+  if (res.status >= 300) {
+    console.error(colors.red(res.statusText + ' (' + res.status + ')'));
+    throw new Error(res.statusText + ' (' + res.status + ')')
+  }
+  if (!obj || !obj.jobId) {
+    console.error(colors.red('No jobId! Something went wrong!'));
+    throw new Error('No jobId! Something went wrong!')
+  }
+  let job;
+  while (true) {
+    job = await getJob({
+      apiEndpoint: opt.apiEndpoint,
+      apiKey: opt.apiKey,
+      projectId: opt.branch
+    }, obj.jobId);
+    if (job && !job.timeouted) {
+      await sleep(2000);
+      continue
+    }
+    if (job && job.timeouted) {
+      console.error(colors.red('Job timeouted!'));
+      throw new Error('Job timeouted!')
+    }
+    break
+  }
+  console.log(colors.green('merging branch successful'));
+};
+
+// handleError removed (unused)
+
+const mergeBranch = async (opt) => {
+  const branches = await getBranches(opt);
+  let b;
+  if (isValidUuid(opt.branch)) b = branches.find((br) => br.id === opt.branch);
+  if (!b) b = branches.find((br) => br.name === opt.branch);
+  if (!b) {
+    throw new Error(`Branch ${opt.branch} not found!`)
+  }
+  opt.branch = b.id;
+  await merge(opt);
+};
+
+module.exports = mergeBranch;

package/dist/cjs/migrate.js

@@ -0,0 +1,239 @@
+'use strict';
+
+var fs = require('node:fs');
+var path = require('node:path');
+var flatten = require('flat');
+var colors = require('colors');
+var request = require('./request.js');
+var getRemoteLanguages = require('./getRemoteLanguages.js');
+var os = require('node:os');
+var mapLimit = require('./mapLimit.js');
+
+const getDirectories = (srcpath) => {
+  return fs.readdirSync(srcpath).filter(function (file) {
+    return fs.statSync(path.join(srcpath, file)).isDirectory()
+  })
+};
+
+const getFiles = (srcpath) => {
+  return fs.readdirSync(srcpath).filter(function (file) {
+    return !fs.statSync(path.join(srcpath, file)).isDirectory()
+  })
+};
+
+const load = async (namespaces) => {
+  await Promise.all(namespaces.map(async (ns) => {
+    try {
+      const data = await fs.promises.readFile(ns.path, 'utf8');
+      ns.value = flatten(JSON.parse(data));
+    } catch (err) {
+      console.error(colors.red(err.stack));
+      ns.value = {};
+    }
+  }));
+  return namespaces
+};
+
+const parseLanguage = async (p) => {
+  const dirs = getDirectories(p);
+  const namespaces = [];
+  dirs.forEach((lng) => {
+    const files = getFiles(path.join(p, lng));
+    files.forEach((file) => {
+      if (path.extname(file) !== '.json') return
+      namespaces.push({
+        language: lng,
+        namespace: path.basename(file, '.json'),
+        path: path.join(p, lng, file)
+      });
+    });
+  });
+  return await load(namespaces)
+};
+
+const transfer = async (opt, ns) => {
+  let url = `${opt.apiEndpoint}/update/{{projectId}}/{{version}}/{{lng}}/{{ns}}`
+    .replace('{{projectId}}', opt.projectId)
+    .replace('{{ver}}', opt.version)
+    .replace('{{version}}', opt.version)
+    .replace('{{language}}', ns.language)
+    .replace('{{lng}}', ns.language)
+    .replace('{{ns}}', ns.namespace)
+    .replace('{{namespace}}', ns.namespace);
+
+  console.log(colors.yellow(`transfering ${opt.version}/${ns.language}/${ns.namespace}...`));
+
+  if (!opt.replace) url = url.replace('/update/', '/missing/');
+
+  const data = ns.value;
+  const keysToSend = Object.keys(data).length;
+  if (keysToSend === 0) return
+
+  const payloadKeysLimit = 1000;
+
+  async function send (d, so, isFirst, isRetrying = false) {
+    const queryParams = new URLSearchParams();
+    if (so) {
+      queryParams.append('omitstatsgeneration', 'true');
+    }
+    if (isFirst && opt.replace) {
+      queryParams.append('replace', 'true');
+    }
+    const queryString = queryParams.size > 0 ? '?' + queryParams.toString() : '';
+    try {
+      const { res, obj } = await request(url + queryString, {
+        method: 'post',
+        body: d,
+        headers: {
+          Authorization: opt.apiKey
+        }
+      });
+      if (url.indexOf('/missing/') > -1 && res.status === 412) {
+        console.log(colors.green(`transfered ${Object.keys(d).length} keys ${opt.version}/${ns.language}/${ns.namespace} (but all keys already existed)...`));
+        return
+      }
+      if (res.status === 504 && !isRetrying) {
+        await new Promise(resolve => setTimeout(resolve, 3000));
+        return send(d, so, isFirst, true)
+      }
+      if (res.status >= 300 && res.status !== 412) {
+        if (obj && (obj.errorMessage || obj.message)) {
+          throw new Error((obj.errorMessage || obj.message))
+        }
+        throw new Error(res.statusText + ' (' + res.status + ')')
+      }
+      console.log(colors.green(`transfered ${Object.keys(d).length} keys ${opt.version}/${ns.language}/${ns.namespace}...`));
+    } catch (err) {
+      console.log(colors.red(`transfer failed for ${Object.keys(d).length} keys ${opt.version}/${ns.language}/${ns.namespace}...`));
+      throw err
+    }
+  }
+
+  if (keysToSend > payloadKeysLimit) {
+    const keysInObj = Object.keys(data);
+    let isFirst = true;
+    while (keysInObj.length > payloadKeysLimit) {
+      const pagedData = {};
+      keysInObj.splice(0, payloadKeysLimit).forEach((k) => { pagedData[k] = data[k]; });
+      const hasMoreKeys = keysInObj.length > 0;
+      await send(pagedData, hasMoreKeys, isFirst);
+      isFirst = false;
+    }
+    if (keysInObj.length === 0) return
+    const finalPagedData = {};
+    keysInObj.splice(0, keysInObj.length).forEach((k) => { finalPagedData[k] = data[k]; });
+    await send(finalPagedData, false, isFirst);
+    return
+  }
+
+  await send(data, false, true);
+};
+
+const upload = async (opt, nss) => {
+  const concurrency = os.cpus().length;
+  if (!opt.referenceLanguage) {
+    await mapLimit(nss, concurrency, async (ns) => transfer(opt, ns));
+    return
+  }
+
+  const nssRefLng = nss.filter((n) => n.language === opt.referenceLanguage);
+  const nssNonRefLng = nss.filter((n) => n.language !== opt.referenceLanguage);
+
+  // Reference language first, then others, but each group in parallel
+  await mapLimit(nssRefLng, concurrency, async (ns) => transfer(opt, ns));
+  await mapLimit(nssNonRefLng, concurrency, async (ns) => transfer(opt, ns));
+};
+
+const addLanguage = async (opt, l) => {
+  const url = opt.apiEndpoint + '/language/' + opt.projectId + '/' + l;
+  try {
+    const { res } = await request(url, {
+      method: 'post',
+      headers: {
+        Authorization: opt.apiKey
+      }
+    });
+    if (res.status >= 300 && res.status !== 412) throw new Error(res.statusText + ' (' + res.status + ')')
+    console.log(colors.green(`added language ${l}...`));
+  } catch (err) {
+    console.log(colors.red(`failed to add language ${l}...`));
+    throw err
+  }
+};
+
+const migrate = async (opt) => {
+  if (opt.format !== 'json') {
+    throw new Error(`Format ${opt.format} is not accepted!`)
+  }
+
+  opt.apiEndpoint = opt.apiEndpoint || 'https://api.locize.app';
+
+  if (opt.language) {
+    const files = getFiles(opt.path);
+    const namespaces = files.map((file) => ({
+      language: opt.language,
+      namespace: path.basename(file, '.json'),
+      path: path.join(opt.path, file)
+    }));
+    let nss;
+    try {
+      nss = await load(namespaces);
+    } catch (err) {
+      console.error(colors.red(err.stack));
+      process.exit(1);
+    }
+    try {
+      await upload(opt, nss);
+      console.log(colors.green('FINISHED'));
+    } catch (err) {
+      console.error(colors.red(err.stack));
+      process.exit(1);
+    }
+    return
+  }
+
+  if (opt.parseLanguage) {
+    let nss;
+    try {
+      nss = await parseLanguage(opt.path);
+    } catch (err) {
+      console.error(colors.red(err.stack));
+      process.exit(1);
+    }
+    let remoteLanguages;
+    try {
+      remoteLanguages = await getRemoteLanguages(opt);
+    } catch (err) {
+      console.error(colors.red(err.stack));
+      process.exit(1);
+    }
+    const localLanguages = [];
+    nss.forEach((n) => {
+      if (localLanguages.indexOf(n.language) < 0) localLanguages.push(n.language);
+    });
+    const notExistingLanguages = localLanguages.filter((l) => remoteLanguages.indexOf(l) < 0);
+    if (notExistingLanguages.length === 0) {
+      try {
+        await upload(opt, nss);
+        console.log(colors.green('FINISHED'));
+      } catch (err) {
+        console.error(colors.red(err.stack));
+        process.exit(1);
+      }
+      return
+    }
+    try {
+      for (const l of notExistingLanguages) {
+        await addLanguage(opt, l);
+      }
+      await new Promise(resolve => setTimeout(resolve, 5000));
+      await upload(opt, nss);
+      console.log(colors.green('FINISHED'));
+    } catch (err) {
+      console.error(colors.red(err.stack));
+      process.exit(1);
+    }
+  }
+};
+
+module.exports = migrate;

package/dist/cjs/missing.js

@@ -0,0 +1,162 @@
+'use strict';
+
+var colors = require('colors');
+var path = require('node:path');
+var request = require('./request.js');
+var formats = require('./formats.js');
+var getRemoteLanguages = require('./getRemoteLanguages.js');
+var parseLocalReference = require('./parseLocalReference.js');
+var parseLocalLanguages = require('./parseLocalLanguages.js');
+var getRemoteNamespace = require('./getRemoteNamespace.js');
+var os = require('node:os');
+var mapLimit = require('./mapLimit.js');
+
+const reversedFileExtensionsMap = formats.reversedFileExtensionsMap;
+
+const compareNamespace = (local, remote) => {
+  const diff = {
+    toAdd: []
+  };
+  local = local || {};
+  remote = remote || {};
+  Object.keys(local).forEach((k) => {
+    if (remote[k] === '' && local[k] === '') return
+    if (!remote[k]) {
+      diff.toAdd.push(k);
+    }
+  });
+  return diff
+};
+
+const compareNamespaces = async (opt, localNamespaces) => {
+  return await Promise.all(localNamespaces.map(async (ns) => {
+    const { result: remoteNamespace } = await getRemoteNamespace(opt, ns.language, ns.namespace);
+    const diff = compareNamespace(ns.content, remoteNamespace);
+    ns.diff = diff;
+    ns.remoteContent = remoteNamespace;
+    return ns
+  }))
+};
+
+const saveMissing = async (opt, lng, ns) => {
+  const data = {};
+  ns.diff.toAdd.forEach((k) => { data[k] = ns.content[k]; });
+  if (Object.keys(data).length === 0 || opt.dry) return
+  const payloadKeysLimit = 1000;
+  async function send (d, isRetrying = false) {
+    const { res, obj } = await request(opt.apiEndpoint + '/missing/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns.namespace, {
+      method: 'post',
+      body: d,
+      headers: {
+        Authorization: opt.apiKey
+      }
+    });
+    if (res.status === 504 && !isRetrying) {
+      await new Promise(resolve => setTimeout(resolve, 3000));
+      return send(d, true)
+    }
+    if (res.status >= 300 && res.status !== 412) {
+      if (obj && (obj.errorMessage || obj.message)) {
+        throw new Error((obj.errorMessage || obj.message))
+      }
+      throw new Error(res.statusText + ' (' + res.status + ')')
+    }
+    await new Promise(resolve => setTimeout(resolve, 1000));
+  }
+  if (Object.keys(data).length > payloadKeysLimit) {
+    const keysInObj = Object.keys(data);
+    while (keysInObj.length > payloadKeysLimit) {
+      const pagedData = {};
+      keysInObj.splice(0, payloadKeysLimit).forEach((k) => { pagedData[k] = data[k]; });
+      await send(pagedData);
+    }
+    if (keysInObj.length === 0) return
+    const finalPagedData = {};
+    keysInObj.splice(0, keysInObj.length).forEach((k) => { finalPagedData[k] = data[k]; });
+    await send(finalPagedData);
+    return
+  }
+  await send(data);
+};
+
+const handleError = (err) => {
+  if (err) {
+    console.error(colors.red(err.stack));
+    process.exit(1);
+  }
+};
+
+const handleMissing = async (opt, localNamespaces) => {
+  if (!localNamespaces || localNamespaces.length === 0) {
+    handleError(new Error('No local namespaces found!'));
+    return
+  }
+  let compared;
+  try {
+    compared = await compareNamespaces(opt, localNamespaces);
+  } catch (err) {
+    handleError(err);
+    return
+  }
+  const concurrency = os.cpus().length;
+  await mapLimit(compared, concurrency, async (ns) => {
+    if (ns.diff.toAdd.length > 0) {
+      console.log(colors.green(`adding ${ns.diff.toAdd.length} keys in ${ns.language}/${ns.namespace}...`));
+      if (opt.dry) console.log(colors.green(`would add ${ns.diff.toAdd.join(', ')} in ${ns.language}/${ns.namespace}...`));
+    }
+    try {
+      await saveMissing(opt, ns.language, ns);
+    } catch (err) {
+      handleError(err);
+      // Don't return here, continue with others
+    }
+  });
+  console.log(colors.green('FINISHED'));
+};
+
+const missing = async (opt) => {
+  if (!reversedFileExtensionsMap[opt.format]) {
+    handleError(new Error(`${opt.format} is not a valid format!`));
+    return
+  }
+  if (opt.namespace && opt.namespace.indexOf(',') > 0) {
+    opt.namespaces = opt.namespace.split(',');
+    delete opt.namespace;
+  }
+  opt.pathMaskInterpolationPrefix = opt.pathMaskInterpolationPrefix || '{{';
+  opt.pathMaskInterpolationSuffix = opt.pathMaskInterpolationSuffix || '}}';
+  opt.pathMask = opt.pathMask || `${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}${path.sep}${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`;
+  opt.languageFolderPrefix = opt.languageFolderPrefix || '';
+  opt.pathMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, `${opt.languageFolderPrefix}${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`);
+  let remoteLanguages;
+  try {
+    remoteLanguages = await getRemoteLanguages(opt);
+  } catch (err) {
+    handleError(err);
+    return
+  }
+  if (opt.referenceLanguageOnly && opt.language && opt.referenceLanguage !== opt.language) {
+    opt.referenceLanguage = opt.language;
+  }
+  if (opt.referenceLanguageOnly) {
+    let localNamespaces;
+    try {
+      localNamespaces = await parseLocalReference(opt);
+    } catch (err) {
+      handleError(err);
+      return
+    }
+    await handleMissing(opt, localNamespaces);
+    return
+  }
+  let localNamespaces;
+  try {
+    localNamespaces = await parseLocalLanguages(opt, remoteLanguages);
+  } catch (err) {
+    handleError(err);
+    return
+  }
+  await handleMissing(opt, localNamespaces);
+};
+
+module.exports = missing;