locize-cli 11.0.0 → 12.0.0

package/download.js

@@ -1,516 +0,0 @@
-const colors = require('colors')
-const { mkdirp } = require('mkdirp')
-const rimraf = require('rimraf')
-const request = require('./request')
-const fs = require('fs')
-const path = require('path')
-const async = require('async')
-const flatten = require('flat')
-const getRemoteNamespace = require('./getRemoteNamespace')
-const getRemoteLanguages = require('./getRemoteLanguages')
-const convertToDesiredFormat = require('./convertToDesiredFormat')
-const formats = require('./formats')
-const getProjectStats = require('./getProjectStats')
-const reversedFileExtensionsMap = formats.reversedFileExtensionsMap
-const locize2xcstrings = require('locize-xcstrings/cjs/locize2xcstrings')
-const getBranches = require('./getBranches')
-const isValidUuid = require('./isValidUuid')
-
-function getInfosInUrl (download) {
-  const splitted = download.key.split('/')
-  const version = splitted[download.isPrivate ? 2 : 1]
-  const language = splitted[download.isPrivate ? 3 : 2]
-  const namespace = splitted[download.isPrivate ? 4 : 3]
-  return { version, language, namespace }
-}
-
-function handleDownload (opt, url, err, res, downloads, cb) {
-  if (err || (downloads && (downloads.errorMessage || downloads.message))) {
-    if (!cb) console.log(colors.red(`download failed for ${url} to ${opt.path}...`))
-
-    if (err) {
-      if (!cb) { console.error(colors.red(err.message)); process.exit(1) }
-      if (cb) cb(err)
-      return
-    }
-    if (downloads && (downloads.errorMessage || downloads.message)) {
-      if (!cb) { console.error(colors.red((downloads.errorMessage || downloads.message))); process.exit(1) }
-      if (cb) cb(new Error((downloads.errorMessage || downloads.message)))
-      return
-    }
-  }
-  if (res.status >= 300) {
-    if (!cb) { console.error(colors.red(res.statusText + ' (' + res.status + ')')); process.exit(1) }
-    if (cb) cb(new Error(res.statusText + ' (' + res.status + ')'))
-    return
-  }
-
-  if (opt.format === 'xcstrings') { // 1 file per namespace including all languages
-    const downloadsByNamespace = {}
-    downloads.forEach((download) => {
-      const { version, namespace } = getInfosInUrl(download)
-      opt.isPrivate = download.isPrivate
-
-      downloadsByNamespace[version] = downloadsByNamespace[version] || {}
-      downloadsByNamespace[version][namespace] = downloadsByNamespace[version][namespace] || []
-      downloadsByNamespace[version][namespace].push(download)
-    })
-
-    async.eachSeries(Object.keys(downloadsByNamespace), (version, clb) => {
-      async.eachLimit(Object.keys(downloadsByNamespace[version]), 20, (ns, clb) => {
-        if (opt.namespace && opt.namespace !== ns) return clb(null)
-        if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(ns) < 0) return clb(null)
-
-        const locizeData = {
-          sourceLng: opt.referenceLanguage,
-          resources: {}
-        }
-        async.eachLimit(downloadsByNamespace[version][ns], 20, (download, clb2) => {
-          const { language } = getInfosInUrl(download)
-          getRemoteNamespace(opt, language, ns, (err, ns, lastModified) => {
-            if (err) return clb2(err)
-
-            if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
-              return clb2(null)
-            }
-
-            locizeData.resources[language] = ns
-            clb2()
-          })
-        }, (err) => {
-          if (err) return clb(err)
-
-          try {
-            const converted = locize2xcstrings(locizeData)
-
-            const filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, '').replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, ns) + reversedFileExtensionsMap[opt.format]
-            let mkdirPath
-            if (filledMask.lastIndexOf(path.sep) > 0) {
-              mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep))
-            }
-
-            function logAndClb (err) {
-              if (err) return clb(err)
-              if (!cb) console.log(colors.green(`downloaded ${version}/${ns} to ${opt.path}...`))
-              if (clb) clb(null)
-            }
-
-            const fileContent = (opt.format !== 'xlsx' && !converted.endsWith('\n')) ? (converted + '\n') : converted
-            if (!opt.version) {
-              if (mkdirPath) mkdirp.sync(path.join(opt.path, version, mkdirPath))
-              fs.writeFile(path.join(opt.path, version, filledMask), fileContent, logAndClb)
-              return
-            }
-
-            if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath))
-            fs.writeFile(path.join(opt.path, filledMask), converted, logAndClb)
-          } catch (e) {
-            err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '')
-            return clb(err)
-          }
-        })
-      }, clb)
-    }, (err) => {
-      if (err) {
-        if (!cb) {
-          console.error(colors.red(err.message))
-          process.exit(1)
-        }
-        if (cb) cb(err)
-        return
-      }
-
-      if (cb) cb(null)
-    })
-  } else { // 1 file per namespace/lng
-    async.eachLimit(downloads, 20, (download, clb) => {
-      const { version, language, namespace } = getInfosInUrl(download)
-      opt.isPrivate = download.isPrivate
-
-      if (opt.namespace && opt.namespace !== namespace) return clb(null)
-      if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null)
-
-      getRemoteNamespace(opt, language, namespace, (err, ns, lastModified) => {
-        if (err) return clb(err)
-
-        if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
-          return clb(null)
-        }
-
-        convertToDesiredFormat(opt, namespace, language, ns, lastModified, (err, converted) => {
-          if (err) {
-            err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '')
-            return clb(err)
-          }
-          let filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, language).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format]
-          let mkdirPath
-          if (filledMask.lastIndexOf(path.sep) > 0) {
-            mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep))
-          }
-          const fileContent = (opt.format !== 'xlsx' && !converted.endsWith('\n')) ? (converted + '\n') : converted
-          if (!opt.version) {
-            if (mkdirPath) mkdirp.sync(path.join(opt.path, version, mkdirPath))
-            fs.writeFile(path.join(opt.path, version, filledMask), fileContent, clb)
-            return
-          }
-          if (!opt.language) {
-            if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath))
-            fs.writeFile(path.join(opt.path, filledMask), fileContent, clb)
-            return
-          }
-
-          if (opt.languageFolderPrefix && filledMask.indexOf(path.sep) > 0) filledMask = filledMask.replace(opt.languageFolderPrefix + language, '')
-          // if (opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`) > opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`) && filledMask.lastIndexOf(path.sep) > 0) {
-          //   mkdirp.sync(path.join(opt.path, filledMask.substring(0, filledMask.lastIndexOf(path.sep))));
-          // }
-          const parentDir = path.dirname(path.join(opt.path, filledMask))
-          mkdirp.sync(parentDir)
-          fs.writeFile(path.join(opt.path, filledMask), fileContent, clb)
-        })
-      })
-    }, (err) => {
-      if (err) {
-        if (!cb) {
-          console.error(colors.red(err.message))
-          process.exit(1)
-        }
-        if (cb) cb(err)
-        return
-      }
-
-      if (!cb) console.log(colors.green(`downloaded ${url} to ${opt.path}...`))
-      if (cb) cb(null)
-    })
-  }
-}
-
-function handlePull (opt, toDownload, cb) {
-  const url = opt.apiEndpoint + '/pull/' + opt.projectId + '/' + opt.version
-
-  if (opt.format === 'xcstrings') { // 1 file per namespace including all languages
-    const downloadsByNamespace = {}
-    toDownload.forEach((download) => {
-      const { namespace } = download
-      downloadsByNamespace[namespace] = downloadsByNamespace[namespace] || []
-      downloadsByNamespace[namespace].push(download)
-    })
-
-    async.eachLimit(Object.keys(downloadsByNamespace), 5, (namespace, clb) => {
-      if (opt.namespace && opt.namespace !== namespace) return clb(null)
-      if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null)
-
-      const locizeData = {
-        sourceLng: opt.referenceLanguage,
-        resources: {}
-      }
-
-      async.eachLimit(downloadsByNamespace[namespace], 5, (download, clb2) => {
-        const { language } = download
-        opt.raw = true
-        getRemoteNamespace(opt, language, namespace, (err, ns, lastModified) => {
-          if (err) return clb2(err)
-
-          if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
-            return clb2(null)
-          }
-
-          locizeData.resources[language] = ns
-          clb2()
-        })
-      }, (err) => {
-        if (err) return clb(err)
-
-        try {
-          const result = locize2xcstrings(locizeData)
-          const converted = JSON.stringify(result, null, 2)
-
-          const filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, '').replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format]
-          let mkdirPath
-          if (filledMask.lastIndexOf(path.sep) > 0) {
-            mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep))
-          }
-
-          function logAndClb (err) {
-            if (err) return clb(err)
-            if (!cb) console.log(colors.green(`downloaded ${opt.version}/${namespace} to ${opt.path}...`))
-            if (clb) clb(null)
-          }
-
-          if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath))
-          const fileContent = (opt.format !== 'xlsx' && !converted.endsWith('\n')) ? (converted + '\n') : converted
-          fs.writeFile(path.join(opt.path, filledMask), fileContent, logAndClb)
-        } catch (e) {
-          err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '')
-          return clb(err)
-        }
-      })
-    }, (err) => {
-      if (err) {
-        if (!cb) {
-          console.error(colors.red(err.message))
-          process.exit(1)
-        }
-        if (cb) cb(err)
-        return
-      }
-
-      if (cb) cb(null)
-    })
-  } else { // 1 file per namespace/lng
-    async.eachLimit(toDownload, 5, (download, clb) => {
-      const lng = download.language
-      const namespace = download.namespace
-
-      if (opt.namespace && opt.namespace !== namespace) return clb(null)
-      if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null)
-
-      getRemoteNamespace(opt, lng, namespace, (err, ns, lastModified) => {
-        if (err) return clb(err)
-
-        if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
-          return clb(null)
-        }
-
-        convertToDesiredFormat(opt, namespace, lng, ns, lastModified, (err, converted) => {
-          if (err) {
-            err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '')
-            return clb(err)
-          }
-          let filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, lng).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format]
-          let mkdirPath
-          if (filledMask.lastIndexOf(path.sep) > 0) {
-            mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep))
-          }
-          const fileContent = (opt.format !== 'xlsx' && !converted.endsWith('\n')) ? (converted + '\n') : converted
-          if (!opt.language) {
-            if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath))
-            fs.writeFile(path.join(opt.path, filledMask), fileContent, clb)
-            return
-          }
-
-          if (opt.languageFolderPrefix && filledMask.indexOf(path.sep) > 0) filledMask = filledMask.replace(opt.languageFolderPrefix + lng, '')
-          // if (opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`) > opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`) && filledMask.lastIndexOf(path.sep) > 0) {
-          //   mkdirp.sync(path.join(opt.path, filledMask.substring(0, filledMask.lastIndexOf(path.sep))));
-          // }
-          const parentDir = path.dirname(path.join(opt.path, filledMask))
-          mkdirp.sync(parentDir)
-          fs.writeFile(path.join(opt.path, filledMask), fileContent, clb)
-        })
-      })
-    }, (err) => {
-      if (err) {
-        if (!cb) {
-          console.error(colors.red(err.message))
-          process.exit(1)
-        }
-        if (cb) cb(err)
-        return
-      }
-
-      if (!cb) console.log(colors.green(`downloaded ${url} to ${opt.path}...`))
-      if (cb) cb(null)
-    })
-  }
-}
-
-const handleError = (err, cb) => {
-  if (!cb && err) {
-    console.error(colors.red(err.message))
-    process.exit(1)
-  }
-  if (cb) cb(err)
-}
-
-const filterDownloadsLanguages = (opt, downloads) => {
-  if (opt.languages) {
-    downloads = downloads.filter((d) => {
-      const splitted = d.key.split('/')
-      // const p = splitted[d.isPrivate ? 1 : 0];
-      // const v = splitted[d.isPrivate ? 2 : 1];
-      const l = splitted[d.isPrivate ? 3 : 2]
-      const n = splitted[d.isPrivate ? 4 : 3]
-      return opt.languages.indexOf(l) > -1 && (!opt.namespace || opt.namespace === n)
-    })
-  }
-  return downloads
-}
-
-const filterDownloads = (opt, downloads) => {
-  if (opt.skipEmpty) return filterDownloadsLanguages(opt, downloads.filter((d) => d.size > 2))
-  if (downloads.length < 1) return downloads
-
-  const allNamespaces = []
-  const downloadMap = {}
-  downloads.forEach((d) => {
-    const splitted = d.key.split('/')
-    const p = splitted[d.isPrivate ? 1 : 0]
-    const v = splitted[d.isPrivate ? 2 : 1]
-    const l = splitted[d.isPrivate ? 3 : 2]
-    const n = splitted[d.isPrivate ? 4 : 3]
-    downloadMap[p] = downloadMap[p] || {}
-    downloadMap[p][v] = downloadMap[p][v] || {}
-    downloadMap[p][v][l] = downloadMap[p][v][l] || {}
-    downloadMap[p][v][l][n] = d
-    if (allNamespaces.indexOf(n) < 0) allNamespaces.push(n)
-  })
-  Object.keys(downloadMap).forEach((projectId) => {
-    Object.keys(downloadMap[projectId]).forEach((version) => {
-      Object.keys(downloadMap[projectId][version]).forEach((language) => {
-        allNamespaces.forEach((namespace) => {
-          if (!downloadMap[projectId][version][language][namespace]) {
-            downloads.push({
-              url: `${opt.apiEndpoint}/${projectId}/${version}/${language}/${namespace}`,
-              key: `${projectId}/${version}/${language}/${namespace}`,
-              lastModified: '1960-01-01T00:00:00.000Z',
-              size: 0
-            })
-          }
-        })
-      })
-    })
-  })
-  return filterDownloadsLanguages(opt, downloads)
-}
-
-const continueToDownload = (opt, cb) => {
-  let url = opt.apiEndpoint + '/download/' + opt.projectId
-
-  if (opt.namespace && opt.namespace.indexOf(',') > 0 && opt.namespace.indexOf(' ') < 0) {
-    opt.namespaces = opt.namespace.split(',')
-    delete opt.namespace
-  }
-
-  if (opt.version) {
-    url += '/' + opt.version
-    if (!opt.languages && opt.language) {
-      url += '/' + opt.language
-      if (opt.namespace) {
-        url += '/' + opt.namespace
-      }
-    }
-  }
-
-  if (opt.clean) rimraf.sync(path.join(opt.path, '*'))
-
-  mkdirp.sync(opt.path)
-
-  if (!cb) console.log(colors.yellow(`downloading ${url} to ${opt.path}...`))
-
-  getRemoteLanguages(opt, (err) => {
-    if (err) return handleError(err, cb)
-
-    if (!opt.unpublished) {
-      request(url, {
-        method: 'get',
-        headers: opt.apiKey
-          ? {
-              Authorization: opt.apiKey
-            }
-          : undefined
-      }, (err, res, obj) => {
-        if (res && res.status === 401) {
-          opt.apiKey = null
-          request(url, {
-            method: 'get',
-          }, (err, res, obj) => {
-            obj = filterDownloads(opt, obj || [])
-            handleDownload(opt, url, err, res, obj, cb)
-          })
-          return
-        }
-
-        if (obj.length > 0) {
-          obj = filterDownloads(opt, obj || [])
-          return handleDownload(opt, url, err, res, obj, cb)
-        }
-
-        getProjectStats(opt, (err, res) => {
-          if (err) return handleError(err, cb)
-          if (!res) return handleError(new Error('Nothing found!'), cb)
-          if (!res[opt.version]) return handleError(new Error(`Version "${opt.version}" not found!`), cb)
-
-          obj = filterDownloads(opt, obj || [])
-          return handleDownload(opt, url, err, res, obj, cb)
-        })
-      })
-      return
-    }
-
-    getProjectStats(opt, (err, res) => {
-      if (err) return handleError(err, cb)
-      if (!res) return handleError(new Error('Nothing found!'), cb)
-      if (!res[opt.version]) return handleError(new Error(`Version "${opt.version}" not found!`), cb)
-
-      const toDownload = []
-      const lngsToCheck = opt.language ? [opt.language] : Object.keys(res[opt.version])
-      lngsToCheck.forEach((l) => {
-        if (opt.namespaces) {
-          opt.namespaces.forEach((n) => {
-            if (!res[opt.version][l][n]) return
-            if (opt.skipEmpty && res[opt.version][l][n].segmentsTranslated === 0) return
-            toDownload.push({ language: l, namespace: n })
-          })
-        } else if (opt.namespace) {
-          if (!res[opt.version][l][opt.namespace]) return
-          if (opt.skipEmpty && res[opt.version][l][opt.namespace].segmentsTranslated === 0) return
-          toDownload.push({ language: l, namespace: opt.namespace })
-        } else {
-          Object.keys(res[opt.version][l]).forEach((n) => {
-            if (opt.skipEmpty && res[opt.version][l][n].segmentsTranslated === 0) return
-            toDownload.push({ language: l, namespace: n })
-          })
-        }
-      })
-      handlePull(opt, toDownload, cb)
-    })
-  })
-}
-
-const download = (opt, cb) => {
-  opt.format = opt.format || 'json'
-  if (!reversedFileExtensionsMap[opt.format]) {
-    return handleError(new Error(`${opt.format} is not a valid format!`), cb)
-  }
-
-  if (opt.skipEmpty === undefined) opt.skipEmpty = true
-  opt.apiEndpoint = opt.apiEndpoint || 'https://api.locize.app'
-  opt.version = opt.version || 'latest'
-  opt.languageFolderPrefix = opt.languageFolderPrefix || ''
-  opt.path = opt.path || opt.target
-  opt.pathMaskInterpolationPrefix = opt.pathMaskInterpolationPrefix || '{{'
-  opt.pathMaskInterpolationSuffix = opt.pathMaskInterpolationSuffix || '}}'
-  opt.pathMask = opt.pathMask || `${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}${path.sep}${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`
-  opt.pathMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, `${opt.languageFolderPrefix}${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`)
-  if (opt.overriddenOnly) {
-    opt.unpublished = true
-  }
-  if (opt.unpublished && !opt.apiKey) {
-    return handleError(new Error('Please provide also an api-key!'), cb)
-  }
-
-  if (opt.branch === '') {
-    return handleError(new Error('The branch name seems invalid!'), cb)
-  }
-
-  if (opt.branch) {
-    getBranches(opt, (err, branches) => {
-      if (err) return handleError(err, cb)
-
-      let b
-      if (isValidUuid(opt.branch)) b = branches.find((br) => br.id === opt.branch)
-      if (!b) b = branches.find((br) => br.name === opt.branch)
-      if (!b) {
-        return handleError(new Error(`Branch ${opt.branch} not found!`), cb)
-      }
-      opt.projectId = b.id
-      opt.version = b.version
-
-      continueToDownload(opt, cb)
-    })
-    return
-  }
-
-  continueToDownload(opt, cb)
-}
-
-module.exports = download