@vocab/phrase 1.0.1 → 1.2.0

package/README.md

@@ -184,6 +184,14 @@ Configuration can either be passed into the Node API directly or be gathered fro
 **vocab.config.js**
 
 ```js
+function capitalize(element) {
+  return element.toUpperCase();
+}
+
+function pad(message) {
+  return '[' + message + ']';
+}
+
 module.exports = {
   devLanguage: 'en',
   languages: [
@@ -192,11 +200,25 @@ module.exports = {
     { name: 'en-US', extends: 'en' },
     { name: 'fr-FR' }
   ],
+  /**
+   * An array of languages to generate based off translations for existing languages
+   * Default: []
+   */
+  generatedLanguages: [
+    {
+      name: 'generatedLangauge',
+      extends: 'en',
+      generator: {
+        transformElement: capitalize,
+        transformMessage: pad
+      }
+    }
+  ],
   /**
    * The root directory to compile and validate translations
    * Default: Current working directory
    */
-  projectRoot: './example/';
+  projectRoot: './example/',
   /**
    * A custom suffix to name vocab translation directories
    * Default: '.vocab'
@@ -209,6 +231,131 @@ module.exports = {
 };
 ```
 
+## Generated languages
+
+Vocab supports the creation of generated languages via the `generatedLanguages` config.
+
+Generated languages are created by running a message `generator` over every translation message in an existing translation.
+A `generator` may contain a `transformElement` function, a `transformMessage` function, or both.
+Both of these functions accept a single string parameter and return a string.
+
+`transformElement` is applied to string literal values contained within `MessageFormatElement`s.
+A `MessageFormatElement` is an object representing a node in the AST of a compiled translation message.
+Simply put, any text that would end up being translated by a translator, i.e. anything that is not part of the [ICU Message syntax], will be passed to `transformElement`.
+An example of a use case for this function would be adding [diacritics] to every letter in order to stress your UI from a vertical line-height perspective.
+
+`transformMessage` receives the entire translation message _after_ `transformElement` has been applied to its individual elements.
+An example of a use case for this function would be adding padding text to the start/end of your messages in order to easily identify which text in your app has not been extracted into a `translations.json` file.
+
+By default, a generated language's messages will be based off the `devLanguage`'s messages, but this can be overridden by providing an `extends` value that references another language.
+
+**vocab.config.js**
+
+```js
+function capitalize(message) {
+  return message.toUpperCase();
+}
+
+function pad(message) {
+  return '[' + message + ']';
+}
+
+module.exports = {
+  devLanguage: 'en',
+  languages: [{ name: 'en' }, { name: 'fr' }],
+  generatedLanguages: [
+    {
+      name: 'generatedLanguage',
+      extends: 'en',
+      generator: {
+        transformElement: capitalize,
+        transformMessage: pad
+      }
+    }
+  ]
+};
+```
+
+Generated languages are consumed the same way as regular languages.
+Any Vocab API that accepts a `language` parameter will work with a generated language as well as a regular language.
+
+**App.tsx**
+
+```tsx
+const App = () => (
+  <VocabProvider language="generatedLanguage">
+    ...
+  </VocabProvider>
+);
+```
+
+[icu message syntax]: https://formatjs.io/docs/intl-messageformat/#message-syntax
+[diacritics]: https://en.wikipedia.org/wiki/Diacritic
+
+## Pseudo-localization
+
+The `@vocab/pseudo-localize` package exports low-level functions that can be used for pseudo-localization of translation messages.
+
+```ts
+import {
+  extendVowels,
+  padString,
+  pseudoLocalize,
+  substituteCharacters
+} from '@vocab/pseudo-localize';
+
+const message = 'Hello';
+
+// [Hello]
+const paddedMessage = padString(message);
+
+// Ḩẽƚƚö
+const substitutedMessage = substituteCharacters(message);
+
+// Heelloo
+const extendedMessage = extendVowels(message);
+
+// Extend the message and then substitute characters
+// Ḩẽẽƚƚöö
+const pseudoLocalizedMessage = pseudoLocalize(message);
+```
+
+Pseudo-localization is a transformation that can be applied to a translation message.
+Vocab's implementation of this transformation contains the following elements:
+
+- _Start and end markers (`padString`):_ All strings are encapsulated in `[` and `]`. If a developer doesn’t see these characters they know the string has been clipped by an inflexible UI element.
+- _Transformation of ASCII characters to extended character equivalents (`substituteCharacters`):_ Stresses the UI from a vertical line-height perspective, tests font and encoding support, and weeds out strings that haven’t been externalized correctly (they will not have the pseudo-localization applied to them).
+- _Padding text (`extendVowels`):_ Simulates translation-induced expansion. Vocab's implementation of this involves repeating vowels (and `y`) to simulate a 40% expansion in the message's length.
+
+This Netflix technology [blog post][blog post] inspired Vocab's implementation of this
+functionality.
+
+### Generating a pseudo-localized language using Vocab
+
+Vocab can generate a pseudo-localized language via the [`generatedLanguages` config][generated languages config], either via the webpack plugin or your `vocab.config.js` file.
+`@vocab/pseudo-localize` exports a `generator` that can be used directly in your config.
+
+**vocab.config.js**
+
+```js
+const { generator } = require('@vocab/pseudo-localize');
+
+module.exports = {
+  devLanguage: 'en',
+  languages: [{ name: 'en' }, { name: 'fr' }],
+  generatedLanguages: [
+    {
+      name: 'pseudo',
+      extends: 'en',
+      generator
+    }
+  ]
+};
+```
+
+[blog post]: https://netflixtechblog.com/pseudo-localization-netflix-12fff76fbcbe
+[generated languages config]: #generated-languages
+
 ## Use without React
 
 If you need to use Vocab outside of React, you can access the returned Vocab file directly. You'll then be responsible for when to load translations and how to update on translation load.
@@ -275,19 +422,81 @@ Or to re-run the compiler when files change use:
 $ vocab compile --watch
 ```
 
-## External translation tooling
+## External Translation Tooling
 
 Vocab can be used to synchronize your translations with translations from a remote translation platform.
 
-| Platform                                     | Environment Variables               |
-| -------------------------------------------- | ----------------------------------- |
-| [Phrase](https://developers.phrase.com/api/) | PHRASE_PROJECT_ID, PHRASE_API_TOKEN |
+| Platform | Environment Variables               |
+| -------- | ----------------------------------- |
+| [Phrase] | PHRASE_PROJECT_ID, PHRASE_API_TOKEN |
 
 ```bash
 $ vocab push --branch my-branch
 $ vocab pull --branch my-branch
 ```
 
+### [Phrase] Platform Features
+
+#### Delete Unused keys
+
+When uploading translations, Phrase identifies keys that exist in the Phrase project, but were not
+referenced in the upload. These keys can be deleted from Phrase by providing the
+`---delete-unused-keys` flag to `vocab push`. E.g.
+
+```sh
+$ vocab push --branch my-branch --delete-unused-keys
+```
+
+[phrase]: https://developers.phrase.com/api/
+
+#### [Tags]
+
+`vocab push` supports uploading [tags] to Phrase.
+
+Tags can be added to an individual key via the `tags` property:
+
+```jsonc
+// translations.json
+{
+  "Hello": {
+    "message": "Hello",
+    "tags": ["greeting", "home_page"]
+  },
+  "Goodbye": {
+    "message": "Goodbye",
+    "tags": ["home_page"]
+  }
+}
+```
+
+Tags can also be added under a top-level `_meta` field. This will result in the tags applying to all
+keys specified in the file:
+
+```jsonc
+// translations.json
+{
+  "_meta": {
+    "tags": ["home_page"]
+  },
+  "Hello": {
+    "message": "Hello",
+    "tags": ["greeting"]
+  },
+  "Goodbye": {
+    "message": "Goodbye"
+  }
+}
+```
+
+In the above example, both the `Hello` and `Goodbye` keys would have the `home_page` tag attached to
+them, but only the `Hello` key would have the `usage_greeting` tag attached to it.
+
+**NOTE**: Only tags specified on keys in your [`devLanguage`][configuration] will be uploaded.
+Tags on keys in other languages will be ignored.
+
+[tags]: https://support.phrase.com/hc/en-us/articles/5822598372252-Tags-Strings-
+[configuration]: #Configuration
+
 ## Troubleshooting
 
 ### Problem: Passed locale is being ignored or using en-US instead

package/dist/declarations/src/csv.d.ts

@@ -0,0 +1,10 @@
+import type { TranslationsByLanguage } from '@vocab/types';
+export declare function translationsToCsv(translations: TranslationsByLanguage, devLanguage: string): {
+    csvString: string;
+    localeMapping: {
+        [k: string]: number;
+    };
+    keyIndex: number;
+    commentIndex: number;
+    tagColumn: number;
+};

package/dist/declarations/src/phrase-api.d.ts

@@ -1,7 +1,12 @@
-import { TranslationsByKey } from './../../types/src/index';
 import type { TranslationsByLanguage } from '@vocab/types';
 import fetch from 'node-fetch';
-export declare function callPhrase(relativePath: string, options?: Parameters<typeof fetch>[1]): Promise<any>;
+export declare function callPhrase<T = any>(relativePath: string, options?: Parameters<typeof fetch>[1]): Promise<T>;
 export declare function pullAllTranslations(branch: string): Promise<TranslationsByLanguage>;
-export declare function pushTranslationsByLocale(contents: TranslationsByKey, locale: string, branch: string): Promise<void>;
+export declare function pushTranslations(translationsByLanguage: TranslationsByLanguage, { devLanguage, branch }: {
+    devLanguage: string;
+    branch: string;
+}): Promise<{
+    uploadId: string;
+}>;
+export declare function deleteUnusedKeys(uploadId: string, branch: string): Promise<void>;
 export declare function ensureBranch(branch: string): Promise<void>;

package/dist/declarations/src/pull-translations.d.ts

@@ -1,6 +1,7 @@
 import type { UserConfig } from '@vocab/types';
 interface PullOptions {
     branch?: string;
+    deleteUnusedKeys?: boolean;
 }
 export declare function pull({ branch }: PullOptions, config: UserConfig): Promise<void>;
 export {};

package/dist/declarations/src/push-translations.d.ts

@@ -1,9 +1,11 @@
 import { UserConfig } from '@vocab/types';
 interface PushOptions {
     branch: string;
+    deleteUnusedKeys?: boolean;
 }
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
-export declare function push({ branch }: PushOptions, config: UserConfig): Promise<void>;
+export declare function push({ branch, deleteUnusedKeys }: PushOptions, config: UserConfig): Promise<void>;
 export {};

package/dist/vocab-phrase.cjs.dev.js

@@ -9,6 +9,7 @@ var FormData = require('form-data');
 var fetch = require('node-fetch');
 var chalk = require('chalk');
 var debug = require('debug');
+var sync = require('csv-stringify/sync');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }
 
@@ -27,6 +28,44 @@ const log = (...params) => {
   console.log(chalk__default['default'].yellow('Vocab'), ...params);
 };
 
+function translationsToCsv(translations, devLanguage) {
+  const languages = Object.keys(translations);
+  const altLanguages = languages.filter(language => language !== devLanguage); // Ensure languages are ordered for locale mapping
+
+  const orderedLanguages = [devLanguage, ...altLanguages];
+  const devLanguageTranslations = translations[devLanguage];
+  const csv = Object.entries(devLanguageTranslations).map(([key, {
+    message,
+    description,
+    tags
+  }]) => {
+    const altTranslationMessages = altLanguages.map(language => {
+      var _translations$languag, _translations$languag2;
+
+      return (_translations$languag = translations[language]) === null || _translations$languag === void 0 ? void 0 : (_translations$languag2 = _translations$languag[key]) === null || _translations$languag2 === void 0 ? void 0 : _translations$languag2.message;
+    });
+    return [message, ...altTranslationMessages, key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
+  });
+  const csvString = sync.stringify(csv, {
+    delimiter: ',',
+    header: false
+  }); // Column indices start at 1
+
+  const localeMapping = Object.fromEntries(orderedLanguages.map((language, index) => [language, index + 1]));
+  const keyIndex = orderedLanguages.length + 1;
+  const commentIndex = keyIndex + 1;
+  const tagColumn = commentIndex + 1;
+  return {
+    csvString,
+    localeMapping,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  };
+}
+
+/* eslint-disable no-console */
+
 function _callPhrase(path, options = {}) {
   const phraseApiToken = process.env.PHRASE_API_TOKEN;
 
@@ -44,14 +83,14 @@ function _callPhrase(path, options = {}) {
   }).then(async response => {
     console.log(`${path}: ${response.status} - ${response.statusText}`);
     console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
-    console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
+    trace('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
     // console.log(Array.from(r.headers.entries()));
 
     try {
       var _response$headers$get;
 
       const result = await response.json();
-      console.log(`Internal Result (Length: ${result.length})\n`);
+      trace(`Internal Result (Length: ${result.length})\n`);
 
       if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
         var _response$headers$get2, _response$headers$get3;
@@ -59,10 +98,10 @@ function _callPhrase(path, options = {}) {
         const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
 
         if (!nextPageUrl) {
-          throw new Error('Cant parse next page URL');
+          throw new Error("Can't parse next page URL");
         }
 
-        console.log('Results recieved with next page: ', nextPageUrl);
+        console.log('Results received with next page: ', nextPageUrl);
         const nextPageResult = await _callPhrase(nextPageUrl, options);
         return [...result, ...nextPageResult];
       }
@@ -109,23 +148,70 @@ async function pullAllTranslations(branch) {
 
   return translations;
 }
-async function pushTranslationsByLocale(contents, locale, branch) {
+async function pushTranslations(translationsByLanguage, {
+  devLanguage,
+  branch
+}) {
   const formData = new FormData__default['default']();
-  const fileContents = Buffer.from(JSON.stringify(contents));
+  const {
+    csvString,
+    localeMapping,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  } = translationsToCsv(translationsByLanguage, devLanguage);
+  const fileContents = Buffer.from(csvString);
   formData.append('file', fileContents, {
-    contentType: 'application/json',
-    filename: `${locale}.json`
+    contentType: 'text/csv',
+    filename: 'translations.csv'
   });
-  formData.append('file_format', 'json');
-  formData.append('locale_id', locale);
+  formData.append('file_format', 'csv');
   formData.append('branch', branch);
   formData.append('update_translations', 'true');
-  trace('Starting to upload:', locale);
-  await callPhrase(`uploads`, {
+
+  for (const [locale, index] of Object.entries(localeMapping)) {
+    formData.append(`locale_mapping[${locale}]`, index);
+  }
+
+  formData.append('format_options[key_index]', keyIndex);
+  formData.append('format_options[comment_index]', commentIndex);
+  formData.append('format_options[tag_column]', tagColumn);
+  formData.append('format_options[enable_pluralization]', 'false');
+  log('Uploading translations');
+  const result = await callPhrase(`uploads`, {
     method: 'POST',
     body: formData
   });
-  log('Successfully Uploaded:', locale, '\n');
+  trace('Upload result:\n', result);
+
+  if (result && 'id' in result) {
+    log('Upload ID:', result.id, '\n');
+    log('Successfully Uploaded\n');
+  } else {
+    log(`Error uploading: ${result === null || result === void 0 ? void 0 : result.message}\n`);
+    log('Response:', result);
+    throw new Error('Error uploading');
+  }
+
+  return {
+    uploadId: result.id
+  };
+}
+async function deleteUnusedKeys(uploadId, branch) {
+  const query = `unmentioned_in_upload:${uploadId}`;
+  const {
+    records_affected
+  } = await callPhrase('keys', {
+    method: 'DELETE',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      branch,
+      q: query
+    })
+  });
+  log('Successfully deleted', records_affected, 'unused keys from branch', branch);
 }
 async function ensureBranch(branch) {
   await callPhrase(`branches`, {
@@ -137,7 +223,7 @@ async function ensureBranch(branch) {
       name: branch
     })
   });
-  trace('Created branch:', branch);
+  log('Created branch:', branch);
 }
 
 async function pull({
@@ -148,9 +234,17 @@ async function pull({
   const alternativeLanguages = core.getAltLanguages(config);
   const allPhraseTranslations = await pullAllTranslations(branch);
   trace(`Pulling translations from Phrase for languages ${config.devLanguage} and ${alternativeLanguages.join(', ')}`);
+  const phraseLanguages = Object.keys(allPhraseTranslations);
+  trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
+
+  if (!phraseLanguages.includes(config.devLanguage)) {
+    throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
+  }
+
   const allVocabTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
 
   for (const loadedTranslation of allVocabTranslations) {
@@ -168,6 +262,11 @@ async function pull({
       defaultValues[key] = { ...defaultValues[key],
         ...allPhraseTranslations[config.devLanguage][core.getUniqueKey(key, loadedTranslation.namespace)]
       };
+    } // Only write a `_meta` field if necessary
+
+
+    if (Object.keys(loadedTranslation.metadata).length > 0) {
+      defaultValues._meta = loadedTranslation.metadata;
     }
 
     await writeFile(loadedTranslation.filePath, `${JSON.stringify(defaultValues, null, 2)}\n`);
@@ -205,14 +304,17 @@ async function pull({
 }
 
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
 async function push({
-  branch
+  branch,
+  deleteUnusedKeys: deleteUnusedKeys$1
 }, config) {
   const allLanguageTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
   trace(`Pushing translations to branch ${branch}`);
   const allLanguages = config.languages.map(v => v.name);
@@ -232,17 +334,37 @@ async function push({
         phraseTranslations[language] = {};
       }
 
+      const {
+        metadata: {
+          tags: sharedTags = []
+        }
+      } = loadedTranslation;
+
       for (const localKey of Object.keys(localTranslations)) {
         const phraseKey = core.getUniqueKey(localKey, loadedTranslation.namespace);
-        phraseTranslations[language][phraseKey] = localTranslations[localKey];
+        const {
+          tags = [],
+          ...localTranslation
+        } = localTranslations[localKey];
+
+        if (language === config.devLanguage) {
+          localTranslation.tags = [...tags, ...sharedTags];
+        }
+
+        phraseTranslations[language][phraseKey] = localTranslation;
       }
     }
   }
 
-  for (const language of allLanguages) {
-    if (phraseTranslations[language]) {
-      await pushTranslationsByLocale(phraseTranslations[language], language, branch);
-    }
+  const {
+    uploadId
+  } = await pushTranslations(phraseTranslations, {
+    devLanguage: config.devLanguage,
+    branch
+  });
+
+  if (deleteUnusedKeys$1) {
+    await deleteUnusedKeys(uploadId, branch);
   }
 }