@vocab/phrase 0.0.0-phrase-pull-dev-language-202281412540 → 0.0.0-tags-support-2023174816

package/dist/declarations/src/csv.d.ts

@@ -0,0 +1,10 @@
+import type { TranslationsByLanguage } from '@vocab/types';
+export declare function translationsToCsv(translations: TranslationsByLanguage, devLanguage: string): {
+    csvString: string;
+    localeMapping: {
+        [k: string]: number;
+    };
+    keyIndex: number;
+    commentIndex: number;
+    tagColumn: number;
+};

package/dist/declarations/src/phrase-api.d.ts

@@ -1,7 +1,12 @@
-import { TranslationsByKey } from './../../types/src/index';
 import type { TranslationsByLanguage } from '@vocab/types';
 import fetch from 'node-fetch';
-export declare function callPhrase(relativePath: string, options?: Parameters<typeof fetch>[1]): Promise<any>;
+export declare function callPhrase<T = any>(relativePath: string, options?: Parameters<typeof fetch>[1]): Promise<T>;
 export declare function pullAllTranslations(branch: string): Promise<TranslationsByLanguage>;
-export declare function pushTranslationsByLocale(contents: TranslationsByKey, locale: string, branch: string): Promise<void>;
+export declare function pushTranslations(translationsByLanguage: TranslationsByLanguage, { devLanguage, branch }: {
+    devLanguage: string;
+    branch: string;
+}): Promise<{
+    uploadId: string;
+}>;
+export declare function deleteUnusedKeys(uploadId: string, branch: string): Promise<void>;
 export declare function ensureBranch(branch: string): Promise<void>;

package/dist/declarations/src/pull-translations.d.ts

@@ -1,6 +1,7 @@
 import type { UserConfig } from '@vocab/types';
 interface PullOptions {
     branch?: string;
+    deleteUnusedKeys?: boolean;
 }
 export declare function pull({ branch }: PullOptions, config: UserConfig): Promise<void>;
 export {};

package/dist/declarations/src/push-translations.d.ts

@@ -1,9 +1,11 @@
 import { UserConfig } from '@vocab/types';
 interface PushOptions {
     branch: string;
+    deleteUnusedKeys?: boolean;
 }
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
-export declare function push({ branch }: PushOptions, config: UserConfig): Promise<void>;
+export declare function push({ branch, deleteUnusedKeys }: PushOptions, config: UserConfig): Promise<void>;
 export {};

package/dist/vocab-phrase.cjs.dev.js

@@ -9,6 +9,7 @@ var FormData = require('form-data');
 var fetch = require('node-fetch');
 var chalk = require('chalk');
 var debug = require('debug');
+var sync = require('csv-stringify/sync');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }
 
@@ -27,6 +28,44 @@ const log = (...params) => {
   console.log(chalk__default['default'].yellow('Vocab'), ...params);
 };
 
+function translationsToCsv(translations, devLanguage) {
+  const languages = Object.keys(translations);
+  const altLanguages = languages.filter(language => language !== devLanguage);
+  const devLanguageTranslations = translations[devLanguage];
+  const csv = Object.entries(devLanguageTranslations).map(([key, {
+    message,
+    description,
+    tags
+  }]) => {
+    const altTranslationMessages = altLanguages.map(language => {
+      var _translations$languag, _translations$languag2;
+
+      return (_translations$languag = translations[language]) === null || _translations$languag === void 0 ? void 0 : (_translations$languag2 = _translations$languag[key]) === null || _translations$languag2 === void 0 ? void 0 : _translations$languag2.message;
+    });
+    return [message, ...altTranslationMessages, key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
+  }); // Not spreading `languages` to ensure correct ordering of dev language first
+  // then alt languages
+
+  const csvString = sync.stringify(csv, {
+    delimiter: ',',
+    header: false
+  }); // Column indices start at 1
+
+  const localeMapping = Object.fromEntries(languages.map((language, index) => [language, index + 1]));
+  const keyIndex = languages.length + 1;
+  const commentIndex = keyIndex + 1;
+  const tagColumn = commentIndex + 1;
+  return {
+    csvString,
+    localeMapping,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  };
+}
+
+/* eslint-disable no-console */
+
 function _callPhrase(path, options = {}) {
   const phraseApiToken = process.env.PHRASE_API_TOKEN;
 
@@ -44,14 +83,14 @@ function _callPhrase(path, options = {}) {
   }).then(async response => {
     console.log(`${path}: ${response.status} - ${response.statusText}`);
     console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
-    console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
+    trace('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
     // console.log(Array.from(r.headers.entries()));
 
     try {
       var _response$headers$get;
 
       const result = await response.json();
-      console.log(`Internal Result (Length: ${result.length})\n`);
+      trace(`Internal Result (Length: ${result.length})\n`);
 
       if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
         var _response$headers$get2, _response$headers$get3;
@@ -59,10 +98,10 @@ function _callPhrase(path, options = {}) {
         const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
 
         if (!nextPageUrl) {
-          throw new Error('Cant parse next page URL');
+          throw new Error("Can't parse next page URL");
         }
 
-        console.log('Results recieved with next page: ', nextPageUrl);
+        console.log('Results received with next page: ', nextPageUrl);
         const nextPageResult = await _callPhrase(nextPageUrl, options);
         return [...result, ...nextPageResult];
       }
@@ -109,23 +148,73 @@ async function pullAllTranslations(branch) {
 
   return translations;
 }
-async function pushTranslationsByLocale(contents, locale, branch) {
+async function pushTranslations(translationsByLanguage, {
+  devLanguage,
+  branch
+}) {
   const formData = new FormData__default['default']();
-  const fileContents = Buffer.from(JSON.stringify(contents));
+  const {
+    csvString,
+    localeMapping,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  } = translationsToCsv(translationsByLanguage, devLanguage);
+  const fileContents = Buffer.from(csvString);
   formData.append('file', fileContents, {
-    contentType: 'application/json',
-    filename: `${locale}.json`
+    contentType: 'text/csv',
+    filename: `translations.csv`
   });
-  formData.append('file_format', 'json');
-  formData.append('locale_id', locale);
+  formData.append('file_format', 'csv');
   formData.append('branch', branch);
   formData.append('update_translations', 'true');
-  trace('Starting to upload:', locale);
-  await callPhrase(`uploads`, {
+
+  for (const [locale, index] of Object.entries(localeMapping)) {
+    formData.append(`locale_mapping[${locale}]`, index);
+  }
+
+  formData.append('format_options[key_index]', keyIndex);
+  formData.append('format_options[comment_index]', commentIndex);
+  formData.append('format_options[tag_column]', tagColumn);
+  formData.append('format_options[enable_pluralization]', 'false');
+  log('Uploading translations');
+  const res = await callPhrase(`uploads`, {
     method: 'POST',
     body: formData
   });
-  log('Successfully Uploaded:', locale, '\n');
+  trace('Upload result:\n', res); // TODO: Figure out error handling
+
+  const {
+    id
+  } = res;
+
+  if (id) {
+    log('Upload ID:', id, '\n');
+    log('Successfully Uploaded\n');
+  } else {
+    log('Error uploading');
+    throw new Error('Error uploading');
+  }
+
+  return {
+    uploadId: id
+  };
+}
+async function deleteUnusedKeys(uploadId, branch) {
+  const query = `unmentioned_in_upload:${uploadId}`;
+  const {
+    records_affected
+  } = await callPhrase('keys', {
+    method: 'DELETE',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      branch,
+      q: query
+    })
+  });
+  log('Successfully deleted', records_affected, 'unused keys from branch', branch);
 }
 async function ensureBranch(branch) {
   await callPhrase(`branches`, {
@@ -137,7 +226,7 @@ async function ensureBranch(branch) {
       name: branch
     })
   });
-  trace('Created branch:', branch);
+  log('Created branch:', branch);
 }
 
 async function pull({
@@ -149,19 +238,16 @@ async function pull({
   const allPhraseTranslations = await pullAllTranslations(branch);
   trace(`Pulling translations from Phrase for languages ${config.devLanguage} and ${alternativeLanguages.join(', ')}`);
   const phraseLanguages = Object.keys(allPhraseTranslations);
-  const phraseLanguagesWithTranslations = phraseLanguages.filter(language => {
-    const phraseTranslationsForLanguage = allPhraseTranslations[language];
-    return Object.keys(phraseTranslationsForLanguage).length > 0;
-  });
-  trace(`Found Phrase translations for languages ${phraseLanguagesWithTranslations.join(', ')}`);
+  trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
 
-  if (!phraseLanguagesWithTranslations.includes(config.devLanguage)) {
-    throw new Error(`Phrase did not return any translations for dev language "${config.devLanguage}".\nEnsure you have configured your Phrase project for your dev language, and have pushed your translations.`);
+  if (!phraseLanguages.includes(config.devLanguage)) {
+    throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
   }
 
   const allVocabTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
 
   for (const loadedTranslation of allVocabTranslations) {
@@ -216,14 +302,17 @@ async function pull({
 }
 
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
 async function push({
-  branch
+  branch,
+  deleteUnusedKeys: deleteUnusedKeys$1
 }, config) {
   const allLanguageTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
   trace(`Pushing translations to branch ${branch}`);
   const allLanguages = config.languages.map(v => v.name);
@@ -250,10 +339,15 @@ async function push({
     }
   }
 
-  for (const language of allLanguages) {
-    if (phraseTranslations[language]) {
-      await pushTranslationsByLocale(phraseTranslations[language], language, branch);
-    }
+  const {
+    uploadId
+  } = await pushTranslations(phraseTranslations, {
+    devLanguage: config.devLanguage,
+    branch
+  });
+
+  if (deleteUnusedKeys$1) {
+    await deleteUnusedKeys(uploadId, branch);
   }
 }
 

package/dist/vocab-phrase.cjs.prod.js

@@ -9,6 +9,7 @@ var FormData = require('form-data');
 var fetch = require('node-fetch');
 var chalk = require('chalk');
 var debug = require('debug');
+var sync = require('csv-stringify/sync');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }
 
@@ -27,6 +28,44 @@ const log = (...params) => {
   console.log(chalk__default['default'].yellow('Vocab'), ...params);
 };
 
+function translationsToCsv(translations, devLanguage) {
+  const languages = Object.keys(translations);
+  const altLanguages = languages.filter(language => language !== devLanguage);
+  const devLanguageTranslations = translations[devLanguage];
+  const csv = Object.entries(devLanguageTranslations).map(([key, {
+    message,
+    description,
+    tags
+  }]) => {
+    const altTranslationMessages = altLanguages.map(language => {
+      var _translations$languag, _translations$languag2;
+
+      return (_translations$languag = translations[language]) === null || _translations$languag === void 0 ? void 0 : (_translations$languag2 = _translations$languag[key]) === null || _translations$languag2 === void 0 ? void 0 : _translations$languag2.message;
+    });
+    return [message, ...altTranslationMessages, key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
+  }); // Not spreading `languages` to ensure correct ordering of dev language first
+  // then alt languages
+
+  const csvString = sync.stringify(csv, {
+    delimiter: ',',
+    header: false
+  }); // Column indices start at 1
+
+  const localeMapping = Object.fromEntries(languages.map((language, index) => [language, index + 1]));
+  const keyIndex = languages.length + 1;
+  const commentIndex = keyIndex + 1;
+  const tagColumn = commentIndex + 1;
+  return {
+    csvString,
+    localeMapping,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  };
+}
+
+/* eslint-disable no-console */
+
 function _callPhrase(path, options = {}) {
   const phraseApiToken = process.env.PHRASE_API_TOKEN;
 
@@ -44,14 +83,14 @@ function _callPhrase(path, options = {}) {
   }).then(async response => {
     console.log(`${path}: ${response.status} - ${response.statusText}`);
     console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
-    console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
+    trace('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
     // console.log(Array.from(r.headers.entries()));
 
     try {
       var _response$headers$get;
 
       const result = await response.json();
-      console.log(`Internal Result (Length: ${result.length})\n`);
+      trace(`Internal Result (Length: ${result.length})\n`);
 
       if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
         var _response$headers$get2, _response$headers$get3;
@@ -59,10 +98,10 @@ function _callPhrase(path, options = {}) {
         const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
 
         if (!nextPageUrl) {
-          throw new Error('Cant parse next page URL');
+          throw new Error("Can't parse next page URL");
         }
 
-        console.log('Results recieved with next page: ', nextPageUrl);
+        console.log('Results received with next page: ', nextPageUrl);
         const nextPageResult = await _callPhrase(nextPageUrl, options);
         return [...result, ...nextPageResult];
       }
@@ -109,23 +148,73 @@ async function pullAllTranslations(branch) {
 
   return translations;
 }
-async function pushTranslationsByLocale(contents, locale, branch) {
+async function pushTranslations(translationsByLanguage, {
+  devLanguage,
+  branch
+}) {
   const formData = new FormData__default['default']();
-  const fileContents = Buffer.from(JSON.stringify(contents));
+  const {
+    csvString,
+    localeMapping,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  } = translationsToCsv(translationsByLanguage, devLanguage);
+  const fileContents = Buffer.from(csvString);
   formData.append('file', fileContents, {
-    contentType: 'application/json',
-    filename: `${locale}.json`
+    contentType: 'text/csv',
+    filename: `translations.csv`
   });
-  formData.append('file_format', 'json');
-  formData.append('locale_id', locale);
+  formData.append('file_format', 'csv');
   formData.append('branch', branch);
   formData.append('update_translations', 'true');
-  trace('Starting to upload:', locale);
-  await callPhrase(`uploads`, {
+
+  for (const [locale, index] of Object.entries(localeMapping)) {
+    formData.append(`locale_mapping[${locale}]`, index);
+  }
+
+  formData.append('format_options[key_index]', keyIndex);
+  formData.append('format_options[comment_index]', commentIndex);
+  formData.append('format_options[tag_column]', tagColumn);
+  formData.append('format_options[enable_pluralization]', 'false');
+  log('Uploading translations');
+  const res = await callPhrase(`uploads`, {
     method: 'POST',
     body: formData
   });
-  log('Successfully Uploaded:', locale, '\n');
+  trace('Upload result:\n', res); // TODO: Figure out error handling
+
+  const {
+    id
+  } = res;
+
+  if (id) {
+    log('Upload ID:', id, '\n');
+    log('Successfully Uploaded\n');
+  } else {
+    log('Error uploading');
+    throw new Error('Error uploading');
+  }
+
+  return {
+    uploadId: id
+  };
+}
+async function deleteUnusedKeys(uploadId, branch) {
+  const query = `unmentioned_in_upload:${uploadId}`;
+  const {
+    records_affected
+  } = await callPhrase('keys', {
+    method: 'DELETE',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      branch,
+      q: query
+    })
+  });
+  log('Successfully deleted', records_affected, 'unused keys from branch', branch);
 }
 async function ensureBranch(branch) {
   await callPhrase(`branches`, {
@@ -137,7 +226,7 @@ async function ensureBranch(branch) {
       name: branch
     })
   });
-  trace('Created branch:', branch);
+  log('Created branch:', branch);
 }
 
 async function pull({
@@ -149,19 +238,16 @@ async function pull({
   const allPhraseTranslations = await pullAllTranslations(branch);
   trace(`Pulling translations from Phrase for languages ${config.devLanguage} and ${alternativeLanguages.join(', ')}`);
   const phraseLanguages = Object.keys(allPhraseTranslations);
-  const phraseLanguagesWithTranslations = phraseLanguages.filter(language => {
-    const phraseTranslationsForLanguage = allPhraseTranslations[language];
-    return Object.keys(phraseTranslationsForLanguage).length > 0;
-  });
-  trace(`Found Phrase translations for languages ${phraseLanguagesWithTranslations.join(', ')}`);
+  trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
 
-  if (!phraseLanguagesWithTranslations.includes(config.devLanguage)) {
-    throw new Error(`Phrase did not return any translations for dev language "${config.devLanguage}".\nEnsure you have configured your Phrase project for your dev language, and have pushed your translations.`);
+  if (!phraseLanguages.includes(config.devLanguage)) {
+    throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
   }
 
   const allVocabTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
 
   for (const loadedTranslation of allVocabTranslations) {
@@ -216,14 +302,17 @@ async function pull({
 }
 
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
 async function push({
-  branch
+  branch,
+  deleteUnusedKeys: deleteUnusedKeys$1
 }, config) {
   const allLanguageTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
   trace(`Pushing translations to branch ${branch}`);
   const allLanguages = config.languages.map(v => v.name);
@@ -250,10 +339,15 @@ async function push({
     }
   }
 
-  for (const language of allLanguages) {
-    if (phraseTranslations[language]) {
-      await pushTranslationsByLocale(phraseTranslations[language], language, branch);
-    }
+  const {
+    uploadId
+  } = await pushTranslations(phraseTranslations, {
+    devLanguage: config.devLanguage,
+    branch
+  });
+
+  if (deleteUnusedKeys$1) {
+    await deleteUnusedKeys(uploadId, branch);
   }
 }