@tandem-language-exchange/content-store 1.1.2 → 1.2.1

package/dist/node.js

@@ -2,7 +2,6 @@
 import {
   S3Client,
   PutObjectCommand,
-  CopyObjectCommand,
   GetObjectCommand
 } from "@aws-sdk/client-s3";
 var ContentStore = class {
@@ -18,14 +17,6 @@ var ContentStore = class {
     });
     this.bucket = cfg.bucket;
   }
-  /** {cms}-{contentType}-{timestamp}.json */
-  buildVersionedKey(cms, contentType, timestamp) {
-    return `${cms}-${contentType}-${timestamp}.json`;
-  }
-  /** {cms}-{contentType}.json  (always points at the latest version) */
-  buildLatestKey(cms, contentType) {
-    return `${cms}-${contentType}.json`;
-  }
   async upload(key, data) {
     await this.client.send(
       new PutObjectCommand({
@@ -37,6 +28,18 @@ var ContentStore = class {
     );
     return key;
   }
+  /** Raw UTF-8 body (e.g. Lingohub file bytes as text). */
+  async uploadRaw(key, body, contentType) {
+    await this.client.send(
+      new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: key,
+        Body: body,
+        ContentType: contentType
+      })
+    );
+    return key;
+  }
   async download(key) {
     const response = await this.client.send(
       new GetObjectCommand({ Bucket: this.bucket, Key: key })
@@ -45,27 +48,104 @@ var ContentStore = class {
     if (!body) throw new Error(`Empty response for key: ${key}`);
     return JSON.parse(body);
   }
-  /**
-   * Copies a versioned object to the "latest" key so that it always reflects
-   * the most recent sync while older timestamped versions are retained.
-   */
-  async copyToLatest(sourceKey, cms, contentType) {
-    const latestKey = this.buildLatestKey(cms, contentType);
-    await this.client.send(
-      new CopyObjectCommand({
-        Bucket: this.bucket,
-        CopySource: `${this.bucket}/${sourceKey}`,
-        Key: latestKey,
-        ContentType: "application/json"
-      })
+  /** Raw UTF-8 body from S3 (no JSON.parse). */
+  async downloadRaw(key) {
+    const response = await this.client.send(
+      new GetObjectCommand({ Bucket: this.bucket, Key: key })
     );
-    return latestKey;
+    const body = await response.Body?.transformToString();
+    if (!body) throw new Error(`Empty response for key: ${key}`);
+    return body;
   }
 };
+var buildCmsObjectKey = (cms, contentType) => {
+  return `${cms}-${contentType}.json`;
+};
+var buildTranslationObjectKey = (project, fileName, locale) => {
+  return `lingohub-${project}.${fileName.replaceAll("[locale]", locale)}`;
+};
 
 // src/shared/bundles.ts
 import fs from "fs/promises";
-import path from "path";
+import path2 from "path";
+
+// src/shared/s3-retry.ts
+function computeDelay(attempt, baseDelayMs, maxDelayMs) {
+  const exponential = baseDelayMs * Math.pow(2, attempt);
+  const jitter = Math.random() * baseDelayMs;
+  return Math.min(exponential + jitter, maxDelayMs);
+}
+function getDefaultS3RetryConfig() {
+  return {
+    maxRetries: parseInt(
+      process.env.S3_RETRY_MAX_RETRIES ?? process.env.RETRY_MAX_RETRIES ?? "5",
+      10
+    ),
+    baseDelayMs: parseInt(
+      process.env.S3_RETRY_BASE_DELAY_MS ?? process.env.RETRY_BASE_DELAY_MS ?? "1000",
+      10
+    ),
+    maxDelayMs: parseInt(
+      process.env.S3_RETRY_MAX_DELAY_MS ?? process.env.RETRY_MAX_DELAY_MS ?? "60000",
+      10
+    )
+  };
+}
+function isRetryableS3DownloadError(err) {
+  if (err === null || err === void 0) return false;
+  if (typeof err === "object") {
+    const e = err;
+    const status = e.$metadata?.httpStatusCode;
+    if (status === 404 || status === 403 || status === 401 || status === 400) {
+      return false;
+    }
+    if (status !== void 0 && (status === 408 || status === 429 || status === 500 || status === 502 || status === 503 || status === 504)) {
+      return true;
+    }
+    const name = e.name ?? "";
+    const code = e.Code ?? "";
+    if (/SlowDown|Throttl|Timeout|TooManyRequests|ServiceUnavailable|InternalError/i.test(
+      name
+    ) || /SlowDown|Throttl/i.test(code)) {
+      return true;
+    }
+  }
+  if (err instanceof Error) {
+    const m = err.message;
+    if (/ECONNRESET|ETIMEDOUT|EPIPE|ECONNREFUSED|socket hang up|getaddrinfo/i.test(
+      m
+    )) {
+      return true;
+    }
+  }
+  return false;
+}
+async function withS3Retry(fn, { maxRetries, baseDelayMs, maxDelayMs }) {
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      if (!isRetryableS3DownloadError(err)) {
+        throw err;
+      }
+      if (attempt === maxRetries) {
+        throw err;
+      }
+      const delay = computeDelay(attempt, baseDelayMs, maxDelayMs);
+      console.warn(
+        `  S3 request failed (attempt ${attempt + 1}/${maxRetries + 1}): ${err instanceof Error ? err.message : String(err)}. Retrying in ${Math.round(delay)}ms\u2026`
+      );
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+  throw new Error("withS3Retry: unreachable");
+}
+async function downloadWithRetry(store, key, cfg) {
+  return withS3Retry(() => store.download(key), cfg);
+}
+async function downloadRawWithRetry(store, key, cfg) {
+  return withS3Retry(() => store.downloadRaw(key), cfg);
+}
 
 // src/shared/trimDepth.ts
 function trimDepth(value, remaining) {
@@ -96,6 +176,301 @@ function trimDepth(value, remaining) {
   return result;
 }
 
+// src/shared/lingohub.ts
+var defaultLocales = ["en", "fr", "de", "es", "it", "pt-br", "ru", "zh-hans", "zh-hant", "ko", "ja"];
+var localeMapping = {
+  ios: {
+    "pt-br": "pt",
+    "zh-hans": "zh-Hans",
+    "zh-hant": "zh-Hant"
+  },
+  android: {
+    "pt-br": "pt",
+    "zh-hans": "zh-Hans-CN",
+    "zh-hant": "zh-TW"
+  }
+};
+var allProjects = {
+  "tandem-(new-website)": [
+    {
+      fileName: "Website.[locale].json",
+      type: "json"
+    }
+  ],
+  "tandem-(website)": [
+    {
+      fileName: "[locale].json",
+      type: "json"
+    },
+    {
+      fileName: "AI.[locale].json",
+      type: "json"
+    },
+    {
+      fileName: "languages.[locale].json",
+      type: "json"
+    }
+  ],
+  "tandem": [
+    {
+      fileName: "InfoPlist.[locale].strings",
+      type: "strings",
+      localeMapping: localeMapping.ios
+    },
+    {
+      fileName: "Localizable.[locale].strings",
+      type: "strings",
+      localeMapping: localeMapping.ios
+    },
+    {
+      fileName: "MainiPad.[locale].strings",
+      type: "strings",
+      localeMapping: localeMapping.ios
+    },
+    {
+      fileName: "Main.[locale].strings",
+      type: "strings",
+      localeMapping: localeMapping.ios
+    }
+  ],
+  "tandem-(android)": [
+    {
+      fileName: "accessibility_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "call_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "cert_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "chat_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "checklist_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "clubs_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "common_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "community_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "correction_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "country_names.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "emoji_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "errors_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "expressions_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "gif_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "guidelines_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "languages_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "localizable2.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "login_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "myprofile_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "onb_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "parties_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "pro_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "pro_screen_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "push_notification_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "reporting_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    },
+    {
+      fileName: "translation_localizable.[locale].xml",
+      type: "xml",
+      localeMapping: localeMapping.android
+    }
+  ],
+  "tandem-(web-invites)": [
+    {
+      fileName: "[locale].json",
+      type: "json"
+    }
+  ]
+};
+
+// src/shared/translationResource.ts
+import path from "path";
+
+// src/shared/utils.ts
+import convert from "xml-js";
+import set from "lodash.set";
+import merge from "lodash.merge";
+var transformObjectToFlat = (data) => {
+  const result = {};
+  const flatten = (obj, path3 = []) => {
+    Object.entries(obj).forEach(([key, value]) => {
+      if (typeof value === "object") {
+        flatten(value, path3.concat(key));
+      } else {
+        result[path3.concat(key).join(".")] = value;
+      }
+    });
+  };
+  flatten(data);
+  return result;
+};
+var convertXMLToJS = (xml) => {
+  const converted = convert.xml2js(xml, {
+    ignoreComment: true,
+    ignoreDeclaration: true,
+    ignoreInstruction: true,
+    compact: true
+  });
+  let mapped = {};
+  converted.resources.string.forEach((item) => {
+    mapped = {
+      ...mapped,
+      [item._attributes.name]: item._text
+    };
+  });
+  return mapped;
+};
+var parseIOSStrings = (strings) => {
+  const parsedObj = {};
+  strings.split("\n").filter((line) => line.startsWith('"') && line.endsWith(";")).map((line) => line.trim().slice(0, -1)).forEach((line) => {
+    let [key, value] = line.split(" = ");
+    if (!key || !value) return;
+    key = key.slice(1, -1);
+    value = value.slice(1, -1);
+    parsedObj[key] = value;
+  });
+  return parsedObj;
+};
+
+// src/shared/translationResource.ts
+function parseTranslationResourceRaw(raw, resource) {
+  if (resource.type === "json") {
+    return JSON.parse(raw);
+  }
+  if (resource.type === "strings") {
+    return parseIOSStrings(raw);
+  }
+  if (resource.type === "xml") {
+    return convertXMLToJS(raw);
+  }
+  throw new Error(`Unsupported resource type: ${resource.type}`);
+}
+function toFlatStringMap(parsed) {
+  if (parsed === null || parsed === void 0) return {};
+  if (typeof parsed === "string") return { value: parsed };
+  if (typeof parsed !== "object") return { value: String(parsed) };
+  if (Array.isArray(parsed)) {
+    const out2 = {};
+    parsed.forEach((v, i) => {
+      out2[String(i)] = typeof v === "object" && v !== null ? JSON.stringify(v) : String(v);
+    });
+    return out2;
+  }
+  const flat = transformObjectToFlat(parsed);
+  const out = {};
+  for (const [k, v] of Object.entries(flat)) {
+    if (v === null || v === void 0) {
+      out[k] = "";
+    } else if (typeof v === "object") {
+      out[k] = JSON.stringify(v);
+    } else {
+      out[k] = String(v);
+    }
+  }
+  return out;
+}
+function translationJsonOutputPath(outputDir, objectKey) {
+  if (objectKey.endsWith(".json")) {
+    return path.resolve(outputDir, objectKey);
+  }
+  return path.resolve(outputDir, `${objectKey}.json`);
+}
+
 // src/shared/bundles.ts
 function getAtPath(obj, dottedPath) {
   const parts = dottedPath.split(".").filter((p) => p.length > 0);
@@ -126,8 +501,8 @@ function isEmptyValue(value) {
   return typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 0;
 }
 function matchesFieldFilter(item, key, expected) {
-  const { path: path2, operator } = parseFieldKey(key);
-  const at = getAtPath(item, path2);
+  const { path: path3, operator } = parseFieldKey(key);
+  const at = getAtPath(item, path3);
   if (operator === "exists") {
     if (expected !== true && expected !== false) return false;
     const empty = !at.found || isEmptyValue(at.value);
@@ -153,26 +528,117 @@ function setNestedAt(target, dottedPath, value) {
   }
   setNestedAt(nested, rest, value);
 }
-async function fetchBundles(store, outputDir, options) {
+async function fetchCmsBundles(store, outputDir, options) {
   const { cms, contentTypes } = options;
+  const retry = options.retry ?? getDefaultS3RetryConfig();
   await fs.mkdir(outputDir, { recursive: true });
   const result = {};
   await Promise.all(
     contentTypes.map(async (contentType) => {
-      const key = store.buildLatestKey(cms, contentType);
-      const data = await store.download(key);
-      const filePath = path.resolve(
-        outputDir,
-        `${cms}-${contentType}.json`
-      );
+      const key = buildCmsObjectKey(cms, contentType);
+      const data = await downloadWithRetry(store, key, retry);
+      const filePath = path2.resolve(outputDir, key);
       await fs.writeFile(filePath, JSON.stringify(data, null, 2), "utf-8");
       result[contentType] = filePath;
     })
   );
   return result;
 }
-async function queryBundle(outputDir, cms, contentType, options = {}) {
-  const filePath = path.resolve(
+async function fetchTranslationBundles(store, outputDir, options) {
+  const { projects, locales } = options;
+  const retry = options.retry ?? getDefaultS3RetryConfig();
+  await fs.mkdir(outputDir, { recursive: true });
+  const result = {};
+  const localesToSync = locales && locales.length ? locales : defaultLocales;
+  await Promise.all(
+    projects.map(async (project) => {
+      const resources = allProjects[project];
+      if (!resources?.length) {
+        return;
+      }
+      const resourceTasks = [];
+      for (const resource of resources) {
+        for (const loc of localesToSync) {
+          const locale = resource.localeMapping && resource.localeMapping[loc] ? resource.localeMapping[loc] : loc;
+          const objectKey = buildTranslationObjectKey(
+            project,
+            resource.fileName,
+            locale
+          );
+          resourceTasks.push({ objectKey, resource });
+        }
+      }
+      await Promise.all(
+        resourceTasks.map(async ({ objectKey, resource }) => {
+          const raw = await downloadRawWithRetry(store, objectKey, retry);
+          const parsed = parseTranslationResourceRaw(raw, resource);
+          const filePath = translationJsonOutputPath(outputDir, objectKey);
+          await fs.mkdir(path2.dirname(filePath), { recursive: true });
+          await fs.writeFile(
+            filePath,
+            JSON.stringify(parsed, null, 2),
+            "utf-8"
+          );
+          if (!result[project]) {
+            result[project] = {};
+          }
+          result[project][objectKey] = filePath;
+        })
+      );
+    })
+  );
+  return result;
+}
+async function fetchMergedTranslationBundles(store, outputDir, options) {
+  const { projects, locales } = options;
+  const retry = options.retry ?? getDefaultS3RetryConfig();
+  await fs.mkdir(outputDir, { recursive: true });
+  const localesToSync = locales && locales.length ? locales : defaultLocales;
+  const tasks = [];
+  for (const project of projects) {
+    const resources = allProjects[project];
+    if (!resources?.length) continue;
+    for (const resource of resources) {
+      for (const loc of localesToSync) {
+        const mappedLocale = resource.localeMapping && resource.localeMapping[loc] ? resource.localeMapping[loc] : loc;
+        const objectKey = buildTranslationObjectKey(
+          project,
+          resource.fileName,
+          mappedLocale
+        );
+        tasks.push({ catalogLocale: loc, objectKey, resource });
+      }
+    }
+  }
+  const results = await Promise.all(
+    tasks.map(async ({ catalogLocale, objectKey, resource }) => {
+      const raw = await downloadRawWithRetry(store, objectKey, retry);
+      const parsed = parseTranslationResourceRaw(raw, resource);
+      const stringMap = toFlatStringMap(parsed);
+      return { catalogLocale, stringMap };
+    })
+  );
+  const merged = {};
+  for (const loc of localesToSync) {
+    merged[loc] = {};
+  }
+  for (const { catalogLocale, stringMap } of results) {
+    Object.assign(merged[catalogLocale], stringMap);
+  }
+  const out = {};
+  for (const loc of localesToSync) {
+    const filePath = path2.resolve(outputDir, `${loc}.json`);
+    await fs.writeFile(
+      filePath,
+      JSON.stringify(merged[loc], null, 2),
+      "utf-8"
+    );
+    out[loc] = filePath;
+  }
+  return out;
+}
+async function queryCmsBundle(outputDir, cms, contentType, options = {}) {
+  const filePath = path2.resolve(
     outputDir,
     `${cms}-${contentType}.json`
   );
@@ -222,21 +688,41 @@ var ContentStoreSDK = class {
    *
    * @returns A map of contentType to absolute file path.
    */
-  async fetchBundles(options) {
-    return fetchBundles(this.store, this.outputDir, options);
+  async fetchCmsBundles(options) {
+    return fetchCmsBundles(this.store, this.outputDir, options);
+  }
+  /**
+   * Downloads translation bundles from S3 and writes them as JSON files
+   * to `outputDir`.
+   *
+   * @returns Per project, a map of S3 object key to absolute file path.
+   */
+  async fetchTranslationBundles(options) {
+    return fetchTranslationBundles(this.store, this.outputDir, options);
+  }
+  /**
+   * Downloads all translation resources for the given projects/locales, parses them,
+   * and writes one merged `{locale}.json` per locale (string key/value map; duplicate keys:
+   * last wins).
+   */
+  async fetchMergedTranslationBundles(options) {
+    return fetchMergedTranslationBundles(this.store, this.outputDir, options);
   }
   /**
    * Queries a previously fetched bundle from the local filesystem.
    */
-  async queryBundle(cms, contentType, options = {}) {
-    return queryBundle(this.outputDir, cms, contentType, options);
+  async queryCmsBundle(cms, contentType, options = {}) {
+    return queryCmsBundle(this.outputDir, cms, contentType, options);
   }
 };
 export {
   ContentStore,
   ContentStoreSDK,
-  fetchBundles,
-  queryBundle,
+  fetchCmsBundles,
+  fetchMergedTranslationBundles,
+  fetchTranslationBundles,
+  getDefaultS3RetryConfig,
+  queryCmsBundle,
   trimDepth
 };
 //# sourceMappingURL=node.js.map