@orderly.network/i18n 3.0.0-beta.1 → 3.0.0-beta.3

package/scripts/generateCsv.js

@@ -0,0 +1,36 @@
+const fs = require("fs-extra");
+const path = require("path");
+const { checkFileExists } = require("./utils");
+const { multiJson2Csv } = require("./json-csv-converter");
+const { defaultLanguages } = require("../dist");
+
+/** Generate a locale CSV file */
+async function generateCsv(inputDir, outputPath) {
+  const headers = [""];
+  const jsonList = [];
+
+  for (const item of defaultLanguages) {
+    headers.push(item.localCode);
+    const json = await fs.readJSON(
+      path.resolve(inputDir, `${item.localCode}.json`),
+      {
+        encoding: "utf8",
+      },
+    );
+    jsonList.push(json);
+  }
+
+  const csv = multiJson2Csv(jsonList, headers);
+
+  const csvPath = path.resolve(outputPath);
+
+  await checkFileExists(outputPath);
+
+  await fs.outputFile(outputPath, csv, { encoding: "utf8" });
+
+  console.log("generateCsv success =>", csvPath);
+}
+
+module.exports = {
+  generateCsv,
+};

package/scripts/generateEnJson.js

@@ -0,0 +1,11 @@
+const fs = require("fs-extra");
+const path = require("path");
+const { en } = require("../dist");
+
+async function generateEnJson() {
+  const outPath = path.resolve(__dirname, "../locales/en.json");
+  const jsonData = JSON.stringify(en, null, 2);
+  await fs.outputFile(outPath, jsonData, { encoding: "utf8" });
+}
+
+generateEnJson();

package/scripts/generateMissingKeys.js

@@ -0,0 +1,49 @@
+const fs = require("fs-extra");
+const path = require("path");
+const { getMissingKeys } = require("./json-csv-converter");
+const { checkFileExists, findJsonFiles } = require("./utils");
+const { LocaleEnum } = require("../dist");
+
+async function generateMissingKeys() {
+  const inputDir = path.resolve(__dirname, "../locales");
+  const jsonFiles = await findJsonFiles(inputDir);
+
+  // Sort input files by locale
+  jsonFiles.sort((a, b) => (b.startsWith(LocaleEnum.en) ? 1 : -1));
+
+  const jsonList = [];
+  const headers = [""];
+
+  for (const file of jsonFiles) {
+    const jsonPath = path.resolve(inputDir, file);
+
+    const json = await fs.readJSON(jsonPath, {
+      encoding: "utf8",
+    });
+
+    jsonList.push(json);
+
+    const fileName = path.basename(file, path.extname(file));
+    headers.push(fileName);
+  }
+
+  const errors = getMissingKeys(jsonList, headers);
+
+  const missingJson = {};
+  for (const locale of Object.values(errors)) {
+    for (const [key, value] of Object.entries(locale)) {
+      missingJson[key] = value;
+    }
+  }
+
+  const outputPath = path.resolve(inputDir, "extend/en.json");
+
+  // don't check file exists
+  // await checkFileExists(outputPath);
+
+  await fs.outputFile(outputPath, JSON.stringify(missingJson, null, 2), {
+    encoding: "utf8",
+  });
+}
+
+generateMissingKeys();

package/scripts/json-csv-converter.js

@@ -0,0 +1,286 @@
+const { validateI18nValue } = require("./utils");
+const fs = require("fs");
+const path = require("path");
+
+const separator = ".";
+
+function multiJson2Csv(jsonList, header) {
+  if (!Array.isArray(jsonList) || jsonList.length === 0) {
+    throw "Object array please.";
+  }
+
+  const ignoreKeys = loadIgnoreKeys();
+  const baseJson = jsonList[0];
+  const baseKeys = Object.keys(baseJson);
+  const errors = {};
+  let result = header.length ? [header] : [];
+  for (const key of baseKeys) {
+    const values = [];
+    for (const [index, json] of jsonList.entries()) {
+      const val = json[key] || "";
+      values.push(val);
+      // Skip validation if key is in ignore list
+      if (!shouldIgnoreKey(key, ignoreKeys)) {
+        const bool = validateI18nValue(val);
+        if (!bool.valid) {
+          const locale = header[index + 1];
+          if (!errors[locale]) {
+            errors[locale] = {};
+          }
+          errors[locale][key] = baseJson[key];
+        }
+      }
+    }
+    result.push(stringsToCsvLine([key, ...values]));
+  }
+  if (Object.keys(errors).length > 0) {
+    throw new Error(
+      "valid i18n value failed, please check the value of the following values: " +
+        JSON.stringify(errors, null, 4),
+    );
+  }
+  return result.join("\n");
+}
+
+function csv2multiJson(csv) {
+  if (typeof csv !== "string") throw "String please.";
+  const json = {};
+
+  const ignoreKeys = loadIgnoreKeys();
+  const lines = csvToLines(csv);
+  const csvLines = lines.filter(Boolean).map(parseCsvLine);
+
+  const headers = csvLines.shift();
+
+  const errors = {};
+  for (const [index, header] of headers.entries()) {
+    json[header] = {};
+    for (const line of csvLines) {
+      const [key, ...values] = line;
+      if (!key) {
+        continue;
+      }
+      const val = values[index];
+      json[header][key] = val;
+      // Skip validation if key is in ignore list
+      if (!shouldIgnoreKey(key, ignoreKeys)) {
+        const bool = validateI18nValue(val);
+        if (!bool.valid) {
+          if (!errors[header]) {
+            errors[header] = {};
+          }
+          errors[header][key] = values[0];
+        }
+      }
+    }
+
+    if (Object.values(json[header]).every((value) => !value)) {
+      delete json[header];
+    }
+  }
+
+  if (Object.keys(errors).length > 0) {
+    throw new Error(
+      "valid i18n value failed, please check the value of the following values: " +
+        JSON.stringify(errors, null, 4),
+    );
+  }
+
+  return json;
+}
+
+function json2Csv(json) {
+  if (typeof json !== "object" || !json) throw "Object please.";
+  return findKeyValues(json).join("\n");
+}
+
+function csv2Json(csv) {
+  if (typeof csv !== "string") throw "String please.";
+  const json = {};
+  parseCsvLines(csv).forEach((line) => {
+    const [path, value] = line;
+    const pathSplit = path.split(separator);
+    deepSet(json, pathSplit, value);
+  });
+  return json;
+}
+
+function findKeyValues(o, prefix, l = 0) {
+  let result = [];
+  for (const key of Object.keys(o)) {
+    const fullKey = prefix ? prefix + separator + key : key;
+    const value = o[key];
+    if (typeof value === "string") {
+      result.push(stringsToCsvLine([fullKey, value]));
+    } else if (typeof value === "object" && value) {
+      result = result.concat(findKeyValues(value, fullKey, l + 1));
+    }
+  }
+  return result;
+}
+
+function stringsToCsvLine(line) {
+  return line.map((s) => `"${s.replace(/"/g, '""')}"`).join(",");
+}
+
+function deepSet(o, path, value) {
+  if (path.length > 1) {
+    const key = path[0];
+    if (!o[key]) {
+      o[key] = {};
+    }
+    deepSet(o[key], path.slice(1), value);
+  } else {
+    o[path[0]] = value;
+  }
+}
+
+function parseCsvLines(csv) {
+  const lines = csvToLines(csv);
+  return lines.filter(Boolean).map(parseCsvLine);
+}
+
+function csvToLines(csv) {
+  return csv.split(/[\r\n]+/);
+}
+
+function parseCsvLine(line) {
+  const result = [];
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    switch (char) {
+      case ",":
+        continue;
+      case '"': {
+        i++;
+        const [str, i2] = parseCsvString(line, i);
+        i = i2;
+        result.push(str);
+        break;
+      }
+      default: {
+        const [str, i2] = parseUnquotedField(line, i);
+        i = i2;
+        result.push(str);
+        break;
+      }
+    }
+  }
+  return result;
+}
+
+/** Parses one field until the next comma or end of line (for unquoted CSV). */
+function parseUnquotedField(line, i) {
+  let result = "";
+  for (; i < line.length; i++) {
+    if (line[i] === ",") break;
+    result += line[i];
+  }
+  return [result, i];
+}
+
+function parseCsvString(line, i) {
+  let result = "";
+  loop: for (; i < line.length; i++) {
+    const char = line[i];
+    switch (char) {
+      case '"':
+        i++;
+        if (line[i] === '"') {
+          result += '"';
+        } else {
+          break loop;
+        }
+        break;
+      default:
+        result += char;
+    }
+  }
+  return [result, i];
+}
+
+function getMissingKeys(jsonList, header) {
+  if (!Array.isArray(jsonList) || jsonList.length === 0) {
+    throw "Object array please.";
+  }
+
+  const baseJson = jsonList[0];
+  const baseKeys = Object.keys(baseJson);
+  const errors = {};
+  for (const key of baseKeys) {
+    for (const [index, json] of jsonList.entries()) {
+      const val = json[key] || "";
+      const bool = validateI18nValue(val);
+      if (!bool.valid) {
+        const locale = header[index + 1];
+        if (!errors[locale]) {
+          errors[locale] = {};
+        }
+        errors[locale][key] = baseJson[key];
+      }
+    }
+  }
+  return errors;
+}
+
+// Load ignore validation keys configuration
+function loadIgnoreKeys() {
+  const configPath = path.join(__dirname, "../validation-ignore.json");
+  try {
+    if (fs.existsSync(configPath)) {
+      const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
+      const ignoreKeys = config.ignoreKeys || [];
+      // Convert config to matcher array: string or regex
+      return ignoreKeys
+        .map((item) => {
+          if (typeof item === "string") {
+            // If string starts and ends with /, treat as regex pattern
+            if (item.startsWith("/") && item.endsWith("/") && item.length > 2) {
+              try {
+                const pattern = item.slice(1, -1); // Remove leading and trailing /
+                return new RegExp(pattern);
+              } catch (error) {
+                console.warn(
+                  `Warning: Invalid regex pattern "${item}": ${error.message}`,
+                );
+                return item;
+              }
+            }
+            return item;
+          }
+          return item;
+        })
+        .filter(Boolean);
+    }
+  } catch (error) {
+    console.warn(
+      `Warning: Failed to load validation-ignore.json: ${error.message}`,
+    );
+  }
+  return [];
+}
+
+// Check if key should be ignored
+function shouldIgnoreKey(key, ignoreKeys) {
+  for (const matcher of ignoreKeys) {
+    if (matcher instanceof RegExp) {
+      if (matcher.test(key)) {
+        return true;
+      }
+    } else if (typeof matcher === "string") {
+      if (matcher === key) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+module.exports = {
+  separator,
+  json2Csv,
+  csv2Json,
+  multiJson2Csv,
+  csv2multiJson,
+  getMissingKeys,
+};

package/scripts/json2csv.js

@@ -0,0 +1,38 @@
+const fs = require("fs-extra");
+const path = require("path");
+const { multiJson2Csv } = require("./json-csv-converter");
+const { checkFileExists, findJsonFiles } = require("./utils");
+const { LocaleEnum } = require("../dist");
+
+/** Convert multiple locale JSON files to a locale CSV file */
+async function json2csv(inputDir, outputPath) {
+  const inputFiles = await findJsonFiles(inputDir);
+
+  // Sort input files by locale
+  inputFiles.sort((a, b) => (b.startsWith(LocaleEnum.en) ? 1 : -1));
+
+  const jsonList = [];
+  const headers = [""];
+  for (const filePath of inputFiles) {
+    const json = await fs.readJSON(path.resolve(inputDir, filePath), {
+      encoding: "utf8",
+    });
+    jsonList.push(json);
+    const fileName = path.basename(filePath, path.extname(filePath));
+    headers.push(fileName);
+  }
+
+  const csv = multiJson2Csv(jsonList, headers);
+
+  const csvPath = path.resolve(outputPath);
+
+  await checkFileExists(csvPath);
+
+  await fs.outputFile(csvPath, csv, { encoding: "utf8" });
+
+  console.log("json2csv success =>", csvPath);
+}
+
+module.exports = {
+  json2csv,
+};

package/scripts/mergeJson.js

@@ -0,0 +1,67 @@
+const fs = require("fs-extra");
+const path = require("path");
+const { checkFileExists, findJsonFiles } = require("./utils");
+const { LocaleEnum } = require("../dist");
+
+/**
+ * Merge default and extend JSON files back into one file
+ * @param {string} inputDir - The input directory containing both default and extend JSON files
+ * @param {string} outputDir - The output directory for merged JSON files
+ */
+async function mergeJson(inputDir, outputDir) {
+  const jsonFiles = await findJsonFiles(inputDir);
+
+  // Sort input files by locale
+  jsonFiles.sort((a, b) => (b.startsWith(LocaleEnum.en) ? 1 : -1));
+  let baseJson = {};
+
+  const extendDir = path.resolve(inputDir, "extend");
+
+  for (const [index, file] of jsonFiles.entries()) {
+    const defaultJsonPath = path.resolve(inputDir, file);
+    const extendJsonPath = path.resolve(extendDir, file);
+
+    // Read default JSON file
+    const defaultJson = await fs.readJSON(defaultJsonPath, {
+      encoding: "utf8",
+    });
+
+    // Read extend JSON file if it exists
+    let extendJson = {};
+    if (await fs.pathExists(extendJsonPath)) {
+      extendJson = await fs.readJSON(extendJsonPath, {
+        encoding: "utf8",
+      });
+    }
+
+    // Merge the JSON objects
+    const mergedJson = { ...defaultJson, ...extendJson };
+
+    let sortedJson = {};
+
+    // base json
+    if (index === 0) {
+      baseJson = mergedJson;
+      sortedJson = mergedJson;
+    } else {
+      for (const key of Object.keys(baseJson)) {
+        sortedJson[key] = mergedJson[key];
+      }
+    }
+
+    const outputPath = path.resolve(outputDir, file);
+
+    await fs.outputFile(outputPath, JSON.stringify(sortedJson, null, 2), {
+      encoding: "utf8",
+    });
+
+    console.log("mergeJson success =>", outputPath);
+  }
+
+  await fs.remove(extendDir);
+  console.log("remove extendDir =>", extendDir);
+}
+
+module.exports = {
+  mergeJson,
+};

package/scripts/separateJson.js

@@ -0,0 +1,50 @@
+const fs = require("fs-extra");
+const path = require("path");
+const { checkFileExists, findJsonFiles } = require("./utils");
+
+/**
+ * Separate json file default and extend key values based on the key
+ * @param {string} inputDir - The input directory for locale JSON files
+ * @param {string} outputDir - The output directory for locale JSON files
+ * @param {string} separateKey - The key to separate the json files
+ */
+async function separateJson(inputDir, outputDir, separateKey) {
+  const jsonFiles = await findJsonFiles(inputDir);
+
+  const separateKeys = separateKey.split(",");
+  for (const file of jsonFiles) {
+    const json = await fs.readJSON(path.resolve(inputDir, file), {
+      encoding: "utf8",
+    });
+
+    const defaultJson = {};
+    const extendJson = {};
+    Object.keys(json).forEach((key) => {
+      if (separateKeys.some((k) => key.startsWith(k))) {
+        extendJson[key] = json[key];
+      } else {
+        defaultJson[key] = json[key];
+      }
+    });
+
+    const jsonPath = path.resolve(outputDir, file);
+    await checkFileExists(jsonPath);
+
+    await fs.outputFile(jsonPath, JSON.stringify(defaultJson, null, 2), {
+      encoding: "utf8",
+    });
+
+    const extendPath = path.resolve(outputDir, "extend", file);
+    await checkFileExists(extendPath);
+
+    await fs.outputFile(extendPath, JSON.stringify(extendJson, null, 2), {
+      encoding: "utf8",
+    });
+
+    console.log("separateJson success =>", jsonPath, extendPath);
+  }
+}
+
+module.exports = {
+  separateJson,
+};

package/scripts/utils.js

@@ -0,0 +1,94 @@
+const fs = require("fs-extra");
+
+async function checkFileExists(filePath) {
+  if (await fs.exists(filePath)) {
+    throw new Error(`${filePath} already exists, please modify file path`);
+  }
+}
+
+const i18nValidErrors = {
+  string: "Value must be a string",
+  empty: "Value cannot be empty",
+  interpolation: "Invalid interpolation",
+  mismatchedClosingTag: (tagName) => `Mismatched closing tag: </${tagName}>`,
+  unclosedTag: (tagName) => `Unclosed tag: <${tagName}>`,
+};
+
+/**
+ * Validate the value of i18n
+ * @param {string} value - The value to validate
+ * @returns {Object} - The validation result
+ */
+function validateI18nValue(value) {
+  if (typeof value !== "string") {
+    return { valid: false, error: i18nValidErrors.string };
+  }
+
+  // 1. check if value is empty
+  if (value.trim() === "") {
+    return { valid: false, error: i18nValidErrors.empty };
+  }
+
+  // 2. check if placeholder format is correct (allow `{{variable}}`)
+  // allow `{{variable}}` or `{{ variable }}`
+  const interpolationRegex = /{{\s*[\w.-]+\s*}}/g;
+  const strippedInterpolation = value.replace(interpolationRegex, "");
+  if (
+    strippedInterpolation.includes("{{") ||
+    strippedInterpolation.includes("}}")
+  ) {
+    return { valid: false, error: i18nValidErrors.interpolation };
+  }
+  // mistaken single-brace placeholders like `{user.name}` (should be `{{user.name}}`)
+  if (/\{\s*[\w.-]+\s*\}/.test(strippedInterpolation)) {
+    return { valid: false, error: i18nValidErrors.interpolation };
+  }
+  // `{` without a closing `}` on the same line segment (typo vs `{{`)
+  if (/\{[^}]*$/.test(strippedInterpolation)) {
+    return { valid: false, error: i18nValidErrors.interpolation };
+  }
+
+  // 3. check if HTML tags are correctly closed (supports attributes, e.g. <script async src="...">)
+  const tagRegex = /<(\/?)\s*([a-zA-Z0-9][\w-]*)[^>]*>/g;
+  let stack = [];
+  let match;
+
+  while ((match = tagRegex.exec(value)) !== null) {
+    const [, slashPrefix, tagName] = match;
+    const fullTag = match[0];
+    const isClosing = slashPrefix === "/";
+    const isSelfClosing = !isClosing && /\/\s*>$/.test(fullTag.trim());
+
+    if (isSelfClosing) {
+      continue;
+    }
+    if (isClosing) {
+      if (stack.length === 0 || stack.pop() !== tagName) {
+        return {
+          valid: false,
+          error: i18nValidErrors.mismatchedClosingTag(tagName),
+        };
+      }
+    } else {
+      stack.push(tagName);
+    }
+  }
+
+  if (stack.length > 0) {
+    return { valid: false, error: i18nValidErrors.unclosedTag(stack.pop()) };
+  }
+
+  return { valid: true, error: null };
+}
+
+async function findJsonFiles(dir) {
+  const files = await fs.readdir(dir);
+  return files.filter((file) => file.endsWith(".json"));
+}
+
+module.exports = {
+  checkFileExists,
+  validateI18nValue,
+  i18nValidErrors,
+  findJsonFiles,
+};