@braze/sanity-plugin-ai-translation 0.2.32 → 0.2.36

package/dist/index.js

@@ -1799,6 +1799,146 @@ const getExistingTranslationId = (translationMetadata, targetLanguage) => transl
     })
   });
 };
+function shouldExcludeField(fieldPath, excludeFields) {
+  for (const pattern of excludeFields) {
+    if (fieldPath === pattern)
+      return !0;
+    if (pattern.startsWith("**.")) {
+      const fieldName = pattern.substring(3), segments = fieldPath.split(".");
+      if (segments[segments.length - 1]?.split("[")[0] === fieldName)
+        return !0;
+    } else if (!pattern.includes(".") && !pattern.includes("[")) {
+      const segments = fieldPath.split(".");
+      if (segments[segments.length - 1]?.split("[")[0] === pattern)
+        return !0;
+    } else if (pattern.includes("[*]")) {
+      const regexPattern = pattern.replace(/\[/g, "\\[").replace(/\]/g, "\\]").replace(/\\\[\*\\\]/g, "\\[\\d+\\]");
+      if (new RegExp(`^${regexPattern}$`).test(fieldPath))
+        return !0;
+    }
+  }
+  return !1;
+}
+function filterExcludedFields(document, excludeFields = [], currentPath = "") {
+  if (!excludeFields.length)
+    return {
+      filteredDocument: document,
+      excludedValues: {}
+    };
+  const filteredDocument = {}, excludedValues = {};
+  let hasExclusions = !1;
+  for (const [key, value] of Object.entries(document)) {
+    const fieldPath = currentPath ? `${currentPath}.${key}` : key;
+    if (shouldExcludeField(fieldPath, excludeFields)) {
+      excludedValues[fieldPath] = value, hasExclusions = !0, devLog.debug("Excluding field from translation", {
+        fieldPath,
+        fieldType: typeof value,
+        value: typeof value == "string" ? value : "[object]"
+      });
+      continue;
+    }
+    if (value && typeof value == "object" && !Array.isArray(value) && value._type !== "reference") {
+      const nested = filterExcludedFields(value, excludeFields, fieldPath);
+      filteredDocument[key] = nested.filteredDocument, Object.assign(excludedValues, nested.excludedValues), Object.keys(nested.excludedValues).length > 0 && (hasExclusions = !0);
+    } else if (Array.isArray(value)) {
+      const filteredArray = [];
+      let arrayHasExclusions = !1;
+      for (let i = 0; i < value.length; i++) {
+        const arrayItem = value[i], itemPath = `${fieldPath}[${i}]`;
+        if (arrayItem && typeof arrayItem == "object" && !Array.isArray(arrayItem) && arrayItem._type !== "reference") {
+          const nested = filterExcludedFields(arrayItem, excludeFields, itemPath);
+          filteredArray[i] = nested.filteredDocument, Object.assign(excludedValues, nested.excludedValues), Object.keys(nested.excludedValues).length > 0 && (arrayHasExclusions = !0);
+        } else
+          filteredArray[i] = arrayItem;
+      }
+      filteredDocument[key] = filteredArray, arrayHasExclusions && (hasExclusions = !0);
+    } else
+      filteredDocument[key] = value;
+  }
+  return hasExclusions && devLog.info("Filtered excluded fields from document", {
+    documentId: document._id || "unknown",
+    excludedFieldCount: Object.keys(excludedValues).length,
+    excludedFields: Object.keys(excludedValues)
+  }), {
+    filteredDocument,
+    excludedValues
+  };
+}
+function restoreExcludedFields(translatedDocument, excludedValues) {
+  if (!Object.keys(excludedValues).length)
+    return translatedDocument;
+  const result = { ...translatedDocument };
+  for (const [fieldPath, value] of Object.entries(excludedValues))
+    setNestedValue(result, fieldPath, value);
+  return devLog.debug("Restored excluded fields to translated document", {
+    documentId: result._id || "unknown",
+    restoredFieldCount: Object.keys(excludedValues).length,
+    restoredFields: Object.keys(excludedValues)
+  }), result;
+}
+function setNestedValue(obj, path, value) {
+  const segments = path.split(".");
+  let current = obj;
+  for (let i = 0; i < segments.length - 1; i++) {
+    const segment = segments[i], arrayMatch2 = segment.match(/^(.+)\[(\d+)\]$/);
+    if (arrayMatch2) {
+      const [, fieldName, indexStr] = arrayMatch2, index = Number.parseInt(indexStr, 10);
+      for ((!(fieldName in current) || !Array.isArray(current[fieldName])) && (current[fieldName] = []); current[fieldName].length <= index; )
+        current[fieldName].push({});
+      current = current[fieldName][index];
+    } else
+      (!(segment in current) || typeof current[segment] != "object") && (current[segment] = {}), current = current[segment];
+  }
+  const finalSegment = segments[segments.length - 1], arrayMatch = finalSegment.match(/^(.+)\[(\d+)\]$/);
+  if (arrayMatch) {
+    const [, fieldName, indexStr] = arrayMatch, index = Number.parseInt(indexStr, 10);
+    for ((!(fieldName in current) || !Array.isArray(current[fieldName])) && (current[fieldName] = []); current[fieldName].length <= index; )
+      current[fieldName].push(null);
+    current[fieldName][index] = value;
+  } else
+    current[finalSegment] = value;
+}
+const COMMON_ENUM_FIELDS = [
+  "status",
+  "state",
+  "type",
+  "category",
+  "priority",
+  "visibility",
+  "role",
+  "permission",
+  "level",
+  "mode",
+  "format",
+  "variant",
+  "size",
+  "theme",
+  "kind",
+  "phase",
+  "stage"
+], PREDEFINED_EXCLUSIONS = {
+  /** Common enum and status fields */
+  enums: [...COMMON_ENUM_FIELDS],
+  /** All metadata and system fields (starting with _) */
+  metadata: ["_id", "_type", "_rev", "_createdAt", "_updatedAt"],
+  /** Common technical and configuration fields */
+  technical: [
+    "slug",
+    "handle",
+    "key",
+    "code",
+    "id",
+    "uuid",
+    "token",
+    "hash",
+    "checksum",
+    "version",
+    "config",
+    "settings"
+  ],
+  /** SEO and URL-related fields that often shouldn't be translated */
+  seo: ["slug", "handle", "permalink", "canonical", "redirect", "robots", "sitemap"]
+};
 function applyTransforms({
   document,
   registry,
@@ -1961,9 +2101,10 @@ class TransformRegistry {
     }), matches[0];
   }
 }
-const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKey, request) => (request.model || "gpt-5") === "gpt-5" ? await callGPT5ResponsesAPI(apiKey, request) : await callChatCompletionsAPI(apiKey, request), callGPT5ResponsesAPI = async (apiKey, request) => {
+const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKey, request) => (request.model || "gpt-5").startsWith("gpt-5") ? await callGPT5ResponsesAPI(apiKey, request) : await callChatCompletionsAPI(apiKey, request), callGPT5ResponsesAPI = async (apiKey, request) => {
   const model = request.model || "gpt-5", requestBody = {
-    model: "gpt-5.1",
+    model: model === "gpt-5" ? "gpt-5.1" : model,
+    // Map gpt-5 to gpt-5.1, use others as-is
     input: request.messages
     // Uses 'input' instead of 'messages'
   };
@@ -1998,7 +2139,8 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
     prompt: result.usage?.input_tokens || 0,
     completion: result.usage?.output_tokens || 0,
     reasoning: result.usage?.output_tokens_details?.reasoning_tokens || 0,
-    total: result.usage?.total_tokens || 0
+    total: result.usage?.total_tokens || 0,
+    cached: result.usage?.input_tokens_details?.cached_tokens || 0
   }, cost = calculateCost(tokensUsed, model);
   let content = "";
   if (result.output && Array.isArray(result.output)) {
@@ -2043,7 +2185,8 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
     prompt: usage.prompt_tokens || 0,
     completion: usage.completion_tokens || 0,
     reasoning: usage.reasoning_tokens || 0,
-    total: usage.total_tokens || 0
+    total: usage.total_tokens || 0,
+    cached: usage.prompt_tokens_details?.cached_tokens || 0
   }, cost = calculateCost(tokensUsed, model);
   return {
     content: result.choices[0].message.content || "",
@@ -2053,27 +2196,47 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
 }, calculateCost = (tokens, model) => {
   const pricing = {
     "gpt-5": {
-      input: 2e-6,
+      input: 125e-8,
+      output: 1e-5,
+      reasoning: 0
+    },
+    "gpt-5.1": {
+      input: 125e-8,
       output: 1e-5,
-      reasoning: 5e-6
+      reasoning: 0
+    },
+    "gpt-5-mini": {
+      input: 25e-8,
+      output: 2e-6,
+      reasoning: 0
+    },
+    "gpt-5-nano": {
+      input: 50000000000000004e-24,
+      output: 40000000000000003e-23,
+      reasoning: 0
+    },
+    "gpt-5-pro": {
+      input: 15e-6,
+      output: 12e-5,
+      reasoning: 0
     },
     "gpt-4o": {
-      input: 5e-6,
-      output: 14999999999999999e-21,
+      input: 25e-7,
+      output: 1e-5,
       reasoning: 0
     },
     "gpt-4-turbo": {
       input: 1e-5,
-      output: 29999999999999997e-21,
+      output: 3e-5,
       reasoning: 0
     },
     "gpt-4": {
-      input: 29999999999999997e-21,
-      output: 59999999999999995e-21,
+      input: 3e-5,
+      output: 6e-5,
       reasoning: 0
     }
-  }, modelPricing = pricing[model] || pricing["gpt-5"];
-  return tokens.prompt * modelPricing.input + tokens.completion * modelPricing.output + tokens.reasoning * modelPricing.reasoning;
+  }, modelPricing = pricing[model] || pricing["gpt-5"], cachedInputRate = model === "gpt-5.1" ? 0.125 / 1e6 : modelPricing.input;
+  return Math.max(0, tokens.prompt - tokens.cached) * modelPricing.input + tokens.cached * cachedInputRate + tokens.completion * modelPricing.output + tokens.reasoning * modelPricing.reasoning;
 }, deserializeDocument = (originalDoc, reviewedText) => {
   const parsed = parseReviewFormat(reviewedText), result = cloneDeep(originalDoc);
   return applyTextChanges(result, parsed), result;
@@ -2089,7 +2252,7 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
         type: "field",
         name: fieldName,
         text: unescapeXml(text2),
-        attributes: {},
+        attributes: { type: "string" },
         children: []
       };
       if (stack.length > 0) {
@@ -2162,7 +2325,13 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
           parent && parent.children.push(node);
         } else
           root.push(node);
-      } else
+      } else if (attributes.type === "string" && content && content.includes(closingTag))
+        if (node.text = unescapeXml(content.replace(closingTag, "")), stack.length > 0) {
+          const parent = stack[stack.length - 1];
+          parent && parent.children.push(node);
+        } else
+          root.push(node);
+      else
         stack.push(node);
       continue;
     }
@@ -2218,7 +2387,10 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
     });
     return;
   }
-  node.text !== void 0 && !node.attributes.readonly && set(doc, path, node.text);
+  node.text !== void 0 && !node.attributes.readonly && (get(doc, path) !== void 0 ? set(doc, path, node.text) : devLog.debug("Skipping field not present in original document", {
+    path: path.join("."),
+    attemptedValue: node.text
+  }));
 }, applyBlockChanges = (block, blockNode) => {
   if (blockNode.attributes._key && block._key !== blockNode.attributes._key) {
     devLog.warn("Block key mismatch, skipping", {
@@ -2249,7 +2421,10 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
     });
     return;
   }
-  fieldNode.text !== void 0 && !fieldNode.attributes.readonly && (block[fieldName] = fieldNode.text);
+  fieldNode.text !== void 0 && !fieldNode.attributes.readonly && (fieldName in block ? block[fieldName] = fieldNode.text : devLog.debug("Skipping block field not present in original", {
+    fieldName,
+    attemptedValue: fieldNode.text
+  }));
 }, applySpanChanges = (span, spanNode) => {
   if (spanNode.attributes._key && span._key !== spanNode.attributes._key) {
     devLog.warn("Span key mismatch, skipping", {
@@ -2294,7 +2469,7 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
       return;
     }
     if (typeof value == "string") {
-      value.length > 0 && lines.push(`${fieldName}: ${escapeXml(value)}`);
+      value.length > 0 && lines.push(`${indentStr}<field name="${fieldName}" type="string">${escapeXml(value)}</field>`);
       return;
     }
     (typeof value == "number" || typeof value == "boolean") && lines.push(`${indentStr}<field name="${fieldName}" type="${typeof value}" readonly="true">${value}</field>`);
@@ -2331,7 +2506,7 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
     return;
   }
   if (typeof value == "string" && value.length > 0) {
-    lines.push(`${indentStr}<field name="${fieldName}">${escapeXml(value)}</field>`);
+    lines.push(`${indentStr}<field name="${fieldName}" type="string">${escapeXml(value)}</field>`);
     return;
   }
   (typeof value == "number" || typeof value == "boolean") && lines.push(`${indentStr}<field name="${fieldName}" type="${typeof value}" readonly="true">${value}</field>`);
@@ -2473,7 +2648,19 @@ const globalTransformRegistry = new TransformRegistry(), callGPT5 = async (apiKe
           `Structure validation failed: ${unexpectedErrors.length} errors. First: ${unexpectedErrors[0].type} at ${unexpectedErrors[0].path.join(".")}`
         );
     }
-    return setJobStatus(dispatch, sourceDocument._id, targetLanguage, "translating_with_openai", {
+    const translationValidation = validateTranslationCompleteness(sourceDocument, result, languageFieldName);
+    return translationValidation.valid || (devLog.warn("Translation completeness warning", {
+      ...logContext,
+      skippedFields: translationValidation.skippedFields,
+      warning: "GPT-5 may have skipped some important fields"
+    }), devLog.info("Skipped field details", {
+      ...logContext,
+      skippedFields: translationValidation.skippedFields.map((field) => ({
+        path: field.path,
+        originalValue: field.originalValue,
+        translatedValue: field.translatedValue
+      }))
+    })), setJobStatus(dispatch, sourceDocument._id, targetLanguage, "translating_with_openai", {
       tokensUsed: response.tokensUsed.total,
       estimatedCost: response.cost
     }), result;
@@ -2514,12 +2701,19 @@ ${sourceText}
 
 ## Instructions
 
-1. Translate all text content into ${targetLanguage}
-2. Follow the guidelines and terminology requirements above
-3. Preserve protected phrases exactly as they appear
-4. Return the document in THE EXACT SAME FORMAT
-5. **CRITICAL**: Only modify text content between tags
-6. Never add, remove, or change any tags, attributes, or structure
+1. **TRANSLATE ALL TEXT FIELDS**: Every field marked with type="string" or containing text must be translated
+2. **DO NOT SKIP SIMPLE FIELDS**: Pay special attention to simple fields like "title", "name", "description" - these are often the most important
+3. Follow the guidelines and terminology requirements above
+4. Preserve protected phrases exactly as they appear
+5. Return the document in THE EXACT SAME FORMAT
+6. **CRITICAL**: Only modify text content between tags
+7. Never add, remove, or change any tags, attributes, or structure
+
+**Common fields that must always be translated:**
+- title, name, heading, label
+- description, caption, alt text
+- All text within <field> tags
+- All text within <span> tags
 
 If you used significant reasoning to make translation decisions, you may include a brief summary:
 <reasoning>
@@ -2527,6 +2721,43 @@ Brief summary of key translation decisions and reasoning
 </reasoning>
 
 Then return the translated document:`;
+}, validateTranslationCompleteness = (sourceDoc, translatedDoc, languageFieldName) => {
+  const skippedFields = [], checkField = (source, translated, path = []) => {
+    if (path.length > 0) {
+      const fieldName = path[path.length - 1];
+      if (fieldName.startsWith("_") || fieldName === languageFieldName)
+        return;
+    }
+    if (typeof source == "string" && typeof translated == "string") {
+      if (source.length > 0 && source === translated) {
+        const fieldName = path[path.length - 1] || "unknown";
+        (["title", "name", "heading", "label", "description", "caption", "alt"].includes(
+          fieldName
+        ) || source.length > 10) && skippedFields.push({
+          path: path.join("."),
+          originalValue: source,
+          translatedValue: translated
+        });
+      }
+      return;
+    }
+    if (source && translated && typeof source == "object" && typeof translated == "object") {
+      if (Array.isArray(source) && Array.isArray(translated)) {
+        const maxLength = Math.min(source.length, translated.length);
+        for (let i = 0; i < maxLength; i++)
+          checkField(source[i], translated[i], [...path, String(i)]);
+      } else if (!Array.isArray(source) && !Array.isArray(translated))
+        for (const key of Object.keys(source))
+          !key.startsWith("_") && key !== languageFieldName && checkField(source[key], translated[key], [
+            ...path,
+            key
+          ]);
+    }
+  };
+  return checkField(sourceDoc, translatedDoc), {
+    valid: skippedFields.length === 0,
+    skippedFields
+  };
 }, assembleLanguageSpecs = ({
   sourceDocument,
   languageFieldName,
@@ -2647,14 +2878,30 @@ ${protectedPhrases.join(", ")}`, buildStyleGuideSection = (globalPrompt, localeP
   translationPrompts,
   translationMode,
   openaiConfig,
-  releaseId
+  releaseId,
+  excludeFields = []
 }) => {
-  devLog.debug("Applying field transforms before translation", {
+  devLog.info("Filtering excluded fields before translation", {
+    documentId: sourceDocument._id,
+    documentType: sourceDocument._type,
+    excludeFields,
+    excludeFieldsLength: excludeFields?.length ?? 0,
+    sourceHasTitle: "title" in sourceDocument,
+    sourceTitleValue: sourceDocument.title
+  });
+  const { filteredDocument, excludedValues } = filterExcludedFields(sourceDocument, excludeFields);
+  devLog.info("After filtering excluded fields", {
+    documentId: sourceDocument._id,
+    filteredHasTitle: "title" in filteredDocument,
+    filteredTitleValue: filteredDocument.title,
+    excludedValuesKeys: Object.keys(excludedValues),
+    excludedValues
+  }), devLog.debug("Applying field transforms before translation", {
     documentId: sourceDocument._id,
     documentType: sourceDocument._type
   });
   const { transformedDocument, originalStructure } = applyTransforms({
-    document: sourceDocument,
+    document: filteredDocument,
     registry: globalTransformRegistry,
     schemaType: sourceDocument._type
   });
@@ -2695,12 +2942,26 @@ ${protectedPhrases.join(", ")}`, buildStyleGuideSection = (globalPrompt, localeP
     documentId: sourceDocument._id,
     documentType: sourceDocument._type
   });
-  const finalDoc = restoreTransforms({
+  const transformedResult = restoreTransforms({
     translatedDocument: translatedDoc,
     originalStructure,
     registry: globalTransformRegistry,
     schemaType: sourceDocument._type
-  }), targetDocumentPublishedId = existingTranslationId || uuid(), targetDocumentVersionId = `versions.${releaseId}.${targetDocumentPublishedId}`, newDoc = {
+  });
+  devLog.info("Before restoring excluded fields", {
+    documentId: sourceDocument._id,
+    excludedFieldCount: Object.keys(excludedValues).length,
+    excludedValuesKeys: Object.keys(excludedValues),
+    transformedResultHasTitle: "title" in transformedResult,
+    transformedResultTitle: transformedResult.title
+  });
+  const finalDoc = restoreExcludedFields(transformedResult, excludedValues);
+  devLog.info("After restoring excluded fields", {
+    documentId: sourceDocument._id,
+    finalDocHasTitle: "title" in finalDoc,
+    finalDocTitle: finalDoc.title
+  });
+  const targetDocumentPublishedId = existingTranslationId || uuid(), targetDocumentVersionId = `versions.${releaseId}.${targetDocumentPublishedId}`, newDoc = {
     ...finalDoc,
     _id: targetDocumentVersionId
   };
@@ -2721,7 +2982,8 @@ ${protectedPhrases.join(", ")}`, buildStyleGuideSection = (globalPrompt, localeP
   translationPrompts,
   translationMode,
   openaiConfig,
-  releaseId
+  releaseId,
+  excludeFields
 }) => {
   const logContext = {
     documentId: sourceDocument._id,
@@ -2757,7 +3019,8 @@ ${protectedPhrases.join(", ")}`, buildStyleGuideSection = (globalPrompt, localeP
         translationPrompts,
         translationMode,
         openaiConfig,
-        releaseId
+        releaseId,
+        excludeFields
       }),
       {
         maxRetries: 3,
@@ -2823,7 +3086,8 @@ ${protectedPhrases.join(", ")}`, buildStyleGuideSection = (globalPrompt, localeP
   references = [],
   maxConcurrentTranslationJobs,
   translationMode,
-  openaiConfig
+  openaiConfig,
+  excludeFields
 }) => {
   const logContext = {
     documentId: sourceDocument._id,
@@ -2889,7 +3153,8 @@ ${protectedPhrases.join(", ")}`, buildStyleGuideSection = (globalPrompt, localeP
                 translationPrompts,
                 translationMode,
                 openaiConfig,
-                releaseId: releaseId2
+                releaseId: releaseId2,
+                excludeFields
               });
               versionedTargetDocumentId && versionedTargetDocumentIds.push(versionedTargetDocumentId);
             }
@@ -2962,7 +3227,8 @@ ${protectedPhrases.join(", ")}`, buildStyleGuideSection = (globalPrompt, localeP
       maxConcurrentTranslationJobs: config.maxConcurrentTranslationJobs,
       maxReferenceDepth: config.maxReferenceDepth,
       translationMode,
-      openaiConfig: config.openai
+      openaiConfig: config.openai,
+      excludeFields: config.excludeFields
     });
   }, [
     document,
@@ -2974,6 +3240,7 @@ ${protectedPhrases.join(", ")}`, buildStyleGuideSection = (globalPrompt, localeP
     config.maxConcurrentTranslationJobs,
     config.maxReferenceDepth,
     config.openai,
+    config.excludeFields,
     references,
     shouldOverwriteFreshTranslations,
     translationMode
@@ -3955,7 +4222,11 @@ const aiTranslationPlugin = definePlugin((config) => (config.fieldTransforms?.le
   }
 })), translationHistorySchema = translationHistory;
 export {
+  COMMON_ENUM_FIELDS,
+  PREDEFINED_EXCLUSIONS,
   aiTranslationPlugin,
+  filterExcludedFields,
+  restoreExcludedFields,
   richTextTransform,
   translationHistorySchema,
   vttCaptionTransform