gt-sanity 2.0.21 → 2.1.0

package/src/serialization/internationalizedArray/__tests__/serializeRoundTrip.test.ts

@@ -0,0 +1,83 @@
+import { Schema } from '@sanity/schema';
+import { SanityDocument } from 'sanity';
+import { describe, expect, test } from 'vitest';
+import { createInternationalizedArrayTypes } from '../../../schema/createInternationalizedArrayTypes';
+import {
+  deserializeDocument,
+  serializeDocument,
+} from '../../../utils/serialize';
+
+const generatedTypes = createInternationalizedArrayTypes({
+  sourceLocale: 'en',
+  locales: ['es'],
+  fieldTypes: ['string', 'text'],
+});
+
+const postType = {
+  name: 'post',
+  title: 'Post',
+  type: 'document',
+  fields: [
+    { name: 'title', title: 'Title', type: 'internationalizedArrayString' },
+    {
+      name: 'description',
+      title: 'Description',
+      type: 'internationalizedArrayText',
+    },
+    { name: 'tags', title: 'Tags', type: 'array', of: [{ type: 'string' }] },
+  ],
+};
+
+const schema: InstanceType<typeof Schema> = new Schema({
+  name: 'test',
+  types: [...generatedTypes, postType] as never,
+});
+
+const item = (type: string, language: string, value: unknown) => ({
+  _key: `key-${language}`,
+  _type: type,
+  language,
+  value,
+});
+
+const doc = {
+  _id: 'drafts.post-1',
+  _type: 'post',
+  _rev: 'rev-1',
+  title: [
+    item('internationalizedArrayStringValue', 'en', 'Hello'),
+    item('internationalizedArrayStringValue', 'es', 'Hola'),
+  ],
+  description: [item('internationalizedArrayTextValue', 'en', 'A description')],
+  tags: ['alpha', 'beta'],
+} as unknown as SanityDocument;
+
+describe('internationalized array serialize round-trip', () => {
+  const serialized = serializeDocument(
+    doc,
+    schema,
+    'en',
+    'internationalizedArray'
+  );
+
+  test('exports only the source-locale value', () => {
+    expect(serialized.content).toContain('Hello');
+    expect(serialized.content).not.toContain('Hola');
+  });
+
+  test('carries the document _type so import can pick the strategy', () => {
+    expect(serialized.content).toContain('content="post"');
+  });
+
+  test('round-trips source values back through the deserializer', () => {
+    const deserialized = deserializeDocument(serialized.content);
+    expect(deserialized.title).toBe('Hello');
+    expect(deserialized.description).toBe('A description');
+    expect(deserialized._type).toBe('post');
+  });
+
+  test('passes non-localized arrays through unchanged', () => {
+    const deserialized = deserializeDocument(serialized.content);
+    expect(deserialized.tags).toEqual(['alpha', 'beta']);
+  });
+});

package/src/serialization/internationalizedArray/collapse.ts

@@ -0,0 +1,52 @@
+import {
+  findLocaleItem,
+  isInternationalizedArrayField,
+  isRecord,
+} from './detect';
+
+/**
+ * Collapse every internationalized-array field in a value down to just its
+ * source-locale `value`, recursing into nested objects/arrays.
+ *
+ * This runs *before* the generic HTML serializer so that the per-locale item
+ * arrays never reach `serializeArray` (which would otherwise emit every
+ * locale). After collapsing, an `internationalizedArrayString` field is a plain
+ * string, an `internationalizedArrayText` is a plain string, Portable Text is a
+ * plain `block` array, and a custom value object is a plain object — all shapes
+ * the existing document-level serializer already round-trips.
+ *
+ * Returns `undefined` when a field has no source-locale entry, so the caller
+ * drops it (nothing to translate) rather than emitting an empty field.
+ */
+export function collapseToSourceLocale(
+  value: unknown,
+  sourceLocale: string
+): unknown {
+  if (isInternationalizedArrayField(value)) {
+    const sourceItem = findLocaleItem(value, sourceLocale);
+    if (!sourceItem) {
+      return undefined;
+    }
+    // Recurse: the source value may itself contain nested localized fields.
+    return collapseToSourceLocale(sourceItem.value, sourceLocale);
+  }
+
+  if (Array.isArray(value)) {
+    return value
+      .map((item) => collapseToSourceLocale(item, sourceLocale))
+      .filter((item) => item !== undefined);
+  }
+
+  if (isRecord(value)) {
+    const collapsed: Record<string, unknown> = {};
+    for (const key of Object.keys(value)) {
+      const collapsedValue = collapseToSourceLocale(value[key], sourceLocale);
+      if (collapsedValue !== undefined) {
+        collapsed[key] = collapsedValue;
+      }
+    }
+    return collapsed;
+  }
+
+  return value;
+}

package/src/serialization/internationalizedArray/detect.ts

@@ -0,0 +1,81 @@
+// Helpers for recognising the `sanity-plugin-internationalized-array` storage
+// shape, where a localized field is stored as an array of per-locale items:
+//
+//   title: [
+//     { _key: 'abc', _type: 'internationalizedArrayStringValue', language: 'en', value: 'Hello' },
+//     { _key: 'def', _type: 'internationalizedArrayStringValue', language: 'es', value: 'Hola' },
+//   ]
+//
+// Detection is shape-based (not schema-based) so the serializer/merger do not
+// have to thread the schema through every call, but it is anchored on the
+// value `_type` naming convention (`<typePrefix>…Value`) so user-defined types
+// that merely share the `{ language, value }` shape are not misdetected.
+
+import { pluginConfig } from '../../adapter/core';
+
+// The reference-plugin prefix. Always recognized (even with a custom
+// `typePrefix`) because compatibility types with this prefix are generated by
+// default and pre-existing upstream data uses it.
+const DEFAULT_TYPE_PREFIX = 'internationalizedArray';
+
+function hasRecognizedTypePrefix(type: string): boolean {
+  return (
+    type.startsWith(DEFAULT_TYPE_PREFIX) ||
+    type.startsWith(pluginConfig.getFieldLevelTypePrefix())
+  );
+}
+
+export const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === 'object' && value !== null && !Array.isArray(value);
+
+export type InternationalizedArrayItem = {
+  _key?: string;
+  _type: string;
+  language: string;
+  value: unknown;
+};
+
+/**
+ * A single `{ _key, _type, language, value }` entry. The `_type` of the
+ * generated value objects is always `<typePrefix><FieldType>Value`
+ * (e.g. `internationalizedArrayStringValue`), so both the prefix and the
+ * `Value` suffix are required — a bare `endsWith('Value')` check would
+ * misdetect user-defined types like `priceValue` that happen to carry
+ * `language`/`value` keys.
+ */
+export function isInternationalizedArrayItem(
+  item: unknown
+): item is InternationalizedArrayItem {
+  return (
+    isRecord(item) &&
+    typeof item._type === 'string' &&
+    hasRecognizedTypePrefix(item._type) &&
+    item._type.endsWith('Value') &&
+    typeof item.language === 'string' &&
+    'value' in item
+  );
+}
+
+/**
+ * A non-empty array where every element is an internationalized-array item.
+ */
+export function isInternationalizedArrayField(
+  value: unknown
+): value is InternationalizedArrayItem[] {
+  return (
+    Array.isArray(value) &&
+    value.length > 0 &&
+    value.every(isInternationalizedArrayItem)
+  );
+}
+
+/**
+ * Returns the item for a given locale, matching on the exact `language` id
+ * (hyphens preserved, e.g. `fr-CA`). Never matches on `_key`, which is random.
+ */
+export function findLocaleItem(
+  field: InternationalizedArrayItem[],
+  locale: string
+): InternationalizedArrayItem | undefined {
+  return field.find((item) => item.language === locale);
+}

package/src/serialization/internationalizedArray/merge.ts

@@ -0,0 +1,149 @@
+import { randomKey } from '../../utils/randomKey';
+import {
+  findLocaleItem,
+  InternationalizedArrayItem,
+  isInternationalizedArrayField,
+  isRecord,
+} from './detect';
+
+/**
+ * Upsert the translated value for `targetLocale` into an internationalized
+ * array, preserving every other locale item and its (random) `_key`.
+ *
+ * Sanity mutations can only address array items by `_key`, and `_key` is random
+ * here, so we resolve the target item by its `language` field and reuse its
+ * existing `_key`; if absent we append a fresh random `_key`.
+ */
+function upsertLocaleItem(
+  baseArray: InternationalizedArrayItem[],
+  translatedValue: unknown,
+  targetLocale: string,
+  sourceLocale: string
+): InternationalizedArrayItem[] {
+  const itemType = (findLocaleItem(baseArray, sourceLocale) ?? baseArray[0])
+    ._type;
+
+  const updated = baseArray.map((item) =>
+    item.language === targetLocale
+      ? { ...item, value: translatedValue }
+      : { ...item }
+  );
+
+  if (!findLocaleItem(updated, targetLocale)) {
+    updated.push({
+      _key: randomKey(),
+      _type: itemType,
+      language: targetLocale,
+      value: translatedValue,
+    });
+  }
+
+  return updated;
+}
+
+/**
+ * Merge a single base value with its translated counterpart, returning the new
+ * value to write, or `undefined` when nothing localized changed at this level.
+ * Recurses through plain objects (by key) and plain arrays (by `_key`) so
+ * internationalized-array fields nested inside other content are also handled.
+ */
+function mergeValue(
+  baseValue: unknown,
+  translatedValue: unknown,
+  targetLocale: string,
+  sourceLocale: string
+): unknown {
+  if (isInternationalizedArrayField(baseValue)) {
+    return upsertLocaleItem(
+      baseValue,
+      translatedValue,
+      targetLocale,
+      sourceLocale
+    );
+  }
+
+  if (Array.isArray(baseValue) && Array.isArray(translatedValue)) {
+    let changed = false;
+    const merged = baseValue.map((item) => {
+      if (!isRecord(item) || typeof item._key !== 'string') {
+        return item;
+      }
+      const translatedItem = translatedValue.find(
+        (candidate) => isRecord(candidate) && candidate._key === item._key
+      );
+      if (!translatedItem) {
+        return item;
+      }
+      const mergedItem = mergeValue(
+        item,
+        translatedItem,
+        targetLocale,
+        sourceLocale
+      );
+      if (mergedItem !== undefined) {
+        changed = true;
+        return mergedItem;
+      }
+      return item;
+    });
+    return changed ? merged : undefined;
+  }
+
+  if (isRecord(baseValue) && isRecord(translatedValue)) {
+    let changed = false;
+    const merged: Record<string, unknown> = { ...baseValue };
+    for (const key of Object.keys(translatedValue)) {
+      if (key.startsWith('_') || !(key in baseValue)) {
+        continue;
+      }
+      const mergedChild = mergeValue(
+        baseValue[key],
+        translatedValue[key],
+        targetLocale,
+        sourceLocale
+      );
+      if (mergedChild !== undefined) {
+        merged[key] = mergedChild;
+        changed = true;
+      }
+    }
+    return changed ? merged : undefined;
+  }
+
+  return undefined;
+}
+
+/**
+ * Compare a base Sanity document against a deserialized translation and produce
+ * a patch object of top-level fields whose internationalized-array content
+ * changed. Each entry is the full merged top-level value (with the target
+ * locale upserted), suitable for `client.patch(id).set(changes)`.
+ *
+ * Only top-level fields that actually changed are returned, so unrelated
+ * content and other locales are left untouched.
+ */
+export function mergeInternationalizedArrays(
+  baseDoc: Record<string, unknown>,
+  translatedFields: Record<string, unknown>,
+  targetLocale: string,
+  sourceLocale: string
+): Record<string, unknown> {
+  const changes: Record<string, unknown> = {};
+
+  for (const key of Object.keys(translatedFields)) {
+    if (key.startsWith('_') || !(key in baseDoc)) {
+      continue;
+    }
+    const merged = mergeValue(
+      baseDoc[key],
+      translatedFields[key],
+      targetLocale,
+      sourceLocale
+    );
+    if (merged !== undefined) {
+      changes[key] = merged;
+    }
+  }
+
+  return changes;
+}

package/src/serialization/types.ts

@@ -14,7 +14,11 @@ export type SerializedDocument = {
   content: string;
 };
 
-export type TranslationLevel = 'document' | 'field';
+// 'document'  – translate the whole document (creates per-locale documents).
+// 'field'     – legacy object-keyed model (e.g. `title: { en, es_ES }`).
+// 'internationalizedArray' – sanity-plugin-internationalized-array shape:
+//               `title: [{ _key, _type, language, value }]`, localized in place.
+export type TranslationLevel = 'document' | 'field' | 'internationalizedArray';
 
 export interface Deserializer {
   deserializeDocument: <

package/src/translation/__tests__/strategy.test.ts

@@ -0,0 +1,47 @@
+import { afterEach, describe, expect, test } from 'vitest';
+import { pluginConfig } from '../../adapter/core';
+import {
+  getTranslationStrategy,
+  getTranslationStrategyForType,
+} from '../strategy';
+import type { SanityDocument } from 'sanity';
+
+const doc = (type: string) =>
+  ({ _id: 'a', _type: type, _rev: 'r' }) as unknown as SanityDocument;
+
+afterEach(() => {
+  pluginConfig.translationLevel = 'document';
+  pluginConfig.fieldLevelDocuments = [];
+});
+
+describe('getTranslationStrategy', () => {
+  test("'document' mode always uses the document-level strategy", () => {
+    pluginConfig.translationLevel = 'document';
+    expect(getTranslationStrategyForType('post').level).toBe('document');
+  });
+
+  test("'internationalizedArray' mode always uses the array strategy", () => {
+    pluginConfig.translationLevel = 'internationalizedArray';
+    expect(getTranslationStrategy(doc('post')).level).toBe(
+      'internationalizedArray'
+    );
+    expect(getTranslationStrategy(doc('page')).level).toBe(
+      'internationalizedArray'
+    );
+  });
+
+  test("'mixed' mode routes by document type", () => {
+    pluginConfig.translationLevel = 'mixed';
+    pluginConfig.fieldLevelDocuments = [{ type: 'post' }];
+    expect(getTranslationStrategyForType('post').level).toBe(
+      'internationalizedArray'
+    );
+    expect(getTranslationStrategyForType('page').level).toBe('document');
+  });
+
+  test('an unknown/undefined type falls back to document-level', () => {
+    pluginConfig.translationLevel = 'mixed';
+    pluginConfig.fieldLevelDocuments = [{ type: 'post' }];
+    expect(getTranslationStrategyForType(undefined).level).toBe('document');
+  });
+});

package/src/translation/importDocument.ts

@@ -1,7 +1,6 @@
-import { pluginConfig } from '../adapter/core';
-import { documentLevelPatch } from '../configuration/baseDocumentLevelConfig/documentLevelPatch';
 import type { GTFile, TranslationFunctionContext } from '../types';
 import { deserializeDocument } from '../utils/serialize';
+import { getTranslationStrategyForType } from './strategy';
 
 export async function importDocument(
   docInfo: GTFile,
@@ -12,12 +11,17 @@ export async function importDocument(
 ) {
   const { client } = context;
   const deserialized = deserializeDocument(document);
-  return documentLevelPatch(
-    docInfo, // versionId is not used here, since we just use the _rev id in the deserialized HTML itself
+  // The serialized HTML carries the document `_type` as a <meta> tag, so the
+  // deserialized doc tells us which strategy to import with. versionId is not
+  // used for the patch since the _rev travels inside the deserialized HTML.
+  const strategy = getTranslationStrategyForType(
+    deserialized._type as string | undefined
+  );
+  return strategy.patch(
+    docInfo,
     deserialized,
     localeId,
     client,
-    pluginConfig.getLanguageField(),
     mergeWithTargetLocale
   );
 }

package/src/translation/strategy.ts

@@ -0,0 +1,92 @@
+import { SanityClient, SanityDocument, Schema } from 'sanity';
+import { pluginConfig } from '../adapter/core';
+import { documentLevelPatch } from '../configuration/baseDocumentLevelConfig/documentLevelPatch';
+import { internationalizedArrayPatch } from '../configuration/internationalizedArrayConfig/internationalizedArrayPatch';
+import type {
+  SerializedDocument,
+  TranslationLevel,
+} from '../serialization/types';
+import type { GTFile } from '../types';
+import { serializeDocument } from '../utils/serialize';
+
+/**
+ * A translation strategy bundles the serialize/patch pair for a given level.
+ * The shared GT upload/enqueue/status/download workflow is identical across
+ * strategies; only how a document is turned into the GT HTML file and how the
+ * translated file is written back to Sanity differs.
+ */
+export type SanityTranslationAdapter = {
+  level: TranslationLevel;
+  serialize: (
+    document: SanityDocument,
+    schema: Schema,
+    baseLanguage: string
+  ) => SerializedDocument;
+  patch: (
+    docInfo: GTFile,
+    deserialized: SanityDocument,
+    localeId: string,
+    client: SanityClient,
+    mergeWithTargetLocale?: boolean
+  ) => Promise<void>;
+};
+
+const documentAdapter: SanityTranslationAdapter = {
+  level: 'document',
+  serialize: (document, schema, baseLanguage) =>
+    serializeDocument(document, schema, baseLanguage, 'document'),
+  patch: (docInfo, deserialized, localeId, client, mergeWithTargetLocale) =>
+    documentLevelPatch(
+      docInfo,
+      deserialized,
+      localeId,
+      client,
+      pluginConfig.getLanguageField(),
+      mergeWithTargetLocale
+    ),
+};
+
+const internationalizedArrayAdapter: SanityTranslationAdapter = {
+  level: 'internationalizedArray',
+  serialize: (document, schema, baseLanguage) =>
+    serializeDocument(document, schema, baseLanguage, 'internationalizedArray'),
+  patch: (docInfo, deserialized, localeId, client) =>
+    internationalizedArrayPatch(docInfo, deserialized, localeId, client),
+};
+
+function matchesFieldLevel(type: string | undefined): boolean {
+  if (!type) {
+    return false;
+  }
+  return pluginConfig
+    .getFieldLevelDocuments()
+    .some((filter) => filter.type === type);
+}
+
+/**
+ * Resolve which strategy a document type uses:
+ * - `document`               → always document-level (default; unchanged).
+ * - `internationalizedArray` → always the array strategy.
+ * - `mixed`                  → array strategy for `fieldLevelDocuments`, else
+ *                              document-level.
+ */
+export function getTranslationStrategyForType(
+  type: string | undefined
+): SanityTranslationAdapter {
+  const level = pluginConfig.getTranslationLevel();
+  if (level === 'internationalizedArray') {
+    return internationalizedArrayAdapter;
+  }
+  if (level === 'mixed') {
+    return matchesFieldLevel(type)
+      ? internationalizedArrayAdapter
+      : documentAdapter;
+  }
+  return documentAdapter;
+}
+
+export function getTranslationStrategy(
+  document: SanityDocument
+): SanityTranslationAdapter {
+  return getTranslationStrategyForType(document._type);
+}

package/src/translation/uploadFiles.ts

@@ -1,7 +1,7 @@
 import type { GTFile, Secrets } from '../types';
 import { gt, overrideConfig } from '../adapter/core';
 import { libraryDefaultLocale } from 'generaltranslation/internal';
-import type { SerializedDocument } from '../serialization';
+import type { SerializedDocument } from '../serialization/types';
 
 // note: this function is used to create a new translation task
 // uploads files & calls the getTranslationTask function

package/src/types.ts

@@ -1,7 +1,7 @@
 // adapted from https://github.com/sanity-io/sanity-translations-tab. See LICENSE.md for more details.
 
 import { SanityClient, Schema, TypedObject } from 'sanity';
-import type { SerializedDocument } from './serialization';
+import type { SerializedDocument } from './serialization/types';
 import { PortableTextTypeComponent } from '@portabletext/to-html';
 import type { DeserializerRule } from '@portabletext/block-tools';
 

package/src/utils/__tests__/batchProcessor.test.ts

@@ -1,5 +1,24 @@
-import { describe, expect, test } from 'vitest';
-import { processBatch } from '../batchProcessor';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import {
+  ImportBatchItem,
+  processBatch,
+  processImportBatch,
+} from '../batchProcessor';
+import { pluginConfig } from '../../adapter/core';
+
+const importCounters = vi.hoisted(() => ({ active: 0, maxActive: 0 }));
+
+vi.mock('../../translation/importDocument', () => ({
+  importDocument: vi.fn(async () => {
+    importCounters.active++;
+    importCounters.maxActive = Math.max(
+      importCounters.maxActive,
+      importCounters.active
+    );
+    await new Promise((resolve) => setTimeout(resolve, 1));
+    importCounters.active--;
+  }),
+}));
 
 describe('processBatch', () => {
   test('serializes items with the same concurrency key', async () => {
@@ -42,3 +61,43 @@ describe('processBatch', () => {
     expect(maxActive).toBe(2);
   });
 });
+
+describe('processImportBatch', () => {
+  const originalLevel = pluginConfig.translationLevel;
+
+  afterEach(() => {
+    pluginConfig.translationLevel = originalLevel;
+    importCounters.active = 0;
+    importCounters.maxActive = 0;
+  });
+
+  const makeItem = (documentId: string, locale: string): ImportBatchItem => ({
+    docInfo: { documentId, versionId: 'rev-1' },
+    locale,
+    data: '',
+    translationContext: {} as ImportBatchItem['translationContext'],
+    key: `${documentId}:${locale}`,
+  });
+
+  test('imports locales of the same document in parallel at document level', async () => {
+    pluginConfig.translationLevel = 'document';
+
+    await processImportBatch([
+      makeItem('article-1', 'es'),
+      makeItem('article-1', 'fr'),
+    ]);
+
+    expect(importCounters.maxActive).toBe(2);
+  });
+
+  test('serializes locales of the same document for in-place strategies', async () => {
+    pluginConfig.translationLevel = 'internationalizedArray';
+
+    await processImportBatch([
+      makeItem('article-1', 'es'),
+      makeItem('article-1', 'fr'),
+    ]);
+
+    expect(importCounters.maxActive).toBe(1);
+  });
+});

package/src/utils/batchProcessor.ts

@@ -1,6 +1,7 @@
 import { GTFile, TranslationFunctionContext } from '../types';
+import { pluginConfig } from '../adapter/core';
 import { importDocument } from '../translation/importDocument';
-import { createStableTranslationKey } from './documentIds';
+import { getPublishedId } from './documentIds';
 
 export interface BatchProcessorOptions<T = unknown, R = unknown> {
   batchSize?: number;
@@ -126,12 +127,16 @@ export async function processImportBatch(
     },
     {
       ...options,
+      // Serialize all locales of the same document when imports may patch the
+      // source document in place: internationalized-array imports do a
+      // read-merge-set, so concurrent locale imports would clobber each other
+      // (last write wins). Document-level imports write to separate per-locale
+      // documents and stay parallel. In 'mixed' mode the per-document strategy
+      // is only known after deserialization, so serialize conservatively.
       getConcurrencyKey: (item: ImportBatchItem) =>
-        createStableTranslationKey(
-          undefined,
-          item.docInfo.documentId,
-          item.locale
-        ),
+        pluginConfig.getTranslationLevel() === 'document'
+          ? undefined
+          : getPublishedId(item.docInfo.documentId),
       onItemSuccess: (item: ImportBatchItem, key: string) => {
         successfulImports.push(key);
         options.onItemSuccess?.(item, key);