gt-sanity 0.0.5 → 1.0.0

package/src/translation/checkTranslationStatus.ts

@@ -0,0 +1,42 @@
+import type { Secrets } from '../types';
+import { gt, overrideConfig } from '../adapter/core';
+
+export async function checkTranslationStatus(
+  fileQueryData: { versionId: string; fileId: string; locale: string }[],
+  downloadStatus: {
+    downloaded: Set<string>;
+    failed: Set<string>;
+    skipped: Set<string>;
+  },
+  secrets: Secrets
+) {
+  overrideConfig(secrets);
+  try {
+    // Only query for files that haven't been downloaded yet
+    const currentQueryData = fileQueryData.filter(
+      (item) =>
+        !downloadStatus.downloaded.has(`${item.fileId}:${item.locale}`) &&
+        !downloadStatus.failed.has(`${item.fileId}:${item.locale}`) &&
+        !downloadStatus.skipped.has(`${item.fileId}:${item.locale}`)
+    );
+
+    // If all files have been downloaded, we're done
+    if (currentQueryData.length === 0) {
+      return true;
+    }
+    // Check for translations
+    const responseData = await gt.checkFileTranslations(currentQueryData);
+
+    const translations = responseData.translations || [];
+
+    // Filter for ready translations
+    const readyTranslations = translations.filter(
+      (translation) => translation.isReady && translation.fileId
+    );
+
+    return readyTranslations;
+  } catch (error) {
+    console.error('Error checking translation status', error);
+    return [];
+  }
+}

package/src/translation/createJobs.ts

@@ -0,0 +1,16 @@
+import { libraryDefaultLocale } from 'generaltranslation/internal';
+import { gt, overrideConfig } from '../adapter/core';
+import type { Secrets } from '../types';
+
+export async function createJobs(
+  uploadResult: Awaited<ReturnType<typeof gt.uploadSourceFiles>>,
+  localeIds: string[],
+  secrets: Secrets
+): Promise<Awaited<ReturnType<typeof gt.enqueueFiles>>> {
+  overrideConfig(secrets);
+  const enqueueResult = await gt.enqueueFiles(uploadResult.uploadedFiles, {
+    sourceLocale: gt.sourceLocale || libraryDefaultLocale,
+    targetLocales: localeIds,
+  });
+  return enqueueResult;
+}

package/src/translation/downloadTranslations.ts

@@ -0,0 +1,68 @@
+import { gt, overrideConfig } from '../adapter/core';
+import type { Secrets } from '../types';
+
+export type BatchedFiles = Array<{
+  documentId: string;
+  versionId: string;
+  translationId: string;
+  locale: string;
+}>;
+
+export type DownloadedFile = {
+  docData: {
+    documentId: string;
+    versionId: string;
+    translationId: string;
+    locale: string;
+  };
+  data: string;
+};
+/**
+ * Downloads multiple translation files in a single batch request
+ * @param files - Array of files to download with their output paths
+ * @param maxRetries - Maximum number of retry attempts
+ * @param retryDelay - Delay between retries in milliseconds
+ * @returns Object containing successful and failed file IDs
+ */
+export async function downloadTranslations(
+  files: BatchedFiles,
+  secrets: Secrets,
+  maxRetries = 3,
+  retryDelay = 1000
+): Promise<DownloadedFile[]> {
+  overrideConfig(secrets);
+  let retries = 0;
+  const fileIds = files.map((file) => file.translationId);
+
+  const map = new Map(files.map((file) => [file.translationId, file]));
+  const result = [] as DownloadedFile[];
+
+  while (retries <= maxRetries) {
+    try {
+      // Download the files
+      const responseData = await gt.downloadFileBatch(fileIds);
+      const downloadedFiles = responseData.files || [];
+
+      // Process each file in the response
+      for (const file of downloadedFiles) {
+        const documentData = map.get(file.id);
+        if (!documentData) {
+          continue;
+        }
+
+        result.push({
+          docData: documentData,
+          data: file.data,
+        });
+      }
+
+      return result;
+    } catch (error) {
+      // Increment retry counter and wait before next attempt
+      retries++;
+      await new Promise((resolve) => setTimeout(resolve, retryDelay));
+    }
+  }
+
+  return result;
+}

package/src/translation/importDocument.ts

@@ -0,0 +1,23 @@
+import { pluginConfig } from '../adapter/core';
+import { documentLevelPatch } from '../configuration/baseDocumentLevelConfig/documentLevelPatch';
+import type { GTFile, TranslationFunctionContext } from '../types';
+import { deserializeDocument } from '../utils/serialize';
+
+export async function importDocument(
+  docInfo: GTFile,
+  localeId: string,
+  document: string,
+  context: TranslationFunctionContext,
+  mergeWithTargetLocale: boolean = false
+) {
+  const { client } = context;
+  const deserialized = deserializeDocument(document);
+  return documentLevelPatch(
+    docInfo, // versionId is not used here, since we just use the _rev id in the deserialized HTML itself
+    deserialized,
+    localeId,
+    client,
+    pluginConfig.getLanguageField(),
+    mergeWithTargetLocale
+  );
+}

package/src/translation/initProject.ts

@@ -0,0 +1,61 @@
+import { gt, overrideConfig } from '../adapter/core';
+import type { Secrets } from '../types';
+
+export async function initProject(
+  uploadResult: Awaited<ReturnType<typeof gt.uploadSourceFiles>>,
+  options: { timeout?: number },
+  secrets: Secrets
+): Promise<boolean> {
+  overrideConfig(secrets);
+  const setupDecision = await Promise.resolve(gt.shouldSetupProject?.())
+    .then((v: any) => v)
+    .catch(() => ({ shouldSetupProject: false }));
+  const shouldSetupProject = Boolean(setupDecision?.shouldSetupProject);
+
+  // Step 2: Setup if needed and poll until complete
+  if (shouldSetupProject) {
+    // Calculate timeout once for setup fetching
+    // Accept number or numeric string, default to 600s
+    const timeoutVal =
+      options?.timeout !== undefined ? Number(options.timeout) : 600;
+    const setupTimeoutMs =
+      (Number.isFinite(timeoutVal) ? timeoutVal : 600) * 1000;
+
+    const { setupJobId } = await gt.setupProject(uploadResult.uploadedFiles);
+
+    const start = Date.now();
+    const pollInterval = 2000;
+
+    let setupCompleted = false;
+    let setupFailedMessage: string | null = null;
+
+    while (true) {
+      const status = await gt.checkSetupStatus(setupJobId);
+
+      if (status.status === 'completed') {
+        setupCompleted = true;
+        break;
+      }
+      if (status.status === 'failed') {
+        setupFailedMessage = status.error?.message || 'Unknown error';
+        break;
+      }
+      if (Date.now() - start > setupTimeoutMs) {
+        setupFailedMessage = 'Timed out while waiting for setup generation';
+        break;
+      }
+      await new Promise((r) => setTimeout(r, pollInterval));
+    }
+
+    if (setupCompleted) {
+      console.log('Setup successfully completed');
+      return true;
+    } else {
+      console.log(`Setup ${setupFailedMessage ? 'failed' : 'timed out'} `);
+      return false;
+    }
+  } else {
+    console.log('Setup not needed');
+  }
+  return true;
+}

package/src/translation/uploadFiles.ts

@@ -0,0 +1,32 @@
+import type { GTFile, Secrets } from '../types';
+import { gt, overrideConfig } from '../adapter/core';
+import { libraryDefaultLocale } from 'generaltranslation/internal';
+import type { SerializedDocument } from '../serialization';
+
+// note: this function is used to create a new translation task
+// uploads files & calls the getTranslationTask function
+export async function uploadFiles(
+  documents: {
+    info: GTFile;
+    serializedDocument: SerializedDocument;
+  }[],
+  secrets: Secrets | null
+): Promise<Awaited<ReturnType<typeof gt.uploadSourceFiles>>> {
+  overrideConfig(secrets);
+  const uploadResult = await gt.uploadSourceFiles(
+    documents.map(({ info, serializedDocument }) => ({
+      source: {
+        content: serializedDocument.content,
+        fileName: `sanity/${info.documentId}`,
+        fileId: info.documentId,
+        fileFormat: 'HTML',
+        locale: gt.sourceLocale || libraryDefaultLocale,
+        versionId: info.versionId || undefined,
+      },
+    })),
+    {
+      sourceLocale: gt.sourceLocale || libraryDefaultLocale,
+    }
+  );
+  return uploadResult;
+}

package/src/types.ts

@@ -1,7 +1,9 @@
+// adapted from https://github.com/sanity-io/sanity-translations-tab. See LICENSE.md for more details.
+
 import { SanityClient, Schema, TypedObject } from 'sanity';
-import { SerializedDocument } from 'sanity-naive-html-serializer';
+import type { SerializedDocument } from './serialization';
 import { PortableTextTypeComponent } from '@portabletext/to-html';
-import { DeserializerRule } from '@sanity/block-tools';
+import type { DeserializerRule } from '@portabletext/block-tools';
 
 export type TranslationTaskLocaleStatus = {
   localeId: string;
@@ -72,15 +74,7 @@ export type GTSerializedDocument = Omit<SerializedDocument, 'name'> & GTFile;
 
 export type ExportForTranslation = (
   documentInfo: GTFile,
-  context: TranslationFunctionContext,
-  serializationOptions?: {
-    additionalStopTypes?: string[];
-    additionalSerializers?: Record<
-      string,
-      PortableTextTypeComponent | undefined
-    >;
-  },
-  languageField?: string
+  context: TranslationFunctionContext
 ) => Promise<GTSerializedDocument>;
 
 export type ImportTranslation = (
@@ -88,15 +82,8 @@ export type ImportTranslation = (
   localeId: string,
   document: string,
   context: TranslationFunctionContext,
-  serializationOptions?: {
-    additionalDeserializers?: Record<
-      string,
-      (value: HTMLElement) => TypedObject
-    >;
-    additionalBlockDeserializers?: DeserializerRule[];
-  },
-  languageField?: string,
-  mergeWithTargetLocale?: boolean
+  mergeWithTargetLocale?: boolean,
+  publish?: boolean
 ) => Promise<void>;
 
 export type TranslationsTabConfigOptions = {

package/src/utils/applyDocuments.ts

@@ -0,0 +1,72 @@
+import { JSONPath } from 'jsonpath-plus';
+import JSONPointer from 'jsonpointer';
+import type { IgnoreFields } from '../adapter/types';
+
+export function applyDocuments(
+  documentId: string,
+  sourceDocument: Record<string, any>,
+  targetDocument: Record<string, any>,
+  ignore: IgnoreFields[]
+) {
+  const ignoreFields = ignore.filter(
+    (field) =>
+      field.documentId === documentId ||
+      field.documentId === undefined ||
+      field.documentId === null
+  );
+
+  // Start with a shallow copy of the source document
+  const mergedDocument = { ...sourceDocument };
+
+  // Merge top-level properties of targetDocument
+  for (const [key, value] of Object.entries(targetDocument)) {
+    mergedDocument[key] = value;
+  }
+
+  // Process ignored fields and restore them from source document
+  for (const ignoreField of ignoreFields) {
+    if (!ignoreField.fields) continue;
+
+    for (const field of ignoreField.fields) {
+      const { property, type } = field;
+
+      try {
+        // Use JSONPath to find matching paths, then JSONPointer to get/set values
+        const sourceResults = JSONPath({
+          json: sourceDocument,
+          path: property,
+          resultType: 'all',
+          flatten: true,
+          wrap: true,
+        });
+
+        if (sourceResults && sourceResults.length > 0) {
+          // Process each matching path
+          sourceResults.forEach((result: { pointer: string; value: any }) => {
+            const sourceValue = result.value;
+
+            // If type is specified, check if it matches the object's _type property
+            if (type !== undefined) {
+              if (
+                typeof sourceValue === 'object' &&
+                sourceValue !== null &&
+                sourceValue._type === type
+              ) {
+                // Type matches, restore the entire object using JSONPointer
+                JSONPointer.set(mergedDocument, result.pointer, sourceValue);
+              }
+            } else {
+              // No type specified, restore the value using JSONPointer
+              JSONPointer.set(mergedDocument, result.pointer, sourceValue);
+            }
+          });
+        }
+      } catch (error) {
+        // Invalid JSONPath, skip this field
+        console.warn(`Invalid JSONPath: ${property}`, error);
+      }
+    }
+  }
+
+  return mergedDocument;
+}

package/src/utils/batchProcessor.ts

@@ -0,0 +1,111 @@
+import { GTFile, TranslationFunctionContext } from '../types';
+import { importDocument } from '../translation/importDocument';
+
+export interface BatchProcessorOptions {
+  batchSize?: number;
+  onProgress?: (current: number, total: number) => void;
+  onItemSuccess?: (item: any, result: any) => void;
+  onItemFailure?: (item: any, error: any) => void;
+}
+
+export interface ImportBatchItem {
+  docInfo: GTFile;
+  locale: string;
+  data: any;
+  translationContext: TranslationFunctionContext;
+  key: string;
+}
+
+export async function processBatch<T>(
+  items: T[],
+  processor: (item: T) => Promise<any>,
+  options: BatchProcessorOptions = {}
+): Promise<{
+  successCount: number;
+  failureCount: number;
+  successfulItems: any[];
+  failedItems: { item: T; error: any }[];
+}> {
+  const { batchSize = 20, onProgress, onItemSuccess, onItemFailure } = options;
+
+  let successCount = 0;
+  let failureCount = 0;
+  const successfulItems: any[] = [];
+  const failedItems: { item: T; error: any }[] = [];
+
+  for (let i = 0; i < items.length; i += batchSize) {
+    const batch = items.slice(i, i + batchSize);
+
+    const batchPromises = batch.map(async (item) => {
+      try {
+        const result = await processor(item);
+        onItemSuccess?.(item, result);
+        return { success: true, item, result };
+      } catch (error) {
+        onItemFailure?.(item, error);
+        return { success: false, item, error };
+      }
+    });
+
+    const batchResults = await Promise.all(batchPromises);
+
+    batchResults.forEach((result) => {
+      if (result.success) {
+        successCount++;
+        successfulItems.push(result.result);
+      } else {
+        failureCount++;
+        failedItems.push({ item: result.item, error: result.error });
+      }
+    });
+
+    onProgress?.(i + batch.length, items.length);
+  }
+
+  return { successCount, failureCount, successfulItems, failedItems };
+}
+
+export async function processImportBatch(
+  items: ImportBatchItem[],
+  options: BatchProcessorOptions = {}
+): Promise<{
+  successCount: number;
+  failureCount: number;
+  successfulImports: string[];
+  failedItems: { item: ImportBatchItem; error: any }[];
+}> {
+  const successfulImports: string[] = [];
+
+  const result = await processBatch(
+    items,
+    async (item: ImportBatchItem) => {
+      await importDocument(
+        item.docInfo,
+        item.locale,
+        item.data,
+        item.translationContext,
+        false
+      );
+      return item.key;
+    },
+    {
+      ...options,
+      onItemSuccess: (item: ImportBatchItem, key: string) => {
+        successfulImports.push(key);
+        options.onItemSuccess?.(item, key);
+      },
+      onItemFailure: (item: ImportBatchItem, error: any) => {
+        console.error(
+          `Failed to import ${item.docInfo.documentId} (${item.locale}):`,
+          error
+        );
+        options.onItemFailure?.(item, error);
+      },
+    }
+  );
+
+  return {
+    ...result,
+    successfulImports,
+  };
+}

package/src/utils/importUtils.ts

@@ -0,0 +1,95 @@
+import { SanityDocument } from 'sanity';
+import { GTFile, Secrets, TranslationFunctionContext } from '../types';
+import {
+  downloadTranslations,
+  BatchedFiles,
+} from '../translation/downloadTranslations';
+import { processImportBatch, ImportBatchItem } from './batchProcessor';
+
+export interface TranslationStatus {
+  progress: number;
+  isReady: boolean;
+  translationId?: string;
+}
+
+export interface ImportResult {
+  successCount: number;
+  failureCount: number;
+  successfulImports: string[];
+}
+
+export interface ImportOptions {
+  filterReadyFiles?: (key: string, status: TranslationStatus) => boolean;
+  onProgress?: (current: number, total: number) => void;
+  onImportSuccess?: (key: string) => void;
+}
+
+export async function getReadyFilesForImport(
+  documents: SanityDocument[],
+  translationStatuses: Map<string, TranslationStatus>,
+  options: ImportOptions = {}
+): Promise<BatchedFiles> {
+  const { filterReadyFiles = () => true } = options;
+  const readyFiles: BatchedFiles = [];
+
+  for (const [key, status] of translationStatuses.entries()) {
+    if (
+      status.isReady &&
+      status.translationId &&
+      filterReadyFiles(key, status)
+    ) {
+      const [documentId, locale] = key.split(':');
+      const document = documents.find(
+        (doc) => (doc._id?.replace('drafts.', '') || doc._id) === documentId
+      );
+
+      if (document) {
+        readyFiles.push({
+          documentId,
+          versionId: document._rev,
+          translationId: status.translationId,
+          locale,
+        });
+      }
+    }
+  }
+
+  return readyFiles;
+}
+
+export async function importTranslations(
+  readyFiles: BatchedFiles,
+  secrets: Secrets,
+  translationContext: TranslationFunctionContext,
+  options: ImportOptions = {}
+): Promise<ImportResult> {
+  if (readyFiles.length === 0) {
+    return { successCount: 0, failureCount: 0, successfulImports: [] };
+  }
+
+  const downloadedFiles = await downloadTranslations(readyFiles, secrets);
+
+  const importItems: ImportBatchItem[] = downloadedFiles.map((file) => ({
+    docInfo: {
+      documentId: file.docData.documentId,
+      versionId: file.docData.versionId,
+    },
+    locale: file.docData.locale,
+    data: file.data,
+    translationContext,
+    key: `${file.docData.documentId}:${file.docData.locale}`,
+  }));
+
+  const result = await processImportBatch(importItems, {
+    onProgress: options.onProgress,
+    onItemSuccess: (item, key) => {
+      options.onImportSuccess?.(key);
+    },
+  });
+
+  return {
+    successCount: result.successCount,
+    failureCount: result.failureCount,
+    successfulImports: result.successfulImports,
+  };
+}

package/src/utils/serialize.ts

@@ -0,0 +1,52 @@
+import { SanityDocument, Schema } from 'sanity';
+import {
+  BaseDocumentDeserializer,
+  BaseDocumentSerializer,
+  defaultStopTypes,
+  customSerializers,
+  customBlockDeserializers,
+} from '../serialization/';
+import { PortableTextHtmlComponents } from '@portabletext/to-html';
+import { pluginConfig } from '../adapter/core';
+import { merge } from 'lodash';
+
+export function deserializeDocument(document: string) {
+  const deserializers = merge(
+    { types: {} },
+    pluginConfig.getAdditionalDeserializers()
+  ) satisfies Partial<PortableTextHtmlComponents>;
+  const blockDeserializers = [
+    ...customBlockDeserializers,
+    ...pluginConfig.getAdditionalBlockDeserializers(),
+  ];
+  const deserialized = BaseDocumentDeserializer.deserializeDocument(
+    document,
+    deserializers,
+    blockDeserializers
+  ) as SanityDocument;
+  return deserialized;
+}
+
+export function serializeDocument(
+  document: SanityDocument,
+  schema: Schema,
+  baseLanguage: string
+) {
+  const stopTypes = [
+    ...defaultStopTypes,
+    ...pluginConfig.getAdditionalStopTypes(),
+  ];
+  const serializers = merge(
+    customSerializers,
+    pluginConfig.getAdditionalSerializers()
+  ) satisfies Partial<PortableTextHtmlComponents>;
+
+  const serialized = BaseDocumentSerializer(schema).serializeDocument(
+    document,
+    'document',
+    baseLanguage,
+    stopTypes,
+    serializers
+  );
+  return serialized;
+}

package/src/utils/shared.ts

@@ -0,0 +1 @@
+export const SECRETS_NAMESPACE = 'generaltranslation.secrets';

package/src/adapter/index.ts

@@ -1,13 +0,0 @@
-import type { Adapter } from '../types';
-
-import { getLocales } from './getLocales';
-import { getTranslationTask } from './getTranslationTask';
-import { getTranslation } from './getTranslation';
-import { createTask } from './createTask';
-
-export const GTAdapter: Adapter = {
-  getLocales,
-  getTranslationTask,
-  createTask,
-  getTranslation,
-};