gt-sanity 0.0.5 → 1.0.0

package/src/serialization/deserialize/BaseDocumentDeserializer.ts

@@ -0,0 +1,171 @@
+// Adapted from https://github.com/sanity-io/sanity-naive-html-serializer
+import { htmlToBlocks } from '@portabletext/block-tools';
+import {
+  customDeserializers,
+  customBlockDeserializers,
+} from '../BaseSerializationConfig';
+import { Deserializer } from '../types';
+import { blockContentType, preprocess } from './helpers';
+import { mergeBlocks } from '../helpers';
+
+export const deserializeArray = (
+  arrayHTML: Element,
+  deserializers: Record<string, any> = customDeserializers,
+  blockDeserializers = customBlockDeserializers
+) => {
+  const output: any[] = [];
+  const children = Array.from(arrayHTML.children);
+  children.forEach((child) => {
+    let deserializedObject: any;
+    try {
+      if (child.tagName?.toLowerCase() === 'span') {
+        deserializedObject = preprocess(child.innerHTML);
+      }
+      //has specific class name or data type, so it's an obj
+      else if (
+        child.className ||
+        child.getAttribute('data-type') === 'object'
+      ) {
+        deserializedObject = deserializeObject(
+          child,
+          deserializers,
+          blockDeserializers
+        );
+        deserializedObject._key = child.id;
+      } else {
+        deserializedObject = htmlToBlocks(child.outerHTML, blockContentType, {
+          rules: blockDeserializers,
+        });
+        deserializedObject = mergeBlocks(deserializedObject);
+        deserializedObject._key = child.id;
+      }
+    } catch (e) {
+      //eslint-disable-next-line no-console
+      console.debug(
+        `Tried to deserialize block: ${child.outerHTML} in an array but failed to identify it! Error: ${e}`
+      );
+    }
+    output.push(deserializedObject);
+  });
+  return output;
+};
+
+export const deserializeObject = (
+  objectHTML: Element,
+  deserializers: Record<string, any> = customDeserializers,
+  blockDeserializers = customBlockDeserializers
+) => {
+  const deserialize = deserializers.types[objectHTML.className];
+  if (deserialize) {
+    return deserialize(objectHTML);
+  }
+
+  const output: Record<string, any> = {};
+  //account for anonymous inline objects
+  if (objectHTML.className) {
+    output._type = objectHTML.className;
+  }
+  const children = Array.from(objectHTML.children);
+
+  children.forEach((child) => {
+    //string field
+    if (child.tagName?.toLowerCase() === 'span') {
+      output[child.className] = preprocess(child.innerHTML);
+    }
+    //richer field, either object or array
+    else if (child.getAttribute('data-level') === 'field') {
+      const deserialized = deserializeHTML(
+        child.outerHTML,
+        deserializers,
+        blockDeserializers
+      );
+      if (deserialized && Object.keys(deserialized).length) {
+        output[child.className] = deserialized;
+      } else {
+        //eslint-disable-next-line no-console
+        console.debug(
+          `Deserializer: Skipping empty or unreadable HTML: ${child.outerHTML}`
+        );
+      }
+    } else if (child.getAttribute('data-type') === 'array') {
+      output[child.className] = deserializeArray(
+        child,
+        deserializers,
+        blockDeserializers
+      );
+    }
+  });
+  return output;
+};
+
+export const deserializeHTML = (
+  html: string,
+  deserializers: Record<string, any>,
+  blockDeserializers: Array<any>
+): Record<string, any> | any[] => {
+  //parent node is always div with classname of field -- get its child
+  let HTMLnode = new DOMParser().parseFromString(html, 'text/html').body
+    .children[0];
+
+  //catch embedded object as a field
+  if (HTMLnode?.getAttribute('data-level') === 'field') {
+    HTMLnode = HTMLnode.children[0];
+  }
+
+  if (!HTMLnode) {
+    return {};
+  }
+
+  let output: Record<string, any> | any[];
+
+  //prioritize custom deserialization
+  const deserialize = deserializers.types[HTMLnode.className];
+  if (deserialize) {
+    output = deserialize(HTMLnode);
+  } else if (HTMLnode.getAttribute('data-type') === 'object') {
+    output = deserializeObject(HTMLnode, deserializers, blockDeserializers);
+  } else if (HTMLnode.getAttribute('data-type') === 'array') {
+    output = deserializeArray(HTMLnode, deserializers, blockDeserializers);
+  } else {
+    output = {};
+    //eslint-disable-next-line no-console
+    console.debug(
+      `Tried to deserialize block ${HTMLnode.outerHTML} but failed to identify it!`
+    );
+  }
+
+  return output;
+};
+
+export const deserializeDocument = (
+  serializedDoc: string,
+  deserializers: Record<string, any> = customDeserializers,
+  blockDeserializers = customBlockDeserializers
+): Record<string, any> => {
+  const metadata: Record<string, any> = {};
+  const head = new DOMParser().parseFromString(serializedDoc, 'text/html').head;
+
+  Array.from(head.children).forEach((metaTag) => {
+    const validTags = ['_id', '_rev', '_type'];
+    const metaName = metaTag.getAttribute('name');
+    if (metaName && validTags.includes(metaName)) {
+      metadata[metaName] = metaTag.getAttribute('content');
+    }
+  });
+
+  const content: Record<string, any> = deserializeHTML(
+    serializedDoc,
+    deserializers,
+    blockDeserializers
+  );
+
+  return {
+    ...content,
+    ...metadata,
+  };
+};
+
+export const BaseDocumentDeserializer: Deserializer = {
+  deserializeDocument,
+  deserializeHTML,
+};

package/src/serialization/deserialize/helpers.ts

@@ -0,0 +1,42 @@
+// Adapted from https://github.com/sanity-io/sanity-naive-html-serializer
+import { htmlToBlocks } from '@portabletext/block-tools';
+import { Schema } from '@sanity/schema';
+import { ObjectField, PortableTextSpan, PortableTextTextBlock } from 'sanity';
+
+const defaultSchema = Schema.compile({
+  name: 'default',
+  types: [
+    {
+      type: 'object',
+      name: 'default',
+      fields: [
+        {
+          name: 'block',
+          type: 'array',
+          of: [{ type: 'block' }],
+        },
+      ],
+    },
+  ],
+});
+
+export const blockContentType = defaultSchema
+  .get('default')
+  .fields.find((field: ObjectField) => field.name === 'block').type;
+
+export const noSchemaWarning = (obj: Element): string =>
+  `WARNING: Unfortunately the deserializer may have issues with this field or object: ${obj.className}.
+  If it's a specific type, you may need to declare  at the top level, or write a custom deserializer.`;
+
+//helper to handle messy input -- take advantage
+//of blockTools' sanitizing behavior for single strings
+export const preprocess = (html: string): string => {
+  const intermediateBlocks = htmlToBlocks(
+    `<p>${html}</p>`,
+    blockContentType
+  ) as PortableTextTextBlock<PortableTextSpan>[];
+  if (!intermediateBlocks.length) {
+    throw new Error(`Error parsing string '${html}'`);
+  }
+  return intermediateBlocks[0].children[0].text;
+};

package/src/serialization/helpers.ts

@@ -0,0 +1,18 @@
+import { PortableTextTextBlock } from 'sanity';
+
+// Helper function to merge multiple blocks
+// Prioritize blocks[0]
+export function mergeBlocks(blocks: PortableTextTextBlock[]) {
+  const mergedBlock = { ...blocks[0] };
+  mergedBlock.markDefs = mergedBlock.markDefs ?? [];
+  for (const [idx, block] of blocks.entries()) {
+    if (idx === 0) {
+      continue;
+    }
+    mergedBlock.children.push(...block.children);
+    mergedBlock.markDefs.push(...(block.markDefs ?? []));
+  }
+  mergedBlock._type = 'block';
+
+  return mergedBlock;
+}

package/src/serialization/index.ts

@@ -0,0 +1,11 @@
+export { BaseDocumentMerger } from './BaseDocumentMerger';
+export { BaseDocumentSerializer } from './serialize';
+export { BaseDocumentDeserializer } from './deserialize/BaseDocumentDeserializer';
+export {
+  defaultStopTypes,
+  customSerializers,
+  customBlockDeserializers,
+} from './BaseSerializationConfig';
+
+export type { SerializedDocument, Deserializer, Merger } from './types';
+export { attachGTData, detachGTData } from './data';

package/src/serialization/serialize/fieldFilters.ts

@@ -0,0 +1,124 @@
+// Adapted from https://github.com/sanity-io/sanity-naive-html-serializer
+
+import { ObjectField, TypedObject } from 'sanity';
+
+const META_FIELDS = ['_key', '_type', '_id'];
+
+/*
+ * Helper. If field-level translation pattern used, only sends over
+ * content from the base language. Works recursively, so if users
+ * use this pattern several layers deep, base language fields will still be found.
+ */
+export const languageObjectFieldFilter = (
+  obj: Record<string, any>,
+  baseLang: string
+): Record<string, any> => {
+  const filterToLangField = (childObj: Record<string, any>) => {
+    const filteredObj: Record<string, any> = {};
+    filteredObj[baseLang] = childObj[baseLang];
+    META_FIELDS.forEach((field) => {
+      if (childObj[field]) {
+        filteredObj[field] = childObj[field];
+      }
+    });
+    return filteredObj;
+  };
+
+  const findBaseLang = (childObj: Record<string, any>): Record<string, any> => {
+    const filteredObj: Record<string, any> = {};
+    META_FIELDS.forEach((field) => {
+      if (childObj[field]) {
+        filteredObj[field] = childObj[field];
+      }
+    });
+
+    for (const key in childObj) {
+      if (childObj.hasOwnProperty(key)) {
+        const value: any = childObj[key];
+        //we've reached a base language field, add it to
+        //what we want to send to translation
+        if (value.hasOwnProperty(baseLang)) {
+          filteredObj[key] = filterToLangField(value);
+        }
+        //we have an array that may have language fields in its objects
+        else if (
+          Array.isArray(value) &&
+          value.length &&
+          typeof value[0] === 'object'
+        ) {
+          //recursively find and filter for any objects that have the base language
+          const validLangObjects = value.reduce((validArr, objInArray) => {
+            if (objInArray._type === 'block') {
+              validArr.push(objInArray);
+            } else if (objInArray.hasOwnProperty(baseLang)) {
+              validArr.push(filterToLangField(objInArray));
+            } else {
+              const filtered = findBaseLang(objInArray);
+              const nonMetaFields = Object.keys(filtered).filter(
+                (objInArrayKey) => META_FIELDS.indexOf(objInArrayKey) === -1
+              );
+              if (nonMetaFields.length) {
+                validArr.push(filtered);
+              }
+            }
+            return validArr;
+          }, []);
+          if (validLangObjects.length) {
+            filteredObj[key] = validLangObjects;
+          }
+        }
+        //we have an object nested in an object
+        //recurse down the tree
+        else if (typeof value === 'object') {
+          const nestedLangObj = findBaseLang(value);
+          const nonMetaFields = Object.keys(nestedLangObj).filter(
+            (nestedObjKey) => META_FIELDS.indexOf(nestedObjKey) === -1
+          );
+          if (nonMetaFields.length) {
+            filteredObj[key] = nestedLangObj;
+          }
+        }
+      }
+    }
+    return filteredObj;
+  };
+
+  //send top level object into recursive function
+  return findBaseLang(obj);
+};
+
+/*
+ * Eliminates stop-types and non-localizable fields
+ * for document-level translation.
+ */
+export const fieldFilter = (
+  obj: Record<string, any>,
+  objFields: ObjectField[],
+  stopTypes: string[]
+): TypedObject => {
+  const filteredObj: TypedObject = { _type: obj._type };
+
+  const fieldFilterFunc = (field: Record<string, any>) => {
+    if (field.localize === false) {
+      return false;
+    } else if (field.type === 'string' || field.type === 'text') {
+      return true;
+    } else if (Array.isArray(obj[field.name])) {
+      return true;
+    } else if (!stopTypes.includes(field.type)) {
+      return true;
+    }
+    return false;
+  };
+
+  const validFields = [
+    ...META_FIELDS,
+    ...objFields?.filter(fieldFilterFunc)?.map((field) => field.name),
+  ];
+  validFields.forEach((field) => {
+    if (obj[field]) {
+      filteredObj[field] = obj[field];
+    }
+  });
+  return filteredObj;
+};

package/src/serialization/serialize/index.ts

@@ -0,0 +1,284 @@
+// Adapted from https://github.com/sanity-io/sanity-naive-html-serializer
+
+import {
+  defaultStopTypes,
+  customSerializers,
+} from '../BaseSerializationConfig';
+import { SanityDocument, TypedObject, Schema } from 'sanity';
+import { TranslationLevel } from '../types';
+import { fieldFilter, languageObjectFieldFilter } from './fieldFilters';
+import { PortableTextTypeComponent, toHTML } from '@portabletext/to-html';
+
+const META_FIELDS = ['_key', '_type', '_id', '_weak'];
+
+export const BaseDocumentSerializer = (schemas: Schema) => {
+  /*
+   * Helper function that allows us to get metadata (like `localize: false`) from schema fields.
+   */
+  const getSchema = (name: string) =>
+    schemas?._original?.types.find((s) => s.name === name) as any;
+
+  const serializeObject = (
+    obj: TypedObject,
+    stopTypes: string[],
+    serializers: Record<string, any>
+  ) => {
+    if (stopTypes.includes(obj._type)) {
+      return '';
+    }
+
+    //if user has declared a custom serializer, use that
+    //instead of this method
+    const hasSerializer =
+      serializers.types && Object.keys(serializers.types).includes(obj._type);
+    if (hasSerializer) {
+      return toHTML([obj], { components: serializers });
+    }
+
+    //we don't need to worry about PT types
+    if (obj._type === 'span' || obj._type === 'block') {
+      return toHTML(obj, { components: serializers });
+    }
+
+    //if schema is available, encode values in the order they're declared in the schema,
+    //since this will likely be more intuitive for a translator.
+    let fieldNames = Object.keys(obj).filter((key) => key !== '_type');
+    const schema = getSchema(obj._type);
+    if (schema && schema.fields) {
+      fieldNames = schema.fields
+        .map((field: Record<string, any>) => field.name)
+        .filter((schemaKey: string) => Object.keys(obj).includes(schemaKey));
+    }
+
+    //account for anonymous inline objects
+    if (typeof obj === 'object' && !obj._type) {
+      obj._type = '';
+    }
+
+    //in some cases, we might recurse through many objects of the same type
+    //we should take all methods necessary to ensure state does not persist
+    //otherwise we risk using old serialization methods on new items
+    const newSerializationMethods: Record<string, PortableTextTypeComponent> =
+      {};
+    const tempType = `${obj._type}__temp_type__${Math.random().toString(36).substring(7)}`;
+    const objToSerialize: TypedObject = { _type: tempType };
+    //for our default serialization method, we only need to
+    //capture metadata. the rest will be recursively turned into strings.
+    META_FIELDS.filter((f) => f !== '_type').forEach((field) => {
+      objToSerialize[field] = obj[field];
+    });
+
+    let innerHTML = '';
+
+    //if it's a custom object, iterate through its keys to find and serialize translatable content
+    fieldNames.forEach((fieldName) => {
+      let htmlField = '';
+
+      if (!META_FIELDS.includes(fieldName)) {
+        const value = obj[fieldName];
+        //strings are either string fields or have recursively been turned
+        //into HTML because they were a nested object or array
+        if (typeof value === 'string') {
+          const htmlRegex = new RegExp(/<("[^"]*"|'[^']*'|[^'">])*>/);
+          if (htmlRegex.test(value)) {
+            htmlField = value;
+          } else {
+            htmlField = `<span class="${fieldName}">${value}</span>`;
+          }
+        }
+
+        //array fields get filtered and its children serialized
+        else if (Array.isArray(value)) {
+          htmlField = serializeArray(value, fieldName, stopTypes, {
+            ...serializers,
+            types: { ...serializers.types },
+          });
+        }
+
+        //this is an object in an object, serialize it first
+        else {
+          const embeddedObject = value as TypedObject;
+          const embeddedObjectSchema = getSchema(embeddedObject._type);
+          let toTranslate = embeddedObject;
+          if (embeddedObjectSchema && embeddedObjectSchema.fields) {
+            toTranslate = fieldFilter(
+              toTranslate,
+              embeddedObjectSchema.fields,
+              stopTypes
+            );
+          }
+          const objHTML = serializeObject(toTranslate, stopTypes, {
+            ...serializers,
+            types: { ...serializers.types },
+          });
+          htmlField = `<div class="${fieldName}" data-level="field">${objHTML}</div>`;
+        }
+
+        innerHTML += htmlField;
+      }
+    });
+
+    if (!innerHTML) {
+      return '';
+    }
+
+    newSerializationMethods[tempType] = ({ value }: { value: TypedObject }) => {
+      let div = `<div class="${value._type.split('__temp_type__')[0]}"`;
+      if (value._key || value._id) {
+        div += `id="${value._key ?? value._id}"`;
+      }
+
+      return [div, ` data-type="object">${innerHTML}</div>`].join('');
+    };
+
+    let serializedBlock = '';
+    try {
+      serializedBlock = toHTML(objToSerialize, {
+        components: {
+          ...serializers,
+          types: {
+            ...serializers.types,
+            ...newSerializationMethods,
+          },
+        },
+      });
+    } catch (err) {
+      //eslint-disable-next-line no-console -- this is a warning
+      console.warn(
+        `Had issues serializing block of type "${obj._type}". Please specify a serialization method for this block in your serialization config. Received error: ${err}`
+      );
+    }
+
+    return serializedBlock;
+  };
+
+  const serializeArray = (
+    fieldContent: Record<string, any>[],
+    fieldName: string,
+    stopTypes: string[],
+    serializers: Record<string, any>
+  ) => {
+    //filter for any blocks that user has indicated
+    //should not be sent for translation
+    const validBlocks = fieldContent.filter(
+      (block) => !stopTypes.includes(block._type)
+    );
+
+    //take out any fields in these blocks that should
+    //not be sent to translation
+    const filteredBlocks = validBlocks.map((block) => {
+      const schema = getSchema(block._type);
+      if (schema && schema.fields) {
+        return fieldFilter(block, schema.fields, stopTypes);
+      }
+      return block;
+    });
+
+    const output = filteredBlocks.map((obj) => {
+      //if object in array is just a string, just return it
+      if (typeof obj === 'string') {
+        return `<span>${obj}</span>`;
+      }
+      //send to serialization method
+      return serializeObject(obj as TypedObject, stopTypes, serializers);
+    });
+
+    //encode this with data-level field
+    return `<div class="${fieldName}" data-type="array">${output.join('')}</div>`;
+  };
+
+  /*
+   * Main parent function: finds fields to translate, and feeds them to appropriate child serialization
+   * methods.
+   */
+  const serializeDocument = (
+    doc: SanityDocument,
+    translationLevel: TranslationLevel = 'document',
+    baseLang = 'en',
+    stopTypes = defaultStopTypes,
+    serializers = customSerializers
+  ) => {
+    const schema = getSchema(doc._type);
+    let filteredObj: Record<string, any> = {};
+
+    //field level translations explicitly send over any fields that
+    //match the base language, regardless of depth
+    if (translationLevel === 'field') {
+      filteredObj = languageObjectFieldFilter(doc, baseLang);
+    }
+    //otherwise, we can refer to the schema and a list of stop types
+    //to determine what should not be sent
+    else {
+      filteredObj = fieldFilter(doc, schema.fields, stopTypes);
+    }
+
+    const serializedFields: Record<string, any> = {};
+
+    for (const key in filteredObj) {
+      if (filteredObj.hasOwnProperty(key) === false) continue;
+      const value: Record<string, any> | Array<any> | string = filteredObj[key];
+
+      if (typeof value === 'string') {
+        serializedFields[key] = value;
+      } else if (Array.isArray(value)) {
+        serializedFields[key] = serializeArray(
+          value,
+          key,
+          stopTypes,
+          serializers
+        );
+      } else if (
+        value &&
+        !stopTypes.find((stopType) => stopType == value?._type)
+      ) {
+        const serialized = serializeObject(
+          value as TypedObject,
+          stopTypes,
+          serializers
+        );
+        serializedFields[key] =
+          `<div class="${key}" data-level='field'>${serialized}</div>`;
+      }
+    }
+
+    //create a valid HTML file
+    const rawHTMLBody = document.createElement('body');
+    rawHTMLBody.innerHTML = serializeObject(
+      serializedFields as TypedObject,
+      stopTypes,
+      serializers
+    );
+
+    const rawHTMLHead = document.createElement('head');
+    const metaFields = ['_id', '_type', '_rev'];
+    //save our metadata as meta tags so we can use them later on
+    metaFields.forEach((field) => {
+      const metaEl = document.createElement('meta');
+      metaEl.setAttribute('name', field);
+      metaEl.setAttribute('content', doc[field] as string);
+      rawHTMLHead.appendChild(metaEl);
+    });
+    //encode version so we can use the correct deserialization methods
+    const versionMeta = document.createElement('meta');
+    versionMeta.setAttribute('name', 'version');
+    versionMeta.setAttribute('content', '3');
+    rawHTMLHead.appendChild(versionMeta);
+
+    const rawHTML = document.createElement('html');
+    rawHTML.appendChild(rawHTMLHead);
+    rawHTML.appendChild(rawHTMLBody);
+
+    return {
+      name: doc._id,
+      content: rawHTML.outerHTML,
+    };
+  };
+
+  return {
+    serializeDocument,
+    fieldFilter,
+    languageObjectFieldFilter,
+    serializeArray,
+    serializeObject,
+  };
+};

package/src/serialization/types.ts

@@ -0,0 +1,41 @@
+// Adapted from https://github.com/sanity-io/sanity-naive-html-serializer
+
+import { ObjectField, SanityDocument, TypedObject, Schema } from 'sanity';
+
+export type SerializedDocument = {
+  name: string;
+  content: string;
+};
+
+export type TranslationLevel = 'document' | 'field';
+
+export interface Deserializer {
+  deserializeDocument: (
+    serializedDoc: string,
+    deserializers?: Record<string, any>,
+    blockDeserializers?: Array<any>
+  ) => Record<string, any>;
+  deserializeHTML: (
+    html: string,
+    deserializers: Record<string, any>,
+    blockDeserializers: Array<any>
+  ) => Record<string, any> | any[];
+}
+
+export interface Merger {
+  fieldLevelMerge: (
+    translatedFields: Record<string, any>,
+    baseDoc: SanityDocument,
+    localeId: string,
+    baseLang: string
+  ) => Record<string, any>;
+  documentLevelMerge: (
+    translatedFields: Record<string, any>,
+    baseDoc: SanityDocument
+  ) => Record<string, any>;
+  reconcileArray: (origArray: any[], translatedArray: any[]) => any[];
+  reconcileObject: (
+    origObject: Record<string, any>,
+    translatedObject: Record<string, any>
+  ) => Record<string, any>;
+}