@temporalio/common 0.19.0-rc.0 → 0.19.2

package/src/converter/data-converter.ts

@@ -1,189 +1,53 @@
-import { ValueError } from '../errors';
-import { str, METADATA_ENCODING_KEY, Payload } from './types';
-import {
-  PayloadConverter,
-  UndefinedPayloadConverter,
-  BinaryPayloadConverter,
-  JsonPayloadConverter,
-} from './payload-converter';
+import { defaultPayloadCodec, PayloadCodec } from './payload-codec';
+import { defaultPayloadConverter, PayloadConverter } from './payload-converter';
 
 /**
- * Used by the framework to serialize/deserialize method parameters that need to be sent over the
- * wire.
+ * When your data (arguments and return values) is sent over the wire and stored by Temporal Server, it is encoded in
+ * binary in a {@link Payload} Protobuf message.
  *
- * Implement this in order to customize worker data serialization or use the default data converter which supports `Uint8Array` and JSON serializables.
+ * The default `DataConverter` supports `undefined`, `Uint8Array`, and JSON serializables (so if
+ * [`JSON.stringify(yourArgOrRetval)`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#description)
+ * works, the default data converter will work). Protobufs are supported via [this
+ * API](https://docs.temporal.io/docs/typescript/data-converters#protobufs).
+ *
+ * Use a custom `DataConverter` to control the contents of your {@link Payload}s. Common reasons for using a custom
+ * `DataConverter` are:
+ * - Converting values that are not supported by the default `DataConverter` (for example, `JSON.stringify()` doesn't
+ *   handle `BigInt`s, so if you want to return `{ total: 1000n }` from a Workflow, Signal, or Activity, you need your
+ *   own `DataConverter`).
+ * - Encrypting values that may contain private information that you don't want stored in plaintext in Temporal Server's
+ *   database.
+ * - Compressing values to reduce disk or network usage.
+ *
+ * To use your custom `DataConverter`, provide it to the {@link WorkflowClient}, {@link Worker}, and
+ * {@link bundleWorkflowCode} (if you use it):
+ * - `new WorkflowClient({ ..., dataConverter })`
+ * - `Worker.create({ ..., dataConverter })`
+ * - `bundleWorkflowCode({ ..., payloadConverterPath })`
  */
 export interface DataConverter {
-  toPayload<T>(value: T): Promise<Payload>;
-
-  fromPayload<T>(payload: Payload): Promise<T>;
-  /**
-   * Implements conversion of a list of values.
-   *
-   * @param values JS values to convert to Payloads.
-   * @return converted value
-   * @throws DataConverterError if conversion of the value passed as parameter failed for any
-   *     reason.
-   */
-  toPayloads(...values: unknown[]): Promise<Payload[] | undefined>;
-
-  /**
-   * Implements conversion of an array of values of different types. Useful for deserializing
-   * arguments of function invocations.
-   *
-   * @param index index of the value in the payloads
-   * @param content serialized value to convert to JS values.
-   * @return converted JS value
-   * @throws DataConverterError if conversion of the data passed as parameter failed for any
-   *     reason.
-   */
-  fromPayloads<T>(index: number, content?: Payload[] | null): Promise<T>;
-
   /**
-   * Sync conversion of single payload, used in the Workflow runtime
+   * Path of a file that has a `payloadConverter` named export.
+   * `payloadConverter` should be an instance of a class that implements {@link PayloadConverter}.
+   * If no path is provided, {@link defaultPayloadConverter} is used.
    */
-  toPayloadSync<T>(value: T): Payload;
+  payloadConverterPath?: string;
 
   /**
-   * Sync conversion from a single payload, used in the Workflow runtime
+   * A {@link PayloadCodec} instance. The default codec is a no-op.
    */
-  fromPayloadSync<T>(payload: Payload): T;
-  /**
-   * Sync conversion of all arguments, used in the Workflow runtime
-   *
-   * Implements conversion of a list of values.
-   *
-   * @param values JS values to convert to Payloads.
-   * @return converted value
-   * @throws DataConverterError if conversion of the value passed as parameter failed for any
-   *     reason.
-   */
-  toPayloadsSync(...values: unknown[]): Payload[] | undefined;
-
-  /**
-   * Sync version of {@link fromPayloads}
-   */
-  fromPayloadsSync<T>(index: number, content?: Payload[] | null): T;
+  payloadCodec?: PayloadCodec;
 }
 
-export class CompositeDataConverter implements DataConverter {
-  readonly converters: PayloadConverter[];
-  readonly converterByEncoding: Map<string, PayloadConverter> = new Map();
-
-  constructor(...converters: PayloadConverter[]) {
-    this.converters = converters;
-    for (const converter of converters) {
-      this.converterByEncoding.set(converter.encodingType, converter);
-    }
-  }
-
-  public async toPayload<T>(value: T): Promise<Payload> {
-    for (const converter of this.converters) {
-      const result = await converter.toData(value);
-      if (result !== undefined) return result;
-    }
-    throw new ValueError(`Cannot serialize ${value}`);
-  }
-
-  public toPayloadSync<T>(value: T): Payload {
-    for (const converter of this.converters) {
-      const result = converter.toDataSync(value);
-      if (result !== undefined) return result;
-    }
-    throw new ValueError(`Cannot serialize ${value}`);
-  }
-
-  public async fromPayload<T>(payload: Payload): Promise<T> {
-    if (payload.metadata === undefined || payload.metadata === null) {
-      throw new ValueError('Missing payload metadata');
-    }
-    const encoding = str(payload.metadata[METADATA_ENCODING_KEY]);
-    const converter = this.converterByEncoding.get(encoding);
-    if (converter === undefined) {
-      throw new ValueError(`Unknown encoding: ${encoding}`);
-    }
-    return await converter.fromData(payload);
-  }
-
-  public fromPayloadSync<T>(payload: Payload): T {
-    if (payload.metadata === undefined || payload.metadata === null) {
-      throw new ValueError('Missing payload metadata');
-    }
-    const encoding = str(payload.metadata[METADATA_ENCODING_KEY]);
-    const converter = this.converterByEncoding.get(encoding);
-    if (converter === undefined) {
-      throw new ValueError(`Unknown encoding: ${encoding}`);
-    }
-    return converter.fromDataSync(payload);
-  }
-
-  public async toPayloads(...values: unknown[]): Promise<Payload[] | undefined> {
-    if (values.length === 0) {
-      return undefined;
-    }
-    return await Promise.all(values.map((value) => this.toPayload(value)));
-  }
-
-  public toPayloadsSync(...values: unknown[]): Payload[] | undefined {
-    if (values.length === 0) {
-      return undefined;
-    }
-    return values.map((value) => this.toPayloadSync(value));
-  }
-
-  public async fromPayloads<T>(index: number, payloads?: Payload[] | null): Promise<T> {
-    // To make adding arguments a backwards compatible change
-    if (payloads === undefined || payloads === null || index >= payloads.length) {
-      return undefined as any;
-    }
-    return await this.fromPayload(payloads[index]);
-  }
-
-  public fromPayloadsSync<T>(index: number, payloads?: Payload[] | null): T {
-    // To make adding arguments a backwards compatible change
-    if (payloads === undefined || payloads === null || index >= payloads.length) {
-      return undefined as any;
-    }
-    return this.fromPayloadSync(payloads[index]);
-  }
-}
-
-export async function arrayFromPayloads(converter: DataConverter, content?: Payload[] | null): Promise<unknown[]> {
-  if (!content) {
-    return [];
-  }
-  return await Promise.all(content.map((payload: Payload) => converter.fromPayload(payload)));
-}
-
-export async function mapToPayloads<K extends string>(
-  converter: DataConverter,
-  source: Record<K, any>
-): Promise<Record<K, Payload>> {
-  return Object.fromEntries(
-    await Promise.all(
-      Object.entries(source).map(async ([k, v]): Promise<[K, Payload]> => [k as K, await converter.toPayload(v)])
-    )
-  ) as Record<K, Payload>;
-}
-
-export function arrayFromPayloadsSync(converter: DataConverter, content?: Payload[] | null): unknown[] {
-  if (!content) {
-    return [];
-  }
-  return content.map((payload: Payload) => converter.fromPayloadSync(payload));
-}
-
-export function mapToPayloadsSync<K extends string>(
-  converter: DataConverter,
-  source: Record<K, any>
-): Record<K, Payload> {
-  return Object.fromEntries(
-    Object.entries(source).map(([k, v]): [K, Payload] => [k as K, converter.toPayloadSync(v)])
-  ) as Record<K, Payload>;
+/**
+ * A {@link DataConverter} that has been loaded via {@link loadDataConverter}.
+ */
+export interface LoadedDataConverter {
+  payloadConverter: PayloadConverter;
+  payloadCodec: PayloadCodec;
 }
 
-export const defaultDataConverter = new CompositeDataConverter(
-  new UndefinedPayloadConverter(),
-  new BinaryPayloadConverter(),
-  new JsonPayloadConverter()
-);
+export const defaultDataConverter: LoadedDataConverter = {
+  payloadConverter: defaultPayloadConverter,
+  payloadCodec: defaultPayloadCodec,
+};

package/src/converter/patch-protobuf-root.ts

@@ -0,0 +1,49 @@
+import { isRecord } from '@temporalio/internal-workflow-common';
+
+/**
+ * Create a version of `root` with non-nested namespaces to match the generated types.
+ * For more information, see:
+ * https://github.com/temporalio/sdk-typescript/blob/main/docs/protobuf-libraries.md#current-solution
+ * @param root Generated by `pbjs -t json-module -w commonjs -o json-module.js *.proto`
+ * @returns A new patched `root`
+ */
+export function patchProtobufRoot<T extends Record<string, unknown>>(root: T): T {
+  return _patchProtobufRoot(root);
+}
+
+function _patchProtobufRoot<T extends Record<string, unknown>>(root: T, name?: string): T {
+  const newRoot = new (root.constructor as any)(isNamespace(root) ? name : {});
+  for (const key in root) {
+    newRoot[key] = root[key];
+  }
+
+  if (isRecord(root.nested)) {
+    for (const typeOrNamespace in root.nested) {
+      const value = root.nested[typeOrNamespace];
+      if (typeOrNamespace in root && !(isType(root[typeOrNamespace]) || isNamespace(root[typeOrNamespace]))) {
+        console.log(
+          `patchRoot warning: overriding property '${typeOrNamespace}' that is used by protobufjs with the '${typeOrNamespace}' protobuf namespace. This may result in protobufjs not working property.`
+        );
+      }
+
+      if (isNamespace(value)) {
+        newRoot[typeOrNamespace] = _patchProtobufRoot(value, typeOrNamespace);
+      } else if (isType(value)) {
+        newRoot[typeOrNamespace] = value;
+      }
+    }
+  }
+
+  return newRoot;
+}
+
+type Type = Record<string, unknown>;
+type Namespace = { nested: Record<string, unknown> };
+
+function isType(value: unknown): value is Type {
+  return isRecord(value) && value.constructor.name === 'Type';
+}
+
+function isNamespace(value: unknown): value is Namespace {
+  return isRecord(value) && value.constructor.name === 'Namespace';
+}

package/src/converter/payload-codec.ts

@@ -0,0 +1,30 @@
+import { Payload } from './types';
+
+/**
+ * `PayloadCodec` is an optional step that happens between the wire and the {@link PayloadConverter}:
+ *
+ * Temporal Server <--> Wire <--> `PayloadCodec` <--> `PayloadConverter` <--> User code
+ *
+ * Implement this to transform an array of {@link Payload}s to/from the format sent over the wire and stored by Temporal Server.
+ * Common transformations are encryption and compression.
+ */
+export interface PayloadCodec {
+  /**
+   * Encode an array of {@link Payload}s for sending over the wire.
+   * @param payloads May have length 0.
+   */
+  encode(payloads: Payload[]): Promise<Payload[]>;
+
+  /**
+   * Decode an array of {@link Payload}s received from the wire.
+   */
+  decode(payloads: Payload[]): Promise<Payload[]>;
+}
+
+/**
+ * No-op implementation of {@link PayloadCodec}.
+ */
+export const defaultPayloadCodec = {
+  encode: async (payloads: Payload[]): Promise<Payload[]> => payloads,
+  decode: async (payloads: Payload[]): Promise<Payload[]> => payloads,
+};

package/src/converter/payload-converter.ts

@@ -1,142 +1,177 @@
-import { ValueError } from '../errors';
-import { u8, str, Payload, encodingTypes, encodingKeys, METADATA_ENCODING_KEY } from './types';
+import { PayloadConverterError, ValueError } from '@temporalio/internal-workflow-common';
+import {
+  BinaryPayloadConverter,
+  JsonPayloadConverter,
+  PayloadConverterWithEncoding,
+  UndefinedPayloadConverter,
+} from './payload-converters';
+import { METADATA_ENCODING_KEY, Payload, str } from './types';
 
 /**
- * Used by the framework to serialize/deserialize method parameters that need to be sent over the
- * wire.
+ * Used by the framework to serialize/deserialize parameters and return values.
  *
- * @author fateev
+ * This is called inside the [Workflow isolate](https://docs.temporal.io/docs/typescript/determinism).
+ * To write async code or use Node APIs (or use packages that use Node APIs), use a {@link PayloadCodec}.
  */
 export interface PayloadConverter {
-  encodingType: string;
-
   /**
-   * TODO: Fix comment in https://github.com/temporalio/sdk-java/blob/85593dbfa99bddcdf54c7196d2b73eeb23e94e9e/temporal-sdk/src/main/java/io/temporal/common/converter/DataConverter.java#L46
-   * Implements conversion of value to payload
-   *
-   * @param value JS value to convert.
-   * @return converted value
-   * @throws DataConverterException if conversion of the value passed as parameter failed for any
-   *     reason.
+   * Converts a value to a {@link Payload}.
+   * @param value The value to convert. Example values include the Workflow args sent by the client and the values returned by a Workflow or Activity.
    */
-  toData(value: unknown): Promise<Payload | undefined>;
+  toPayload<T>(value: T): Payload | undefined;
 
   /**
-   * Implements conversion of payload to value.
-   *
-   * @param content Serialized value to convert to a JS value.
-   * @return converted JS value
-   * @throws DataConverterException if conversion of the data passed as parameter failed for any
-   *     reason.
+   * Converts a {@link Payload} back to a value.
    */
-  fromData<T>(content: Payload): Promise<T>;
+  fromPayload<T>(payload: Payload): T;
+}
+
+export class CompositePayloadConverter implements PayloadConverter {
+  readonly converters: PayloadConverterWithEncoding[];
+  readonly converterByEncoding: Map<string, PayloadConverterWithEncoding> = new Map();
+
+  constructor(...converters: PayloadConverterWithEncoding[]) {
+    this.converters = converters;
+    for (const converter of converters) {
+      this.converterByEncoding.set(converter.encodingType, converter);
+    }
+  }
 
   /**
-   * Synchronous version of {@link toData}, used in the Workflow runtime because
-   * the async version limits the functionality of the runtime.
-   *
-   * Implements conversion of value to payload
+   * Tries to run `.toPayload(value)` on each converter in the order provided at construction.
+   * Returns the first successful result, or `undefined` if there is no converter that can handle the value.
    *
-   * @param value JS value to convert.
-   * @return converted value
-   * @throws DataConverterException if conversion of the value passed as parameter failed for any
-   *     reason.
+   * @throws UnsupportedJsonTypeError
    */
-  toDataSync(value: unknown): Payload | undefined;
+  public toPayload<T>(value: T): Payload | undefined {
+    for (const converter of this.converters) {
+      const result = converter.toPayload(value);
+      if (result !== undefined) {
+        return result;
+      }
+    }
+    return undefined;
+  }
 
   /**
-   * Synchronous version of {@link fromData}, used in the Workflow runtime because
-   * the async version limits the functionality of the runtime.
-   *
-   * Implements conversion of payload to value.
-   *
-   * @param content Serialized value to convert to a JS value.
-   * @return converted JS value
-   * @throws DataConverterException if conversion of the data passed as parameter failed for any
-   *     reason.
+   * Run {@link PayloadConverterWithEncoding.fromPayload} based on the {@link encodingTypes | encoding type} of the {@link Payload}.
    */
-  fromDataSync<T>(content: Payload): T;
-}
-
-export abstract class AsyncFacadePayloadConverter implements PayloadConverter {
-  abstract encodingType: string;
-  abstract toDataSync(value: unknown): Payload | undefined;
-  abstract fromDataSync<T>(content: Payload): T;
-
-  public async toData(value: unknown): Promise<Payload | undefined> {
-    return this.toDataSync(value);
+  public fromPayload<T>(payload: Payload): T {
+    if (payload.metadata === undefined || payload.metadata === null) {
+      throw new ValueError('Missing payload metadata');
+    }
+    const encoding = str(payload.metadata[METADATA_ENCODING_KEY]);
+    const converter = this.converterByEncoding.get(encoding);
+    if (converter === undefined) {
+      throw new ValueError(`Unknown encoding: ${encoding}`);
+    }
+    return converter.fromPayload(payload);
   }
+}
 
-  public async fromData<T>(content: Payload): Promise<T> {
-    return this.fromDataSync(content);
+/**
+ * Tries to convert `value` to a {@link Payload}. Throws if conversion fails.
+ *
+ * @throws {@link PayloadConverterError}
+ */
+export function toPayload(converter: PayloadConverter, value: unknown): Payload {
+  const payload = converter.toPayload(value);
+  if (payload === undefined) {
+    throw new PayloadConverterError(`Failed to convert value: ${value}`);
   }
+  return payload;
 }
 
 /**
- * Converts between JS undefined and NULL Payload
+ * Implements conversion of a list of values.
+ *
+ * @param converter
+ * @param values JS values to convert to Payloads
+ * @return converted values
+ * @throws PayloadConverterError if conversion of the value passed as parameter failed for any
+ *     reason.
  */
-export class UndefinedPayloadConverter extends AsyncFacadePayloadConverter {
-  public encodingType = encodingTypes.METADATA_ENCODING_NULL;
-
-  public toDataSync(value: unknown): Payload | undefined {
-    if (value !== undefined) return undefined; // Can't encode
-    return {
-      metadata: {
-        [METADATA_ENCODING_KEY]: encodingKeys.METADATA_ENCODING_NULL,
-      },
-    };
+export function toPayloads(converter: PayloadConverter, ...values: unknown[]): Payload[] | undefined {
+  if (values.length === 0) {
+    return undefined;
   }
 
-  public fromDataSync<T>(_content: Payload): T {
-    return undefined as any; // Just return undefined
-  }
+  return values.map((value) => toPayload(converter, value));
 }
 
 /**
- * Converts between non-undefined values and serialized JSON Payload
+ * Run {@link PayloadConverter.toPayload} on each value in the map.
+ *
+ * @throws {@link PayloadConverterError} if conversion of any value in the map fails
  */
-export class JsonPayloadConverter extends AsyncFacadePayloadConverter {
-  public encodingType = encodingTypes.METADATA_ENCODING_JSON;
-
-  public toDataSync(value: unknown): Payload | undefined {
-    if (value === undefined) return undefined; // Should be encoded with the UndefinedPayloadConverter
-    return {
-      metadata: {
-        [METADATA_ENCODING_KEY]: encodingKeys.METADATA_ENCODING_JSON,
-      },
-      data: u8(JSON.stringify(value)),
-    };
-  }
+export function mapToPayloads<K extends string>(converter: PayloadConverter, map: Record<K, any>): Record<K, Payload> {
+  return Object.fromEntries(
+    Object.entries(map).map(([k, v]): [K, Payload] => [k as K, toPayload(converter, v)])
+  ) as Record<K, Payload>;
+}
 
-  public fromDataSync<T>(content: Payload): T {
-    if (content.data === undefined || content.data === null) {
-      throw new ValueError('Got payload with no data');
-    }
-    return JSON.parse(str(content.data));
+/**
+ * Implements conversion of an array of values of different types. Useful for deserializing
+ * arguments of function invocations.
+ *
+ * @param converter
+ * @param index index of the value in the payloads
+ * @param payloads serialized value to convert to JS values.
+ * @return converted JS value
+ * @throws {@link PayloadConverterError} if conversion of the data passed as parameter failed for any
+ *     reason.
+ */
+export function fromPayloadsAtIndex<T>(converter: PayloadConverter, index: number, payloads?: Payload[] | null): T {
+  // To make adding arguments a backwards compatible change
+  if (payloads === undefined || payloads === null || index >= payloads.length) {
+    return undefined as any;
   }
+  return converter.fromPayload(payloads[index]);
 }
 
 /**
- * Converts between binary data types and RAW Payload
+ * Run {@link PayloadConverter.fromPayload} on each value in the array.
  */
-export class BinaryPayloadConverter extends AsyncFacadePayloadConverter {
-  public encodingType = encodingTypes.METADATA_ENCODING_RAW;
-
-  public toDataSync(value: unknown): Payload | undefined {
-    // TODO: support any DataView or ArrayBuffer?
-    if (!(value instanceof Uint8Array)) {
-      return undefined;
-    }
-    return {
-      metadata: {
-        [METADATA_ENCODING_KEY]: encodingKeys.METADATA_ENCODING_RAW,
-      },
-      data: value,
-    };
+export function arrayFromPayloads(converter: PayloadConverter, payloads?: Payload[] | null): unknown[] {
+  if (!payloads) {
+    return [];
   }
+  return payloads.map((payload: Payload) => converter.fromPayload(payload));
+}
+
+export function mapFromPayloads<K extends string>(
+  converter: PayloadConverter,
+  map?: Record<K, Payload> | null | undefined
+): Record<K, unknown> | undefined {
+  if (map === undefined || map === null) return undefined;
+  return Object.fromEntries(
+    Object.entries(map).map(([k, payload]): [K, unknown] => {
+      const value = converter.fromPayload(payload as Payload);
+      return [k as K, value];
+    })
+  ) as Record<K, unknown>;
+}
 
-  public fromDataSync<T>(content: Payload): T {
-    // TODO: support any DataView or ArrayBuffer?
-    return content.data as any;
+export const searchAttributePayloadConverter = new JsonPayloadConverter();
+
+export class DefaultPayloadConverter extends CompositePayloadConverter {
+  // Match the order used in other SDKs, but exclude Protobuf converters so that the code, including
+  // `proto3-json-serializer`, doesn't take space in Workflow bundles that don't use Protobufs. To use Protobufs, use
+  // {@link DefaultPayloadConverterWithProtobufs}.
+  //
+  // Go SDK:
+  // https://github.com/temporalio/sdk-go/blob/5e5645f0c550dcf717c095ae32c76a7087d2e985/converter/default_data_converter.go#L28
+  constructor() {
+    super(new UndefinedPayloadConverter(), new BinaryPayloadConverter(), new JsonPayloadConverter());
   }
 }
+
+/**
+ * The default {@link PayloadConverter} used by the SDK.
+ * Supports `Uint8Array` and JSON serializables (so if [`JSON.stringify(yourArgOrRetval)`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#description) works, the default payload converter will work).
+ *
+ * To also support Protobufs, create a custom payload converter with {@link DefaultPayloadConverter}:
+ *
+ * `const myConverter = new DefaultPayloadConverter({ protobufRoot })`
+ */
+export const defaultPayloadConverter = new DefaultPayloadConverter();