@or-sdk/contacts 3.2.2-beta.1771.0 → 3.2.2-beta.1775.0

package/src/api/contactApi.ts

@@ -1,5 +1,6 @@
 import {
   BatchProcessResponseDto,
+  BatchProcessStatus,
   ContactParamsDto,
   ContactRequestDto,
   ContactResponseDto,
@@ -18,17 +19,15 @@ import {
 import { CalApiParams, List } from '@or-sdk/base';
 import { BaseApi } from './baseApi';
 import BatchProcessApi from './batchProcessApi';
-import ContactBookApi from './contactBookApi';
-import { getObjectSizeInBytes, chunkArrByMaxSize } from '../utils';
+import { getObjectSizeInBytes, chunkArrByMaxSize, debouncePromise } from '../utils';
+import { InitCreateBatchResponse, CreateContactsBatchResults } from '../types';
 import { REQUEST_PAYLOAD_MAX_BYTES, FAILED_REQUEST_REPEATS, CONTACTS_DELETE_MAX_AMOUNT } from '../constants';
-import { ApiError } from '../apiError';
-import { DeleteContactsData } from '../types';
+import { CreateContactsBatchError } from '../apiError';
 
 export default class ContactApi extends BaseApi {
   constructor(
     protected readonly apiCall: <T>(params: CalApiParams) => Promise<T>,
-    private batchProcessApi: BatchProcessApi,
-    private bookServiceApi: ContactBookApi
+    private batchProcessApi: BatchProcessApi
   ) {
     super(apiCall);
   }
@@ -91,8 +90,8 @@ export default class ContactApi extends BaseApi {
    * @description Delete Contact
    * @param data
    */
-  deleteMulti(data: DeleteContactsData): Promise<void> {
-    if ((Array.isArray(data.ids) && data.ids.length >= CONTACTS_DELETE_MAX_AMOUNT) || data.all) {
+  deleteMulti(data: DeleteContactMultiParamsDto): Promise<void> {
+    if ((Array.isArray(data.ids) && data.ids.length > CONTACTS_DELETE_MAX_AMOUNT) || data.all) {
       return this.bulkDeleteContacts(data);
     }
     return this.apiCall({
@@ -102,45 +101,6 @@ export default class ContactApi extends BaseApi {
     });
   }
 
-  /**
-   * @description Delete all contacts from a book
-   */
-  async deleteContactsByBook(bookId: string): Promise<void> {
-    const contacts = await this.listContact({ contact_book: bookId });
-    return this.bulkDeleteContacts({
-      ids: contacts.items.map(({ id }) => id),
-      contact_book: bookId,
-    });
-  }
-
-  /**
-   * Initiates bulk of deleting all contacts from a certain book.
-   * The only first batch is going to be executed and the only - its
-   * process id is returned in the response.
-   * Tracking of the batch status is possible with trackBatchProcess
-   * from BatchProcessApi
-   *
-   * @see batchProcessApi.trackBatchProcess
-   */
-  async initDeleteBookContactsBulk(bookId: string) {
-    const { contactsCount } = await this.bookServiceApi.getContactBook(bookId);
-    const contacts = await this.listContact({
-      contact_book: bookId,
-      size: CONTACTS_DELETE_MAX_AMOUNT,
-    });
-
-    const batchProcess = await this.runSingleDeleteContactsBulk({
-      ids: contacts.items.map(({ id }) => id),
-      contact_book: bookId,
-    });
-
-    return {
-      batchId: batchProcess.id,
-      totalBatches: Math.floor(contactsCount / 50),
-      firstBatchSize: CONTACTS_DELETE_MAX_AMOUNT,
-    };
-  }
-
   /**
    * @description Create Contact
    * @param data
@@ -155,18 +115,61 @@ export default class ContactApi extends BaseApi {
 
   /**
    * @description Create Contacts either in single or in multi batch(es), depending on payload size
-   * @param data
    */
-  async bulkCreateContacts(data: CreateMultipleContactsDto): Promise<ContactResponseDto[]> {
+  async bulkCreateContacts(data: CreateMultipleContactsDto): Promise<string[]> {
+    // eslint-disable-next-line no-console
+    console.log('contacts: ', data.contacts.length);
+    const { contacts, ...rest } = data;
+    const contactsMaxSize = REQUEST_PAYLOAD_MAX_BYTES - getObjectSizeInBytes({ ...rest });
+    const contactsChunks = chunkArrByMaxSize(contacts, contactsMaxSize);
+
+    contactsChunks.forEach((chunk, idx) => {
+      // eslint-disable-next-line no-console
+      console.log('size of ', idx, ' chunk', ' is ', getObjectSizeInBytes(chunk));
+    });
+    // eslint-disable-next-line no-console
+    console.log('chunks: ', contactsChunks.length);
+    const results = contactsChunks.length === 1
+      ? [await this.createContactsInSingleBatch(data)]
+      : await this.createContactsInMultiBatches(contactsChunks, rest);
+
+    return results.reduce<string[]>((acc, { contactsIds }) => ([...acc, ...contactsIds]), []);
+  }
+
+  /**
+   * Breaks the contacts to chunks of maximum allowed size (200 KB) and launches create batch
+   * only for the first chunk.
+   * Returns batch id in response by which the batch can be tracked using batch process API.
+   * Note: since create contact batch is a very expensive operations from terms of consuming
+   * DB connections, it is highly recommended to check amount of pending (running) batches
+   * prior to this method execution, and if the amount is greater of 2 the batch initiation
+   * should be deferred.
+   *
+   * @see batchProcessApi.trackBatchProcess
+   * @see batchProcessApi.getPendingBatchProcesses
+   */
+  async initCreateBatch(data: CreateMultipleContactsDto): Promise<InitCreateBatchResponse> {
     const { contacts, ...rest } = data;
     const contactsMaxSize = REQUEST_PAYLOAD_MAX_BYTES - getObjectSizeInBytes({ ...rest });
     const contactsChunks = chunkArrByMaxSize(contacts, contactsMaxSize);
 
-    return contactsChunks.length === 1
-      ? this.createContactsInSingleBatch(data)
-      : this.createContactsInMultiBatches(contactsChunks, rest);
+    const batchProcess = await this.apiCall<BatchProcessResponseDto>({
+      method: 'POST',
+      route: `${this.apiBasePath}/bulk`,
+      data: {
+        contacts: contactsChunks[0],
+        ...rest,
+      },
+    });
+
+    return {
+      totalChunks: contactsChunks.length,
+      firstChunkSize: contactsChunks[0].length,
+      batchId: batchProcess.id,
+    };
   }
 
+
   /**
    * @description Merge two Contacts into one
    * @param id Contact id TO which the data will be merged
@@ -230,102 +233,99 @@ export default class ContactApi extends BaseApi {
     });
   }
 
-  private async getSafelyContactsList(
-    contactIds: string[],
-    bookId: string | undefined,
-    repeats?: number
-  ): Promise<List<ContactResponseDto>> {
-    const contacts = await this.listContact({
-      contactIds,
-      ...(bookId && { contact_book: bookId }),
-    }).catch((e) => {
-      repeats = repeats || 0;
-      if (repeats < FAILED_REQUEST_REPEATS) {
-        return this.getSafelyContactsList(contactIds, bookId, repeats + 1);
-      }
-      throw new Error(e);
-    });
-    return contacts;
-  }
-
-  private async bulkDeleteContacts(data: DeleteContactsData): Promise<void> {
-    const { ids, contact_book, all } = data;
-    if (all) {
-      if (!contact_book) {
-        throw new ApiError(400, 'contact_book should be provided in case if "all" is true');
-      }
-      return await this.deleteContactsByBook(contact_book);
-    }
-    if (!ids) {
-      throw new ApiError(400, 'Provide either "contact_book" or "all"');
-    }
-    const batchSize = CONTACTS_DELETE_MAX_AMOUNT;
-    for (let i = 0;i < ids.length;i += batchSize) {
-      const chunk = ids.slice(i, i + batchSize);
-      await this.runSingleDeleteContactsBulk({
-        ids: chunk,
-        contact_book,
-      });
-    }
-  }
-
-  private async runSingleDeleteContactsBulk(
-    data: DeleteContactMultiParamsDto,
-    withPoling = true
-  ): Promise<BatchProcessResponseDto> {
+  private async bulkDeleteContacts(data: DeleteContactMultiParamsDto): Promise<void> {
     const batchProcess = await this.apiCall<BatchProcessResponseDto>({
       method: 'DELETE',
       route: `${this.apiBasePath}/bulk`,
       data,
     });
-    if (withPoling) {
-      const result = await this.polling(batchProcess.id);
-      return result;
+    await this.polling(batchProcess.id);
+  }
+
+  private async createContactsInMultiBatches(
+    contactsChunks: ContactRequestDto[][],
+    data: Omit<CreateMultipleContactsDto, 'contacts'>
+  ): Promise<CreateContactsBatchResults[]> {
+    const batchPromises = contactsChunks
+      .map((chunkContacts) => async () => this.createContactsInSingleBatch({
+        contacts: chunkContacts,
+        ...data,
+      }));
+
+    const promisesBatchSize = 1;
+    const results: CreateContactsBatchResults[] = [];
+
+    for (let i = 0;i < batchPromises.length;i += promisesBatchSize) {
+      const s = performance.now();
+      try {
+        const chunk = batchPromises.slice(i, i + promisesBatchSize);
+        const chunkResults = await Promise.all(chunk.map((p) => p()));
+        results.push(...chunkResults);
+      } catch (e) {
+        if (e instanceof CreateContactsBatchError) {
+          e.processedBatchIds = results.map(({ batchId }) => batchId);
+        }
+        throw e;
+      }
+
+      const e = performance.now();
+      // eslint-disable-next-line no-console
+      console.log('batch id = ', i, '; awaiting ', promisesBatchSize, ' batches: ', (e - s) / 1000, 's');
     }
-    return batchProcess;
+
+    return results;
   }
 
-  private async createContactsInSingleBatch(data: CreateMultipleContactsDto): Promise<ContactResponseDto[]> {
+  private async createContactsInSingleBatch(data: CreateMultipleContactsDto): Promise<CreateContactsBatchResults> {
+    const s = performance.now();
     const batchProcess = await this.apiCall<BatchProcessResponseDto>({
       method: 'POST',
       route: `${this.apiBasePath}/bulk`,
       data,
     });
     const batchProcessResult = await this.polling(batchProcess.id);
-    const contactIds = batchProcessResult.results.flatMap<string>(i => JSON.parse(i));
-    const importedContacts = await this.getSafelyContactsList(contactIds, data.contact_book);
-    return importedContacts.items;
-  }
-
-  private async createContactsInMultiBatches(
-    contactsChunks: ContactRequestDto[][],
-    data: Omit<CreateMultipleContactsDto, 'contacts'>
-  ): Promise<ContactResponseDto[]> {
-    const batchPromises = contactsChunks.map((chunkContacts) => async () => this.createContactsInSingleBatch({
-      contacts: chunkContacts,
-      ...data,
-    }));
-
-    const results: ContactResponseDto[] = [];
-    for (const fn of batchPromises) {
-      const singleBatchResults = await fn();
-      results.push(...singleBatchResults);
-    }
-    return results;
+    const e = performance.now();
+    // eslint-disable-next-line no-console
+    console.log('single batch time: ', (e - s) / 1000, 's');
+    return {
+      batchId: batchProcess.id,
+      contactsIds: batchProcessResult.results.flatMap<string>(i => JSON.parse(i)),
+    };
   }
 
   /**
    * Pols specific batch process until status of it isn't turned from 'pending to something else.
    */
-  private async polling(batchId: string, repeats?: number): Promise<BatchProcessResponseDto> {
-    const batchProcessState = await this.batchProcessApi.getBatchProcess(batchId).catch((e) => {
-      repeats = repeats || 0;
+  private async polling(batchId: string, repeats = 0): Promise<BatchProcessResponseDto> {
+
+    let batchProcess: BatchProcessResponseDto;
+    try {
+      batchProcess = await debouncePromise(() => this.batchProcessApi.getBatchProcess(batchId), 1000);
+      // eslint-disable-next-line no-console
+      const counter = batchCounters[batchId] || 0;
+      // eslint-disable-next-line no-console
+      console.log('polling counter of ', batchId, ' = ', counter, ' | status = ', batchProcess.status);
+      batchCounters[batchId] = counter + 1;
+
+    } catch (e) {
       if (repeats < FAILED_REQUEST_REPEATS) {
         return this.polling(batchId, repeats + 1);
       }
-      throw new Error(e);
-    });
-    // value 'pending' is used because of https://onereach.atlassian.net/browse/CU-562
-    return batchProcessState.status === 'pending' ? this.polling(batchId) : batchProcessState;
+      throw new CreateContactsBatchError((e as unknown as Error).message, batchId);
+    }
+
+    if (batchProcess.status === BatchProcessStatus.failed) {
+      throw new CreateContactsBatchError(
+        'Could not complete batch process',
+        batchId,
+        batchProcess.messages
+      );
+    }
+
+    return batchProcess.status === BatchProcessStatus.pending
+      ? this.polling(batchId)
+      : batchProcess;
   }
 }
+
+const batchCounters: Record<string, number> = {};

package/src/apiError.ts

@@ -11,3 +11,14 @@ export class ApiError extends Error {
     super(message);
   }
 }
+
+export class CreateContactsBatchError extends Error {
+  constructor(
+    message: string,
+    public readonly failedBatchId: string,
+    public readonly batchMessages?: string[],
+    public processedBatchIds?: string[],
+  ) {
+    super(message);
+  }
+}

package/src/constants.ts

@@ -1,7 +1,13 @@
 export const CONTACTS_SERVICE_KEY = 'contacts-api';
 
-export const REQUEST_PAYLOAD_MAX_BYTES = 128000;
+export const REQUEST_PAYLOAD_MAX_BYTES = 150000;
 
 export const FAILED_REQUEST_REPEATS = 3;
 
-export const CONTACTS_DELETE_MAX_AMOUNT = 50;
+/**
+ * 5 selected experimentally
+ * Since contacts deleting assumes cascading on related field values and deleting
+ * relations in contact_book_contact table, it results in huge amount of queries
+ * which sent simultaneously might kill connection to the DB.
+ */
+export const CONTACTS_DELETE_MAX_AMOUNT = 5;

package/src/types.ts

@@ -1,17 +1,8 @@
 import { Token } from '@or-sdk/base';
-import {
-  ContactBookParamsDto,
-  ListApiParams,
-  OrderParams,
-  DeleteContactMultiParamsDto,
-  BatchProcessStatus,
-} from '@onereach/types-contacts-api';
+import { BatchProcessStatus, ContactBookParamsDto, ListApiParams, OrderParams } from '@onereach/types-contacts-api';
 import { OrderOptions, PaginationOptions } from '@or-sdk/base';
 export * from '@onereach/types-contacts-api';
 
-type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>;
-type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
-
 //re-export types
 export { OrderDirection, PaginationOptions, OrderOptions } from '@or-sdk/base';
 
@@ -40,7 +31,16 @@ export interface ContactBookParams extends AdaptedListParams<ContactBookParamsDt
 export type AdaptedListParams<T extends ListApiParams & OrderParams> =
   Omit<T, 'order' | 'skip' | 'take'> & Partial<PaginationOptions & OrderOptions>;
 
-export type DeleteContactsData = PartialBy<DeleteContactMultiParamsDto, 'ids'> & { all?: boolean;};
+export type CreateContactsBatchResults ={
+  batchId: string;
+  contactsIds: string[];
+};
+
+export type InitCreateBatchResponse = {
+  totalChunks: number;
+  firstChunkSize: number;
+  batchId: string;
+};
 
 export type TrackBatchProcessResponse = {
   status: BatchProcessStatus;

package/src/utils.ts

@@ -43,3 +43,19 @@ export const chunkArrByMaxSize = <T>(arr: T[], maxSize: number): T[][] => {
     ...chunkArrByMaxSize(arr.slice(mid), maxSize),
   ];
 };
+
+export function debouncePromise<T>(
+  caller: () => Promise<T>,
+  delay: number,
+  reject?: (value: string | T | PromiseLike<T>) => void
+): Promise<T> {
+  let timeout: NodeJS.Timeout | null = null;
+
+  return new Promise<T>((res) => {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+    timeout = setTimeout(() => caller().then((x) => res(x)).catch((err) => reject && reject(err)), delay);
+  });
+}
+