@twentyhq/people-data-labs 1.0.0

package/src/logic-functions/utils/build-person-matched-data.ts

@@ -0,0 +1,78 @@
+import { type CoreApiClient } from 'twenty-client-sdk/core';
+
+import { findOrCreateCurrentCompany } from 'src/logic-functions/utils/find-or-create-current-company';
+import { isEmptyEmails } from 'src/logic-functions/utils/is-empty-emails';
+import { isEmptyFullName } from 'src/logic-functions/utils/is-empty-full-name';
+import { isEmptyLinks } from 'src/logic-functions/utils/is-empty-links';
+import { isEmptyPhones } from 'src/logic-functions/utils/is-empty-phones';
+import { isEmptyText } from 'src/logic-functions/utils/is-empty-text';
+import { mapPerson } from 'src/logic-functions/utils/map-person';
+import { pickWritableStandard } from 'src/logic-functions/utils/pick-writable-standard';
+import { type CompanyIdByMatchKeyCache } from 'src/types/company-id-by-match-key-cache';
+import { type PdlPersonData } from 'src/types/pdl-person-data';
+import { type PersonNode } from 'src/types/person-node';
+import { isDefined } from 'src/utils/is-defined';
+import { pruneUndefined } from 'src/utils/prune-undefined';
+
+const PERSON_EMPTY_CHECKS = {
+  name: isEmptyFullName,
+  emails: isEmptyEmails,
+  phones: isEmptyPhones,
+  jobTitle: isEmptyText,
+  linkedinLink: isEmptyLinks,
+};
+
+export const buildPersonMatchedData = async ({
+  client,
+  node,
+  outcome,
+  enrichedAt,
+  companyIdByMatchKeyCache,
+  overrideExistingValues,
+  shouldPersist,
+}: {
+  client: CoreApiClient;
+  node: PersonNode;
+  outcome: { likelihood?: number; data: PdlPersonData };
+  enrichedAt: string;
+  companyIdByMatchKeyCache: CompanyIdByMatchKeyCache;
+  overrideExistingValues: boolean;
+  shouldPersist: boolean;
+}): Promise<{
+  mappedData: Record<string, unknown>;
+  persistData: Record<string, unknown>;
+}> => {
+  const mapped = mapPerson(outcome.data);
+  const mappedData = pruneUndefined({ ...mapped.standard, ...mapped.pdl });
+
+  if (!shouldPersist) {
+    return { mappedData, persistData: {} };
+  }
+
+  const writableStandard = pickWritableStandard({
+    standard: mapped.standard,
+    current: node as unknown as Record<string, unknown>,
+    emptyChecks: PERSON_EMPTY_CHECKS,
+    overrideExistingValues,
+  });
+
+  const currentCompanyId = isDefined(node.company?.id)
+    ? undefined
+    : await findOrCreateCurrentCompany({
+        client,
+        personData: outcome.data,
+        companyIdByMatchKeyCache,
+      });
+
+  const persistData = pruneUndefined({
+    ...writableStandard,
+    ...mapped.pdl,
+    companyId: currentCompanyId,
+    pdlLikelihood: outcome.likelihood,
+    pdlRawPayload: outcome.data,
+    pdlLastEnrichedAt: enrichedAt,
+    pdlEnrichmentStatus: 'MATCHED',
+  });
+
+  return { mappedData, persistData };
+};

package/src/logic-functions/utils/build-person-name-param.ts

@@ -0,0 +1,19 @@
+import { toText } from 'src/logic-functions/utils/to-text';
+import { isDefined } from 'src/utils/is-defined';
+
+export const buildPersonNameParam = ({
+  firstName,
+  lastName,
+}: {
+  firstName: unknown;
+  lastName: unknown;
+}): string | undefined => {
+  const first = toText(firstName);
+  const last = toText(lastName);
+
+  if (!isDefined(first) || !isDefined(last)) {
+    return undefined;
+  }
+
+  return `${first} ${last}`;
+};

package/src/logic-functions/utils/build-phones.ts

@@ -0,0 +1,40 @@
+import { isNonEmptyArray } from '@sniptt/guards';
+
+import { toText } from 'src/logic-functions/utils/to-text';
+import { type PhonesValue } from 'src/types/phones-value';
+import { isDefined } from 'src/utils/is-defined';
+
+export const buildPhones = (
+  phoneCandidates: (string | null | undefined)[],
+): PhonesValue | undefined => {
+  const uniquePhoneNumbers: string[] = [];
+  const seenPhoneNumbers = new Set<string>();
+
+  for (const candidate of phoneCandidates) {
+    const phoneNumber = toText(candidate);
+    if (!isDefined(phoneNumber) || seenPhoneNumbers.has(phoneNumber)) {
+      continue;
+    }
+
+    seenPhoneNumbers.add(phoneNumber);
+    uniquePhoneNumbers.push(phoneNumber);
+  }
+
+  const [primaryPhoneNumber, ...additionalPhoneNumbers] = uniquePhoneNumbers;
+  if (!isDefined(primaryPhoneNumber)) {
+    return undefined;
+  }
+
+  return {
+    primaryPhoneNumber,
+    primaryPhoneCountryCode: '',
+    primaryPhoneCallingCode: '',
+    additionalPhones: isNonEmptyArray(additionalPhoneNumbers)
+      ? additionalPhoneNumbers.map((phoneNumber) => ({
+          number: phoneNumber,
+          countryCode: '',
+          callingCode: '',
+        }))
+      : null,
+  };
+};

package/src/logic-functions/utils/build-skipped-result.ts

@@ -0,0 +1,15 @@
+import { type EnrichResult } from 'src/types/enrich-result';
+
+export const buildSkippedResult = ({
+  recordId,
+  message,
+}: {
+  recordId: string;
+  message: string;
+}): EnrichResult => ({
+  success: true,
+  recordId,
+  status: 'SKIPPED',
+  updatedFields: [],
+  message,
+});

package/src/logic-functions/utils/capitalize-name.ts

@@ -0,0 +1,8 @@
+const NAME_BOUNDARY_REGEX = /(^|[\s'-])(\p{L})/gu;
+
+export const capitalizeName = (value: string): string =>
+  value.replace(
+    NAME_BOUNDARY_REGEX,
+    (_match, boundary: string, letter: string) =>
+      `${boundary}${letter.toUpperCase()}`,
+  );

package/src/logic-functions/utils/charge-matched-enrichments.ts

@@ -0,0 +1,29 @@
+import { chargeCredits } from 'twenty-sdk/billing';
+
+import { BILLING_MARGIN_MULTIPLIER } from 'src/constants/billing-margin-multiplier';
+import { MICRO_CREDITS_PER_DOLLAR } from 'src/constants/micro-credits-per-dollar';
+
+export const chargeMatchedEnrichments = async ({
+  matchedCount,
+  costPerMatchDollars,
+  resourceContext,
+}: {
+  matchedCount: number;
+  costPerMatchDollars: number;
+  resourceContext: string;
+}): Promise<void> => {
+  if (matchedCount === 0) {
+    return;
+  }
+
+  const creditsPerMatchMicro = Math.round(
+    costPerMatchDollars * BILLING_MARGIN_MULTIPLIER * MICRO_CREDITS_PER_DOLLAR,
+  );
+
+  await chargeCredits({
+    creditsUsedMicro: matchedCount * creditsPerMatchMicro,
+    operationType: 'CODE_EXECUTION',
+    quantity: matchedCount,
+    resourceContext,
+  });
+};

package/src/logic-functions/utils/chunk.ts

@@ -0,0 +1,18 @@
+export const chunk = <TItem>({
+  items,
+  size,
+}: {
+  items: TItem[];
+  size: number;
+}): TItem[][] => {
+  if (size <= 0) {
+    return items.length > 0 ? [items] : [];
+  }
+
+  const chunks: TItem[][] = [];
+  for (let index = 0; index < items.length; index += size) {
+    chunks.push(items.slice(index, index + size));
+  }
+
+  return chunks;
+};

package/src/logic-functions/utils/enrich-chunk.ts

@@ -0,0 +1,361 @@
+import { type CoreApiClient } from 'twenty-client-sdk/core';
+
+import { buildErrorResult } from 'src/logic-functions/utils/build-error-result';
+import { buildMatchedResult } from 'src/logic-functions/utils/build-matched-result';
+import { buildNotFoundResult } from 'src/logic-functions/utils/build-not-found-result';
+import { buildSkippedResult } from 'src/logic-functions/utils/build-skipped-result';
+import { chargeMatchedEnrichments } from 'src/logic-functions/utils/charge-matched-enrichments';
+import { INTERNAL_BOOKKEEPING_FIELDS } from 'src/logic-functions/utils/internal-field-names';
+import { nowIso } from 'src/logic-functions/utils/now-iso';
+import { resolveUpdateFieldsMode } from 'src/logic-functions/utils/resolve-update-fields-mode';
+import { type BatchEnrichmentAdapter } from 'src/types/batch-enrichment-adapter';
+import { type BulkEnrichInput } from 'src/types/bulk-enrich-input';
+import { type CompanyIdByMatchKeyCache } from 'src/types/company-id-by-match-key-cache';
+import { type EnrichResult } from 'src/types/enrich-result';
+import { type PdlEnrichResult } from 'src/types/pdl-enrich-result';
+import { isDefined } from 'src/utils/is-defined';
+import { toErrorMessage } from 'src/utils/to-error-message';
+
+const writeErrorStatusWithBackoff = async <TNode, TData, TParams>({
+  adapter,
+  client,
+  recordIds,
+  enrichedAt,
+}: {
+  adapter: BatchEnrichmentAdapter<TNode, TData, TParams>;
+  client: CoreApiClient;
+  recordIds: string[];
+  enrichedAt: string;
+}): Promise<void> => {
+  if (recordIds.length === 0) {
+    return;
+  }
+
+  await adapter
+    .updateManyStatus({
+      client,
+      recordIds,
+      data: { pdlEnrichmentStatus: 'ERROR', pdlLastEnrichedAt: enrichedAt },
+    })
+    .catch(() => undefined);
+};
+
+type MatchedRecord = {
+  recordId: string;
+  mappedData: Record<string, unknown>;
+  persistData: Record<string, unknown>;
+};
+
+const recordMatchedRecords = async <TNode, TData, TParams>({
+  adapter,
+  client,
+  matchedRecords,
+  shouldPersist,
+  resultById,
+}: {
+  adapter: BatchEnrichmentAdapter<TNode, TData, TParams>;
+  client: CoreApiClient;
+  matchedRecords: MatchedRecord[];
+  shouldPersist: boolean;
+  resultById: Map<string, EnrichResult>;
+}): Promise<string[]> => {
+  if (matchedRecords.length === 0) {
+    return [];
+  }
+
+  if (!shouldPersist) {
+    for (const matchedRecord of matchedRecords) {
+      resultById.set(
+        matchedRecord.recordId,
+        buildMatchedResult({
+          recordId: matchedRecord.recordId,
+          updatedFields: [],
+          data: matchedRecord.mappedData,
+        }),
+      );
+    }
+
+    return [];
+  }
+
+  const settledWriteResults = await Promise.allSettled(
+    matchedRecords.map((matchedRecord) =>
+      adapter.updateOne({
+        client,
+        recordId: matchedRecord.recordId,
+        data: matchedRecord.persistData,
+      }),
+    ),
+  );
+
+  const failedRecordIds: string[] = [];
+  for (const [index, writeResult] of settledWriteResults.entries()) {
+    const matchedRecord = matchedRecords[index];
+    if (writeResult.status === 'rejected') {
+      resultById.set(
+        matchedRecord.recordId,
+        buildErrorResult({
+          recordId: matchedRecord.recordId,
+          error: toErrorMessage(writeResult.reason),
+        }),
+      );
+      failedRecordIds.push(matchedRecord.recordId);
+      continue;
+    }
+
+    resultById.set(
+      matchedRecord.recordId,
+      buildMatchedResult({
+        recordId: matchedRecord.recordId,
+        updatedFields: Object.keys(matchedRecord.persistData).filter(
+          (fieldName) => !INTERNAL_BOOKKEEPING_FIELDS.has(fieldName),
+        ),
+        data: matchedRecord.mappedData,
+      }),
+    );
+  }
+
+  return failedRecordIds;
+};
+
+const recordNotFoundRecords = async <TNode, TData, TParams>({
+  adapter,
+  client,
+  notFoundRecordIds,
+  shouldPersist,
+  enrichedAt,
+  resultById,
+}: {
+  adapter: BatchEnrichmentAdapter<TNode, TData, TParams>;
+  client: CoreApiClient;
+  notFoundRecordIds: string[];
+  shouldPersist: boolean;
+  enrichedAt: string;
+  resultById: Map<string, EnrichResult>;
+}): Promise<string[]> => {
+  if (notFoundRecordIds.length === 0) {
+    return [];
+  }
+
+  if (!shouldPersist) {
+    for (const recordId of notFoundRecordIds) {
+      resultById.set(recordId, buildNotFoundResult(recordId));
+    }
+
+    return [];
+  }
+
+  try {
+    await adapter.updateManyStatus({
+      client,
+      recordIds: notFoundRecordIds,
+      data: { pdlEnrichmentStatus: 'NOT_FOUND', pdlLastEnrichedAt: enrichedAt },
+    });
+    for (const recordId of notFoundRecordIds) {
+      resultById.set(recordId, buildNotFoundResult(recordId));
+    }
+
+    return [];
+  } catch (notFoundStatusWriteError) {
+    const notFoundStatusWriteErrorMessage = toErrorMessage(
+      notFoundStatusWriteError,
+    );
+    for (const recordId of notFoundRecordIds) {
+      resultById.set(
+        recordId,
+        buildErrorResult({ recordId, error: notFoundStatusWriteErrorMessage }),
+      );
+    }
+
+    return notFoundRecordIds;
+  }
+};
+
+export const enrichChunk = async <TNode, TData, TParams>({
+  client,
+  recordIds,
+  input,
+  adapter,
+  resultById,
+  companyIdByMatchKeyCache,
+}: {
+  client: CoreApiClient;
+  recordIds: string[];
+  input: BulkEnrichInput;
+  adapter: BatchEnrichmentAdapter<TNode, TData, TParams>;
+  resultById: Map<string, EnrichResult>;
+  companyIdByMatchKeyCache: CompanyIdByMatchKeyCache;
+}): Promise<void> => {
+  const { shouldPersist, overrideExistingValues } = resolveUpdateFieldsMode(
+    input.updateFields,
+  );
+
+  let recordNodes: TNode[];
+  try {
+    recordNodes = await adapter.readRecords({ client, recordIds });
+  } catch (readError) {
+    const readErrorMessage = toErrorMessage(readError);
+    for (const recordId of recordIds) {
+      resultById.set(
+        recordId,
+        buildErrorResult({ recordId, error: readErrorMessage }),
+      );
+    }
+
+    return;
+  }
+
+  const nodeByRecordId = new Map(
+    recordNodes.map((recordNode) => [adapter.getNodeId(recordNode), recordNode]),
+  );
+
+  const recordsToEnrich: { recordId: string; node: TNode; params: TParams }[] =
+    [];
+  for (const recordId of recordIds) {
+    const recordNode = nodeByRecordId.get(recordId);
+    if (!isDefined(recordNode)) {
+      resultById.set(
+        recordId,
+        buildErrorResult({
+          recordId,
+          error: `${adapter.objectNameSingular} ${recordId} not found`,
+        }),
+      );
+      continue;
+    }
+
+    const matchParams = adapter.extractParams({ node: recordNode, input });
+    if (!isDefined(matchParams)) {
+      resultById.set(
+        recordId,
+        buildSkippedResult({
+          recordId,
+          message: adapter.noIdentifierMessage,
+        }),
+      );
+      continue;
+    }
+
+    recordsToEnrich.push({ recordId, node: recordNode, params: matchParams });
+  }
+
+  if (recordsToEnrich.length === 0) {
+    return;
+  }
+
+  const enrichedAt = nowIso();
+  const recordIdsToMarkAsError: string[] = [];
+
+  let pdlEnrichmentOutcomes: PdlEnrichResult<TData>[];
+  try {
+    pdlEnrichmentOutcomes = await adapter.enrichBatch(
+      recordsToEnrich.map((recordToEnrich) => recordToEnrich.params),
+    );
+  } catch (enrichBatchError) {
+    const enrichBatchErrorMessage = toErrorMessage(enrichBatchError);
+    for (const recordToEnrich of recordsToEnrich) {
+      resultById.set(
+        recordToEnrich.recordId,
+        buildErrorResult({
+          recordId: recordToEnrich.recordId,
+          error: enrichBatchErrorMessage,
+        }),
+      );
+    }
+    if (shouldPersist) {
+      await writeErrorStatusWithBackoff({
+        adapter,
+        client,
+        recordIds: recordsToEnrich.map(
+          (recordToEnrich) => recordToEnrich.recordId,
+        ),
+        enrichedAt,
+      });
+    }
+
+    return;
+  }
+
+  await chargeMatchedEnrichments({
+    matchedCount: pdlEnrichmentOutcomes.filter(
+      (enrichmentOutcome) => enrichmentOutcome?.outcome === 'matched',
+    ).length,
+    costPerMatchDollars: adapter.costPerMatchDollars,
+    resourceContext: `pdl/${adapter.objectNameSingular.toLowerCase()}`,
+  });
+
+  const notFoundRecordIds: string[] = [];
+  const matchedRecords: MatchedRecord[] = [];
+
+  for (let index = 0; index < recordsToEnrich.length; index++) {
+    const { recordId, node: recordNode } = recordsToEnrich[index];
+    const enrichmentOutcome = pdlEnrichmentOutcomes[index];
+
+    if (!isDefined(enrichmentOutcome) || enrichmentOutcome.outcome === 'error') {
+      const enrichmentErrorMessage = isDefined(enrichmentOutcome)
+        ? enrichmentOutcome.message
+        : 'People Data Labs returned no response for this record.';
+      resultById.set(
+        recordId,
+        buildErrorResult({ recordId, error: enrichmentErrorMessage }),
+      );
+      recordIdsToMarkAsError.push(recordId);
+      continue;
+    }
+
+    if (enrichmentOutcome.outcome === 'not_found') {
+      notFoundRecordIds.push(recordId);
+      continue;
+    }
+
+    try {
+      const { mappedData, persistData } = await adapter.buildMatchedData({
+        client,
+        node: recordNode,
+        outcome: enrichmentOutcome,
+        enrichedAt,
+        companyIdByMatchKeyCache,
+        overrideExistingValues,
+        shouldPersist,
+      });
+      matchedRecords.push({ recordId, mappedData, persistData });
+    } catch (buildMatchedDataError) {
+      resultById.set(
+        recordId,
+        buildErrorResult({
+          recordId,
+          error: toErrorMessage(buildMatchedDataError),
+        }),
+      );
+      recordIdsToMarkAsError.push(recordId);
+    }
+  }
+
+  const failedMatchedWriteRecordIds = await recordMatchedRecords({
+    adapter,
+    client,
+    matchedRecords,
+    shouldPersist,
+    resultById,
+  });
+  recordIdsToMarkAsError.push(...failedMatchedWriteRecordIds);
+
+  const failedNotFoundWriteRecordIds = await recordNotFoundRecords({
+    adapter,
+    client,
+    notFoundRecordIds,
+    shouldPersist,
+    enrichedAt,
+    resultById,
+  });
+  recordIdsToMarkAsError.push(...failedNotFoundWriteRecordIds);
+
+  if (shouldPersist) {
+    await writeErrorStatusWithBackoff({
+      adapter,
+      client,
+      recordIds: recordIdsToMarkAsError,
+      enrichedAt,
+    });
+  }
+};

package/src/logic-functions/utils/enrich-companies.ts

@@ -0,0 +1,21 @@
+import { postPdlBulkEnrich } from 'src/logic-functions/utils/post-pdl-enrich';
+import { type PdlCompanyData } from 'src/types/pdl-company-data';
+import { type PdlCompanyEnrichParams } from 'src/types/pdl-company-enrich-params';
+import { type PdlEnrichResult } from 'src/types/pdl-enrich-result';
+import { pruneUndefined } from 'src/utils/prune-undefined';
+
+export const enrichCompanies = (
+  params: PdlCompanyEnrichParams[],
+): Promise<PdlEnrichResult<PdlCompanyData>[]> =>
+  postPdlBulkEnrich<PdlCompanyData>({
+    path: '/company/enrich/bulk',
+    requests: params.map((entry) =>
+      pruneUndefined({
+        pdl_id: entry.pdlId,
+        website: entry.website,
+        profile: entry.profile,
+        name: entry.name,
+        min_likelihood: entry.minLikelihood,
+      }),
+    ),
+  });

package/src/logic-functions/utils/enrich-company.ts

@@ -0,0 +1,23 @@
+import { postPdlSingleEnrich } from 'src/logic-functions/utils/post-pdl-single-enrich';
+import { type PdlCompanyData } from 'src/types/pdl-company-data';
+import { type PdlCompanyEnrichParams } from 'src/types/pdl-company-enrich-params';
+import { type PdlEnrichResult } from 'src/types/pdl-enrich-result';
+import { pruneUndefined } from 'src/utils/prune-undefined';
+
+export const enrichCompany = (
+  params: PdlCompanyEnrichParams[],
+): Promise<PdlEnrichResult<PdlCompanyData>[]> =>
+  Promise.all(
+    params.map((entry) =>
+      postPdlSingleEnrich<PdlCompanyData>({
+        path: '/company/enrich',
+        params: pruneUndefined({
+          pdl_id: entry.pdlId,
+          website: entry.website,
+          profile: entry.profile,
+          name: entry.name,
+          min_likelihood: entry.minLikelihood,
+        }),
+      }),
+    ),
+  );

package/src/logic-functions/utils/enrich-people.ts

@@ -0,0 +1,22 @@
+import { postPdlBulkEnrich } from 'src/logic-functions/utils/post-pdl-enrich';
+import { type PdlEnrichResult } from 'src/types/pdl-enrich-result';
+import { type PdlPersonData } from 'src/types/pdl-person-data';
+import { type PdlPersonEnrichParams } from 'src/types/pdl-person-enrich-params';
+import { pruneUndefined } from 'src/utils/prune-undefined';
+
+export const enrichPeople = (
+  params: PdlPersonEnrichParams[],
+): Promise<PdlEnrichResult<PdlPersonData>[]> =>
+  postPdlBulkEnrich<PdlPersonData>({
+    path: '/person/bulk',
+    requests: params.map((entry) =>
+      pruneUndefined({
+        pdl_id: entry.pdlId,
+        profile: entry.profile,
+        email: entry.email,
+        name: entry.name,
+        company: entry.company,
+        min_likelihood: entry.minLikelihood,
+      }),
+    ),
+  });

package/src/logic-functions/utils/enrich-person.ts

@@ -0,0 +1,24 @@
+import { postPdlSingleEnrich } from 'src/logic-functions/utils/post-pdl-single-enrich';
+import { type PdlEnrichResult } from 'src/types/pdl-enrich-result';
+import { type PdlPersonData } from 'src/types/pdl-person-data';
+import { type PdlPersonEnrichParams } from 'src/types/pdl-person-enrich-params';
+import { pruneUndefined } from 'src/utils/prune-undefined';
+
+export const enrichPerson = (
+  params: PdlPersonEnrichParams[],
+): Promise<PdlEnrichResult<PdlPersonData>[]> =>
+  Promise.all(
+    params.map((entry) =>
+      postPdlSingleEnrich<PdlPersonData>({
+        path: '/person/enrich',
+        params: pruneUndefined({
+          pdl_id: entry.pdlId,
+          profile: entry.profile,
+          email: entry.email,
+          name: entry.name,
+          company: entry.company,
+          min_likelihood: entry.minLikelihood,
+        }),
+      }),
+    ),
+  );