@memberjunction/connector-hubspot 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/HubSpotConnector.d.ts +522 -0
- package/dist/HubSpotConnector.js +3140 -0
- package/dist/HubSpotConnector.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/package.json +39 -0
|
@@ -0,0 +1,522 @@
|
|
|
1
|
+
import { type UserInfo } from '@memberjunction/core';
|
|
2
|
+
import type { MJCompanyIntegrationEntity, MJIntegrationObjectEntity } from '@memberjunction/core-entities';
|
|
3
|
+
import { BaseRESTIntegrationConnector, type RESTAuthContext, type RESTResponse, type PaginationState, type PaginationType, type ConnectionTestResult, type DefaultFieldMapping, type DefaultIntegrationConfig, type FetchContext, type FetchBatchResult, type ExternalRecord, type CRUDResult, type CreateRecordContext, type UpdateRecordContext, type UpsertRecordContext, type DeleteRecordContext, type GetRecordContext, type SearchContext, type SearchResult, type ListContext, type ListResult, type IntegrationObjectInfo, type ActionGeneratorConfig, type ExternalObjectSchema, type ExternalFieldSchema, type SourceSchemaInfo, type RateLimitPolicy } from '@memberjunction/integration-engine';
|
|
4
|
+
/** Connection configuration parsed from CompanyIntegration.Configuration JSON */
|
|
5
|
+
export interface HubSpotConnectionConfig {
|
|
6
|
+
/** HubSpot Private App access token (API Key auth — Bearer header). */
|
|
7
|
+
AccessToken: string;
|
|
8
|
+
/** API version string. Default: 'v3' */
|
|
9
|
+
ApiVersion: string;
|
|
10
|
+
/** Maximum retries for rate-limited or failed requests. Default: 5 */
|
|
11
|
+
MaxRetries?: number;
|
|
12
|
+
/** HTTP request timeout in milliseconds. Default: 30000 */
|
|
13
|
+
RequestTimeoutMs?: number;
|
|
14
|
+
/** Minimum milliseconds between API requests (HubSpot: 100 req/10s for private apps). Default: 100 */
|
|
15
|
+
MinRequestIntervalMs?: number;
|
|
16
|
+
}
|
|
17
|
+
/** HubSpot property definition (from /crm/v3/properties/{objectType}) */
|
|
18
|
+
interface HubSpotPropertyDef {
|
|
19
|
+
name: string;
|
|
20
|
+
label: string;
|
|
21
|
+
type: string;
|
|
22
|
+
fieldType: string;
|
|
23
|
+
groupName: string;
|
|
24
|
+
description: string;
|
|
25
|
+
hasUniqueValue: boolean;
|
|
26
|
+
calculated: boolean;
|
|
27
|
+
externalOptions: boolean;
|
|
28
|
+
hidden?: boolean;
|
|
29
|
+
modificationMetadata?: {
|
|
30
|
+
readOnlyValue: boolean;
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Within-scan resume state for the search-based incremental fetch (FetchChangesViaSearch).
|
|
35
|
+
* Serialized into FetchBatchResult.NextCursor so the engine threads it back via
|
|
36
|
+
* FetchContext.CurrentCursor on the next call.
|
|
37
|
+
*
|
|
38
|
+
* - `after` paginates WITHIN a single ≤10k HubSpot search window (the API's own opaque offset).
|
|
39
|
+
* - `anchorDateMs` + `anchorId` re-anchor the NEXT window by a (dateField, hs_object_id) keyset
|
|
40
|
+
* once the 10k cap is hit, so a window larger than 10k — including a >10k cluster that all shares
|
|
41
|
+
* one modified-date — is paged completely instead of stalling on a watermark that cannot advance.
|
|
42
|
+
*/
|
|
43
|
+
interface HubSpotSearchCursor {
|
|
44
|
+
after?: string;
|
|
45
|
+
anchorDateMs?: string;
|
|
46
|
+
anchorId?: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Connector for HubSpot CRM via the HubSpot REST API v3.
|
|
50
|
+
*
|
|
51
|
+
* Extends BaseRESTIntegrationConnector to leverage metadata-driven object/field
|
|
52
|
+
* discovery from IntegrationEngineBase cache and generic pagination handling.
|
|
53
|
+
*
|
|
54
|
+
* Uses Bearer token authentication with a HubSpot Private App access token (API Key auth).
|
|
55
|
+
* Supports cursor-based pagination and automatic response flattening.
|
|
56
|
+
*
|
|
57
|
+
* Configuration JSON (on CompanyIntegration) supports optional rate limit overrides:
|
|
58
|
+
* {
|
|
59
|
+
* "accessToken": "...",
|
|
60
|
+
* "MaxRetries": 5, // optional, default: 5
|
|
61
|
+
* "RequestTimeoutMs": 30000, // optional, default: 30000
|
|
62
|
+
* "MinRequestIntervalMs": 100 // optional, default: 100
|
|
63
|
+
* }
|
|
64
|
+
*
|
|
65
|
+
* Supports full CRUD: Get, Create, Update, Delete, Search, and List operations
|
|
66
|
+
* on all HubSpot CRM object types.
|
|
67
|
+
*/
|
|
68
|
+
export declare class HubSpotConnector extends BaseRESTIntegrationConnector {
|
|
69
|
+
/** Timestamp of the last API request, used for throttling */
|
|
70
|
+
private lastRequestTime;
|
|
71
|
+
/** Resolved config (populated after first Authenticate call) */
|
|
72
|
+
private _config;
|
|
73
|
+
/** Cached auth context — reused within a session to avoid redundant credential loads */
|
|
74
|
+
private _cachedAuth;
|
|
75
|
+
/** Cache of resolved default association typeIds, keyed by `${fromType}/${toType}`. */
|
|
76
|
+
private _assocTypeIdCache;
|
|
77
|
+
private get effectiveMaxRetries();
|
|
78
|
+
private get effectiveRequestTimeoutMs();
|
|
79
|
+
private get effectiveMinRequestIntervalMs();
|
|
80
|
+
get SupportsCreate(): boolean;
|
|
81
|
+
get SupportsUpdate(): boolean;
|
|
82
|
+
get SupportsUpsert(): boolean;
|
|
83
|
+
get SupportsDelete(): boolean;
|
|
84
|
+
get SupportsSearch(): boolean;
|
|
85
|
+
get SupportsListing(): boolean;
|
|
86
|
+
get IntegrationName(): string;
|
|
87
|
+
/** ~10 req/s sustained (honors MinRequestIntervalMs config) with a ~100-request burst window. */
|
|
88
|
+
get RateLimitPolicy(): RateLimitPolicy;
|
|
89
|
+
/** HubSpot rate-limits on a rolling 10-second window; on a 429 that escaped internal retries, back off ~10s. */
|
|
90
|
+
ExtractRetryAfterMs(error: unknown): number | undefined;
|
|
91
|
+
/** HubSpot tolerates modest object-level parallelism; the engine's AIMD controller ramps toward this, with the token-bucket as the real backstop. */
|
|
92
|
+
get MaxConcurrencyHint(): number;
|
|
93
|
+
GetIntegrationObjects(): IntegrationObjectInfo[];
|
|
94
|
+
GetActionGeneratorConfig(): ActionGeneratorConfig | null;
|
|
95
|
+
/** Known standard HubSpot CRM object type IDs → API names. */
|
|
96
|
+
/**
|
|
97
|
+
* All standard CRM object type IDs → names.
|
|
98
|
+
* Fields discovered live via /crm/v3/properties/{objectType}.
|
|
99
|
+
* All support: GET, POST, PATCH, DELETE, search (incremental via hs_lastmodifieddate).
|
|
100
|
+
*/
|
|
101
|
+
private static readonly STANDARD_OBJECTS;
|
|
102
|
+
/**
|
|
103
|
+
* Non-CRM API endpoints that don't follow the /crm/v3/objects pattern.
|
|
104
|
+
* Fields cannot be discovered via /crm/v3/properties — discovered dynamically
|
|
105
|
+
* by fetching one record and inferring fields from the response.
|
|
106
|
+
*/
|
|
107
|
+
private static readonly NON_CRM_OBJECTS;
|
|
108
|
+
/**
|
|
109
|
+
* Association objects use the v4 per-object associations endpoint.
|
|
110
|
+
* Fields are fixed (two FK columns + association_type) — no live discovery API exists.
|
|
111
|
+
*/
|
|
112
|
+
private static readonly ASSOCIATION_OBJECTS;
|
|
113
|
+
/**
|
|
114
|
+
* Discovers all HubSpot objects (standard + custom + non-CRM + associations) via live API and static lists.
|
|
115
|
+
*/
|
|
116
|
+
DiscoverObjects(companyIntegration: MJCompanyIntegrationEntity, contextUser: UserInfo): Promise<ExternalObjectSchema[]>;
|
|
117
|
+
/**
|
|
118
|
+
* Helper to check if an object name is a CRM object (has /crm/v3/properties endpoint).
|
|
119
|
+
*/
|
|
120
|
+
private IsCRMObject;
|
|
121
|
+
/**
|
|
122
|
+
* Finds the non-CRM object config by name.
|
|
123
|
+
*/
|
|
124
|
+
private GetNonCRMObject;
|
|
125
|
+
/**
|
|
126
|
+
* Returns association object config if objectName is an association table, null otherwise.
|
|
127
|
+
*/
|
|
128
|
+
private GetAssociationObject;
|
|
129
|
+
/**
|
|
130
|
+
* Discovers all fields on a HubSpot object via the Properties API.
|
|
131
|
+
* Returns field types, constraints, PKs, and read-only flags from live metadata.
|
|
132
|
+
*/
|
|
133
|
+
DiscoverFields(companyIntegration: MJCompanyIntegrationEntity, objectName: string, contextUser: UserInfo): Promise<ExternalFieldSchema[]>;
|
|
134
|
+
/**
|
|
135
|
+
* Discovers fields for non-CRM objects by fetching the first page of results
|
|
136
|
+
* and inferring field names/types from the response.
|
|
137
|
+
*/
|
|
138
|
+
/**
|
|
139
|
+
* Non-CRM and association objects have fixed, documented schemas.
|
|
140
|
+
* - Association objects: return both composite PK fields from ASSOCIATION_OBJECTS config.
|
|
141
|
+
* - Non-CRM objects: return the PK field from NON_CRM_OBJECTS config.
|
|
142
|
+
* IntrospectSchema's DB-fallback supplements with the full field list from metadata.
|
|
143
|
+
* No live API sampling needed.
|
|
144
|
+
*/
|
|
145
|
+
private DiscoverNonCRMFields;
|
|
146
|
+
/**
|
|
147
|
+
* Priority-ordered list of HubSpot "last changed" timestamp field names, used to
|
|
148
|
+
* populate SourceObjectInfo.IncrementalWatermarkField. Every name here is a field
|
|
149
|
+
* the connector ALREADY declares on its objects — CRM objects expose
|
|
150
|
+
* `hs_lastmodifieddate` (contacts use the legacy `lastmodifieddate`), while non-CRM
|
|
151
|
+
* REST objects expose `updatedAt`. Provable-only: the watermark is set on an object
|
|
152
|
+
* solely from that object's own declared field list, never invented.
|
|
153
|
+
*/
|
|
154
|
+
private static readonly WATERMARK_FIELD_CANDIDATES;
|
|
155
|
+
/**
|
|
156
|
+
* Promotes an object's own declared "last changed" timestamp field into the
|
|
157
|
+
* IncrementalWatermarkField slot. Returns the first candidate present in the
|
|
158
|
+
* supplied field-name set, or undefined when the object declares none — an honest
|
|
159
|
+
* gap rather than a fabricated watermark. Matching is case-insensitive so a
|
|
160
|
+
* DB-cached field list (which may differ in casing) still resolves.
|
|
161
|
+
*/
|
|
162
|
+
private PickIncrementalWatermarkField;
|
|
163
|
+
/**
|
|
164
|
+
* Full schema introspection — discovers all objects and their fields from the live API.
|
|
165
|
+
*/
|
|
166
|
+
IntrospectSchema(companyIntegration: MJCompanyIntegrationEntity, contextUser: UserInfo): Promise<SourceSchemaInfo>;
|
|
167
|
+
/**
|
|
168
|
+
* Retrieves a single record by ExternalID (HubSpot object ID).
|
|
169
|
+
*/
|
|
170
|
+
GetRecord(ctx: GetRecordContext): Promise<ExternalRecord | null>;
|
|
171
|
+
/**
|
|
172
|
+
* Creates a new record in HubSpot.
|
|
173
|
+
* Routes association objects to the v4 batch/create endpoint instead of v3 objects.
|
|
174
|
+
*/
|
|
175
|
+
CreateRecord(ctx: CreateRecordContext): Promise<CRUDResult>;
|
|
176
|
+
/**
|
|
177
|
+
* Updates an existing record in HubSpot by ExternalID.
|
|
178
|
+
* For association objects, re-creates the association (idempotent in HubSpot).
|
|
179
|
+
*/
|
|
180
|
+
UpdateRecord(ctx: UpdateRecordContext): Promise<CRUDResult>;
|
|
181
|
+
/**
|
|
182
|
+
* Idempotently creates-or-updates a record keyed by a unique business property
|
|
183
|
+
* (default: the object's `UpsertKey` metadata, e.g. 'email' for contacts).
|
|
184
|
+
*
|
|
185
|
+
* Uses HubSpot's batch/upsert endpoint with a batch of one. This is the ONLY HubSpot
|
|
186
|
+
* single-call idempotent path verified against the live API: the single-record
|
|
187
|
+
* PATCH .../{id}?idProperty=email does NOT create-on-missing (returns 404), while
|
|
188
|
+
* POST .../batch/upsert creates-on-missing and updates-on-existing with a 2xx (no 409).
|
|
189
|
+
* A batch of one sidesteps the documented batch caveats (whole-batch-409 on concurrent
|
|
190
|
+
* batches, no partial upserts) that only bite multi-input batches.
|
|
191
|
+
*
|
|
192
|
+
* This *defines the error out of existence*: a search-then-create sequence has a window in
|
|
193
|
+
* which a concurrent writer can create the same email-keyed contact, yielding
|
|
194
|
+
* `409 Contact already exists`. Rather than catch and special-case that 409, the single keyed
|
|
195
|
+
* upsert removes the window entirely — the collision is no longer a condition the caller (or
|
|
196
|
+
* this code) ever has to handle.
|
|
197
|
+
*/
|
|
198
|
+
Upsert(ctx: UpsertRecordContext): Promise<CRUDResult>;
|
|
199
|
+
/**
|
|
200
|
+
* Deletes (archives) a record in HubSpot by ExternalID.
|
|
201
|
+
* Routes association objects to the v4 batch/archive endpoint instead of v3 objects.
|
|
202
|
+
*/
|
|
203
|
+
DeleteRecord(ctx: DeleteRecordContext): Promise<CRUDResult>;
|
|
204
|
+
/**
|
|
205
|
+
* Creates an association in HubSpot using the v4 batch/create endpoint.
|
|
206
|
+
* ExternalID returned is "{leftID}|{rightID}" matching the pull ExternalID format.
|
|
207
|
+
*/
|
|
208
|
+
private CreateAssociation;
|
|
209
|
+
/**
|
|
210
|
+
* Removes an association in HubSpot using the v4 batch/archive endpoint.
|
|
211
|
+
* ExternalID must be "{leftID}|{rightID}" — the same format stored by pull sync.
|
|
212
|
+
*/
|
|
213
|
+
private DeleteAssociation;
|
|
214
|
+
/**
|
|
215
|
+
* Searches HubSpot objects using the CRM search API.
|
|
216
|
+
*/
|
|
217
|
+
SearchRecords(ctx: SearchContext): Promise<SearchResult>;
|
|
218
|
+
/**
|
|
219
|
+
* Lists records from a HubSpot object with cursor-based pagination.
|
|
220
|
+
*/
|
|
221
|
+
ListRecords(ctx: ListContext): Promise<ListResult>;
|
|
222
|
+
/** Converts a raw HubSpot API object to an ExternalRecord. */
|
|
223
|
+
private RawToExternalRecord;
|
|
224
|
+
/** Validates a CRUD response and throws on non-2xx status. */
|
|
225
|
+
private ValidateCRUDResponse;
|
|
226
|
+
/**
|
|
227
|
+
* Resolves the v4 wire from/to object types for an association. Uses explicit fromType/toType
|
|
228
|
+
* config when present; otherwise falls back to apiPath segment order. Both
|
|
229
|
+
* CreateAssociation and DeleteAssociation share this so create and archive always agree.
|
|
230
|
+
*/
|
|
231
|
+
private GetAssociationWireTypes;
|
|
232
|
+
/**
|
|
233
|
+
* Resolves the default HUBSPOT_DEFINED association typeId for a (fromType, toType) pair via
|
|
234
|
+
* GET /crm/v4/associations/{fromType}/{toType}/labels, cached per pair for the connector's life.
|
|
235
|
+
* Picks the unlabeled HUBSPOT_DEFINED entry (label === null) as the plain default; if none is
|
|
236
|
+
* unlabeled, falls back to the sole/first HUBSPOT_DEFINED entry. Returns null on lookup failure
|
|
237
|
+
* or when no HUBSPOT_DEFINED entry exists — callers MUST treat null as a hard error (never
|
|
238
|
+
* silently send empty types).
|
|
239
|
+
*/
|
|
240
|
+
private ResolveAssociationTypeId;
|
|
241
|
+
/**
|
|
242
|
+
* Validates a v4 association batch/create response BODY (not just the HTTP status).
|
|
243
|
+
* HubSpot returns 2xx even when zero associations are created — on the legacy empty-`types`
|
|
244
|
+
* no-op (empty results, no errors) and on validation failures (empty results + numErrors).
|
|
245
|
+
* Returns null when the operation genuinely completed; otherwise a human-readable error.
|
|
246
|
+
* Predicate verified against live HubSpot batch/create responses.
|
|
247
|
+
*/
|
|
248
|
+
private GetAssociationBatchError;
|
|
249
|
+
/**
|
|
250
|
+
* Validates a v3 batch/upsert response BODY (not just the HTTP status). HubSpot's batch
|
|
251
|
+
* envelope can return a 2xx while reporting per-input failures via `numErrors`/`errors`,
|
|
252
|
+
* an incomplete `status`, or an empty `results` array. Returns null when the upsert
|
|
253
|
+
* genuinely produced a record; otherwise a human-readable error. Mirrors the
|
|
254
|
+
* GetAssociationBatchError precedent — never trust a bare 2xx on a batch endpoint.
|
|
255
|
+
*/
|
|
256
|
+
private GetBatchUpsertError;
|
|
257
|
+
/** Builds a CRUDResult for error responses. */
|
|
258
|
+
private BuildCRUDErrorResult;
|
|
259
|
+
protected Authenticate(companyIntegration: MJCompanyIntegrationEntity, contextUser: UserInfo): Promise<RESTAuthContext>;
|
|
260
|
+
protected BuildHeaders(auth: RESTAuthContext): Record<string, string>;
|
|
261
|
+
protected MakeHTTPRequest(_auth: RESTAuthContext, url: string, method: string, headers: Record<string, string>, body?: unknown): Promise<RESTResponse>;
|
|
262
|
+
protected NormalizeResponse(rawBody: unknown, responseDataKey: string | null): Record<string, unknown>[];
|
|
263
|
+
protected ExtractPaginationInfo(rawBody: unknown, _paginationType: PaginationType, _currentPage: number, _currentOffset: number, _pageSize: number): PaginationState;
|
|
264
|
+
protected GetBaseURL(_companyIntegration: MJCompanyIntegrationEntity, _auth: RESTAuthContext): string;
|
|
265
|
+
/**
|
|
266
|
+
* Overrides base pagination URL building to use HubSpot's parameter names.
|
|
267
|
+
* HubSpot uses `after` for cursor pagination (not `cursor`), and needs
|
|
268
|
+
* `limit` instead of `pageSize`. Also appends `properties` query param.
|
|
269
|
+
*/
|
|
270
|
+
protected BuildPaginatedURL(basePath: string, obj: MJIntegrationObjectEntity, _page: number, _offset: number, cursor?: string): string;
|
|
271
|
+
/** Tests connectivity by authenticating and fetching 1 contact. */
|
|
272
|
+
TestConnection(companyIntegration: MJCompanyIntegrationEntity, _contextUser: UserInfo): Promise<ConnectionTestResult>;
|
|
273
|
+
GetDefaultFieldMappings(objectName: string, _entityName: string): DefaultFieldMapping[];
|
|
274
|
+
GetDefaultConfiguration(): DefaultIntegrationConfig;
|
|
275
|
+
/** Converts a HubSpot property definition to ExternalFieldSchema format */
|
|
276
|
+
MapPropertyToField(prop: HubSpotPropertyDef): {
|
|
277
|
+
Name: string;
|
|
278
|
+
Label: string;
|
|
279
|
+
DataType: string;
|
|
280
|
+
IsRequired: boolean;
|
|
281
|
+
IsUniqueKey: boolean;
|
|
282
|
+
IsReadOnly: boolean;
|
|
283
|
+
};
|
|
284
|
+
/** Maps HubSpot type + fieldType to a simplified data type string */
|
|
285
|
+
MapHubSpotType(type: string, fieldType: string): string;
|
|
286
|
+
/**
|
|
287
|
+
* Builds a HubSpotConnectionConfig from credentials and optional overrides
|
|
288
|
+
* from the CompanyIntegration Configuration JSON.
|
|
289
|
+
*/
|
|
290
|
+
private BuildConnectionConfig;
|
|
291
|
+
/**
|
|
292
|
+
* Parses optional performance overrides from Configuration JSON and applies
|
|
293
|
+
* them to the provided config object. Invalid/missing values are silently ignored.
|
|
294
|
+
*/
|
|
295
|
+
private ApplyConfigOverrides;
|
|
296
|
+
/**
|
|
297
|
+
* Reads credentials from CompanyIntegration.CredentialID -> Credential.Values JSON,
|
|
298
|
+
* or falls back to CompanyIntegration Configuration JSON for backwards compat.
|
|
299
|
+
*/
|
|
300
|
+
private LoadCredentials;
|
|
301
|
+
/** Loads credentials from a Credential entity by ID. */
|
|
302
|
+
private LoadFromCredentialEntity;
|
|
303
|
+
/** Parses a JSON string to extract HubSpot credentials. Returns null if no token found. */
|
|
304
|
+
private ParseCredentialJson;
|
|
305
|
+
/**
|
|
306
|
+
* Flattens a HubSpot CRM record from the nested format:
|
|
307
|
+
* { id, properties: { field1, field2 }, createdAt, updatedAt, archived }
|
|
308
|
+
* into a flat record with all properties at the top level,
|
|
309
|
+
* plus system fields (hs_object_id, createdAt, updatedAt, archived).
|
|
310
|
+
*/
|
|
311
|
+
private FlattenHubSpotRecord;
|
|
312
|
+
/**
|
|
313
|
+
* Overrides FetchChanges to support three fetch strategies:
|
|
314
|
+
*
|
|
315
|
+
* 1. **Association objects** → v4 per-object associations endpoint
|
|
316
|
+
* 2. **Incremental sync** (watermark set) → HubSpot search API with server-side
|
|
317
|
+
* `hs_lastmodifieddate >= watermark` filter
|
|
318
|
+
* 3. **Full load** (no watermark / first sync) → standard list API via base class
|
|
319
|
+
*/
|
|
320
|
+
FetchChanges(ctx: FetchContext): Promise<FetchBatchResult>;
|
|
321
|
+
/**
|
|
322
|
+
* Full-load path for CRM objects (first sync, no watermark).
|
|
323
|
+
* Uses the CRM list endpoint with property expansion and FlattenHubSpotRecord so that
|
|
324
|
+
* ExternalID is correctly built from hs_object_id (mirroring FetchChangesViaSearch).
|
|
325
|
+
* The base class FetchChanges cannot be used here because it reads raw[field] without
|
|
326
|
+
* flattening, causing ExternalID="" for every record.
|
|
327
|
+
*/
|
|
328
|
+
private FetchCRMFullLoad;
|
|
329
|
+
/**
|
|
330
|
+
* Fetches all pages from a single non-CRM API endpoint URL, accumulating all records.
|
|
331
|
+
* Used as a building block for both flat and parameterized endpoint fetches.
|
|
332
|
+
*/
|
|
333
|
+
private FetchAllPagesFromURL;
|
|
334
|
+
/**
|
|
335
|
+
* Handles parameterized endpoints (apiPath with {placeholder}) by fan-out:
|
|
336
|
+
* fetches all parent records, then fetches children for each parent ID.
|
|
337
|
+
* All child records are accumulated and returned as a single batch (HasMore: false).
|
|
338
|
+
*
|
|
339
|
+
* Skips objects where parentObject is not found in NON_CRM_OBJECTS (config error).
|
|
340
|
+
* Skips objects with {appId} placeholder — these require Developer App configuration.
|
|
341
|
+
*/
|
|
342
|
+
private FetchParameterizedChanges;
|
|
343
|
+
/**
|
|
344
|
+
* Fetches records from non-CRM HubSpot APIs (Marketing, CMS, Files, etc.).
|
|
345
|
+
* These endpoints use standard REST list pagination, not the CRM search API.
|
|
346
|
+
* Watermark filtering is client-side based on date fields in the response.
|
|
347
|
+
*
|
|
348
|
+
* Dispatches to FetchParameterizedChanges when apiPath contains {placeholder}.
|
|
349
|
+
*/
|
|
350
|
+
private FetchNonCRMChanges;
|
|
351
|
+
/**
|
|
352
|
+
* Find the latest date value across common date fields in a set of records.
|
|
353
|
+
*/
|
|
354
|
+
private FindLatestDateInFields;
|
|
355
|
+
/**
|
|
356
|
+
* Fetches changed records using the HubSpot search API with server-side date filtering.
|
|
357
|
+
* Much more efficient than fetching ALL records and filtering client-side.
|
|
358
|
+
*
|
|
359
|
+
* Handles the search API's 10,000-results-per-window hard cap by keyset re-anchoring: results
|
|
360
|
+
* are sorted by (dateField, hs_object_id) ASCENDING, paginated within a window by the API's
|
|
361
|
+
* opaque `after` offset, and once that offset hits the 10k cap the NEXT window re-anchors with a
|
|
362
|
+
* compound filter `(dateField > anchor) OR (dateField == anchor AND hs_object_id > anchorId)`.
|
|
363
|
+
* This makes an incremental window — or a bulk-import cluster of >10k records that all share one
|
|
364
|
+
* `hs_lastmodifieddate` — page through completely in a single sync, instead of the watermark
|
|
365
|
+
* stalling on a same-timestamp cluster it can never advance past (which silently lost records).
|
|
366
|
+
* The date GTE watermark remains the primary filter throughout, so incremental sync is preserved.
|
|
367
|
+
*
|
|
368
|
+
* LIVE-VERIFY (confirm during the credentialed run against a real >10k same-timestamp cluster):
|
|
369
|
+
* 1. hs_object_id GT/EQ comparison in v3 search is NUMERIC, not lexicographic. HIGHEST STAKES — if
|
|
370
|
+
* lexicographic, '2' > '10000' and the keyset would skip records. (hs_object_id is a sequential
|
|
371
|
+
* 64-bit integer / number-typed property, so numeric is expected, but prove it across an
|
|
372
|
+
* id-magnitude boundary, e.g. ids 9, 10, 100, 1000 within one timestamp cluster.)
|
|
373
|
+
* 2. Datetime filter values accept epoch-millis-as-string for EQ/GT/GTE (the pre-existing GTE
|
|
374
|
+
* watermark filter already relies on this, so a regression here would also break prior behavior).
|
|
375
|
+
* 3. The compound (dateField ASC, hs_object_id ASC) sort is honored deterministically across pages
|
|
376
|
+
* and object types, so the last raw result is the true (date,id)-max keyset boundary.
|
|
377
|
+
* 4. `total` reflects the CURRENT filterGroups per re-anchored query (not a cached original count).
|
|
378
|
+
*/
|
|
379
|
+
protected FetchChangesViaSearch(ctx: FetchContext): Promise<FetchBatchResult>;
|
|
380
|
+
/**
|
|
381
|
+
* Parses the {@link HubSpotSearchCursor} threaded via FetchContext.CurrentCursor. Tolerates a
|
|
382
|
+
* legacy raw `after` string (pre-keyset format) by treating it as a plain window offset, so an
|
|
383
|
+
* in-flight sync mid-upgrade degrades gracefully rather than throwing.
|
|
384
|
+
*/
|
|
385
|
+
protected parseSearchCursor(raw: string | undefined): HubSpotSearchCursor;
|
|
386
|
+
/**
|
|
387
|
+
* Extracts the keyset anchor (the last record's dateField-as-epoch-ms and hs_object_id) from a
|
|
388
|
+
* batch of raw search results. Because results are sorted (dateField, hs_object_id) ASCENDING,
|
|
389
|
+
* the last element is the maximum position and therefore the resume point for the next window.
|
|
390
|
+
* Returns undefined fields when the batch is empty or the values can't be parsed.
|
|
391
|
+
*/
|
|
392
|
+
private extractSearchAnchor;
|
|
393
|
+
/**
|
|
394
|
+
* Normalizes a HubSpot datetime property value to an epoch-millis string for use in a search
|
|
395
|
+
* filter. Accepts both an ISO-8601 string (the usual v3 shape) and a bare epoch-millis numeric
|
|
396
|
+
* string (some endpoints/properties). Returns undefined when unparseable — callers then skip
|
|
397
|
+
* re-anchoring rather than seeking from a NaN position.
|
|
398
|
+
*/
|
|
399
|
+
private toEpochMs;
|
|
400
|
+
/**
|
|
401
|
+
* Pure decision for the next search window, given this page's pagination + total. No network.
|
|
402
|
+
*
|
|
403
|
+
* - If more pages remain inside the current ≤10k window, advance the API `after` offset and keep
|
|
404
|
+
* the same anchor.
|
|
405
|
+
* - Otherwise the window is exhausted (the API stopped returning `after`, or it reached the 10k
|
|
406
|
+
* cap). If records still match the current filter (`total > cap`), re-anchor the next window on
|
|
407
|
+
* the last record's (dateField, hs_object_id) keyset; else the scan is complete.
|
|
408
|
+
*
|
|
409
|
+
* `total` is the count matching the CURRENT filter, so after each re-anchor it shrinks by roughly
|
|
410
|
+
* one window until it falls to/under the cap — guaranteeing termination with no skipped records
|
|
411
|
+
* (the anchor's id strictly increases) and no duplicates (the keyset predicate excludes it).
|
|
412
|
+
*/
|
|
413
|
+
protected computeSearchResume(args: {
|
|
414
|
+
incoming: HubSpotSearchCursor;
|
|
415
|
+
pagingNextAfter: string | undefined;
|
|
416
|
+
total: number;
|
|
417
|
+
lastAnchorDateMs: string | undefined;
|
|
418
|
+
lastAnchorId: string | undefined;
|
|
419
|
+
}): {
|
|
420
|
+
nextCursor: HubSpotSearchCursor | undefined;
|
|
421
|
+
hasMore: boolean;
|
|
422
|
+
stalled?: boolean;
|
|
423
|
+
};
|
|
424
|
+
/**
|
|
425
|
+
* Detects archived (deleted) CRM records since the given watermark.
|
|
426
|
+
*
|
|
427
|
+
* Uses the HubSpot search API with `archived: true` so the same server-side
|
|
428
|
+
* GTE watermark filter applies — only records archived/modified since the last
|
|
429
|
+
* sync are returned. Returns them with `IsDeleted: true` so the integration
|
|
430
|
+
* engine routes them through the delete pipeline.
|
|
431
|
+
*
|
|
432
|
+
* Falls back to empty on any API error (e.g. object type doesn't support
|
|
433
|
+
* archived search) so delete detection degrades gracefully rather than
|
|
434
|
+
* blocking the active-record sync.
|
|
435
|
+
*/
|
|
436
|
+
private FetchArchivedCRMChanges;
|
|
437
|
+
/** Returns the watermark date field name for a given object type. */
|
|
438
|
+
private GetWatermarkField;
|
|
439
|
+
/** Finds the latest date value across records for a given field name. */
|
|
440
|
+
private FindLatestDate;
|
|
441
|
+
/**
|
|
442
|
+
* Fetches association records in batches by iterating over synced parent (from-side)
|
|
443
|
+
* objects and calling HubSpot's v4 per-object associations endpoint.
|
|
444
|
+
*
|
|
445
|
+
* Uses ctx.CurrentOffset to track parent position across batch calls, so the engine
|
|
446
|
+
* can page through all parents without truncating records.
|
|
447
|
+
*/
|
|
448
|
+
/**
|
|
449
|
+
* Fetches association records using the HubSpot v4 batch/read endpoint.
|
|
450
|
+
* Batches up to 100 parent IDs per request instead of one GET per parent,
|
|
451
|
+
* reducing API calls from O(n) to O(n/100).
|
|
452
|
+
*/
|
|
453
|
+
private FetchAssociationChanges;
|
|
454
|
+
/**
|
|
455
|
+
* Calls POST /crm/v4/associations/{fromType}/{toType}/batch/read with up to 100 parent IDs.
|
|
456
|
+
* Response format: { results: [{ from: { id }, to: [{ toObjectId, associationTypes }] }] }
|
|
457
|
+
*/
|
|
458
|
+
private FetchAssociationBatch;
|
|
459
|
+
/**
|
|
460
|
+
* Converts a HubSpot v4 association result item into a flat record suitable for storage.
|
|
461
|
+
* v4 format: { toObjectId: number, associationTypes: [{ label, typeId, category }] }
|
|
462
|
+
*/
|
|
463
|
+
private FlattenAssociationRecord;
|
|
464
|
+
/**
|
|
465
|
+
* Parses a v4 associations APIPath to extract from/to object type names.
|
|
466
|
+
* E.g., "/crm/v4/associations/contacts/companies" → { fromType: "contacts", toType: "companies" }
|
|
467
|
+
*/
|
|
468
|
+
private ParseAssociationPath;
|
|
469
|
+
/**
|
|
470
|
+
* Loads hs_object_id values for all synced records of a given HubSpot object type
|
|
471
|
+
* by finding its entity map and querying the local MJ entity.
|
|
472
|
+
*/
|
|
473
|
+
private LoadAssociationParentIDs;
|
|
474
|
+
/**
|
|
475
|
+
* Extracts the HubSpot object name from an API path.
|
|
476
|
+
* E.g., "/crm/v3/objects/contacts" -> "contacts"
|
|
477
|
+
*/
|
|
478
|
+
private ExtractObjectNameFromPath;
|
|
479
|
+
/**
|
|
480
|
+
* Returns the known field names for a HubSpot object type, derived from
|
|
481
|
+
* the HUBSPOT_OBJECTS metadata (single source of truth).
|
|
482
|
+
*/
|
|
483
|
+
private GetObjectFieldNames;
|
|
484
|
+
/**
|
|
485
|
+
* Returns the configured upsert key (unique business property to match on) for an
|
|
486
|
+
* object from HUBSPOT_OBJECTS metadata, or undefined if the object declares none.
|
|
487
|
+
* Used by Upsert to default the idProperty when the caller doesn't override it.
|
|
488
|
+
*/
|
|
489
|
+
private GetUpsertKey;
|
|
490
|
+
/**
|
|
491
|
+
* Returns the effective property list for a HubSpot CRM request.
|
|
492
|
+
*
|
|
493
|
+
* When `requestedFields` (from FetchContext.RequestedSourceFields) is provided it
|
|
494
|
+
* contains the source fields from active field maps, including any custom properties.
|
|
495
|
+
* We merge those with the essential system properties so watermark tracking always works.
|
|
496
|
+
*
|
|
497
|
+
* Falls back to the static HUBSPOT_OBJECTS field list when no requestedFields are given.
|
|
498
|
+
*
|
|
499
|
+
* Note: hs_object_id is NOT included — it's the top-level `id` field on every HubSpot
|
|
500
|
+
* response and is injected by FlattenHubSpotRecord regardless of `?properties=`.
|
|
501
|
+
* Essential system properties (always included):
|
|
502
|
+
* - GetWatermarkField(objectName) — the per-object modified-date property (object-specific;
|
|
503
|
+
* contacts use 'lastmodifieddate', all others use 'hs_lastmodifieddate')
|
|
504
|
+
* - createdate — creation timestamp
|
|
505
|
+
*/
|
|
506
|
+
private BuildEffectiveProperties;
|
|
507
|
+
/**
|
|
508
|
+
* Builds the `properties` query parameter for a HubSpot object type.
|
|
509
|
+
* Accepts optional `requestedFields` from FetchContext to include custom-mapped properties.
|
|
510
|
+
* Returns empty string if no properties are configured for the object.
|
|
511
|
+
*/
|
|
512
|
+
private BuildPropertiesParam;
|
|
513
|
+
/** Executes an HTTP request with a timeout and optional JSON body. */
|
|
514
|
+
private FetchWithTimeout;
|
|
515
|
+
/** Calculates retry delay from Retry-After header or exponential backoff. */
|
|
516
|
+
private CalculateRetryDelay;
|
|
517
|
+
/** Converts a fetch Response + parsed body into a RESTResponse. */
|
|
518
|
+
private BuildRESTResponse;
|
|
519
|
+
/** Returns a promise that resolves after the specified number of milliseconds. */
|
|
520
|
+
private Sleep;
|
|
521
|
+
}
|
|
522
|
+
export {};
|