@memberjunction/integration-engine 0.0.1 → 5.9.0

package/dist/BaseRESTIntegrationConnector.d.ts

@@ -0,0 +1,222 @@
+import { type UserInfo } from '@memberjunction/core';
+import type { MJCompanyIntegrationEntity, MJIntegrationObjectEntity } from '@memberjunction/core-entities';
+import { BaseIntegrationConnector, type ExternalObjectSchema, type ExternalFieldSchema, type FetchContext, type FetchBatchResult } from './BaseIntegrationConnector.js';
+import type { SourceSchemaInfo } from './types.js';
+/** Authentication context returned by Authenticate(). Concrete connectors extend this. */
+export interface RESTAuthContext {
+    /** Bearer or session token */
+    Token?: string;
+    /** Session ID for session-based APIs */
+    SessionID?: string;
+    /** When the auth token expires */
+    ExpiresAt?: Date;
+    /** Allow connector-specific auth properties */
+    [key: string]: unknown;
+}
+/** Normalized HTTP response from MakeHTTPRequest() */
+export interface RESTResponse {
+    /** HTTP status code */
+    Status: number;
+    /** Parsed response body */
+    Body: unknown;
+    /** Response headers (lowercase keys) */
+    Headers: Record<string, string>;
+}
+/** Pagination state extracted from a vendor response */
+export interface PaginationState {
+    /** Whether more pages remain */
+    HasMore: boolean;
+    /** Opaque cursor for cursor-based pagination */
+    NextCursor?: string;
+    /** Numeric offset for offset-based pagination */
+    NextOffset?: number;
+    /** Page number for page-based pagination */
+    NextPage?: number;
+    /** Total records reported by the API, if available */
+    TotalRecords?: number;
+}
+/** Pagination type values matching MJIntegrationObjectEntity.PaginationType */
+export type PaginationType = 'Cursor' | 'None' | 'Offset' | 'PageNumber';
+/**
+ * Abstract base class for REST API integration connectors.
+ *
+ * Implements the generic REST sync pattern: reads IntegrationObject/Field
+ * metadata from the MJ database, handles pagination, template variable
+ * resolution (per-parent iteration), and converts raw API responses to
+ * ExternalRecord format.
+ *
+ * Concrete connectors (YourMembership, Salesforce, HubSpot, etc.) extend
+ * this class and implement only auth, HTTP transport, and response normalization.
+ */
+export declare abstract class BaseRESTIntegrationConnector extends BaseIntegrationConnector {
+    /**
+     * Authenticate with the external system and return an auth context.
+     * Called once per FetchChanges invocation; the returned context is
+     * passed to BuildHeaders and MakeHTTPRequest for every request.
+     */
+    protected abstract Authenticate(companyIntegration: MJCompanyIntegrationEntity): Promise<RESTAuthContext>;
+    /**
+     * Build HTTP headers for an API request, including auth headers.
+     * Called before every HTTP request.
+     */
+    protected abstract BuildHeaders(auth: RESTAuthContext): Record<string, string>;
+    /**
+     * Execute an HTTP request. The concrete connector owns the transport
+     * layer (fetch, axios, got, etc.).
+     */
+    protected abstract MakeHTTPRequest(auth: RESTAuthContext, url: string, method: string, headers: Record<string, string>, body?: unknown): Promise<RESTResponse>;
+    /**
+     * Extract the data array from the vendor-specific response envelope.
+     * @param rawBody - The parsed response body
+     * @param responseDataKey - The key to extract data from, or null for root-level arrays
+     * @returns Array of raw record objects
+     */
+    protected abstract NormalizeResponse(rawBody: unknown, responseDataKey: string | null): Record<string, unknown>[];
+    /**
+     * Extract pagination state from the vendor-specific response.
+     * @param rawBody - The parsed response body
+     * @param paginationType - The pagination strategy for this object
+     * @param currentPage - Current page number (1-based)
+     * @param currentOffset - Current record offset
+     * @param pageSize - Page size used in the request
+     * @returns Pagination state indicating whether more data is available
+     */
+    protected abstract ExtractPaginationInfo(rawBody: unknown, paginationType: PaginationType, currentPage: number, currentOffset: number, pageSize: number): PaginationState;
+    /**
+     * Get the base URL for API requests (e.g., "https://api.example.com/v1").
+     * Combined with the object's APIPath to form the full request URL.
+     */
+    protected abstract GetBaseURL(companyIntegration: MJCompanyIntegrationEntity): string;
+    /**
+     * Discovers available objects from the IntegrationEngineBase cache.
+     */
+    DiscoverObjects(companyIntegration: MJCompanyIntegrationEntity, _contextUser: UserInfo): Promise<ExternalObjectSchema[]>;
+    /**
+     * Discovers fields for a specific object from the IntegrationEngineBase cache.
+     */
+    DiscoverFields(companyIntegration: MJCompanyIntegrationEntity, objectName: string, _contextUser: UserInfo): Promise<ExternalFieldSchema[]>;
+    /**
+     * Builds SourceSchemaInfo from IntegrationEngineBase cached metadata.
+     * Provides richer metadata than the base class implementation, including
+     * FK relationships derived from RelatedIntegrationObjectID.
+     */
+    IntrospectSchema(companyIntegration: MJCompanyIntegrationEntity, _contextUser: UserInfo): Promise<SourceSchemaInfo>;
+    /**
+     * Fetches records from the external REST API using metadata-driven configuration.
+     * Handles both flat endpoints and template-variable (per-parent) endpoints.
+     */
+    FetchChanges(ctx: FetchContext): Promise<FetchBatchResult>;
+    /**
+     * Fetches records from a flat endpoint (no template variable substitution).
+     */
+    private FetchFlat;
+    /**
+     * Fetches records from a template-variable endpoint by iterating over parent records.
+     * Identifies the parent object from FK field metadata, loads parent IDs from the
+     * local database, then fetches child records per parent.
+     */
+    private FetchWithTemplateVars;
+    /**
+     * Resolves which template variable maps to which FK field + parent object.
+     *
+     * Resolution strategy (in order):
+     * 1. Explicit: FK field with RelatedIntegrationObjectID that matches a template var name
+     * 2. PK fallback: finds a sibling integration object whose primary key field name
+     *    matches the template var, allowing resolution without explicit FK metadata
+     *
+     * Returns null if no match is found by either strategy.
+     */
+    private ResolveParentInfo;
+    /**
+     * Fallback resolution: finds a sibling integration object whose primary key
+     * field name matches one of the template variables (case-insensitive).
+     * E.g., {ProfileID} matches Members object which has PK field "ProfileID".
+     */
+    private ResolveParentByPKMatch;
+    /**
+     * Loads parent record IDs from the local MJ database.
+     * Uses IntegrationEngineBase cache for object/field metadata lookups,
+     * then queries the MJ entity via RunView for actual synced record IDs.
+     */
+    private LoadParentIDs;
+    /**
+     * Fetches records from a single URL with pagination support.
+     * Loops through pages until no more data or maxRecords is reached.
+     */
+    private FetchWithPagination;
+    /**
+     * Fetches a single non-paginated page.
+     */
+    private FetchSinglePage;
+    /**
+     * Iterates through paginated responses until exhausted or maxRecords is reached.
+     */
+    private FetchPaginatedLoop;
+    /**
+     * Combines baseURL and apiPath into a full URL.
+     */
+    private BuildFullURL;
+    /**
+     * Appends pagination parameters to a URL based on the object's PaginationType.
+     * Override in subclasses to use vendor-specific parameter names.
+     */
+    protected BuildPaginatedURL(basePath: string, obj: MJIntegrationObjectEntity, page: number, offset: number, cursor?: string): string;
+    /**
+     * Appends default query parameters from the IntegrationObject metadata.
+     * DefaultQueryParams is stored as a JSON object like {"key": "value"}.
+     * Automatically skips params whose key (case-insensitive) already appears
+     * in the URL to avoid duplicates with pagination params.
+     */
+    protected AppendDefaultQueryParams(url: string, obj: MJIntegrationObjectEntity): string;
+    /**
+     * Extracts query parameter keys from a URL, returned as a Set of lowercase strings.
+     * Used to detect duplicates between pagination params and DefaultQueryParams.
+     */
+    private ExtractURLParamKeys;
+    /**
+     * Detects template variables in an API path. Template variables use
+     * the format {VariableName} (e.g., "/profiles/{ProfileID}/events").
+     */
+    private DetectTemplateVars;
+    /**
+     * Substitutes a single template variable in a path.
+     */
+    private SubstituteTemplateVars;
+    /**
+     * Gets an IntegrationObject from the engine's cache by integration ID and object name.
+     * Throws if not found.
+     */
+    private GetCachedObject;
+    /**
+     * Gets IntegrationObjectField records from the engine's cache for a given object ID.
+     * Returns only active fields sorted by Sequence.
+     */
+    private GetCachedFields;
+    /**
+     * Converts a raw API record object to the ExternalRecord format.
+     */
+    private ToExternalRecord;
+    /**
+     * Finds the primary key field name from a set of IntegrationObjectField records.
+     * Falls back to "ID" if no PK field is explicitly marked.
+     */
+    private FindPrimaryKeyFieldName;
+    /**
+     * Converts an IntegrationObjectFieldEntity to the ExternalFieldSchema format.
+     */
+    private FieldEntityToSchema;
+    /**
+     * Builds a SourceObjectInfo from an IntegrationObject and its fields.
+     * Resolves FK relationships using the RelatedIntegrationObjectID.
+     */
+    private BuildSourceObjectInfo;
+    /**
+     * Builds SourceRelationshipInfo entries from FK fields.
+     */
+    private BuildRelationships;
+    /**
+     * Validates an HTTP response and throws a descriptive error on non-2xx status.
+     */
+    private ValidateHTTPResponse;
+}
+//# sourceMappingURL=BaseRESTIntegrationConnector.d.ts.map

package/dist/BaseRESTIntegrationConnector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"BaseRESTIntegrationConnector.d.ts","sourceRoot":"","sources":["../src/BaseRESTIntegrationConnector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAE9D,OAAO,KAAK,EAAE,0BAA0B,EAAE,yBAAyB,EAAkC,MAAM,+BAA+B,CAAC;AAE3I,OAAO,EACH,wBAAwB,EAExB,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,EACxB,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACxB,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAkB,gBAAgB,EAA4C,MAAM,YAAY,CAAC;AAI7G,0FAA0F;AAC1F,MAAM,WAAW,eAAe;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,+CAA+C;IAC/C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,sDAAsD;AACtD,MAAM,WAAW,YAAY;IACzB,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,wCAAwC;IACxC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC5B,gCAAgC;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,+EAA+E;AAC/E,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,GAAG,YAAY,CAAC;AAazE;;;;;;;;;;GAUG;AACH,8BAAsB,4BAA6B,SAAQ,wBAAwB;IAI/E;;;;OAIG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,CAC3B,kBAAkB,EAAE,0BAA0B,GAC/C,OAAO,CAAC,eAAe,CAAC;IAE3B;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,CAC3B,IAAI,EAAE,eAAe,GACtB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,eAAe,CAC9B,IAAI,EAAE,eAAe,EACrB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,IAAI,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,CAAC;IAExB;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAChC,OAAO,EAAE,OAAO,EAChB,eAAe,EAAE,MAAM,GAAG,IAAI,GAC/B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;IAE5B;;;;;;;;OAQG;IACH,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CACpC,OAAO,EAAE,OAAO,EAChB,cAAc,EAAE,cAAc,EAC9B,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,MAAM,GACjB,eAAe;IAElB;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,UAAU,CACzB,kBAAkB,EAAE,0BAA0B,GAC/C,MAAM;IAIT;;OAEG;IACU,eAAe,CACxB,kBAAkB,EAAE,0BAA0B,EAC9C,YAAY,EAAE,QAAQ,GACvB,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAclC;;OAEG;IACU,cAAc,CACvB,kBAAkB,EAAE,0BAA0B,EAC9C,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,QAAQ,GACvB,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAOjC;;;;OAIG;IACU,gBAAgB,CACzB,kBAAkB,EAAE,0BAA0B,EAC9C,YAAY,EAAE,QAAQ,GACvB,OAAO,CAAC,gBAAgB,CAAC;IAc5B;;;OAGG;IACU,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiBvE;;OAEG;YACW,SAAS;IAmBvB;;;;OAIG;YACW,qBAAqB;IAsCnC;;;;;;;;;OASG;IACH,OAAO,CAAC,iBAAiB;IAwBzB;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IA2B9B;;;;OAIG;YACW,aAAa;IAyC3B;;;OAGG;YACW,mBAAmB;IAajC;;OAEG;YACW,eAAe;IAc7B;;OAEG;YACW,kBAAkB;IAiEhC;;OAEG;IACH,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,SAAS,CAAC,iBAAiB,CACvB,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,yBAAyB,EAC9B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,MAAM,GAChB,MAAM;IAiBT;;;;;OAKG;IACH,SAAS,CAAC,wBAAwB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,yBAAyB,GAAG,MAAM;IA0BvF;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAgB3B;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAK1B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAM9B;;;OAGG;IACH,OAAO,CAAC,eAAe;IAQvB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAQvB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAK/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAc3B;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IA8B7B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAwB1B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CAW/B"}