@memberjunction/integration-engine 5.49.0 → 5.51.0

package/dist/MatchEngine.d.ts

@@ -21,6 +21,52 @@ export declare class MatchEngine {
      * @returns Updated mapped records with resolved ChangeType and MatchedMJRecordID
      */
     Resolve(records: MappedRecord[], entityMap: ICompanyIntegrationEntityMap, fieldMaps: ICompanyIntegrationFieldMap[], contextUser: UserInfo, provider?: IMetadataProvider): Promise<MappedRecord[]>;
+    /**
+     * Issues every lookup this batch needs as ONE batched read, and builds both indexes from it.
+     *
+     * The legs are independent — the record-map read and each criteria-shape group's read share
+     * no data dependency — so they go out together via {@link RunView.RunViews} rather than as
+     * separate awaits. Result order matches param order, which is how each leg is attributed back.
+     *
+     * A `null` index means "the read failed, fall back to the per-record query" and is NOT the
+     * same as an empty index, which means "asked, and there is nothing". Conflating them would
+     * treat every record as unmapped and turn an incremental sync into a batch of duplicate
+     * creates, so the two are kept distinct all the way down.
+     */
+    private PrefetchBatchLookups;
+    /**
+     * The read that resolves an entire batch of external IDs against the record map, or null when
+     * the batch carries no usable external ID (nothing to ask, so no query is issued).
+     */
+    private buildRecordMapViewParams;
+    /**
+     * Builds the record-map index from its leg of the batched read.
+     *
+     * Returns an index containing only the IDs that HAVE a mapping, so a miss is represented by
+     * absence — exactly what the per-record lookup needed to distinguish. Returns null if the read
+     * failed, which makes callers fall back to their original per-record query rather than
+     * silently treating every record as unmapped.
+     *
+     * The index carries a second, normalized view of the same rows. The `IN (…)` read is evaluated
+     * by the database under its own collation, so a row stored as `abc` comes back for a requested
+     * `ABC`; pairing the result up in JavaScript with `===` would then miss it. See
+     * {@link normalizeExternalID}.
+     */
+    private buildRecordMapIndexFromResult;
+    /** Builds the exact + normalized views of a record-map read, separating out ambiguous folds. */
+    private buildRecordMapIndex;
+    /**
+     * True when the platform's `=` on string columns is case- and trailing-blank-insensitive, so
+     * the JS-side fold in {@link normalizeExternalID} reproduces what the database already did.
+     *
+     * SQL Server, under the default `SQL_Latin1_General_CP1_CI_AS` collation, is. PostgreSQL is
+     * not: `text`/`varchar` equality there is exact on both counts. Folding on PG would make the
+     * batch index return a mapping the per-record query would never return — the record would be
+     * UPDATED onto someone else's row instead of created, which is the one error direction this
+     * whole index is built to avoid. A provider with no dialect (client-side) gets `false`, the
+     * conservative answer.
+     */
+    private get platformFoldsStringEquality();
     /**
      * Resolves a single record by checking for an existing MJ match.
      */
@@ -34,30 +80,126 @@ export declare class MatchEngine {
      */
     private ResolveExistingRecord;
     /**
-     * Attempts to find an existing MJ record by key field matching, then falls back
-     * to the CompanyIntegrationRecordMap.
+     * Attempts to find an existing MJ record by identity (PK) or, failing that, by the
+     * configured key fields — then falls back to the CompanyIntegrationRecordMap.
      *
-     * For composite-PK entities (no auto-generated ID), key-field matching is always
-     * attempted (even when no key fields are configured) so that PK fields themselves
-     * serve as the unique match criteria. This handles entities like InsightTopics
-     * where (person_id, topic) together form the natural key.
+     * The direct lookup is attempted whenever it CAN resolve something:
+     *  - the mapped record carries a complete PK (single OR composite) — the identity case,
+     *    and the same key the apply path will address the row by; or
+     *  - key fields are configured — the fallback case, for mapping external records onto
+     *    pre-existing MJ rows whose PK the external system does not know.
+     *
+     * Previously a single-PK entity with no configured key fields skipped this entirely and
+     * relied on the record map alone — so a record whose map row was missing (or truncated;
+     * see `LoadAllRecordMaps`) re-CREATED a row whose PK already existed. Integration shadow
+     * tables are exactly this shape: a single soft PK holding the external ID.
      */
     private FindExistingRecord;
     /**
-     * Searches for an existing MJ record using key field values.
+     * Answers the identity/key lookup from the batch prefetch when it can, and only issues the
+     * per-record query when the prefetch neither found the record nor proved it absent.
+     */
+    private LookupByKeyFields;
+    /**
+     * Returns the entity's primary-key fields (soft PKs included — integration shadow tables
+     * carry no physical key, so `PrimaryKeys` is populated from `additionalSchemaInfo` and is
+     * still the natural identity).
+     */
+    private primaryKeyFieldsFor;
+    /** True when the mapped data carries a value for EVERY PK field — i.e. it asserts an identity. */
+    private hasCompleteMappedPrimaryKey;
+    /**
+     * Searches for an existing MJ record, using ONE definition of identity.
+     *
+     * **The unified rule (tasks.md PR 2 item 3).** The entity's primary key — including a
+     * soft PK, which is what integration shadow tables carry — is the single definition of
+     * record identity, for matching AND for saving. Configured key fields (`IsKeyField`) are
+     * a *fallback lookup* for records whose PK the mapped data does not carry, never a
+     * competing identity.
      *
-     * For composite-PK entities, all PK fields are added to the WHERE filter to
-     * guarantee a unique match, and the returned ID is a '|'-delimited composite
-     * of all PK field values (matching the ExternalID format used by connectors).
+     * Concretely: when the mapped record carries a COMPLETE PK, match on the PK alone. The
+     * apply path already addresses the record by its PK (`CreateRecord.extractMappedPrimaryKey`
+     * → `InnerLoad`), so matching on anything else is exactly how a record could match one row
+     * and then be written to another — or match a row whose PK then collides on save. When the
+     * PK is absent or partial (e.g. mapping external records onto pre-existing MJ rows keyed by
+     * a server-assigned UUID), fall back to the configured key fields, plus whatever PK parts
+     * ARE present to narrow the result.
      *
-     * For single-PK entities, behaviour is unchanged: filter by configured key fields,
-     * return the single PK value as a plain string.
+     * The returned ID is always the PK values in `PrimaryKeys` order, '|'-joined — the same
+     * format `ExternalID`/`EntityRecordID` use — so the caller can load the row directly.
      */
     private FindByKeyFields;
     /**
-     * Builds a SQL filter clause from key field values on a mapped record.
+     * Builds the field/value criteria that identify a record, applying the item-3 identity rule:
+     * a COMPLETE primary key matches on the PK alone; otherwise the configured key fields plus
+     * whatever PK parts are present. Returns null when nothing can be matched on.
+     *
+     * Field/value pairs (rather than pre-rendered SQL) so the same criteria can be rendered as a
+     * filter clause AND used as a local lookup key when a whole batch is resolved in one query.
+     */
+    private BuildMatchCriteria;
+    /**
+     * Quotes a value as a SQL string literal for a `RunView` `ExtraFilter`.
+     *
+     * `ExtraFilter` is SQL text, not a parameter list, so literals must be escaped here rather
+     * than bound. Where the provider is a database provider this defers to its dialect — the same
+     * rule `RecordMapBatch`'s read path uses, so the two cannot disagree about a value.
+     * `IMetadataProvider` in general carries no dialect (a client-side provider has none), so the
+     * fallback is ANSI quote-doubling, which is what both supported platforms do.
+     *
+     * Every value quoted here — external IDs and key-field values alike — is compared against an
+     * nvarchar column, so it goes through {@link quoteTextLiteral} rather than the dialect's bare
+     * `QuoteStringLiteral`: on SQL Server the bare form is a *varchar* literal and silently drops
+     * any character outside the database's collation codepage before the comparison.
+     */
+    private quoteLiteral;
+    /**
+     * Renders criteria as a filter clause.
+     *
+     * ANSI double-quoted identifiers — portable across SQL Server (QUOTED_IDENTIFIER ON, the
+     * driver default) and Postgres (exact-case; integration columns are lowercase). Plain
+     * identifiers break on a column named for a reserved word (e.g. a soft PK named
+     * `open`/`order`); brackets would fix SQL Server but break Postgres, so double-quote.
+     */
+    private CriteriaToSQL;
+    /**
+     * Order-independent local lookup key for a criteria set (and for a row read back for it).
+     *
+     * `\0` delimits both within and between pairs, because neither a field name nor a value can
+     * contain it and no printable delimiter can make that guarantee. With a space, field `A B`
+     * = `C` and field `A` = `B C` both render `A B C` — and since this key is what decides which
+     * row answers which criteria, a collision attributes a row to the wrong record.
+     */
+    private CriteriaKey;
+    /**
+     * Resolves the identity/key lookup for a WHOLE batch in one query per criteria shape,
+     * instead of one query per record (tasks.md PR 2 item 5).
+     *
+     * Records are grouped by entity + the exact set of fields they match on, so every record in
+     * a group produces a structurally identical AND-clause; the group's clauses are OR-ed into a
+     * single read and the rows attributed back locally by the same field/value pairs that built
+     * the filter. A 500-record batch of an integration shadow table (single soft PK) collapses
+     * from 500 reads to 1.
+     *
+     * Attribution is deliberately conservative. `Matched` holds records we found a row for;
+     * `Unmatched` holds records we can PROVE have no row — only populated when the group's read
+     * returned nothing at all, so a record whose row exists but whose local key comparison
+     * differs from the database's (case-insensitive collation, numeric/date formatting) is left
+     * in neither set and falls back to its own query. A false "unmatched" would create a
+     * duplicate, so it is never inferred.
+     *
+     * This half is pure: it builds the reads without issuing them, so every group goes out as one
+     * leg of the batch's single {@link RunView.RunViews} call. {@link MatchEngine.buildKeyMatchIndex}
+     * attributes the results back.
+     */
+    private buildKeyMatchGroups;
+    /**
+     * Attributes the batched group reads back to their criteria, positionally, in the order
+     * {@link MatchEngine.buildKeyMatchGroups} emitted them.
+     *
+     * Returns null if any group read failed — callers then use the original per-record path.
      */
-    private BuildKeyFieldFilter;
+    private buildKeyMatchIndex;
     /**
      * Checks the CompanyIntegrationRecordMap for a previous external↔MJ mapping.
      */

package/dist/MatchEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"MatchEngine.d.ts","sourceRoot":"","sources":["../src/MatchEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAqB,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC3F,OAAO,KAAK,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,KAAK,EAAE,YAAY,EAAsB,MAAM,YAAY,CAAC;AAGnE;;;GAGG;AACH,qBAAa,WAAW;IACpB,gFAAgF;IAChF,OAAO,CAAC,SAAS,CAAC,CAAoB;IAEtC,4FAA4F;IAC5F,SAAS,KAAK,aAAa,IAAI,iBAAiB,CAE/C;IAED;;;;;;;;;OASG;IACU,OAAO,CAChB,OAAO,EAAE,YAAY,EAAE,EACvB,SAAS,EAAE,4BAA4B,EACvC,SAAS,EAAE,2BAA2B,EAAE,EACxC,WAAW,EAAE,QAAQ,EACrB,QAAQ,CAAC,EAAE,iBAAiB,GAC7B,OAAO,CAAC,YAAY,EAAE,CAAC;IAmB1B;;OAEG;YACW,mBAAmB;IAsBjC;;OAEG;YACW,oBAAoB;IAmBlC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAW7B;;;;;;;;OAQG;YACW,kBAAkB;IAuBhC;;;;;;;;;OASG;YACW,eAAe;IAqD7B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAiB3B;;OAEG;YACW,kBAAkB;CA0BnC"}
+{"version":3,"file":"MatchEngine.d.ts","sourceRoot":"","sources":["../src/MatchEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwB,iBAAiB,EAA6D,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACzJ,OAAO,KAAK,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,KAAK,EAAE,YAAY,EAAsB,MAAM,YAAY,CAAC;AA8EnE;;;GAGG;AACH,qBAAa,WAAW;IACpB,gFAAgF;IAChF,OAAO,CAAC,SAAS,CAAC,CAAoB;IAEtC,4FAA4F;IAC5F,SAAS,KAAK,aAAa,IAAI,iBAAiB,CAE/C;IAED;;;;;;;;;OASG;IACU,OAAO,CAChB,OAAO,EAAE,YAAY,EAAE,EACvB,SAAS,EAAE,4BAA4B,EACvC,SAAS,EAAE,2BAA2B,EAAE,EACxC,WAAW,EAAE,QAAQ,EACrB,QAAQ,CAAC,EAAE,iBAAiB,GAC7B,OAAO,CAAC,YAAY,EAAE,CAAC;IAkC1B;;;;;;;;;;;OAWG;YACW,oBAAoB;IA+BlC;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAyBhC;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,6BAA6B;IAOrC,gGAAgG;IAChG,OAAO,CAAC,mBAAmB;IA6B3B;;;;;;;;;;OAUG;IACH,OAAO,KAAK,2BAA2B,GAMtC;IAED;;OAEG;YACW,mBAAmB;IAwBjC;;OAEG;YACW,oBAAoB;IAqBlC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAW7B;;;;;;;;;;;;;;OAcG;YACW,kBAAkB;IAsBhC;;;OAGG;YACW,iBAAiB;IAkB/B;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IAK3B,kGAAkG;IAClG,OAAO,CAAC,2BAA2B;IAMnC;;;;;;;;;;;;;;;;;;;OAmBG;YACW,eAAe;IAyB7B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAyC1B;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IAOpB;;;;;;;OAOG;IACH,OAAO,CAAC,aAAa;IAMrB;;;;;;;OAOG;IACH,OAAO,CAAC,WAAW;IAQnB;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,OAAO,CAAC,mBAAmB;IAgD3B;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IA8B1B;;OAEG;YACW,kBAAkB;CA8CnC"}