@memberjunction/server 6.2.0-edge.6 → 6.2.0-edge.7

package/src/integration/EntityMapLifecycle.ts

@@ -4,7 +4,8 @@
  * Selection removal DISABLES (never deletes — data is kept), re-selection
  * re-enables, and a refresh's new objects arrive disabled-by-default.
  */
-import { CompositeKey, IMetadataProvider, LogError, RunView, UserInfo } from '@memberjunction/core';
+import { CompositeKey, IMetadataProvider, LogError, RunView, UserInfo, type DatabaseProviderBase } from '@memberjunction/core';
+import { EscapeSQLString } from '@memberjunction/global';
 import type {
     MJCompanyIntegrationEntityMapEntity,
     MJCompanyIntegrationFieldMapEntity,
@@ -284,9 +285,9 @@ export interface FieldMapReconcilePlan {
  * `decideAbsentDeactivations` in the engine.
  *
  * The rules:
- *  - **New column → `Active`.** A refresh is an explicit request to bring the source's current shape
- *    in, so a column it finds is adopted rather than queued for approval. `autoEnableNewColumns:
- *    false` gates it for a connection that wants to review first.
+ *  - **New column → `Inactive` (default).** A column the refresh finds gets a field map, disabled,
+ *    and the user turns it on. `autoEnableNewColumns: true` adopts it as `Active` instead, for a
+ *    connection that wants a refresh to bring the source's current shape straight in.
  *
  *    This is REFRESH ONLY. A column first seen mid-SYNC is never auto-created — it is captured as a
  *    candidate with its statistics and needs acceptance before any DDL runs (see
@@ -301,7 +302,9 @@ export function DecideFieldMapReconcile(
     activeFieldNames: readonly string[],
     existing: readonly ExistingFieldMapRead[],
     mapEnabled: boolean,
-    autoEnableNewColumns = true,
+    // Defaults to the CONTRACT (everything.txt): a refresh's new columns arrive disabled and
+    // the user turns them on. An omitted argument must not silently adopt.
+    autoEnableNewColumns = false,
 ): FieldMapReconcilePlan {
     const plan: FieldMapReconcilePlan = { Create: [], Enable: [], Disable: [] };
     const existingByLower = new Map(
@@ -334,7 +337,227 @@ export function decideFieldMapReconcile(
     activeFieldNames: readonly string[],
     existing: readonly ExistingFieldMapRead[],
     mapEnabled: boolean,
-    autoEnableNewColumns = true,
+    autoEnableNewColumns = false,
 ): FieldMapReconcilePlan {
     return DecideFieldMapReconcile(activeFieldNames, existing, mapEnabled, autoEnableNewColumns);
 }
+
+// ── Re-keying: when a connector upgrade changes what identifies a row ────────────────────────
+//
+// A connector release can change an object's primary key — PheedLoop 1.4.6 makes Attendees
+// `code + eventCode` because the object is fetched once per event and an attendee at two events
+// comes back twice with the same `code`.
+//
+// That is not an ordinary schema change. `BaseRESTIntegrationConnector.ToExternalRecord` builds
+// `ExternalID` by joining every IsPrimaryKey field with '|' in Sequence order, so changing the key
+// changes the identity of every row the source will ever send again. Nothing can match a row
+// already stored: the RecordMap is keyed `EntityID|ExternalID`, and the fallback
+// (`MatchEngine.FindByKeyFields`) queries the destination table by key fields that are NULL on
+// rows written before the change. The next sync therefore inserts the whole source ALONGSIDE the
+// rows already there, leaving two copies of everything and nothing to say which is current.
+//
+// So a re-key means the object's stored data has to go, and the source reloaded under the new
+// identity. This is the same reasoning as U10's watermark reset one step further: not just
+// "re-fetch everything", but "re-fetch everything into a table that is not holding rows the
+// re-fetch can never reconcile with".
+
+const ENTITY_RECORD_MAPS = 'MJ: Company Integration Record Maps';
+
+/** A catalog or entity field, as much of one as the key decision needs. */
+export interface KeyCandidateField {
+    Name: string;
+    IsPrimaryKey: boolean | null;
+    Sequence: number | null;
+}
+
+/**
+ * The key that decides identity, in the order identity uses.
+ *
+ * Deliberately NOT `SourceObjectInfo.PrimaryKeyFields`. That is built in
+ * `buildSourceSchemaFromPersistedRows` as `fields.filter(f => f.IsPrimaryKey).map(f => f.Name)`
+ * over `GetIntegrationObjectFields`, which returns rows in load order with NO Sequence sort — so
+ * its order is not the order `ToExternalRecord` joins in, and comparing against it would miss a
+ * reorder and could invent one.
+ */
+export function IdentityKeyFields(fields: ReadonlyArray<KeyCandidateField>): string[] {
+    return fields
+        .filter(f => f.IsPrimaryKey === true)
+        .slice()
+        .sort((a, b) => (a.Sequence ?? 0) - (b.Sequence ?? 0))
+        .map(f => f.Name);
+}
+
+/**
+ * The key the catalog declares for an object, or `null` when its field rows were not read.
+ *
+ * An object with no Active field rows is unread, not keyless: a per-connection object whose fields
+ * were never warmed answers `[]`, and a continuing object with a built table had columns when it
+ * was built. Reporting that as "no key" would make {@link DecideRekeyed} see every keyed object as
+ * re-keyed, and the evolution would delete its record maps and rows.
+ */
+export function CatalogIdentityKey(
+    fields: ReadonlyArray<KeyCandidateField & { Status: string | null }>,
+): string[] | null {
+    const active = fields.filter(f => f.Status === 'Active');
+    return active.length === 0 ? null : IdentityKeyFields(active);
+}
+
+/**
+ * Has the identity changed between the key the tables were BUILT with and the key the catalog
+ * now declares?
+ *
+ * Ordered, not set-based: `code|eventCode` and `eventCode|code` are different identity strings,
+ * so a key that keeps its columns and reorders them strands rows exactly like one that gains a
+ * column.
+ *
+ * Returns false when the built key is EMPTY. That is "we could not read it", not "it had no key" —
+ * an object genuinely without a key never had stable identities to strand, and treating unknown as
+ * empty would re-key every table on a workspace whose entity metadata failed to load.
+ *
+ * Returns false when the catalog key is `null` — the catalog side could not be read (see
+ * {@link CatalogIdentityKey}). An empty ARRAY is different: the catalog was read and declares no
+ * key, so the object moves to content-hash identity, and that is a re-key.
+ */
+export function DecideRekeyed(builtKey: readonly string[], catalogKey: readonly string[] | null): boolean {
+    if (builtKey.length === 0 || catalogKey === null) return false;
+    const norm = (s: string) => (s ?? '').trim().toLowerCase();
+    if (builtKey.length !== catalogKey.length) return true;
+    return builtKey.some((n, i) => norm(n) !== norm(catalogKey[i]));
+}
+
+/** Record maps per chunk. Each chunk is one read and one transaction, so this bounds both. */
+export const REKEY_CLEAR_CHUNK = 500;
+
+/**
+ * The key values a record map's `EntityRecordID` names, or `null` when it cannot name exactly one
+ * row. The inverse of the engine's `composeEntityRecordID`: a single-column key is the value
+ * itself; a composite one is '|'-joined in PrimaryKeys order.
+ *
+ * A composite id that splits into more parts than the key has columns held a value that itself
+ * contained '|', and its split would name some other row. An empty part is a null key value. Both
+ * are refused rather than matched.
+ */
+function keyValuesOf(entityRecordID: string | null | undefined, keyColumns: number): string[] | null {
+    if (!entityRecordID) return null;
+    const parts = keyColumns === 1 ? [entityRecordID] : entityRecordID.split('|');
+    return parts.length === keyColumns && parts.every(p => p.length > 0) ? parts : null;
+}
+
+/**
+ * Clear what ties a re-keyed object's stored rows to the source, so the next sync rebuilds it:
+ * this connection's rows, and the record maps that point at them.
+ *
+ * Scoped by the record maps, never the table. Two connections to the same vendor resolve to one
+ * schema and therefore one table, and nothing on the table records which connection wrote a row.
+ * The connection's record maps are the only thing that does, so they decide which rows go: each
+ * map's `EntityRecordID` is the row's key, and the delete names exactly those keys. A row no map
+ * of this connection points at is left alone.
+ *
+ * In chunks of {@link REKEY_CLEAR_CHUNK} maps, read in ID order and resumed after the last ID, so
+ * neither the read nor a statement grows with the table. Each chunk deletes its rows and then its
+ * maps in ONE transaction: the rows have to go while the maps still say which rows they are, and a
+ * failure between the two must not leave maps pointing at deleted rows (the next sync would try to
+ * UPDATE them, and fail per record). Set-based, like the engine's `touchLastReconciledAt`, whose
+ * `IN` list of dialect-quoted key literals this follows.
+ *
+ * Returns the object names fully cleared. A failed chunk rolls back, is logged, and leaves its
+ * object out. So does a map whose id cannot name one row (see {@link keyValuesOf}): it stays in
+ * place, and so does its row. A table that could not be cleared must not look like one that was.
+ */
+export async function ClearRekeyedObjectData(
+    targets: ReadonlyArray<{
+        ExternalObjectName: string;
+        EntityMapID: string;
+        EntityID: string;
+        SchemaName: string;
+        BaseTable: string;
+    }>,
+    companyIntegrationID: string,
+    contextUser: UserInfo,
+    provider: DatabaseProviderBase,
+    chunkSize = REKEY_CLEAR_CHUNK,
+): Promise<string[]> {
+    const cleared: string[] = [];
+    if (targets.length === 0) return cleared;
+
+    // Resolved from metadata, never spelled out. A hardcoded `__mj.CompanyIntegrationRecordMap`
+    // would be a second place the schema is declared, and the one place nothing would update.
+    const rmEntity = provider.EntityByName(ENTITY_RECORD_MAPS);
+    if (!rmEntity?.SchemaName || !rmEntity.BaseTable) {
+        LogError(`[EntityMapLifecycle] Cannot clear re-keyed objects: '${ENTITY_RECORD_MAPS}' is not registered`);
+        return cleared;
+    }
+    const d = provider.Dialect;
+    const recordMaps = `${d.QuoteIdentifier(rmEntity.SchemaName)}.${d.QuoteIdentifier(rmEntity.BaseTable)}`;
+    const literal = (value: string) => d.QuoteStringLiteral(value);
+
+    for (const t of targets) {
+        const key = t.EntityID ? provider.EntityByID(t.EntityID)?.PrimaryKeys ?? [] : [];
+        if (!t.SchemaName || !t.BaseTable || key.length === 0) {
+            LogError(`[EntityMapLifecycle] Skipping re-key clear for ${t.ExternalObjectName}: no resolved table or key`);
+            continue;
+        }
+        const table = `${d.QuoteIdentifier(t.SchemaName)}.${d.QuoteIdentifier(t.BaseTable)}`;
+        const scope = `CompanyIntegrationID='${EscapeSQLString(companyIntegrationID)}' AND EntityID='${EscapeSQLString(t.EntityID)}'`;
+        const mapScope = `${d.QuoteIdentifier('CompanyIntegrationID')}=${literal(companyIntegrationID)} ` +
+            `AND ${d.QuoteIdentifier('EntityID')}=${literal(t.EntityID)}`;
+        let unmatched = 0;
+        let after: string | null = null;
+        try {
+            for (;;) {
+                const page = await new RunView().RunView<{ ID: string; EntityRecordID: string }>({
+                    EntityName: ENTITY_RECORD_MAPS,
+                    ExtraFilter: after ? `${scope} AND ID > '${EscapeSQLString(after)}'` : scope,
+                    Fields: ['ID', 'EntityRecordID'],
+                    OrderBy: 'ID',
+                    MaxRows: chunkSize,
+                    ResultType: 'simple',
+                    BypassCache: true,
+                    SkipTotalRowCount: true,
+                }, contextUser);
+                if (!page.Success) throw new Error(`record maps could not be read: ${page.ErrorMessage ?? 'unknown error'}`);
+                if (page.Results.length === 0) break;
+                after = page.Results[page.Results.length - 1].ID;
+
+                const matched = page.Results
+                    .map(m => ({ MapID: m.ID, Values: keyValuesOf(m.EntityRecordID, key.length) }))
+                    .filter((m): m is { MapID: string; Values: string[] } => m.Values !== null);
+                unmatched += page.Results.length - matched.length;
+                if (matched.length > 0) {
+                    const rows = key.length === 1
+                        ? `${d.QuoteIdentifier(key[0].Name)} IN (${matched.map(m => literal(m.Values[0])).join(',')})`
+                        : matched
+                            .map(m => `(${key.map((k, i) => `${d.QuoteIdentifier(k.Name)}=${literal(m.Values[i])}`).join(' AND ')})`)
+                            .join(' OR ');
+                    await provider.BeginTransaction();
+                    try {
+                        await provider.ExecuteSQL(`DELETE FROM ${table} WHERE ${rows}`, undefined, undefined, contextUser);
+                        await provider.ExecuteSQL(
+                            `DELETE FROM ${recordMaps} WHERE ${mapScope} AND ${d.QuoteIdentifier('ID')} IN (${matched.map(m => literal(m.MapID)).join(',')})`,
+                            undefined, undefined, contextUser
+                        );
+                        await provider.CommitTransaction();
+                    } catch (err) {
+                        await provider.RollbackTransaction();
+                        throw err;
+                    }
+                }
+                if (page.Results.length < chunkSize) break;
+            }
+            if (unmatched > 0) {
+                LogError(
+                    `[EntityMapLifecycle] Re-key clear of ${t.ExternalObjectName} left ${unmatched} record map(s) ` +
+                    `whose id does not name one row of its ${key.length}-column key; those rows were kept`
+                );
+                continue;
+            }
+            cleared.push(t.ExternalObjectName);
+        } catch (err) {
+            LogError(
+                `[EntityMapLifecycle] Failed to clear re-keyed object ${t.ExternalObjectName}: ` +
+                `${err instanceof Error ? err.message : String(err)}`
+            );
+        }
+    }
+    return cleared;
+}

package/src/integration/InactiveRowWarnings.ts

@@ -51,6 +51,56 @@ const isActive = (row: NamedStatusRow): boolean => row.Status === 'Active';
  * Returns an empty array when everything declared and in scope is Active.
  */
 export function ComputeInactiveRowWarnings(input: InactiveRowWarningInput): string[] {
+    return computeInactiveRowWarnings(input);
+}
+
+/**
+ * The catalog, reduced to the three reads this warning needs. `IntegrationEngineBase` satisfies it
+ * structurally, so the resolver passes its engine instance and a test passes a literal — which is the
+ * point: the collection used to live inside the resolver's schema REBUILD, and the one apply path that
+ * skips the rebuild (the batch, which prefetches its schema) therefore lost the warnings entirely.
+ * They never depended on the rebuild; they are the catalog's own Status rows.
+ */
+export interface CatalogStatusReader {
+    /** Active objects of this integration — the ones an apply materializes. */
+    GetActiveIntegrationObjects(integrationID: string): ReadonlyArray<{ ID: string; Name: string }>;
+    /** EVERY object of this integration, Active or not. */
+    GetIntegrationObjectsByIntegrationID(integrationID: string): ReadonlyArray<{ Name: string; Status: string | null }>;
+    /** Every field of one object, Active or not. */
+    GetIntegrationObjectFields(objectID: string): ReadonlyArray<{ Name: string; Status: string | null }>;
+}
+
+/**
+ * Gathers the rows {@link ComputeInactiveRowWarnings} needs from the catalog and returns its verdict.
+ *
+ * `requestedNames` empty/undefined means "everything active", which deliberately does not report
+ * deactivated OBJECTS — see {@link InactiveRowWarningInput.RequestedNames}.
+ */
+export function CollectInactiveRowWarnings(
+    catalog: CatalogStatusReader,
+    integrationID: string,
+    requestedNames?: string[] | null,
+): string[] {
+    const filter = requestedNames && requestedNames.length > 0
+        ? new Set(requestedNames.map(n => n.toLowerCase()))
+        : null;
+    // Fields only for the objects this apply materializes: a non-materialized object is reported by
+    // the object warning, not by one line per field it also lost.
+    const FieldsByObjectName: Record<string, NamedStatusRow[]> = {};
+    for (const io of catalog.GetActiveIntegrationObjects(integrationID)) {
+        if (filter && !filter.has(io.Name.toLowerCase())) continue;
+        FieldsByObjectName[io.Name] = catalog.GetIntegrationObjectFields(io.ID)
+            .map(f => ({ Name: f.Name, Status: f.Status }));
+    }
+    return computeInactiveRowWarnings({
+        RequestedNames: requestedNames ?? null,
+        AllObjects: catalog.GetIntegrationObjectsByIntegrationID(integrationID)
+            .map(io => ({ Name: io.Name, Status: io.Status })),
+        FieldsByObjectName,
+    });
+}
+
+function computeInactiveRowWarnings(input: InactiveRowWarningInput): string[] {
     const warnings: string[] = [];
     const requested = input.RequestedNames && input.RequestedNames.length > 0
         ? new Set(input.RequestedNames.map(n => n.toLowerCase()))

package/src/integration/ObjectNamesByID.ts

@@ -0,0 +1,97 @@
+/**
+ * A table selection's object ids, resolved to object names — against the CONNECTION's catalog first.
+ *
+ * Inside a catalog scope, `IntegrationListSourceObjects` hands the picker per-connection
+ * (CompanyIntegrationObject) ids. The apply paths looked ids up only in the shared
+ * `MJ: Integration Objects` catalog, so an id-only selection matched nothing: every pick was dropped
+ * with a LogError, the migration ran with an empty table list, CodeGen and the restart still ran,
+ * and the apply reported Success with 0 tables.
+ *
+ * Three tiers, cheapest first:
+ *   1. the scope's loaded catalog, no query — every per-connection object, keyed by its own id AND
+ *      by the shared IntegrationObjectID it carries, so an id from either catalog resolves;
+ *   2. a direct, uncached read of `MJ: Company Integration Objects` for this connection, for ids the
+ *      loaded catalog does not hold yet (a row written earlier in the same run);
+ *   3. the shared catalog, exactly as before — and the only tier outside a catalog scope.
+ *
+ * Keys are upper-cased ids; values are object names. An id no tier resolves is simply absent: the
+ * caller decides whether an empty resolution is fatal (an apply refuses it).
+ */
+import { RunView, type UserInfo } from '@memberjunction/core';
+import { EscapeSQLString } from '@memberjunction/global';
+import {
+    IntegrationEngineBase,
+    ENTITY_COMPANY_INTEGRATION_OBJECTS,
+    type CompanyIntegrationObjectRow,
+} from '@memberjunction/integration-engine-base';
+import { CurrentCatalogCI } from '@memberjunction/integration-engine';
+
+/** Ids per per-connection read. Each read is one `IN (…)` list, so this bounds the filter's length. */
+export const OBJECT_ID_LOOKUP_CHUNK = 200;
+
+/** What the lookup reads off a per-connection object. */
+type CatalogIdentity = Pick<CompanyIntegrationObjectRow, 'ID' | 'Name' | 'IntegrationObjectID'>;
+
+const quote = (value: string): string => `'${EscapeSQLString(value)}'`;
+
+/**
+ * Resolves object ids to names for the connection in catalog scope (or the shared catalog when
+ * there is none). Never throws for a missing id; a failed read leaves its ids unresolved.
+ */
+export async function LookupObjectNamesByID(
+    ids: ReadonlyArray<string | null | undefined> | undefined,
+    user: UserInfo
+): Promise<Map<string, string>> {
+    const map = new Map<string, string>();
+    const wanted = [...new Set((ids ?? [])
+        .filter((id): id is string => typeof id === 'string' && id.length > 0)
+        .map(id => id.toUpperCase()))];
+    if (wanted.length === 0) return map;
+
+    const ciID = CurrentCatalogCI();
+    if (ciID) {
+        const engine = IntegrationEngineBase.Instance;
+        if (engine.HasCompanyIntegrationCatalog(ciID)) {
+            // GetCompanyIntegrationObjects presents CompanyIntegrationObjectRow projections typed as
+            // the shared entity (asReadOnlyObject); the per-connection columns are on every row.
+            const rows = engine.GetCompanyIntegrationObjects(ciID) as unknown as CatalogIdentity[];
+            for (const row of rows) {
+                if (row.ID) map.set(String(row.ID).toUpperCase(), row.Name);
+                if (row.IntegrationObjectID) map.set(String(row.IntegrationObjectID).toUpperCase(), row.Name);
+            }
+        }
+    }
+
+    const rv = new RunView();
+    let missing = wanted.filter(id => !map.has(id));
+    if (missing.length > 0 && ciID) {
+        for (let i = 0; i < missing.length; i += OBJECT_ID_LOOKUP_CHUNK) {
+            const chunk = missing.slice(i, i + OBJECT_ID_LOOKUP_CHUNK);
+            const result = await rv.RunView<{ ID: string; Name: string }>({
+                EntityName: ENTITY_COMPANY_INTEGRATION_OBJECTS,
+                ExtraFilter: `CompanyIntegrationID=${quote(ciID)} AND ID IN (${chunk.map(quote).join(',')})`,
+                ResultType: 'simple',
+                Fields: ['ID', 'Name'],
+                BypassCache: true,
+                IgnoreMaxRows: true,
+            }, user);
+            if (result.Success) {
+                for (const r of result.Results) map.set(String(r.ID).toUpperCase(), r.Name);
+            }
+        }
+        missing = wanted.filter(id => !map.has(id));
+    }
+
+    if (missing.length > 0) {
+        const result = await rv.RunView<{ ID: string; Name: string }>({
+            EntityName: 'MJ: Integration Objects',
+            ExtraFilter: missing.map(id => `ID=${quote(id)}`).join(' OR '),
+            ResultType: 'simple',
+            Fields: ['ID', 'Name'],
+        }, user);
+        if (result.Success) {
+            for (const r of result.Results) map.set(String(r.ID).toUpperCase(), r.Name);
+        }
+    }
+    return map;
+}

package/src/integration/PostRestartIntrospect.ts

@@ -0,0 +1,49 @@
+/**
+ * The post-restart introspection, run against the connection's own catalog with its fields warm.
+ *
+ * After the RSU restarts the process, the consumer introspects the connector to build the entity
+ * and field maps. That runs at process start, outside any request, so nothing has entered a scope
+ * for it, and nothing has loaded this connection's catalog either. Two things have to be true for
+ * the connector to see the connection's fields:
+ *
+ *  - the scope must be entered ASYNC (`WithCatalogScope`), which loads the connection's objects;
+ *    a synchronous `RunInCatalogScope` loads nothing, so on a fresh process the reads fall back to
+ *    the shared catalog and every column this connection's discovery sampled gets no field map;
+ *  - each object's FIELD rows must be warm. A scope loads objects, not fields: only the sync loop
+ *    warms fields, so a connector reading them here (`GetCachedFields`) gets per-connection objects
+ *    with no fields at all, and the consumer creates no field maps.
+ *
+ * So this enters the scope, warms and pins every per-connection object the pending work names,
+ * introspects, and releases the pins whether or not the introspection succeeded.
+ */
+import type { UserInfo } from '@memberjunction/core';
+import { IntegrationEngineBase } from '@memberjunction/integration-engine-base';
+import { UnpinCatalogObjects, WarmCatalogObject, WithCatalogScope } from '@memberjunction/integration-engine';
+
+/**
+ * Run `introspect` in `companyIntegrationID`'s catalog scope, with the fields of each named object
+ * warm. A name the connection has no object of its own for is skipped: it reads the shared catalog,
+ * as it always did. A failed warm throws; it is never read as an object with no fields.
+ */
+export async function IntrospectInCatalogScope<T>(
+    companyIntegrationID: string,
+    sourceObjectNames: readonly string[],
+    introspect: () => Promise<T>,
+    user: UserInfo,
+): Promise<T> {
+    return WithCatalogScope(companyIntegrationID, async () => {
+        const engine = IntegrationEngineBase.Instance;
+        const pinned: string[] = [];
+        try {
+            for (const name of sourceObjectNames) {
+                const own = engine.GetCompanyIntegrationObject(companyIntegrationID, name);
+                if (!own) continue;
+                await WarmCatalogObject(own.ID, user);
+                pinned.push(own.ID);
+            }
+            return await introspect();
+        } finally {
+            UnpinCatalogObjects(pinned);
+        }
+    }, user);
+}