@memberjunction/server 5.28.0 → 5.30.0

package/src/resolvers/IntegrationDiscoveryResolver.ts

@@ -4,6 +4,7 @@ import { CronExpressionHelper } from "@memberjunction/scheduling-engine";
 import {
     MJCompanyIntegrationEntity,
     MJIntegrationEntity,
+    MJIntegrationObjectEntity,
     MJCredentialEntity,
     MJCompanyIntegrationEntityMapEntity,
     MJCompanyIntegrationFieldMapEntity,
@@ -27,6 +28,7 @@ import {
     SourceSchemaInfo,
     IntegrationSchemaSync
 } from "@memberjunction/integration-engine";
+import { IntegrationEngineBase } from "@memberjunction/integration-engine-base";
 import {
     SchemaBuilder,
     TypeMapper,
@@ -75,7 +77,8 @@ class ApplySchemaBatchItemInput {
 
 @InputType()
 class SourceObjectInput {
-    @Field({ description: 'Source object ID (IntegrationObject.ID)' }) SourceObjectID: string;
+    @Field({ nullable: true, description: 'Existing IntegrationObject.ID. Either SourceObjectID or SourceObjectName must be provided; if both, ID wins.' }) SourceObjectID?: string;
+    @Field({ nullable: true, description: 'External object name (e.g. "Account"). Use when the object has no IntegrationObject row yet — the server will create one via describe+persist.' }) SourceObjectName?: string;
     @Field(() => [String], { nullable: true, description: 'Optional field selection. Empty/null = all fields (including any new ones). Only specified fields get field maps.' }) Fields?: string[];
 }
 
@@ -692,6 +695,34 @@ function isValidEntityMapStatus(value: string): value is EntityMapStatus {
     return (VALID_ENTITY_MAP_STATUSES as readonly string[]).includes(value);
 }
 
+// ─── List Source Objects (Full-Catalog Picker) ──────────────────────────────
+// Returns every object the source system exposes (e.g. all ~1,800 Salesforce
+// sobjects), merged with any existing IntegrationObject metadata so the UI
+// can show which objects are already registered versus newly discoverable.
+// Intentionally cheap: one global describe call, no per-object describes.
+
+@ObjectType()
+class ListSourceObjectsItem {
+    @Field() Name: string;
+    @Field() Label: string;
+    @Field({ nullable: true }) Description?: string;
+    @Field() SupportsIncrementalSync: boolean;
+    @Field() SupportsWrite: boolean;
+    /** True when an IntegrationObject row already exists for this object. */
+    @Field() AlreadyPersisted: boolean;
+    /** IntegrationObject.ID — populated only when AlreadyPersisted is true. */
+    @Field({ nullable: true }) IntegrationObjectID?: string;
+    /** True when the source system flags this as user/custom (e.g. SF __c names). */
+    @Field() IsCustom: boolean;
+}
+
+@ObjectType()
+class ListSourceObjectsOutput {
+    @Field() Success: boolean;
+    @Field() Message: string;
+    @Field(() => [ListSourceObjectsItem], { nullable: true }) Objects?: ListSourceObjectsItem[];
+}
+
 /**
  * GraphQL resolver for integration discovery operations.
  * Provides endpoints to test connections, discover objects, and discover fields
@@ -773,6 +804,109 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         }
     }
 
+    /**
+     * Full-catalog picker endpoint: returns every object the source system
+     * exposes (e.g. all ~1,800 Salesforce sobjects) merged with flags showing
+     * which ones already have IntegrationObject rows in MJ. Cheap by design —
+     * one global discovery call per source, no per-object describes. Per-object
+     * describe runs later, at selection time, inside IntegrationApplyAllBatch.
+     */
+    @Query(() => ListSourceObjectsOutput)
+    async IntegrationListSourceObjects(
+        @Arg("companyIntegrationID") companyIntegrationID: string,
+        @Ctx() ctx: AppContext
+    ): Promise<ListSourceObjectsOutput> {
+        try {
+            const user = this.getAuthenticatedUser(ctx);
+            const { connector, companyIntegration } = await this.resolveConnector(companyIntegrationID, user);
+
+            // Use the engine cache for already-persisted IntegrationObject
+            // rows — single in-memory read instead of a per-call DB roundtrip.
+            await IntegrationEngine.Instance.Config(false, user);
+            const existingObjects = IntegrationEngineBase.Instance
+                .GetIntegrationObjectsByIntegrationID(companyIntegration.IntegrationID);
+
+            const discoverObjects = connector.DiscoverObjects.bind(connector) as
+                (ci: unknown, u: unknown) => Promise<ExternalObjectSchema[]>;
+            const liveObjects = await discoverObjects(companyIntegration, user);
+
+            const existingByName = new Map<string, { ID: string; IsCustom: boolean }>();
+            for (const row of existingObjects) {
+                existingByName.set(row.Name, { ID: row.ID, IsCustom: !!row.IsCustom });
+            }
+
+            // Live is SoT. When the probe succeeds, show only live objects
+            // (the persisted IntegrationObject ID is overlaid by name when
+            // there's a match). When the probe returns nothing (transient
+            // SI failure, rate limit, expired session), fall back to the
+            // engine cache so the user isn't stuck with an empty picker.
+            const sourceObjects = liveObjects.length > 0
+                ? liveObjects
+                : existingObjects.map(row => ({
+                    Name: row.Name,
+                    Label: row.Name,
+                    Description: undefined,
+                    SupportsIncrementalSync: true,
+                    SupportsWrite: true,
+                  }) as ExternalObjectSchema);
+
+            const merged: ListSourceObjectsItem[] = sourceObjects.map(o => {
+                const existing = existingByName.get(o.Name);
+                return {
+                    Name: o.Name,
+                    Label: o.Label,
+                    Description: o.Description,
+                    SupportsIncrementalSync: o.SupportsIncrementalSync,
+                    SupportsWrite: o.SupportsWrite,
+                    AlreadyPersisted: existing != null,
+                    IntegrationObjectID: existing?.ID,
+                    IsCustom: this.isCustomObjectName(o.Name, existing?.IsCustom),
+                };
+            });
+            merged.sort((a, b) => a.Name.localeCompare(b.Name));
+
+            return {
+                Success: true,
+                Message: `Listed ${merged.length} source objects (${liveObjects.length} from live probe, ${existingByName.size} already persisted)`,
+                Objects: merged,
+            };
+        } catch (e) {
+            LogError(`IntegrationListSourceObjects error: ${e}`);
+            return { Success: false, Message: this.formatError(e) };
+        }
+    }
+
+    private async loadIntegrationObjectsByIntegrationID(
+        integrationID: string,
+        user: UserInfo
+    ): Promise<Array<{ ID: string; Name: string; IsCustom: boolean }>> {
+        const rv = new RunView();
+        const result = await rv.RunView<MJIntegrationObjectEntity>({
+            EntityName: 'MJ: Integration Objects',
+            ExtraFilter: `IntegrationID='${integrationID}'`,
+            ResultType: 'entity_object',
+        }, user);
+        if (!result.Success) {
+            LogError(`loadIntegrationObjectsByIntegrationID failed: ${result.ErrorMessage}`);
+            return [];
+        }
+        return result.Results.map(r => ({
+            ID: r.ID,
+            Name: r.Name,
+            IsCustom: r.IsCustom === true,
+        }));
+    }
+
+    /**
+     * Heuristic for flagging custom objects in the UI. Existing rows carry an
+     * IsCustom column; newly-discovered ones don't, so fall back to the SF
+     * `__c` suffix convention (harmless on systems where it doesn't apply).
+     */
+    private isCustomObjectName(name: string, existingIsCustom: boolean | undefined): boolean {
+        if (existingIsCustom != null) return existingIsCustom;
+        return name.endsWith('__c');
+    }
+
     /**
      * Discovers fields on a specific external object.
      */
@@ -1023,6 +1157,15 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         // but the connector's GetIntegrationObjects() always has them.
         const connectorDescriptions = this.buildDescriptionLookup(connector);
 
+        // Track all drop reasons so we can emit one summary line at the end
+        // instead of forcing the caller to scan O(N) LogError lines to figure
+        // out how many selections actually made it. The picker → ApplyAll →
+        // RSU pipeline already had three layers of silent O(N) drops; this
+        // makes them at least summarised.
+        const droppedNotInSchema: string[] = [];
+        const droppedNoFields: string[] = [];
+        const droppedNoPrimaryKey: string[] = [];
+
         const results: TargetTableConfig[] = [];
         for (const obj of objects) {
             const sourceObj = sourceSchema.Objects.find(o => o.ExternalName.toLowerCase() === obj.SourceObjectName.toLowerCase());
@@ -1031,6 +1174,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             // If the object wasn't discovered in IntrospectSchema (e.g. API error), skip it
             // rather than generating a broken table with no columns and a fallback PK.
             if (!sourceObj) {
+                droppedNotInSchema.push(obj.SourceObjectName);
                 LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — not found in source schema (IntrospectSchema may have failed for this object)`);
                 continue;
             }
@@ -1043,17 +1187,27 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 !selectedFieldSet || selectedFieldSet.has(f.Name.toLowerCase()) || f.IsPrimaryKey
             );
 
-            const columns: TargetColumnConfig[] = sourceFields.map(f => ({
-                SourceFieldName: f.Name,
-                TargetColumnName: f.Name.replace(/[^A-Za-z0-9_]/g, '_'),
-                TargetSqlType: mapper.MapSourceType(f.SourceType, platform, f),
-                IsNullable: !f.IsRequired,
-                MaxLength: f.MaxLength,
-                Precision: f.Precision,
-                Scale: f.Scale,
-                DefaultValue: f.DefaultValue,
-                Description: f.Description ?? objDescriptions?.fields.get(f.Name.toLowerCase()),
-            }));
+            const columns: TargetColumnConfig[] = sourceFields.map(f => {
+                const targetSqlType = mapper.MapSourceType(f.SourceType, platform, f);
+                return {
+                    SourceFieldName: f.Name,
+                    TargetColumnName: f.Name.replace(/[^A-Za-z0-9_]/g, '_'),
+                    TargetSqlType: targetSqlType,
+                    // Synced shadow tables must NOT enforce NOT NULL on non-PK
+                    // columns. The external system (SF, HubSpot, etc.) is the
+                    // source of truth for business data, not for MJ's schema
+                    // constraints — and its describe output often declares
+                    // fields required when real records actually have nulls
+                    // (deprecated, calculated, or edge-case fields). Enforcing
+                    // NOT NULL here just aborts entire batches on one bad row.
+                    IsNullable: !f.IsPrimaryKey,
+                    MaxLength: f.MaxLength,
+                    Precision: f.Precision,
+                    Scale: f.Scale,
+                    DefaultValue: this.formatSqlDefault(f.DefaultValue, targetSqlType),
+                    Description: f.Description ?? objDescriptions?.fields.get(f.Name.toLowerCase()),
+                };
+            });
 
             const primaryKeyFields = sourceObj.Fields
                 .filter(f => f.IsPrimaryKey)
@@ -1062,6 +1216,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             // If no columns were discovered, skip rather than generating a broken table
             // (DDL with UNIQUE ([ID]) on a non-existent column will always fail).
             if (columns.length === 0 && primaryKeyFields.length === 0) {
+                droppedNoFields.push(obj.SourceObjectName);
                 LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — 0 fields discovered (live API likely failed and no DB-cached fields available)`);
                 continue;
             }
@@ -1069,6 +1224,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             // If columns exist but no PK was found, log diagnostic info and skip rather than
             // generating broken DDL with UNIQUE ([ID]) on a non-existent column.
             if (primaryKeyFields.length === 0 && columns.length > 0) {
+                droppedNoPrimaryKey.push(obj.SourceObjectName);
                 const fieldNames = sourceObj.Fields.map(f => `${f.Name}(pk=${f.IsPrimaryKey})`).join(', ');
                 LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — ${columns.length} columns but NO primary key field found. Fields: [${fieldNames}]`);
                 continue;
@@ -1085,6 +1241,28 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 SoftForeignKeys: []
             });
         }
+
+        // Single-line summary of every drop that happened during this call.
+        // Without this, callers see N individual LogError lines and have to
+        // count them by hand to know how much got lost. With it, the gap
+        // between "selections requested" and "tables generated" is a one-line
+        // grep target (`buildTargetConfigs summary`) that names which objects
+        // were lost and why.
+        const totalRequested = objects.length;
+        const totalAccepted = results.length;
+        const totalDropped = totalRequested - totalAccepted;
+        if (totalDropped > 0) {
+            const fmt = (arr: string[]): string =>
+                arr.length === 0
+                    ? '0'
+                    : `${arr.length} (${arr.slice(0, 5).join(', ')}${arr.length > 5 ? `, +${arr.length - 5} more` : ''})`;
+            console.warn(
+                `[buildTargetConfigs summary] requested=${totalRequested}, accepted=${totalAccepted}, dropped=${totalDropped} ` +
+                `(notInSchema=${fmt(droppedNotInSchema)}, noFields=${fmt(droppedNoFields)}, noPK=${fmt(droppedNoPrimaryKey)})`
+            );
+        } else {
+            console.log(`[buildTargetConfigs summary] requested=${totalRequested}, accepted=${totalAccepted} (all selections produced target configs)`);
+        }
         return results;
     }
 
@@ -1116,6 +1294,49 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         return result;
     }
 
+    /**
+     * Format a raw default-value from source schema (SF describe, etc.) into a
+     * SQL-literal string appropriate for the target column's SQL type.
+     *
+     * The DDLGenerator splats DefaultValue raw into `... DEFAULT ${value}`, so
+     * the caller MUST pre-quote/pre-coerce. Previously this layer passed SF's
+     * `String(defaultValue)` through unchanged, which produced invalid T-SQL
+     * like `DEFAULT false` on BIT columns and `DEFAULT Diagonal` on strings.
+     *
+     * Rules:
+     *  - null/undefined/empty → undefined (no DEFAULT clause emitted)
+     *  - Known SQL expressions (GETDATE(), CURRENT_TIMESTAMP, NEWID(), NULL) → pass through
+     *  - Numeric-looking strings → pass through
+     *  - Booleans on BIT/BOOLEAN columns → '1' / '0'
+     *  - Everything else → quoted string literal with single-quote escaping
+     */
+    private formatSqlDefault(raw: string | null | undefined, targetSqlType: string): string | undefined {
+        if (raw == null) return undefined;
+        const trimmed = String(raw).trim();
+        if (trimmed === '') return undefined;
+
+        const upperType = targetSqlType.toUpperCase();
+        const isBit = upperType.includes('BIT') || upperType.includes('BOOLEAN');
+
+        // Preserve SQL keywords / well-known function calls
+        const sqlFunctionRegex = /^(NULL|CURRENT_TIMESTAMP|CURRENT_DATE|CURRENT_TIME|GETDATE\(\)|GETUTCDATE\(\)|SYSUTCDATETIME\(\)|SYSDATETIME\(\)|NEWID\(\)|NEWSEQUENTIALID\(\))$/i;
+        if (sqlFunctionRegex.test(trimmed)) return trimmed.toUpperCase();
+
+        // Booleans
+        if (/^(true|false)$/i.test(trimmed)) {
+            const isTrue = trimmed.toLowerCase() === 'true';
+            if (isBit) return isTrue ? '1' : '0';
+            // Non-bit column holding a boolean word — quote it as a string
+            return isTrue ? "'true'" : "'false'";
+        }
+
+        // Numeric literal (int, decimal, scientific notation)
+        if (/^-?\d+(\.\d+)?([eE][+-]?\d+)?$/.test(trimmed)) return trimmed;
+
+        // String literal — escape single quotes by doubling them
+        return `'${trimmed.replace(/'/g, "''")}'`;
+    }
+
     private buildDescriptionLookup(connector?: BaseIntegrationConnector): Map<string, { objectDescription?: string; fields: Map<string, string> }> {
         const result = new Map<string, { objectDescription?: string; fields: Map<string, string> }>();
         if (!connector) return result;
@@ -1131,6 +1352,73 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         return result;
     }
 
+    /**
+     * Decides whether an apply call should use the filtered-introspection flow.
+     * Salesforce has ~1,800 sobjects and the global describe is prohibitively
+     * expensive; other connectors have dozens and the legacy "describe all then
+     * pick" behavior is fine. The flow also engages when the client opts in by
+     * sending a SourceObjectName (which the SF full-catalog picker does).
+     */
+    private shouldUseFilteredIntrospection(
+        connector: BaseIntegrationConnector,
+        sourceObjects: SourceObjectInput[]
+    ): boolean {
+        const isSalesforce = connector.IntegrationName === 'Salesforce';
+        const clientSentNames = sourceObjects.some(so => !!so.SourceObjectName);
+        return isSalesforce && clientSentNames;
+    }
+
+    /**
+     * Builds a selection plan from SourceObjectInput[] for the filtered flow.
+     * Each entry resolves to { Name, Fields }, with Name coming from either:
+     *   - SourceObjectName directly (newly-picked from full-catalog picker), or
+     *   - A one-shot DB lookup of SourceObjectID → IntegrationObject.Name
+     * Never fails on missing rows — such entries are silently dropped (the
+     * caller raises on empty selection).
+     */
+    private async resolveSelectionPlan(
+        sourceObjects: SourceObjectInput[],
+        user: UserInfo
+    ): Promise<Array<{ Name: string; Fields?: string[] }>> {
+        const idsToLookup = sourceObjects
+            .filter(so => !so.SourceObjectName && so.SourceObjectID)
+            .map(so => so.SourceObjectID!);
+
+        const idToName = new Map<string, string>();
+        if (idsToLookup.length > 0) {
+            const rv = new RunView();
+            const result = await rv.RunView<{ ID: string; Name: string }>({
+                EntityName: 'MJ: Integration Objects',
+                ExtraFilter: idsToLookup.map(id => `ID='${id}'`).join(' OR '),
+                ResultType: 'simple',
+                Fields: ['ID', 'Name'],
+            }, user);
+            if (result.Success) {
+                for (const row of result.Results) {
+                    idToName.set(row.ID.toUpperCase(), row.Name);
+                }
+            }
+        }
+
+        const plan: Array<{ Name: string; Fields?: string[] }> = [];
+        for (const so of sourceObjects) {
+            const name = so.SourceObjectName
+                ?? (so.SourceObjectID ? idToName.get(so.SourceObjectID.toUpperCase()) : undefined);
+            if (name) plan.push({ Name: name, Fields: so.Fields });
+        }
+        return plan;
+    }
+
+    /**
+     * Aligns caller-supplied names to the source schema's ExternalName casing.
+     * Keeps the original when no match is found so downstream steps can still
+     * raise a targeted error rather than silently drop the object.
+     */
+    private normalizeNamesAgainstSchema(names: string[], sourceSchema: SourceSchemaInfo): string[] {
+        const map = new Map(sourceSchema.Objects.map(o => [o.ExternalName.toLowerCase(), o.ExternalName]));
+        return names.map(n => map.get(n.toLowerCase()) ?? n);
+    }
+
     /**
      * Resolves source object IDs to exact names from the DB, and normalizes names
      * to match the source schema's ExternalName casing. Call once at each entry point.
@@ -1142,7 +1430,12 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         integrationID: string,
         user: UserInfo
     ): Promise<string[]> {
-        // If IDs provided, resolve them to names from IntegrationObject records
+        // PRESERVED for backward compat with older call sites; new code should
+        // use resolveSourceObjectsToNames which handles per-item ID/Name fallback
+        // without silently dropping items that have a name but no ID (or an ID
+        // that doesn't match an IntegrationObject row yet — the picker can send
+        // newly-discovered objects with no persisted row).
+        void integrationID;
         if (ids && ids.length > 0) {
             const rv = new RunView();
             const result = await rv.RunView<{ ID: string; Name: string }>({
@@ -1155,16 +1448,90 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 return result.Results.map(r => r.Name);
             }
         }
-
-        // Otherwise normalize provided names against source schema casing
         if (names && names.length > 0) {
             const nameMap = new Map(sourceSchema.Objects.map(o => [o.ExternalName.toLowerCase(), o.ExternalName]));
             return names.map(n => nameMap.get(n.toLowerCase()) ?? n);
         }
-
         return [];
     }
 
+    /**
+     * Per-item ID/name resolver for picker selections.
+     *
+     * Each `SourceObjectInput` from the picker may carry SourceObjectID
+     * (for objects with an existing IntegrationObject row), SourceObjectName
+     * (for newly-discovered objects with no persisted row yet), or both.
+     *
+     * The legacy `resolveSourceObjectNames` only honored the IDs path:
+     * `ids.map(...)` produced a SQL `WHERE ID IN (...)` and returned only
+     * the matched rows — name-only selections and ID-misses were silently
+     * dropped, with no surfaced log line. On real syncs this collapsed
+     * 1156 picker selections to 420 IntegrationObjects to 181 generated
+     * tables. Two silent O(N) data losses, invisible to users.
+     *
+     * This resolver:
+     *   - looks up names for selections that have an ID
+     *   - falls back to the SourceObjectName for selections without an ID
+     *     (or whose ID didn't match) — normalizing case against the source
+     *     schema when available
+     *   - LogErrors loudly when a selection truly can't be resolved (no ID
+     *     match AND no name) so the drop is visible in the run output
+     *   - returns names in the same order as the input, with the count of
+     *     dropped items so the caller can decide whether to abort or warn
+     */
+    private async resolveSourceObjectsToNames(
+        sourceObjects: SourceObjectInput[],
+        sourceSchema: SourceSchemaInfo,
+        user: UserInfo
+    ): Promise<{ names: string[]; droppedCount: number; sourceObjects: SourceObjectInput[] }> {
+        // Look up names for any selections with an ID
+        const idsToLookup = sourceObjects
+            .map(so => so.SourceObjectID)
+            .filter((id): id is string => typeof id === 'string' && id.length > 0);
+        const idToName = new Map<string, string>();
+        if (idsToLookup.length > 0) {
+            const rv = new RunView();
+            const result = await rv.RunView<{ ID: string; Name: string }>({
+                EntityName: 'MJ: Integration Objects',
+                ExtraFilter: idsToLookup.map(id => `ID='${id}'`).join(' OR '),
+                ResultType: 'simple',
+                Fields: ['ID', 'Name'],
+            }, user);
+            if (result.Success) {
+                for (const r of result.Results) idToName.set(r.ID, r.Name);
+            }
+        }
+
+        const schemaNameMap = new Map(sourceSchema.Objects.map(o => [o.ExternalName.toLowerCase(), o.ExternalName]));
+        const resolvedNames: string[] = [];
+        const resolvedSourceObjects: SourceObjectInput[] = [];
+        const dropped: SourceObjectInput[] = [];
+        for (const so of sourceObjects) {
+            let name: string | undefined;
+            if (so.SourceObjectID && idToName.has(so.SourceObjectID)) {
+                name = idToName.get(so.SourceObjectID);
+            } else if (so.SourceObjectName) {
+                // Normalize case against the schema when the connector reports it
+                name = schemaNameMap.get(so.SourceObjectName.toLowerCase()) ?? so.SourceObjectName;
+            }
+            if (name) {
+                resolvedNames.push(name);
+                resolvedSourceObjects.push(so);
+            } else {
+                dropped.push(so);
+            }
+        }
+        if (dropped.length > 0) {
+            const sample = dropped.slice(0, 5).map(d => `{id=${d.SourceObjectID ?? '∅'}, name=${d.SourceObjectName ?? '∅'}}`).join(', ');
+            LogError(
+                `[resolveSourceObjectsToNames] Dropped ${dropped.length} of ${sourceObjects.length} selection(s) ` +
+                `— neither SourceObjectID matched an IntegrationObject row nor was a SourceObjectName provided. ` +
+                `Sample: ${sample}${dropped.length > 5 ? ` (+${dropped.length - 5} more)` : ''}.`
+            );
+        }
+        return { names: resolvedNames, droppedCount: dropped.length, sourceObjects: resolvedSourceObjects };
+    }
+
     /**
      * Resolves SourceObjectID/SourceObjectName on SchemaPreviewObjectInput array.
      * Mutates the objects in place — sets SourceObjectName from ID if provided.
@@ -1924,18 +2291,20 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 console.warn(`[IntegrationApplyAll] Schema persistence warning (non-fatal): ${msg}`);
             }
 
-            const objectIDs = input.SourceObjects.map(so => so.SourceObjectID);
-            const resolvedNames = await this.resolveSourceObjectNames(objectIDs, undefined, sourceSchema, companyIntegration.IntegrationID, user);
+            const resolved = await this.resolveSourceObjectsToNames(input.SourceObjects, sourceSchema, user);
+            const resolvedNames = resolved.names;
 
-            // Build SchemaPreviewObjectInput with Fields carried from SourceObjectInput
-            const fieldsByID = new Map(input.SourceObjects.map(so => [so.SourceObjectID, so.Fields]));
+            // Build SchemaPreviewObjectInput with Fields from the matching
+            // SourceObjectInput (resolved.sourceObjects is order-aligned with names).
+            // Previously this stripped to IDs only, which silently dropped any
+            // selection without an IntegrationObject row yet (newly discovered).
             const objects = resolvedNames.map((name, i) => {
                 const obj = new SchemaPreviewObjectInput();
                 obj.SourceObjectName = name;
                 obj.SchemaName = schemaName;
                 obj.TableName = name.replace(/[^A-Za-z0-9_]/g, '_');
                 obj.EntityName = name.replace(/([a-z])([A-Z])/g, '$1 $2').replace(/_/g, ' ');
-                obj.Fields = fieldsByID.get(objectIDs[i]) ?? undefined;
+                obj.Fields = resolved.sourceObjects[i].Fields ?? undefined;
                 return obj;
             });
 
@@ -1950,11 +2319,13 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
             const pendingWorkDir = join(rsuWorkDir, '.rsu_pending');
             const pendingFilePath = join(pendingWorkDir, `${Date.now()}.json`);
 
-            // Build per-object field map for pending file (null = all fields)
+            // Build per-object field map for pending file (null = all fields).
+            // resolved.sourceObjects is order-aligned with resolvedNames after the
+            // resolveSourceObjectsToNames refactor — pair them directly instead
+            // of looking up by ID (which broke for name-only selections).
             const sourceObjectFields: Record<string, string[] | null> = {};
-            for (const so of input.SourceObjects) {
-                const resolvedName = resolvedNames[objectIDs.indexOf(so.SourceObjectID)];
-                if (resolvedName) sourceObjectFields[resolvedName] = so.Fields ?? null;
+            for (let i = 0; i < resolvedNames.length; i++) {
+                sourceObjectFields[resolvedNames[i]] = resolved.sourceObjects[i].Fields ?? null;
             }
 
             const pendingPayload = {
@@ -2004,7 +2375,13 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 );
                 const createdMapIDs = entityMapsCreated.map(em => em.EntityMapID).filter(Boolean);
                 const scopedMapIDs = input.SyncScope === 'all' ? undefined : createdMapIDs;
-                const syncRunID = input.StartSync !== false
+                // Skip sync when SyncScope='created' but 0 new maps were
+                // created — otherwise empty EntityMapIDs falls through engine's
+                // `length > 0` gate and runs a full integration sync against
+                // every existing entity map (the 459-record-on-0-map-apply bug).
+                const shouldStartSync = input.StartSync !== false &&
+                    (input.SyncScope === 'all' || createdMapIDs.length > 0);
+                const syncRunID = shouldStartSync
                     ? await this.startSyncAfterApply(input.CompanyIntegrationID, user, scopedMapIDs, input.FullSync)
                     : null;
 
@@ -2231,13 +2608,26 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         platform: 'sqlserver' | 'postgresql',
         user: UserInfo,
         skipGitCommit: boolean,
-        skipRestart: boolean
+        skipRestart: boolean,
+        prefetchedSourceSchema?: SourceSchemaInfo
     ): Promise<{ schemaOutput: SchemaBuilderOutput; rsuInput: RSUPipelineInput }> {
         const { connector, companyIntegration } = await this.resolveConnector(companyIntegrationID, user);
 
-        const introspect = connector.IntrospectSchema.bind(connector) as
-            (ci: unknown, u: unknown) => Promise<SourceSchemaInfo>;
-        const sourceSchema = await introspect(companyIntegration, user);
+        // If the caller already ran IntrospectSchema (e.g. IntegrationApplyAllBatch),
+        // reuse it. The legacy path was running introspect TWICE per apply — once
+        // in the resolver and once here — which doubled probe time on connectors
+        // like Sage Intacct AND silently dropped selections when the second pass
+        // returned fewer objects than the first (rate limits, transient errors).
+        // The picked items would then fail to match `filteredSchema` below and
+        // get silently stripped before reaching buildTargetConfigs.
+        let sourceSchema: SourceSchemaInfo;
+        if (prefetchedSourceSchema) {
+            sourceSchema = prefetchedSourceSchema;
+        } else {
+            const introspect = connector.IntrospectSchema.bind(connector) as
+                (ci: unknown, u: unknown) => Promise<SourceSchemaInfo>;
+            sourceSchema = await introspect(companyIntegration, user);
+        }
 
         // Normalize names to match source schema casing
         const nameMap = new Map(sourceSchema.Objects.map(o => [o.ExternalName.toLowerCase(), o.ExternalName]));
@@ -3090,33 +3480,134 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
                 input.Connectors.map(async (connInput) => {
                     const { connector, companyIntegration } = await this.resolveConnector(connInput.CompanyIntegrationID, user);
                     const schemaName = this.deriveSchemaName(companyIntegration.Integration);
+                    console.log(
+                        `[IntegrationApplyAllBatch] connector=${companyIntegration.Integration} ` +
+                        `received ${connInput.SourceObjects.length} selections: ` +
+                        connInput.SourceObjects.map(so =>
+                            `{id=${so.SourceObjectID ?? '∅'}, name=${so.SourceObjectName ?? '∅'}}`
+                        ).slice(0, 30).join(', ') +
+                        (connInput.SourceObjects.length > 30 ? `, ... (+${connInput.SourceObjects.length - 30} more)` : '')
+                    );
+
+                    // Branch: Salesforce's full-catalog picker sends SourceObjectName
+                    // for freshly-discovered objects and uses the filtered describe
+                    // path to avoid a ~70s global describe on every apply. Other
+                    // connectors (HubSpot, YourMembership, etc.) retain the legacy
+                    // ID-only flow that describes and persists the entire schema.
+                    const useFilteredFlow = this.shouldUseFilteredIntrospection(connector, connInput.SourceObjects);
+
+                    let sourceSchema: SourceSchemaInfo;
+                    let resolvedNames: string[];
+                    const fieldsByName = new Map<string, string[] | null | undefined>();
+
+                    if (useFilteredFlow) {
+                        // Salesforce path — describe only selected objects, persist only those
+                        const selectionPlan = await this.resolveSelectionPlan(connInput.SourceObjects, user);
+                        const selectionNames = selectionPlan.map(p => p.Name);
+                        if (selectionNames.length === 0) {
+                            throw new Error('No source objects selected — every SourceObject must have either SourceObjectID or SourceObjectName set');
+                        }
 
-                    // Resolve object IDs to names with per-object Fields
-                    const sourceSchema = await (connector.IntrospectSchema.bind(connector) as
-                        (ci: unknown, u: unknown) => Promise<SourceSchemaInfo>)(companyIntegration, user);
-                    const objectIDs = connInput.SourceObjects.map(so => so.SourceObjectID);
-                    const resolvedNames = await this.resolveSourceObjectNames(objectIDs, undefined, sourceSchema, companyIntegration.IntegrationID, user);
+                        sourceSchema = await (connector.IntrospectSchema.bind(connector) as
+                            (ci: unknown, u: unknown, opts: { ObjectNames?: string[] }) => Promise<SourceSchemaInfo>)(
+                                companyIntegration, user, { ObjectNames: selectionNames }
+                            );
+
+                        try {
+                            const persistResult = await IntegrationSchemaSync.PersistDiscoveredSchema({
+                                IntegrationID: companyIntegration.IntegrationID,
+                                SourceSchema: sourceSchema,
+                                ContextUser: user,
+                            });
+                            console.log(
+                                `[IntegrationApplyAllBatch] Persisted describe for ${companyIntegration.Integration} (${selectionNames.length} selected): ` +
+                                `${persistResult.ObjectsCreated} new, ${persistResult.FieldsCreated} new fields, ` +
+                                `${persistResult.ObjectsUpdated} updated, ${persistResult.FieldsUpdated} updated fields`
+                            );
+                        } catch (persistErr) {
+                            LogError(`IntegrationApplyAllBatch: PersistDiscoveredSchema failed for ${companyIntegration.Integration}: ${persistErr}`);
+                        }
 
-                    const fieldsByID = new Map(connInput.SourceObjects.map(so => [so.SourceObjectID, so.Fields]));
-                    const objects = resolvedNames.map((name, i) => {
+                        resolvedNames = this.normalizeNamesAgainstSchema(selectionNames, sourceSchema);
+                        for (const p of selectionPlan) {
+                            fieldsByName.set(p.Name.toLowerCase(), p.Fields);
+                        }
+                    } else {
+                        // Legacy path (HubSpot, YourMembership, Sage Intacct, etc.)
+                        // — describe all, persist all, then resolve by either
+                        // SourceObjectID (legacy clients) OR SourceObjectName
+                        // (newly-discovered objects from connectors that probe
+                        // their full catalog at picker time, e.g. SI's 666
+                        // candidates). Without this fallback, freshly-probed
+                        // selections silently drop.
+                        sourceSchema = await (connector.IntrospectSchema.bind(connector) as
+                            (ci: unknown, u: unknown) => Promise<SourceSchemaInfo>)(companyIntegration, user);
+
+                        try {
+                            const persistResult = await IntegrationSchemaSync.PersistDiscoveredSchema({
+                                IntegrationID: companyIntegration.IntegrationID,
+                                SourceSchema: sourceSchema,
+                                ContextUser: user,
+                            });
+                            console.log(
+                                `[IntegrationApplyAllBatch] Persisted discovered schema for ${companyIntegration.Integration}: ` +
+                                `${persistResult.ObjectsCreated} new objects, ${persistResult.FieldsCreated} new fields, ` +
+                                `${persistResult.ObjectsUpdated} updated objects, ${persistResult.FieldsUpdated} updated fields`
+                            );
+                        } catch (persistErr) {
+                            LogError(`IntegrationApplyAllBatch: PersistDiscoveredSchema failed for ${companyIntegration.Integration}: ${persistErr}`);
+                        }
+
+                        // Resolve names from BOTH ID lookups and direct names.
+                        // Direct names skip the IntegrationObject DB roundtrip
+                        // since the selection plan already has the API code.
+                        const idsOnly = connInput.SourceObjects.map(so => so.SourceObjectID).filter((x): x is string => !!x);
+                        const directNames = connInput.SourceObjects.map(so => so.SourceObjectName).filter((x): x is string => !!x);
+                        const namesFromIds = idsOnly.length > 0
+                            ? await this.resolveSourceObjectNames(idsOnly, undefined, sourceSchema, companyIntegration.IntegrationID, user)
+                            : [];
+                        const normalizedDirect = this.normalizeNamesAgainstSchema(directNames, sourceSchema);
+
+                        // Preserve original picker order while deduping
+                        const seen = new Set<string>();
+                        resolvedNames = [];
+                        const orderedSources: SourceObjectInput[] = [];
+                        let idCursor = 0;
+                        let nameCursor = 0;
+                        for (const so of connInput.SourceObjects) {
+                            const resolved = so.SourceObjectName
+                                ? normalizedDirect[nameCursor++]
+                                : (so.SourceObjectID ? namesFromIds[idCursor++] : undefined);
+                            if (!resolved) continue;
+                            const key = resolved.toLowerCase();
+                            if (seen.has(key)) continue;
+                            seen.add(key);
+                            resolvedNames.push(resolved);
+                            orderedSources.push(so);
+                        }
+                        for (let i = 0; i < resolvedNames.length; i++) {
+                            fieldsByName.set(resolvedNames[i].toLowerCase(), orderedSources[i].Fields);
+                        }
+                    }
+
+                    const objects = resolvedNames.map(name => {
                         const obj = new SchemaPreviewObjectInput();
                         obj.SourceObjectName = name;
                         obj.SchemaName = schemaName;
                         obj.TableName = name.replace(/[^A-Za-z0-9_]/g, '_');
                         obj.EntityName = name.replace(/([a-z])([A-Z])/g, '$1 $2').replace(/_/g, ' ');
-                        obj.Fields = fieldsByID.get(objectIDs[i]) ?? undefined;
+                        obj.Fields = fieldsByName.get(name.toLowerCase()) ?? undefined;
                         return obj;
                     });
 
                     const { schemaOutput, rsuInput } = await this.buildSchemaForConnector(
-                        connInput.CompanyIntegrationID, objects, validatedPlatform, user, skipGitCommit, skipRestart
+                        connInput.CompanyIntegrationID, objects, validatedPlatform, user, skipGitCommit, skipRestart, sourceSchema
                     );
 
                     // Build per-object field map for pending file
                     const sourceObjectFields: Record<string, string[] | null> = {};
-                    for (const so of connInput.SourceObjects) {
-                        const resolvedName = resolvedNames[objectIDs.indexOf(so.SourceObjectID)];
-                        if (resolvedName) sourceObjectFields[resolvedName] = so.Fields ?? null;
+                    for (const name of resolvedNames) {
+                        sourceObjectFields[name] = fieldsByName.get(name.toLowerCase()) ?? null;
                     }
 
                     // Inject post-restart pending work payload
@@ -3238,7 +3729,16 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
 
                     const createdMapIDs = entityMapsCreated.map(em => em.EntityMapID).filter(Boolean);
                     const scopedMapIDs = input.SyncScope === 'all' ? undefined : createdMapIDs;
-                    const syncRunID = input.StartSync !== false
+
+                    // Skip sync entirely when SyncScope='created' (default) but
+                    // no new maps were created. Otherwise the engine sees an
+                    // empty EntityMapIDs array, falls through its `length > 0`
+                    // gate, and runs a FULL integration sync — silently re-
+                    // pulling every existing map. That's why a 0-new-map apply
+                    // could trigger a 459-record sync against the 71 existing.
+                    const shouldStartSync = input.StartSync !== false &&
+                        (input.SyncScope === 'all' || createdMapIDs.length > 0);
+                    const syncRunID = shouldStartSync
                         ? await this.startSyncAfterApply(build.connInput.CompanyIntegrationID, user, scopedMapIDs, input.FullSync)
                         : null;
                     if (syncRunID) connResult.SyncRunID = syncRunID;