@memberjunction/integration-connectors 5.29.0 → 5.30.0

package/dist/SalesforceConnector.js

@@ -4,9 +4,11 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
     else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
+var SalesforceConnector_1;
 import jwt from 'jsonwebtoken';
 import { RegisterClass } from '@memberjunction/global';
 import { Metadata } from '@memberjunction/core';
+import { IntegrationEngineBase } from '@memberjunction/integration-engine-base';
 import { BaseIntegrationConnector, BaseRESTIntegrationConnector, } from '@memberjunction/integration-engine';
 // ─── Constants ────────────────────────────────────────────────────────
 const DEFAULT_API_VERSION = '61.0';
@@ -30,6 +32,38 @@ const SYSTEM_READ_ONLY_FIELDS = new Set([
 ]);
 /** Fields always included in SOQL SELECT */
 const REQUIRED_SOQL_FIELDS = ['Id', 'SystemModstamp', 'IsDeleted', 'LastModifiedById'];
+/**
+ * Objects that SF exposes through the main `/sobjects/` endpoint but that only
+ * function correctly through the Tooling API. The Data API describes them as
+ * `queryable=true` so they'd otherwise land in the picker, but syncing them
+ * produces one or more of: duplicate-key violations (pagination returns the
+ * same record across pages), sentinel `000000000000000AAA` IDs, or
+ * `MALFORMED_QUERY` on fields like `Metadata`/`FullName` that SF permits only
+ * one-row-at-a-time in Tooling queries.
+ *
+ * Observed causing errors in production syncs; blacklisting here eliminates
+ * ~960 errors per cold apply and ~10 per incremental without any value loss
+ * (these are metadata/telemetry tables, not business data).
+ */
+const TOOLING_API_DENYLIST = new Set([
+    'EntityDefinition',
+    'DataType',
+    'AuraDefinitionInfo',
+    'AuraDefinitionBundleInfo',
+    'FormulaFunction',
+    'AppDefinition',
+    'UserSetupEntityAccess',
+    'PlatformEventUsageMetric',
+    'EventBusSubscriber',
+    'ApexClass',
+    'ApexPage',
+    'ApexTrigger',
+    'ApexComponent',
+    'Publisher',
+    'ExternalString',
+    'CustomHttpHeader',
+    'FormulaFunctionCategory',
+]);
 // ─── Salesforce Object Metadata for Action Generation ─────────────────
 const SALESFORCE_OBJECTS = [
     {
@@ -183,11 +217,10 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
         // ── Rate limit / governor tracking ──────────────────────────────
         this.lastRequestTime = 0;
         this.governorState = { CurrentUsage: 0, DailyLimit: 15000, LastUpdated: 0 };
-        // ── Pagination state for SOQL queryMore ─────────────────────────
-        this.queryLocator = null;
         // ── Config ──────────────────────────────────────────────────────
         this._config = null;
     }
+    static { SalesforceConnector_1 = this; }
     get effectiveMaxRetries() { return this._config?.MaxRetries ?? DEFAULT_MAX_RETRIES; }
     get effectiveRequestTimeoutMs() { return this._config?.RequestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; }
     get effectiveMinRequestIntervalMs() { return this._config?.MinRequestIntervalMs ?? DEFAULT_MIN_REQUEST_INTERVAL_MS; }
@@ -236,7 +269,10 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
     // ─── Discovery (Live from SF Describe API) ───────────────────────
     /**
      * Discovers available objects by calling the SF SObjects API.
-     * Returns all queryable objects including custom objects (__c).
+     * Returns user-relevant queryable objects: standard CRM objects + custom
+     * (__c). Filters out audit/system noise (~1,866 → ~150-300 typically).
+     * Set MJ_SALESFORCE_INCLUDE_ALL_SOBJECTS=true to bypass the filter and
+     * return the full catalog (useful for debugging or unusual integrations).
      */
     async DiscoverObjects(companyIntegration, contextUser) {
         const auth = await this.Authenticate(companyIntegration, contextUser);
@@ -246,9 +282,21 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
         this.ValidateResponse(response, url);
         const body = response.Body;
         const sobjects = body.sobjects ?? [];
-        return sobjects
-            .filter(obj => obj.queryable)
-            .map(obj => ({
+        const includeAll = process.env.MJ_SALESFORCE_INCLUDE_ALL_SOBJECTS === 'true';
+        const filtered = sobjects.filter(obj => {
+            if (!obj.queryable)
+                return false;
+            if (TOOLING_API_DENYLIST.has(obj.name))
+                return false;
+            if (includeAll)
+                return true;
+            return this.isUserRelevantSObject(obj);
+        });
+        if (!includeAll) {
+            const removed = sobjects.length - filtered.length;
+            console.log(`[Salesforce] DiscoverObjects: filtered ${sobjects.length} → ${filtered.length} (excluded ${removed} system/audit objects; set MJ_SALESFORCE_INCLUDE_ALL_SOBJECTS=true to bypass)`);
+        }
+        return filtered.map(obj => ({
             Name: obj.name,
             Label: obj.label,
             Description: obj.custom ? `Custom object: ${obj.label}` : undefined,
@@ -256,6 +304,61 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
             SupportsWrite: obj.createable || obj.updateable,
         }));
     }
+    /**
+     * Heuristic for "user-relevant" SF object: customer data (custom objects
+     * with __c suffix) OR createable standard CRM objects, excluding audit
+     * tables, system metadata, and managed-package telemetry.
+     *
+     * Standard SF orgs return ~1,866 sobjects from describeGlobal — most are
+     * audit/internal noise that don't represent business data the user wants
+     * to sync into MJ. This drops them to ~150-300.
+     */
+    isUserRelevantSObject(obj) {
+        // Custom objects always pass — they're customer-defined data
+        if (obj.custom)
+            return true;
+        const name = obj.name;
+        // STRICT exclusions only. Don't filter anything that could possibly
+        // hold customer data. Each exclusion below is a category SF defines
+        // as pure audit/internal/metadata with no customer-meaningful rows:
+        //
+        //   *ChangeEvent: CDC stream, transient, replicated by replication API
+        //   *Feed:        Chatter feed entries (separate "Feed" sync if wanted)
+        //   *History:     audit tables — every value-change is one row
+        //   *FieldHistory: same as *History but per-field tracking
+        //   *Share:       SF row-level access control entries (security metadata)
+        //   *OwnerSharingRule / *CriteriaBasedSharingRule: security rules
+        //   *PermissionSet*: profile/permission internals
+        //
+        // Things we used to exclude that we now LET THROUGH because they CAN
+        // be customer data: EmailMessage (email log), CaseComment / FeedComment
+        // (customer interactions), EntitySubscription (notification subs),
+        // Vote (idea/feedback), Tag (content tagging), Solution (knowledge),
+        // ProcessInstance (workflow approvals), Domain (tenant config).
+        if (/(?:ChangeEvent|Feed|History|FieldHistory|Share|OwnerSharingRule|CriteriaBasedSharingRule)$/.test(name)) {
+            return false;
+        }
+        // SF tooling/setup metadata: Apex code, permissions, setup audit, login
+        // history, async job framework, sandbox/cron internals, network/site
+        // metadata, theme/branding, package licenses. None of these are
+        // business data the integration should sync.
+        if (/^(Apex|Permission|Setup|Login|Async|Sandbox|Auth|Network|Stamp|Site|FlowDefinition|FlowInterview|FlowVariableView|EventLog|CronTrigger|StreamingChannel|InstalledMobileApp|UserPackageLicense|PackageLicense|Theme)/.test(name)) {
+            return false;
+        }
+        // Specific system catalog objects — schema-of-the-schema metadata,
+        // not customer data. These are readable but represent SF's own
+        // structural definitions, never user-entered records.
+        if (/^(EntityDefinition|FieldDefinition|EntityParticle|RelationshipInfo|RelationshipDomain|StandardAction|UserAppMenuItem|UserListView|UserPreference|UserShare|GroupMember|FiscalYearSettings|Period|RecordType|BusinessProcess|PicklistValueInfo)$/.test(name)) {
+            return false;
+        }
+        // NOTE: we intentionally do NOT exclude `!obj.createable`. Many SF
+        // objects are flagged non-createable because they're auto-populated
+        // by SF (rollups, attachment-link junctions, history-style records)
+        // but they DO carry real customer data we want to sync. The
+        // targeted exclusions above already cover the audit/security/CDC
+        // categories that have no business value.
+        return true;
+    }
     /**
      * Discovers fields on a specific SF object via the Describe API.
      * Skips compound fields (address, location) — their component fields are included individually.
@@ -274,50 +377,208 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
     }
     /**
      * Full schema introspection — builds object graph with relationships.
+     *
+     * If `options.ObjectNames` is provided, only those objects are described
+     * (fast path for user-selected subsets). Without a filter, every
+     * queryable sobject is described — ~70s even with parallelism on a
+     * large org — and the result is cached per-org for 5 minutes so
+     * back-to-back resolver calls don't re-describe.
+     *
+     * Filtered calls bypass the cache on purpose: the subset is typically
+     * small and cheap, and the full-schema cache would shadow newer results
+     * if a user selects a previously-unknown object.
      */
-    async IntrospectSchema(companyIntegration, contextUser) {
-        const objects = await this.DiscoverObjects(companyIntegration, contextUser);
+    async IntrospectSchema(companyIntegration, contextUser, options) {
+        const filtered = options?.ObjectNames && options.ObjectNames.length > 0;
+        if (filtered) {
+            return this.DoIntrospectSchema(companyIntegration, contextUser, options);
+        }
+        // Dedupe back-to-back introspection requests from the same org.
+        // Three separate resolvers call IntrospectSchema; without this, the
+        // UI's "discover then apply" flow re-describes every object twice.
+        const cacheKey = companyIntegration.ID;
+        const cached = SalesforceConnector_1.introspectCache.get(cacheKey);
+        const now = Date.now();
+        if (cached && cached.expiresAt > now) {
+            const remainingSec = Math.round((cached.expiresAt - now) / 1000);
+            console.log(`[Salesforce] IntrospectSchema: reusing in-flight/cached result (${remainingSec}s TTL remaining)`);
+            return cached.promise;
+        }
+        const promise = this.DoIntrospectSchema(companyIntegration, contextUser);
+        SalesforceConnector_1.introspectCache.set(cacheKey, {
+            promise,
+            expiresAt: now + SalesforceConnector_1.INTROSPECT_CACHE_TTL_MS,
+        });
+        // If the run fails, evict so the next request can retry
+        promise.catch(() => SalesforceConnector_1.introspectCache.delete(cacheKey));
+        return promise;
+    }
+    async DoIntrospectSchema(companyIntegration, contextUser, options) {
+        const allObjects = await this.DiscoverObjects(companyIntegration, contextUser);
+        const wanted = options?.ObjectNames && options.ObjectNames.length > 0
+            ? new Set(options.ObjectNames)
+            : null;
+        const objects = wanted ? allObjects.filter(o => wanted.has(o.Name)) : allObjects;
+        const total = objects.length;
+        const startMs = Date.now();
+        const CONCURRENCY = 8;
+        console.log(`[Salesforce] IntrospectSchema: describing ${total} queryable objects (parallel×${CONCURRENCY})...`);
         const result = { Objects: [] };
-        for (const obj of objects) {
-            try {
-                const sourceObj = await this.BuildSourceObjectFromDescribe(companyIntegration, contextUser, obj.Name);
-                result.Objects.push(sourceObj);
-            }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                console.warn(`[Salesforce] Skipping "${obj.Name}" during introspection: ${msg}`);
+        let nextIdx = 0;
+        let succeeded = 0;
+        let skipped = 0;
+        const worker = async () => {
+            while (true) {
+                const myIdx = nextIdx++;
+                if (myIdx >= total)
+                    return;
+                const obj = objects[myIdx];
+                try {
+                    const sourceObj = await this.BuildSourceObjectFromDescribe(companyIntegration, contextUser, obj.Name);
+                    result.Objects.push(sourceObj);
+                    succeeded++;
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    skipped++;
+                    console.warn(`[Salesforce] Skipping "${obj.Name}" during introspection: ${msg}`);
+                }
+                const done = succeeded + skipped;
+                if (done % 100 === 0 || done === total) {
+                    const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1);
+                    const etaSec = done < total
+                        ? (((Date.now() - startMs) / done) * (total - done) / 1000).toFixed(0)
+                        : '0';
+                    console.log(`[Salesforce] IntrospectSchema progress: ${done}/${total} (ok=${succeeded}, skipped=${skipped}) — ${elapsedSec}s elapsed, ~${etaSec}s remaining`);
+                }
             }
-        }
+        };
+        await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
+        console.log(`[Salesforce] IntrospectSchema complete: ${succeeded}/${total} objects in ${((Date.now() - startMs) / 1000).toFixed(1)}s`);
         return result;
     }
+    // ─── Introspection cache (module-scoped via static) ──────────────
+    // Resolver creates a fresh connector per request, so instance-level
+    // caching is useless. Static Map survives across requests in the same
+    // process. TTL is modest to allow schema refresh on demand.
+    static { this.INTROSPECT_CACHE_TTL_MS = 5 * 60 * 1000; }
+    static { this.introspectCache = new Map(); }
     // ─── FetchChanges (SOQL-based — overrides base class entirely) ───
     /**
      * Fetches changed records using SOQL queries with SystemModstamp watermarks.
      * Completely overrides the base class REST pagination because SF uses
      * SOQL + queryMore, not standard REST list endpoints.
+     *
+     * Dispatches to family-specific fetch routines based on the
+     * IntegrationObject metadata's `DefaultQueryParams.api_family` hint:
+     * tooling → Tooling API, analytics_report/dashboard → Analytics API,
+     * bulk_* / composite → not intended for listing (returns empty batch),
+     * everything else → standard SObject SOQL flow.
      */
     async FetchChanges(ctx) {
         const auth = await this.Authenticate(ctx.CompanyIntegration, ctx.ContextUser);
         const batchSize = ctx.BatchSize || this.effectiveBatchSize;
-        // If we have a queryLocator from a previous call, use queryMore
-        if (this.queryLocator && ctx.CurrentCursor) {
+        const family = this.ResolveAPIFamily(ctx.CompanyIntegration.IntegrationID, ctx.ObjectName);
+        // Continuation: SF returned a nextRecordsUrl on the previous call,
+        // and the engine passed it back as CurrentCursor. Re-issue against
+        // the same query state via /query/<locator> rather than re-running
+        // the original SOQL (which would just return the first page again
+        // and silently truncate the dataset to one batch). Pre-existing bug:
+        // a dead `this.queryLocator` member shadowed this branch with an
+        // always-false condition, so every "next batch" call re-executed
+        // the initial SOQL and produced duplicate first-page results until
+        // the engine's duplicate-batch guard aborted the entity.
+        if (ctx.CurrentCursor) {
             return this.FetchNextPage(auth, ctx.CurrentCursor);
         }
-        // Build and execute SOQL query
-        const fields = await this.GetQueryableFieldNames(auth, ctx.ObjectName);
+        if (family === 'analytics_report' || family === 'analytics_dashboard') {
+            return this.FetchAnalyticsList(auth, family, ctx.ObjectName);
+        }
+        if (family === 'bulk_ingest' || family === 'bulk_query') {
+            return this.FetchBulkJobs(auth, family, ctx.ObjectName);
+        }
+        if (family === 'composite') {
+            // Composite is a per-request construct, not a queryable endpoint
+            return { Records: [], HasMore: false };
+        }
+        if (family === 'knowledge') {
+            const page = ctx.CurrentPage ?? 1;
+            return this.FetchKnowledgeArticles(auth, ctx.ObjectName, ctx.WatermarkValue, batchSize, page);
+        }
+        // Standard SObject (sobject) and Tooling (tooling) both use SOQL —
+        // just against different endpoints.
+        const fields = await this.GetQueryableFieldNames(auth, ctx.ObjectName, family);
         const soql = this.BuildSOQLQuery(ctx.ObjectName, fields, ctx.WatermarkValue, batchSize, true);
-        return this.ExecuteSOQLQuery(auth, soql, ctx.ObjectName);
+        return this.ExecuteSOQLQuery(auth, soql, ctx.ObjectName, family);
+    }
+    /**
+     * Resolves the API family for a given IntegrationObject by reading its
+     * `DefaultQueryParams.api_family` flag from the engine cache. Defaults to
+     * `sobject` when no metadata is available (common in unit tests).
+     */
+    ResolveAPIFamily(integrationID, objectName) {
+        try {
+            const obj = IntegrationEngineBase.Instance.GetIntegrationObject(integrationID, objectName);
+            if (!obj || !obj.DefaultQueryParams)
+                return 'sobject';
+            const parsed = JSON.parse(obj.DefaultQueryParams);
+            if (this.IsValidFamily(parsed.api_family))
+                return parsed.api_family;
+            return 'sobject';
+        }
+        catch {
+            return 'sobject';
+        }
+    }
+    IsValidFamily(v) {
+        return v === 'sobject' || v === 'tooling' || v === 'knowledge'
+            || v === 'analytics_report' || v === 'analytics_dashboard'
+            || v === 'bulk_ingest' || v === 'bulk_query' || v === 'composite';
+    }
+    /**
+     * Returns the REST path prefix for an SObject in the given API family.
+     * Used by CRUD operations and field discovery.
+     */
+    SObjectBasePath(auth, family) {
+        const base = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}`;
+        return family === 'tooling' ? `${base}/tooling/sobjects` : `${base}/sobjects`;
+    }
+    /**
+     * Returns the SOQL query endpoint for the given API family.
+     * Standard uses /query + /queryAll; Tooling uses /tooling/query.
+     */
+    SOQLEndpoint(auth, family, includeDeleted) {
+        const base = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}`;
+        if (family === 'tooling')
+            return `${base}/tooling/query`;
+        return includeDeleted ? `${base}/queryAll` : `${base}/query`;
     }
     // ─── CRUD Operations ─────────────────────────────────────────────
     /**
-     * Creates a new record in Salesforce.
+     * Creates a new record in Salesforce. Dispatches to the appropriate API
+     * family endpoint (standard SObjects, Tooling, Bulk Ingest/Query jobs, or
+     * Composite) based on the IntegrationObject metadata.
      */
     async CreateRecord(ctx) {
         const companyIntegration = ctx.CompanyIntegration;
         const contextUser = ctx.ContextUser;
         const auth = await this.Authenticate(companyIntegration, contextUser);
+        const family = this.ResolveAPIFamily(companyIntegration.IntegrationID, ctx.ObjectName);
+        if (family === 'bulk_ingest' || family === 'bulk_query') {
+            return this.CreateBulkJob(auth, family, ctx.Attributes);
+        }
+        if (family === 'composite') {
+            return this.ExecuteCompositeRequest(auth, ctx.Attributes);
+        }
+        if (family === 'analytics_report' || family === 'analytics_dashboard' || family === 'knowledge') {
+            return {
+                Success: false,
+                ErrorMessage: `[Salesforce] Create is not supported for API family "${family}" (${ctx.ObjectName})`,
+                StatusCode: 405,
+            };
+        }
         const headers = { ...this.BuildHeaders(auth), 'Content-Type': 'application/json' };
-        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/sobjects/${ctx.ObjectName}/`;
+        const url = `${this.SObjectBasePath(auth, family)}/${ctx.ObjectName}/`;
         const body = this.StripReadOnlyFields(ctx.Attributes);
         const response = await this.MakeHTTPRequest(auth, url, 'POST', headers, body);
         if (response.Status === 201 || (response.Status >= 200 && response.Status < 300)) {
@@ -332,13 +593,23 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
     }
     /**
      * Updates an existing record in Salesforce (PATCH — only changed fields).
+     * Honors API family routing; Tooling uses `/tooling/sobjects/`, everything
+     * else uses standard `/sobjects/`.
      */
     async UpdateRecord(ctx) {
         const companyIntegration = ctx.CompanyIntegration;
         const contextUser = ctx.ContextUser;
         const auth = await this.Authenticate(companyIntegration, contextUser);
+        const family = this.ResolveAPIFamily(companyIntegration.IntegrationID, ctx.ObjectName);
+        if (family !== 'sobject' && family !== 'tooling') {
+            return {
+                Success: false,
+                ErrorMessage: `[Salesforce] Update is not supported for API family "${family}" (${ctx.ObjectName})`,
+                StatusCode: 405,
+            };
+        }
         const headers = { ...this.BuildHeaders(auth), 'Content-Type': 'application/json' };
-        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/sobjects/${ctx.ObjectName}/${ctx.ExternalID}`;
+        const url = `${this.SObjectBasePath(auth, family)}/${ctx.ObjectName}/${ctx.ExternalID}`;
         const body = this.StripReadOnlyFields(ctx.Attributes);
         const response = await this.MakeHTTPRequest(auth, url, 'PATCH', headers, body);
         if (response.Status === 204 || (response.Status >= 200 && response.Status < 300)) {
@@ -351,14 +622,28 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
         return this.HandleWriteError(response, 'UpdateRecord', ctx.ObjectName, ctx.ExternalID, auth, headers, url, body);
     }
     /**
-     * Deletes a record from Salesforce.
+     * Deletes a record from Salesforce. For Bulk Job families, delete aborts
+     * the job. For standard and Tooling SObjects, performs a soft delete
+     * (Recycle Bin, 15-day retention). Analytics/Composite/Knowledge do not
+     * support deletion through this path.
      */
     async DeleteRecord(ctx) {
         const companyIntegration = ctx.CompanyIntegration;
         const contextUser = ctx.ContextUser;
         const auth = await this.Authenticate(companyIntegration, contextUser);
+        const family = this.ResolveAPIFamily(companyIntegration.IntegrationID, ctx.ObjectName);
+        if (family === 'bulk_ingest' || family === 'bulk_query') {
+            return this.AbortBulkJob(auth, family, ctx.ExternalID);
+        }
+        if (family !== 'sobject' && family !== 'tooling') {
+            return {
+                Success: false,
+                ErrorMessage: `[Salesforce] Delete is not supported for API family "${family}" (${ctx.ObjectName})`,
+                StatusCode: 405,
+            };
+        }
         const headers = this.BuildHeaders(auth);
-        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/sobjects/${ctx.ObjectName}/${ctx.ExternalID}`;
+        const url = `${this.SObjectBasePath(auth, family)}/${ctx.ObjectName}/${ctx.ExternalID}`;
         const response = await this.MakeHTTPRequest(auth, url, 'DELETE', headers);
         if (response.Status === 204 || (response.Status >= 200 && response.Status < 300)) {
             return { Success: true, ExternalID: ctx.ExternalID, StatusCode: response.Status };
@@ -370,14 +655,28 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
         return this.BuildCRUDError(response, 'DeleteRecord', ctx.ObjectName);
     }
     /**
-     * Retrieves a single record by its Salesforce ID.
+     * Retrieves a single record by its Salesforce ID. Routes to the API family
+     * endpoint indicated by IntegrationObject metadata.
      */
     async GetRecord(ctx) {
         const companyIntegration = ctx.CompanyIntegration;
         const contextUser = ctx.ContextUser;
         const auth = await this.Authenticate(companyIntegration, contextUser);
+        const family = this.ResolveAPIFamily(companyIntegration.IntegrationID, ctx.ObjectName);
+        if (family === 'analytics_report') {
+            return this.GetAnalyticsReport(auth, ctx.ExternalID);
+        }
+        if (family === 'analytics_dashboard') {
+            return this.GetAnalyticsDashboard(auth, ctx.ExternalID);
+        }
+        if (family === 'bulk_ingest' || family === 'bulk_query') {
+            return this.GetBulkJob(auth, family, ctx.ExternalID);
+        }
+        if (family === 'composite') {
+            return null;
+        }
         const headers = this.BuildHeaders(auth);
-        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/sobjects/${ctx.ObjectName}/${ctx.ExternalID}`;
+        const url = `${this.SObjectBasePath(auth, family)}/${ctx.ObjectName}/${ctx.ExternalID}`;
         const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
         if (response.Status === 404)
             return null;
@@ -388,12 +687,18 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
     // ─── Search (SOQL) ───────────────────────────────────────────────
     /**
      * Searches records using SOQL WHERE clauses built from the provided filters.
+     * Search is supported for Standard SObjects and Tooling SObjects; other API
+     * families return an empty result.
      */
     async SearchRecords(ctx) {
         const companyIntegration = ctx.CompanyIntegration;
         const contextUser = ctx.ContextUser;
         const auth = await this.Authenticate(companyIntegration, contextUser);
-        const fields = await this.GetQueryableFieldNames(auth, ctx.ObjectName);
+        const family = this.ResolveAPIFamily(companyIntegration.IntegrationID, ctx.ObjectName);
+        if (family !== 'sobject' && family !== 'tooling') {
+            return { Records: [], TotalCount: 0, HasMore: false };
+        }
+        const fields = await this.GetQueryableFieldNames(auth, ctx.ObjectName, family);
         const whereClause = this.BuildWhereClauseFromFilters(ctx.Filters);
         const limit = ctx.PageSize ?? 100;
         const offset = ctx.Page != null && ctx.Page > 1 ? (ctx.Page - 1) * limit : 0;
@@ -405,7 +710,7 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
         soql += ` LIMIT ${limit}`;
         if (offset > 0)
             soql += ` OFFSET ${offset}`;
-        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/query?q=${encodeURIComponent(soql)}`;
+        const url = `${this.SOQLEndpoint(auth, family, false)}?q=${encodeURIComponent(soql)}`;
         const headers = this.BuildHeaders(auth);
         const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
         this.ValidateResponse(response, url);
@@ -548,21 +853,60 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
      * Builds a SOQL query string with optional watermark filter.
      */
     BuildSOQLQuery(objectName, fields, watermarkValue, batchSize, includeDeleted) {
-        const dedupedFields = [...new Set([...REQUIRED_SOQL_FIELDS, ...fields])];
+        // Many SF system/tooling/meta objects lack one or more of the "standard"
+        // audit fields (SystemModstamp, IsDeleted, LastModifiedById). Assuming
+        // they're always present produces `INVALID_FIELD` errors. Build the
+        // SELECT from what the object's describe actually exposes.
+        const available = new Set(fields);
+        const requiredPresent = REQUIRED_SOQL_FIELDS.filter(f => available.has(f));
+        const dedupedFields = [...new Set([...requiredPresent, ...fields])];
+        // Pick the best available watermark/ordering column.
+        // Preference: SystemModstamp > LastModifiedDate > CreatedDate > (none)
+        const watermarkCol = available.has('SystemModstamp')
+            ? 'SystemModstamp'
+            : available.has('LastModifiedDate')
+                ? 'LastModifiedDate'
+                : available.has('CreatedDate')
+                    ? 'CreatedDate'
+                    : null;
         let soql = `SELECT ${dedupedFields.join(', ')} FROM ${objectName}`;
-        if (watermarkValue) {
-            soql += ` WHERE SystemModstamp > ${this.FormatSOQLDateTime(watermarkValue)}`;
+        if (watermarkValue && watermarkCol) {
+            // `>=` not `>` — strict greater-than misses records modified
+            // at exactly the watermark instant. SF's SystemModstamp has
+            // millisecond precision but bulk updates can produce multiple
+            // records with the identical modstamp; only the last one shapes
+            // the saved watermark. Without `>=`, any record colliding on
+            // that exact ms after watermark save is permanently dropped
+            // (the watermark advances past it next sync). Engine dedupe
+            // handles the cheap cost of re-fetching boundary records.
+            soql += ` WHERE ${watermarkCol} >= ${this.FormatSOQLDateTime(watermarkValue)}`;
+        }
+        if (watermarkCol) {
+            soql += ` ORDER BY ${watermarkCol} ASC`;
         }
-        soql += ' ORDER BY SystemModstamp ASC';
-        soql += ` LIMIT ${batchSize}`;
+        // NO `LIMIT batchSize`. SF's REST API natively paginates the result
+        // via `done` / `nextRecordsUrl` — the engine loop drives subsequent
+        // pages. A SOQL LIMIT here would cap the ENTIRE result set at
+        // batchSize records and SF would (correctly) report done=true at
+        // that count, silently dropping every record past the limit. The
+        // worst part: incremental syncs advance the watermark past the
+        // dropped records, so they're never re-fetched on subsequent runs.
+        // Per-page batch size is controlled by the `Sforce-Query-Options`
+        // header (or SF default) and does not need a SOQL LIMIT.
+        // _ = batchSize  // intentionally unused — preserved param for API stability
+        void batchSize;
         return soql;
     }
     /**
      * Executes a SOQL query and returns records as a FetchBatchResult.
+     * For the standard SObject family this uses `queryAll` so that soft-deleted
+     * records are returned (IsDeleted=true). For the Tooling family, the
+     * Tooling API's `/tooling/query` endpoint is used — `queryAll` is not
+     * supported there.
      */
-    async ExecuteSOQLQuery(auth, soql, objectName) {
-        // Use queryAll to include deleted records (IsDeleted=true)
-        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/queryAll?q=${encodeURIComponent(soql)}`;
+    async ExecuteSOQLQuery(auth, soql, objectName, family = 'sobject') {
+        const includeDeleted = family === 'sobject';
+        const url = `${this.SOQLEndpoint(auth, family, includeDeleted)}?q=${encodeURIComponent(soql)}`;
         const headers = this.BuildHeaders(auth);
         const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
         this.ValidateResponse(response, url);
@@ -588,7 +932,35 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
      * Parses a SOQL query response into a FetchBatchResult.
      */
     ParseSOQLResponse(body, objectName) {
-        const records = (body.records ?? []).map(r => {
+        const rawRecords = body.records ?? [];
+        // Some Salesforce system/metadata objects (e.g. TabDefinition,
+        // FormulaFunctionAllowedType) return many rows that all share the
+        // placeholder `Id = '000000000000000AAA'` — SF treats Id as
+        // non-unique on those objects, but MJ's auto-generated UQ_<table>_PK
+        // rejects the duplicates and produces one error per record after
+        // the first. Dedupe on Id within the batch (keep first occurrence)
+        // before returning to the engine.
+        const seenIds = new Set();
+        const dedupedRaw = [];
+        let duplicatesDropped = 0;
+        for (const r of rawRecords) {
+            const raw = r;
+            const id = raw['Id'];
+            if (id) {
+                if (seenIds.has(id)) {
+                    duplicatesDropped++;
+                    continue;
+                }
+                seenIds.add(id);
+            }
+            dedupedRaw.push(r);
+        }
+        if (duplicatesDropped > 0) {
+            console.warn(`[Salesforce] ${objectName}: dropped ${duplicatesDropped} record(s) with duplicate Id ` +
+                `(SF returned non-unique Ids for this object — typical for system/metadata sObjects ` +
+                `like TabDefinition, FormulaFunctionAllowedType where Id is a placeholder).`);
+        }
+        const records = dedupedRaw.map(r => {
             const raw = r;
             const record = this.RawToExternalRecord(raw, objectName);
             record.IsDeleted = raw['IsDeleted'] === true;
@@ -598,7 +970,7 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
             return record;
         });
         // Compute new watermark from the max SystemModstamp in this batch
-        const newWatermark = this.ExtractMaxWatermark(body.records ?? []);
+        const newWatermark = this.ExtractMaxWatermark(dedupedRaw);
         return {
             Records: records,
             HasMore: !body.done,
@@ -607,7 +979,23 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
         };
     }
     /**
-     * Extracts the maximum SystemModstamp value from a batch of records.
+     * Extracts the maximum SystemModstamp value from a batch of records and
+     * advances it by 1ms so the next sync's `WHERE SystemModstamp >= <wm>`
+     * filter excludes the boundary cluster.
+     *
+     * Why advance: the SOQL filter uses `>=` (not `>`) on purpose so we don't
+     * drop records that share the modstamp instant of the previous max. But
+     * if the saved watermark equals the new max, every subsequent incremental
+     * re-pulls the same boundary cluster forever and the watermark plateaus.
+     * Observed in the wild: 1,800 EmailMessage records bulk-imported with
+     * identical SystemModstamp re-fetched on every run with `Updated` count
+     * never decreasing.
+     *
+     * SF's SystemModstamp is millisecond-precision, so adding 1ms cannot
+     * skip a real record — there is nothing scheduled between `max` and
+     * `max + 1ms`. The connector returns the advanced value as
+     * `NewWatermarkValue`; the engine persists it; the next run picks up
+     * cleanly past the cluster.
      */
     ExtractMaxWatermark(records) {
         let maxTimestamp = null;
@@ -618,13 +1006,20 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
                 maxTimestamp = stamp;
             }
         }
-        return maxTimestamp;
+        if (!maxTimestamp)
+            return null;
+        const parsed = new Date(maxTimestamp);
+        if (Number.isNaN(parsed.getTime()))
+            return maxTimestamp;
+        return new Date(parsed.getTime() + 1).toISOString();
     }
     /**
-     * Gets queryable field names for a SF object via the Describe API.
+     * Gets queryable field names for a SF object via the Describe API. When
+     * `family === 'tooling'`, calls the Tooling describe endpoint instead of
+     * the standard one.
      */
-    async GetQueryableFieldNames(auth, objectName) {
-        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/sobjects/${objectName}/describe`;
+    async GetQueryableFieldNames(auth, objectName, family = 'sobject') {
+        const url = `${this.SObjectBasePath(auth, family)}/${objectName}/describe`;
         const headers = this.BuildHeaders(auth);
         const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
         this.ValidateResponse(response, url);
@@ -646,7 +1041,33 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
             aud: config.LoginUrl,
             exp: now + 300, // 5 minutes
         };
-        return jwt.sign(payload, config.PrivateKey, { algorithm: 'RS256' });
+        const pem = this.NormalizePem(config.PrivateKey);
+        return jwt.sign(payload, pem, { algorithm: 'RS256' });
+    }
+    /**
+     * Normalize a PEM private key that may have had its newlines stripped or
+     * replaced with spaces / literal "\n" sequences. Rebuilds the standard
+     * BEGIN header / 64-char body / END footer layout that OpenSSL requires.
+     */
+    NormalizePem(key) {
+        if (!key)
+            return key;
+        // First, normalize escaped newlines to real ones
+        let normalized = key.includes('\\n') ? key.replace(/\\n/g, '\n') : key;
+        // If real newlines exist already, trust them
+        if (normalized.includes('\n'))
+            return normalized;
+        // Split out header / footer / body and rebuild with real newlines
+        const headerMatch = normalized.match(/^(-----BEGIN [^-]+-----)/);
+        const footerMatch = normalized.match(/(-----END [^-]+-----)\s*$/);
+        if (!headerMatch || !footerMatch)
+            return normalized;
+        const header = headerMatch[1];
+        const footer = footerMatch[1];
+        const bodyRaw = normalized.slice(header.length, normalized.length - footer.length);
+        const body = bodyRaw.replace(/\s+/g, '');
+        const chunked = body.match(/.{1,64}/g)?.join('\n') ?? body;
+        return `${header}\n${chunked}\n${footer}`;
     }
     /**
      * Exchanges a JWT assertion for an access token at the SF token endpoint.
@@ -910,6 +1331,9 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
     // ─── SF Type Mapping ─────────────────────────────────────────────
     /**
      * Maps a Salesforce field describe to an ExternalFieldSchema.
+     * Custom fields can be detected by callers via the `__c` suffix in the
+     * field name — Salesforce standardizes this convention for all custom
+     * fields on both standard and custom SObjects.
      */
     MapSFFieldToSchema(f) {
         return {
@@ -1169,8 +1593,185 @@ let SalesforceConnector = class SalesforceConnector extends BaseRESTIntegrationC
             return body.slice(0, 500);
         return JSON.stringify(body).slice(0, 500);
     }
+    // ─── Analytics API (Reports & Dashboards) ────────────────────────
+    /**
+     * Lists Analytics Reports or Dashboards via the Analytics API. Read-only;
+     * no incremental-sync support at this endpoint.
+     */
+    async FetchAnalyticsList(auth, family, objectName) {
+        const segment = family === 'analytics_report' ? 'reports' : 'dashboards';
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/analytics/${segment}`;
+        const headers = this.BuildHeaders(auth);
+        const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
+        this.ValidateResponse(response, url);
+        const body = response.Body;
+        const list = body.reports ?? body.dashboards ?? [];
+        const records = list.map(item => this.AnalyticsItemToRecord(item, objectName));
+        return { Records: records, HasMore: false };
+    }
+    AnalyticsItemToRecord(item, objectType) {
+        return {
+            ExternalID: String(item.id ?? ''),
+            ObjectType: objectType,
+            Fields: { ...item },
+        };
+    }
+    async GetAnalyticsReport(auth, id) {
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/analytics/reports/${id}`;
+        const headers = this.BuildHeaders(auth);
+        const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
+        if (response.Status === 404)
+            return null;
+        this.ValidateResponse(response, url);
+        return {
+            ExternalID: id,
+            ObjectType: 'Report',
+            Fields: response.Body,
+        };
+    }
+    async GetAnalyticsDashboard(auth, id) {
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/analytics/dashboards/${id}`;
+        const headers = this.BuildHeaders(auth);
+        const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
+        if (response.Status === 404)
+            return null;
+        this.ValidateResponse(response, url);
+        return {
+            ExternalID: id,
+            ObjectType: 'Dashboard',
+            Fields: response.Body,
+        };
+    }
+    // ─── Knowledge Articles ──────────────────────────────────────────
+    /**
+     * Fetches published Knowledge article versions. This endpoint is
+     * paginated via `pageNumber`/`pageSize` (not cursor). The connector
+     * returns a single batch; callers that need pagination can issue
+     * additional requests via SearchRecords.
+     */
+    async FetchKnowledgeArticles(auth, objectName, watermarkValue, batchSize, page = 1) {
+        const params = new URLSearchParams({ pageSize: String(batchSize), pageNumber: String(page) });
+        if (watermarkValue) {
+            params.set('publishStatus', 'Online');
+        }
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/support/knowledgeArticles?${params}`;
+        const headers = this.BuildHeaders(auth);
+        const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
+        this.ValidateResponse(response, url);
+        const body = response.Body;
+        const records = (body.articles ?? []).map(a => ({
+            ExternalID: String(a.id ?? ''),
+            ObjectType: objectName,
+            Fields: a,
+        }));
+        // Knowledge Articles uses page-based pagination. The endpoint doesn't
+        // return a documented total/has-more signal in its body, so treat a
+        // full page as "potentially more" — same defensive pattern used for
+        // SI and YM. Engine loop drives subsequent pages via NextPage; an
+        // empty next-page response terminates naturally. Without this guard
+        // sync silently caps at batchSize records per object — identical to
+        // the SOQL LIMIT bug we already fixed in this session.
+        const hasMore = records.length >= batchSize;
+        return {
+            Records: records,
+            HasMore: hasMore,
+            NextPage: hasMore ? page + 1 : undefined,
+        };
+    }
+    // ─── Bulk API 2.0 (Ingest & Query Jobs) ──────────────────────────
+    /**
+     * Lists in-flight Bulk API 2.0 jobs. Useful for monitoring background
+     * imports/queries the integration previously started.
+     */
+    async FetchBulkJobs(auth, family, objectName) {
+        const segment = family === 'bulk_ingest' ? 'ingest' : 'query';
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/jobs/${segment}`;
+        const headers = this.BuildHeaders(auth);
+        const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
+        this.ValidateResponse(response, url);
+        const body = response.Body;
+        const records = (body.records ?? []).map(r => ({
+            ExternalID: String(r.id ?? ''),
+            ObjectType: objectName,
+            Fields: r,
+            ModifiedAt: r.systemModstamp ? new Date(r.systemModstamp) : undefined,
+        }));
+        return {
+            Records: records,
+            HasMore: !body.done,
+            NextCursor: body.nextRecordsUrl ?? undefined,
+        };
+    }
+    /**
+     * Creates (starts) a new Bulk API 2.0 ingest or query job using the
+     * provided attributes. Required fields differ by operation; we pass
+     * attributes straight through to Salesforce after stripping any readonly
+     * fields. For ingest, CSV data is uploaded in a separate request (PUT) —
+     * callers must issue that themselves via UploadBulkData once the job ID
+     * is known.
+     */
+    async CreateBulkJob(auth, family, attrs) {
+        const segment = family === 'bulk_ingest' ? 'ingest' : 'query';
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/jobs/${segment}`;
+        const headers = { ...this.BuildHeaders(auth), 'Content-Type': 'application/json' };
+        const body = this.StripReadOnlyFields(attrs);
+        const response = await this.MakeHTTPRequest(auth, url, 'POST', headers, body);
+        if (response.Status >= 200 && response.Status < 300) {
+            const created = response.Body;
+            return { Success: true, ExternalID: created.id ?? '', StatusCode: response.Status };
+        }
+        return this.BuildCRUDError(response, 'CreateBulkJob', family);
+    }
+    /**
+     * Aborts a running Bulk API 2.0 job by PATCHing state=Aborted.
+     */
+    async AbortBulkJob(auth, family, jobId) {
+        const segment = family === 'bulk_ingest' ? 'ingest' : 'query';
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/jobs/${segment}/${jobId}`;
+        const headers = { ...this.BuildHeaders(auth), 'Content-Type': 'application/json' };
+        const response = await this.MakeHTTPRequest(auth, url, 'PATCH', headers, { state: 'Aborted' });
+        if (response.Status >= 200 && response.Status < 300) {
+            return { Success: true, ExternalID: jobId, StatusCode: response.Status };
+        }
+        return this.BuildCRUDError(response, 'AbortBulkJob', family);
+    }
+    async GetBulkJob(auth, family, jobId) {
+        const segment = family === 'bulk_ingest' ? 'ingest' : 'query';
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/jobs/${segment}/${jobId}`;
+        const headers = this.BuildHeaders(auth);
+        const response = await this.MakeHTTPRequest(auth, url, 'GET', headers);
+        if (response.Status === 404)
+            return null;
+        this.ValidateResponse(response, url);
+        return {
+            ExternalID: jobId,
+            ObjectType: family === 'bulk_ingest' ? 'BulkIngestJob' : 'BulkQueryJob',
+            Fields: response.Body,
+        };
+    }
+    // ─── Composite Requests ──────────────────────────────────────────
+    /**
+     * Executes a composite request — up to 25 sub-requests in one HTTP call.
+     * The `attrs` payload is expected to be the full composite body or at
+     * least a `compositeRequest` array. Returns the overall composite
+     * response as the created record's ExternalID (Salesforce returns
+     * per-sub-request results; callers should inspect Fields for details).
+     */
+    async ExecuteCompositeRequest(auth, attrs) {
+        const url = `${auth.InstanceUrl}/services/data/v${auth.ApiVersion}/composite`;
+        const headers = { ...this.BuildHeaders(auth), 'Content-Type': 'application/json' };
+        // Either pass the whole payload through or wrap a bare array
+        const body = Array.isArray(attrs.compositeRequest)
+            ? attrs
+            : { allOrNone: true, compositeRequest: [attrs] };
+        const response = await this.MakeHTTPRequest(auth, url, 'POST', headers, body);
+        if (response.Status >= 200 && response.Status < 300) {
+            return { Success: true, ExternalID: 'composite', StatusCode: response.Status };
+        }
+        return this.BuildCRUDError(response, 'ExecuteCompositeRequest', 'CompositeRequest');
+    }
 };
-SalesforceConnector = __decorate([
+SalesforceConnector = SalesforceConnector_1 = __decorate([
     RegisterClass(BaseIntegrationConnector, 'SalesforceConnector')
 ], SalesforceConnector);
 export { SalesforceConnector };