@memberjunction/server 5.42.0 → 5.44.0

package/src/index.ts

@@ -40,6 +40,7 @@ import createMSSQLConfig from './orm.js';
 import { setupRESTEndpoints } from './rest/setupRESTEndpoints.js';
 import { createOAuthCallbackHandler } from './rest/OAuthCallbackHandler.js';
 import { createSignatureWebhookHandler } from './rest/SignatureWebhookHandler.js';
+import { createMediaStreamRouter } from './rest/MediaStreamHandler.js';
 import { createMagicLinkHandler, registerMagicLinkAuthProvider, MAGIC_LINK_MOUNT_PATH } from './auth/magicLink/index.js';
 
 import { resolve } from 'node:path';
@@ -162,6 +163,8 @@ export * from './resolvers/ISAEntityResolver.js';
 export * from './resolvers/ArtifactFileResolver.js';
 export * from './resolvers/FileCategoryResolver.js';
 export * from './resolvers/FileResolver.js';
+export * from './rest/MediaAccessKeys.js';
+export * from './rest/MediaStreamHandler.js';
 export * from './resolvers/InfoResolver.js';
 export * from './resolvers/PotentialDuplicateRecordResolver.js';
 export * from './resolvers/RunTestResolver.js';
@@ -863,7 +866,11 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
       onConnect: async (ctx) => {
         try {
           const token = String(ctx.connectionParams?.Authorization);
-          const userPayload = await getUserPayload(token, undefined, dataSources);
+          // Carry API keys from connectionParams so API-key / MCP / Node clients can authenticate the socket
+          // (validated the same way as the HTTP x-mj-api-key / x-mj-user-api-key headers).
+          const systemApiKey = ctx.connectionParams?.['x-mj-api-key'] ? String(ctx.connectionParams['x-mj-api-key']) : undefined;
+          const userApiKey = ctx.connectionParams?.['x-mj-user-api-key'] ? String(ctx.connectionParams['x-mj-user-api-key']) : undefined;
+          const userPayload = await getUserPayload(token, undefined, dataSources, undefined, systemApiKey, userApiKey);
 
           // Store validated payload on the connection for use in context()
           ctx.extra.userPayload = userPayload;
@@ -987,6 +994,13 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
   app.use('/esignature', cors<cors.CorsRequest>(), createSignatureWebhookHandler());
   startupLog.LogIf('verbose', '[eSignature] Webhook route registered at /esignature/webhook/:driverKey');
 
+  // ─── Authenticated media streaming (token-gated, registered BEFORE auth) ─────
+  // The `?token=` query param (minted by CreateMediaAccessToken after a per-user permission
+  // check) is the capability — the route verifies it itself, so it does NOT use the MJ bearer
+  // auth middleware (an <audio>/<video> element can't send Authorization headers).
+  app.use('/media', cors<cors.CorsRequest>(), createMediaStreamRouter());
+  startupLog.LogIf('verbose', '[Media] Streaming route registered at /media/:fileId');
+
   // ─── Magic-link routes (MJ-issued, app-scoped external access) ───────────
   // Public router (JWKS + redeem) mounts BEFORE the auth middleware; the
   // authenticated invite-creation router mounts AFTER it (see below).

package/src/resolvers/ExecuteRemoteOperationResolver.ts

@@ -1,10 +1,14 @@
-import { Arg, Ctx, Field, InputType, Mutation, ObjectType, Resolver } from 'type-graphql';
+import { Arg, Ctx, Field, ID, InputType, Mutation, ObjectType, PubSub, PubSubEngine, Resolver, Root, Subscription } from 'type-graphql';
 import { MJGlobal } from '@memberjunction/global';
-import { BaseRemotableOperation, RemoteOpInvokeMode } from '@memberjunction/core';
+import { BaseRemotableOperation, RemoteOpInvokeMode, RemoteOpProgress } from '@memberjunction/core';
+import { RemoteOperationEngineBase } from '@memberjunction/core-entities';
 import { ResolverBase } from '../generic/ResolverBase.js';
 import { AppContext } from '../types.js';
 import { GetReadWriteProvider } from '../util.js';
 
+/** PubSub topic carrying RemoteOpProgress envelopes for attached over-the-wire callers (RO-3). */
+const REMOTE_OP_PROGRESS_TOPIC = 'RemoteOperationProgress';
+
 /** Input for the generic Remote Operation transport mutation. */
 @InputType()
 export class ExecuteRemoteOperationInput {
@@ -16,6 +20,25 @@ export class ExecuteRemoteOperationInput {
 
     @Field()
     invokeMode: string;
+
+    /**
+     * Optional client-generated channel id. When set (attached mode over the wire), the server publishes every
+     * `RemoteOpProgress` the operation emits to the `RemoteOperationProgress` subscription filtered by this id,
+     * so the caller's `onProgress` fires live while the operation runs. Omitted for sync / no-progress calls.
+     */
+    @Field(() => String, { nullable: true })
+    progressChannelId?: string;
+}
+
+/** A single streamed progress envelope delivered over the `RemoteOperationProgress` subscription. */
+@ObjectType()
+export class RemoteOperationProgressNotification {
+    @Field()
+    ChannelId: string;
+
+    /** JSON-serialized {@link RemoteOpProgress}. */
+    @Field()
+    ProgressJSON: string;
 }
 
 /** Result of the generic Remote Operation transport mutation. */
@@ -50,6 +73,7 @@ export class ExecuteRemoteOperationResolver extends ResolverBase {
     async ExecuteRemoteOperation(
         @Arg('input') input: ExecuteRemoteOperationInput,
         @Ctx() ctx: AppContext,
+        @PubSub() pubSub: PubSubEngine,
     ): Promise<ExecuteRemoteOperationResultType> {
         try {
             const key = input.operationKey?.trim();
@@ -78,6 +102,15 @@ export class ExecuteRemoteOperationResolver extends ResolverBase {
                 return fail('NO_USER', 'Not authenticated');
             }
 
+            // (4) Metadata gate (defense-in-depth): honor the op's MJ: Remote Operations Status/approval, so a
+            // registered-but-disabled (or unapproved AI) op is rejected even though its class is still in code.
+            const provider = GetReadWriteProvider(ctx.providers);
+            await RemoteOperationEngineBase.Instance.Config(false, user, provider);
+            const invokability = RemoteOperationEngineBase.Instance.IsInvokable(key);
+            if (!invokability.Invokable) {
+                return fail(invokability.ResultCode ?? 'OPERATION_UNAVAILABLE', invokability.Reason ?? `Operation '${key}' is unavailable`);
+            }
+
             let parsedInput: unknown;
             try {
                 parsedInput = input.inputJSON ? JSON.parse(input.inputJSON) : {};
@@ -85,9 +118,16 @@ export class ExecuteRemoteOperationResolver extends ResolverBase {
                 return fail('INVALID_INPUT_JSON', `Invalid input JSON: ${e instanceof Error ? e.message : String(e)}`);
             }
 
+            // Attached over-the-wire (RO-3): forward each emitted RemoteOpProgress to the progress channel so a
+            // remote caller's onProgress fires live. No-op when the caller didn't open a channel (sync calls).
+            const onProgress = input.progressChannelId
+                ? (p: RemoteOpProgress) => {
+                      void pubSub.publish(REMOTE_OP_PROGRESS_TOPIC, { ChannelId: input.progressChannelId!, ProgressJSON: JSON.stringify(p) });
+                  }
+                : undefined;
+
             // Route through the per-request provider (server in-process path; runs the op's Authorize + InternalExecute).
-            const provider = GetReadWriteProvider(ctx.providers);
-            const result = await provider.RouteOperation(key, parsedInput, { user, mode: input.invokeMode as RemoteOpInvokeMode });
+            const result = await provider.RouteOperation(key, parsedInput, { user, mode: input.invokeMode as RemoteOpInvokeMode, onProgress });
 
             return {
                 success: result.Success,
@@ -100,6 +140,22 @@ export class ExecuteRemoteOperationResolver extends ResolverBase {
             return fail('ERROR', e instanceof Error ? e.message : String(e));
         }
     }
+
+    /**
+     * Subscription the client opens (with a self-generated `channelId`) immediately before calling
+     * `ExecuteRemoteOperation` with the same id in `progressChannelId`. The server publishes one event per
+     * `RemoteOpProgress` the operation emits; filtered by `channelId` so concurrent calls never interleave.
+     */
+    @Subscription(() => RemoteOperationProgressNotification, {
+        topics: REMOTE_OP_PROGRESS_TOPIC,
+        filter: ({ payload, args }: { payload: RemoteOperationProgressNotification; args: { channelId: string } }) => payload.ChannelId === args.channelId,
+    })
+    RemoteOperationProgress(
+        @Root() notification: RemoteOperationProgressNotification,
+        @Arg('channelId', () => ID) _channelId: string,
+    ): RemoteOperationProgressNotification {
+        return notification;
+    }
 }
 
 /** Builds a failed result. */

package/src/resolvers/FileResolver.ts

@@ -1,6 +1,7 @@
-import { EntityPermissionType, FieldValueCollection, EntitySaveOptions } from '@memberjunction/core';
+import { EntityPermissionType, FieldValueCollection, EntitySaveOptions, LogError, UserInfo } from '@memberjunction/core';
 import { NormalizeUUID } from '@memberjunction/global';
 import { MJFileEntity, MJFileStorageProviderEntity, MJFileStorageAccountEntity } from '@memberjunction/core-entities';
+import { readRealtimeRecordingFile } from '@memberjunction/ai-agents';
 import {
   AppContext,
   Arg,
@@ -8,6 +9,7 @@ import {
   DeleteOptionsInput,
   Field,
   FieldResolver,
+  Float,
   InputType,
   Int,
   Mutation,
@@ -37,6 +39,9 @@ import {
 import { CreateMJFileInput, MJFileResolver as FileResolverBase, MJFile_, UpdateMJFileInput } from '../generated/generated.js';
 import { FieldMapper } from '@memberjunction/graphql-dataprovider';
 import { GetReadOnlyProvider } from '../util.js';
+import { configInfo } from '../config.js';
+import { MediaAccessKeyManager } from '../rest/MediaAccessKeys.js';
+import { deriveSidecarPath, parsePeaksSidecar } from './peaksSidecar.js';
 
 @InputType()
 export class CreateUploadURLInput {
@@ -312,8 +317,161 @@ export class SearchAcrossAccountsPayload {
   failedAccounts: number;
 }
 
+/**
+ * Result of {@link FileResolver.GetFileContents} — the bytes of an `MJ: Files` record returned as
+ * base64, read server-side through authenticated MJStorage (never a public pre-signed link).
+ */
+@ObjectType()
+export class FileContentsResult {
+  @Field(() => Boolean)
+  Success: boolean;
+
+  @Field(() => String, { nullable: true })
+  Base64?: string;
+
+  @Field(() => String, { nullable: true })
+  MimeType?: string;
+
+  @Field(() => String, { nullable: true })
+  ErrorMessage?: string;
+}
+
+/**
+ * Result of {@link FileResolver.CreateMediaAccessToken} — a short-lived signed token plus the
+ * authenticated streaming URL the browser hands to a `<audio>`/`<video>` element. The token is
+ * minted only after a per-user permission check on the `MJ: Files` row; the `/media/:fileId`
+ * route re-verifies it (signature + expiry + file match) on each Range request.
+ */
+@ObjectType()
+export class MediaAccessTokenResult {
+  @Field(() => Boolean)
+  Success: boolean;
+
+  @Field(() => String, { nullable: true })
+  Token?: string;
+
+  @Field(() => String, { nullable: true })
+  Url?: string;
+
+  @Field(() => Date, { nullable: true })
+  ExpiresAt?: Date;
+
+  /**
+   * The file's MIME type (from the already-loaded `MJ: Files` row). Lets the wrapper choose
+   * `<audio>` vs `<video>` WITHOUT re-downloading any bytes; the stream itself sets Content-Type.
+   */
+  @Field(() => String, { nullable: true })
+  MimeType?: string;
+
+  /**
+   * Optional precomputed waveform peaks (normalized `0..1`, one per rendered bar) read from a
+   * `peaks.json` sidecar that sits beside the file in storage. When present, the player renders the
+   * real waveform instantly with NO client-side fetch/decode of the audio. Best-effort: a missing or
+   * malformed sidecar simply omits this field — it never blocks token minting or fails the mutation.
+   */
+  @Field(() => [Float], { nullable: true })
+  Peaks?: number[];
+
+  @Field(() => String, { nullable: true })
+  ErrorMessage?: string;
+}
+
 @Resolver(MJFile_)
 export class FileResolver extends FileResolverBase {
+  /**
+   * Resolves the server's public base URL the same way callbacks/magic-link do:
+   * `publicUrl` if configured (e.g. an ngrok URL), else `baseUrl:graphqlPort`. Trailing
+   * slash stripped so callers can append `/media/...`.
+   */
+  private resolvePublicBaseUrl(): string {
+    const base = configInfo.publicUrl || `${configInfo.baseUrl}:${configInfo.graphqlPort}${configInfo.graphqlRootPath || ''}`;
+    return base.replace(/\/+$/, '');
+  }
+
+  /**
+   * Mints a short-lived signed media-access token for an `MJ: Files` record and returns the
+   * authenticated streaming URL (`<publicBase>/media/<fileId>?token=<token>`). Permission-gated
+   * IDENTICALLY to {@link GetFileContents}: the file is loaded under the CALLING USER's context, so
+   * MJ row-level security determines access. The returned token is the capability — the streaming
+   * route re-verifies it without re-checking row-level access. Never throws to the client.
+   *
+   * @param fileId The `MJ: Files` id to grant streaming access to.
+   * @returns `{ Success, Token?, Url?, ExpiresAt?, ErrorMessage? }`.
+   */
+  @Mutation(() => MediaAccessTokenResult)
+  async CreateMediaAccessToken(@Arg('fileId', () => String) fileId: string, @Ctx() context: AppContext): Promise<MediaAccessTokenResult> {
+    try {
+      const provider = GetReadOnlyProvider(context.providers, { allowFallbackToReadWrite: true });
+      const contextUser = this.GetUserFromPayload(context.userPayload);
+
+      // Permission gate: load the file under the USER's context. A failed load means the file
+      // does not exist OR the user lacks read access under MJ row-level security.
+      const fileEntity = await provider.GetEntityObject<MJFileEntity>('MJ: Files', contextUser);
+      const loaded = await fileEntity.Load(fileId);
+      if (!loaded) {
+        return { Success: false, ErrorMessage: 'You do not have access to this file or it does not exist.' };
+      }
+
+      // Access authorized — mint the capability token.
+      const { Token, ExpiresAt } = MediaAccessKeyManager.Instance.Sign(fileId, contextUser.ID);
+      const url = `${this.resolvePublicBaseUrl()}/media/${encodeURIComponent(fileId)}?token=${encodeURIComponent(Token)}`;
+
+      // Best-effort: surface precomputed waveform peaks from a peaks.json sidecar beside the file, so
+      // the player renders the real waveform instantly without fetching/decoding the audio. Never
+      // blocks token minting — any failure just omits Peaks.
+      const peaks = await this.tryReadPeaksSidecar(fileEntity, contextUser);
+
+      return { Success: true, Token, Url: url, ExpiresAt, MimeType: fileEntity.ContentType ?? undefined, Peaks: peaks };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      LogError(`CreateMediaAccessToken failed (file ${fileId}): ${message}`);
+      return { Success: false, ErrorMessage: message };
+    }
+  }
+
+  /**
+   * Best-effort read of a `peaks.json` waveform sidecar that sits in the SAME storage folder as the
+   * given file (a JSON array of normalized `0..1` numbers, written at capture time). Derives the
+   * folder from the file's `ProviderKey` (strips the final path segment), reads `<folder>/peaks.json`
+   * via the file's own storage driver, parses + validates it, and returns sanitized peaks. Returns
+   * `undefined` on ANY failure (no ProviderKey, no sidecar, parse error, garbage) — the caller treats
+   * peaks as a pure optimization and must never let a sidecar problem affect token minting.
+   *
+   * @param file The already-loaded (under the user's context) `MJ: Files` row.
+   * @param contextUser The calling user — used to resolve the storage driver.
+   * @returns Sanitized `0..1` peaks (length-capped), or `undefined`.
+   */
+  private async tryReadPeaksSidecar(file: MJFileEntity, contextUser: UserInfo): Promise<number[] | undefined> {
+    try {
+      // The sidecar lives next to the recording: replace the final path segment with peaks.json.
+      const sidecarPath = deriveSidecarPath(file.ProviderKey);
+      if (!sidecarPath) {
+        return undefined;
+      }
+
+      // Resolve the file's storage account → driver (mirror of readRealtimeRecordingFile's pattern).
+      await FileStorageEngine.Instance.Config(false, contextUser);
+      let accounts = FileStorageEngine.Instance.GetAccountsByProviderID(file.ProviderID);
+      if (accounts.length === 0) {
+        await FileStorageEngine.Instance.Config(true, contextUser);
+        accounts = FileStorageEngine.Instance.GetAccountsByProviderID(file.ProviderID);
+      }
+      const account = accounts[0];
+      if (!account) {
+        return undefined;
+      }
+      const driver = await FileStorageEngine.Instance.GetDriver(account.ID, contextUser);
+      const bytes = await driver.GetObject({ fullPath: sidecarPath });
+      if (!bytes || bytes.length === 0) {
+        return undefined;
+      }
+      return parsePeaksSidecar(bytes);
+    } catch {
+      // No sidecar / unreadable / parse failure — peaks are optional, never surface the error.
+      return undefined;
+    }
+  }
+
   /**
    * Builds UserContextOptions for storage operations that may require OAuth authentication.
    * This passes the current user's ID and context to allow the storage utilities to
@@ -431,6 +589,46 @@ export class FileResolver extends FileResolverBase {
     return url;
   }
 
+  /**
+   * Returns an `MJ: Files` record's bytes as base64, read server-side through authenticated MJStorage
+   * (`GetObject`) — NOT a public pre-signed link. Permission-gated: the file is first loaded under the
+   * calling user's context, so MJ row-level security determines access. Never throws to the client.
+   *
+   * @param fileId The `MJ: Files` id whose bytes to return.
+   * @returns `{ Success, Base64?, MimeType?, ErrorMessage? }`.
+   */
+  @Query(() => FileContentsResult)
+  async GetFileContents(@Arg('fileId', () => String) fileId: string, @Ctx() context: AppContext): Promise<FileContentsResult> {
+    try {
+      const provider = GetReadOnlyProvider(context.providers, { allowFallbackToReadWrite: true });
+      const contextUser = this.GetUserFromPayload(context.userPayload);
+
+      // Permission gate FIRST: load the file record under the USER's context. A failed load means the
+      // file does not exist OR the user lacks read access under MJ row-level security.
+      const fileEntity = await provider.GetEntityObject<MJFileEntity>('MJ: Files', contextUser);
+      const loaded = await fileEntity.Load(fileId);
+      if (!loaded) {
+        return { Success: false, ErrorMessage: 'You do not have access to this file or it does not exist.' };
+      }
+
+      // Read the bytes via authenticated MJStorage (server-side GetObject on the file's own account).
+      const result = await readRealtimeRecordingFile(fileId, contextUser, provider);
+      if (!result) {
+        return { Success: false, ErrorMessage: 'The file could not be read from storage.' };
+      }
+
+      return {
+        Success: true,
+        Base64: result.Bytes.toString('base64'),
+        MimeType: result.MimeType ?? fileEntity.ContentType ?? undefined,
+      };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      LogError(`GetFileContents failed (file ${fileId}): ${message}`);
+      return { Success: false, ErrorMessage: message };
+    }
+  }
+
   @Mutation(() => MJFile_)
   async UpdateFile(@Arg('input', () => UpdateMJFileInput) input: UpdateMJFileInput, @Ctx() context: AppContext, @PubSub() pubSub: PubSubEngine) {
     // if the name is changing, rename the target object as well

package/src/resolvers/IntegrationDiscoveryResolver.ts

@@ -1,5 +1,5 @@
 import { Resolver, Query, Mutation, Arg, Ctx, ObjectType, Field, InputType, Int, Float } from "type-graphql";
-import { CompositeKey, DatabaseProviderBase, LocalCacheManager, Metadata, RunView, UserInfo, LogError, LogStatus, IMetadataProvider, TransactionGroupBase } from "@memberjunction/core";
+import { CompositeKey, DatabaseProviderBase, EntityInfo, LocalCacheManager, Metadata, RunView, UserInfo, LogError, LogStatus, IMetadataProvider, TransactionGroupBase } from "@memberjunction/core";
 import { GetReadOnlyProvider, GetReadWriteProvider } from "../util.js";
 import { CronExpressionHelper } from "@memberjunction/scheduling-engine";
 import {
@@ -3451,9 +3451,29 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
     ): Promise<ApplyAllEntityMapCreated[]> {
         const results: ApplyAllEntityMapCreated[] = [];
 
+        // PERF (large-catalog ApplyAll, e.g. Salesforce 1695 objects): precompute O(1) lookups ONCE before
+        // the per-object loop. The per-object path previously did an O(N) `provider.Entities.find` + a
+        // BypassCache RunView for the existing entity map PER object → O(N²) scans + N DB round-trips, so a
+        // 1695-object connector burned its whole budget in ApplyAll and FullSync never started. One entity
+        // index + one batched entity-map query make both O(1) per object.
+        const entityByKey = new Map(
+            provider.Entities.map(e => [`${e.SchemaName.toLowerCase()}|${e.BaseTable.toLowerCase()}`, e] as const)
+        );
+        const existingMapByExternalName = new Map<string, MJCompanyIntegrationEntityMapEntity>();
+        const allMaps = await new RunView().RunView<MJCompanyIntegrationEntityMapEntity>({
+            EntityName: 'MJ: Company Integration Entity Maps',
+            ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}'`,
+            ResultType: 'entity_object',
+            BypassCache: true, // idempotency must read COMMITTED state, not a possibly-stale filtered cache
+        }, user);
+        if (allMaps.Success) {
+            for (const m of allMaps.Results) existingMapByExternalName.set((m.ExternalObjectName ?? '').toLowerCase(), m);
+        }
+
         for (const obj of objects) {
             const entityMapResult = await this.createSingleEntityMap(
-                companyIntegrationID, obj, connector, companyIntegration, schemaName, user, provider, defaultSyncDirection
+                companyIntegrationID, obj, connector, companyIntegration, schemaName, user, provider, defaultSyncDirection,
+                entityByKey, existingMapByExternalName
             );
             if (entityMapResult) {
                 results.push(entityMapResult);
@@ -3471,10 +3491,14 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         schemaName: string,
         user: UserInfo,
         md: IMetadataProvider,
-        defaultSyncDirection: string = 'Pull'
+        defaultSyncDirection: string = 'Pull',
+        entityByKey?: Map<string, EntityInfo>,
+        existingMapByExternalName?: Map<string, MJCompanyIntegrationEntityMapEntity>
     ): Promise<ApplyAllEntityMapCreated | null> {
-        // Find the entity by schema + table name
-        const entityInfo = md.Entities.find(
+        // Find the entity by schema + table name. PERF: O(1) precomputed index when provided
+        // (large-catalog ApplyAll); falls back to the O(N) scan when called without it.
+        const entityInfo = entityByKey?.get(`${schemaName.toLowerCase()}|${obj.TableName.toLowerCase()}`)
+          ?? md.Entities.find(
             e => e.SchemaName.toLowerCase() === schemaName.toLowerCase()
               && e.BaseTable.toLowerCase() === obj.TableName.toLowerCase()
         );
@@ -3488,19 +3512,27 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         // multiplies the maps in lockstep (N applies → N duplicate maps per object), which
         // silently corrupts the record-map 1:1 completeness gate and makes the forward sync
         // process each object N times. Reuse the existing (connection, external object) map instead.
-        const escapedObjectName = obj.SourceObjectName.replace(/'/g, "''");
-        const existingMapResult = await new RunView().RunView<MJCompanyIntegrationEntityMapEntity>({
-            EntityName: 'MJ: Company Integration Entity Maps',
-            ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND ExternalObjectName='${escapedObjectName}'`,
-            OrderBy: '__mj_CreatedAt ASC',
-            MaxRows: 1,
-            ResultType: 'entity_object',
-            BypassCache: true, // idempotency must read COMMITTED state, not a possibly-stale filtered cache
-        }, user);
+        // PERF: use the precomputed existing-map index when provided (one batched query for the whole
+        // ApplyAll instead of one BypassCache RunView per object); falls back to a per-object query.
+        let existing: MJCompanyIntegrationEntityMapEntity | undefined;
+        if (existingMapByExternalName) {
+            existing = existingMapByExternalName.get(obj.SourceObjectName.toLowerCase());
+        } else {
+            const escapedObjectName = obj.SourceObjectName.replace(/'/g, "''");
+            const existingMapResult = await new RunView().RunView<MJCompanyIntegrationEntityMapEntity>({
+                EntityName: 'MJ: Company Integration Entity Maps',
+                ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND ExternalObjectName='${escapedObjectName}'`,
+                OrderBy: '__mj_CreatedAt ASC',
+                MaxRows: 1,
+                ResultType: 'entity_object',
+                BypassCache: true, // idempotency must read COMMITTED state, not a possibly-stale filtered cache
+            }, user);
+            if (existingMapResult.Success && existingMapResult.Results.length > 0) existing = existingMapResult.Results[0];
+        }
 
         let em: MJCompanyIntegrationEntityMapEntity;
-        if (existingMapResult.Success && existingMapResult.Results.length > 0) {
-            em = existingMapResult.Results[0]; // reuse — keeps the map stable across re-applies
+        if (existing) {
+            em = existing; // reuse — keeps the map stable across re-applies
         } else {
             em = await md.GetEntityObject<MJCompanyIntegrationEntityMapEntity>('MJ: Company Integration Entity Maps', user);
             em.NewRecord();
@@ -3634,26 +3666,70 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
      * still surfaces every object/field; only materialization is capped. Per-table column count = the selected
      * field subset, or the object's full discovered field count when all fields are selected.
      */
-    private enforceSchemaLimits(
+    private async enforceSchemaLimits(
         objects: SchemaPreviewObjectInput[],
         filteredSchema: SourceSchemaInfo,
-    ): void {
-        const envInt = (name: string): number | null => {
-            const v = parseInt(process.env[name] ?? '', 10);
-            return Number.isFinite(v) && v > 0 ? v : null;
-        };
+        companyIntegration: MJCompanyIntegrationEntity,
+        user: UserInfo,
+        md: IMetadataProvider,
+    ): Promise<string[]> {
+        const warnings: string[] = [];
         const fullCountByName = new Map(filteredSchema.Objects.map(o => [o.ExternalName.toLowerCase(), o.Fields.length]));
-        // Delegate the cap decision to the engine's pure, unit-tested decideSchemaLimitViolations.
+        // Use the MAX of the input object's Fields and the filtered-schema count — the batch path can pass
+        // objects with empty Fields (the real columns come from filteredSchema, which is what buildSchema
+        // materializes). `?? ` would wrongly read an empty [] as 0 (0 is not nullish); Math.max is robust to
+        // either source being empty and reflects the count that will actually become columns.
+        const colCount = (o: SchemaPreviewObjectInput): number =>
+            Math.max(fullCountByName.get(o.SourceObjectName.toLowerCase()) ?? 0, o.Fields?.length ?? 0);
+
+        // ── Per-table COLUMN limit: DISABLE-the-offender, NOT reject-the-apply. The limit is clamped at
+        // startup to ≤ IntegrationEngine.MAX_COLUMNS_CEILING (SQL Server's 1024 minus framework column
+        // headroom). An object exceeding it is set Status='Disabled' (reversible — it re-enables on a later
+        // discovery if it has shrunk back under the limit) and dropped from THIS apply, so the materializable
+        // objects still sync instead of the whole apply failing CREATE TABLE. One CLI warning per disabled object.
+        const limit = IntegrationEngine.Instance.MaxColumnsPerTable;
+        const overLimit = objects.filter(o => colCount(o) > limit);
+        if (overLimit.length > 0) {
+            const ioIDByName = new Map(
+                IntegrationEngineBase.Instance.GetActiveIntegrationObjects(companyIntegration.IntegrationID)
+                    .map(io => [io.Name.toLowerCase(), io.ID] as [string, string])
+            );
+            for (const o of overLimit) {
+                warnings.push(`Disabled "${o.SourceObjectName}" (${colCount(o)} columns > limit ${limit}) — too wide to materialize as one table; reduce its columns or it stays disabled.`);
+                const ioID = ioIDByName.get(o.SourceObjectName.toLowerCase());
+                if (ioID) {
+                    const io = await md.GetEntityObject<MJIntegrationObjectEntity>('MJ: Integration Objects', user);
+                    if (await io.Load(ioID)) {
+                        io.Status = 'Disabled';
+                        if (!await io.Save()) {
+                            LogError(`enforceSchemaLimits: failed to disable over-wide IO '${o.SourceObjectName}': ${io.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+                        }
+                    }
+                }
+            }
+            // Drop the disabled objects from BOTH the shared objects array (used downstream by
+            // buildTargetConfigs + createEntityAndFieldMaps) and filteredSchema, in place.
+            const disabled = new Set(overLimit.map(o => o.SourceObjectName.toLowerCase()));
+            for (let i = objects.length - 1; i >= 0; i--) {
+                if (disabled.has(objects[i].SourceObjectName.toLowerCase())) objects.splice(i, 1);
+            }
+            filteredSchema.Objects = filteredSchema.Objects.filter(o => !disabled.has(o.ExternalName.toLowerCase()));
+        }
+
+        // MaxTables remains an operator/env REJECT guardrail (unchanged), evaluated on the surviving objects.
+        // Columns are handled above via disable, so the pure decision is asked about tables only here.
+        const maxTables = ((): number | null => {
+            const v = parseInt(process.env.MJ_INTEGRATION_MAX_TABLES ?? '', 10);
+            return Number.isFinite(v) && v > 0 ? v : null;
+        })();
         const violations = decideSchemaLimitViolations({
             TableCount: objects.length,
-            ColumnCountByTable: objects.map(o => ({
-                Name: o.SourceObjectName,
-                ColumnCount: o.Fields?.length ?? fullCountByName.get(o.SourceObjectName.toLowerCase()) ?? 0,
-            })),
-            MaxTables: envInt('MJ_INTEGRATION_MAX_TABLES'),
-            MaxColumnsPerTable: envInt('MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE'),
+            ColumnCountByTable: [],
+            MaxTables: maxTables,
+            MaxColumnsPerTable: null,
         });
         if (violations.length > 0) throw new Error(violations.join(' '));
+        return warnings;
     }
 
     private async buildSchemaForConnector(
@@ -3717,7 +3793,7 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
         // §B — reject an over-limit table/column SELECTION before ANY table is materialized. This is the
         // single shared gate for ApplyAll / ApplyAllBatch / ApplySchemaBatch (all route through here).
         // Caps are operator/env guardrails (MJ_INTEGRATION_MAX_TABLES / _MAX_COLUMNS_PER_TABLE).
-        this.enforceSchemaLimits(objects, filteredSchema);
+        const limitWarnings = await this.enforceSchemaLimits(objects, filteredSchema, companyIntegration, user, provider);
 
         const targetConfigs = this.buildTargetConfigs(objects, filteredSchema, platform, connector);
 
@@ -3736,6 +3812,8 @@ export class IntegrationDiscoveryResolver extends ResolverBase {
 
         const builder = new SchemaBuilder();
         const schemaOutput = builder.BuildSchema(input);
+        // Surface the auto-disabled over-wide objects (if any) in the apply's Warnings → CLI output.
+        if (limitWarnings.length > 0) schemaOutput.Warnings.unshift(...limitWarnings);
 
         if (schemaOutput.Errors.length > 0) {
             throw new Error(`Schema generation failed: ${schemaOutput.Errors.join('; ')}`);