@memberjunction/server 5.40.2 → 5.42.0

package/dist/resolvers/IntegrationDiscoveryResolver.js

@@ -11,16 +11,18 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
     return function (target, key) { decorator(target, key, paramIndex); }
 };
 var IntegrationDiscoveryResolver_1;
-import { Resolver, Query, Mutation, Arg, Ctx, ObjectType, Field, InputType } from "type-graphql";
+import { Resolver, Query, Mutation, Arg, Ctx, ObjectType, Field, InputType, Int, Float } from "type-graphql";
 import { CompositeKey, LocalCacheManager, Metadata, RunView, LogError, LogStatus } from "@memberjunction/core";
 import { GetReadOnlyProvider, GetReadWriteProvider } from "../util.js";
 import { CronExpressionHelper } from "@memberjunction/scheduling-engine";
-import { ConnectorFactory, IntegrationEngine, IntegrationSchemaSync, IntegrationConnectorCreationPipeline, IntegrationActionGenerator } from "@memberjunction/integration-engine";
+import { ConnectorFactory, IntegrationEngine, IntegrationSchemaSync, decideSchemaLimitViolations, IntegrationConnectorCreationPipeline, IntegrationActionGenerator } from "@memberjunction/integration-engine";
 import { IntegrationEngineBase } from "@memberjunction/integration-engine-base";
 import { SchemaBuilder, TypeMapper, SchemaEvolution } from "@memberjunction/integration-schema-builder";
 import { RuntimeSchemaManager } from "@memberjunction/schema-engine";
 import { IntegrationProgressReader } from "@memberjunction/integration-progress-artifacts";
 import { ResolverBase } from "../generic/ResolverBase.js";
+import { IntegrationCustomColumnPromoter } from "../integration/CustomColumnPromoter.js";
+import { RequireSystemUser } from "../directives/RequireSystemUser.js";
 import { UserCache } from "@memberjunction/sqlserver-dataprovider";
 // ─── RSU Pipeline Output Types ──────────────────────────────────────────────
 let RSUStepOutput = class RSUStepOutput {
@@ -1105,6 +1107,188 @@ __decorate([
 MutationResultOutput = __decorate([
     ObjectType()
 ], MutationResultOutput);
+// ─── Typed sync-config (rate-limit / concurrency / time-budget as STRUCTURED fields, not a raw
+//     Configuration JSON blob). These map to the CompanyIntegration.Configuration keys the engine
+//     reads at runtime, so they are customizable per-connection via the API instead of hidden code
+//     constants. Set merges (preserves other Configuration keys); Get reads them back typed. ──────
+let IntegrationSyncConfigInput = class IntegrationSyncConfigInput {
+};
+__decorate([
+    Field(() => Int, { nullable: true, description: 'Entity maps processed concurrently within a dependency layer (clamped 1-16). Default 1 (sequential).' }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigInput.prototype, "SyncConcurrency", void 0);
+__decorate([
+    Field(() => Int, { nullable: true, description: 'Upper bound the per-layer AIMD controller ramps toward. Default = connector MaxConcurrencyHint.' }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigInput.prototype, "MaxConcurrency", void 0);
+__decorate([
+    Field(() => Float, { nullable: true, description: 'Override the source rate limit (requests/sec). Default = connector RateLimitPolicy / derived.' }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigInput.prototype, "RateLimitTokensPerSec", void 0);
+__decorate([
+    Field(() => Int, { nullable: true, description: 'Override the rate-limiter burst capacity (tokens).' }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigInput.prototype, "RateLimitBurst", void 0);
+__decorate([
+    Field(() => Boolean, { nullable: true, description: '§4 cross-layer pipelining: a child map starts when ITS parents finish, not the whole layer.' }),
+    __metadata("design:type", Boolean)
+], IntegrationSyncConfigInput.prototype, "CrossLayerPipeline", void 0);
+__decorate([
+    Field(() => Boolean, { nullable: true, description: 'Merkle/partition hash-diff reconcile for watermark-less change detection (buffers the set in RAM).' }),
+    __metadata("design:type", Boolean)
+], IntegrationSyncConfigInput.prototype, "PartitionReconcile", void 0);
+__decorate([
+    Field(() => Int, { nullable: true, description: 'Time budget (ms) for stage-2 streaming field discovery before it stops and uses what it gathered.' }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigInput.prototype, "DiscoveryTimeBudgetMs", void 0);
+__decorate([
+    Field(() => Int, { nullable: true, description: 'Batch size for stage-2 streaming field discovery (records per FetchChanges page). Default 500.' }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigInput.prototype, "DiscoveryBatchSize", void 0);
+__decorate([
+    Field(() => Int, { nullable: true, description: 'Max records sampled in stage-2 streaming field discovery (a column corpus + PK guess; NOT a full scan). Default 500.' }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigInput.prototype, "DiscoveryMaxRecords", void 0);
+__decorate([
+    Field(() => Boolean, { nullable: true, description: '§7 Comprehensive refresh deactivates Declared objects/fields ABSENT from an AUTHORITATIVE discovery (reversible). Default false.' }),
+    __metadata("design:type", Boolean)
+], IntegrationSyncConfigInput.prototype, "DeactivateAbsent", void 0);
+IntegrationSyncConfigInput = __decorate([
+    InputType()
+], IntegrationSyncConfigInput);
+let IntegrationSyncConfigOutput = class IntegrationSyncConfigOutput {
+};
+__decorate([
+    Field(),
+    __metadata("design:type", Boolean)
+], IntegrationSyncConfigOutput.prototype, "Success", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], IntegrationSyncConfigOutput.prototype, "Message", void 0);
+__decorate([
+    Field(() => Int, { nullable: true }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigOutput.prototype, "SyncConcurrency", void 0);
+__decorate([
+    Field(() => Int, { nullable: true }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigOutput.prototype, "MaxConcurrency", void 0);
+__decorate([
+    Field(() => Float, { nullable: true }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigOutput.prototype, "RateLimitTokensPerSec", void 0);
+__decorate([
+    Field(() => Int, { nullable: true }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigOutput.prototype, "RateLimitBurst", void 0);
+__decorate([
+    Field(() => Boolean, { nullable: true }),
+    __metadata("design:type", Boolean)
+], IntegrationSyncConfigOutput.prototype, "CrossLayerPipeline", void 0);
+__decorate([
+    Field(() => Boolean, { nullable: true }),
+    __metadata("design:type", Boolean)
+], IntegrationSyncConfigOutput.prototype, "PartitionReconcile", void 0);
+__decorate([
+    Field(() => Int, { nullable: true }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigOutput.prototype, "DiscoveryTimeBudgetMs", void 0);
+__decorate([
+    Field(() => Int, { nullable: true }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigOutput.prototype, "DiscoveryBatchSize", void 0);
+__decorate([
+    Field(() => Int, { nullable: true }),
+    __metadata("design:type", Number)
+], IntegrationSyncConfigOutput.prototype, "DiscoveryMaxRecords", void 0);
+__decorate([
+    Field(() => Boolean, { nullable: true }),
+    __metadata("design:type", Boolean)
+], IntegrationSyncConfigOutput.prototype, "DeactivateAbsent", void 0);
+IntegrationSyncConfigOutput = __decorate([
+    ObjectType()
+], IntegrationSyncConfigOutput);
+let CustomColumnCandidate = class CustomColumnCandidate {
+};
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], CustomColumnCandidate.prototype, "EntityName", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], CustomColumnCandidate.prototype, "SourceKey", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], CustomColumnCandidate.prototype, "ColumnName", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], CustomColumnCandidate.prototype, "InferredType", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", Boolean)
+], CustomColumnCandidate.prototype, "NeedsColumn", void 0);
+CustomColumnCandidate = __decorate([
+    ObjectType()
+], CustomColumnCandidate);
+let CustomColumnCandidatesOutput = class CustomColumnCandidatesOutput {
+};
+__decorate([
+    Field(),
+    __metadata("design:type", Boolean)
+], CustomColumnCandidatesOutput.prototype, "Success", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], CustomColumnCandidatesOutput.prototype, "Message", void 0);
+__decorate([
+    Field(() => [CustomColumnCandidate]),
+    __metadata("design:type", Array)
+], CustomColumnCandidatesOutput.prototype, "Candidates", void 0);
+CustomColumnCandidatesOutput = __decorate([
+    ObjectType()
+], CustomColumnCandidatesOutput);
+let PromotedColumn = class PromotedColumn {
+};
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], PromotedColumn.prototype, "EntityName", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], PromotedColumn.prototype, "ColumnName", void 0);
+PromotedColumn = __decorate([
+    ObjectType()
+], PromotedColumn);
+let PromoteCustomColumnsOutput = class PromoteCustomColumnsOutput {
+};
+__decorate([
+    Field(),
+    __metadata("design:type", Boolean)
+], PromoteCustomColumnsOutput.prototype, "Success", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", String)
+], PromoteCustomColumnsOutput.prototype, "Message", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", Boolean)
+], PromoteCustomColumnsOutput.prototype, "Promoted", void 0);
+__decorate([
+    Field(() => [PromotedColumn]),
+    __metadata("design:type", Array)
+], PromoteCustomColumnsOutput.prototype, "ColumnsAdded", void 0);
+__decorate([
+    Field(),
+    __metadata("design:type", Boolean)
+], PromoteCustomColumnsOutput.prototype, "SchemaUpdatePending", void 0);
+PromoteCustomColumnsOutput = __decorate([
+    ObjectType()
+], PromoteCustomColumnsOutput);
 let FieldMapInput = class FieldMapInput {
 };
 __decorate([
@@ -1261,6 +1445,14 @@ __decorate([
     Field({ nullable: true }),
     __metadata("design:type", Boolean)
 ], CreateScheduleInput.prototype, "FullSync", void 0);
+__decorate([
+    Field({ nullable: true, defaultValue: 'sync' }),
+    __metadata("design:type", String)
+], CreateScheduleInput.prototype, "JobKind", void 0);
+__decorate([
+    Field({ nullable: true }),
+    __metadata("design:type", Boolean)
+], CreateScheduleInput.prototype, "DeactivateAbsent", void 0);
 CreateScheduleInput = __decorate([
     InputType()
 ], CreateScheduleInput);
@@ -2203,7 +2395,7 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
      * both on stdout (visible in the MJAPI log file) and in a per-run
      * `<cwd>/logs/integration-runs/<runID>/progress.jsonl` artifact.
      */
-    async IntegrationRefreshConnectorSchema(companyIntegrationID, universalPKConvention, ctx) {
+    async IntegrationRefreshConnectorSchema(companyIntegrationID, universalPKConvention, deactivateAbsent, ctx) {
         try {
             const user = this.getAuthenticatedUser(ctx);
             const provider = GetReadWriteProvider(ctx.providers);
@@ -2221,6 +2413,11 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
                 UniversalPKConvention: universalPKConvention || undefined,
                 ConsoleMirror: true,
                 TriggerType: 'Manual',
+                // §7 — explicit RefreshConnectorSchema is a comprehensive re-discovery: default to
+                // deactivating objects/fields the source no longer exposes (reversible). Precedence:
+                // explicit arg > persisted Configuration.deactivateAbsent (set via IntegrationSetSyncConfig) >
+                // comprehensive default (true). Caller can opt out per-call or per-connection.
+                DeactivateAbsent: deactivateAbsent ?? this.readConfigBool(companyIntegration.Configuration, 'deactivateAbsent') ?? true,
             };
             const result = await pipeline.Run(runOpts);
             // Refresh the metadata cache so subsequent reads see the new IO/IOF
@@ -2502,14 +2699,20 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
                     SourceFieldName: f.Name,
                     TargetColumnName: f.Name.replace(/[^A-Za-z0-9_]/g, '_'),
                     TargetSqlType: targetSqlType,
-                    // Synced shadow tables must NOT enforce NOT NULL on non-PK
-                    // columns. The external system (SF, HubSpot, etc.) is the
-                    // source of truth for business data, not for MJ's schema
-                    // constraints — and its describe output often declares
-                    // fields required when real records actually have nulls
-                    // (deprecated, calculated, or edge-case fields). Enforcing
-                    // NOT NULL here just aborts entire batches on one bad row.
-                    IsNullable: !f.IsPrimaryKey,
+                    // Synced shadow tables must NOT enforce NOT NULL on ANY
+                    // column — including the PK. The external system (SF, HubSpot,
+                    // GrowthZone, etc.) is the source of truth for business data,
+                    // not for MJ's schema constraints — and its describe output
+                    // often declares fields required when real records actually
+                    // have nulls (deprecated, calculated, or edge-case fields).
+                    // Integration PKs are SOFT (tracked via SchemaBuilder.SoftPrimaryKeys
+                    // for upsert/dedup; identity falls back to a content-hash when the
+                    // PK is null/partial — §4). Emitting the PK column NOT NULL breaks
+                    // that fallback: a source row with a null PK (e.g. nested/derived
+                    // records like event sponsors, contact phones) aborts the insert
+                    // before content-hash can save it. So the soft-PK column is nullable
+                    // too; uniqueness/identity is enforced logically, not by the DDL.
+                    IsNullable: true,
                     MaxLength: f.MaxLength,
                     Precision: f.Precision,
                     Scale: f.Scale,
@@ -2527,12 +2730,15 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
                 LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — 0 fields discovered (live API likely failed and no DB-cached fields available)`);
                 continue;
             }
-            // If columns exist but no PK was found, log diagnostic info and skip rather than
-            // generating broken DDL with UNIQUE ([ID]) on a non-existent column.
+            // Provable-only: no PK we could prove from the streamed data (single OR composite) means
+            // the object is NOT added — and we say so clearly. We never fabricate a key (e.g. "all
+            // columns as the PK"); a wrong identity is worse than an honest omission. The fix for a
+            // missing PK is to STREAM MORE DATA at discovery time so the stats can prove one, not to
+            // invent a key here.
             if (primaryKeyFields.length === 0 && columns.length > 0) {
                 droppedNoPrimaryKey.push(obj.SourceObjectName);
                 const fieldNames = sourceObj.Fields.map(f => `${f.Name}(pk=${f.IsPrimaryKey})`).join(', ');
-                LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — ${columns.length} columns but NO primary key field found. Fields: [${fieldNames}]`);
+                LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — ${columns.length} columns but NO provable primary key. Fields: [${fieldNames}]`);
                 continue;
             }
             results.push({
@@ -2992,7 +3198,12 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
      */
     buildSourceSchemaFromPersistedRows(integrationID, requestedNames) {
         const engine = IntegrationEngineBase.Instance;
-        const ios = engine.GetIntegrationObjectsByIntegrationID(integrationID);
+        // ACTIVE-only materialization: an object/field a given tenant doesn't expose is marked
+        // Status='Inactive' by discovery (the phantom-skip), and MUST NOT be materialized — creating
+        // empty tables/columns for absent objects wastes storage AND, more importantly, blows up the
+        // per-entity CodeGen + advancedGen (AI form-layout) time on every ApplyAll. The sync path
+        // already filters active (GetActiveIntegrationObjects); this build site now matches it.
+        const ios = engine.GetActiveIntegrationObjects(integrationID);
         const filter = requestedNames && requestedNames.length > 0
             ? new Set(requestedNames.map(n => n.toLowerCase()))
             : null;
@@ -3005,7 +3216,8 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
         for (const io of ios) {
             if (filter && !filter.has(io.Name.toLowerCase()))
                 continue;
-            const iofs = engine.GetIntegrationObjectFields(io.ID);
+            // Active fields only — an inactive (source-absent / deactivated) field is not materialized.
+            const iofs = engine.GetIntegrationObjectFields(io.ID).filter(iof => iof.Status === 'Active');
             const fields = iofs.map(iof => {
                 const targetIOName = iof.RelatedIntegrationObjectID
                     ? ioByID.get(iof.RelatedIntegrationObjectID) ?? null
@@ -3394,6 +3606,170 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
             return { Success: false, Message: this.formatError(e) };
         }
     }
+    /**
+     * Sets the per-connection sync tuning (rate limit, concurrency, time budget, pipeline flags) as
+     * STRUCTURED typed fields, merged into CompanyIntegration.Configuration (other keys preserved).
+     * These are the exact keys the IntegrationEngine reads at runtime, so they become customizable
+     * via the API instead of hidden code constants. Returns the merged config typed.
+     */
+    async IntegrationSetSyncConfig(companyIntegrationID, config, ctx) {
+        try {
+            const user = this.getAuthenticatedUser(ctx);
+            const md = GetReadWriteProvider(ctx.providers, { allowFallbackToReadOnly: true });
+            const ci = await md.GetEntityObject('MJ: Company Integrations', user);
+            if (!await ci.InnerLoad(CompositeKey.FromID(companyIntegrationID))) {
+                return { Success: false, Message: 'CompanyIntegration not found' };
+            }
+            let cfg = {};
+            try {
+                if (ci.Configuration)
+                    cfg = JSON.parse(ci.Configuration);
+            }
+            catch {
+                cfg = {};
+            }
+            const set = (key, val) => { if (val !== undefined && val !== null)
+                cfg[key] = val; };
+            set('syncConcurrency', config.SyncConcurrency);
+            set('maxConcurrency', config.MaxConcurrency);
+            set('rateLimitTokensPerSec', config.RateLimitTokensPerSec);
+            set('rateLimitBurst', config.RateLimitBurst);
+            set('crossLayerPipeline', config.CrossLayerPipeline);
+            set('partitionReconcile', config.PartitionReconcile);
+            set('discoveryTimeBudgetMs', config.DiscoveryTimeBudgetMs);
+            set('discoveryBatchSize', config.DiscoveryBatchSize);
+            set('discoveryMaxRecords', config.DiscoveryMaxRecords);
+            set('deactivateAbsent', config.DeactivateAbsent);
+            ci.Configuration = JSON.stringify(cfg);
+            if (!await ci.Save())
+                return { Success: false, Message: `Failed to save: ${ci.LatestResult?.CompleteMessage ?? 'unknown'}` };
+            return { Success: true, Message: 'Sync config updated', ...this.readSyncConfig(cfg) };
+        }
+        catch (e) {
+            LogError(`IntegrationSetSyncConfig error: ${e}`);
+            return { Success: false, Message: this.formatError(e) };
+        }
+    }
+    /** Reads the per-connection sync tuning back as STRUCTURED typed fields. */
+    async IntegrationGetSyncConfig(companyIntegrationID, ctx) {
+        try {
+            const user = this.getAuthenticatedUser(ctx);
+            const md = GetReadWriteProvider(ctx.providers, { allowFallbackToReadOnly: true });
+            const ci = await md.GetEntityObject('MJ: Company Integrations', user);
+            if (!await ci.InnerLoad(CompositeKey.FromID(companyIntegrationID))) {
+                return { Success: false, Message: 'CompanyIntegration not found' };
+            }
+            let cfg = {};
+            try {
+                if (ci.Configuration)
+                    cfg = JSON.parse(ci.Configuration);
+            }
+            catch {
+                cfg = {};
+            }
+            return { Success: true, Message: 'OK', ...this.readSyncConfig(cfg) };
+        }
+        catch (e) {
+            LogError(`IntegrationGetSyncConfig error: ${e}`);
+            return { Success: false, Message: this.formatError(e) };
+        }
+    }
+    /** Reads a single boolean key from a CompanyIntegration.Configuration JSON string (undefined if absent/malformed). */
+    readConfigBool(configuration, key) {
+        try {
+            if (!configuration)
+                return undefined;
+            const v = JSON.parse(configuration)[key];
+            return typeof v === 'boolean' ? v : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /** Extracts the typed sync-config fields from a parsed Configuration object (type-guarded). */
+    readSyncConfig(cfg) {
+        const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : undefined);
+        const bool = (v) => (typeof v === 'boolean' ? v : undefined);
+        return {
+            SyncConcurrency: num(cfg.syncConcurrency),
+            MaxConcurrency: num(cfg.maxConcurrency),
+            RateLimitTokensPerSec: num(cfg.rateLimitTokensPerSec),
+            RateLimitBurst: num(cfg.rateLimitBurst),
+            CrossLayerPipeline: bool(cfg.crossLayerPipeline),
+            PartitionReconcile: bool(cfg.partitionReconcile),
+            DiscoveryTimeBudgetMs: num(cfg.discoveryTimeBudgetMs),
+            DiscoveryBatchSize: num(cfg.discoveryBatchSize),
+            DiscoveryMaxRecords: num(cfg.discoveryMaxRecords),
+            DeactivateAbsent: bool(cfg.deactivateAbsent),
+        };
+    }
+    /** The MJ entity names this connection has entity maps for (for whole-connection scope). */
+    async getMappedEntityNames(companyIntegrationID, user) {
+        const rv = new RunView();
+        const res = await rv.RunView({
+            EntityName: 'MJ: Company Integration Entity Maps',
+            ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}'`,
+            Fields: ['Entity'],
+            ResultType: 'simple',
+        }, user);
+        if (!res.Success)
+            return [];
+        return Array.from(new Set((res.Results ?? []).map(r => r.Entity).filter((e) => !!e)));
+    }
+    /**
+     * Lists the custom-column CANDIDATES captured in the overflow column awaiting promotion — the "new
+     * columns found" for a connection. READ-ONLY (no schema change, no RSU). Scope to one entity via
+     * entityName, or omit to list across all the connection's mapped entities. Computed live (overflow keys
+     * minus already-mapped/already-a-column), so it is inherently deduped against anything a concurrent
+     * discovery already promoted.
+     */
+    async IntegrationListCustomColumnCandidates(companyIntegrationID, entityName, ctx) {
+        try {
+            const user = this.getAuthenticatedUser(ctx);
+            const provider = GetReadWriteProvider(ctx.providers, { allowFallbackToReadOnly: true });
+            const promoter = new IntegrationCustomColumnPromoter(user, provider);
+            const entityNames = entityName ? [entityName] : await this.getMappedEntityNames(companyIntegrationID, user);
+            const candidates = [];
+            for (const en of entityNames) {
+                candidates.push(...await promoter.ListCandidates(companyIntegrationID, en));
+            }
+            return { Success: true, Message: `${candidates.length} candidate column(s) found`, Candidates: candidates };
+        }
+        catch (e) {
+            LogError(`IntegrationListCustomColumnCandidates error: ${e}`);
+            return { Success: false, Message: this.formatError(e), Candidates: [] };
+        }
+    }
+    /**
+     * On-demand promotion of captured custom columns: runs RSU (ADD COLUMN + register EntityField + field map),
+     * which may require a server restart to expose them over GraphQL. This is the USER-ACCEPTED trigger — by
+     * default a sync only CAPTURES to the overflow column (auto-promote is opt-in per connection via
+     * Configuration.autoPromoteCustomColumns). Scope via entityNames, or omit to promote across all mapped
+     * entities. Idempotent: already-promoted/mapped keys are skipped (safe to re-run / run alongside discovery).
+     */
+    async IntegrationPromoteCustomColumns(companyIntegrationID, entityNames, ctx) {
+        try {
+            const user = this.getAuthenticatedUser(ctx);
+            const provider = GetReadWriteProvider(ctx.providers);
+            const targets = (entityNames && entityNames.length > 0) ? entityNames : await this.getMappedEntityNames(companyIntegrationID, user);
+            if (targets.length === 0) {
+                return { Success: true, Message: 'No mapped entities to promote', Promoted: false, ColumnsAdded: [], SchemaUpdatePending: false };
+            }
+            const promoter = new IntegrationCustomColumnPromoter(user, provider);
+            const result = await promoter.PromoteForSync(companyIntegrationID, targets);
+            return {
+                Success: true,
+                Message: result.Promoted ? `Promoted ${result.ColumnsAdded.length} column(s)` : 'No columns required promotion',
+                Promoted: result.Promoted,
+                ColumnsAdded: result.ColumnsAdded.map(c => ({ EntityName: c.EntityName, ColumnName: c.ColumnName })),
+                SchemaUpdatePending: result.SchemaUpdatePending,
+            };
+        }
+        catch (e) {
+            LogError(`IntegrationPromoteCustomColumns error: ${e}`);
+            return { Success: false, Message: this.formatError(e), Promoted: false, ColumnsAdded: [], SchemaUpdatePending: false };
+        }
+    }
     /**
      * Soft-deletes a CompanyIntegration by setting IsActive=false.
      */
@@ -4040,6 +4416,35 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
      * Build schema artifacts for a single connector's objects.
      * Shared by IntegrationApplySchema (single) and IntegrationApplySchemaBatch (batch).
      */
+    /**
+     * §B — enforce OPTIONAL table/column caps at the create-tables (RSU) gate. These are OPERATOR
+     * (deployment) guardrails read from ENV — `MJ_INTEGRATION_MAX_TABLES` / `MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE`
+     * (absent or ≤0 = unbounded, the common case). They are DELIBERATELY env-only, NOT per-connection
+     * `Configuration`/GraphQL: a guardrail a user can raise via the same API they apply with is toothless.
+     * THROWS a clear error when the selection exceeds a cap so NOTHING partial is created — the caller surfaces
+     * it and the user narrows the selection (the cap itself is an operator concern). NEVER truncates. Discovery
+     * still surfaces every object/field; only materialization is capped. Per-table column count = the selected
+     * field subset, or the object's full discovered field count when all fields are selected.
+     */
+    enforceSchemaLimits(objects, filteredSchema) {
+        const envInt = (name) => {
+            const v = parseInt(process.env[name] ?? '', 10);
+            return Number.isFinite(v) && v > 0 ? v : null;
+        };
+        const fullCountByName = new Map(filteredSchema.Objects.map(o => [o.ExternalName.toLowerCase(), o.Fields.length]));
+        // Delegate the cap decision to the engine's pure, unit-tested decideSchemaLimitViolations.
+        const violations = decideSchemaLimitViolations({
+            TableCount: objects.length,
+            ColumnCountByTable: objects.map(o => ({
+                Name: o.SourceObjectName,
+                ColumnCount: o.Fields?.length ?? fullCountByName.get(o.SourceObjectName.toLowerCase()) ?? 0,
+            })),
+            MaxTables: envInt('MJ_INTEGRATION_MAX_TABLES'),
+            MaxColumnsPerTable: envInt('MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE'),
+        });
+        if (violations.length > 0)
+            throw new Error(violations.join(' '));
+    }
     async buildSchemaForConnector(companyIntegrationID, objects, platform, user, skipGitCommit, skipRestart, provider, prefetchedSourceSchema) {
         const { connector, companyIntegration } = await this.resolveConnector(companyIntegrationID, user, provider);
         // Source-schema resolution order:
@@ -4082,6 +4487,10 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
         const filteredSchema = {
             Objects: sourceSchema.Objects.filter(o => requestedNames.has(o.ExternalName))
         };
+        // §B — reject an over-limit table/column SELECTION before ANY table is materialized. This is the
+        // single shared gate for ApplyAll / ApplyAllBatch / ApplySchemaBatch (all route through here).
+        // Caps are operator/env guardrails (MJ_INTEGRATION_MAX_TABLES / _MAX_COLUMNS_PER_TABLE).
+        this.enforceSchemaLimits(objects, filteredSchema);
         const targetConfigs = this.buildTargetConfigs(objects, filteredSchema, platform, connector);
         const input = {
             SourceSchema: filteredSchema,
@@ -4129,6 +4538,12 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
                 syncOptions.EntityMapIDs = entityMapIDs;
             if (syncDirection)
                 syncOptions.SyncDirection = syncDirection;
+            // Capture the fire instant BEFORE launching the sync. The run-record lookup below
+            // finds the run by recency (StartedAt >= firedAt) rather than by transient status, so
+            // a fast run (0-record / empty connector / quick failure) that finishes before we poll
+            // is still reported with its real RunID instead of an untrackable null. RunSync stamps
+            // StartedAt with an app-side `new Date()`, so this clock is consistent with the row.
+            const firedAt = new Date();
             // Fire and forget — progress is tracked inside IntegrationEngine
             const syncPromise = IntegrationEngine.Instance.RunSync(companyIntegrationID, user, 'Manual', undefined, undefined, Object.keys(syncOptions).length > 0 ? syncOptions : undefined);
             syncPromise
@@ -4157,22 +4572,43 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
                     });
                 }
             });
-            // Small delay to let the run record get created
-            await new Promise(resolve => setTimeout(resolve, 200));
-            const rv = new RunView();
-            const runResult = await rv.RunView({
-                EntityName: 'MJ: Company Integration Runs',
-                ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND Status='In Progress'`,
-                OrderBy: '__mj_CreatedAt DESC',
-                MaxRows: 1,
-                ResultType: 'simple',
-                Fields: ['ID', 'Status', 'StartedAt']
-            }, user);
-            const run = runResult.Success && runResult.Results.length > 0 ? runResult.Results[0] : null;
+            // Resolve the run record by RECENCY, not by transient 'In Progress' status. The prior
+            // implementation slept a fixed 200ms then filtered Status='In Progress' — when the run
+            // finished in under 200ms (an empty connector, a 0-record sync, or a fast synchronous
+            // failure) there was no 'In Progress' row to find, so it returned Success:true with a
+            // null RunID: an optimistic, untrackable result that read as "started" when nothing
+            // could be followed. Poll briefly for ANY run for this connector stamped at/after the
+            // fire instant; that catches both the still-running and the already-finished cases.
+            const firedAtFilter = firedAt.toISOString();
+            let run = null;
+            for (let attempt = 0; attempt < 15 && !run; attempt++) {
+                await new Promise(resolve => setTimeout(resolve, 200));
+                const rv = new RunView();
+                const runResult = await rv.RunView({
+                    EntityName: 'MJ: Company Integration Runs',
+                    ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND StartedAt >= '${firedAtFilter}'`,
+                    OrderBy: 'StartedAt DESC',
+                    MaxRows: 1,
+                    ResultType: 'simple',
+                    Fields: ['ID', 'Status', 'StartedAt']
+                }, user);
+                if (runResult.Success && runResult.Results.length > 0) {
+                    run = runResult.Results[0];
+                }
+            }
+            // No run record after the poll window ⇒ the sync genuinely did not start (it no-op'd
+            // before creating a run). Report that honestly rather than claiming Success with a null
+            // RunID — a caller/scheduler can act on a false, but a true+null silently strands it.
+            if (!run) {
+                return {
+                    Success: false,
+                    Message: 'Sync did not start — no run record was created (the connector may be gated or misconfigured)'
+                };
+            }
             return {
                 Success: true,
                 Message: 'Sync started',
-                RunID: run?.ID
+                RunID: run.ID
             };
         }
         catch (e) {
@@ -4275,16 +4711,21 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
             const user = this.getAuthenticatedUser(ctx);
             const md = GetReadWriteProvider(ctx.providers, { allowFallbackToReadOnly: true });
             const rv = new RunView();
-            // Find IntegrationSync job type
+            // §13 — select the driver by job-kind: 'discovery' = schema-only refresh on a cron;
+            // 'sync' (default) = the data-moving RunSync job.
+            const jobKind = (input.JobKind ?? 'sync').toLowerCase();
+            const driverClass = jobKind === 'discovery'
+                ? 'IntegrationDiscoveryScheduledJobDriver'
+                : 'IntegrationSyncScheduledJobDriver';
             const jobTypeResult = await rv.RunView({
                 EntityName: 'MJ: Scheduled Job Types',
-                ExtraFilter: `DriverClass='IntegrationSyncScheduledJobDriver'`,
+                ExtraFilter: `DriverClass='${driverClass}'`,
                 MaxRows: 1,
                 ResultType: 'simple',
                 Fields: ['ID']
             }, user);
             if (!jobTypeResult.Success || jobTypeResult.Results.length === 0) {
-                return { Success: false, Message: 'IntegrationSync scheduled job type not found' };
+                return { Success: false, Message: `Scheduled job type not found for driver '${driverClass}' (kind='${jobKind}')` };
             }
             const jobTypeID = jobTypeResult.Results[0].ID;
             const job = await md.GetEntityObject('MJ: Scheduled Jobs', user);
@@ -4298,10 +4739,18 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
             job.Status = 'Active';
             job.OwnerUserID = user.ID;
             const jobConfig = { CompanyIntegrationID: input.CompanyIntegrationID };
-            if (input.SyncDirection)
-                jobConfig.SyncDirection = input.SyncDirection;
-            if (input.FullSync)
-                jobConfig.FullSync = input.FullSync;
+            if (jobKind === 'discovery') {
+                // Discovery job: schema-only. SyncDirection/FullSync are meaningless here; carry the
+                // deactivate toggle instead (driver defaults it to true when omitted).
+                if (input.DeactivateAbsent !== undefined && input.DeactivateAbsent !== null)
+                    jobConfig.DeactivateAbsent = input.DeactivateAbsent;
+            }
+            else {
+                if (input.SyncDirection)
+                    jobConfig.SyncDirection = input.SyncDirection;
+                if (input.FullSync)
+                    jobConfig.FullSync = input.FullSync;
+            }
             job.Configuration = JSON.stringify(jobConfig);
             job.NextRunAt = CronExpressionHelper.GetNextRunTime(input.CronExpression, input.Timezone || 'UTC');
             if (!await job.Save())
@@ -5477,12 +5926,18 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
             if (!submitted) {
                 return { Success: false, Message: 'Transaction failed — all deletes rolled back' };
             }
-            if (deleteData) {
-                LogError(`IntegrationDeleteConnection: deleteData=true requested but table deletion not yet implemented for ${companyIntegrationID}`);
-            }
+            // deleteData=true asks us to also DROP the physical mirror tables. We deliberately do NOT:
+            // RuntimeSchemaManager rejects DROP TABLE on the __mj schema by design (a destructive-op
+            // safety guard), and there is no DROP generator. Dropping integration tables is real data
+            // loss and must be an explicit, separately-designed operation — not a silent side effect of
+            // removing a connection. So we delete the connection + all METADATA (maps/fields/schedules/
+            // credential) and RETAIN the data tables, and we say so honestly rather than claim otherwise.
+            const dataNote = deleteData
+                ? ' The physical data tables were RETAINED (dropping integration tables is not performed automatically — it is a separate, explicit operation).'
+                : '';
             return {
                 Success: true,
-                Message: `Deleted connection and all associated records`,
+                Message: `Deleted connection and all associated metadata records.${dataNote}`,
                 EntityMapsDeleted: entityMapsDeleted,
                 FieldMapsDeleted: fieldMapsDeleted,
                 SchedulesDeleted: schedulesDeleted,
@@ -5699,15 +6154,18 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationTestConnection", null);
 __decorate([
     Mutation(() => RefreshConnectorSchemaOutput),
+    RequireSystemUser(),
     __param(0, Arg("companyIntegrationID")),
     __param(1, Arg("universalPKConvention", { nullable: true, description: "Optional vendor-wide PK convention hint (e.g. 'id' for HubSpot)" })),
-    __param(2, Ctx()),
+    __param(2, Arg("deactivateAbsent", { nullable: true, description: "Comprehensive refresh (default true): objects/fields ABSENT from this discovery are deactivated (Status='Disabled', never deleted, reversible on a later rediscovery). Pass false for a scoped/partial discovery so it never disables what it didn't probe." })),
+    __param(3, Ctx()),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, String, Object]),
+    __metadata("design:paramtypes", [String, String, Boolean, Object]),
     __metadata("design:returntype", Promise)
 ], IntegrationDiscoveryResolver.prototype, "IntegrationRefreshConnectorSchema", null);
 __decorate([
     Mutation(() => IntegrationGenerateActionOutput),
+    RequireSystemUser(),
     __param(0, Arg("integrationName")),
     __param(1, Arg("objectName")),
     __param(2, Arg("verb", { nullable: true, description: "Optional CRUD verb (Get|Create|Update|Delete|Search|List). Omit to generate all applicable verbs for the object." })),
@@ -5755,6 +6213,7 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationListConnections", null);
 __decorate([
     Mutation(() => CreateConnectionOutput),
+    RequireSystemUser(),
     __param(0, Arg("input")),
     __param(1, Arg("testConnection", () => Boolean, { defaultValue: false })),
     __param(2, Arg("runSchemaRefresh", () => Boolean, { defaultValue: true, description: "When true (default) and TestConnection succeeds, automatically runs IntegrationConnectorCreationPipeline (live introspect → persist Declared/Discovered/Custom → SoftPKClassifier). The intermittent server-side work the wizard's Forward step represents." })),
@@ -5778,6 +6237,42 @@ __decorate([
     __metadata("design:paramtypes", [String, String, String, String, Boolean, Boolean, String, Object]),
     __metadata("design:returntype", Promise)
 ], IntegrationDiscoveryResolver.prototype, "IntegrationUpdateConnection", null);
+__decorate([
+    Mutation(() => IntegrationSyncConfigOutput),
+    __param(0, Arg("companyIntegrationID")),
+    __param(1, Arg("config", () => IntegrationSyncConfigInput)),
+    __param(2, Ctx()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, IntegrationSyncConfigInput, Object]),
+    __metadata("design:returntype", Promise)
+], IntegrationDiscoveryResolver.prototype, "IntegrationSetSyncConfig", null);
+__decorate([
+    Query(() => IntegrationSyncConfigOutput),
+    __param(0, Arg("companyIntegrationID")),
+    __param(1, Ctx()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, Object]),
+    __metadata("design:returntype", Promise)
+], IntegrationDiscoveryResolver.prototype, "IntegrationGetSyncConfig", null);
+__decorate([
+    Query(() => CustomColumnCandidatesOutput),
+    __param(0, Arg("companyIntegrationID")),
+    __param(1, Arg("entityName", { nullable: true })),
+    __param(2, Ctx()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, String, Object]),
+    __metadata("design:returntype", Promise)
+], IntegrationDiscoveryResolver.prototype, "IntegrationListCustomColumnCandidates", null);
+__decorate([
+    Mutation(() => PromoteCustomColumnsOutput),
+    RequireSystemUser(),
+    __param(0, Arg("companyIntegrationID")),
+    __param(1, Arg("entityNames", () => [String], { nullable: true })),
+    __param(2, Ctx()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, Array, Object]),
+    __metadata("design:returntype", Promise)
+], IntegrationDiscoveryResolver.prototype, "IntegrationPromoteCustomColumns", null);
 __decorate([
     Mutation(() => MutationResultOutput),
     __param(0, Arg("companyIntegrationID")),
@@ -5805,6 +6300,7 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationCreateEntityMaps", null);
 __decorate([
     Mutation(() => ApplySchemaOutput),
+    RequireSystemUser(),
     __param(0, Arg("companyIntegrationID")),
     __param(1, Arg("objects", () => [SchemaPreviewObjectInput])),
     __param(2, Arg("platform", { defaultValue: "sqlserver" })),
@@ -5817,6 +6313,7 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationApplySchema", null);
 __decorate([
     Mutation(() => ApplySchemaBatchOutput),
+    RequireSystemUser(),
     __param(0, Arg("items", () => [ApplySchemaBatchItemInput])),
     __param(1, Arg("platform", { defaultValue: "sqlserver" })),
     __param(2, Arg("skipGitCommit", { defaultValue: false })),
@@ -5828,6 +6325,7 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationApplySchemaBatch", null);
 __decorate([
     Mutation(() => ApplyAllOutput),
+    RequireSystemUser(),
     __param(0, Arg("input")),
     __param(1, Arg("platform", { defaultValue: "sqlserver" })),
     __param(2, Arg("skipGitCommit", { defaultValue: false })),
@@ -5839,6 +6337,7 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationApplyAll", null);
 __decorate([
     Mutation(() => StartSyncOutput),
+    RequireSystemUser(),
     __param(0, Arg("companyIntegrationID")),
     __param(1, Arg("webhookURL", { nullable: true })),
     __param(2, Arg("fullSync", () => Boolean, { defaultValue: false, description: 'If true, ignores watermarks and re-fetches all records from the source' })),
@@ -5859,6 +6358,7 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationCancelSync", null);
 __decorate([
     Mutation(() => WriteRecordOutput),
+    RequireSystemUser(),
     __param(0, Arg("companyIntegrationID")),
     __param(1, Arg("objectName")),
     __param(2, Arg("operation", () => String, { description: 'create, update, or delete' })),
@@ -6016,6 +6516,7 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationGetConnectorCapabilities", null);
 __decorate([
     Mutation(() => ApplyAllBatchOutput),
+    RequireSystemUser(),
     __param(0, Arg("input")),
     __param(1, Arg("platform", { defaultValue: "sqlserver" })),
     __param(2, Arg("skipGitCommit", { defaultValue: false })),
@@ -6036,6 +6537,7 @@ __decorate([
 ], IntegrationDiscoveryResolver.prototype, "IntegrationDeleteConnection", null);
 __decorate([
     Mutation(() => SchemaEvolutionOutput),
+    RequireSystemUser(),
     __param(0, Arg("companyIntegrationID")),
     __param(1, Arg("platform", { defaultValue: "sqlserver" })),
     __param(2, Arg("skipGitCommit", { defaultValue: false })),