@memberjunction/server 5.41.0 → 5.43.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/dist/agentSessions/SessionJanitor.d.ts.map +1 -1
  2. package/dist/agentSessions/SessionJanitor.js +5 -1
  3. package/dist/agentSessions/SessionJanitor.js.map +1 -1
  4. package/dist/agentSessions/SessionManager.d.ts +15 -2
  5. package/dist/agentSessions/SessionManager.d.ts.map +1 -1
  6. package/dist/agentSessions/SessionManager.js +50 -11
  7. package/dist/agentSessions/SessionManager.js.map +1 -1
  8. package/dist/agentSessions/index.d.ts +1 -0
  9. package/dist/agentSessions/index.d.ts.map +1 -1
  10. package/dist/agentSessions/index.js +5 -0
  11. package/dist/agentSessions/index.js.map +1 -1
  12. package/dist/agentSessions/remoteBrowserGoalEngine.d.ts +102 -0
  13. package/dist/agentSessions/remoteBrowserGoalEngine.d.ts.map +1 -0
  14. package/dist/agentSessions/remoteBrowserGoalEngine.js +164 -0
  15. package/dist/agentSessions/remoteBrowserGoalEngine.js.map +1 -0
  16. package/dist/agentSessions/remoteBrowserGoalRegistry.d.ts +66 -0
  17. package/dist/agentSessions/remoteBrowserGoalRegistry.d.ts.map +1 -0
  18. package/dist/agentSessions/remoteBrowserGoalRegistry.js +86 -0
  19. package/dist/agentSessions/remoteBrowserGoalRegistry.js.map +1 -0
  20. package/dist/auth/initializeProviders.d.ts.map +1 -1
  21. package/dist/auth/initializeProviders.js +4 -7
  22. package/dist/auth/initializeProviders.js.map +1 -1
  23. package/dist/config.d.ts +7 -0
  24. package/dist/config.d.ts.map +1 -1
  25. package/dist/config.js +12 -2
  26. package/dist/config.js.map +1 -1
  27. package/dist/generated/generated.d.ts +612 -0
  28. package/dist/generated/generated.d.ts.map +1 -1
  29. package/dist/generated/generated.js +6934 -3626
  30. package/dist/generated/generated.js.map +1 -1
  31. package/dist/index.d.ts.map +1 -1
  32. package/dist/index.js +38 -17
  33. package/dist/index.js.map +1 -1
  34. package/dist/integration/CustomColumnPromoter.d.ts +108 -0
  35. package/dist/integration/CustomColumnPromoter.d.ts.map +1 -0
  36. package/dist/integration/CustomColumnPromoter.js +508 -0
  37. package/dist/integration/CustomColumnPromoter.js.map +1 -0
  38. package/dist/logging/StartupLogger.d.ts +57 -1
  39. package/dist/logging/StartupLogger.d.ts.map +1 -1
  40. package/dist/logging/StartupLogger.js +115 -6
  41. package/dist/logging/StartupLogger.js.map +1 -1
  42. package/dist/resolvers/ExecuteRemoteOperationResolver.d.ts +46 -0
  43. package/dist/resolvers/ExecuteRemoteOperationResolver.d.ts.map +1 -0
  44. package/dist/resolvers/ExecuteRemoteOperationResolver.js +192 -0
  45. package/dist/resolvers/ExecuteRemoteOperationResolver.js.map +1 -0
  46. package/dist/resolvers/IntegrationDiscoveryResolver.d.ts +101 -1
  47. package/dist/resolvers/IntegrationDiscoveryResolver.d.ts.map +1 -1
  48. package/dist/resolvers/IntegrationDiscoveryResolver.js +629 -58
  49. package/dist/resolvers/IntegrationDiscoveryResolver.js.map +1 -1
  50. package/dist/resolvers/RealtimeBridgeResolver.d.ts +130 -0
  51. package/dist/resolvers/RealtimeBridgeResolver.d.ts.map +1 -0
  52. package/dist/resolvers/RealtimeBridgeResolver.js +580 -0
  53. package/dist/resolvers/RealtimeBridgeResolver.js.map +1 -0
  54. package/dist/resolvers/RemoteBrowserActionResolver.d.ts +78 -5
  55. package/dist/resolvers/RemoteBrowserActionResolver.d.ts.map +1 -1
  56. package/dist/resolvers/RemoteBrowserActionResolver.js +227 -15
  57. package/dist/resolvers/RemoteBrowserActionResolver.js.map +1 -1
  58. package/dist/resolvers/RunAIAgentResolver.d.ts.map +1 -1
  59. package/dist/resolvers/RunAIAgentResolver.js +0 -7
  60. package/dist/resolvers/RunAIAgentResolver.js.map +1 -1
  61. package/dist/resolvers/RunTemplateResolver.d.ts +8 -0
  62. package/dist/resolvers/RunTemplateResolver.d.ts.map +1 -1
  63. package/dist/resolvers/RunTemplateResolver.js +8 -0
  64. package/dist/resolvers/RunTemplateResolver.js.map +1 -1
  65. package/dist/rest/setupRESTEndpoints.d.ts.map +1 -1
  66. package/dist/rest/setupRESTEndpoints.js +5 -3
  67. package/dist/rest/setupRESTEndpoints.js.map +1 -1
  68. package/package.json +83 -78
  69. package/src/__tests__/RealtimeBridgeResolver.test.ts +100 -0
  70. package/src/__tests__/RemoteBrowserAudioStream.test.ts +8 -1
  71. package/src/__tests__/RemoteBrowserGoalEngine.test.ts +144 -0
  72. package/src/__tests__/remoteBrowserGoalRegistry.test.ts +81 -0
  73. package/src/agentSessions/SessionJanitor.ts +15 -1
  74. package/src/agentSessions/SessionManager.ts +59 -10
  75. package/src/agentSessions/index.ts +5 -0
  76. package/src/agentSessions/remoteBrowserGoalEngine.ts +191 -0
  77. package/src/agentSessions/remoteBrowserGoalRegistry.ts +122 -0
  78. package/src/auth/initializeProviders.ts +4 -7
  79. package/src/config.ts +13 -2
  80. package/src/generated/generated.ts +2502 -198
  81. package/src/index.ts +38 -17
  82. package/src/integration/CustomColumnPromoter.ts +597 -0
  83. package/src/logging/StartupLogger.ts +130 -6
  84. package/src/resolvers/ExecuteRemoteOperationResolver.ts +164 -0
  85. package/src/resolvers/IntegrationDiscoveryResolver.ts +490 -53
  86. package/src/resolvers/RealtimeBridgeResolver.ts +493 -0
  87. package/src/resolvers/RemoteBrowserActionResolver.ts +232 -17
  88. package/src/resolvers/RunAIAgentResolver.ts +1 -9
  89. package/src/resolvers/RunTemplateResolver.ts +8 -0
  90. package/src/rest/setupRESTEndpoints.ts +5 -3
@@ -11,16 +11,18 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
11
11
  return function (target, key) { decorator(target, key, paramIndex); }
12
12
  };
13
13
  var IntegrationDiscoveryResolver_1;
14
- import { Resolver, Query, Mutation, Arg, Ctx, ObjectType, Field, InputType } from "type-graphql";
14
+ import { Resolver, Query, Mutation, Arg, Ctx, ObjectType, Field, InputType, Int, Float } from "type-graphql";
15
15
  import { CompositeKey, LocalCacheManager, Metadata, RunView, LogError, LogStatus } from "@memberjunction/core";
16
16
  import { GetReadOnlyProvider, GetReadWriteProvider } from "../util.js";
17
17
  import { CronExpressionHelper } from "@memberjunction/scheduling-engine";
18
- import { ConnectorFactory, IntegrationEngine, IntegrationSchemaSync, IntegrationConnectorCreationPipeline, IntegrationActionGenerator } from "@memberjunction/integration-engine";
18
+ import { ConnectorFactory, IntegrationEngine, IntegrationSchemaSync, decideSchemaLimitViolations, IntegrationConnectorCreationPipeline, IntegrationActionGenerator } from "@memberjunction/integration-engine";
19
19
  import { IntegrationEngineBase } from "@memberjunction/integration-engine-base";
20
20
  import { SchemaBuilder, TypeMapper, SchemaEvolution } from "@memberjunction/integration-schema-builder";
21
21
  import { RuntimeSchemaManager } from "@memberjunction/schema-engine";
22
22
  import { IntegrationProgressReader } from "@memberjunction/integration-progress-artifacts";
23
23
  import { ResolverBase } from "../generic/ResolverBase.js";
24
+ import { IntegrationCustomColumnPromoter } from "../integration/CustomColumnPromoter.js";
25
+ import { RequireSystemUser } from "../directives/RequireSystemUser.js";
24
26
  import { UserCache } from "@memberjunction/sqlserver-dataprovider";
25
27
  // ─── RSU Pipeline Output Types ──────────────────────────────────────────────
26
28
  let RSUStepOutput = class RSUStepOutput {
@@ -1105,6 +1107,188 @@ __decorate([
1105
1107
  MutationResultOutput = __decorate([
1106
1108
  ObjectType()
1107
1109
  ], MutationResultOutput);
1110
+ // ─── Typed sync-config (rate-limit / concurrency / time-budget as STRUCTURED fields, not a raw
1111
+ // Configuration JSON blob). These map to the CompanyIntegration.Configuration keys the engine
1112
+ // reads at runtime, so they are customizable per-connection via the API instead of hidden code
1113
+ // constants. Set merges (preserves other Configuration keys); Get reads them back typed. ──────
1114
+ let IntegrationSyncConfigInput = class IntegrationSyncConfigInput {
1115
+ };
1116
+ __decorate([
1117
+ Field(() => Int, { nullable: true, description: 'Entity maps processed concurrently within a dependency layer (clamped 1-16). Default 1 (sequential).' }),
1118
+ __metadata("design:type", Number)
1119
+ ], IntegrationSyncConfigInput.prototype, "SyncConcurrency", void 0);
1120
+ __decorate([
1121
+ Field(() => Int, { nullable: true, description: 'Upper bound the per-layer AIMD controller ramps toward. Default = connector MaxConcurrencyHint.' }),
1122
+ __metadata("design:type", Number)
1123
+ ], IntegrationSyncConfigInput.prototype, "MaxConcurrency", void 0);
1124
+ __decorate([
1125
+ Field(() => Float, { nullable: true, description: 'Override the source rate limit (requests/sec). Default = connector RateLimitPolicy / derived.' }),
1126
+ __metadata("design:type", Number)
1127
+ ], IntegrationSyncConfigInput.prototype, "RateLimitTokensPerSec", void 0);
1128
+ __decorate([
1129
+ Field(() => Int, { nullable: true, description: 'Override the rate-limiter burst capacity (tokens).' }),
1130
+ __metadata("design:type", Number)
1131
+ ], IntegrationSyncConfigInput.prototype, "RateLimitBurst", void 0);
1132
+ __decorate([
1133
+ Field(() => Boolean, { nullable: true, description: '§4 cross-layer pipelining: a child map starts when ITS parents finish, not the whole layer.' }),
1134
+ __metadata("design:type", Boolean)
1135
+ ], IntegrationSyncConfigInput.prototype, "CrossLayerPipeline", void 0);
1136
+ __decorate([
1137
+ Field(() => Boolean, { nullable: true, description: 'Merkle/partition hash-diff reconcile for watermark-less change detection (buffers the set in RAM).' }),
1138
+ __metadata("design:type", Boolean)
1139
+ ], IntegrationSyncConfigInput.prototype, "PartitionReconcile", void 0);
1140
+ __decorate([
1141
+ Field(() => Int, { nullable: true, description: 'Time budget (ms) for stage-2 streaming field discovery before it stops and uses what it gathered.' }),
1142
+ __metadata("design:type", Number)
1143
+ ], IntegrationSyncConfigInput.prototype, "DiscoveryTimeBudgetMs", void 0);
1144
+ __decorate([
1145
+ Field(() => Int, { nullable: true, description: 'Batch size for stage-2 streaming field discovery (records per FetchChanges page). Default 500.' }),
1146
+ __metadata("design:type", Number)
1147
+ ], IntegrationSyncConfigInput.prototype, "DiscoveryBatchSize", void 0);
1148
+ __decorate([
1149
+ Field(() => Int, { nullable: true, description: 'Max records sampled in stage-2 streaming field discovery (a column corpus + PK guess; NOT a full scan). Default 500.' }),
1150
+ __metadata("design:type", Number)
1151
+ ], IntegrationSyncConfigInput.prototype, "DiscoveryMaxRecords", void 0);
1152
+ __decorate([
1153
+ Field(() => Boolean, { nullable: true, description: '§7 Comprehensive refresh deactivates Declared objects/fields ABSENT from an AUTHORITATIVE discovery (reversible). Default false.' }),
1154
+ __metadata("design:type", Boolean)
1155
+ ], IntegrationSyncConfigInput.prototype, "DeactivateAbsent", void 0);
1156
+ IntegrationSyncConfigInput = __decorate([
1157
+ InputType()
1158
+ ], IntegrationSyncConfigInput);
1159
+ let IntegrationSyncConfigOutput = class IntegrationSyncConfigOutput {
1160
+ };
1161
+ __decorate([
1162
+ Field(),
1163
+ __metadata("design:type", Boolean)
1164
+ ], IntegrationSyncConfigOutput.prototype, "Success", void 0);
1165
+ __decorate([
1166
+ Field(),
1167
+ __metadata("design:type", String)
1168
+ ], IntegrationSyncConfigOutput.prototype, "Message", void 0);
1169
+ __decorate([
1170
+ Field(() => Int, { nullable: true }),
1171
+ __metadata("design:type", Number)
1172
+ ], IntegrationSyncConfigOutput.prototype, "SyncConcurrency", void 0);
1173
+ __decorate([
1174
+ Field(() => Int, { nullable: true }),
1175
+ __metadata("design:type", Number)
1176
+ ], IntegrationSyncConfigOutput.prototype, "MaxConcurrency", void 0);
1177
+ __decorate([
1178
+ Field(() => Float, { nullable: true }),
1179
+ __metadata("design:type", Number)
1180
+ ], IntegrationSyncConfigOutput.prototype, "RateLimitTokensPerSec", void 0);
1181
+ __decorate([
1182
+ Field(() => Int, { nullable: true }),
1183
+ __metadata("design:type", Number)
1184
+ ], IntegrationSyncConfigOutput.prototype, "RateLimitBurst", void 0);
1185
+ __decorate([
1186
+ Field(() => Boolean, { nullable: true }),
1187
+ __metadata("design:type", Boolean)
1188
+ ], IntegrationSyncConfigOutput.prototype, "CrossLayerPipeline", void 0);
1189
+ __decorate([
1190
+ Field(() => Boolean, { nullable: true }),
1191
+ __metadata("design:type", Boolean)
1192
+ ], IntegrationSyncConfigOutput.prototype, "PartitionReconcile", void 0);
1193
+ __decorate([
1194
+ Field(() => Int, { nullable: true }),
1195
+ __metadata("design:type", Number)
1196
+ ], IntegrationSyncConfigOutput.prototype, "DiscoveryTimeBudgetMs", void 0);
1197
+ __decorate([
1198
+ Field(() => Int, { nullable: true }),
1199
+ __metadata("design:type", Number)
1200
+ ], IntegrationSyncConfigOutput.prototype, "DiscoveryBatchSize", void 0);
1201
+ __decorate([
1202
+ Field(() => Int, { nullable: true }),
1203
+ __metadata("design:type", Number)
1204
+ ], IntegrationSyncConfigOutput.prototype, "DiscoveryMaxRecords", void 0);
1205
+ __decorate([
1206
+ Field(() => Boolean, { nullable: true }),
1207
+ __metadata("design:type", Boolean)
1208
+ ], IntegrationSyncConfigOutput.prototype, "DeactivateAbsent", void 0);
1209
+ IntegrationSyncConfigOutput = __decorate([
1210
+ ObjectType()
1211
+ ], IntegrationSyncConfigOutput);
1212
+ let CustomColumnCandidate = class CustomColumnCandidate {
1213
+ };
1214
+ __decorate([
1215
+ Field(),
1216
+ __metadata("design:type", String)
1217
+ ], CustomColumnCandidate.prototype, "EntityName", void 0);
1218
+ __decorate([
1219
+ Field(),
1220
+ __metadata("design:type", String)
1221
+ ], CustomColumnCandidate.prototype, "SourceKey", void 0);
1222
+ __decorate([
1223
+ Field(),
1224
+ __metadata("design:type", String)
1225
+ ], CustomColumnCandidate.prototype, "ColumnName", void 0);
1226
+ __decorate([
1227
+ Field(),
1228
+ __metadata("design:type", String)
1229
+ ], CustomColumnCandidate.prototype, "InferredType", void 0);
1230
+ __decorate([
1231
+ Field(),
1232
+ __metadata("design:type", Boolean)
1233
+ ], CustomColumnCandidate.prototype, "NeedsColumn", void 0);
1234
+ CustomColumnCandidate = __decorate([
1235
+ ObjectType()
1236
+ ], CustomColumnCandidate);
1237
+ let CustomColumnCandidatesOutput = class CustomColumnCandidatesOutput {
1238
+ };
1239
+ __decorate([
1240
+ Field(),
1241
+ __metadata("design:type", Boolean)
1242
+ ], CustomColumnCandidatesOutput.prototype, "Success", void 0);
1243
+ __decorate([
1244
+ Field(),
1245
+ __metadata("design:type", String)
1246
+ ], CustomColumnCandidatesOutput.prototype, "Message", void 0);
1247
+ __decorate([
1248
+ Field(() => [CustomColumnCandidate]),
1249
+ __metadata("design:type", Array)
1250
+ ], CustomColumnCandidatesOutput.prototype, "Candidates", void 0);
1251
+ CustomColumnCandidatesOutput = __decorate([
1252
+ ObjectType()
1253
+ ], CustomColumnCandidatesOutput);
1254
+ let PromotedColumn = class PromotedColumn {
1255
+ };
1256
+ __decorate([
1257
+ Field(),
1258
+ __metadata("design:type", String)
1259
+ ], PromotedColumn.prototype, "EntityName", void 0);
1260
+ __decorate([
1261
+ Field(),
1262
+ __metadata("design:type", String)
1263
+ ], PromotedColumn.prototype, "ColumnName", void 0);
1264
+ PromotedColumn = __decorate([
1265
+ ObjectType()
1266
+ ], PromotedColumn);
1267
+ let PromoteCustomColumnsOutput = class PromoteCustomColumnsOutput {
1268
+ };
1269
+ __decorate([
1270
+ Field(),
1271
+ __metadata("design:type", Boolean)
1272
+ ], PromoteCustomColumnsOutput.prototype, "Success", void 0);
1273
+ __decorate([
1274
+ Field(),
1275
+ __metadata("design:type", String)
1276
+ ], PromoteCustomColumnsOutput.prototype, "Message", void 0);
1277
+ __decorate([
1278
+ Field(),
1279
+ __metadata("design:type", Boolean)
1280
+ ], PromoteCustomColumnsOutput.prototype, "Promoted", void 0);
1281
+ __decorate([
1282
+ Field(() => [PromotedColumn]),
1283
+ __metadata("design:type", Array)
1284
+ ], PromoteCustomColumnsOutput.prototype, "ColumnsAdded", void 0);
1285
+ __decorate([
1286
+ Field(),
1287
+ __metadata("design:type", Boolean)
1288
+ ], PromoteCustomColumnsOutput.prototype, "SchemaUpdatePending", void 0);
1289
+ PromoteCustomColumnsOutput = __decorate([
1290
+ ObjectType()
1291
+ ], PromoteCustomColumnsOutput);
1108
1292
  let FieldMapInput = class FieldMapInput {
1109
1293
  };
1110
1294
  __decorate([
@@ -1261,6 +1445,14 @@ __decorate([
1261
1445
  Field({ nullable: true }),
1262
1446
  __metadata("design:type", Boolean)
1263
1447
  ], CreateScheduleInput.prototype, "FullSync", void 0);
1448
+ __decorate([
1449
+ Field({ nullable: true, defaultValue: 'sync' }),
1450
+ __metadata("design:type", String)
1451
+ ], CreateScheduleInput.prototype, "JobKind", void 0);
1452
+ __decorate([
1453
+ Field({ nullable: true }),
1454
+ __metadata("design:type", Boolean)
1455
+ ], CreateScheduleInput.prototype, "DeactivateAbsent", void 0);
1264
1456
  CreateScheduleInput = __decorate([
1265
1457
  InputType()
1266
1458
  ], CreateScheduleInput);
@@ -2203,7 +2395,7 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
2203
2395
  * both on stdout (visible in the MJAPI log file) and in a per-run
2204
2396
  * `<cwd>/logs/integration-runs/<runID>/progress.jsonl` artifact.
2205
2397
  */
2206
- async IntegrationRefreshConnectorSchema(companyIntegrationID, universalPKConvention, ctx) {
2398
+ async IntegrationRefreshConnectorSchema(companyIntegrationID, universalPKConvention, deactivateAbsent, ctx) {
2207
2399
  try {
2208
2400
  const user = this.getAuthenticatedUser(ctx);
2209
2401
  const provider = GetReadWriteProvider(ctx.providers);
@@ -2221,6 +2413,11 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
2221
2413
  UniversalPKConvention: universalPKConvention || undefined,
2222
2414
  ConsoleMirror: true,
2223
2415
  TriggerType: 'Manual',
2416
+ // §7 — explicit RefreshConnectorSchema is a comprehensive re-discovery: default to
2417
+ // deactivating objects/fields the source no longer exposes (reversible). Precedence:
2418
+ // explicit arg > persisted Configuration.deactivateAbsent (set via IntegrationSetSyncConfig) >
2419
+ // comprehensive default (true). Caller can opt out per-call or per-connection.
2420
+ DeactivateAbsent: deactivateAbsent ?? this.readConfigBool(companyIntegration.Configuration, 'deactivateAbsent') ?? true,
2224
2421
  };
2225
2422
  const result = await pipeline.Run(runOpts);
2226
2423
  // Refresh the metadata cache so subsequent reads see the new IO/IOF
@@ -2502,14 +2699,20 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
2502
2699
  SourceFieldName: f.Name,
2503
2700
  TargetColumnName: f.Name.replace(/[^A-Za-z0-9_]/g, '_'),
2504
2701
  TargetSqlType: targetSqlType,
2505
- // Synced shadow tables must NOT enforce NOT NULL on non-PK
2506
- // columns. The external system (SF, HubSpot, etc.) is the
2507
- // source of truth for business data, not for MJ's schema
2508
- // constraints — and its describe output often declares
2509
- // fields required when real records actually have nulls
2510
- // (deprecated, calculated, or edge-case fields). Enforcing
2511
- // NOT NULL here just aborts entire batches on one bad row.
2512
- IsNullable: !f.IsPrimaryKey,
2702
+ // Synced shadow tables must NOT enforce NOT NULL on ANY
2703
+ // column — including the PK. The external system (SF, HubSpot,
2704
+ // GrowthZone, etc.) is the source of truth for business data,
2705
+ // not for MJ's schema constraints — and its describe output
2706
+ // often declares fields required when real records actually
2707
+ // have nulls (deprecated, calculated, or edge-case fields).
2708
+ // Integration PKs are SOFT (tracked via SchemaBuilder.SoftPrimaryKeys
2709
+ // for upsert/dedup; identity falls back to a content-hash when the
2710
+ // PK is null/partial — §4). Emitting the PK column NOT NULL breaks
2711
+ // that fallback: a source row with a null PK (e.g. nested/derived
2712
+ // records like event sponsors, contact phones) aborts the insert
2713
+ // before content-hash can save it. So the soft-PK column is nullable
2714
+ // too; uniqueness/identity is enforced logically, not by the DDL.
2715
+ IsNullable: true,
2513
2716
  MaxLength: f.MaxLength,
2514
2717
  Precision: f.Precision,
2515
2718
  Scale: f.Scale,
@@ -2527,12 +2730,15 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
2527
2730
  LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — 0 fields discovered (live API likely failed and no DB-cached fields available)`);
2528
2731
  continue;
2529
2732
  }
2530
- // If columns exist but no PK was found, log diagnostic info and skip rather than
2531
- // generating broken DDL with UNIQUE ([ID]) on a non-existent column.
2733
+ // Provable-only: no PK we could prove from the streamed data (single OR composite) means
2734
+ // the object is NOT added and we say so clearly. We never fabricate a key (e.g. "all
2735
+ // columns as the PK"); a wrong identity is worse than an honest omission. The fix for a
2736
+ // missing PK is to STREAM MORE DATA at discovery time so the stats can prove one, not to
2737
+ // invent a key here.
2532
2738
  if (primaryKeyFields.length === 0 && columns.length > 0) {
2533
2739
  droppedNoPrimaryKey.push(obj.SourceObjectName);
2534
2740
  const fieldNames = sourceObj.Fields.map(f => `${f.Name}(pk=${f.IsPrimaryKey})`).join(', ');
2535
- LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — ${columns.length} columns but NO primary key field found. Fields: [${fieldNames}]`);
2741
+ LogError(`[buildTargetConfigs] Skipping "${obj.SourceObjectName}" — ${columns.length} columns but NO provable primary key. Fields: [${fieldNames}]`);
2536
2742
  continue;
2537
2743
  }
2538
2744
  results.push({
@@ -2992,7 +3198,12 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
2992
3198
  */
2993
3199
  buildSourceSchemaFromPersistedRows(integrationID, requestedNames) {
2994
3200
  const engine = IntegrationEngineBase.Instance;
2995
- const ios = engine.GetIntegrationObjectsByIntegrationID(integrationID);
3201
+ // ACTIVE-only materialization: an object/field a given tenant doesn't expose is marked
3202
+ // Status='Inactive' by discovery (the phantom-skip), and MUST NOT be materialized — creating
3203
+ // empty tables/columns for absent objects wastes storage AND, more importantly, blows up the
3204
+ // per-entity CodeGen + advancedGen (AI form-layout) time on every ApplyAll. The sync path
3205
+ // already filters active (GetActiveIntegrationObjects); this build site now matches it.
3206
+ const ios = engine.GetActiveIntegrationObjects(integrationID);
2996
3207
  const filter = requestedNames && requestedNames.length > 0
2997
3208
  ? new Set(requestedNames.map(n => n.toLowerCase()))
2998
3209
  : null;
@@ -3005,7 +3216,8 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
3005
3216
  for (const io of ios) {
3006
3217
  if (filter && !filter.has(io.Name.toLowerCase()))
3007
3218
  continue;
3008
- const iofs = engine.GetIntegrationObjectFields(io.ID);
3219
+ // Active fields only — an inactive (source-absent / deactivated) field is not materialized.
3220
+ const iofs = engine.GetIntegrationObjectFields(io.ID).filter(iof => iof.Status === 'Active');
3009
3221
  const fields = iofs.map(iof => {
3010
3222
  const targetIOName = iof.RelatedIntegrationObjectID
3011
3223
  ? ioByID.get(iof.RelatedIntegrationObjectID) ?? null
@@ -3394,6 +3606,170 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
3394
3606
  return { Success: false, Message: this.formatError(e) };
3395
3607
  }
3396
3608
  }
3609
+ /**
3610
+ * Sets the per-connection sync tuning (rate limit, concurrency, time budget, pipeline flags) as
3611
+ * STRUCTURED typed fields, merged into CompanyIntegration.Configuration (other keys preserved).
3612
+ * These are the exact keys the IntegrationEngine reads at runtime, so they become customizable
3613
+ * via the API instead of hidden code constants. Returns the merged config typed.
3614
+ */
3615
+ async IntegrationSetSyncConfig(companyIntegrationID, config, ctx) {
3616
+ try {
3617
+ const user = this.getAuthenticatedUser(ctx);
3618
+ const md = GetReadWriteProvider(ctx.providers, { allowFallbackToReadOnly: true });
3619
+ const ci = await md.GetEntityObject('MJ: Company Integrations', user);
3620
+ if (!await ci.InnerLoad(CompositeKey.FromID(companyIntegrationID))) {
3621
+ return { Success: false, Message: 'CompanyIntegration not found' };
3622
+ }
3623
+ let cfg = {};
3624
+ try {
3625
+ if (ci.Configuration)
3626
+ cfg = JSON.parse(ci.Configuration);
3627
+ }
3628
+ catch {
3629
+ cfg = {};
3630
+ }
3631
+ const set = (key, val) => { if (val !== undefined && val !== null)
3632
+ cfg[key] = val; };
3633
+ set('syncConcurrency', config.SyncConcurrency);
3634
+ set('maxConcurrency', config.MaxConcurrency);
3635
+ set('rateLimitTokensPerSec', config.RateLimitTokensPerSec);
3636
+ set('rateLimitBurst', config.RateLimitBurst);
3637
+ set('crossLayerPipeline', config.CrossLayerPipeline);
3638
+ set('partitionReconcile', config.PartitionReconcile);
3639
+ set('discoveryTimeBudgetMs', config.DiscoveryTimeBudgetMs);
3640
+ set('discoveryBatchSize', config.DiscoveryBatchSize);
3641
+ set('discoveryMaxRecords', config.DiscoveryMaxRecords);
3642
+ set('deactivateAbsent', config.DeactivateAbsent);
3643
+ ci.Configuration = JSON.stringify(cfg);
3644
+ if (!await ci.Save())
3645
+ return { Success: false, Message: `Failed to save: ${ci.LatestResult?.CompleteMessage ?? 'unknown'}` };
3646
+ return { Success: true, Message: 'Sync config updated', ...this.readSyncConfig(cfg) };
3647
+ }
3648
+ catch (e) {
3649
+ LogError(`IntegrationSetSyncConfig error: ${e}`);
3650
+ return { Success: false, Message: this.formatError(e) };
3651
+ }
3652
+ }
3653
+ /** Reads the per-connection sync tuning back as STRUCTURED typed fields. */
3654
+ async IntegrationGetSyncConfig(companyIntegrationID, ctx) {
3655
+ try {
3656
+ const user = this.getAuthenticatedUser(ctx);
3657
+ const md = GetReadWriteProvider(ctx.providers, { allowFallbackToReadOnly: true });
3658
+ const ci = await md.GetEntityObject('MJ: Company Integrations', user);
3659
+ if (!await ci.InnerLoad(CompositeKey.FromID(companyIntegrationID))) {
3660
+ return { Success: false, Message: 'CompanyIntegration not found' };
3661
+ }
3662
+ let cfg = {};
3663
+ try {
3664
+ if (ci.Configuration)
3665
+ cfg = JSON.parse(ci.Configuration);
3666
+ }
3667
+ catch {
3668
+ cfg = {};
3669
+ }
3670
+ return { Success: true, Message: 'OK', ...this.readSyncConfig(cfg) };
3671
+ }
3672
+ catch (e) {
3673
+ LogError(`IntegrationGetSyncConfig error: ${e}`);
3674
+ return { Success: false, Message: this.formatError(e) };
3675
+ }
3676
+ }
3677
+ /** Reads a single boolean key from a CompanyIntegration.Configuration JSON string (undefined if absent/malformed). */
3678
+ readConfigBool(configuration, key) {
3679
+ try {
3680
+ if (!configuration)
3681
+ return undefined;
3682
+ const v = JSON.parse(configuration)[key];
3683
+ return typeof v === 'boolean' ? v : undefined;
3684
+ }
3685
+ catch {
3686
+ return undefined;
3687
+ }
3688
+ }
3689
+ /** Extracts the typed sync-config fields from a parsed Configuration object (type-guarded). */
3690
+ readSyncConfig(cfg) {
3691
+ const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : undefined);
3692
+ const bool = (v) => (typeof v === 'boolean' ? v : undefined);
3693
+ return {
3694
+ SyncConcurrency: num(cfg.syncConcurrency),
3695
+ MaxConcurrency: num(cfg.maxConcurrency),
3696
+ RateLimitTokensPerSec: num(cfg.rateLimitTokensPerSec),
3697
+ RateLimitBurst: num(cfg.rateLimitBurst),
3698
+ CrossLayerPipeline: bool(cfg.crossLayerPipeline),
3699
+ PartitionReconcile: bool(cfg.partitionReconcile),
3700
+ DiscoveryTimeBudgetMs: num(cfg.discoveryTimeBudgetMs),
3701
+ DiscoveryBatchSize: num(cfg.discoveryBatchSize),
3702
+ DiscoveryMaxRecords: num(cfg.discoveryMaxRecords),
3703
+ DeactivateAbsent: bool(cfg.deactivateAbsent),
3704
+ };
3705
+ }
3706
+ /** The MJ entity names this connection has entity maps for (for whole-connection scope). */
3707
+ async getMappedEntityNames(companyIntegrationID, user) {
3708
+ const rv = new RunView();
3709
+ const res = await rv.RunView({
3710
+ EntityName: 'MJ: Company Integration Entity Maps',
3711
+ ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}'`,
3712
+ Fields: ['Entity'],
3713
+ ResultType: 'simple',
3714
+ }, user);
3715
+ if (!res.Success)
3716
+ return [];
3717
+ return Array.from(new Set((res.Results ?? []).map(r => r.Entity).filter((e) => !!e)));
3718
+ }
3719
+ /**
3720
+ * Lists the custom-column CANDIDATES captured in the overflow column awaiting promotion — the "new
3721
+ * columns found" for a connection. READ-ONLY (no schema change, no RSU). Scope to one entity via
3722
+ * entityName, or omit to list across all the connection's mapped entities. Computed live (overflow keys
3723
+ * minus already-mapped/already-a-column), so it is inherently deduped against anything a concurrent
3724
+ * discovery already promoted.
3725
+ */
3726
+ async IntegrationListCustomColumnCandidates(companyIntegrationID, entityName, ctx) {
3727
+ try {
3728
+ const user = this.getAuthenticatedUser(ctx);
3729
+ const provider = GetReadWriteProvider(ctx.providers, { allowFallbackToReadOnly: true });
3730
+ const promoter = new IntegrationCustomColumnPromoter(user, provider);
3731
+ const entityNames = entityName ? [entityName] : await this.getMappedEntityNames(companyIntegrationID, user);
3732
+ const candidates = [];
3733
+ for (const en of entityNames) {
3734
+ candidates.push(...await promoter.ListCandidates(companyIntegrationID, en));
3735
+ }
3736
+ return { Success: true, Message: `${candidates.length} candidate column(s) found`, Candidates: candidates };
3737
+ }
3738
+ catch (e) {
3739
+ LogError(`IntegrationListCustomColumnCandidates error: ${e}`);
3740
+ return { Success: false, Message: this.formatError(e), Candidates: [] };
3741
+ }
3742
+ }
3743
+ /**
3744
+ * On-demand promotion of captured custom columns: runs RSU (ADD COLUMN + register EntityField + field map),
3745
+ * which may require a server restart to expose them over GraphQL. This is the USER-ACCEPTED trigger — by
3746
+ * default a sync only CAPTURES to the overflow column (auto-promote is opt-in per connection via
3747
+ * Configuration.autoPromoteCustomColumns). Scope via entityNames, or omit to promote across all mapped
3748
+ * entities. Idempotent: already-promoted/mapped keys are skipped (safe to re-run / run alongside discovery).
3749
+ */
3750
+ async IntegrationPromoteCustomColumns(companyIntegrationID, entityNames, ctx) {
3751
+ try {
3752
+ const user = this.getAuthenticatedUser(ctx);
3753
+ const provider = GetReadWriteProvider(ctx.providers);
3754
+ const targets = (entityNames && entityNames.length > 0) ? entityNames : await this.getMappedEntityNames(companyIntegrationID, user);
3755
+ if (targets.length === 0) {
3756
+ return { Success: true, Message: 'No mapped entities to promote', Promoted: false, ColumnsAdded: [], SchemaUpdatePending: false };
3757
+ }
3758
+ const promoter = new IntegrationCustomColumnPromoter(user, provider);
3759
+ const result = await promoter.PromoteForSync(companyIntegrationID, targets);
3760
+ return {
3761
+ Success: true,
3762
+ Message: result.Promoted ? `Promoted ${result.ColumnsAdded.length} column(s)` : 'No columns required promotion',
3763
+ Promoted: result.Promoted,
3764
+ ColumnsAdded: result.ColumnsAdded.map(c => ({ EntityName: c.EntityName, ColumnName: c.ColumnName })),
3765
+ SchemaUpdatePending: result.SchemaUpdatePending,
3766
+ };
3767
+ }
3768
+ catch (e) {
3769
+ LogError(`IntegrationPromoteCustomColumns error: ${e}`);
3770
+ return { Success: false, Message: this.formatError(e), Promoted: false, ColumnsAdded: [], SchemaUpdatePending: false };
3771
+ }
3772
+ }
3397
3773
  /**
3398
3774
  * Soft-deletes a CompanyIntegration by setting IsActive=false.
3399
3775
  */
@@ -3903,8 +4279,25 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
3903
4279
  */
3904
4280
  async createEntityAndFieldMaps(companyIntegrationID, objects, connector, companyIntegration, schemaName, user, provider, defaultSyncDirection = 'Pull') {
3905
4281
  const results = [];
4282
+ // PERF (large-catalog ApplyAll, e.g. Salesforce 1695 objects): precompute O(1) lookups ONCE before
4283
+ // the per-object loop. The per-object path previously did an O(N) `provider.Entities.find` + a
4284
+ // BypassCache RunView for the existing entity map PER object → O(N²) scans + N DB round-trips, so a
4285
+ // 1695-object connector burned its whole budget in ApplyAll and FullSync never started. One entity
4286
+ // index + one batched entity-map query make both O(1) per object.
4287
+ const entityByKey = new Map(provider.Entities.map(e => [`${e.SchemaName.toLowerCase()}|${e.BaseTable.toLowerCase()}`, e]));
4288
+ const existingMapByExternalName = new Map();
4289
+ const allMaps = await new RunView().RunView({
4290
+ EntityName: 'MJ: Company Integration Entity Maps',
4291
+ ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}'`,
4292
+ ResultType: 'entity_object',
4293
+ BypassCache: true, // idempotency must read COMMITTED state, not a possibly-stale filtered cache
4294
+ }, user);
4295
+ if (allMaps.Success) {
4296
+ for (const m of allMaps.Results)
4297
+ existingMapByExternalName.set((m.ExternalObjectName ?? '').toLowerCase(), m);
4298
+ }
3906
4299
  for (const obj of objects) {
3907
- const entityMapResult = await this.createSingleEntityMap(companyIntegrationID, obj, connector, companyIntegration, schemaName, user, provider, defaultSyncDirection);
4300
+ const entityMapResult = await this.createSingleEntityMap(companyIntegrationID, obj, connector, companyIntegration, schemaName, user, provider, defaultSyncDirection, entityByKey, existingMapByExternalName);
3908
4301
  if (entityMapResult) {
3909
4302
  results.push(entityMapResult);
3910
4303
  }
@@ -3912,10 +4305,12 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
3912
4305
  return results;
3913
4306
  }
3914
4307
  /** Creates a single entity map + field maps for one source object. */
3915
- async createSingleEntityMap(companyIntegrationID, obj, connector, companyIntegration, schemaName, user, md, defaultSyncDirection = 'Pull') {
3916
- // Find the entity by schema + table name
3917
- const entityInfo = md.Entities.find(e => e.SchemaName.toLowerCase() === schemaName.toLowerCase()
3918
- && e.BaseTable.toLowerCase() === obj.TableName.toLowerCase());
4308
+ async createSingleEntityMap(companyIntegrationID, obj, connector, companyIntegration, schemaName, user, md, defaultSyncDirection = 'Pull', entityByKey, existingMapByExternalName) {
4309
+ // Find the entity by schema + table name. PERF: O(1) precomputed index when provided
4310
+ // (large-catalog ApplyAll); falls back to the O(N) scan when called without it.
4311
+ const entityInfo = entityByKey?.get(`${schemaName.toLowerCase()}|${obj.TableName.toLowerCase()}`)
4312
+ ?? md.Entities.find(e => e.SchemaName.toLowerCase() === schemaName.toLowerCase()
4313
+ && e.BaseTable.toLowerCase() === obj.TableName.toLowerCase());
3919
4314
  if (!entityInfo) {
3920
4315
  LogError(`IntegrationApplyAll: entity not found for ${schemaName}.${obj.TableName}`);
3921
4316
  return null;
@@ -3925,18 +4320,28 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
3925
4320
  // multiplies the maps in lockstep (N applies → N duplicate maps per object), which
3926
4321
  // silently corrupts the record-map 1:1 completeness gate and makes the forward sync
3927
4322
  // process each object N times. Reuse the existing (connection, external object) map instead.
3928
- const escapedObjectName = obj.SourceObjectName.replace(/'/g, "''");
3929
- const existingMapResult = await new RunView().RunView({
3930
- EntityName: 'MJ: Company Integration Entity Maps',
3931
- ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND ExternalObjectName='${escapedObjectName}'`,
3932
- OrderBy: '__mj_CreatedAt ASC',
3933
- MaxRows: 1,
3934
- ResultType: 'entity_object',
3935
- BypassCache: true, // idempotency must read COMMITTED state, not a possibly-stale filtered cache
3936
- }, user);
4323
+ // PERF: use the precomputed existing-map index when provided (one batched query for the whole
4324
+ // ApplyAll instead of one BypassCache RunView per object); falls back to a per-object query.
4325
+ let existing;
4326
+ if (existingMapByExternalName) {
4327
+ existing = existingMapByExternalName.get(obj.SourceObjectName.toLowerCase());
4328
+ }
4329
+ else {
4330
+ const escapedObjectName = obj.SourceObjectName.replace(/'/g, "''");
4331
+ const existingMapResult = await new RunView().RunView({
4332
+ EntityName: 'MJ: Company Integration Entity Maps',
4333
+ ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND ExternalObjectName='${escapedObjectName}'`,
4334
+ OrderBy: '__mj_CreatedAt ASC',
4335
+ MaxRows: 1,
4336
+ ResultType: 'entity_object',
4337
+ BypassCache: true, // idempotency must read COMMITTED state, not a possibly-stale filtered cache
4338
+ }, user);
4339
+ if (existingMapResult.Success && existingMapResult.Results.length > 0)
4340
+ existing = existingMapResult.Results[0];
4341
+ }
3937
4342
  let em;
3938
- if (existingMapResult.Success && existingMapResult.Results.length > 0) {
3939
- em = existingMapResult.Results[0]; // reuse — keeps the map stable across re-applies
4343
+ if (existing) {
4344
+ em = existing; // reuse — keeps the map stable across re-applies
3940
4345
  }
3941
4346
  else {
3942
4347
  em = await md.GetEntityObject('MJ: Company Integration Entity Maps', user);
@@ -4040,6 +4445,72 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
4040
4445
  * Build schema artifacts for a single connector's objects.
4041
4446
  * Shared by IntegrationApplySchema (single) and IntegrationApplySchemaBatch (batch).
4042
4447
  */
4448
+ /**
4449
+ * §B — enforce OPTIONAL table/column caps at the create-tables (RSU) gate. These are OPERATOR
4450
+ * (deployment) guardrails read from ENV — `MJ_INTEGRATION_MAX_TABLES` / `MJ_INTEGRATION_MAX_COLUMNS_PER_TABLE`
4451
+ * (absent or ≤0 = unbounded, the common case). They are DELIBERATELY env-only, NOT per-connection
4452
+ * `Configuration`/GraphQL: a guardrail a user can raise via the same API they apply with is toothless.
4453
+ * THROWS a clear error when the selection exceeds a cap so NOTHING partial is created — the caller surfaces
4454
+ * it and the user narrows the selection (the cap itself is an operator concern). NEVER truncates. Discovery
4455
+ * still surfaces every object/field; only materialization is capped. Per-table column count = the selected
4456
+ * field subset, or the object's full discovered field count when all fields are selected.
4457
+ */
4458
+ async enforceSchemaLimits(objects, filteredSchema, companyIntegration, user, md) {
4459
+ const warnings = [];
4460
+ const fullCountByName = new Map(filteredSchema.Objects.map(o => [o.ExternalName.toLowerCase(), o.Fields.length]));
4461
+ // Use the MAX of the input object's Fields and the filtered-schema count — the batch path can pass
4462
+ // objects with empty Fields (the real columns come from filteredSchema, which is what buildSchema
4463
+ // materializes). `?? ` would wrongly read an empty [] as 0 (0 is not nullish); Math.max is robust to
4464
+ // either source being empty and reflects the count that will actually become columns.
4465
+ const colCount = (o) => Math.max(fullCountByName.get(o.SourceObjectName.toLowerCase()) ?? 0, o.Fields?.length ?? 0);
4466
+ // ── Per-table COLUMN limit: DISABLE-the-offender, NOT reject-the-apply. The limit is clamped at
4467
+ // startup to ≤ IntegrationEngine.MAX_COLUMNS_CEILING (SQL Server's 1024 minus framework column
4468
+ // headroom). An object exceeding it is set Status='Disabled' (reversible — it re-enables on a later
4469
+ // discovery if it has shrunk back under the limit) and dropped from THIS apply, so the materializable
4470
+ // objects still sync instead of the whole apply failing CREATE TABLE. One CLI warning per disabled object.
4471
+ const limit = IntegrationEngine.Instance.MaxColumnsPerTable;
4472
+ const overLimit = objects.filter(o => colCount(o) > limit);
4473
+ if (overLimit.length > 0) {
4474
+ const ioIDByName = new Map(IntegrationEngineBase.Instance.GetActiveIntegrationObjects(companyIntegration.IntegrationID)
4475
+ .map(io => [io.Name.toLowerCase(), io.ID]));
4476
+ for (const o of overLimit) {
4477
+ warnings.push(`Disabled "${o.SourceObjectName}" (${colCount(o)} columns > limit ${limit}) — too wide to materialize as one table; reduce its columns or it stays disabled.`);
4478
+ const ioID = ioIDByName.get(o.SourceObjectName.toLowerCase());
4479
+ if (ioID) {
4480
+ const io = await md.GetEntityObject('MJ: Integration Objects', user);
4481
+ if (await io.Load(ioID)) {
4482
+ io.Status = 'Disabled';
4483
+ if (!await io.Save()) {
4484
+ LogError(`enforceSchemaLimits: failed to disable over-wide IO '${o.SourceObjectName}': ${io.LatestResult?.CompleteMessage ?? 'unknown error'}`);
4485
+ }
4486
+ }
4487
+ }
4488
+ }
4489
+ // Drop the disabled objects from BOTH the shared objects array (used downstream by
4490
+ // buildTargetConfigs + createEntityAndFieldMaps) and filteredSchema, in place.
4491
+ const disabled = new Set(overLimit.map(o => o.SourceObjectName.toLowerCase()));
4492
+ for (let i = objects.length - 1; i >= 0; i--) {
4493
+ if (disabled.has(objects[i].SourceObjectName.toLowerCase()))
4494
+ objects.splice(i, 1);
4495
+ }
4496
+ filteredSchema.Objects = filteredSchema.Objects.filter(o => !disabled.has(o.ExternalName.toLowerCase()));
4497
+ }
4498
+ // MaxTables remains an operator/env REJECT guardrail (unchanged), evaluated on the surviving objects.
4499
+ // Columns are handled above via disable, so the pure decision is asked about tables only here.
4500
+ const maxTables = (() => {
4501
+ const v = parseInt(process.env.MJ_INTEGRATION_MAX_TABLES ?? '', 10);
4502
+ return Number.isFinite(v) && v > 0 ? v : null;
4503
+ })();
4504
+ const violations = decideSchemaLimitViolations({
4505
+ TableCount: objects.length,
4506
+ ColumnCountByTable: [],
4507
+ MaxTables: maxTables,
4508
+ MaxColumnsPerTable: null,
4509
+ });
4510
+ if (violations.length > 0)
4511
+ throw new Error(violations.join(' '));
4512
+ return warnings;
4513
+ }
4043
4514
  async buildSchemaForConnector(companyIntegrationID, objects, platform, user, skipGitCommit, skipRestart, provider, prefetchedSourceSchema) {
4044
4515
  const { connector, companyIntegration } = await this.resolveConnector(companyIntegrationID, user, provider);
4045
4516
  // Source-schema resolution order:
@@ -4082,6 +4553,10 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
4082
4553
  const filteredSchema = {
4083
4554
  Objects: sourceSchema.Objects.filter(o => requestedNames.has(o.ExternalName))
4084
4555
  };
4556
+ // §B — reject an over-limit table/column SELECTION before ANY table is materialized. This is the
4557
+ // single shared gate for ApplyAll / ApplyAllBatch / ApplySchemaBatch (all route through here).
4558
+ // Caps are operator/env guardrails (MJ_INTEGRATION_MAX_TABLES / _MAX_COLUMNS_PER_TABLE).
4559
+ const limitWarnings = await this.enforceSchemaLimits(objects, filteredSchema, companyIntegration, user, provider);
4085
4560
  const targetConfigs = this.buildTargetConfigs(objects, filteredSchema, platform, connector);
4086
4561
  const input = {
4087
4562
  SourceSchema: filteredSchema,
@@ -4097,6 +4572,9 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
4097
4572
  };
4098
4573
  const builder = new SchemaBuilder();
4099
4574
  const schemaOutput = builder.BuildSchema(input);
4575
+ // Surface the auto-disabled over-wide objects (if any) in the apply's Warnings → CLI output.
4576
+ if (limitWarnings.length > 0)
4577
+ schemaOutput.Warnings.unshift(...limitWarnings);
4100
4578
  if (schemaOutput.Errors.length > 0) {
4101
4579
  throw new Error(`Schema generation failed: ${schemaOutput.Errors.join('; ')}`);
4102
4580
  }
@@ -4129,6 +4607,12 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
4129
4607
  syncOptions.EntityMapIDs = entityMapIDs;
4130
4608
  if (syncDirection)
4131
4609
  syncOptions.SyncDirection = syncDirection;
4610
+ // Capture the fire instant BEFORE launching the sync. The run-record lookup below
4611
+ // finds the run by recency (StartedAt >= firedAt) rather than by transient status, so
4612
+ // a fast run (0-record / empty connector / quick failure) that finishes before we poll
4613
+ // is still reported with its real RunID instead of an untrackable null. RunSync stamps
4614
+ // StartedAt with an app-side `new Date()`, so this clock is consistent with the row.
4615
+ const firedAt = new Date();
4132
4616
  // Fire and forget — progress is tracked inside IntegrationEngine
4133
4617
  const syncPromise = IntegrationEngine.Instance.RunSync(companyIntegrationID, user, 'Manual', undefined, undefined, Object.keys(syncOptions).length > 0 ? syncOptions : undefined);
4134
4618
  syncPromise
@@ -4157,22 +4641,43 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
4157
4641
  });
4158
4642
  }
4159
4643
  });
4160
- // Small delay to let the run record get created
4161
- await new Promise(resolve => setTimeout(resolve, 200));
4162
- const rv = new RunView();
4163
- const runResult = await rv.RunView({
4164
- EntityName: 'MJ: Company Integration Runs',
4165
- ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND Status='In Progress'`,
4166
- OrderBy: '__mj_CreatedAt DESC',
4167
- MaxRows: 1,
4168
- ResultType: 'simple',
4169
- Fields: ['ID', 'Status', 'StartedAt']
4170
- }, user);
4171
- const run = runResult.Success && runResult.Results.length > 0 ? runResult.Results[0] : null;
4644
+ // Resolve the run record by RECENCY, not by transient 'In Progress' status. The prior
4645
+ // implementation slept a fixed 200ms then filtered Status='In Progress' — when the run
4646
+ // finished in under 200ms (an empty connector, a 0-record sync, or a fast synchronous
4647
+ // failure) there was no 'In Progress' row to find, so it returned Success:true with a
4648
+ // null RunID: an optimistic, untrackable result that read as "started" when nothing
4649
+ // could be followed. Poll briefly for ANY run for this connector stamped at/after the
4650
+ // fire instant; that catches both the still-running and the already-finished cases.
4651
+ const firedAtFilter = firedAt.toISOString();
4652
+ let run = null;
4653
+ for (let attempt = 0; attempt < 15 && !run; attempt++) {
4654
+ await new Promise(resolve => setTimeout(resolve, 200));
4655
+ const rv = new RunView();
4656
+ const runResult = await rv.RunView({
4657
+ EntityName: 'MJ: Company Integration Runs',
4658
+ ExtraFilter: `CompanyIntegrationID='${companyIntegrationID}' AND StartedAt >= '${firedAtFilter}'`,
4659
+ OrderBy: 'StartedAt DESC',
4660
+ MaxRows: 1,
4661
+ ResultType: 'simple',
4662
+ Fields: ['ID', 'Status', 'StartedAt']
4663
+ }, user);
4664
+ if (runResult.Success && runResult.Results.length > 0) {
4665
+ run = runResult.Results[0];
4666
+ }
4667
+ }
4668
+ // No run record after the poll window ⇒ the sync genuinely did not start (it no-op'd
4669
+ // before creating a run). Report that honestly rather than claiming Success with a null
4670
+ // RunID — a caller/scheduler can act on a false, but a true+null silently strands it.
4671
+ if (!run) {
4672
+ return {
4673
+ Success: false,
4674
+ Message: 'Sync did not start — no run record was created (the connector may be gated or misconfigured)'
4675
+ };
4676
+ }
4172
4677
  return {
4173
4678
  Success: true,
4174
4679
  Message: 'Sync started',
4175
- RunID: run?.ID
4680
+ RunID: run.ID
4176
4681
  };
4177
4682
  }
4178
4683
  catch (e) {
@@ -4275,16 +4780,21 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
4275
4780
  const user = this.getAuthenticatedUser(ctx);
4276
4781
  const md = GetReadWriteProvider(ctx.providers, { allowFallbackToReadOnly: true });
4277
4782
  const rv = new RunView();
4278
- // Find IntegrationSync job type
4783
+ // §13 select the driver by job-kind: 'discovery' = schema-only refresh on a cron;
4784
+ // 'sync' (default) = the data-moving RunSync job.
4785
+ const jobKind = (input.JobKind ?? 'sync').toLowerCase();
4786
+ const driverClass = jobKind === 'discovery'
4787
+ ? 'IntegrationDiscoveryScheduledJobDriver'
4788
+ : 'IntegrationSyncScheduledJobDriver';
4279
4789
  const jobTypeResult = await rv.RunView({
4280
4790
  EntityName: 'MJ: Scheduled Job Types',
4281
- ExtraFilter: `DriverClass='IntegrationSyncScheduledJobDriver'`,
4791
+ ExtraFilter: `DriverClass='${driverClass}'`,
4282
4792
  MaxRows: 1,
4283
4793
  ResultType: 'simple',
4284
4794
  Fields: ['ID']
4285
4795
  }, user);
4286
4796
  if (!jobTypeResult.Success || jobTypeResult.Results.length === 0) {
4287
- return { Success: false, Message: 'IntegrationSync scheduled job type not found' };
4797
+ return { Success: false, Message: `Scheduled job type not found for driver '${driverClass}' (kind='${jobKind}')` };
4288
4798
  }
4289
4799
  const jobTypeID = jobTypeResult.Results[0].ID;
4290
4800
  const job = await md.GetEntityObject('MJ: Scheduled Jobs', user);
@@ -4298,10 +4808,18 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
4298
4808
  job.Status = 'Active';
4299
4809
  job.OwnerUserID = user.ID;
4300
4810
  const jobConfig = { CompanyIntegrationID: input.CompanyIntegrationID };
4301
- if (input.SyncDirection)
4302
- jobConfig.SyncDirection = input.SyncDirection;
4303
- if (input.FullSync)
4304
- jobConfig.FullSync = input.FullSync;
4811
+ if (jobKind === 'discovery') {
4812
+ // Discovery job: schema-only. SyncDirection/FullSync are meaningless here; carry the
4813
+ // deactivate toggle instead (driver defaults it to true when omitted).
4814
+ if (input.DeactivateAbsent !== undefined && input.DeactivateAbsent !== null)
4815
+ jobConfig.DeactivateAbsent = input.DeactivateAbsent;
4816
+ }
4817
+ else {
4818
+ if (input.SyncDirection)
4819
+ jobConfig.SyncDirection = input.SyncDirection;
4820
+ if (input.FullSync)
4821
+ jobConfig.FullSync = input.FullSync;
4822
+ }
4305
4823
  job.Configuration = JSON.stringify(jobConfig);
4306
4824
  job.NextRunAt = CronExpressionHelper.GetNextRunTime(input.CronExpression, input.Timezone || 'UTC');
4307
4825
  if (!await job.Save())
@@ -5477,12 +5995,18 @@ let IntegrationDiscoveryResolver = class IntegrationDiscoveryResolver extends Re
5477
5995
  if (!submitted) {
5478
5996
  return { Success: false, Message: 'Transaction failed — all deletes rolled back' };
5479
5997
  }
5480
- if (deleteData) {
5481
- LogError(`IntegrationDeleteConnection: deleteData=true requested but table deletion not yet implemented for ${companyIntegrationID}`);
5482
- }
5998
+ // deleteData=true asks us to also DROP the physical mirror tables. We deliberately do NOT:
5999
+ // RuntimeSchemaManager rejects DROP TABLE on the __mj schema by design (a destructive-op
6000
+ // safety guard), and there is no DROP generator. Dropping integration tables is real data
6001
+ // loss and must be an explicit, separately-designed operation — not a silent side effect of
6002
+ // removing a connection. So we delete the connection + all METADATA (maps/fields/schedules/
6003
+ // credential) and RETAIN the data tables, and we say so honestly rather than claim otherwise.
6004
+ const dataNote = deleteData
6005
+ ? ' The physical data tables were RETAINED (dropping integration tables is not performed automatically — it is a separate, explicit operation).'
6006
+ : '';
5483
6007
  return {
5484
6008
  Success: true,
5485
- Message: `Deleted connection and all associated records`,
6009
+ Message: `Deleted connection and all associated metadata records.${dataNote}`,
5486
6010
  EntityMapsDeleted: entityMapsDeleted,
5487
6011
  FieldMapsDeleted: fieldMapsDeleted,
5488
6012
  SchedulesDeleted: schedulesDeleted,
@@ -5699,15 +6223,18 @@ __decorate([
5699
6223
  ], IntegrationDiscoveryResolver.prototype, "IntegrationTestConnection", null);
5700
6224
  __decorate([
5701
6225
  Mutation(() => RefreshConnectorSchemaOutput),
6226
+ RequireSystemUser(),
5702
6227
  __param(0, Arg("companyIntegrationID")),
5703
6228
  __param(1, Arg("universalPKConvention", { nullable: true, description: "Optional vendor-wide PK convention hint (e.g. 'id' for HubSpot)" })),
5704
- __param(2, Ctx()),
6229
+ __param(2, Arg("deactivateAbsent", { nullable: true, description: "Comprehensive refresh (default true): objects/fields ABSENT from this discovery are deactivated (Status='Disabled', never deleted, reversible on a later rediscovery). Pass false for a scoped/partial discovery so it never disables what it didn't probe." })),
6230
+ __param(3, Ctx()),
5705
6231
  __metadata("design:type", Function),
5706
- __metadata("design:paramtypes", [String, String, Object]),
6232
+ __metadata("design:paramtypes", [String, String, Boolean, Object]),
5707
6233
  __metadata("design:returntype", Promise)
5708
6234
  ], IntegrationDiscoveryResolver.prototype, "IntegrationRefreshConnectorSchema", null);
5709
6235
  __decorate([
5710
6236
  Mutation(() => IntegrationGenerateActionOutput),
6237
+ RequireSystemUser(),
5711
6238
  __param(0, Arg("integrationName")),
5712
6239
  __param(1, Arg("objectName")),
5713
6240
  __param(2, Arg("verb", { nullable: true, description: "Optional CRUD verb (Get|Create|Update|Delete|Search|List). Omit to generate all applicable verbs for the object." })),
@@ -5755,6 +6282,7 @@ __decorate([
5755
6282
  ], IntegrationDiscoveryResolver.prototype, "IntegrationListConnections", null);
5756
6283
  __decorate([
5757
6284
  Mutation(() => CreateConnectionOutput),
6285
+ RequireSystemUser(),
5758
6286
  __param(0, Arg("input")),
5759
6287
  __param(1, Arg("testConnection", () => Boolean, { defaultValue: false })),
5760
6288
  __param(2, Arg("runSchemaRefresh", () => Boolean, { defaultValue: true, description: "When true (default) and TestConnection succeeds, automatically runs IntegrationConnectorCreationPipeline (live introspect → persist Declared/Discovered/Custom → SoftPKClassifier). The intermittent server-side work the wizard's Forward step represents." })),
@@ -5778,6 +6306,42 @@ __decorate([
5778
6306
  __metadata("design:paramtypes", [String, String, String, String, Boolean, Boolean, String, Object]),
5779
6307
  __metadata("design:returntype", Promise)
5780
6308
  ], IntegrationDiscoveryResolver.prototype, "IntegrationUpdateConnection", null);
6309
+ __decorate([
6310
+ Mutation(() => IntegrationSyncConfigOutput),
6311
+ __param(0, Arg("companyIntegrationID")),
6312
+ __param(1, Arg("config", () => IntegrationSyncConfigInput)),
6313
+ __param(2, Ctx()),
6314
+ __metadata("design:type", Function),
6315
+ __metadata("design:paramtypes", [String, IntegrationSyncConfigInput, Object]),
6316
+ __metadata("design:returntype", Promise)
6317
+ ], IntegrationDiscoveryResolver.prototype, "IntegrationSetSyncConfig", null);
6318
+ __decorate([
6319
+ Query(() => IntegrationSyncConfigOutput),
6320
+ __param(0, Arg("companyIntegrationID")),
6321
+ __param(1, Ctx()),
6322
+ __metadata("design:type", Function),
6323
+ __metadata("design:paramtypes", [String, Object]),
6324
+ __metadata("design:returntype", Promise)
6325
+ ], IntegrationDiscoveryResolver.prototype, "IntegrationGetSyncConfig", null);
6326
+ __decorate([
6327
+ Query(() => CustomColumnCandidatesOutput),
6328
+ __param(0, Arg("companyIntegrationID")),
6329
+ __param(1, Arg("entityName", { nullable: true })),
6330
+ __param(2, Ctx()),
6331
+ __metadata("design:type", Function),
6332
+ __metadata("design:paramtypes", [String, String, Object]),
6333
+ __metadata("design:returntype", Promise)
6334
+ ], IntegrationDiscoveryResolver.prototype, "IntegrationListCustomColumnCandidates", null);
6335
+ __decorate([
6336
+ Mutation(() => PromoteCustomColumnsOutput),
6337
+ RequireSystemUser(),
6338
+ __param(0, Arg("companyIntegrationID")),
6339
+ __param(1, Arg("entityNames", () => [String], { nullable: true })),
6340
+ __param(2, Ctx()),
6341
+ __metadata("design:type", Function),
6342
+ __metadata("design:paramtypes", [String, Array, Object]),
6343
+ __metadata("design:returntype", Promise)
6344
+ ], IntegrationDiscoveryResolver.prototype, "IntegrationPromoteCustomColumns", null);
5781
6345
  __decorate([
5782
6346
  Mutation(() => MutationResultOutput),
5783
6347
  __param(0, Arg("companyIntegrationID")),
@@ -5805,6 +6369,7 @@ __decorate([
5805
6369
  ], IntegrationDiscoveryResolver.prototype, "IntegrationCreateEntityMaps", null);
5806
6370
  __decorate([
5807
6371
  Mutation(() => ApplySchemaOutput),
6372
+ RequireSystemUser(),
5808
6373
  __param(0, Arg("companyIntegrationID")),
5809
6374
  __param(1, Arg("objects", () => [SchemaPreviewObjectInput])),
5810
6375
  __param(2, Arg("platform", { defaultValue: "sqlserver" })),
@@ -5817,6 +6382,7 @@ __decorate([
5817
6382
  ], IntegrationDiscoveryResolver.prototype, "IntegrationApplySchema", null);
5818
6383
  __decorate([
5819
6384
  Mutation(() => ApplySchemaBatchOutput),
6385
+ RequireSystemUser(),
5820
6386
  __param(0, Arg("items", () => [ApplySchemaBatchItemInput])),
5821
6387
  __param(1, Arg("platform", { defaultValue: "sqlserver" })),
5822
6388
  __param(2, Arg("skipGitCommit", { defaultValue: false })),
@@ -5828,6 +6394,7 @@ __decorate([
5828
6394
  ], IntegrationDiscoveryResolver.prototype, "IntegrationApplySchemaBatch", null);
5829
6395
  __decorate([
5830
6396
  Mutation(() => ApplyAllOutput),
6397
+ RequireSystemUser(),
5831
6398
  __param(0, Arg("input")),
5832
6399
  __param(1, Arg("platform", { defaultValue: "sqlserver" })),
5833
6400
  __param(2, Arg("skipGitCommit", { defaultValue: false })),
@@ -5839,6 +6406,7 @@ __decorate([
5839
6406
  ], IntegrationDiscoveryResolver.prototype, "IntegrationApplyAll", null);
5840
6407
  __decorate([
5841
6408
  Mutation(() => StartSyncOutput),
6409
+ RequireSystemUser(),
5842
6410
  __param(0, Arg("companyIntegrationID")),
5843
6411
  __param(1, Arg("webhookURL", { nullable: true })),
5844
6412
  __param(2, Arg("fullSync", () => Boolean, { defaultValue: false, description: 'If true, ignores watermarks and re-fetches all records from the source' })),
@@ -5859,6 +6427,7 @@ __decorate([
5859
6427
  ], IntegrationDiscoveryResolver.prototype, "IntegrationCancelSync", null);
5860
6428
  __decorate([
5861
6429
  Mutation(() => WriteRecordOutput),
6430
+ RequireSystemUser(),
5862
6431
  __param(0, Arg("companyIntegrationID")),
5863
6432
  __param(1, Arg("objectName")),
5864
6433
  __param(2, Arg("operation", () => String, { description: 'create, update, or delete' })),
@@ -6016,6 +6585,7 @@ __decorate([
6016
6585
  ], IntegrationDiscoveryResolver.prototype, "IntegrationGetConnectorCapabilities", null);
6017
6586
  __decorate([
6018
6587
  Mutation(() => ApplyAllBatchOutput),
6588
+ RequireSystemUser(),
6019
6589
  __param(0, Arg("input")),
6020
6590
  __param(1, Arg("platform", { defaultValue: "sqlserver" })),
6021
6591
  __param(2, Arg("skipGitCommit", { defaultValue: false })),
@@ -6036,6 +6606,7 @@ __decorate([
6036
6606
  ], IntegrationDiscoveryResolver.prototype, "IntegrationDeleteConnection", null);
6037
6607
  __decorate([
6038
6608
  Mutation(() => SchemaEvolutionOutput),
6609
+ RequireSystemUser(),
6039
6610
  __param(0, Arg("companyIntegrationID")),
6040
6611
  __param(1, Arg("platform", { defaultValue: "sqlserver" })),
6041
6612
  __param(2, Arg("skipGitCommit", { defaultValue: false })),