@memberjunction/integration-connectors 5.29.0 → 5.30.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
6
6
  };
7
7
  import { RegisterClass } from '@memberjunction/global';
8
8
  import { Metadata } from '@memberjunction/core';
9
+ import { IntegrationEngineBase } from '@memberjunction/integration-engine-base';
9
10
  import { BaseIntegrationConnector, } from '@memberjunction/integration-engine';
10
11
  // ─── Constants ────────────────────────────────────────────────────────
11
12
  /** Sage Intacct XML Gateway endpoint */
@@ -20,10 +21,155 @@ const DEFAULT_MAX_RETRIES = 3;
20
21
  const DEFAULT_REQUEST_TIMEOUT_MS = 30000;
21
22
  /** Default minimum milliseconds between API requests */
22
23
  const DEFAULT_MIN_REQUEST_INTERVAL_MS = 200;
23
- /** Default page size for readByQuery */
24
- const DEFAULT_PAGE_SIZE = 100;
24
+ /** Default page size for readByQuery — SI's documented max. */
25
+ // Direct probing against the live tenant proved that pageSize=1000 (and
26
+ // even 2000) is accepted; the previously-observed DL02000001 errors were
27
+ // from SQL-style ORDER BY in the WHERE clause, NOT from large pageSizes.
28
+ // Critically, SI's readByQuery does NOT scan in PK-ascending order — at
29
+ // pageSize=100 against APBILL it returned RECORDNOs 19..1354, silently
30
+ // dropping 12..18 because those weren't in SI's first 100 of natural
31
+ // order. PK-cursor advancement (PK > maxOfBatch) then made the missing
32
+ // records permanently invisible. Using SI's documented maximum minimizes
33
+ // the chance any object exceeds a single page; for the few that do (e.g.
34
+ // GLBATCH with thousands of rows) SI provides a server-side resultId
35
+ // that maintains scan consistency across readMore calls.
36
+ const DEFAULT_PAGE_SIZE = 1000;
25
37
  /** Session lifetime — Sage Intacct sessions last ~10 minutes; refresh at 8 min */
26
38
  const SESSION_LIFETIME_MS = 8 * 60 * 1000;
39
+ // ─── Range-chunked pagination constants ───────────────────────────────
40
+ //
41
+ // SI's readByQuery cannot guarantee PK-ascending scan order, and SI rejects
42
+ // both `<orderby>` (XSD parse error) and SQL-style `ORDER BY` (DL02000001).
43
+ // Without ordering, the only way to GUARANTEE complete record coverage for
44
+ // numeric-PK objects is to walk RECORDNO in fixed numeric ranges:
45
+ //
46
+ // chunk 0: RECORDNO ∈ [0, CHUNK)
47
+ // chunk 1: RECORDNO ∈ [CHUNK, 2*CHUNK)
48
+ // chunk 2: RECORDNO ∈ [2*CHUNK, 3*CHUNK)
49
+ // ...
50
+ // stop when N consecutive chunks return zero records
51
+ //
52
+ // Within a chunk, if the response is a full page we know the chunk is too
53
+ // dense (more records exist than we can safely retrieve in one page given
54
+ // SI's scattered ordering). We HALVE the chunk and retry — recursive
55
+ // subdivision down to a hard floor.
56
+ //
57
+ // Initial chunk size: 5000 RECORDNOs. For typical SI density (~0.1
58
+ // records per RECORDNO unit, observed on APBILL: 148 records over 1342
59
+ // units = density 0.11), a 5000 chunk holds ~550 records, well under
60
+ // page-size. For dense objects (density up to 1.0), halving once gets us
61
+ // to 2500 then 1250 (still fits in page-size=1000? No — needs another
62
+ // halve to 625). The floor MIN_CHUNK protects against pathological
63
+ // density (e.g. composite-PK objects with >1 record per RECORDNO unit,
64
+ // which shouldn't exist for SI but is defensive).
65
+ /** Initial RECORDNO range size when starting range-chunked walk. */
66
+ const RANGE_INITIAL_CHUNK = 5000;
67
+ /** Hard floor on chunk size during recursive halving. Throws if a chunk
68
+ * at this size still returns a full page (signals the impossible case
69
+ * of >pageSize records in <RANGE_MIN_CHUNK RECORDNOs). */
70
+ const RANGE_MIN_CHUNK = 100;
71
+ /** Starting probe value for upper-bound discovery. The first probe asks SI
72
+ * whether any record has RECORDNO >= this value. Cheap to start small. */
73
+ const RANGE_DISCOVERY_INITIAL_PROBE = 10000;
74
+ /** Multiplier applied each iteration when the previous probe DID find
75
+ * records. Geometric growth → log_RANGE_DISCOVERY_GROWTH(maxRecordno)
76
+ * HTTP calls upfront, bounded at log10(SANITY_CAP) ≤ 11 calls. */
77
+ const RANGE_DISCOVERY_GROWTH = 10;
78
+ /** Hard sanity cap on max RECORDNO probe. If exponential probing exceeds
79
+ * this without finding an empty range, throws — guards against runaway
80
+ * loops when SI behaves pathologically. 100 billion is well beyond any
81
+ * plausible real-world tenant's RECORDNO assignment. */
82
+ const RANGE_DISCOVERY_SANITY_CAP = 100_000_000_000;
83
+ /** Number of attempts for the discovery probe before giving up. Each
84
+ * retry uses exponential backoff. A persistent failure THROWS rather
85
+ * than silently treating the transient error as "no records exist past
86
+ * this point" — fail-stop semantics over silent under-coverage. */
87
+ const RANGE_DISCOVERY_MAX_ATTEMPTS = 3;
88
+ /** Backoff between discovery probe retries. Doubled on each retry. */
89
+ const RANGE_DISCOVERY_RETRY_BACKOFF_MS = 500;
90
+ /** Sub-range verification confirms each "completed" chunk's record count
91
+ * by independently querying its two halves and checking the sum matches.
92
+ * This catches SI inconsistencies where a chunk query reports "complete"
93
+ * with N records while sub-ranges sum to a different value. Skipped for
94
+ * chunks already at minimum size since further splitting isn't possible. */
95
+ const RANGE_VERIFICATION_ENABLED = true;
96
+ /** Cursor prefix for range-chunked walks. Format:
97
+ * `RANGE:<lo>:<chunkSize>:<upperBound>:<maxWatermarkB64>`
98
+ * Watermark is base64-encoded to allow `:` inside it. */
99
+ const RANGE_CURSOR_PREFIX = 'RANGE:';
100
+ /**
101
+ * Sage Intacct objects that exist only as children of a parent record. They
102
+ * cannot be queried directly via `readByQuery` — SI returns them inline as
103
+ * <line> elements within their parent's response. Mirroring them as
104
+ * standalone tables produces empty result sets and confusing sync runs, so
105
+ * we hide them from the picker and surface a clear error if a caller still
106
+ * tries to fetch them.
107
+ *
108
+ * Map: child object name → parent object name. Match is case-insensitive.
109
+ * Add entries here as more parent/child families come up. Patterns:
110
+ * - `*Detail` / `*Details` — line-item children of transactions
111
+ * - `*Line` — same family, older naming
112
+ * - `*Item` — line items on POs/SOs/Invoices
113
+ */
114
+ const SAGE_INTACCT_CHILD_OBJECTS = {
115
+ // Existing items / line-level entries
116
+ 'APBILLITEM': 'APBILL',
117
+ 'APBILLITEMS': 'APBILL',
118
+ 'ARINVOICEITEM': 'ARINVOICE',
119
+ 'ARINVOICEITEMS': 'ARINVOICE',
120
+ 'ARPAYMENTITEM': 'ARPAYMENT',
121
+ 'ARPAYMENTITEMS': 'ARPAYMENT',
122
+ 'APPAYMENTITEM': 'APPAYMENT',
123
+ 'APPAYMENTITEMS': 'APPAYMENT',
124
+ 'GLBATCHENTRY': 'GLBATCH',
125
+ 'GLBATCHENTRIES': 'GLBATCH',
126
+ 'JOURNALENTRYITEM': 'GLBATCH',
127
+ 'PURCHASINGDOCUMENTENTRY': 'PURCHASINGDOCUMENT',
128
+ 'SODOCUMENTENTRY': 'SODOCUMENT',
129
+ 'STOREORDERDETAIL': 'STOREORDER',
130
+ 'STOREORDERDETAILS': 'STOREORDER',
131
+ // Detail/line tables observed in production picker as silent skips —
132
+ // these never have standalone schema; SI returns them inline with
133
+ // the parent record. Mapping them to parents removes them from
134
+ // pickers and prevents "0 fields discovered" failures at apply time.
135
+ 'APBILLDETAIL': 'APBILL',
136
+ 'APPAYMENTDETAIL': 'APPAYMENT',
137
+ 'APDETAIL': 'APBILL',
138
+ 'ARDETAIL': 'ARINVOICE',
139
+ 'ARADJUSTMENTDETAIL': 'ARADJUSTMENT',
140
+ 'ARINVOICEPAYMENT': 'ARINVOICE',
141
+ 'ARRETAINAGERELEASEENTRY': 'ARRETAINAGERELEASE',
142
+ 'APRETAINAGERELEASEENTRY': 'APRETAINAGERELEASE',
143
+ 'APBILLPAYMENT': 'APBILL',
144
+ 'APPOSTEDADVANCE': 'APPAYMENT',
145
+ 'ARPOSTEDOVERPAYMENT': 'ARPAYMENT',
146
+ 'ALLOCATIONENTRY': 'ALLOCATION',
147
+ 'CONTRACTBILLINGSCHEDULEENTRY': 'CONTRACTBILLINGSCHEDULE',
148
+ 'CONTRACTREVENUESCHEDULEENTRY': 'CONTRACTREVENUESCHEDULE',
149
+ 'CONTRACTREVENUETEMPLATEENTRY': 'CONTRACTREVENUETEMPLATE',
150
+ 'CONTRACTBILLINGTEMPLATEENTRY': 'CONTRACTBILLINGTEMPLATE',
151
+ 'CONTRACTEXPENSESCHEDULEENTRY': 'CONTRACTEXPENSE',
152
+ 'CONTRACTNEGATIVEBILLINGENTRY': 'CONTRACTNEGATIVEBILLING',
153
+ 'CONTRACTUSAGEBILLING': 'CONTRACTUSAGE',
154
+ 'CONTRACTMEABUNDLEENTRY': 'CONTRACT',
155
+ 'CHANGEREQUESTENTRY': 'CHANGEREQUEST',
156
+ 'TIMESHEETENTRY': 'TIMESHEET',
157
+ // Audit/history line tables — children of audit records
158
+ 'AUDITHISTORY': 'AUDITRECORD',
159
+ 'COSTHISTORY': 'ITEM',
160
+ 'LANDEDCOSTHISTORY': 'PURCHASINGDOCUMENT',
161
+ 'BUYTOORDERHISTORY': 'PURCHASINGDOCUMENT',
162
+ 'DROPSHIPHISTORY': 'SODOCUMENT',
163
+ // System/ID-mapping helpers exposed but not standalone-syncable
164
+ 'ROLEUSERS': 'ROLE',
165
+ 'ROLEGROUPS': 'ROLE',
166
+ 'ROLEPOLICYASSIGNMENT': 'ROLE',
167
+ 'PARTNERFIELDMAP': 'PARTNER',
168
+ 'COMPLIANCETASKITEM': 'CONTRACTCOMPLIANCETASKITEM',
169
+ 'CONTRACTCOMPLIANCETASKITEM': 'CONTRACT',
170
+ 'CONTRACTCOMPLIANCENOTE': 'CONTRACT',
171
+ 'VENDORENTITYCONTACTS': 'VENDOR',
172
+ };
27
173
  /** Sage Intacct data type to generic type mapping */
28
174
  const INTACCT_TYPE_MAP = {
29
175
  'string': 'string',
@@ -172,21 +318,29 @@ const SAGE_INTACCT_OBJECTS = [
172
318
  ],
173
319
  },
174
320
  ];
175
- /** Primary key field name for each known Sage Intacct object */
321
+ /** Primary key field name for each known Sage Intacct object.
322
+ * This is a fallback used only when the IntegrationObject metadata does NOT
323
+ * carry a DefaultQueryParams.pk_field hint. Prefer the metadata over this map. */
176
324
  const OBJECT_PK_MAP = {
177
325
  CUSTOMER: 'CUSTOMERID',
178
326
  VENDOR: 'VENDORID',
179
327
  GLACCOUNT: 'ACCOUNTNO',
328
+ ACCOUNT: 'ACCOUNTNO',
180
329
  APBILL: 'RECORDNO',
181
330
  ARINVOICE: 'RECORDNO',
182
331
  PROJECT: 'PROJECTID',
183
332
  EMPLOYEE: 'EMPLOYEEID',
184
333
  DEPARTMENT: 'DEPARTMENTID',
185
334
  CLASS: 'CLASSID',
335
+ LOCATION: 'LOCATIONID',
336
+ ITEM: 'ITEMID',
337
+ CONTACT: 'CONTACTNAME',
338
+ WAREHOUSE: 'WAREHOUSEID',
339
+ USER: 'LOGIN',
340
+ CURRENCY: 'CURRENCYCODE',
341
+ TASK: 'TASKID',
342
+ CONTRACT: 'CONTRACTID',
186
343
  };
187
- /** Objects that use legacy API functions (create_*, update_*, delete_*) instead of the
188
- * generic CRUD API. These require different XML structures. */
189
- const LEGACY_FUNCTION_OBJECTS = new Set(['APBILL', 'ARINVOICE']);
190
344
  // ─── Connector Implementation ─────────────────────────────────────────
191
345
  /**
192
346
  * Connector for the Sage Intacct Web Services API (XML over HTTPS).
@@ -318,66 +472,760 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
318
472
  * Falls back to the known object list if the API call fails.
319
473
  */
320
474
  async DiscoverObjects(companyIntegration, contextUser) {
321
- try {
322
- const session = await this.GetSession(companyIntegration, contextUser);
323
- const xml = this.BuildInspectRequest(session, '*');
324
- const response = await this.SendXMLRequest(session, xml);
325
- const types = this.ParseInspectObjectsResponse(response);
326
- return types.map(name => ({
327
- Name: name,
328
- Label: this.FormatLabel(name),
329
- SupportsIncrementalSync: true,
330
- SupportsWrite: true,
331
- }));
332
- }
333
- catch {
334
- // Fallback to known objects
335
- return SAGE_INTACCT_OBJECTS.map(obj => ({
475
+ // Strategy: union the curated static catalog with whatever SI's
476
+ // dynamic `inspect *` endpoint returns. The dynamic list usually
477
+ // contains many object names — some are valid API codes (CUSTOMER,
478
+ // VENDOR, ARINVOICE, GLACCOUNT...) that the per-object describe
479
+ // endpoint accepts cleanly, others are human display names ("AP
480
+ // bill", "Work queue") that get rejected at describe time. Rather
481
+ // than guess which is which up-front, we present everything and let
482
+ // the describe phase weed out invalid names with logged warnings.
483
+ // The curated catalog is unioned in so the picker always shows the
484
+ // well-known objects even if dynamic discovery fails outright (e.g.
485
+ // auth/network problem on the inspect call).
486
+ const merged = new Map();
487
+ // Seed with curated catalog — these have known-good API codes and
488
+ // rich field metadata. They win on collisions.
489
+ for (const obj of SAGE_INTACCT_OBJECTS) {
490
+ merged.set(obj.Name, {
336
491
  Name: obj.Name,
337
492
  Label: obj.DisplayName,
338
493
  Description: obj.Description,
339
494
  SupportsIncrementalSync: true,
340
495
  SupportsWrite: obj.SupportsWrite,
341
- }));
496
+ });
497
+ }
498
+ // Layer dynamic results on top — anything new gets added; anything
499
+ // that collides with a curated entry is ignored (curated wins).
500
+ try {
501
+ const session = await this.GetSession(companyIntegration, contextUser);
502
+ const xml = this.BuildInspectRequest(session, '*');
503
+ const response = await this.SendXMLRequest(session, xml);
504
+ const candidates = this.ParseInspectObjectsResponse(response);
505
+ // Per-object probe: candidates from inspect-* are a mix of real
506
+ // API codes and display-name-derived guesses. The only reliable
507
+ // way to know which actually exist as API objects is to probe
508
+ // each one with `inspect <name>` and keep the ones SI accepts.
509
+ // Done in parallel (8-way) so 600+ probes don't sequence
510
+ // serially. Curated names are skipped (already known good).
511
+ const toProbe = candidates.filter(n => !merged.has(n));
512
+ console.log(`[SageIntacct] DiscoverObjects: probing ${toProbe.length} candidate object names...`);
513
+ const validatedNames = await this.ProbeObjectNames(session, toProbe);
514
+ console.log(`[SageIntacct] DiscoverObjects: ${validatedNames.length}/${toProbe.length} candidates validated as real API objects`);
515
+ for (const name of validatedNames) {
516
+ if (!merged.has(name)) {
517
+ merged.set(name, {
518
+ Name: name,
519
+ Label: this.FormatLabel(name),
520
+ SupportsIncrementalSync: true,
521
+ SupportsWrite: true,
522
+ });
523
+ }
524
+ }
525
+ }
526
+ catch (err) {
527
+ // Dynamic discovery failed — fall through with curated set only.
528
+ // Logged so operators know the picker is showing the smaller set.
529
+ const msg = err instanceof Error ? err.message : String(err);
530
+ console.warn(`[SageIntacct] DiscoverObjects: dynamic inspect failed (${msg}); returning curated catalog only`);
531
+ }
532
+ // Filter out known child-only objects — they can't be fetched
533
+ // independently from SI's API. Surfacing them in the picker leads
534
+ // users to select objects that always come back empty.
535
+ const filtered = [];
536
+ const skippedChildren = [];
537
+ for (const obj of merged.values()) {
538
+ const parent = SAGE_INTACCT_CHILD_OBJECTS[obj.Name.toUpperCase()];
539
+ if (parent) {
540
+ skippedChildren.push(`${obj.Name} (child of ${parent})`);
541
+ continue;
542
+ }
543
+ filtered.push(obj);
342
544
  }
545
+ if (skippedChildren.length > 0) {
546
+ console.log(`[SageIntacct] DiscoverObjects: filtered ${skippedChildren.length} known child-only objects from picker — ` +
547
+ `they sync inline via their parent records.`);
548
+ }
549
+ return filtered;
343
550
  }
344
551
  /**
345
- * Discovers fields on a specific object using the inspect API function.
552
+ * Probes a list of candidate object names against SI's per-object
553
+ * `inspect <name>` endpoint to determine which are real API objects vs
554
+ * display-name labels with no API equivalent. Runs in parallel with a
555
+ * bounded worker pool so the 600+ probe round-trips don't serialize.
556
+ *
557
+ * Returns only the names that responded successfully — failures (including
558
+ * "Object type X not found") are silently dropped. This is intentional:
559
+ * the goal is to populate the picker with names the user can actually
560
+ * sync, not surface every SI-side error.
561
+ */
562
+ async ProbeObjectNames(session, candidates) {
563
+ if (candidates.length === 0)
564
+ return [];
565
+ const concurrency = 8;
566
+ const validated = [];
567
+ let cursor = 0;
568
+ let zeroFieldCount = 0;
569
+ const errorSamples = [];
570
+ let totalErrors = 0;
571
+ const worker = async () => {
572
+ while (true) {
573
+ const idx = cursor++;
574
+ if (idx >= candidates.length)
575
+ return;
576
+ const name = candidates[idx];
577
+ try {
578
+ const xml = this.BuildInspectRequest(session, name);
579
+ const response = await this.SendXMLRequest(session, xml);
580
+ // Successful response = real API object. Empty inspect
581
+ // fields don't mean it's not syncable — many SI objects
582
+ // (especially platform/contract/role types) return no
583
+ // schema via inspect but DO return data via readByQuery.
584
+ // The earlier "require ≥1 field" check was too strict
585
+ // and silently dropped 242 real objects per probe pass.
586
+ this.CheckForErrors(response);
587
+ validated.push(name);
588
+ if (this.ParseInspectFieldsResponse(response).length === 0) {
589
+ zeroFieldCount++;
590
+ }
591
+ }
592
+ catch (err) {
593
+ totalErrors++;
594
+ if (errorSamples.length < 3) {
595
+ const msg = err instanceof Error ? err.message : String(err);
596
+ errorSamples.push(`"${name}": ${msg.slice(0, 200)}`);
597
+ }
598
+ }
599
+ }
600
+ };
601
+ await Promise.all(Array.from({ length: concurrency }, () => worker()));
602
+ console.log(`[SageIntacct] ProbeObjectNames: ${validated.length} validated, ` +
603
+ `${zeroFieldCount} returned no fields, ${totalErrors} errored`);
604
+ if (errorSamples.length > 0) {
605
+ console.log(`[SageIntacct] ProbeObjectNames sample errors:\n ${errorSamples.join('\n ')}`);
606
+ }
607
+ return validated;
608
+ }
609
+ /**
610
+ * Discovers fields on a specific object by combining the `inspect` API
611
+ * call (standard fields) with a `lookup` call (which exposes custom fields
612
+ * via the ISCUSTOM flag). Custom fields are returned with IsCustom=true so
613
+ * the sync engine can flag them in IntegrationObjectField metadata.
346
614
  */
347
615
  async DiscoverFields(companyIntegration, objectName, contextUser) {
348
616
  const session = await this.GetSession(companyIntegration, contextUser);
349
- const xml = this.BuildInspectRequest(session, objectName);
350
- const response = await this.SendXMLRequest(session, xml);
351
- const fields = this.ParseInspectFieldsResponse(response);
352
- return fields.map(f => ({
353
- Name: f.Name,
354
- Label: f.Label || this.FormatLabel(f.Name),
355
- DataType: INTACCT_TYPE_MAP[f.DataType.toLowerCase()] ?? 'string',
356
- IsRequired: f.IsRequired,
357
- IsUniqueKey: f.Name === this.GetPrimaryKeyField(objectName),
358
- IsReadOnly: f.IsReadOnly,
359
- }));
617
+ const inspectXml = this.BuildInspectRequest(session, objectName);
618
+ const inspectResponse = await this.SendXMLRequest(session, inspectXml);
619
+ let standardFields = this.ParseInspectFieldsResponse(inspectResponse);
620
+ // Always pull lookup. Two reasons: (1) it's the source of ISCUSTOM
621
+ // flags for custom fields, (2) it's the deterministic fallback when
622
+ // inspect returns an empty <Fields/> (common for SI platform/role/
623
+ // contract-template objects whose schema isn't surfaced via inspect
624
+ // but IS via lookup). Same shape, different SI metadata API surface.
625
+ let lookupFields = [];
626
+ let customFieldNames = new Set();
627
+ try {
628
+ const lookupXml = this.BuildLookupRequest(session, objectName);
629
+ const lookupResponse = await this.SendXMLRequest(session, lookupXml);
630
+ lookupFields = this.ParseLookupFieldsResponse(lookupResponse);
631
+ for (const f of lookupFields) {
632
+ if (f.IsCustom)
633
+ customFieldNames.add(f.Name);
634
+ }
635
+ }
636
+ catch {
637
+ // Lookup not supported for this object — proceed with inspect only.
638
+ }
639
+ // Lookup-as-fallback: when inspect returned no fields but lookup did,
640
+ // promote lookup as the primary field source. Lookup-derived fields
641
+ // are deterministic SI metadata, no inference.
642
+ if (standardFields.length === 0 && lookupFields.length > 0) {
643
+ console.log(`[SageIntacct] DiscoverFields: ${objectName} — inspect returned 0 fields, using ${lookupFields.length} fields from lookup API`);
644
+ standardFields = lookupFields;
645
+ }
646
+ const pkField = this.GetPrimaryKeyField(objectName, companyIntegration.IntegrationID);
647
+ // Curated catalog overlay: when SAGE_INTACCT_OBJECTS has a record for
648
+ // this object, build a quick lookup of curated field metadata. Where
649
+ // inspect is missing data (no MaxLength, no description, etc.) we fill
650
+ // from the curated entry. Live inspect data still wins on direct
651
+ // conflicts — curation is a supplement, not a source of truth.
652
+ const curatedObj = SAGE_INTACCT_OBJECTS.find(o => o.Name.toLowerCase() === objectName.toLowerCase());
653
+ const curatedFieldMap = new Map();
654
+ if (curatedObj) {
655
+ for (const cf of curatedObj.Fields) {
656
+ curatedFieldMap.set(cf.Name.toLowerCase(), cf);
657
+ }
658
+ }
659
+ const result = standardFields.map(f => {
660
+ const isCustom = customFieldNames.has(f.Name) || f.IsCustom;
661
+ if (isCustom) {
662
+ console.debug(`[SageIntacct] Custom field detected on ${objectName}: ${f.Name}`);
663
+ }
664
+ const curated = curatedFieldMap.get(f.Name.toLowerCase());
665
+ return {
666
+ Name: f.Name,
667
+ Label: f.Label || curated?.DisplayName || this.FormatLabel(f.Name),
668
+ Description: curated?.Description,
669
+ DataType: INTACCT_TYPE_MAP[f.DataType.toLowerCase()] ?? curated?.Type ?? 'string',
670
+ IsRequired: f.IsRequired || curated?.IsRequired === true,
671
+ IsUniqueKey: f.Name === pkField || curated?.IsPrimaryKey === true,
672
+ IsReadOnly: f.IsReadOnly || curated?.IsReadOnly === true,
673
+ IsForeignKey: !!f.References,
674
+ ForeignKeyTarget: f.References ?? null,
675
+ MaxLength: f.MaxLength ?? null,
676
+ Precision: f.Precision ?? null,
677
+ Scale: f.Scale ?? null,
678
+ DefaultValue: f.DefaultValue ?? null,
679
+ };
680
+ });
681
+ // Append curated fields that inspect didn't return — happens when SI
682
+ // returns a partial field list for legacy objects, or when curation
683
+ // covers a known field that the per-tenant SI instance omits.
684
+ const inspectNames = new Set(result.map(f => f.Name.toLowerCase()));
685
+ if (curatedObj) {
686
+ for (const cf of curatedObj.Fields) {
687
+ if (!inspectNames.has(cf.Name.toLowerCase())) {
688
+ result.push({
689
+ Name: cf.Name,
690
+ Label: cf.DisplayName ?? cf.Name,
691
+ Description: cf.Description,
692
+ DataType: cf.Type ?? 'string',
693
+ IsRequired: cf.IsRequired === true,
694
+ IsUniqueKey: cf.IsPrimaryKey === true,
695
+ IsReadOnly: cf.IsReadOnly === true,
696
+ });
697
+ inspectNames.add(cf.Name.toLowerCase());
698
+ }
699
+ }
700
+ }
701
+ // Engine cache fallback. SI's per-object inspect frequently returns
702
+ // empty <Fields/> for platform/contract/role objects that ARE real
703
+ // and ARE syncable via readByQuery — they just don't expose schema.
704
+ // For those, hydrate constraints from previously-persisted
705
+ // IntegrationObjectField rows so the user's prior describe survives
706
+ // a transient SI gap. Live > curated > IOF cache (precedence order).
707
+ const knownNames = new Set(result.map(f => f.Name.toLowerCase()));
708
+ try {
709
+ const integrationObj = IntegrationEngineBase.Instance.GetIntegrationObject(companyIntegration.IntegrationID, objectName);
710
+ if (integrationObj) {
711
+ const cachedFields = IntegrationEngineBase.Instance.GetIntegrationObjectFields(integrationObj.ID);
712
+ let appended = 0;
713
+ for (const cf of cachedFields) {
714
+ if (cf.Status !== 'Active')
715
+ continue;
716
+ if (knownNames.has(cf.Name.toLowerCase()))
717
+ continue;
718
+ result.push({
719
+ Name: cf.Name,
720
+ Label: cf.DisplayName ?? cf.Name,
721
+ Description: cf.Description ?? undefined,
722
+ DataType: cf.Type ?? 'string',
723
+ IsRequired: !!cf.IsRequired,
724
+ IsUniqueKey: !!cf.IsPrimaryKey,
725
+ IsReadOnly: !!cf.IsReadOnly,
726
+ MaxLength: cf.Length ?? null,
727
+ Precision: cf.Precision ?? null,
728
+ Scale: cf.Scale ?? null,
729
+ DefaultValue: cf.DefaultValue ?? null,
730
+ });
731
+ knownNames.add(cf.Name.toLowerCase());
732
+ appended++;
733
+ }
734
+ if (appended > 0) {
735
+ console.log(`[SageIntacct] DiscoverFields: hydrated ${appended} fields on ${objectName} from IntegrationObjectField cache (live inspect was missing them)`);
736
+ }
737
+ }
738
+ }
739
+ catch (err) {
740
+ // Engine may not be configured; live result alone is fine.
741
+ console.warn(`[SageIntacct] DiscoverFields: IOF cache fallback unavailable for ${objectName}: ${err instanceof Error ? err.message : String(err)}`);
742
+ }
743
+ // Final defensive dedupe by lowercased Name. Belt-and-suspenders for
744
+ // the (IntegrationObjectID, Name) unique key — every code path that
745
+ // ever adds to `result` is now guarded.
746
+ const seen = new Set();
747
+ return result.filter(f => {
748
+ const key = f.Name.toLowerCase();
749
+ if (seen.has(key)) {
750
+ console.warn(`[SageIntacct] DiscoverFields: dropping duplicate field "${f.Name}" on ${objectName}`);
751
+ return false;
752
+ }
753
+ seen.add(key);
754
+ return true;
755
+ });
756
+ }
757
+ BuildLookupRequest(session, objectName) {
758
+ return this.WrapInSessionRequest(session, `
759
+ <function controlid="${CONTROL_ID_PREFIX}-lookup-${Date.now()}">
760
+ <lookup>
761
+ <object>${this.EscapeXmlValue(objectName)}</object>
762
+ </lookup>
763
+ </function>`);
764
+ }
765
+ /**
766
+ * Full parser for SI's `lookup <object>` response. Returns the same shape
767
+ * as ParseInspectFieldsResponse so callers can use lookup as a drop-in
768
+ * fallback when inspect comes back empty. SI's lookup endpoint frequently
769
+ * returns a populated `<Fields>` block for objects whose `inspect` returns
770
+ * an empty `<Fields/>` — same metadata, different API surface, same
771
+ * deterministic source-of-truth from SI.
772
+ *
773
+ * Lookup uses uppercase tags (`<NAME>`, `<DATATYPE>`, `<ISCUSTOM>`) where
774
+ * inspect uses mixed case (`<Name>`, `<DataType>`). Tag extraction tries
775
+ * both casings to be robust against per-tenant API quirks.
776
+ */
777
+ ParseLookupFieldsResponse(xml) {
778
+ this.CheckForErrors(xml);
779
+ const fields = [];
780
+ const seen = new Set();
781
+ const fieldRegex = /<Field>\s*([\s\S]*?)\s*<\/Field>/gi;
782
+ let match;
783
+ while ((match = fieldRegex.exec(xml)) !== null) {
784
+ const frag = match[1];
785
+ const name = this.ExtractXmlValueFromFragment(frag, 'NAME')
786
+ || this.ExtractXmlValueFromFragment(frag, 'Name');
787
+ if (!name)
788
+ continue;
789
+ const key = name.toLowerCase();
790
+ if (seen.has(key))
791
+ continue;
792
+ seen.add(key);
793
+ const dataType = this.ExtractXmlValueFromFragment(frag, 'DATATYPE')
794
+ || this.ExtractXmlValueFromFragment(frag, 'DataType')
795
+ || 'string';
796
+ const label = this.ExtractXmlValueFromFragment(frag, 'DISPLAYLABEL')
797
+ || this.ExtractXmlValueFromFragment(frag, 'DisplayLabel')
798
+ || name;
799
+ const isRequired = (this.ExtractXmlValueFromFragment(frag, 'REQUIRED')
800
+ || this.ExtractXmlValueFromFragment(frag, 'Required')).toLowerCase() === 'true';
801
+ const isReadOnly = (this.ExtractXmlValueFromFragment(frag, 'READONLY')
802
+ || this.ExtractXmlValueFromFragment(frag, 'ReadOnly')).toLowerCase() === 'true';
803
+ const isCustom = (this.ExtractXmlValueFromFragment(frag, 'ISCUSTOM')
804
+ || this.ExtractXmlValueFromFragment(frag, 'IsCustom')).toLowerCase() === 'true';
805
+ const maxLength = this.ParseNumericTagFromFragment(frag, 'MAXLENGTH')
806
+ ?? this.ParseNumericTagFromFragment(frag, 'MaxLength')
807
+ ?? this.ParseNumericTagFromFragment(frag, 'LENGTH')
808
+ ?? this.ParseNumericTagFromFragment(frag, 'Length');
809
+ const precision = this.ParseNumericTagFromFragment(frag, 'PRECISION')
810
+ ?? this.ParseNumericTagFromFragment(frag, 'Precision');
811
+ const scale = this.ParseNumericTagFromFragment(frag, 'SCALE')
812
+ ?? this.ParseNumericTagFromFragment(frag, 'Scale');
813
+ const defaultValueRaw = this.ExtractXmlValueFromFragment(frag, 'DEFAULTVALUE')
814
+ || this.ExtractXmlValueFromFragment(frag, 'DefaultValue');
815
+ const referencesRaw = this.ExtractXmlValueFromFragment(frag, 'REFERENCES')
816
+ || this.ExtractXmlValueFromFragment(frag, 'References');
817
+ fields.push({
818
+ Name: name, Label: label, DataType: dataType,
819
+ IsRequired: isRequired, IsReadOnly: isReadOnly, IsCustom: isCustom,
820
+ MaxLength: maxLength,
821
+ Precision: precision,
822
+ Scale: scale,
823
+ DefaultValue: defaultValueRaw && defaultValueRaw.length > 0 ? defaultValueRaw : null,
824
+ References: referencesRaw && referencesRaw.length > 0 ? referencesRaw : null,
825
+ });
826
+ }
827
+ return fields;
360
828
  }
361
829
  // ─── FetchChanges ────────────────────────────────────────────────
362
830
  /**
363
- * Fetches records using readByQuery with WHENMODIFIED watermark filtering.
364
- * Supports pagination via readMore with resultId tokens.
831
+ * Fetches records from a Sage Intacct object. Uses one of two strategies
832
+ * based on the object's primary key shape:
833
+ *
834
+ * - Numeric PK (RECORDNO): walks RECORDNO in fixed numeric ranges with
835
+ * recursive halving on dense chunks. Bulletproof — no dependence on
836
+ * SI's scan order, pagination signals, or page-size limits.
837
+ *
838
+ * - String PK (CUSTOMERID, VENDORID, etc.): single readByQuery + readMore
839
+ * loop using SI's server-side resultId. Hard fails (instead of silently
840
+ * dropping records) if SI returns a full page without a resultId.
365
841
  */
366
842
  async FetchChanges(ctx) {
843
+ // Guardrail: if a caller somehow ends up trying to fetch a child-only
844
+ // object directly (entity map left over from a previous picker, or a
845
+ // hand-set IntegrationObject row), bail with a clear message rather
846
+ // than producing an empty result that looks like "the table is empty".
847
+ const parentForChild = SAGE_INTACCT_CHILD_OBJECTS[ctx.ObjectName.toUpperCase()];
848
+ if (parentForChild) {
849
+ console.warn(`[SageIntacct] FetchChanges: "${ctx.ObjectName}" is a child of "${parentForChild}" and cannot be fetched directly. ` +
850
+ `Sync the parent object instead — child rows arrive inline in the parent's response.`);
851
+ return { Records: [], HasMore: false };
852
+ }
367
853
  const session = await this.GetSession(ctx.CompanyIntegration, ctx.ContextUser);
368
- const pkField = this.GetPrimaryKeyField(ctx.ObjectName);
369
- const pageSize = Math.min(ctx.BatchSize || DEFAULT_PAGE_SIZE, DEFAULT_PAGE_SIZE);
370
- // If we have a cursor (resultId) from a previous call, use readMore
854
+ const pkField = this.GetPrimaryKeyField(ctx.ObjectName, ctx.CompanyIntegration.IntegrationID);
855
+ const pageSize = Math.max(ctx.BatchSize || 0, DEFAULT_PAGE_SIZE);
856
+ // Dispatch by PK shape. RECORDNO (universal SI numeric row id) gets
857
+ // the bulletproof range-chunked walk. Everything else (string PKs)
858
+ // takes the single-pull-with-readMore path.
859
+ if (pkField === 'RECORDNO') {
860
+ return this.FetchChangesByRange(session, ctx, pageSize);
861
+ }
862
+ return this.FetchChangesSinglePull(session, ctx, pageSize, pkField);
863
+ }
864
+ /**
865
+ * Range-chunked walk over numeric RECORDNO. Each FetchChanges call
866
+ * processes ONE chunk (one HTTP request to SI). The engine drives the
867
+ * walk by passing the returned NextCursor back on the next call. Cursor
868
+ * encodes (lo, chunkSize, upperBound, maxWatermarkSeen).
869
+ *
870
+ * Termination: walk completes when `lo >= upperBound`. The upperBound is
871
+ * discovered ONCE at the start of the walk via exponential probe of SI's
872
+ * actual MAX RECORDNO — no empty-chunk heuristic, no "what if records
873
+ * exist past N" doubt. The walk covers exactly [0, MAX_RECORDNO + safety
874
+ * margin), every RECORDNO accounted for.
875
+ *
876
+ * Density adaptation: full-page response → halve chunk, re-query same lo.
877
+ * Watermark: tracked in cursor; emitted only on final batch.
878
+ */
879
+ async FetchChangesByRange(session, ctx, pageSize) {
880
+ let state = this.parseRangeCursor(ctx.CurrentCursor);
881
+ if (!state) {
882
+ // First call of this walk — discover the actual upper bound on
883
+ // RECORDNO so termination is precise rather than heuristic.
884
+ const upperBound = await this.discoverUpperBound(session, ctx);
885
+ // Start with chunkSize = min(upperBound, RANGE_INITIAL_CHUNK)
886
+ // so small objects walk in a single chunk (no wasted traversal),
887
+ // while huge ranges still cap at a sane starting point that won't
888
+ // require many density halvings to fit in pageSize.
889
+ const initialChunkSize = Math.max(Math.min(upperBound, RANGE_INITIAL_CHUNK), RANGE_MIN_CHUNK);
890
+ state = {
891
+ lo: 0,
892
+ chunkSize: initialChunkSize,
893
+ upperBound,
894
+ maxWatermark: undefined,
895
+ };
896
+ }
897
+ // If this call lands at-or-past the discovered upper bound, the walk
898
+ // is over even before we make the first request. Emit the watermark
899
+ // and terminate. (This shouldn't happen normally — advanceRangeWalk
900
+ // catches it first — but defends against cursors hand-crafted by
901
+ // tooling or replays.)
902
+ if (state.lo >= state.upperBound) {
903
+ console.log(`[SageIntacct] ${ctx.ObjectName}: RANGE walk complete — lo=${state.lo} >= upperBound=${state.upperBound}. Final watermark: ${state.maxWatermark ?? 'none'}`);
904
+ return { Records: [], HasMore: false, NewWatermarkValue: state.maxWatermark };
905
+ }
906
+ const hi = Math.min(state.lo + state.chunkSize, state.upperBound);
907
+ const filter = this.buildRangeFilter(ctx, state.lo, hi);
908
+ console.log(`[SageIntacct] ${ctx.ObjectName}: RANGE [${state.lo}, ${hi}) chunkSize=${state.chunkSize} upperBound=${state.upperBound} — filter="${filter}"`);
909
+ const xml = this.BuildReadByQueryRequest(session, ctx.ObjectName, filter, pageSize);
910
+ const response = await this.SendXMLRequest(session, xml);
911
+ const result = this.ParseReadByQueryResponseRaw(response, ctx.ObjectName, pageSize);
912
+ // Density check: a full page back means SI may have more records in
913
+ // this RECORDNO range than fit in one page. Because SI doesn't scan
914
+ // PK-ascending, the records we got could be any random subset of the
915
+ // chunk's true contents — keeping them risks a permanent miss of the
916
+ // un-returned records. Discard, halve, re-query.
917
+ if (result.records.length >= pageSize) {
918
+ return this.handleDenseChunk(ctx, state, pageSize);
919
+ }
920
+ // Sub-range verification: independently query [lo, mid) and [mid, hi)
921
+ // and confirm their counts sum to the main query's count. Catches SI
922
+ // inconsistencies where a chunk reports "complete" while sub-ranges
923
+ // tell a different story. Throws on mismatch (fail-stop) — partial
924
+ // sync data is worse than no sync data.
925
+ if (RANGE_VERIFICATION_ENABLED && (hi - state.lo) >= 2 * RANGE_MIN_CHUNK) {
926
+ const verificationOutcome = await this.verifyChunk(session, ctx, state.lo, hi, pageSize, result.records.length);
927
+ if (verificationOutcome === 'sub-range-dense') {
928
+ // Either half is dense — drill down via standard halving.
929
+ return this.handleDenseChunk(ctx, state, pageSize);
930
+ }
931
+ // verificationOutcome === 'verified' → sums match; main count is trustworthy.
932
+ }
933
+ // Chunk fully captured. Compute its contribution to the watermark and
934
+ // decide whether to advance to the next chunk or terminate.
935
+ return this.advanceRangeWalk(ctx, state, result.records);
936
+ }
937
+ /** Sub-range verification: query both halves of [lo, hi) independently
938
+ * and confirm their counts sum to `mainCount`. Returns 'verified' when
939
+ * sums match, 'sub-range-dense' when either half is at-or-above
940
+ * pageSize (signalling that further drill-down is needed). Throws on
941
+ * count mismatch — that's an SI inconsistency we refuse to paper over. */
942
+ async verifyChunk(session, ctx, lo, hi, pageSize, mainCount) {
943
+ const mid = lo + Math.floor((hi - lo) / 2);
944
+ const leftFilter = this.buildRangeFilter(ctx, lo, mid);
945
+ const rightFilter = this.buildRangeFilter(ctx, mid, hi);
946
+ const [leftRaw, rightRaw] = await Promise.all([
947
+ this.SendXMLRequest(session, this.BuildReadByQueryRequest(session, ctx.ObjectName, leftFilter, pageSize)),
948
+ this.SendXMLRequest(session, this.BuildReadByQueryRequest(session, ctx.ObjectName, rightFilter, pageSize)),
949
+ ]);
950
+ const left = this.ParseReadByQueryResponseRaw(leftRaw, ctx.ObjectName, pageSize);
951
+ const right = this.ParseReadByQueryResponseRaw(rightRaw, ctx.ObjectName, pageSize);
952
+ // If either half is dense, we can't trust the parent's count yet —
953
+ // drill into smaller chunks. The caller handles this via the standard
954
+ // dense-halving path.
955
+ if (left.records.length >= pageSize || right.records.length >= pageSize) {
956
+ console.warn(`[SageIntacct] ${ctx.ObjectName}: verification of [${lo}, ${hi}) revealed dense sub-range ` +
957
+ `(left=${left.records.length}, right=${right.records.length}). Drilling down.`);
958
+ return 'sub-range-dense';
959
+ }
960
+ const sumOfHalves = left.records.length + right.records.length;
961
+ if (sumOfHalves !== mainCount) {
962
+ throw new Error(`[SageIntacct] ${ctx.ObjectName}: SI INCONSISTENCY in [${lo}, ${hi}). ` +
963
+ `Main query returned ${mainCount} records, but sub-ranges ` +
964
+ `[${lo}, ${mid}) + [${mid}, ${hi}) returned ${left.records.length} + ${right.records.length} = ${sumOfHalves}. ` +
965
+ `SI is hiding records — refusing to continue sync with potentially incomplete data. ` +
966
+ `Re-run sync; if persistent, this object may need manual investigation.`);
967
+ }
968
+ // Sums match — main count verified.
969
+ return 'verified';
970
+ }
971
+ /** Exponential probe of SI for the highest RECORDNO that exists on this
972
+ * object. Returns an upper bound (always strictly greater than the true
973
+ * max). Costs O(log_RANGE_DISCOVERY_GROWTH(maxRecordno)) HTTP calls —
974
+ * typically 1-5 calls for any plausible tenant.
975
+ *
976
+ * Uses pageSize=1 because we only care whether ANY record exists at or
977
+ * above the probe point; the records themselves are discarded. Does NOT
978
+ * apply WHENMODIFIED filter — we want the absolute upper bound on
979
+ * RECORDNO regardless of when it was modified, so that incremental
980
+ * syncs still cover the full RECORDNO space (each chunk inside the
981
+ * walk applies its own WHENMODIFIED filter).
982
+ *
983
+ * Each probe is retried up to RANGE_DISCOVERY_MAX_ATTEMPTS times on
984
+ * network/transport failure before throwing. A retried-but-still-failed
985
+ * probe THROWS rather than silently treating the failure as "no records
986
+ * exist past this point" — fail-stop matches the user requirement that
987
+ * partial sync is worse than no sync. */
988
+ async discoverUpperBound(session, ctx) {
989
+ let probe = RANGE_DISCOVERY_INITIAL_PROBE;
990
+ while (probe <= RANGE_DISCOVERY_SANITY_CAP) {
991
+ const result = await this.probeRecordnoExistsRetried(session, ctx, probe);
992
+ if (result.records.length === 0 && result.numRemaining === 0) {
993
+ console.log(`[SageIntacct] ${ctx.ObjectName}: discovered upperBound=${probe} (no records at or above this RECORDNO)`);
994
+ return probe;
995
+ }
996
+ probe *= RANGE_DISCOVERY_GROWTH;
997
+ }
998
+ throw new Error(`[SageIntacct] ${ctx.ObjectName}: max RECORDNO exceeds sanity cap of ${RANGE_DISCOVERY_SANITY_CAP}. ` +
999
+ `This object's RECORDNOs are pathologically large; raise RANGE_DISCOVERY_SANITY_CAP.`);
1000
+ }
1001
+ /** Single discovery probe with retry on TRANSPORT errors only. SI-side
1002
+ * errors (permission denied, syntax invalid, schema rejected) are
1003
+ * deterministic and don't benefit from retry — they're propagated
1004
+ * immediately so the caller (sync engine) can record the object as
1005
+ * failed and move on. Throws after RANGE_DISCOVERY_MAX_ATTEMPTS
1006
+ * exhausted on transport — does NOT silently treat persistent failure
1007
+ * as "empty result." */
1008
+ async probeRecordnoExistsRetried(session, ctx, probe) {
1009
+ const filter = `RECORDNO >= ${probe}`;
1010
+ let lastError;
1011
+ for (let attempt = 1; attempt <= RANGE_DISCOVERY_MAX_ATTEMPTS; attempt++) {
1012
+ try {
1013
+ const xml = this.BuildReadByQueryRequest(session, ctx.ObjectName, filter, 1);
1014
+ const response = await this.SendXMLRequest(session, xml);
1015
+ return this.ParseReadByQueryResponseRaw(response, ctx.ObjectName, 1);
1016
+ }
1017
+ catch (err) {
1018
+ lastError = err;
1019
+ // SI-side errors (permission, schema, query syntax) are
1020
+ // deterministic — retrying gives the same answer. Surface
1021
+ // immediately so the engine can move on to the next object.
1022
+ if (this.isSageIntacctApiError(err)) {
1023
+ throw err;
1024
+ }
1025
+ const willRetry = attempt < RANGE_DISCOVERY_MAX_ATTEMPTS;
1026
+ console.warn(`[SageIntacct] ${ctx.ObjectName}: discovery probe at RECORDNO >= ${probe} ` +
1027
+ `transport attempt ${attempt}/${RANGE_DISCOVERY_MAX_ATTEMPTS} failed: ${err?.message ?? String(err)}` +
1028
+ (willRetry ? ` — retrying after ${RANGE_DISCOVERY_RETRY_BACKOFF_MS * attempt}ms` : ' — giving up'));
1029
+ if (willRetry) {
1030
+ await new Promise(resolve => setTimeout(resolve, RANGE_DISCOVERY_RETRY_BACKOFF_MS * attempt));
1031
+ }
1032
+ }
1033
+ }
1034
+ throw new Error(`[SageIntacct] ${ctx.ObjectName}: discovery probe at RECORDNO >= ${probe} failed ` +
1035
+ `after ${RANGE_DISCOVERY_MAX_ATTEMPTS} transport attempts. Refusing to assume "no records exist" ` +
1036
+ `from a transport-level failure — sync would silently undercount. Last error: ` +
1037
+ `${lastError?.message ?? String(lastError)}`);
1038
+ }
1039
+ /** Returns true when the error originated from SI's API response (a
1040
+ * thrown structured error from CheckForErrors), false for transport /
1041
+ * network / parser failures. SI-side errors are not worth retrying. */
1042
+ isSageIntacctApiError(err) {
1043
+ const msg = err?.message;
1044
+ if (typeof msg !== 'string')
1045
+ return false;
1046
+ return msg.startsWith('Sage Intacct error ') || msg.startsWith('Sage Intacct API error');
1047
+ }
1048
+ /** Halve the current chunk and re-query its lower half on the next call.
1049
+ * Throws if already at RANGE_MIN_CHUNK — that signals the impossible
1050
+ * case where SI has more records in 100 RECORDNOs than fit in our
1051
+ * configured page size, and we cannot guarantee complete sync. */
1052
+ handleDenseChunk(ctx, state, pageSize) {
1053
+ const halved = Math.max(Math.floor(state.chunkSize / 2), RANGE_MIN_CHUNK);
1054
+ if (halved === state.chunkSize) {
1055
+ throw new Error(`[SageIntacct] ${ctx.ObjectName}: chunk at minimum size ${state.chunkSize} ` +
1056
+ `still returns ≥${pageSize} records. Cannot guarantee complete sync. ` +
1057
+ `Increase DEFAULT_PAGE_SIZE or lower RANGE_MIN_CHUNK.`);
1058
+ }
1059
+ console.warn(`[SageIntacct] ${ctx.ObjectName}: dense chunk at [${state.lo}, ${state.lo + state.chunkSize}) — ` +
1060
+ `halving to ${halved} and retrying. (Records from this query are discarded; they will be re-fetched in halved chunks.)`);
1061
+ // Re-query the SAME lo with smaller chunkSize. upperBound unchanged.
1062
+ const nextCursor = this.buildRangeCursor({ ...state, chunkSize: halved });
1063
+ return { Records: [], HasMore: true, NextCursor: nextCursor };
1064
+ }
1065
+ /** Chunk completed safely. Advance to next chunk or terminate the walk
1066
+ * when we cross the discovered upperBound. */
1067
+ advanceRangeWalk(ctx, state, records) {
1068
+ const externalRecords = records.map(r => this.toExternalRecord(r, ctx.ObjectName, 'RECORDNO'));
1069
+ const chunkWatermark = this.ComputeWatermark(records);
1070
+ const newMaxWatermark = this.maxWatermarkString(state.maxWatermark, chunkWatermark);
1071
+ const nextLo = state.lo + state.chunkSize;
1072
+ if (nextLo >= state.upperBound) {
1073
+ console.log(`[SageIntacct] ${ctx.ObjectName}: RANGE walk complete — covered [0, ${state.upperBound}). Final watermark: ${newMaxWatermark ?? 'none'}`);
1074
+ return {
1075
+ Records: externalRecords,
1076
+ HasMore: false,
1077
+ NewWatermarkValue: newMaxWatermark,
1078
+ };
1079
+ }
1080
+ const nextCursor = this.buildRangeCursor({
1081
+ lo: nextLo,
1082
+ chunkSize: RANGE_INITIAL_CHUNK, // grow back to default after dense recovery
1083
+ upperBound: state.upperBound, // carried forward unchanged
1084
+ maxWatermark: newMaxWatermark,
1085
+ });
1086
+ return {
1087
+ Records: externalRecords,
1088
+ HasMore: true,
1089
+ NextCursor: nextCursor,
1090
+ };
1091
+ }
1092
+ /** Single-page fetch + readMore loop for objects with string PKs (CUSTOMER,
1093
+ * VENDOR, GLACCOUNT, etc.). These are typically small lookup tables; if
1094
+ * one ever exceeds a single page, we trust SI's resultId. If SI returns
1095
+ * a full page without a resultId, we HARD FAIL rather than silently
1096
+ * drop records via PK-cursor pagination. */
1097
+ async FetchChangesSinglePull(session, ctx, pageSize, pkField) {
1098
+ // readMore continuation — SI maintains scan-order consistency on its
1099
+ // server-side resultId, so this path is safe.
371
1100
  if (ctx.CurrentCursor) {
372
1101
  return this.FetchMoreRecords(session, ctx.CurrentCursor, ctx.ObjectName, pkField);
373
1102
  }
374
- // Build the query filter
375
- const filter = ctx.WatermarkValue
376
- ? `WHENMODIFIED >= '${ctx.WatermarkValue}'`
377
- : '';
1103
+ // Initial fetch with watermark filter only.
1104
+ const hasWhenModified = this.ObjectHasWhenModified(ctx.CompanyIntegration.IntegrationID, ctx.ObjectName);
1105
+ const filterParts = [];
1106
+ if (ctx.WatermarkValue && hasWhenModified) {
1107
+ filterParts.push(`WHENMODIFIED >= '${this.NormalizeSageIntacctTimestamp(ctx.WatermarkValue)}'`);
1108
+ }
1109
+ const filter = filterParts.join(' AND ');
1110
+ console.log(`[SageIntacct] ${ctx.ObjectName}: SINGLE-PULL fetch (${pkField}) — filter="${filter}"`);
378
1111
  const xml = this.BuildReadByQueryRequest(session, ctx.ObjectName, filter, pageSize);
379
1112
  const response = await this.SendXMLRequest(session, xml);
380
- return this.ParseReadByQueryResponse(response, ctx.ObjectName, pkField);
1113
+ const raw = this.ParseReadByQueryResponseRaw(response, ctx.ObjectName, pageSize);
1114
+ const externalRecords = raw.records.map(r => this.toExternalRecord(r, ctx.ObjectName, pkField));
1115
+ const watermark = this.ComputeWatermark(raw.records);
1116
+ // Safe terminal cases: small response (definitely complete) OR full
1117
+ // page WITH resultId (readMore handles the rest deterministically).
1118
+ if (raw.records.length < pageSize) {
1119
+ return { Records: externalRecords, HasMore: false, NewWatermarkValue: watermark };
1120
+ }
1121
+ if (raw.resultId) {
1122
+ return { Records: externalRecords, HasMore: true, NextCursor: raw.resultId };
1123
+ }
1124
+ // Dangerous case: full page, no resultId, string PK. PK-cursor
1125
+ // pagination would silently drop records (SI's scan order is not
1126
+ // PK-ascending). We REFUSE to proceed rather than corrupt the sync.
1127
+ throw new Error(`[SageIntacct] ${ctx.ObjectName}: SI returned ${raw.records.length} records ` +
1128
+ `(full page) with no resultId and the PK (${pkField}) is non-numeric. ` +
1129
+ `Cannot safely paginate without ordering guarantees. ` +
1130
+ `Either (a) raise DEFAULT_PAGE_SIZE so this object fits in one page, ` +
1131
+ `or (b) add ${ctx.ObjectName} to the numeric-PK code path if it has a RECORDNO column.`);
1132
+ }
1133
+ parseRangeCursor(cursor) {
1134
+ if (!cursor || !cursor.startsWith(RANGE_CURSOR_PREFIX))
1135
+ return null;
1136
+ const body = cursor.slice(RANGE_CURSOR_PREFIX.length);
1137
+ const parts = body.split(':');
1138
+ if (parts.length < 4)
1139
+ return null;
1140
+ const lo = parseInt(parts[0], 10);
1141
+ const chunkSize = parseInt(parts[1], 10);
1142
+ const upperBound = parseInt(parts[2], 10);
1143
+ const maxWatermarkB64 = parts[3];
1144
+ if (!Number.isFinite(lo) || !Number.isFinite(chunkSize) || !Number.isFinite(upperBound))
1145
+ return null;
1146
+ const maxWatermark = maxWatermarkB64 ? Buffer.from(maxWatermarkB64, 'base64').toString('utf8') : undefined;
1147
+ return { lo, chunkSize, upperBound, maxWatermark };
1148
+ }
1149
+ buildRangeCursor(state) {
1150
+ const wmB64 = state.maxWatermark ? Buffer.from(state.maxWatermark, 'utf8').toString('base64') : '';
1151
+ return `${RANGE_CURSOR_PREFIX}${state.lo}:${state.chunkSize}:${state.upperBound}:${wmB64}`;
1152
+ }
1153
+ buildRangeFilter(ctx, lo, hi) {
1154
+ const hasWhenModified = this.ObjectHasWhenModified(ctx.CompanyIntegration.IntegrationID, ctx.ObjectName);
1155
+ const parts = [];
1156
+ if (ctx.WatermarkValue && hasWhenModified) {
1157
+ parts.push(`WHENMODIFIED >= '${this.NormalizeSageIntacctTimestamp(ctx.WatermarkValue)}'`);
1158
+ }
1159
+ parts.push(`RECORDNO >= ${lo}`);
1160
+ parts.push(`RECORDNO < ${hi}`);
1161
+ return parts.join(' AND ');
1162
+ }
1163
+ /** Converts a watermark value to SI's expected datetime literal format
1164
+ * (`MM/DD/YYYY HH:mm:ss`). The integration engine may pass watermarks
1165
+ * in ISO 8601 (e.g. `2026-04-26T01:00:25.170Z`) when no prior SI-format
1166
+ * watermark exists for an object — substituting that ISO string straight
1167
+ * into a `WHENMODIFIED >= '...'` filter triggers SI's generic
1168
+ * DL02000001 "There was an error processing the request" rejection.
1169
+ * This converts to SI's accepted format using UTC components for
1170
+ * deterministic results regardless of the host server's timezone. */
1171
+ NormalizeSageIntacctTimestamp(value) {
1172
+ // Already in SI's MM/DD/YYYY format — pass through unchanged.
1173
+ if (/^\d{1,2}\/\d{1,2}\/\d{4}/.test(value)) {
1174
+ return value;
1175
+ }
1176
+ const d = new Date(value);
1177
+ if (Number.isNaN(d.getTime())) {
1178
+ // Can't parse — let SI reject it loudly rather than silently
1179
+ // re-format something we don't understand.
1180
+ return value;
1181
+ }
1182
+ const pad = (n) => String(n).padStart(2, '0');
1183
+ return `${pad(d.getUTCMonth() + 1)}/${pad(d.getUTCDate())}/${d.getUTCFullYear()} ` +
1184
+ `${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
1185
+ }
1186
+ toExternalRecord(record, objectName, pkField) {
1187
+ return {
1188
+ ExternalID: String(record[pkField] ?? ''),
1189
+ ObjectType: objectName,
1190
+ Fields: record,
1191
+ ModifiedAt: record['WHENMODIFIED'] ? new Date(String(record['WHENMODIFIED'])) : undefined,
1192
+ };
1193
+ }
1194
+ /** Returns the lex-greater of two ISO/SI watermark strings, or undefined
1195
+ * if both are absent. SI's WHENMODIFIED format (`MM/DD/YYYY HH:mm:ss`)
1196
+ * doesn't sort lex-correctly across years/months in pure string compare,
1197
+ * so we parse to Date for comparison. */
1198
+ maxWatermarkString(a, b) {
1199
+ if (!a)
1200
+ return b;
1201
+ if (!b)
1202
+ return a;
1203
+ const da = new Date(a);
1204
+ const db = new Date(b);
1205
+ if (Number.isNaN(da.getTime()))
1206
+ return b;
1207
+ if (Number.isNaN(db.getTime()))
1208
+ return a;
1209
+ return da.getTime() >= db.getTime() ? a : b;
1210
+ }
1211
+ /**
1212
+ * Returns true when the IntegrationObject's discovered field set includes
1213
+ * WHENMODIFIED. Used to skip the watermark filter for objects that don't
1214
+ * support it (most SI platform/lookup objects).
1215
+ */
1216
+ ObjectHasWhenModified(integrationID, objectName) {
1217
+ try {
1218
+ const obj = IntegrationEngineBase.Instance.GetIntegrationObject(integrationID, objectName);
1219
+ if (!obj)
1220
+ return true; // unknown — assume yes, preserve existing behavior
1221
+ const fields = IntegrationEngineBase.Instance.GetIntegrationObjectFields(obj.ID);
1222
+ if (!fields || fields.length === 0)
1223
+ return true; // no metadata yet — preserve behavior
1224
+ return fields.some(f => f.Name.toUpperCase() === 'WHENMODIFIED');
1225
+ }
1226
+ catch {
1227
+ return true;
1228
+ }
381
1229
  }
382
1230
  // ─── CRUD Operations ─────────────────────────────────────────────
383
1231
  async CreateRecord(ctx) {
@@ -398,7 +1246,8 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
398
1246
  const companyIntegration = ctx.CompanyIntegration;
399
1247
  const contextUser = ctx.ContextUser;
400
1248
  const session = await this.GetSession(companyIntegration, contextUser);
401
- const xml = this.BuildUpdateRequest(session, ctx.ObjectName, ctx.ExternalID, ctx.Attributes);
1249
+ const pkField = this.GetPrimaryKeyField(ctx.ObjectName, companyIntegration.IntegrationID);
1250
+ const xml = this.BuildUpdateRequest(session, ctx.ObjectName, ctx.ExternalID, ctx.Attributes, pkField);
402
1251
  try {
403
1252
  await this.SendXMLRequest(session, xml);
404
1253
  return { Success: true, ExternalID: ctx.ExternalID, StatusCode: 200 };
@@ -424,7 +1273,7 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
424
1273
  const companyIntegration = ctx.CompanyIntegration;
425
1274
  const contextUser = ctx.ContextUser;
426
1275
  const session = await this.GetSession(companyIntegration, contextUser);
427
- const pkField = this.GetPrimaryKeyField(ctx.ObjectName);
1276
+ const pkField = this.GetPrimaryKeyField(ctx.ObjectName, companyIntegration.IntegrationID);
428
1277
  const xml = this.BuildReadRequest(session, ctx.ObjectName, ctx.ExternalID);
429
1278
  try {
430
1279
  const response = await this.SendXMLRequest(session, xml);
@@ -445,13 +1294,13 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
445
1294
  const companyIntegration = ctx.CompanyIntegration;
446
1295
  const contextUser = ctx.ContextUser;
447
1296
  const session = await this.GetSession(companyIntegration, contextUser);
448
- const pkField = this.GetPrimaryKeyField(ctx.ObjectName);
1297
+ const pkField = this.GetPrimaryKeyField(ctx.ObjectName, companyIntegration.IntegrationID);
449
1298
  const pageSize = ctx.PageSize ?? DEFAULT_PAGE_SIZE;
450
1299
  const filterParts = Object.entries(ctx.Filters).map(([field, value]) => `${field} = '${this.EscapeXmlValue(String(value))}'`);
451
1300
  const filter = filterParts.join(' AND ');
452
- const xml = this.BuildReadByQueryRequest(session, ctx.ObjectName, filter, pageSize);
1301
+ const xml = this.BuildReadByQueryRequest(session, ctx.ObjectName, filter, pageSize, pkField);
453
1302
  const response = await this.SendXMLRequest(session, xml);
454
- const result = this.ParseReadByQueryResponse(response, ctx.ObjectName, pkField);
1303
+ const result = this.ParseReadByQueryResponse(response, ctx.ObjectName, pkField, pageSize);
455
1304
  return {
456
1305
  Records: result.Records,
457
1306
  TotalCount: result.Records.length + (result.HasMore ? 1 : 0), // Intacct doesn't give exact total in readByQuery
@@ -462,7 +1311,7 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
462
1311
  const companyIntegration = ctx.CompanyIntegration;
463
1312
  const contextUser = ctx.ContextUser;
464
1313
  const session = await this.GetSession(companyIntegration, contextUser);
465
- const pkField = this.GetPrimaryKeyField(ctx.ObjectName);
1314
+ const pkField = this.GetPrimaryKeyField(ctx.ObjectName, companyIntegration.IntegrationID);
466
1315
  const pageSize = ctx.PageSize ?? DEFAULT_PAGE_SIZE;
467
1316
  if (ctx.Cursor) {
468
1317
  const result = await this.FetchMoreRecords(session, ctx.Cursor, ctx.ObjectName, pkField);
@@ -475,9 +1324,9 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
475
1324
  const filter = ctx.Filter
476
1325
  ? Object.entries(ctx.Filter).map(([k, v]) => `${k} = '${this.EscapeXmlValue(String(v))}'`).join(' AND ')
477
1326
  : '';
478
- const xml = this.BuildReadByQueryRequest(session, ctx.ObjectName, filter, pageSize);
1327
+ const xml = this.BuildReadByQueryRequest(session, ctx.ObjectName, filter, pageSize, pkField);
479
1328
  const response = await this.SendXMLRequest(session, xml);
480
- const result = this.ParseReadByQueryResponse(response, ctx.ObjectName, pkField);
1329
+ const result = this.ParseReadByQueryResponse(response, ctx.ObjectName, pkField, pageSize);
481
1330
  return {
482
1331
  Records: result.Records,
483
1332
  HasMore: result.HasMore,
@@ -595,7 +1444,16 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
595
1444
  </operation>
596
1445
  </request>`;
597
1446
  }
598
- BuildReadByQueryRequest(session, objectName, filter, pageSize) {
1447
+ BuildReadByQueryRequest(session, objectName, filter, pageSize, _orderByField) {
1448
+ // SI's readByQuery XSD does NOT accept an <orderby> child element —
1449
+ // every request including it errors with "Element 'orderby' is not
1450
+ // expected" and returns 0 records. Ordering, when needed, must be
1451
+ // expressed in the WHERE/<query> clause via SI's SOQL-like syntax
1452
+ // ("ORDER BY <field> ASC" appended to the query expression). The
1453
+ // caller can append "ORDER BY <field>" inside `filter` if it wants
1454
+ // deterministic ordering. The orderByField parameter is preserved
1455
+ // for API stability but no longer emits the rejected XML element.
1456
+ void _orderByField;
599
1457
  return this.WrapInSessionRequest(session, `
600
1458
  <function controlid="${CONTROL_ID_PREFIX}-readByQuery-${Date.now()}">
601
1459
  <readByQuery>
@@ -635,9 +1493,9 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
635
1493
  </create>
636
1494
  </function>`);
637
1495
  }
638
- BuildUpdateRequest(session, objectName, key, attributes) {
639
- const pkField = this.GetPrimaryKeyField(objectName);
640
- const fieldsXml = this.AttributesToXml({ [pkField]: key, ...attributes });
1496
+ BuildUpdateRequest(session, objectName, key, attributes, pkField) {
1497
+ const resolvedPk = pkField ?? this.GetPrimaryKeyField(objectName);
1498
+ const fieldsXml = this.AttributesToXml({ [resolvedPk]: key, ...attributes });
641
1499
  return this.WrapInSessionRequest(session, `
642
1500
  <function controlid="${CONTROL_ID_PREFIX}-update-${Date.now()}">
643
1501
  <update>
@@ -703,11 +1561,25 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
703
1561
  Config: config,
704
1562
  };
705
1563
  }
706
- ParseReadByQueryResponse(xml, objectName, pkField) {
1564
+ /** Lower-level parser: returns just the raw records + SI's signals
1565
+ * without applying any pagination logic. Used by the range-chunked and
1566
+ * single-pull paths which decide pagination for themselves. */
1567
+ ParseReadByQueryResponseRaw(xml, objectName, pageSize) {
1568
+ this.CheckForErrors(xml);
1569
+ const numRemaining = parseInt(this.ExtractXmlValue(xml, 'numremaining') || '0', 10);
1570
+ const resultId = this.ExtractXmlValue(xml, 'resultId') || undefined;
1571
+ const records = this.ExtractRecords(xml, objectName);
1572
+ console.log(`[SageIntacct] ${objectName}: SI response — records.length=${records.length}, ` +
1573
+ `numremaining=${numRemaining}, resultId=${resultId ? 'present' : 'absent'}, pageSize=${pageSize}`);
1574
+ return { records, numRemaining, resultId };
1575
+ }
1576
+ ParseReadByQueryResponse(xml, objectName, pkField, pageSize = DEFAULT_PAGE_SIZE) {
707
1577
  this.CheckForErrors(xml);
708
1578
  const numRemaining = parseInt(this.ExtractXmlValue(xml, 'numremaining') || '0', 10);
709
1579
  const resultId = this.ExtractXmlValue(xml, 'resultId') || undefined;
710
1580
  const records = this.ExtractRecords(xml, objectName);
1581
+ console.log(`[SageIntacct] ${objectName}: SI response — records.length=${records.length}, ` +
1582
+ `numremaining=${numRemaining}, resultId=${resultId ? 'present' : 'absent'}, pageSize=${pageSize}`);
711
1583
  const externalRecords = records.map(r => ({
712
1584
  ExternalID: String(r[pkField] ?? ''),
713
1585
  ObjectType: objectName,
@@ -716,11 +1588,54 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
716
1588
  }));
717
1589
  // Compute watermark from the latest WHENMODIFIED in this batch
718
1590
  const newWatermark = this.ComputeWatermark(records);
1591
+ // Legacy parser used by SearchRecords/ListRecords/FetchMoreRecords
1592
+ // (browse/search APIs that use SI's resultId pagination directly).
1593
+ // The sync engine path uses the bulletproof range-chunked walk in
1594
+ // FetchChangesByRange — it never reaches this branch.
1595
+ //
1596
+ // Safe pagination signals:
1597
+ // - SI returned a resultId → readMore against that resultId
1598
+ // - SI returned numremaining=0 AND records.length < pageSize → done
1599
+ //
1600
+ // Unsafe case: full page with no resultId. The previous PKCURSOR
1601
+ // fallback (PK > maxPK on next call) silently dropped records when
1602
+ // SI's scan order wasn't PK-ascending, which is the actual default.
1603
+ // Better to fail loudly than corrupt the dataset.
1604
+ const pageWasFull = records.length >= pageSize;
1605
+ const apiSaysMore = numRemaining > 0;
1606
+ let hasMore = false;
1607
+ let nextCursor;
1608
+ if (resultId && (apiSaysMore || pageWasFull)) {
1609
+ hasMore = true;
1610
+ nextCursor = resultId;
1611
+ }
1612
+ else if (pageWasFull) {
1613
+ throw new Error(`[SageIntacct] ${objectName}: SI returned a full page (${records.length} records) ` +
1614
+ `with no resultId. Cannot safely paginate — SI's readByQuery does not scan in ` +
1615
+ `PK-ascending order, so PK-cursor advancement would silently drop records. ` +
1616
+ `Use FetchChanges (range-chunked sync) for complete coverage, or raise ` +
1617
+ `pageSize so this query fits in a single page.`);
1618
+ }
1619
+ // Suppress unused-warning when the legacy MaxPkValue helper is not
1620
+ // referenced from this branch. Other callers (probe scripts) may
1621
+ // still invoke it via reflection.
1622
+ void pkField;
1623
+ // Critical: only advance the watermark on the FINAL batch of an
1624
+ // entity's fetch. Intermediate batches must keep the original
1625
+ // watermark filter so PK-cursor pagination doesn't shrink the
1626
+ // WHENMODIFIED window and silently drop records modified between
1627
+ // the original watermark and the batch's max. The engine sets
1628
+ // currentWatermark from NewWatermarkValue immediately and that
1629
+ // becomes the next batch's WHERE filter — fine for the FINAL
1630
+ // batch (entity fully drained), broken for any mid-pagination
1631
+ // batch (next call still needs the original watermark to find
1632
+ // the rest of the records).
1633
+ const newWatermarkForReturn = hasMore ? undefined : newWatermark;
719
1634
  return {
720
1635
  Records: externalRecords,
721
- HasMore: numRemaining > 0,
722
- NewWatermarkValue: newWatermark,
723
- NextCursor: numRemaining > 0 ? resultId : undefined,
1636
+ HasMore: hasMore,
1637
+ NewWatermarkValue: newWatermarkForReturn,
1638
+ NextCursor: nextCursor,
724
1639
  };
725
1640
  }
726
1641
  ParseSingleRecord(xml, objectName) {
@@ -728,21 +1643,103 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
728
1643
  const records = this.ExtractRecords(xml, objectName);
729
1644
  return records.length > 0 ? records[0] : null;
730
1645
  }
1646
+ /**
1647
+ * Returns the maximum PK value across the records, picking the right
1648
+ * comparator (numeric vs lexicographic) based on whether all values
1649
+ * look like integers. Returns undefined when records is empty or no
1650
+ * record has a usable PK value.
1651
+ */
1652
+ MaxPkValue(records, pkField) {
1653
+ const values = [];
1654
+ for (const r of records) {
1655
+ const v = r[pkField];
1656
+ if (v == null)
1657
+ continue;
1658
+ if (typeof v === 'number' && Number.isFinite(v))
1659
+ values.push(v);
1660
+ else if (typeof v === 'string' && v.length > 0)
1661
+ values.push(v);
1662
+ }
1663
+ if (values.length === 0)
1664
+ return undefined;
1665
+ const allNumericStrings = values.every(v => typeof v === 'number' || (typeof v === 'string' && /^-?\d+$/.test(v)));
1666
+ if (allNumericStrings) {
1667
+ return values.reduce((m, v) => {
1668
+ const n = typeof v === 'number' ? v : parseInt(v, 10);
1669
+ return n > m ? n : m;
1670
+ }, Number.NEGATIVE_INFINITY);
1671
+ }
1672
+ // String comparison — return lexicographically largest
1673
+ return values.reduce((m, v) => {
1674
+ const s = String(v);
1675
+ return s > m ? s : m;
1676
+ }, '');
1677
+ }
731
1678
  ParseInspectObjectsResponse(xml) {
732
1679
  this.CheckForErrors(xml);
733
1680
  const objectNames = [];
734
- // inspect with '*' returns <type>OBJECTNAME</type> elements
1681
+ const skipped = [];
1682
+ // inspect with '*' returns <type>OBJECTNAME</type> elements containing
1683
+ // a mix of (a) real API codes that work directly (CUSTOMER, APBILL),
1684
+ // (b) human display names where the API code is a "strip spaces +
1685
+ // uppercase" transform of the display name ("AP bill" → APBILL,
1686
+ // "Order entry transaction" → ORDERENTRYTRANSACTION), and (c) proper-
1687
+ // case display-only labels that have no API equivalent ("Channel",
1688
+ // "User", "Entity"). We can't tell (b) from (c) up-front, so this
1689
+ // method returns BOTH: real-looking API codes pass straight through,
1690
+ // and display-name-shaped entries are normalized into candidate API
1691
+ // codes. The caller then probes each candidate against per-object
1692
+ // inspect to keep only the ones SI actually accepts.
735
1693
  const typeRegex = /<type[^>]*>([^<]+)<\/type>/gi;
1694
+ const apiNameRegex = /^[A-Z][A-Z0-9_]*$/;
1695
+ const seen = new Set();
736
1696
  let match;
737
1697
  while ((match = typeRegex.exec(xml)) !== null) {
738
- objectNames.push(match[1].trim());
1698
+ const raw = match[1].trim();
1699
+ if (!raw)
1700
+ continue;
1701
+ // Already an API-code-shaped name → keep verbatim
1702
+ if (apiNameRegex.test(raw)) {
1703
+ if (!seen.has(raw)) {
1704
+ seen.add(raw);
1705
+ objectNames.push(raw);
1706
+ }
1707
+ continue;
1708
+ }
1709
+ // Display-name-shaped → derive a candidate API code by stripping
1710
+ // whitespace and uppercasing. SI accepts these for many objects
1711
+ // where the inspect-* list shows the display label.
1712
+ const candidate = raw.replace(/[\s\-]+/g, '').toUpperCase();
1713
+ if (!candidate || !apiNameRegex.test(candidate)) {
1714
+ skipped.push(raw);
1715
+ continue;
1716
+ }
1717
+ if (!seen.has(candidate)) {
1718
+ seen.add(candidate);
1719
+ objectNames.push(candidate);
1720
+ }
1721
+ }
1722
+ if (skipped.length > 0) {
1723
+ console.log(`[SageIntacct] ParseInspectObjectsResponse: filtered ${skipped.length} non-API-code names ` +
1724
+ `(e.g. "${skipped.slice(0, 3).join('", "')}"${skipped.length > 3 ? ', ...' : ''}) — ` +
1725
+ `these entries cannot be inspected via the per-object endpoint.`);
739
1726
  }
740
1727
  return objectNames;
741
1728
  }
742
1729
  ParseInspectFieldsResponse(xml) {
743
1730
  this.CheckForErrors(xml);
744
1731
  const fields = [];
1732
+ // SI's inspect can emit the same <Name> in multiple <Field> blocks
1733
+ // (nested aliases, legacy + current paths for the same column, etc.).
1734
+ // The unique key on IntegrationObjectField is (IntegrationObjectID,
1735
+ // Name) so duplicates blow up at insert time. Dedupe here, first
1736
+ // occurrence wins.
1737
+ const seenNames = new Set();
745
1738
  // inspect returns <Field> elements with <Name>, <DataType>, etc.
1739
+ // SI also exposes (when present): <MaxLength>, <Precision>, <Scale>,
1740
+ // <DefaultValue>, <References> (FK target object name). Surface them
1741
+ // so downstream DDL generation has real constraints to work with
1742
+ // instead of guessing precision/length and producing brittle types.
746
1743
  const fieldRegex = /<Field>\s*([\s\S]*?)\s*<\/Field>/gi;
747
1744
  let match;
748
1745
  while ((match = fieldRegex.exec(xml)) !== null) {
@@ -755,12 +1752,50 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
755
1752
  const isRequired = this.ExtractXmlValueFromFragment(fieldXml, 'Required') === 'true';
756
1753
  const isReadOnly = this.ExtractXmlValueFromFragment(fieldXml, 'ReadOnly') === 'true'
757
1754
  || this.ExtractXmlValueFromFragment(fieldXml, 'SystemGenerated') === 'true';
1755
+ const isCustom = this.ExtractXmlValueFromFragment(fieldXml, 'IsCustom').toLowerCase() === 'true'
1756
+ || this.ExtractXmlValueFromFragment(fieldXml, 'ISCUSTOM').toLowerCase() === 'true';
1757
+ const maxLength = this.ParseNumericTagFromFragment(fieldXml, 'MaxLength')
1758
+ ?? this.ParseNumericTagFromFragment(fieldXml, 'Length');
1759
+ const precision = this.ParseNumericTagFromFragment(fieldXml, 'Precision');
1760
+ const scale = this.ParseNumericTagFromFragment(fieldXml, 'Scale');
1761
+ const defaultValueRaw = this.ExtractXmlValueFromFragment(fieldXml, 'DefaultValue');
1762
+ const defaultValue = defaultValueRaw && defaultValueRaw.length > 0 ? defaultValueRaw : null;
1763
+ const referencesRaw = this.ExtractXmlValueFromFragment(fieldXml, 'References')
1764
+ || this.ExtractXmlValueFromFragment(fieldXml, 'ReferencesObject');
1765
+ const references = referencesRaw && referencesRaw.length > 0 ? referencesRaw : null;
758
1766
  if (name) {
759
- fields.push({ Name: name, Label: label, DataType: dataType, IsRequired: isRequired, IsReadOnly: isReadOnly });
1767
+ const key = name.toLowerCase();
1768
+ if (seenNames.has(key))
1769
+ continue;
1770
+ seenNames.add(key);
1771
+ fields.push({
1772
+ Name: name, Label: label, DataType: dataType,
1773
+ IsRequired: isRequired, IsReadOnly: isReadOnly,
1774
+ IsCustom: isCustom,
1775
+ MaxLength: maxLength,
1776
+ Precision: precision,
1777
+ Scale: scale,
1778
+ DefaultValue: defaultValue,
1779
+ References: references,
1780
+ });
760
1781
  }
761
1782
  }
762
1783
  return fields;
763
1784
  }
1785
+ /**
1786
+ * Pulls a tag value out of a fragment and parses it as a number. Returns
1787
+ * null when the tag is absent, empty, or doesn't parse. Used for fields
1788
+ * like MaxLength / Precision / Scale that SI reports inconsistently
1789
+ * across object families (some return numbers, some omit, some return
1790
+ * empty strings).
1791
+ */
1792
+ ParseNumericTagFromFragment(fragment, tagName) {
1793
+ const raw = this.ExtractXmlValueFromFragment(fragment, tagName);
1794
+ if (!raw || raw.length === 0)
1795
+ return null;
1796
+ const n = Number(raw);
1797
+ return Number.isFinite(n) ? n : null;
1798
+ }
764
1799
  // ─── HTTP Transport ──────────────────────────────────────────────
765
1800
  /**
766
1801
  * Sends an XML request to the Sage Intacct API using a session.
@@ -940,11 +1975,46 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
940
1975
  .replace(/'/g, '&apos;');
941
1976
  }
942
1977
  /**
943
- * Gets the primary key field name for a Sage Intacct object.
1978
+ * Gets the primary key field name for a Sage Intacct object, preferring
1979
+ * the IntegrationObject metadata (DefaultQueryParams.pk_field or the
1980
+ * IntegrationObjectField marked IsPrimaryKey) over the built-in fallback
1981
+ * map. Returns 'RECORDNO' when nothing else is available.
1982
+ *
1983
+ * This is called frequently — the lookup is cheap (pure engine cache) and
1984
+ * avoids hardcoded switch statements on object names.
944
1985
  */
945
- GetPrimaryKeyField(objectName) {
1986
+ GetPrimaryKeyField(objectName, integrationID) {
1987
+ if (integrationID) {
1988
+ const fromMetadata = this.resolvePkFromMetadata(integrationID, objectName);
1989
+ if (fromMetadata)
1990
+ return fromMetadata;
1991
+ }
946
1992
  return OBJECT_PK_MAP[objectName.toUpperCase()] || 'RECORDNO';
947
1993
  }
1994
+ /**
1995
+ * Resolves the PK for an object from the engine cache.
1996
+ * Checks (in order): DefaultQueryParams.pk_field, then the field record
1997
+ * with IsPrimaryKey=true. Returns null when nothing is found.
1998
+ */
1999
+ resolvePkFromMetadata(integrationID, objectName) {
2000
+ try {
2001
+ const obj = IntegrationEngineBase.Instance.GetIntegrationObject(integrationID, objectName);
2002
+ if (!obj)
2003
+ return null;
2004
+ if (obj.DefaultQueryParams) {
2005
+ const parsed = JSON.parse(obj.DefaultQueryParams);
2006
+ if (parsed.pk_field && typeof parsed.pk_field === 'string') {
2007
+ return parsed.pk_field;
2008
+ }
2009
+ }
2010
+ const fields = IntegrationEngineBase.Instance.GetIntegrationObjectFields(obj.ID);
2011
+ const pkField = fields.find(f => f.IsPrimaryKey && f.Status === 'Active');
2012
+ return pkField ? pkField.Name : null;
2013
+ }
2014
+ catch {
2015
+ return null;
2016
+ }
2017
+ }
948
2018
  /**
949
2019
  * Formats an API object name into a human-readable label.
950
2020
  */
@@ -976,7 +2046,10 @@ let SageIntacctConnector = class SageIntacctConnector extends BaseIntegrationCon
976
2046
  async FetchMoreRecords(session, resultId, objectName, pkField) {
977
2047
  const xml = this.BuildReadMoreRequest(session, resultId);
978
2048
  const response = await this.SendXMLRequest(session, xml);
979
- return this.ParseReadByQueryResponse(response, objectName, pkField);
2049
+ // readMore inherits the original page size; the engine batch-size
2050
+ // doesn't change mid-stream. DEFAULT_PAGE_SIZE is the server-side
2051
+ // default and matches what BuildReadByQueryRequest uses.
2052
+ return this.ParseReadByQueryResponse(response, objectName, pkField, DEFAULT_PAGE_SIZE);
980
2053
  }
981
2054
  /**
982
2055
  * Throttles API requests to respect rate limits.