@elisra-devops/docgen-data-provider 1.138.0 → 1.140.0

package/src/modules/TicketsDataProvider.ts

@@ -6,6 +6,8 @@ import { QueryType } from '../models/tfs-data';
 import { QueryAllTypes } from '../models/tfs-data';
 import { Column } from '../models/tfs-data';
 import { value } from '../models/tfs-data';
+import { TestSteps } from '../models/tfs-data';
+import TestStepParserHelper from '../utils/testStepParserHelper';
 
 import logger from '../utils/logger';
 const pLimit = require('p-limit');
@@ -96,9 +98,11 @@ export default class TicketsDataProvider {
   token: string = '';
   queriesList: Array<any> = new Array<any>();
   private limit = pLimit(10);
-  // Set once a workitemsbatch call reports one of HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS
-  // as unknown for this project, so only the first chunk pays the retry cost.
-  private historicalFieldsOverride: string[] | null = null;
+  // Resolved once per project via resolveHistoricalTestPhaseField, so a
+  // compare's baseline and compareTo snapshots (and repeated queries against
+  // the same provider instance) only pay the discovery call once. Value is
+  // the real reference name, or null if the project defines no such field.
+  private historicalTestPhaseFieldByProject = new Map<string, Promise<string | null>>();
 
   constructor(orgUrl: string, token: string) {
     this.orgUrl = orgUrl;
@@ -1786,6 +1790,58 @@ export default class TicketsDataProvider {
     return String(value).trim();
   }
 
+  /**
+   * Stores rich-text fields (Description, Steps) with only whitespace trimmed, preserving markup
+   * so the report renderer can clean/render it faithfully. Equality between two snapshots must go
+   * through canonicalizeHistoricalRichTextForCompare instead of comparing this value directly.
+   */
+  private rawHistoricalRichText(value: unknown): string {
+    if (value == null) {
+      return '';
+    }
+    if (typeof value === 'string') {
+      return value.trim();
+    }
+    return this.normalizeHistoricalCompareValue(value);
+  }
+
+  /**
+   * Canonicalizes a rich-text field for equality comparison only. Intentionally identical to the
+   * whitespace collapse previously applied at storage time, so Changed/No-changes verdicts and
+   * summary counts are unaffected by storing the raw value for rendering.
+   */
+  private canonicalizeHistoricalRichTextForCompare(value: unknown): string {
+    return this.normalizeHistoricalCompareValue(value);
+  }
+
+  /**
+   * Parses baseline/compareTo Steps XML into per-step Action/Expected arrays for a changed test
+   * case, so the report can render readable steps instead of raw Steps XML. Best-effort: a parse
+   * failure (malformed XML, shared-step lookup error) must not fail the whole comparison, so it
+   * logs and falls back to leaving the caller with only the raw XML strings.
+   */
+  private async parseHistoricalStepsForDiff(
+    testStepParserHelper: TestStepParserHelper,
+    baselineStepsXml: string,
+    compareToStepsXml: string,
+  ): Promise<{ baselineSteps: TestSteps[]; compareToSteps: TestSteps[] } | null> {
+    try {
+      const emptyRevisionMap = new Map<number, number>();
+      const [baselineSteps, compareToSteps] = await Promise.all([
+        baselineStepsXml
+          ? testStepParserHelper.parseTestSteps(baselineStepsXml, emptyRevisionMap)
+          : Promise.resolve([]),
+        compareToStepsXml
+          ? testStepParserHelper.parseTestSteps(compareToStepsXml, emptyRevisionMap)
+          : Promise.resolve([]),
+      ]);
+      return { baselineSteps, compareToSteps };
+    } catch (err: any) {
+      logger.warn(`Failed to parse historical Steps XML for diff rendering: ${err?.message || err}`);
+      return null;
+    }
+  }
+
   private normalizeTestPhaseValue(value: unknown): string {
     const rendered = this.normalizeHistoricalCompareValue(value);
     if (!rendered) {
@@ -1988,25 +2044,61 @@ export default class TicketsDataProvider {
     return /TF51535|Cannot find field/i.test(message);
   }
 
-  // ADO's TF51535 names the specific field it couldn't find, e.g.
-  // "TF51535: Cannot find field 'Elisra.TestPhase'." — different networks
-  // define this optional field under different reference names (Elisra.* vs
-  // Custom.*), so drop only the one ADO actually rejected rather than every
-  // optional field, or a network that defines Custom.TestPhase but not
-  // Elisra.TestPhase would lose Test Phase comparison entirely for no reason.
-  private extractHistoricalUnknownFieldName(error: any): string | null {
-    const message = this.historicalErrorMessage(error);
-    const quoted = message.match(/Cannot find field\s*'([^']+)'/i);
-    if (quoted?.[1]) return quoted[1];
-    return HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.find((field) => message.includes(field)) || null;
+  // Different networks define the "Test Phase" custom field under different
+  // reference names (Elisra.TestPhase vs Custom.TestPhase, or something else
+  // entirely on a third network) — hardcoding candidate names and guessing is
+  // inherently fragile. Resolve the project's real field list once (Fields -
+  // List: GET {org}/{project}/_apis/wit/fields, supported since old TFS API
+  // versions) and use whichever reference name actually matches, by
+  // reference name or by friendly name. Resolves to null if the project
+  // defines no such field at all.
+  //
+  // Caches the in-flight promise, not just the resolved value: a compare's
+  // baseline and compareTo snapshots run concurrently against the same
+  // provider instance, and both would otherwise read the cache before either
+  // had a value to store, issuing the discovery request twice.
+  private resolveHistoricalTestPhaseField(project: string, apiVersion: string | null): Promise<string | null> {
+    const cached = this.historicalTestPhaseFieldByProject.get(project);
+    if (cached) return cached;
+    const promise = this.discoverHistoricalTestPhaseField(project, apiVersion);
+    this.historicalTestPhaseFieldByProject.set(project, promise);
+    return promise;
+  }
+
+  private async discoverHistoricalTestPhaseField(
+    project: string,
+    apiVersion: string | null,
+  ): Promise<string | null> {
+    const fieldsUrl = this.appendApiVersion(`${this.orgUrl}${project}/_apis/wit/fields`, apiVersion);
+    try {
+      const response = await TFSServices.getItemContent(fieldsUrl, this.token);
+      const fields = Array.isArray(response?.value) ? response.value : [];
+      const match =
+        fields.find((field: any) =>
+          HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.some(
+            (candidate) => String(field?.referenceName || '').toLowerCase() === candidate.toLowerCase(),
+          ),
+        ) || fields.find((field: any) => /test\s*phase/i.test(String(field?.name || '')));
+      const resolved = match?.referenceName ? String(match.referenceName) : null;
+      if (!resolved) {
+        logger.warn(`[historical-workitems] no Test Phase field defined for project '${project}'`);
+      }
+      return resolved;
+    } catch (error: any) {
+      logger.warn(
+        `[historical-workitems] could not list fields for project '${project}' to resolve the ` +
+          `Test Phase field name, omitting it: ${this.historicalErrorMessage(error)}`,
+      );
+      return null;
+    }
   }
 
-  private fetchHistoricalFieldsBatchAttempt(
+  private fetchHistoricalFieldsBatch(
     workItemsBatchUrl: string,
     idChunk: number[],
     asOf: string,
+    project: string,
     fieldsForRequest: string[],
-    attemptsRemaining: number,
   ): Promise<any> {
     return this.limit(() =>
       TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
@@ -2016,57 +2108,31 @@ export default class TicketsDataProvider {
         errorPolicy: 'Omit',
       }),
     ).catch((error: any) => {
-      if (attemptsRemaining <= 0 || !this.isHistoricalUnknownFieldError(error)) {
-        throw error;
-      }
-      const unknownField = this.extractHistoricalUnknownFieldName(error);
-      // Only strip fields this call actually recognizes as droppable
-      // (named explicitly, or one of the known optional fields as a
-      // last-resort fallback if the message doesn't parse) — never strip a
-      // required field, so an unrelated schema issue still surfaces as an
-      // error instead of silently losing data.
-      const reducedFields = unknownField
-        ? fieldsForRequest.filter((field) => field !== unknownField)
-        : fieldsForRequest.filter((field) => !HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.includes(field));
-      if (reducedFields.length === fieldsForRequest.length) {
+      // Defensive net: the resolved field name came from the project's own
+      // field list, so this shouldn't fire, but if it somehow still does,
+      // fall back to the required fields and poison the cache so later
+      // chunks/queries for this project stop requesting it too.
+      const hasOptionalField = HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.some((f) => fieldsForRequest.includes(f))
+        || fieldsForRequest.some((f) => !HISTORICAL_REQUIRED_WORK_ITEM_FIELDS.includes(f));
+      if (!hasOptionalField || !this.isHistoricalUnknownFieldError(error)) {
         throw error;
       }
-      // Concurrent chunks can each discover a different bad field before any
-      // of them updates the shared override — intersect rather than
-      // overwrite, so one chunk's retry never re-requests a field another
-      // chunk already proved invalid for this project.
-      this.historicalFieldsOverride = this.historicalFieldsOverride
-        ? this.historicalFieldsOverride.filter((field) => reducedFields.includes(field))
-        : reducedFields;
       logger.warn(
-        `[historical-workitems] field${unknownField ? ` '${unknownField}'` : '(s)'} not defined ` +
-          `for this project; retrying workitemsbatch without it: ${this.historicalErrorMessage(error)}`,
+        `[historical-workitems] resolved Test Phase field was rejected by workitemsbatch for ` +
+          `project '${project}'; retrying without it: ${this.historicalErrorMessage(error)}`,
       );
-      return this.fetchHistoricalFieldsBatchAttempt(
-        workItemsBatchUrl,
-        idChunk,
-        asOf,
-        reducedFields,
-        attemptsRemaining - 1,
+      this.historicalTestPhaseFieldByProject.set(project, Promise.resolve(null));
+      return this.limit(() =>
+        TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
+          ids: idChunk,
+          asOf,
+          fields: HISTORICAL_REQUIRED_WORK_ITEM_FIELDS,
+          errorPolicy: 'Omit',
+        }),
       );
     });
   }
 
-  private fetchHistoricalFieldsBatch(
-    workItemsBatchUrl: string,
-    idChunk: number[],
-    asOf: string,
-  ): Promise<any> {
-    const fieldsForRequest = this.historicalFieldsOverride || HISTORICAL_WORK_ITEM_FIELDS;
-    return this.fetchHistoricalFieldsBatchAttempt(
-      workItemsBatchUrl,
-      idChunk,
-      asOf,
-      fieldsForRequest,
-      HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.length,
-    );
-  }
-
   private async fetchHistoricalWorkItemsBatch(
     project: string,
     ids: number[],
@@ -2077,6 +2143,12 @@ export default class TicketsDataProvider {
       `${this.orgUrl}${project}/_apis/wit/workitemsbatch`,
       apiVersion,
     );
+    // Resolved once per project, before any chunk starts, so every chunk
+    // shares the same field list — no per-request guessing or retry race.
+    const testPhaseField = await this.resolveHistoricalTestPhaseField(project, apiVersion);
+    const fieldsForRequest = testPhaseField
+      ? [...HISTORICAL_REQUIRED_WORK_ITEM_FIELDS, testPhaseField]
+      : HISTORICAL_REQUIRED_WORK_ITEM_FIELDS;
     const idChunks = this.chunkHistoricalWorkItemIds(ids);
     try {
       const chunkResults = await Promise.all(
@@ -2093,7 +2165,7 @@ export default class TicketsDataProvider {
           // two units against the shared concurrency cap, not one — otherwise
           // the real ceiling on simultaneous ADO calls would silently double.
           Promise.all([
-            this.fetchHistoricalFieldsBatch(workItemsBatchUrl, idChunk, asOf),
+            this.fetchHistoricalFieldsBatch(workItemsBatchUrl, idChunk, asOf, project, fieldsForRequest),
             this.limit(() =>
               TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
                 ids: idChunk,
@@ -2266,8 +2338,8 @@ export default class TicketsDataProvider {
       iterationPath: this.normalizeHistoricalCompareValue(fields['System.IterationPath']),
       versionId: this.toHistoricalRevision(workItem?.rev ?? fields['System.Rev']),
       versionTimestamp: this.normalizeHistoricalCompareValue(fields['System.ChangedDate']),
-      description: this.normalizeHistoricalCompareValue(fields['System.Description']),
-      steps: this.normalizeHistoricalCompareValue(fields['Microsoft.VSTS.TCM.Steps']),
+      description: this.rawHistoricalRichText(fields['System.Description']),
+      steps: this.rawHistoricalRichText(fields['Microsoft.VSTS.TCM.Steps']),
       testPhase: this.normalizeTestPhaseValue(testPhaseRaw),
       relatedLinkCount,
       workItemUrl: `${this.orgUrl}${project}/_workitems/edit/${id}`,
@@ -2448,8 +2520,9 @@ export default class TicketsDataProvider {
       ...Array.from(compareTo.snapshotMap.keys()),
     ]);
     const sortedIds = Array.from(allIds.values()).sort((a, b) => a - b);
+    const testStepParserHelper = new TestStepParserHelper(this.orgUrl, this.token);
 
-    const rows = sortedIds.map((id) => {
+    const rows = await Promise.all(sortedIds.map(async (id) => {
       const baselineRow = baseline.snapshotMap.get(id) || null;
       const compareToRow = compareTo.snapshotMap.get(id) || null;
       const workItemType = compareToRow?.workItemType || baselineRow?.workItemType || '';
@@ -2487,7 +2560,15 @@ export default class TicketsDataProvider {
       const safeCompareTo = compareToRow as HistoricalWorkItemSnapshot;
       const changedFields: string[] = [];
 
-      if (safeBaseline.description !== safeCompareTo.description) {
+      const descriptionChanged =
+        this.canonicalizeHistoricalRichTextForCompare(safeBaseline.description) !==
+        this.canonicalizeHistoricalRichTextForCompare(safeCompareTo.description);
+      const stepsChanged =
+        isTestCase &&
+        this.canonicalizeHistoricalRichTextForCompare(safeBaseline.steps) !==
+          this.canonicalizeHistoricalRichTextForCompare(safeCompareTo.steps);
+
+      if (descriptionChanged) {
         changedFields.push('Description');
       }
       if (safeBaseline.title !== safeCompareTo.title) {
@@ -2496,7 +2577,7 @@ export default class TicketsDataProvider {
       if (safeBaseline.state !== safeCompareTo.state) {
         changedFields.push('State');
       }
-      if (isTestCase && safeBaseline.steps !== safeCompareTo.steps) {
+      if (stepsChanged) {
         changedFields.push('Steps');
       }
       if (safeBaseline.testPhase !== safeCompareTo.testPhase) {
@@ -2506,28 +2587,41 @@ export default class TicketsDataProvider {
         changedFields.push('Related Link Count');
       }
 
-      const differences = changedFields.map((field) => {
-        switch (field) {
-          case 'Description':
-            return { field, baseline: safeBaseline.description, compareTo: safeCompareTo.description };
-          case 'Title':
-            return { field, baseline: safeBaseline.title, compareTo: safeCompareTo.title };
-          case 'State':
-            return { field, baseline: safeBaseline.state, compareTo: safeCompareTo.state };
-          case 'Steps':
-            return { field, baseline: safeBaseline.steps, compareTo: safeCompareTo.steps };
-          case 'Test Phase':
-            return { field, baseline: safeBaseline.testPhase, compareTo: safeCompareTo.testPhase };
-          case 'Related Link Count':
-            return {
-              field,
-              baseline: String(safeBaseline.relatedLinkCount),
-              compareTo: String(safeCompareTo.relatedLinkCount),
-            };
-          default:
-            return { field, baseline: '', compareTo: '' };
-        }
-      });
+      const differences = await Promise.all(
+        changedFields.map(async (field) => {
+          switch (field) {
+            case 'Description':
+              return { field, baseline: safeBaseline.description, compareTo: safeCompareTo.description };
+            case 'Title':
+              return { field, baseline: safeBaseline.title, compareTo: safeCompareTo.title };
+            case 'State':
+              return { field, baseline: safeBaseline.state, compareTo: safeCompareTo.state };
+            case 'Steps': {
+              const parsed = await this.parseHistoricalStepsForDiff(
+                testStepParserHelper,
+                safeBaseline.steps,
+                safeCompareTo.steps,
+              );
+              return {
+                field,
+                baseline: safeBaseline.steps,
+                compareTo: safeCompareTo.steps,
+                ...(parsed || {}),
+              };
+            }
+            case 'Test Phase':
+              return { field, baseline: safeBaseline.testPhase, compareTo: safeCompareTo.testPhase };
+            case 'Related Link Count':
+              return {
+                field,
+                baseline: String(safeBaseline.relatedLinkCount),
+                compareTo: String(safeCompareTo.relatedLinkCount),
+              };
+            default:
+              return { field, baseline: '', compareTo: '' };
+          }
+        }),
+      );
 
       return {
         id,
@@ -2540,7 +2634,7 @@ export default class TicketsDataProvider {
         differences,
         workItemUrl: safeCompareTo.workItemUrl || safeBaseline.workItemUrl,
       };
-    });
+    }));
 
     const summary = rows.reduce(
       (acc, row) => {