@elisra-devops/docgen-data-provider 1.137.1 → 1.139.0

package/bin/modules/TicketsDataProvider.d.ts

@@ -10,6 +10,7 @@ export default class TicketsDataProvider {
     token: string;
     queriesList: Array<any>;
     private limit;
+    private historicalFieldsOverride;
     constructor(orgUrl: string, token: string);
     /**
      * Extracts the Azure DevOps project name from a WIQL URL.
@@ -201,6 +202,25 @@ export default class TicketsDataProvider {
     }>;
     private normalizeHistoricalAsOf;
     private normalizeHistoricalCompareValue;
+    /**
+     * Stores rich-text fields (Description, Steps) with only whitespace trimmed, preserving markup
+     * so the report renderer can clean/render it faithfully. Equality between two snapshots must go
+     * through canonicalizeHistoricalRichTextForCompare instead of comparing this value directly.
+     */
+    private rawHistoricalRichText;
+    /**
+     * Canonicalizes a rich-text field for equality comparison only. Intentionally identical to the
+     * whitespace collapse previously applied at storage time, so Changed/No-changes verdicts and
+     * summary counts are unaffected by storing the raw value for rendering.
+     */
+    private canonicalizeHistoricalRichTextForCompare;
+    /**
+     * Parses baseline/compareTo Steps XML into per-step Action/Expected arrays for a changed test
+     * case, so the report can render readable steps instead of raw Steps XML. Best-effort: a parse
+     * failure (malformed XML, shared-step lookup error) must not fail the whole comparison, so it
+     * logs and falls back to leaving the caller with only the raw XML strings.
+     */
+    private parseHistoricalStepsForDiff;
     private normalizeTestPhaseValue;
     private isTestCaseType;
     private toHistoricalRevision;
@@ -214,6 +234,10 @@ export default class TicketsDataProvider {
     private normalizeHistoricalQueryRoot;
     private historicalErrorMessage;
     private isHistoricalMissingWorkItemError;
+    private isHistoricalUnknownFieldError;
+    private extractHistoricalUnknownFieldName;
+    private fetchHistoricalFieldsBatchAttempt;
+    private fetchHistoricalFieldsBatch;
     private fetchHistoricalWorkItemsBatch;
     private executeHistoricalQueryAtAsOf;
     private toHistoricalWorkItemSnapshot;

package/bin/modules/TicketsDataProvider.js

@@ -8,11 +8,12 @@ const tfs_data_3 = require("../models/tfs-data");
 const tfs_data_4 = require("../models/tfs-data");
 const tfs_data_5 = require("../models/tfs-data");
 const tfs_data_6 = require("../models/tfs-data");
+const testStepParserHelper_1 = require("../utils/testStepParserHelper");
 const logger_1 = require("../utils/logger");
 const pLimit = require('p-limit');
 const HISTORICAL_WIT_API_VERSIONS = ['7.1', '5.1', null];
 const HISTORICAL_BATCH_MAX_IDS = 200;
-const HISTORICAL_WORK_ITEM_FIELDS = [
+const HISTORICAL_REQUIRED_WORK_ITEM_FIELDS = [
     'System.Id',
     'System.WorkItemType',
     'System.Title',
@@ -23,8 +24,19 @@ const HISTORICAL_WORK_ITEM_FIELDS = [
     'System.ChangedDate',
     'System.Description',
     'Microsoft.VSTS.TCM.Steps',
-    'Elisra.TestPhase',
-    'Custom.TestPhase',
+];
+// Process-template-specific: not every project defines either of these.
+// workitemsbatch's `fields` filter validates every requested reference name
+// against the project's schema and throws TF51535 ("Cannot find field ...")
+// if one is missing — unlike an unfiltered per-item GET, which just omits an
+// unknown field from the response. toHistoricalWorkItemSnapshot already
+// reads TestPhase via a 4-way fallback expecting it might be absent, so it's
+// safe to drop these on a TF51535 and retry without them (see
+// fetchHistoricalWorkItemsBatch).
+const HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS = ['Elisra.TestPhase', 'Custom.TestPhase'];
+const HISTORICAL_WORK_ITEM_FIELDS = [
+    ...HISTORICAL_REQUIRED_WORK_ITEM_FIELDS,
+    ...HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS,
 ];
 /** Default fields fetched per work item in tree/flat query parsing. */
 const WI_DEFAULT_FIELDS = 'System.Description,System.Title,System.WorkItemType,Microsoft.VSTS.TCM.ReproSteps,Microsoft.VSTS.CMMI.Symptom';
@@ -34,6 +46,9 @@ class TicketsDataProvider {
         this.token = '';
         this.queriesList = new Array();
         this.limit = pLimit(10);
+        // Set once a workitemsbatch call reports one of HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS
+        // as unknown for this project, so only the first chunk pays the retry cost.
+        this.historicalFieldsOverride = null;
         this.orgUrl = orgUrl;
         this.token = token;
     }
@@ -1395,6 +1410,52 @@ class TicketsDataProvider {
         }
         return String(value).trim();
     }
+    /**
+     * Stores rich-text fields (Description, Steps) with only whitespace trimmed, preserving markup
+     * so the report renderer can clean/render it faithfully. Equality between two snapshots must go
+     * through canonicalizeHistoricalRichTextForCompare instead of comparing this value directly.
+     */
+    rawHistoricalRichText(value) {
+        if (value == null) {
+            return '';
+        }
+        if (typeof value === 'string') {
+            return value.trim();
+        }
+        return this.normalizeHistoricalCompareValue(value);
+    }
+    /**
+     * Canonicalizes a rich-text field for equality comparison only. Intentionally identical to the
+     * whitespace collapse previously applied at storage time, so Changed/No-changes verdicts and
+     * summary counts are unaffected by storing the raw value for rendering.
+     */
+    canonicalizeHistoricalRichTextForCompare(value) {
+        return this.normalizeHistoricalCompareValue(value);
+    }
+    /**
+     * Parses baseline/compareTo Steps XML into per-step Action/Expected arrays for a changed test
+     * case, so the report can render readable steps instead of raw Steps XML. Best-effort: a parse
+     * failure (malformed XML, shared-step lookup error) must not fail the whole comparison, so it
+     * logs and falls back to leaving the caller with only the raw XML strings.
+     */
+    async parseHistoricalStepsForDiff(testStepParserHelper, baselineStepsXml, compareToStepsXml) {
+        try {
+            const emptyRevisionMap = new Map();
+            const [baselineSteps, compareToSteps] = await Promise.all([
+                baselineStepsXml
+                    ? testStepParserHelper.parseTestSteps(baselineStepsXml, emptyRevisionMap)
+                    : Promise.resolve([]),
+                compareToStepsXml
+                    ? testStepParserHelper.parseTestSteps(compareToStepsXml, emptyRevisionMap)
+                    : Promise.resolve([]),
+            ]);
+            return { baselineSteps, compareToSteps };
+        }
+        catch (err) {
+            logger_1.default.warn(`Failed to parse historical Steps XML for diff rendering: ${(err === null || err === void 0 ? void 0 : err.message) || err}`);
+            return null;
+        }
+    }
     normalizeTestPhaseValue(value) {
         const rendered = this.normalizeHistoricalCompareValue(value);
         if (!rendered) {
@@ -1571,6 +1632,61 @@ class TicketsDataProvider {
         }
         return status === 404 || status === 410;
     }
+    isHistoricalUnknownFieldError(error) {
+        const message = this.historicalErrorMessage(error);
+        return /TF51535|Cannot find field/i.test(message);
+    }
+    // ADO's TF51535 names the specific field it couldn't find, e.g.
+    // "TF51535: Cannot find field 'Elisra.TestPhase'." — different networks
+    // define this optional field under different reference names (Elisra.* vs
+    // Custom.*), so drop only the one ADO actually rejected rather than every
+    // optional field, or a network that defines Custom.TestPhase but not
+    // Elisra.TestPhase would lose Test Phase comparison entirely for no reason.
+    extractHistoricalUnknownFieldName(error) {
+        const message = this.historicalErrorMessage(error);
+        const quoted = message.match(/Cannot find field\s*'([^']+)'/i);
+        if (quoted === null || quoted === void 0 ? void 0 : quoted[1])
+            return quoted[1];
+        return HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.find((field) => message.includes(field)) || null;
+    }
+    fetchHistoricalFieldsBatchAttempt(workItemsBatchUrl, idChunk, asOf, fieldsForRequest, attemptsRemaining) {
+        return this.limit(() => tfs_1.TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
+            ids: idChunk,
+            asOf,
+            fields: fieldsForRequest,
+            errorPolicy: 'Omit',
+        })).catch((error) => {
+            if (attemptsRemaining <= 0 || !this.isHistoricalUnknownFieldError(error)) {
+                throw error;
+            }
+            const unknownField = this.extractHistoricalUnknownFieldName(error);
+            // Only strip fields this call actually recognizes as droppable
+            // (named explicitly, or one of the known optional fields as a
+            // last-resort fallback if the message doesn't parse) — never strip a
+            // required field, so an unrelated schema issue still surfaces as an
+            // error instead of silently losing data.
+            const reducedFields = unknownField
+                ? fieldsForRequest.filter((field) => field !== unknownField)
+                : fieldsForRequest.filter((field) => !HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.includes(field));
+            if (reducedFields.length === fieldsForRequest.length) {
+                throw error;
+            }
+            // Concurrent chunks can each discover a different bad field before any
+            // of them updates the shared override — intersect rather than
+            // overwrite, so one chunk's retry never re-requests a field another
+            // chunk already proved invalid for this project.
+            this.historicalFieldsOverride = this.historicalFieldsOverride
+                ? this.historicalFieldsOverride.filter((field) => reducedFields.includes(field))
+                : reducedFields;
+            logger_1.default.warn(`[historical-workitems] field${unknownField ? ` '${unknownField}'` : '(s)'} not defined ` +
+                `for this project; retrying workitemsbatch without it: ${this.historicalErrorMessage(error)}`);
+            return this.fetchHistoricalFieldsBatchAttempt(workItemsBatchUrl, idChunk, asOf, reducedFields, attemptsRemaining - 1);
+        });
+    }
+    fetchHistoricalFieldsBatch(workItemsBatchUrl, idChunk, asOf) {
+        const fieldsForRequest = this.historicalFieldsOverride || HISTORICAL_WORK_ITEM_FIELDS;
+        return this.fetchHistoricalFieldsBatchAttempt(workItemsBatchUrl, idChunk, asOf, fieldsForRequest, HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.length);
+    }
     async fetchHistoricalWorkItemsBatch(project, ids, asOf, apiVersion) {
         const workItemsBatchUrl = this.appendApiVersion(`${this.orgUrl}${project}/_apis/wit/workitemsbatch`, apiVersion);
         const idChunks = this.chunkHistoricalWorkItemIds(ids);
@@ -1588,12 +1704,7 @@ class TicketsDataProvider {
             // two units against the shared concurrency cap, not one — otherwise
             // the real ceiling on simultaneous ADO calls would silently double.
             Promise.all([
-                this.limit(() => tfs_1.TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
-                    ids: idChunk,
-                    asOf,
-                    fields: HISTORICAL_WORK_ITEM_FIELDS,
-                    errorPolicy: 'Omit',
-                })),
+                this.fetchHistoricalFieldsBatch(workItemsBatchUrl, idChunk, asOf),
                 this.limit(() => tfs_1.TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
                     ids: idChunk,
                     asOf,
@@ -1716,8 +1827,8 @@ class TicketsDataProvider {
             iterationPath: this.normalizeHistoricalCompareValue(fields['System.IterationPath']),
             versionId: this.toHistoricalRevision((_d = workItem === null || workItem === void 0 ? void 0 : workItem.rev) !== null && _d !== void 0 ? _d : fields['System.Rev']),
             versionTimestamp: this.normalizeHistoricalCompareValue(fields['System.ChangedDate']),
-            description: this.normalizeHistoricalCompareValue(fields['System.Description']),
-            steps: this.normalizeHistoricalCompareValue(fields['Microsoft.VSTS.TCM.Steps']),
+            description: this.rawHistoricalRichText(fields['System.Description']),
+            steps: this.rawHistoricalRichText(fields['Microsoft.VSTS.TCM.Steps']),
             testPhase: this.normalizeTestPhaseValue(testPhaseRaw),
             relatedLinkCount,
             workItemUrl: `${this.orgUrl}${project}/_workitems/edit/${id}`,
@@ -1853,7 +1964,8 @@ class TicketsDataProvider {
             ...Array.from(compareTo.snapshotMap.keys()),
         ]);
         const sortedIds = Array.from(allIds.values()).sort((a, b) => a - b);
-        const rows = sortedIds.map((id) => {
+        const testStepParserHelper = new testStepParserHelper_1.default(this.orgUrl, this.token);
+        const rows = await Promise.all(sortedIds.map(async (id) => {
             const baselineRow = baseline.snapshotMap.get(id) || null;
             const compareToRow = compareTo.snapshotMap.get(id) || null;
             const workItemType = (compareToRow === null || compareToRow === void 0 ? void 0 : compareToRow.workItemType) || (baselineRow === null || baselineRow === void 0 ? void 0 : baselineRow.workItemType) || '';
@@ -1887,7 +1999,12 @@ class TicketsDataProvider {
             const safeBaseline = baselineRow;
             const safeCompareTo = compareToRow;
             const changedFields = [];
-            if (safeBaseline.description !== safeCompareTo.description) {
+            const descriptionChanged = this.canonicalizeHistoricalRichTextForCompare(safeBaseline.description) !==
+                this.canonicalizeHistoricalRichTextForCompare(safeCompareTo.description);
+            const stepsChanged = isTestCase &&
+                this.canonicalizeHistoricalRichTextForCompare(safeBaseline.steps) !==
+                    this.canonicalizeHistoricalRichTextForCompare(safeCompareTo.steps);
+            if (descriptionChanged) {
                 changedFields.push('Description');
             }
             if (safeBaseline.title !== safeCompareTo.title) {
@@ -1896,7 +2013,7 @@ class TicketsDataProvider {
             if (safeBaseline.state !== safeCompareTo.state) {
                 changedFields.push('State');
             }
-            if (isTestCase && safeBaseline.steps !== safeCompareTo.steps) {
+            if (stepsChanged) {
                 changedFields.push('Steps');
             }
             if (safeBaseline.testPhase !== safeCompareTo.testPhase) {
@@ -1905,7 +2022,7 @@ class TicketsDataProvider {
             if (isTestCase && safeBaseline.relatedLinkCount !== safeCompareTo.relatedLinkCount) {
                 changedFields.push('Related Link Count');
             }
-            const differences = changedFields.map((field) => {
+            const differences = await Promise.all(changedFields.map(async (field) => {
                 switch (field) {
                     case 'Description':
                         return { field, baseline: safeBaseline.description, compareTo: safeCompareTo.description };
@@ -1913,8 +2030,10 @@ class TicketsDataProvider {
                         return { field, baseline: safeBaseline.title, compareTo: safeCompareTo.title };
                     case 'State':
                         return { field, baseline: safeBaseline.state, compareTo: safeCompareTo.state };
-                    case 'Steps':
-                        return { field, baseline: safeBaseline.steps, compareTo: safeCompareTo.steps };
+                    case 'Steps': {
+                        const parsed = await this.parseHistoricalStepsForDiff(testStepParserHelper, safeBaseline.steps, safeCompareTo.steps);
+                        return Object.assign({ field, baseline: safeBaseline.steps, compareTo: safeCompareTo.steps }, (parsed || {}));
+                    }
                     case 'Test Phase':
                         return { field, baseline: safeBaseline.testPhase, compareTo: safeCompareTo.testPhase };
                     case 'Related Link Count':
@@ -1926,7 +2045,7 @@ class TicketsDataProvider {
                     default:
                         return { field, baseline: '', compareTo: '' };
                 }
-            });
+            }));
             return {
                 id,
                 workItemType,
@@ -1938,7 +2057,7 @@ class TicketsDataProvider {
                 differences,
                 workItemUrl: safeCompareTo.workItemUrl || safeBaseline.workItemUrl,
             };
-        });
+        }));
         const summary = rows.reduce((acc, row) => {
             if (row.compareStatus === 'Added')
                 acc.addedCount += 1;