@elisra-devops/docgen-data-provider 1.137.0 → 1.138.0

package/bin/modules/TicketsDataProvider.d.ts

@@ -10,6 +10,7 @@ export default class TicketsDataProvider {
     token: string;
     queriesList: Array<any>;
     private limit;
+    private historicalFieldsOverride;
     constructor(orgUrl: string, token: string);
     /**
      * Extracts the Azure DevOps project name from a WIQL URL.
@@ -214,6 +215,10 @@ export default class TicketsDataProvider {
     private normalizeHistoricalQueryRoot;
     private historicalErrorMessage;
     private isHistoricalMissingWorkItemError;
+    private isHistoricalUnknownFieldError;
+    private extractHistoricalUnknownFieldName;
+    private fetchHistoricalFieldsBatchAttempt;
+    private fetchHistoricalFieldsBatch;
     private fetchHistoricalWorkItemsBatch;
     private executeHistoricalQueryAtAsOf;
     private toHistoricalWorkItemSnapshot;

package/bin/modules/TicketsDataProvider.js

@@ -12,7 +12,7 @@ const logger_1 = require("../utils/logger");
 const pLimit = require('p-limit');
 const HISTORICAL_WIT_API_VERSIONS = ['7.1', '5.1', null];
 const HISTORICAL_BATCH_MAX_IDS = 200;
-const HISTORICAL_WORK_ITEM_FIELDS = [
+const HISTORICAL_REQUIRED_WORK_ITEM_FIELDS = [
     'System.Id',
     'System.WorkItemType',
     'System.Title',
@@ -23,8 +23,19 @@ const HISTORICAL_WORK_ITEM_FIELDS = [
     'System.ChangedDate',
     'System.Description',
     'Microsoft.VSTS.TCM.Steps',
-    'Elisra.TestPhase',
-    'Custom.TestPhase',
+];
+// Process-template-specific: not every project defines either of these.
+// workitemsbatch's `fields` filter validates every requested reference name
+// against the project's schema and throws TF51535 ("Cannot find field ...")
+// if one is missing — unlike an unfiltered per-item GET, which just omits an
+// unknown field from the response. toHistoricalWorkItemSnapshot already
+// reads TestPhase via a 4-way fallback expecting it might be absent, so it's
+// safe to drop these on a TF51535 and retry without them (see
+// fetchHistoricalWorkItemsBatch).
+const HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS = ['Elisra.TestPhase', 'Custom.TestPhase'];
+const HISTORICAL_WORK_ITEM_FIELDS = [
+    ...HISTORICAL_REQUIRED_WORK_ITEM_FIELDS,
+    ...HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS,
 ];
 /** Default fields fetched per work item in tree/flat query parsing. */
 const WI_DEFAULT_FIELDS = 'System.Description,System.Title,System.WorkItemType,Microsoft.VSTS.TCM.ReproSteps,Microsoft.VSTS.CMMI.Symptom';
@@ -34,6 +45,9 @@ class TicketsDataProvider {
         this.token = '';
         this.queriesList = new Array();
         this.limit = pLimit(10);
+        // Set once a workitemsbatch call reports one of HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS
+        // as unknown for this project, so only the first chunk pays the retry cost.
+        this.historicalFieldsOverride = null;
         this.orgUrl = orgUrl;
         this.token = token;
     }
@@ -1571,19 +1585,106 @@ class TicketsDataProvider {
         }
         return status === 404 || status === 410;
     }
+    isHistoricalUnknownFieldError(error) {
+        const message = this.historicalErrorMessage(error);
+        return /TF51535|Cannot find field/i.test(message);
+    }
+    // ADO's TF51535 names the specific field it couldn't find, e.g.
+    // "TF51535: Cannot find field 'Elisra.TestPhase'." — different networks
+    // define this optional field under different reference names (Elisra.* vs
+    // Custom.*), so drop only the one ADO actually rejected rather than every
+    // optional field, or a network that defines Custom.TestPhase but not
+    // Elisra.TestPhase would lose Test Phase comparison entirely for no reason.
+    extractHistoricalUnknownFieldName(error) {
+        const message = this.historicalErrorMessage(error);
+        const quoted = message.match(/Cannot find field\s*'([^']+)'/i);
+        if (quoted === null || quoted === void 0 ? void 0 : quoted[1])
+            return quoted[1];
+        return HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.find((field) => message.includes(field)) || null;
+    }
+    fetchHistoricalFieldsBatchAttempt(workItemsBatchUrl, idChunk, asOf, fieldsForRequest, attemptsRemaining) {
+        return this.limit(() => tfs_1.TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
+            ids: idChunk,
+            asOf,
+            fields: fieldsForRequest,
+            errorPolicy: 'Omit',
+        })).catch((error) => {
+            if (attemptsRemaining <= 0 || !this.isHistoricalUnknownFieldError(error)) {
+                throw error;
+            }
+            const unknownField = this.extractHistoricalUnknownFieldName(error);
+            // Only strip fields this call actually recognizes as droppable
+            // (named explicitly, or one of the known optional fields as a
+            // last-resort fallback if the message doesn't parse) — never strip a
+            // required field, so an unrelated schema issue still surfaces as an
+            // error instead of silently losing data.
+            const reducedFields = unknownField
+                ? fieldsForRequest.filter((field) => field !== unknownField)
+                : fieldsForRequest.filter((field) => !HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.includes(field));
+            if (reducedFields.length === fieldsForRequest.length) {
+                throw error;
+            }
+            // Concurrent chunks can each discover a different bad field before any
+            // of them updates the shared override — intersect rather than
+            // overwrite, so one chunk's retry never re-requests a field another
+            // chunk already proved invalid for this project.
+            this.historicalFieldsOverride = this.historicalFieldsOverride
+                ? this.historicalFieldsOverride.filter((field) => reducedFields.includes(field))
+                : reducedFields;
+            logger_1.default.warn(`[historical-workitems] field${unknownField ? ` '${unknownField}'` : '(s)'} not defined ` +
+                `for this project; retrying workitemsbatch without it: ${this.historicalErrorMessage(error)}`);
+            return this.fetchHistoricalFieldsBatchAttempt(workItemsBatchUrl, idChunk, asOf, reducedFields, attemptsRemaining - 1);
+        });
+    }
+    fetchHistoricalFieldsBatch(workItemsBatchUrl, idChunk, asOf) {
+        const fieldsForRequest = this.historicalFieldsOverride || HISTORICAL_WORK_ITEM_FIELDS;
+        return this.fetchHistoricalFieldsBatchAttempt(workItemsBatchUrl, idChunk, asOf, fieldsForRequest, HISTORICAL_OPTIONAL_WORK_ITEM_FIELDS.length);
+    }
     async fetchHistoricalWorkItemsBatch(project, ids, asOf, apiVersion) {
         const workItemsBatchUrl = this.appendApiVersion(`${this.orgUrl}${project}/_apis/wit/workitemsbatch`, apiVersion);
         const idChunks = this.chunkHistoricalWorkItemIds(ids);
         try {
-            const batchResponses = await Promise.all(idChunks.map((idChunk) => this.limit(() => tfs_1.TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
-                ids: idChunk,
-                asOf,
-                $expand: 'Relations',
-                fields: HISTORICAL_WORK_ITEM_FIELDS,
-            }))));
+            const chunkResults = await Promise.all(idChunks.map((idChunk) => 
+            // ADO rejects a workitemsbatch request that combines `fields` with
+            // `$expand` in the same call ("the expand parameter can not be used
+            // with the fields parameter"), so fields and relations are fetched
+            // in two separate batched passes and merged by id below.
+            // errorPolicy: 'Omit' keeps one inaccessible/deleted work item from
+            // failing the whole chunk (ADO's default is 'Fail'), matching the
+            // per-item fallback's tolerance for missing items.
+            // Each pass gets its own this.limit(...) slot (rather than wrapping
+            // both in one) so the two concurrent calls per chunk still count as
+            // two units against the shared concurrency cap, not one — otherwise
+            // the real ceiling on simultaneous ADO calls would silently double.
+            Promise.all([
+                this.fetchHistoricalFieldsBatch(workItemsBatchUrl, idChunk, asOf),
+                this.limit(() => tfs_1.TFSServices.getItemContent(workItemsBatchUrl, this.token, 'post', {
+                    ids: idChunk,
+                    asOf,
+                    $expand: 'Relations',
+                    errorPolicy: 'Omit',
+                })),
+            ]).then(([fieldsBatch, relationsBatch]) => {
+                const relationsById = new Map();
+                (Array.isArray(relationsBatch === null || relationsBatch === void 0 ? void 0 : relationsBatch.value) ? relationsBatch.value : []).forEach((item) => {
+                    const relId = Number(item === null || item === void 0 ? void 0 : item.id);
+                    if (Number.isFinite(relId)) {
+                        relationsById.set(relId, Array.isArray(item === null || item === void 0 ? void 0 : item.relations) ? item.relations : []);
+                    }
+                });
+                const fieldsValue = Array.isArray(fieldsBatch === null || fieldsBatch === void 0 ? void 0 : fieldsBatch.value) ? fieldsBatch.value : [];
+                const returnedIds = new Set(fieldsValue.map((item) => Number(item === null || item === void 0 ? void 0 : item.id)));
+                const items = fieldsValue.map((item) => (Object.assign(Object.assign({}, item), { relations: relationsById.get(Number(item === null || item === void 0 ? void 0 : item.id)) || [] })));
+                const skippedWorkItemIds = idChunk.filter((id) => !returnedIds.has(id));
+                if (skippedWorkItemIds.length > 0) {
+                    logger_1.default.warn(`[historical-workitems] workitemsbatch omitted ${skippedWorkItemIds.length} ` +
+                        `inaccessible/missing work item(s) for asOf ${asOf}: ${skippedWorkItemIds.join(', ')}`);
+                }
+                return { items, skippedWorkItemIds };
+            })));
             return {
-                items: batchResponses.flatMap((batch) => (Array.isArray(batch === null || batch === void 0 ? void 0 : batch.value) ? batch.value : [])),
-                skippedWorkItemIds: [],
+                items: chunkResults.flatMap((chunk) => chunk.items),
+                skippedWorkItemIds: chunkResults.flatMap((chunk) => chunk.skippedWorkItemIds),
             };
         }
         catch (error) {