@elisra-devops/docgen-data-provider 1.142.0 → 1.143.0

package/src/modules/TicketsDataProvider.ts

@@ -10,7 +10,7 @@ import { TestSteps } from '../models/tfs-data';
 import TestStepParserHelper from '../utils/testStepParserHelper';
 
 import logger from '../utils/logger';
-import { describeError } from '../helpers/requestContext';
+import { describeError, logCaughtError } from '../helpers/requestContext';
 const pLimit = require('p-limit');
 
 type FallbackFetchOutcome = {
@@ -492,7 +492,7 @@ export default class TicketsDataProvider {
           break;
       }
     } catch (err: any) {
-      logger.error('Error occurred during fetching shared queries:', describeError(err));
+      logCaughtError('Error occurred during fetching shared queries:', err);
       throw err;
     }
   }
@@ -528,7 +528,7 @@ export default class TicketsDataProvider {
           })
       );
     } catch (err: any) {
-      logger.error('Error occurred during fetching fields by type:', describeError(err));
+      logCaughtError('Error occurred during fetching fields by type:', err);
       throw err;
     }
   }
@@ -559,7 +559,7 @@ export default class TicketsDataProvider {
       }
       return { types, states: Array.from(stateSet) };
     } catch (err: any) {
-      logger.error('Error fetching work item type states:', describeError(err));
+      logCaughtError('Error fetching work item type states:', err);
       throw err;
     }
   }
@@ -1287,7 +1287,7 @@ export default class TicketsDataProvider {
           break;
       }
     } catch (err: any) {
-      logger.error(`Could not fetch query results for ${wiqlHref}:`, describeError(err));
+      logCaughtError(`Could not fetch query results for ${wiqlHref}:`, err);
     }
   }
 
@@ -1663,7 +1663,7 @@ export default class TicketsDataProvider {
       }
       return [...workItemsResultMap.values()];
     } catch (error: any) {
-      logger.error('could not parse requested flat query:', describeError(error));
+      logCaughtError('could not parse requested flat query:', error);
     }
   }
 
@@ -3563,7 +3563,7 @@ export default class TicketsDataProvider {
 
       item.fields = { ...parsedFields };
     } catch (err: any) {
-      logger.error('Cannot filter columns:', describeError(err));
+      logCaughtError('Cannot filter columns:', err);
       throw err;
     }
   }
@@ -3618,7 +3618,7 @@ export default class TicketsDataProvider {
 
       return workItemTypesWithIcons;
     } catch (err: any) {
-      logger.error('Error occurred during fetching work item types:', describeError(err));
+      logCaughtError('Error occurred during fetching work item types:', err);
       throw err;
     }
   }
@@ -3793,7 +3793,7 @@ export default class TicketsDataProvider {
         totalCount: workItemIds.length,
       };
     } catch (err: any) {
-      logger.error('Could not fetch categorized requirements:', describeError(err));
+      logCaughtError('Could not fetch categorized requirements:', err);
       throw err;
     }
   }
@@ -3888,7 +3888,7 @@ export default class TicketsDataProvider {
           const relations = qr?.workItemRelations || [];
           result['req-test'] = await resolveQuerySides(cols, relations, true);
         } catch (err: any) {
-          logger.error('GetTraceColumnsByType[req-test] failed:', describeError(err));
+          logCaughtError('GetTraceColumnsByType[req-test] failed:', err);
           result['req-test'] = { Requirement: [], 'Test Case': [] };
         }
       }
@@ -3900,14 +3900,14 @@ export default class TicketsDataProvider {
           const relations = qr?.workItemRelations || [];
           result['test-req'] = await resolveQuerySides(cols, relations, false);
         } catch (err: any) {
-          logger.error('GetTraceColumnsByType[test-req] failed:', describeError(err));
+          logCaughtError('GetTraceColumnsByType[test-req] failed:', err);
           result['test-req'] = { Requirement: [], 'Test Case': [] };
         }
       }
 
       return result;
     } catch (err: any) {
-      logger.error('GetTraceColumnsByType failed:', describeError(err));
+      logCaughtError('GetTraceColumnsByType failed:', err);
       return {};
     }
   }

package/src/tests/helpers/requestContext.test.ts

@@ -1,3 +1,4 @@
+import logger from '../../utils/logger';
 import {
   sanitizeUrl,
   summarizeBody,
@@ -5,6 +6,9 @@ import {
   buildAdoRequestContext,
   annotateAdoError,
   maskWiqlLiterals,
+  templatePath,
+  describeError,
+  logCaughtError,
 } from '../../helpers/requestContext';
 
 describe('maskWiqlLiterals', () => {
@@ -200,3 +204,49 @@ describe('annotateAdoError', () => {
     expect(() => annotateAdoError(undefined as any, { url: 'u' })).not.toThrow();
   });
 });
+
+describe('templatePath', () => {
+  it('drops origin and query and collapses numeric / guid segments', () => {
+    expect(templatePath('https://dev.azure.com/org/p/_apis/wit/workitems/123?api-version=7')).toBe(
+      '/org/p/_apis/wit/workitems/{id}'
+    );
+    expect(templatePath('https://h/a/3f2504e0-4f89-11d3-9a0c-0305e82c3301/b')).toBe('/a/{id}/b');
+  });
+  it('copes with a relative or missing url', () => {
+    expect(templatePath('/rel/42?x=1')).toBe('/rel/{id}');
+    expect(templatePath(undefined)).toBe('(unknown url)');
+  });
+});
+
+describe('describeError', () => {
+  it('carries the original message separately so the dashboard err.message is not the combined line', () => {
+    expect(describeError(new Error('boom'))).toMatchObject({ message: 'boom', errMessage: 'boom' });
+  });
+});
+
+describe('logCaughtError', () => {
+  beforeEach(() => {
+    jest.spyOn(logger, 'warn').mockImplementation((() => logger) as any);
+    jest.spyOn(logger, 'error').mockImplementation((() => logger) as any);
+  });
+  afterEach(() => jest.restoreAllMocks());
+
+  it('logs at warn when TFSServices already reported the error', () => {
+    const err = annotateAdoError(new Error('x'), { method: 'GET', url: 'https://h/y', reported: true });
+    logCaughtError('Could not fetch Y:', err);
+    expect(logger.warn).toHaveBeenCalledWith('Could not fetch Y:', expect.objectContaining({ message: 'x' }));
+    expect(logger.error).not.toHaveBeenCalled();
+  });
+
+  it('stays at error when the error was annotated but not reported (printError=false)', () => {
+    const err = annotateAdoError(new Error('x'), { method: 'GET', url: 'https://h/y', reported: false });
+    logCaughtError('Could not fetch Y:', err);
+    expect(logger.error).toHaveBeenCalledTimes(1);
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+
+  it('stays at error for a non-ADO error', () => {
+    logCaughtError('Could not fetch Y:', new Error('plain'));
+    expect(logger.error).toHaveBeenCalledTimes(1);
+  });
+});

package/src/tests/helpers/tfs.test.ts

@@ -1115,6 +1115,7 @@ describe('TFSServices request-context annotation', () => {
       url: 'https://dev.azure.com/org/_apis/wit/queries/q1',
       status: 404,
       attempt: 1,
+      reported: false,
       responseExcerpt: 'TF401232: Work item 5 does not exist',
     });
     expect(logger.error).not.toHaveBeenCalled();
@@ -1127,13 +1128,33 @@ describe('TFSServices request-context annotation', () => {
 
     expect(logger.error).toHaveBeenCalledTimes(1);
     expect(logger.error).toHaveBeenCalledWith(
-      'ADO request failed',
+      'ADO request failed: GET /org/_apis/x -> 404',
       expect.objectContaining({
-        adoRequest: expect.objectContaining({ method: 'GET', url: 'https://dev.azure.com/org/_apis/x', status: 404 }),
+        adoRequest: expect.objectContaining({
+          method: 'GET',
+          url: 'https://dev.azure.com/org/_apis/x',
+          status: 404,
+          reported: true,
+        }),
       })
     );
   });
 
+  it('puts status and templated path in the message so 401 and 404 group separately', async () => {
+    const e401: any = new Error('Request failed with status code 401');
+    e401.response = { status: 401, data: {} };
+    mockAxiosInstance.request.mockRejectedValueOnce(e401);
+    await TFSServices.getItemContent('https://dev.azure.com/org/p/_apis/wit/workitems/123', 'pat').catch(() => undefined);
+    mockAxiosInstance.request.mockRejectedValueOnce(notFound());
+    await TFSServices.getItemContent('https://dev.azure.com/org/p/_apis/wit/workitems/456', 'pat').catch(() => undefined);
+
+    const messages = (logger.error as jest.Mock).mock.calls.map((c) => c[0]);
+    expect(messages).toEqual([
+      'ADO request failed: GET /org/p/_apis/wit/workitems/{id} -> 401',
+      'ADO request failed: GET /org/p/_apis/wit/workitems/{id} -> 404',
+    ]);
+  });
+
   it('never leaks the PAT: credentials are applied to the axios config, not the description', async () => {
     mockAxiosInstance.request.mockRejectedValue(notFound());
 

package/src/tests/modules/ResultDataProvider.test.ts

@@ -6032,6 +6032,7 @@ describe('ResultDataProvider', () => {
       // record with the response summary as meta (docgen-log-redaction / Phase 2/4).
       expect(logger.error).toHaveBeenCalledWith('Error during getCombinedResultsSummary:', {
         message: 'boom',
+        errMessage: 'boom',
         stack: expect.any(String),
         responseSummary: '{"detail":"bad"}',
       });

package/src/tests/utils/logger.test.ts

@@ -1,6 +1,6 @@
 import * as winston from 'winston';
 import Transport = require('winston-transport');
-import { redact, DiagnosticsTransport } from '../../utils/logger';
+import { redact, DiagnosticsTransport, isSensitiveKey, skipUncaptured } from '../../utils/logger';
 import { installLogSink, LogSink, DiagnosticEvent } from '../../utils/logSink';
 import { runContextStore } from '../../utils/runContext';
 
@@ -379,3 +379,37 @@ describe('DiagnosticsTransport capture policy (Phase 6b)', () => {
     expect(events[0].message).toBe('verbose debug');
   });
 });
+
+describe('isSensitiveKey', () => {
+  it.each(['token', 'accessToken', 'x-docgen-ingest-token', 'PAT', 'pat', 'password', 'DB_PASSWORD', 'Authorization', 'minioSecretKey', 'minioAccessKey', 'apiKey', 'clientSecret', 'cookie'])(
+    'redacts %s',
+    (key) => expect(isSensitiveKey(key)).toBe(true)
+  );
+  it.each(['areaPath', 'IterationPath', 'path', 'filePath', 'patch', 'dispatch', 'compatibility', 'tokenCount', 'System.AreaPath', 'url'])(
+    'keeps %s',
+    (key) => expect(isSensitiveKey(key)).toBe(false)
+  );
+  it('redact() leaves areaPath visible but scrubs a token', () => {
+    const out = applyRedact({ message: 'm', areaPath: 'P\\Area', accessToken: 'abc' });
+    expect(out.areaPath).toBe('P\\Area');
+    expect(out.accessToken).toBe('[REDACTED]');
+  });
+});
+
+describe('skipUncaptured', () => {
+  const run = (level: string) => (skipUncaptured() as any).transform({ level, message: 'm' });
+  it('keeps what stdout prints and always keeps warn/error', () => {
+    expect(run('error')).toBeTruthy();
+    expect(run('warn')).toBeTruthy();
+    expect(run('info')).toBeTruthy();
+  });
+  it('drops debug in normal mode', () => {
+    expect(run('debug')).toBe(false);
+  });
+  it('keeps debug inside a verbose run', () => {
+    const { runContextStore } = require('../../utils/runContext');
+    runContextStore.run({ runId: 'r1', captureMode: 'verbose' }, () => {
+      expect(run('debug')).toBeTruthy();
+    });
+  });
+});

package/src/utils/logger.ts

@@ -29,7 +29,24 @@ export const withRunContext = winston.format((info) => {
 // "accesskey" (not just "minioaccesskey") so this also catches an *AccessKeyId-style field —
 // a real gap found in api-gate's copy during the Phase 5 manifest work: only the *SecretKey
 // sibling was covered before, via "secret". Applied here too to keep the four copies in sync.
-const SENSITIVE_KEY = /token|pat|password|secret|authorization|accesskey|minioaccesskey|miniosecretkey/i;
+// Word-based, not substring: the key is split into lower-case words (camelCase, kebab-case and
+// snake_case alike) and judged by how it *ends*. A substring test matched "pat" inside
+// "areaPath" / "path" / "patch" and blanked ordinary ADO fields; this keeps `accessToken`,
+// `x-docgen-ingest-token`, `minioSecretKey`, `PAT`, `password` redacted while `areaPath` stays.
+const SENSITIVE_TAILS = ['token', 'password', 'secret', 'authorization', 'accesskey', 'secretkey', 'apikey'];
+const SENSITIVE_LAST_WORDS = new Set(['pat', 'pwd', 'cookie', 'tokens', 'passwords', 'secrets']);
+export function isSensitiveKey(key: string): boolean {
+  const words = key
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean);
+  if (words.length === 0) return false;
+  if (SENSITIVE_LAST_WORDS.has(words[words.length - 1])) return true;
+  const tail = words.slice(-2).join('');
+  return SENSITIVE_TAILS.some((t) => tail.endsWith(t));
+}
 
 // A per-key try/catch on the *read*, not just around the whole loop: a getter that throws
 // (e.g. `{ get boom() { throw ... } }`) would otherwise abort redaction for every remaining
@@ -56,7 +73,7 @@ export function redactValue(value: unknown, depth = 0, seen = new WeakSet<object
       out[key] = '[Unreadable property]';
       continue;
     }
-    out[key] = SENSITIVE_KEY.test(key) ? '[REDACTED]' : redactValue(read.value, depth + 1, seen);
+    out[key] = isSensitiveKey(key) ? '[REDACTED]' : redactValue(read.value, depth + 1, seen);
   }
   return out;
 }
@@ -75,7 +92,7 @@ export const redact = winston.format((info) => {
       ? '[Unreadable property]'
       : // A top-level primitive (e.g. token: "abc") has no children for redactValue to walk into —
         // the key itself has to be checked here too, not just inside the recursive object walk.
-        SENSITIVE_KEY.test(key)
+        isSensitiveKey(key)
         ? '[REDACTED]'
         : redactValue(read.value);
     try {
@@ -216,7 +233,9 @@ export class DiagnosticsTransport extends Transport {
           message,
           err: hasErr
             ? {
-                message,
+                // The error's own message when the call site supplied it (describeError); else
+                // the combined winston message, which is all a bare logger.error(err) leaves.
+                message: clamp(info.errMessage, MAX_MESSAGE_LEN) ?? message,
                 code: typeof info.code === 'string' ? info.code : undefined,
                 stack: clamp(info.stack, MAX_STACK_LEN),
               }
@@ -248,11 +267,29 @@ const useJson = (process.env.LOG_FORMAT || 'text').toLowerCase() === 'json';
 // for free; info/warn/error volume (Phase 4's focus) is unaffected either way.
 const CONSOLE_LOG_LEVEL = process.env.LOG_LEVEL || 'info';
 
+// Drops a record before the (comparatively expensive) redact/splat/format work when nothing
+// would consume it: stdout wouldn't print it, and DiagnosticsTransport wouldn't persist it
+// (only debug/info under a verbose/retain-on-failure run are captured below warn). The logger's
+// own level gate must stay 'debug' for that capture, so this is the cheap early exit instead.
+const LEVEL_RANK: Record<string, number> = { error: 0, warn: 1, info: 2, http: 3, verbose: 4, debug: 5, silly: 6 };
+export const skipUncaptured = winston.format((info) => {
+  const printRank = LEVEL_RANK[CONSOLE_LOG_LEVEL];
+  const rank = LEVEL_RANK[String(info.level)];
+  if (printRank === undefined || rank === undefined || rank <= printRank) return info;
+  const mode = runContextStore.getStore()?.captureMode;
+  const captured =
+    DIAGNOSTICS_CAPTURE_ENABLED &&
+    (info.level === 'debug' || info.level === 'info') &&
+    (mode === 'verbose' || mode === 'retain-on-failure');
+  return captured ? info : false;
+});
+
 logger = winston.createLogger({
   level: 'debug',
   defaultMeta: { service: '@elisra-devops/docgen-data-provider', version: readOwnVersion() },
   format: useJson
     ? winston.format.combine(
+        skipUncaptured(),
         winston.format.errors({ stack: true }),
         winston.format.timestamp(),
         withRunContext(),
@@ -261,6 +298,7 @@ logger = winston.createLogger({
         winston.format.json()
       )
     : winston.format.combine(
+        skipUncaptured(),
         winston.format.errors({ stack: true }),
         winston.format.timestamp(),
         withRunContext(),