@quenty/nevermore-cli 4.43.4 → 4.45.0

package/src/utils/testing/parsers/batch-log-renderer.test.ts

@@ -0,0 +1,330 @@
+import { describe, it, expect } from 'vitest';
+import { OutputHelper } from '@quenty/cli-output-helpers';
+import {
+  renderBatchLog,
+  toRenderableLines,
+  type RenderableLogLine,
+} from './batch-log-renderer.js';
+import { type BatchPackageResult } from './batch-log-parser.js';
+
+const SLUGS = new Map([
+  ['alpha', '@quenty/alpha'],
+  ['beta', '@quenty/beta'],
+]);
+
+/** Plain OUTPUT lines, which is what most of a run is. */
+function plain(...lines: string[]): RenderableLogLine[] {
+  return lines.map((line) => ({ line, severity: 'output' as const }));
+}
+
+function render(
+  lines: RenderableLogLine[],
+  results?: Map<string, BatchPackageResult>
+): string[] {
+  return renderBatchLog(lines, {
+    slugToPackage: SLUGS,
+    results,
+    useGroups: true,
+    color: false,
+  });
+}
+
+function result(over: Partial<BatchPackageResult>): BatchPackageResult {
+  return {
+    slug: 'alpha',
+    success: true,
+    logs: '',
+    tracebackCount: 0,
+    ...over,
+  };
+}
+
+describe('renderBatchLog', () => {
+  it('wraps each package section in a group of its own', () => {
+    const out = render(
+      plain(
+        '===BATCH_TEST_BEGIN alpha===',
+        'alpha ran',
+        '===BATCH_TEST_END alpha PASS 10==='
+      )
+    );
+
+    expect(out).toEqual([
+      '::group::@quenty/alpha',
+      'alpha ran',
+      '::endgroup::',
+    ]);
+  });
+
+  it('keeps output that belongs to no package, outside any group', () => {
+    // The whole point. Output between two sections used to be discarded by the
+    // parser, so a crash landing between packages reached nobody.
+    const out = render(
+      plain(
+        'harness starting up',
+        '===BATCH_TEST_BEGIN alpha===',
+        'alpha ran',
+        '===BATCH_TEST_END alpha PASS 10===',
+        'something exploded between packages',
+        '===BATCH_TEST_BEGIN beta===',
+        'beta ran',
+        '===BATCH_TEST_END beta PASS 20===',
+        'trailing note after the last package'
+      )
+    );
+
+    expect(out).toEqual([
+      'harness starting up',
+      '::group::@quenty/alpha',
+      'alpha ran',
+      '::endgroup::',
+      'something exploded between packages',
+      '::group::@quenty/beta',
+      'beta ran',
+      '::endgroup::',
+      'trailing note after the last package',
+    ]);
+  });
+
+  it('emits one group per package rather than one per start', () => {
+    // The bug this replaces: every package was grouped when it started, which
+    // in an aggregated batch is all of them at once, before the task existed.
+    const out = render(
+      plain(
+        '===BATCH_TEST_BEGIN alpha===',
+        '===BATCH_TEST_END alpha PASS 10===',
+        '===BATCH_TEST_BEGIN beta===',
+        '===BATCH_TEST_END beta PASS 20==='
+      )
+    );
+
+    expect(out.filter((line) => line === '::endgroup::')).toHaveLength(2);
+    expect(out.indexOf('::group::@quenty/beta')).toBeGreaterThan(
+      out.indexOf('::endgroup::')
+    );
+  });
+
+  it('closes a section left open by a missing END', () => {
+    // Groups must balance whatever the log did, or every later line renders
+    // inside a package it has nothing to do with.
+    const out = render(
+      plain(
+        '===BATCH_TEST_BEGIN alpha===',
+        'alpha ran',
+        '===BATCH_TEST_BEGIN beta===',
+        'beta ran',
+        '===BATCH_TEST_END beta PASS 20==='
+      )
+    );
+
+    expect(out.filter((line) => line.startsWith('::group::'))).toHaveLength(2);
+    expect(out.filter((line) => line === '::endgroup::')).toHaveLength(2);
+  });
+
+  it('puts a dropped BEGIN section inside its group, output and all', () => {
+    // Truncation eats the head of the window, so the first surviving section
+    // announces itself only at its END — after its output has been emitted.
+    // Heading it there would leave a group holding nothing but the verdict
+    // while hundreds of its lines sat loose above it.
+    const out = render(
+      plain(
+        'alpha line one',
+        'alpha line two',
+        '===BATCH_TEST_END alpha PASS 10==='
+      )
+    );
+
+    expect(out).toEqual([
+      '::group::@quenty/alpha',
+      'alpha line one',
+      'alpha line two',
+      '::endgroup::',
+    ]);
+  });
+
+  it('leaves earlier top-level output above a dropped BEGIN section', () => {
+    // Only the run since the last section closed belongs to the orphan; output
+    // from before that is not retroactively swept into it.
+    const out = render(
+      plain(
+        '===BATCH_TEST_BEGIN alpha===',
+        'alpha ran',
+        '===BATCH_TEST_END alpha PASS 10===',
+        'loose line between packages',
+        'beta output with no begin',
+        '===BATCH_TEST_END beta PASS 20==='
+      )
+    );
+
+    expect(out).toEqual([
+      '::group::@quenty/alpha',
+      'alpha ran',
+      '::endgroup::',
+      '::group::@quenty/beta',
+      'loose line between packages',
+      'beta output with no begin',
+      '::endgroup::',
+    ]);
+  });
+
+  it('carries the verdict in the group title, and the reason in its body', () => {
+    // A group is read collapsed first, so the title has to say which ones are
+    // worth opening. The reason is a sentence and stays in the body.
+    const results = new Map([
+      [
+        '@quenty/alpha',
+        result({
+          success: false,
+          error: 'no jest report in output',
+          testCounts: { passed: 3, failed: 1, total: 4 },
+          durationMs: 1500,
+        }),
+      ],
+    ]);
+
+    const out = render(
+      plain(
+        '===BATCH_TEST_BEGIN alpha===',
+        'alpha ran',
+        '===BATCH_TEST_END alpha FAIL 10==='
+      ),
+      results
+    );
+
+    expect(out).toEqual([
+      '::group::@quenty/alpha - ❌ FAILED (3/4) (1.5s)',
+      'alpha ran',
+      'no jest report in output',
+      '::endgroup::',
+    ]);
+  });
+
+  it('titles a passing package with its counts and duration', () => {
+    const results = new Map([
+      [
+        '@quenty/alpha',
+        result({
+          success: true,
+          testCounts: { passed: 68, failed: 0, total: 68 },
+          durationMs: 1500,
+        }),
+      ],
+    ]);
+
+    const out = render(
+      plain(
+        '===BATCH_TEST_BEGIN alpha===',
+        '===BATCH_TEST_END alpha PASS 1==='
+      ),
+      results
+    );
+
+    expect(out[0]).toBe('::group::@quenty/alpha - ✅ Passed (68/68) (1.5s)');
+  });
+
+  it('titles a package with only its name when nothing is known about it', () => {
+    const out = render(
+      plain('===BATCH_TEST_BEGIN alpha===', '===BATCH_TEST_END alpha PASS 1===')
+    );
+
+    expect(out[0]).toBe('::group::@quenty/alpha');
+  });
+
+  it('drops the summary marker and its payload', () => {
+    // Framing for the parser, not output a person reads.
+    const out = render(
+      plain(
+        'real output',
+        '===BATCH_TEST_SUMMARY===',
+        '[{"slug":"alpha","success":true}]'
+      )
+    );
+
+    expect(out).toEqual(['real output']);
+  });
+
+  it('leaves plain output untouched so jest keeps its own color', () => {
+    const out = renderBatchLog(plain('✓ passed'), {
+      slugToPackage: SLUGS,
+      useGroups: true,
+      color: true,
+    });
+
+    expect(out).toEqual(['✓ passed']);
+  });
+
+  it('colors warnings and errors by the severity the API reported', () => {
+    const out = renderBatchLog(
+      [
+        { line: 'plain', severity: 'output' },
+        { line: 'careful', severity: 'warning' },
+        { line: 'broken', severity: 'error' },
+      ],
+      { slugToPackage: SLUGS, useGroups: true, color: true }
+    );
+
+    // Compared against the formatters themselves rather than asserting the text
+    // changed: chalk emits no escapes on a non-TTY, so "it differs" would pass
+    // or fail on where the suite runs rather than on what the renderer did.
+    expect(out[0]).toBe('plain');
+    expect(out[1]).toBe(OutputHelper.formatWarning('careful'));
+    expect(out[2]).toBe(OutputHelper.formatError('broken'));
+  });
+
+  it('uses a plain header instead of group markers outside CI', () => {
+    const out = renderBatchLog(
+      plain(
+        '===BATCH_TEST_BEGIN alpha===',
+        'ran',
+        '===BATCH_TEST_END alpha PASS 1==='
+      ),
+      { slugToPackage: SLUGS, useGroups: false, color: false }
+    );
+
+    expect(out.some((line) => line.includes('@quenty/alpha'))).toBe(true);
+    expect(out).not.toContain('::group::@quenty/alpha');
+    expect(out).not.toContain('::endgroup::');
+  });
+});
+
+describe('toRenderableLines', () => {
+  it('gives every line of a message the severity of that message', () => {
+    // An error arrives with its traceback attached as one message. Classifying
+    // by line would leave the continuation lines looking like ordinary output.
+    const lines = toRenderableLines([
+      { message: 'fine', messageType: 'OUTPUT' },
+      {
+        message:
+          "TaskScript:1: boom\nStack Begin\nScript 'X', Line 1\nStack End",
+        messageType: 'ERROR',
+      },
+    ]);
+
+    expect(lines).toHaveLength(5);
+    expect(lines[0]).toEqual({ line: 'fine', severity: 'output' });
+    expect(lines.slice(1).every((l) => l.severity === 'error')).toBe(true);
+    expect(lines[2].line).toBe('Stack Begin');
+  });
+
+  it('treats an untyped message as ordinary output', () => {
+    // The flat log view carries no type, and a missing type must not read as
+    // a failure.
+    expect(toRenderableLines([{ message: 'hi' }])).toEqual([
+      { line: 'hi', severity: 'output' },
+    ]);
+  });
+
+  it('produces exactly the lines the parser tokenizes', () => {
+    // The renderer maps token i onto line i, which only holds if this split is
+    // the same one the fetch's joined text would produce.
+    const messages = [
+      { message: 'one', messageType: 'OUTPUT' },
+      { message: 'two\nthree', messageType: 'WARNING' },
+    ];
+    const joined = messages.map((m) => m.message).join('\n');
+
+    expect(toRenderableLines(messages).map((l) => l.line)).toEqual(
+      joined.split('\n')
+    );
+  });
+});

package/src/utils/testing/parsers/batch-log-renderer.ts

@@ -0,0 +1,329 @@
+import { OutputHelper } from '@quenty/cli-output-helpers';
+import { formatDurationMs } from '@quenty/cli-output-helpers/cli-utils';
+import {
+  formatStatusText,
+  resolveResultStatus,
+  statusIcon,
+  type ResultStatusInput,
+} from '@quenty/cli-output-helpers/reporting';
+import {
+  type BatchPackageResult,
+  tokenizeBatchLog,
+} from './batch-log-parser.js';
+
+/**
+ * One line of a batch run's output, carrying the severity Open Cloud reported
+ * for the message it came from.
+ *
+ * Severity is per *message*, and a message can be several lines — an error
+ * arrives with its whole traceback attached — so every line of a message
+ * inherits it. That is why this is not derived from the text: guessing at
+ * severity from patterns misreads a traceback's continuation lines, while the
+ * API already says which of OUTPUT, WARNING and ERROR each message was.
+ */
+export interface RenderableLogLine {
+  line: string;
+  severity: 'output' | 'warning' | 'error';
+}
+
+export interface BatchLogRenderOptions {
+  /** Section slug to the package name a reader recognizes, for group titles. */
+  slugToPackage: Map<string, string>;
+  /**
+   * Per-package verdicts, printed as each section closes so a package's result
+   * sits with the output that produced it rather than in a later block.
+   *
+   * Keyed by package name, as `parseBatchTestLogs` returns it — sections are
+   * named by slug, so a lookup here goes through `slugToPackage` first.
+   */
+  results?: Map<string, BatchPackageResult>;
+  /** Emit GitHub `::group::` markers. Off outside CI, where they are noise. */
+  useGroups: boolean;
+  /** Colorize by severity. Off when NO_COLOR is set. */
+  color: boolean;
+}
+
+/**
+ * Split typed messages into the lines the parser tokenizes, keeping severity.
+ *
+ * The join is the same one `getRawTaskLogsAsync` performs, so the resulting
+ * lines are positionally identical to the parser's — which is what lets the
+ * token at index i describe the line at index i.
+ */
+export function toRenderableLines(
+  messages: Array<{ message: string; messageType?: string }>
+): RenderableLogLine[] {
+  const lines: RenderableLogLine[] = [];
+
+  for (const entry of messages) {
+    const severity =
+      entry.messageType === 'ERROR'
+        ? 'error'
+        : entry.messageType === 'WARNING'
+        ? 'warning'
+        : 'output';
+
+    for (const line of entry.message.split('\n')) {
+      lines.push({ line, severity });
+    }
+  }
+
+  return lines;
+}
+
+/**
+ * Render a whole batch run's output in arrival order, with each package's
+ * section wrapped in a collapsible group.
+ *
+ * Every line the run produced is emitted, including the ones that belong to no
+ * package: output between two sections, or before the first BEGIN, used to be
+ * discarded by the parser and so never reached anyone. Those lines are exactly
+ * the ones worth seeing when a batch misbehaves — a crash between packages, or
+ * a harness message printed before the first test — so they are rendered at the
+ * top level, outside any group, rather than dropped.
+ *
+ * Groups are injected around the parser's own BEGIN/END tokens rather than
+ * opened as each package starts. An aggregated batch starts every package at
+ * once, before the cloud task exists, so grouping by start time produced one
+ * empty group per package and put every line under the last of them.
+ */
+export function renderBatchLog(
+  lines: RenderableLogLine[],
+  options: BatchLogRenderOptions
+): string[] {
+  const tokens = tokenizeBatchLog(lines.map((entry) => entry.line));
+  const out: string[] = [];
+  /** Slugs that reached the output, so the rest can be listed after it. */
+  const rendered = new Set<string>();
+  let openSlug: string | undefined;
+  /**
+   * Where the run of lines belonging to no open section began.
+   *
+   * A section whose BEGIN the log window dropped only announces itself at its
+   * END, by which point its output has already been emitted. Knowing where that
+   * run started lets the header be spliced in above it, so the output lands
+   * inside its own group rather than the group holding only a verdict.
+   */
+  let orphanStart = 0;
+
+  const closeGroup = (): void => {
+    if (openSlug === undefined) {
+      return;
+    }
+    if (options.useGroups) {
+      out.push('::endgroup::');
+    }
+    openSlug = undefined;
+    orphanStart = out.length;
+  };
+
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+
+    if (token.kind === 'begin') {
+      // A second BEGIN with a section still open means the first never closed;
+      // close it here so the groups stay balanced whatever the log did.
+      closeGroup();
+      openSlug = token.slug;
+      rendered.add(token.slug);
+      out.push(...openGroup(token.slug, options));
+      continue;
+    }
+
+    if (token.kind === 'end') {
+      if (openSlug === undefined) {
+        // An END whose BEGIN the log window dropped. Its output is already in
+        // `out`, so the header goes in above it rather than here — otherwise
+        // the group holds nothing but the verdict while the output it belongs
+        // to sits loose above it.
+        out.splice(orphanStart, 0, ...openGroup(token.slug, options));
+        openSlug = token.slug;
+      }
+      rendered.add(token.slug);
+      out.push(...verdictLines(token.slug, options));
+      closeGroup();
+      continue;
+    }
+
+    // The summary marker and its JSON payload are framing for the parser, not
+    // output anyone reads.
+    if (token.kind === 'summary' || token.kind === 'summaryPayload') {
+      continue;
+    }
+
+    out.push(colorize(lines[i], options.color));
+  }
+
+  closeGroup();
+  out.push(...sectionlessGroups(rendered, options));
+
+  return out;
+}
+
+/**
+ * Groups for packages that never appeared in the log at all.
+ *
+ * A dropped log window takes whole sections with it, markers included, and such
+ * a package used to render nothing: no group, no verdict, no reason — only a
+ * row in the summary table saying FAILED with nothing behind it. The result is
+ * known regardless, because it came back in the run's return value, so it is
+ * shown in the same shape every other package is.
+ */
+function sectionlessGroups(
+  rendered: Set<string>,
+  options: BatchLogRenderOptions
+): string[] {
+  const out: string[] = [];
+
+  for (const [slug, name] of options.slugToPackage) {
+    if (rendered.has(slug)) {
+      continue;
+    }
+    const result = options.results?.get(name);
+    if (!result) {
+      continue;
+    }
+
+    out.push(...openGroup(slug, options));
+    out.push(...verdictLines(slug, options));
+    // The group is otherwise empty, so it says why rather than leaving a reader
+    // to work out that a package showing no output is one whose output is gone.
+    out.push(
+      options.color ? OutputHelper.formatDim(NO_SECTION_NOTE) : NO_SECTION_NOTE
+    );
+    if (options.useGroups) {
+      out.push('::endgroup::');
+    }
+  }
+
+  return out;
+}
+
+const NO_SECTION_NOTE =
+  "This package printed nothing that survived the run's log window. " +
+  'Its verdict and counts came back in the batch summary instead.';
+
+/**
+ * Render everything one batch execution produced, ready to print.
+ *
+ * The binding between what a run reports and what the renderer is told about it
+ * lives here rather than at the call site, so a field the outcome carries and
+ * the render forgets — the results that put a verdict in every group title, say
+ * — is a mistake one test can catch.
+ *
+ * Structural rather than typed against `BatchExecutionOutcome`: the job context
+ * that defines it already imports this module.
+ */
+export function renderBatchOutcome(
+  outcome: {
+    lines: RenderableLogLine[];
+    results: Map<string, BatchPackageResult>;
+    slugToPackage: Map<string, string>;
+  },
+  presentation: { useGroups: boolean; color: boolean }
+): string[] {
+  return renderBatchLog(outcome.lines, {
+    slugToPackage: outcome.slugToPackage,
+    results: outcome.results,
+    useGroups: presentation.useGroups,
+    color: presentation.color,
+  });
+}
+
+/**
+ * Header for one package's section, carrying its verdict.
+ *
+ * The status is in the title because a group is read collapsed: a reader
+ * scanning 78 of them wants to know which to open, and a title that is only a
+ * package name makes them all look alike. The counts and duration are formatted
+ * with the same helpers the PR comment table uses, so the two cannot drift into
+ * describing the same run differently.
+ */
+function openGroup(slug: string, options: BatchLogRenderOptions): string[] {
+  const name = options.slugToPackage.get(slug) ?? slug;
+  const result = options.results?.get(name);
+  const title = result ? `${name} - ${describeResult(result)}` : name;
+
+  if (options.useGroups) {
+    return [`::group::${title}`];
+  }
+
+  return [OutputHelper.formatDim(`──────── ${title} ────────`)];
+}
+
+/**
+ * "⚠️ Passed (35/35) - logs lost (1.2s)", the shape every other reporter uses.
+ *
+ * Resolved by the shared status resolver rather than worked out here, so a
+ * group title, a summary-table row and a PR comment row describing the same run
+ * cannot disagree about what state it is in or what that state is called. This
+ * only picks the icon style and puts the duration on the end.
+ */
+function describeResult(result: BatchPackageResult): string {
+  const status = resolveResultStatus(toStatusInput(result));
+  const duration =
+    result.durationMs === undefined
+      ? ''
+      : `(${formatDurationMs(result.durationMs)})`;
+
+  return [
+    statusIcon(status.severity, 'emoji'),
+    formatStatusText(status),
+    duration,
+  ]
+    .filter(Boolean)
+    .join(' ');
+}
+
+/**
+ * A batch result in the shape the status resolver reads.
+ *
+ * The parser's result is not a `PackageResult` — it is per-section, and one
+ * section is not a package's whole run — so the two fields the resolver needs
+ * are handed over rather than the type being widened to fit.
+ */
+function toStatusInput(result: BatchPackageResult): ResultStatusInput {
+  return {
+    success: result.success,
+    progressSummary: result.testCounts
+      ? {
+          kind: 'test-counts',
+          passed: result.testCounts.passed,
+          failed: result.testCounts.failed,
+          total: result.testCounts.total,
+        }
+      : undefined,
+    caveats: result.logsLost ? ['logs-lost'] : undefined,
+  };
+}
+
+/**
+ * Why a package failed, printed where its output ends.
+ *
+ * Only the reason: the verdict itself is in the group's title now, and saying
+ * it twice in six lines of output reads as two separate findings. A reason is
+ * a sentence and belongs in the body.
+ */
+function verdictLines(slug: string, options: BatchLogRenderOptions): string[] {
+  const name = options.slugToPackage.get(slug);
+  const result = name ? options.results?.get(name) : undefined;
+  if (!result?.error) {
+    return [];
+  }
+
+  return [
+    options.color ? OutputHelper.formatError(result.error) : result.error,
+  ];
+}
+
+function colorize(entry: RenderableLogLine, color: boolean): string {
+  if (!color || entry.severity === 'output') {
+    // Left exactly as it arrived: jest-lua colors its own output, and
+    // re-wrapping it would fight those escapes.
+    return entry.line;
+  }
+
+  return entry.severity === 'error'
+    ? OutputHelper.formatError(entry.line)
+    : OutputHelper.formatWarning(entry.line);
+}

package/src/utils/testing/runner/combined-project-generator.ts

@@ -125,7 +125,7 @@ export async function generateCombinedProjectAsync(options: {
   const luneScriptPath = resolvePackagePath(
     import.meta.url,
     'build-scripts',
-    'combine-test-places.luau'
+    'combine-test-places.lua'
   );
 
   // Build args: <outputPath> <slug1> <rbxl1> <script1> <slug2> <rbxl2> <script2> ...