@quenty/nevermore-cli 4.43.4 → 4.45.0

package/src/utils/testing/batch-run.e2e.test.ts

@@ -0,0 +1,425 @@
+/**
+ * End-to-end test for one aggregated batch run.
+ *
+ * Every piece of this path has unit tests — the parser, the renderer, the job
+ * context — and none of them cover the thing that actually goes wrong, which is
+ * the seams between them. What a reader sees is produced by a log travelling
+ * through all three, and until the execution returned data rather than printing
+ * it, that could not be asserted on without spying on `console.log`.
+ *
+ * So this drives the whole path from the one end that is not ours: a transport
+ * handing back the messages a real Open Cloud task produces, complete with the
+ * message types, the multi-line error and the trailing JSON summary. Everything
+ * downstream is the real code, and the assertions are on what a person would
+ * read and on what each package's result came out as.
+ */
+
+import { describe, it, expect, vi } from 'vitest';
+import { OutputHelper } from '@quenty/cli-output-helpers';
+import { BatchScriptJobContext } from '../job-context/batch-script-job-context.js';
+import {
+  type BuiltPlace,
+  type Deployment,
+  type JobContext,
+  type JobLogs,
+  type RunScriptOptions,
+  type ScriptRunResult,
+} from '../job-context/job-context.js';
+import { type TaskLogMessage } from '../open-cloud/open-cloud-client.js';
+import { type BatchTarget } from '../batch/changed-packages-utils.js';
+import { renderBatchOutcome } from './parsers/batch-log-renderer.js';
+
+const { SLUG_MAP } = vi.hoisted(() => ({
+  SLUG_MAP: new Map([
+    ['@quenty/alpha', 'alpha'],
+    ['@quenty/beta', 'beta'],
+  ]),
+}));
+
+// The only mocked collaborator: building the combined place shells out to rojo
+// and lune, which is a different subject from what this test is about. What it
+// produces that matters downstream is the slug map.
+vi.mock('./runner/combined-project-generator.js', () => ({
+  generateCombinedProjectAsync: vi.fn(async () => ({
+    rbxlPath: '/tmp/batch.rbxl',
+    slugMap: SLUG_MAP,
+    primaryTarget: { placeId: 1, universeId: 2 },
+    buildContext: { cleanupAsync: async () => {} },
+  })),
+}));
+
+/**
+ * The summary the Luau runner prints last and also returns.
+ *
+ * `counts` are present for both packages: the runner reads them off what the
+ * test script returned, or off what NevermoreTestRunnerUtils left in its state
+ * module when the script returned nothing.
+ */
+const SUMMARY_ENTRIES = [
+  {
+    slug: 'alpha',
+    success: true,
+    durationMs: 1500,
+    counts: {
+      passed: 68,
+      failed: 0,
+      skipped: 0,
+      total: 68,
+      suitesPassed: 4,
+      suitesFailed: 0,
+      suitesTotal: 4,
+    },
+    ranJest: true,
+  },
+  {
+    slug: 'beta',
+    success: false,
+    durationMs: 900,
+    error: 'beta had a failing suite',
+    counts: {
+      passed: 3,
+      failed: 1,
+      skipped: 0,
+      total: 4,
+      suitesPassed: 0,
+      suitesFailed: 1,
+      suitesTotal: 1,
+    },
+    ranJest: true,
+  },
+];
+
+const SUMMARY_JSON = JSON.stringify(SUMMARY_ENTRIES);
+
+/**
+ * What the task printed, in the shape Open Cloud delivers it: typed messages,
+ * one of which carries a whole traceback, and none of which are guaranteed to
+ * fall inside a package's section.
+ */
+const TASK_MESSAGES: TaskLogMessage[] = [
+  { message: '[BatchTest] running 2 packages', messageType: 'OUTPUT' },
+  { message: '===BATCH_TEST_BEGIN alpha===', messageType: 'OUTPUT' },
+  {
+    message: 'PASS ServerScriptService.alpha.Thing.spec',
+    messageType: 'OUTPUT',
+  },
+  { message: '===BATCH_TEST_END alpha PASS 1500===', messageType: 'OUTPUT' },
+  {
+    message:
+      "TaskScript:12: a leaked task exploded\nStack Begin\nScript 'Foo', Line 3\nStack End",
+    messageType: 'ERROR',
+  },
+  { message: '===BATCH_TEST_BEGIN beta===', messageType: 'OUTPUT' },
+  {
+    message: '[BatchTest] beta: beta had a failing suite',
+    messageType: 'WARNING',
+  },
+  { message: '===BATCH_TEST_END beta FAIL 900===', messageType: 'OUTPUT' },
+  { message: '===BATCH_TEST_SUMMARY===', messageType: 'OUTPUT' },
+  { message: SUMMARY_JSON, messageType: 'OUTPUT' },
+];
+
+/** A transport that runs nothing and reports the messages it was handed. */
+class FakeTransport implements JobContext {
+  public scriptContent?: string;
+  public deployCount = 0;
+  public runCount = 0;
+
+  constructor(
+    private readonly _messages: TaskLogMessage[],
+    private readonly _returnValues?: unknown[]
+  ) {}
+
+  async buildPlaceAsync(): Promise<BuiltPlace> {
+    throw new Error('the batch context builds the combined place itself');
+  }
+
+  async deployBuiltPlaceAsync(): Promise<Deployment> {
+    this.deployCount++;
+    return {};
+  }
+
+  async runScriptAsync(
+    _deployment: Deployment,
+    options: RunScriptOptions
+  ): Promise<ScriptRunResult> {
+    this.runCount++;
+    this.scriptContent = options.scriptContent;
+    return {
+      success: true,
+      taskState: 'COMPLETE',
+      returnValues: this._returnValues,
+    };
+  }
+
+  async getLogsAsync(): Promise<JobLogs> {
+    const text = this._messages.map((m) => m.message).join('\n');
+    return {
+      text,
+      messages: this._messages,
+      stats: {
+        requests: 1,
+        pages: 1,
+        entries: 1,
+        messages: this._messages.length,
+        chars: text.length,
+      },
+    };
+  }
+
+  async releaseAsync(): Promise<void> {}
+  async releaseBuiltPlaceAsync(): Promise<void> {}
+  async disposeAsync(): Promise<void> {}
+}
+
+function createBatch(
+  messages: TaskLogMessage[] = TASK_MESSAGES,
+  returnValues?: unknown[]
+) {
+  const targets = [...SLUG_MAP.keys()].map((name) => ({
+    name,
+    packageName: name,
+    path: `/repo/src/${name.split('/')[1]}`,
+    target: { placeId: 1, universeId: 2 },
+    places: [],
+  })) as unknown as BatchTarget[];
+
+  const transport = new FakeTransport(messages, returnValues);
+  const batch = new BatchScriptJobContext(transport, targets, {
+    repoRoot: '/repo',
+  });
+
+  return { batch, transport, targets };
+}
+
+/** What the per-package loop asks the context for, for one package. */
+async function runPackageAsync(
+  batch: BatchScriptJobContext,
+  packageName: string
+): Promise<ScriptRunResult> {
+  const deployment = await batch.deployBuiltPlaceAsync({
+    builtPlace: { rbxlPath: '/tmp/batch.rbxl', target: {} } as BuiltPlace,
+    packageName,
+    packagePath: '/repo',
+  });
+
+  return batch.runScriptAsync(deployment, {
+    scriptContent: '',
+    packageName,
+  });
+}
+
+describe('an aggregated batch run, end to end', () => {
+  it('renders the whole run: every line, in order, grouped and titled', async () => {
+    const { batch } = createBatch();
+
+    const outcome = await batch.getExecutionOutcomeAsync();
+    const rendered = renderBatchOutcome(outcome, {
+      useGroups: true,
+      color: false,
+    });
+
+    expect(rendered).toEqual([
+      // Before any package began, so it belongs to none of them.
+      '[BatchTest] running 2 packages',
+      '::group::@quenty/alpha - ✅ Passed (68/68) (1.5s)',
+      'PASS ServerScriptService.alpha.Thing.spec',
+      '::endgroup::',
+      // Between two packages. This is the output the parser used to discard,
+      // and the only trace of a crash that belongs to no section.
+      'TaskScript:12: a leaked task exploded',
+      'Stack Begin',
+      "Script 'Foo', Line 3",
+      'Stack End',
+      '::group::@quenty/beta - ❌ FAILED (3/4) (900ms)',
+      '[BatchTest] beta: beta had a failing suite',
+      'beta had a failing suite',
+      '::endgroup::',
+      // The summary marker and its payload are framing, not output.
+    ]);
+  });
+
+  it('colors by the severity the transport reported, jest output untouched', async () => {
+    const { batch } = createBatch();
+
+    const outcome = await batch.getExecutionOutcomeAsync();
+    const rendered = renderBatchOutcome(outcome, {
+      useGroups: false,
+      color: true,
+    });
+
+    expect(rendered).toContain(
+      OutputHelper.formatError('TaskScript:12: a leaked task exploded')
+    );
+    expect(rendered).toContain(
+      OutputHelper.formatWarning('[BatchTest] beta: beta had a failing suite')
+    );
+    expect(rendered).toContain('PASS ServerScriptService.alpha.Thing.spec');
+  });
+
+  it('gives each package its own verdict, counts and duration', async () => {
+    const { batch } = createBatch();
+
+    const alpha = await runPackageAsync(batch, '@quenty/alpha');
+    const beta = await runPackageAsync(batch, '@quenty/beta');
+
+    expect(alpha.success).toBe(true);
+    expect(alpha.durationMs).toBe(1500);
+    expect(alpha.testResults).toMatchObject({
+      passed: 68,
+      failed: 0,
+      total: 68,
+    });
+
+    expect(beta.success).toBe(false);
+    expect(beta.durationMs).toBe(900);
+    expect(beta.errorMessage).toContain('beta had a failing suite');
+    expect(beta.testResults).toMatchObject({ passed: 3, failed: 1, total: 4 });
+  });
+
+  it('builds, uploads and executes once for the whole batch', async () => {
+    const { batch, transport } = createBatch();
+
+    await batch.getExecutionOutcomeAsync();
+    await runPackageAsync(batch, '@quenty/alpha');
+    await runPackageAsync(batch, '@quenty/beta');
+
+    expect(transport.deployCount).toBe(1);
+    expect(transport.runCount).toBe(1);
+    // The script the packages ran is the template with this batch's slugs in
+    // it — the one substitution the context makes, and nothing runs if it is
+    // wrong.
+    expect(transport.scriptContent).toContain('["alpha","beta"]');
+    expect(transport.scriptContent).not.toContain('{{ PACKAGE_SLUGS_JSON }}');
+  });
+
+  it('judges a package whose whole section the log window dropped', async () => {
+    // The case this path exists for. beta printed nothing that survived — no
+    // BEGIN, no END, no jest report — so before the summary came back as a
+    // return value it was failed for having no attributable output, and
+    // rendered nothing at all: no group, no verdict, no reason.
+    const { batch } = createBatch(
+      [
+        { message: '===BATCH_TEST_BEGIN alpha===', messageType: 'OUTPUT' },
+        {
+          message: 'PASS ServerScriptService.alpha.Thing.spec',
+          messageType: 'OUTPUT',
+        },
+        {
+          message: '===BATCH_TEST_END alpha PASS 1500===',
+          messageType: 'OUTPUT',
+        },
+      ],
+      [SUMMARY_ENTRIES]
+    );
+
+    const outcome = await batch.getExecutionOutcomeAsync();
+    const beta = outcome.results.get('@quenty/beta');
+
+    expect(beta?.testCounts).toEqual({ passed: 3, failed: 1, total: 4 });
+    expect(beta?.countsSource).toBe('returned');
+    expect(beta?.logsLost).toBe(true);
+    // Not failed for the absence of a section, which is what used to happen.
+    expect(beta?.error).not.toContain('no output could be attributed');
+
+    const rendered = renderBatchOutcome(outcome, {
+      useGroups: true,
+      color: false,
+    });
+
+    expect(rendered).toContain(
+      '::group::@quenty/beta - ❌ FAILED (3/4) - logs lost (900ms)'
+    );
+    expect(rendered.join('\n')).toContain('survived the run');
+  });
+
+  it('marks a pass judged on counts alone as one, rather than an ordinary pass', async () => {
+    // Trustworthy counts, but nothing to check for tracebacks against and
+    // nothing to read if someone goes looking, so it does not get to look like
+    // every other green package in the list.
+    const { batch } = createBatch(
+      [{ message: 'nothing survived', messageType: 'OUTPUT' }],
+      [
+        [
+          {
+            slug: 'alpha',
+            success: true,
+            durationMs: 1200,
+            counts: {
+              passed: 35,
+              failed: 0,
+              skipped: 0,
+              total: 35,
+              suitesPassed: 2,
+              suitesFailed: 0,
+              suitesTotal: 2,
+            },
+            ranJest: true,
+          },
+        ],
+      ]
+    );
+
+    const outcome = await batch.getExecutionOutcomeAsync();
+
+    expect(outcome.results.get('@quenty/alpha')?.success).toBe(true);
+    expect(
+      renderBatchOutcome(outcome, { useGroups: true, color: false })
+    ).toContain(
+      '::group::@quenty/alpha - ⚠️ Passed (35/35) - logs lost (1.2s)'
+    );
+  });
+
+  it('prefers the returned summary over the printed one', async () => {
+    // Both channels carry the same summary. The returned one is read because it
+    // is the copy the engine's log buffer cannot truncate — so when they
+    // disagree, which only happens if the printed one arrived damaged, the
+    // returned one wins.
+    const { batch } = createBatch(TASK_MESSAGES, [
+      [
+        {
+          slug: 'alpha',
+          success: true,
+          durationMs: 4200,
+          counts: {
+            passed: 99,
+            failed: 0,
+            skipped: 0,
+            total: 99,
+            suitesPassed: 1,
+            suitesFailed: 0,
+            suitesTotal: 1,
+          },
+          ranJest: true,
+        },
+      ],
+    ]);
+
+    const alpha = (await batch.getExecutionOutcomeAsync()).results.get(
+      '@quenty/alpha'
+    );
+
+    expect(alpha?.testCounts?.total).toBe(99);
+    expect(alpha?.durationMs).toBe(4200);
+  });
+
+  it('reports what reading the log turned up without printing it', async () => {
+    // A log window that dropped everything but the summary: the run is still
+    // judged, and the shortfall is handed back as a diagnostic for the command
+    // to place after the output rather than printed from inside the transport.
+    const { batch } = createBatch([
+      { message: '===BATCH_TEST_SUMMARY===', messageType: 'OUTPUT' },
+      { message: SUMMARY_JSON, messageType: 'OUTPUT' },
+    ]);
+
+    const outcome = await batch.getExecutionOutcomeAsync();
+
+    expect(outcome.diagnostics.length).toBeGreaterThan(0);
+    expect(outcome.diagnostics.map((d) => d.message).join('\n')).toContain(
+      'could not attribute any of it to a package section'
+    );
+    // Still counted, because the counts came back in the summary rather than
+    // being scraped out of the section that went missing.
+    expect(outcome.results.get('@quenty/alpha')?.testCounts?.total).toBe(68);
+  });
+});

package/src/utils/testing/batch-test-runner-template.test.ts

@@ -0,0 +1,62 @@
+/**
+ * Guards the two names the batch runner template spells out by hand.
+ *
+ * The template is raw Luau with no loader, so it cannot import anything: it
+ * recognizes a results table by a format string and finds the runner's state by
+ * a module name, both written as literals. Nothing else connects those literals
+ * to the Luau they describe — rename either side and the batch silently loses
+ * every package's counts, which looks exactly like a truncated log.
+ */
+
+import { describe, it, expect } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import { resolvePackagePath } from '@quenty/nevermore-template-helpers';
+
+const TEMPLATE = fs.readFileSync(
+  resolvePackagePath(import.meta.url, 'templates', 'batch-test-runner.lua'),
+  'utf-8'
+);
+
+/** The nevermore-test-runner package this template is written against. */
+const RUNNER_DIR = path.resolve(
+  resolvePackagePath(import.meta.url, '..', '..'),
+  'src',
+  'nevermore-test-runner',
+  'src',
+  'Server'
+);
+
+function literal(name: string): string {
+  const match = new RegExp(`local ${name} = "([^"]+)"`).exec(TEMPLATE);
+  expect(match, `${name} is not declared in the template`).toBeTruthy();
+  return match![1];
+}
+
+describe('batch-test-runner.lua', () => {
+  it('names a state module that exists in the test runner package', () => {
+    const moduleName = literal('STATE_MODULE_NAME');
+
+    expect(
+      fs.existsSync(
+        path.join(RUNNER_DIR, 'NevermoreTestRunnerUtils', `${moduleName}.lua`)
+      )
+    ).toBe(true);
+  });
+
+  it('agrees with the test runner about the results format tag', () => {
+    const results = fs.readFileSync(
+      path.join(RUNNER_DIR, 'NevermoreTestResults.lua'),
+      'utf-8'
+    );
+
+    expect(results).toContain(`local FORMAT = "${literal('RESULTS_FORMAT')}"`);
+  });
+
+  it('returns its summary as well as printing it', () => {
+    // The return value is the copy the engine's log buffer cannot truncate.
+    // Losing it would leave the printed summary as the only channel, which is
+    // the failure the whole path exists to survive.
+    expect(TEMPLATE.trimEnd().endsWith('return results')).toBe(true);
+  });
+});

package/src/utils/testing/log-fetch-stats.test.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect } from 'vitest';
+import { describeLogVolume, type LogFetchStats } from './log-fetch-stats.js';
+
+const STATS: LogFetchStats = {
+  requests: 1,
+  pages: 1,
+  entries: 1,
+  messages: 7627,
+  chars: 276989,
+};
+
+describe('describeLogVolume', () => {
+  it('reports lines and chars alongside the shape of the fetch', () => {
+    const description = describeLogVolume('a\nb\nc', STATS);
+
+    expect(description).toContain('5 chars, 3 line(s) received');
+    expect(description).toContain('1 API call(s)');
+    expect(description).toContain('1 page(s)');
+    expect(description).toContain('1 log entry(ies)');
+    expect(description).toContain('7627 message(s)');
+  });
+
+  it('separates entries from messages', () => {
+    // The two are orders of magnitude apart and only one of them is a line
+    // count. A reader that takes "1 entry" for "1 line" concludes the run
+    // printed nothing, when 7627 messages arrived inside that entry.
+    const description = describeLogVolume('', STATS);
+
+    expect(description).toContain('1 log entry(ies), 7627 message(s)');
+  });
+
+  it('counts no lines in empty logs', () => {
+    // Splitting '' yields one empty line, which would report a run that
+    // produced nothing as having produced a line.
+    expect(describeLogVolume('', STATS)).toContain('0 chars, 0 line(s)');
+  });
+
+  it('says the fetch is unmeasured rather than empty when it is unknown', () => {
+    // A local run reads its logs off the Studio bridge, where there is no
+    // request to count. Zeroes there would read as a fetch that returned
+    // nothing, which is a different fault entirely.
+    const description = describeLogVolume('a\nb', undefined);
+
+    expect(description).toContain('3 chars, 2 line(s) received');
+    expect(description).toContain('no fetch stats');
+    expect(description).not.toContain('0 API call(s)');
+  });
+});

package/src/utils/testing/log-fetch-stats.ts

@@ -0,0 +1,55 @@
+/**
+ * What it took to pull a run's logs, recorded so a parser that finds nothing in
+ * them can say where the shortfall happened.
+ *
+ * Output goes missing between a task and the parser at several caps that look
+ * identical from the parser's side — the engine drops messages past a session
+ * buffer it never reports, the API pages them, and a fetch can be retried. A
+ * count of chars alone cannot tell a run that printed little from one whose
+ * output the engine discarded, so the fetch reports its own shape too.
+ */
+export interface LogFetchStats {
+  /**
+   * HTTP GETs issued to the logs endpoint for this fetch, retries included.
+   * Empty logs are retried, so this exceeds `pages` when the first tries came
+   * back with nothing.
+   */
+  requests: number;
+  /** Pages walked via `nextPageToken` on the attempt that produced the logs. */
+  pages: number;
+  /**
+   * `luauExecutionSessionTaskLog` objects across those pages. Each one carries
+   * many messages, so this stays far below the message count and is not a line
+   * count — the API's per-page limit applies to these, not to messages.
+   */
+  entries: number;
+  /** Engine messages inside those entries: one per `print`/`warn` that survived. */
+  messages: number;
+  /** Length of the joined log text handed to the parser. */
+  chars: number;
+}
+
+/**
+ * Describe how much log text arrived and what the fetch had to do to get it,
+ * for a diagnostic that has already decided something is missing.
+ *
+ * `stats` is absent for logs that did not come from Open Cloud — a local run
+ * reads them off the Studio bridge, where there is no request to count — so the
+ * volume that can always be measured is reported separately from the fetch.
+ */
+export function describeLogVolume(
+  rawLogs: string,
+  stats: LogFetchStats | undefined
+): string {
+  const lines = rawLogs.length === 0 ? 0 : rawLogs.split('\n').length;
+  const volume = `${rawLogs.length} chars, ${lines} line(s) received`;
+
+  if (!stats) {
+    return `${volume}; no fetch stats (logs did not come from Open Cloud)`;
+  }
+
+  return (
+    `${volume}; fetched in ${stats.requests} API call(s) over ${stats.pages} ` +
+    `page(s), ${stats.entries} log entry(ies), ${stats.messages} message(s)`
+  );
+}