@quenty/nevermore-cli 4.43.4 → 4.45.0

package/src/utils/open-cloud/open-cloud-logs.characterization.test.ts

@@ -0,0 +1,277 @@
+/**
+ * Characterization tests for how Open Cloud actually delivers a Luau execution
+ * task's logs and return value. Nothing here tests our code: every assertion is
+ * a claim about Roblox's behaviour that our log handling is built on, written
+ * so the claim fails out loud if Roblox changes it.
+ *
+ * These claims decide real things — whether a truncated run is worth
+ * re-fetching, whether `maxPageSize` is worth sending, whether a batch package
+ * with no output was dropped or never ran — and none of them are documented by
+ * Roblox. Prose in a gotchas file cannot notice when it goes stale. This can.
+ *
+ * Disabled by default: it spends real Open Cloud quota. To run it:
+ *
+ *   NEVERMORE_CHARACTERIZATION=1 npm test -- open-cloud-logs
+ *
+ * It reads the key through the same credential store the CLI uses, so
+ * `nevermore auth login` is the only setup. The universe and place default to
+ * the nevermore-test-runner test target and can be pointed elsewhere with
+ * NEVERMORE_CHARACTERIZATION_UNIVERSE / _PLACE.
+ */
+
+import { describe, it, expect, beforeAll } from 'vitest';
+import { loadStoredApiKeyAsync } from '@quenty/nevermore-cli-helpers';
+import {
+  OpenCloudClient,
+  getTaskReturnValues,
+  type LuauTask,
+} from './open-cloud-client.js';
+import { RateLimiter } from './rate-limiter.js';
+
+const ENABLED = process.env.NEVERMORE_CHARACTERIZATION === '1';
+
+/** The nevermore-test-runner test target, which exists to be run against. */
+const UNIVERSE_ID = Number(
+  process.env.NEVERMORE_CHARACTERIZATION_UNIVERSE ?? 9716264427
+);
+const PLACE_ID = Number(
+  process.env.NEVERMORE_CHARACTERIZATION_PLACE ?? 138951894938783
+);
+
+const LINES_THAT_FIT = 5_000;
+const LINES_AT_CAPACITY = 8_000;
+const LINES_THAT_OVERFLOW_LESS = 10_000;
+const LINES_THAT_OVERFLOW = 20_000;
+
+/**
+ * Retention is bounded by memory, and each retained message costs this many
+ * bytes beyond its own text — timestamp, type and framing. Derived by holding
+ * the print count at 20,000 and varying only the payload width, then solving
+ * `chars + k * messages` for the k that makes that budget constant.
+ */
+const PER_MESSAGE_OVERHEAD_BYTES = 47;
+
+/** A printed line that names its own position, so survivors identify themselves. */
+const SEQ = new RegExp('^SEQ (' + String.raw`\d` + '+)');
+
+describe.runIf(ENABLED)('Open Cloud task log delivery', () => {
+  let client: OpenCloudClient;
+  let placeVersion: number;
+
+  beforeAll(async () => {
+    const apiKey = await loadStoredApiKeyAsync();
+    if (!apiKey) {
+      throw new Error(
+        'No stored Open Cloud API key. Run `nevermore auth login` first.'
+      );
+    }
+
+    client = new OpenCloudClient({ apiKey, rateLimiter: new RateLimiter() });
+    placeVersion = await client.resolveLatestPlaceVersionAsync(
+      UNIVERSE_ID,
+      PLACE_ID,
+      'saved'
+    );
+  }, 120_000);
+
+  /**
+   * Run a task printing `lines` numbered messages, each optionally padded to
+   * widen it, and report what came back.
+   */
+  async function printLinesAsync(lines: number, padChars = 0) {
+    const script = padChars
+      ? `local pad = string.rep("p", ${padChars}) for i = 1, ${lines} do print("SEQ " .. i .. " " .. pad) end`
+      : `for i = 1, ${lines} do print("SEQ " .. i) end`;
+
+    const task = await client.createExecutionTaskAsync(
+      UNIVERSE_ID,
+      PLACE_ID,
+      placeVersion,
+      script,
+      300_000
+    );
+    const completed = await client.pollTaskCompletionAsync(task.path);
+    expect(completed.state).toBe('COMPLETE');
+
+    const fetched = await client.getRawTaskLogsAsync(completed.path);
+    const seq = fetched.text
+      .split('\n')
+      .map((line) => SEQ.exec(line.trim())?.[1])
+      .filter((n): n is string => n !== undefined)
+      .map(Number);
+
+    return { task: completed, ...fetched, seq };
+  }
+
+  /** Bytes the buffer spent on what it kept, per-message overhead included. */
+  function retainedBudget(run: { stats: { chars: number; messages: number } }) {
+    return run.stats.chars + PER_MESSAGE_OVERHEAD_BYTES * run.stats.messages;
+  }
+
+  it('delivers a small run complete and in order', async () => {
+    const run = await printLinesAsync(LINES_THAT_FIT);
+
+    expect(run.seq).toHaveLength(LINES_THAT_FIT);
+    expect(run.seq[0]).toBe(1);
+    expect(run.seq[run.seq.length - 1]).toBe(LINES_THAT_FIT);
+    // Arrival order is chronological, so no client-side sort is needed — and
+    // sorting by createTime would be wrong, since a whole frame shares one.
+    expect(run.seq).toEqual([...run.seq].sort((a, b) => a - b));
+  }, 300_000);
+
+  it('truncates a large run by dropping the head, keeping a contiguous tail', async () => {
+    // Says a short log is missing output rather than proof a run printed
+    // little, and rules out the loss being a page we failed to request — an
+    // unrequested page would have held the head. It is why the batch parser
+    // closes sections on their END marker.
+    const run = await printLinesAsync(LINES_THAT_OVERFLOW);
+
+    expect(run.seq.length).toBeLessThan(LINES_THAT_OVERFLOW);
+    expect(run.seq[run.seq.length - 1]).toBe(LINES_THAT_OVERFLOW);
+    expect(run.seq[0]).toBeGreaterThan(1);
+    expect(run.seq).toEqual(
+      Array.from({ length: run.seq.length }, (_, i) => run.seq[0] + i)
+    );
+  }, 300_000);
+
+  it('bounds retention by memory, not by number of print calls', async () => {
+    // Both runs make exactly the same number of print calls. If the buffer held
+    // a fixed number of messages they would retain the same count; the padded
+    // one keeps a small fraction of it, so the budget is bytes.
+    const narrow = await printLinesAsync(LINES_THAT_OVERFLOW);
+    const wide = await printLinesAsync(LINES_THAT_OVERFLOW, 500);
+
+    expect(wide.stats.messages).toBeLessThan(narrow.stats.messages / 5);
+    expect(wide.stats.chars).toBeGreaterThan(narrow.stats.chars);
+
+    // And it is one budget in both: bytes kept, plus a fixed cost per message,
+    // lands on the same number however wide the messages are. ~450 KB.
+    expect(retainedBudget(wide)).toBeGreaterThan(retainedBudget(narrow) * 0.95);
+    expect(retainedBudget(wide)).toBeLessThan(retainedBudget(narrow) * 1.05);
+    expect(retainedBudget(narrow)).toBeGreaterThan(400_000);
+    expect(retainedBudget(narrow)).toBeLessThan(500_000);
+  }, 300_000);
+
+  it('keeps the same amount however much is printed, at one message width', async () => {
+    // The corollary: at a fixed message size, printing twice as much does not
+    // get twice as much back. A long run cannot be made to report itself by
+    // printing more carefully, which is why counts travel in the return value.
+    const atCapacity = await printLinesAsync(LINES_AT_CAPACITY);
+    expect(atCapacity.seq).toHaveLength(LINES_AT_CAPACITY);
+
+    const over = await printLinesAsync(LINES_THAT_OVERFLOW_LESS);
+    const wayOver = await printLinesAsync(LINES_THAT_OVERFLOW);
+
+    expect(over.seq.length).toBeLessThan(LINES_THAT_OVERFLOW_LESS);
+    expect(wayOver.seq.length).toBeLessThan(LINES_THAT_OVERFLOW);
+    expect(wayOver.seq.length).toBeGreaterThan(over.seq.length * 0.9);
+    expect(wayOver.seq.length).toBeLessThan(over.seq.length * 1.1);
+  }, 300_000);
+
+  it('returns only what the task printed, with nothing of its own mixed in', async () => {
+    // Rules out engine or internal messages taking room in the buffer: every
+    // message that came back is one of ours, so a retained count short of a
+    // round number is the byte budget, not foreign output.
+    const run = await printLinesAsync(LINES_THAT_OVERFLOW);
+
+    expect(run.seq).toHaveLength(run.stats.messages);
+  }, 300_000);
+
+  it('delivers the same messages structured or flat, and never a second page', async () => {
+    // Why we send no maxPageSize, and why the view is a free choice: asking for
+    // 10 still returns the whole window in one page. A page size that counted
+    // messages would cap this at 10 and hand back a token for the rest.
+    const run = await printLinesAsync(LINES_THAT_OVERFLOW);
+
+    const structured = await fetchOnePageAsync(run.task.path, {
+      view: 'STRUCTURED',
+    });
+    const flat = await fetchOnePageAsync(run.task.path, {});
+
+    // Same messages in the same order — the view changes the envelope, not the
+    // delivery.
+    expect(flat.messages).toEqual(structured.messages);
+    expect(structured.messages).toHaveLength(run.stats.messages);
+
+    const variants: Array<Record<string, string>> = [
+      { view: 'STRUCTURED', maxPageSize: '10' },
+      { view: 'STRUCTURED', maxPageSize: '100' },
+      { maxPageSize: '10' },
+    ];
+
+    for (const params of variants) {
+      const page = await fetchOnePageAsync(run.task.path, params);
+      expect(page.nextPageToken).toBeUndefined();
+      expect(page.messages).toEqual(structured.messages);
+    }
+  }, 300_000);
+
+  it('delivers a large return value, but drops the task entirely past a limit', async () => {
+    // Why the returned failure list is capped rather than trusted to arrive: an
+    // oversize return value does not come back truncated, it takes the whole
+    // run with it — and such a task carries no output and no message saying so,
+    // which is why "nothing came back" must stay distinguishable from "the
+    // script returned nothing".
+    const twoMegabytes = await returnStringAsync(2 * 1024 * 1024);
+    expect(twoMegabytes.state).toBe('COMPLETE');
+    expect(getTaskReturnValues(twoMegabytes)).toHaveLength(1);
+
+    const fiveMegabytes = await returnStringAsync(5 * 1024 * 1024);
+    expect(fiveMegabytes.state).not.toBe('COMPLETE');
+    expect(getTaskReturnValues(fiveMegabytes)).toBeUndefined();
+  }, 300_000);
+
+  /** Run a task returning a string of `bytes`, and wait for whatever happens. */
+  async function returnStringAsync(bytes: number): Promise<LuauTask> {
+    const task = await client.createExecutionTaskAsync(
+      UNIVERSE_ID,
+      PLACE_ID,
+      placeVersion,
+      `return string.rep("x", ${bytes})`,
+      300_000
+    );
+
+    // Returns on any terminal state, failure included — which is one of the
+    // outcomes under test here, not an error.
+    return client.pollTaskCompletionAsync(task.path);
+  }
+
+  /**
+   * One raw request, bypassing the client so the query can be varied. Returns
+   * only what the claims above are about.
+   */
+  async function fetchOnePageAsync(
+    taskPath: string,
+    params: Record<string, string>
+  ): Promise<{ nextPageToken?: string; messages: string[] }> {
+    const apiKey = (await loadStoredApiKeyAsync())!;
+    const url = new URL(`https://apis.roblox.com/cloud/v2/${taskPath}/logs`);
+    for (const [key, value] of Object.entries(params)) {
+      url.searchParams.set(key, value);
+    }
+
+    const response = await fetch(url.toString(), {
+      headers: { 'X-API-Key': apiKey },
+    });
+    expect(response.ok).toBe(true);
+
+    const data = (await response.json()) as {
+      luauExecutionSessionTaskLogs?: Array<{
+        messages?: string[];
+        structuredMessages?: Array<{ message: string }>;
+      }>;
+      nextPageToken?: string;
+    };
+
+    const messages: string[] = [];
+    for (const entry of data.luauExecutionSessionTaskLogs ?? []) {
+      if (entry.structuredMessages?.length) {
+        messages.push(...entry.structuredMessages.map((m) => m.message));
+      } else if (entry.messages?.length) {
+        messages.push(...entry.messages);
+      }
+    }
+
+    return { nextPageToken: data.nextPageToken || undefined, messages };
+  }
+});

package/src/utils/package-manager-utils.test.ts

@@ -106,6 +106,33 @@ describe('package-manager-utils', () => {
       expect(await detectPackageManagerAsync(root)).toBe('pnpm');
     });
 
+    it('ignores a lockfile sitting in the home directory', async () => {
+      // npm leaves a package-lock.json behind after one `npm install` run in a
+      // home directory, and walking to the filesystem root adopted it — every
+      // project below it then installed with the one tool that rejects this
+      // repo's `workspace:` ranges. This test was the symptom: the temp dir
+      // lives under the home directory on Windows, so "defaults to pnpm" here
+      // failed on a machine that had one and passed everywhere else.
+      const home = await fs.realpath(os.homedir());
+      const marker = path.join(home, 'package-lock.json');
+      const alreadyThere = await fs
+        .access(marker)
+        .then(() => true)
+        .catch(() => false);
+
+      if (!alreadyThere) {
+        await fs.writeFile(marker, '{}');
+      }
+
+      try {
+        expect(await detectPackageManagerAsync(root)).toBe('pnpm');
+      } finally {
+        if (!alreadyThere) {
+          await fs.rm(marker, { force: true });
+        }
+      }
+    });
+
     it('still honors an npm project', async () => {
       await writeFileAsync('package.json', JSON.stringify({ name: 'game' }));
       await writeFileAsync('package-lock.json', '{}');

package/src/utils/package-manager-utils.ts

@@ -4,6 +4,7 @@
  */
 
 import * as fs from 'fs/promises';
+import * as os from 'os';
 import * as path from 'path';
 
 export const PACKAGE_MANAGERS = ['pnpm', 'npm', 'yarn', 'bun'] as const;
@@ -80,9 +81,20 @@ async function detectInDirectoryAsync(
 export async function detectPackageManagerAsync(
   startDirectory: string
 ): Promise<PackageManager> {
-  let directory = path.resolve(startDirectory);
+  let directory = await resolveRealPathAsync(path.resolve(startDirectory));
+  // Walking to the filesystem root adopts lockfiles that belong to nobody. A
+  // stray package-lock.json in a home directory — npm leaves one behind after a
+  // single `npm install` run there — made every project below it detect as npm,
+  // and in this monorepo that means installing with the one tool that rejects
+  // `workspace:` ranges outright. Nothing above the home directory can be part
+  // of the project, so the search stops there.
+  const boundary = await resolveRealPathAsync(os.homedir());
 
   for (;;) {
+    if (boundary !== undefined && isSamePath(directory, boundary)) {
+      break;
+    }
+
     const detected = await detectInDirectoryAsync(directory);
     if (detected) {
       return detected;
@@ -98,6 +110,27 @@ export async function detectPackageManagerAsync(
   return DEFAULT_PACKAGE_MANAGER;
 }
 
+/**
+ * Resolve symlinks and, on Windows, 8.3 short names — `C:\Users\JAMESO~1` and
+ * `C:\Users\James Onnen` are the same directory and must compare equal.
+ * Returns the input unchanged when the path does not exist.
+ */
+async function resolveRealPathAsync(target: string): Promise<string> {
+  try {
+    return await fs.realpath(target);
+  } catch {
+    return target;
+  }
+}
+
+/** Windows paths are case-insensitive; POSIX ones are not. */
+function isSamePath(left: string, right: string): boolean {
+  if (process.platform === 'win32') {
+    return left.toLowerCase() === right.toLowerCase();
+  }
+  return left === right;
+}
+
 /**
  * Builds the command that adds dependencies to a project. Only npm spells this
  * `install`; the rest treat a bare `install` as "install the lockfile" and need