@quenty/nevermore-cli 4.43.4 → 4.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/build-scripts/{combine-test-places.luau → combine-test-places.lua} +3 -3
- package/build-scripts/{transform-inject-deploy-metadata.luau → transform-inject-deploy-metadata.lua} +3 -3
- package/dist/commands/batch-command/batch-deploy-command.js +4 -4
- package/dist/commands/batch-command/batch-deploy-command.js.map +1 -1
- package/dist/commands/batch-command/batch-test-command.d.ts.map +1 -1
- package/dist/commands/batch-command/batch-test-command.js +76 -19
- package/dist/commands/batch-command/batch-test-command.js.map +1 -1
- package/dist/commands/deploy-command/deploy-init-utils.d.ts.map +1 -1
- package/dist/commands/deploy-command/deploy-init-utils.js +7 -1
- package/dist/commands/deploy-command/deploy-init-utils.js.map +1 -1
- package/dist/commands/deploy-command/index.js +1 -1
- package/dist/commands/deploy-command/index.js.map +1 -1
- package/dist/utils/deploy/deploy-github-columns.d.ts +12 -2
- package/dist/utils/deploy/deploy-github-columns.d.ts.map +1 -1
- package/dist/utils/deploy/deploy-github-columns.js +11 -4
- package/dist/utils/deploy/deploy-github-columns.js.map +1 -1
- package/dist/utils/deploy/deploy-metadata.js +1 -1
- package/dist/utils/deploy/deploy-metadata.js.map +1 -1
- package/dist/utils/job-context/base-job-context.d.ts +2 -2
- package/dist/utils/job-context/base-job-context.d.ts.map +1 -1
- package/dist/utils/job-context/base-job-context.js +1 -1
- package/dist/utils/job-context/base-job-context.js.map +1 -1
- package/dist/utils/job-context/batch-script-job-context.d.ts +37 -2
- package/dist/utils/job-context/batch-script-job-context.d.ts.map +1 -1
- package/dist/utils/job-context/batch-script-job-context.js +59 -22
- package/dist/utils/job-context/batch-script-job-context.js.map +1 -1
- package/dist/utils/job-context/cloud-job-context.d.ts +2 -2
- package/dist/utils/job-context/cloud-job-context.d.ts.map +1 -1
- package/dist/utils/job-context/cloud-job-context.js +10 -5
- package/dist/utils/job-context/cloud-job-context.js.map +1 -1
- package/dist/utils/job-context/cloud-job-context.test.d.ts +7 -0
- package/dist/utils/job-context/cloud-job-context.test.d.ts.map +1 -0
- package/dist/utils/job-context/cloud-job-context.test.js +91 -0
- package/dist/utils/job-context/cloud-job-context.test.js.map +1 -0
- package/dist/utils/job-context/job-context.d.ts +70 -2
- package/dist/utils/job-context/job-context.d.ts.map +1 -1
- package/dist/utils/job-context/local-job-context.d.ts +2 -2
- package/dist/utils/job-context/local-job-context.d.ts.map +1 -1
- package/dist/utils/job-context/local-job-context.js +4 -2
- package/dist/utils/job-context/local-job-context.js.map +1 -1
- package/dist/utils/job-context/local-job-context.test.d.ts +7 -0
- package/dist/utils/job-context/local-job-context.test.d.ts.map +1 -0
- package/dist/utils/job-context/local-job-context.test.js +68 -0
- package/dist/utils/job-context/local-job-context.test.js.map +1 -0
- package/dist/utils/open-cloud/open-cloud-client.d.ts +50 -7
- package/dist/utils/open-cloud/open-cloud-client.d.ts.map +1 -1
- package/dist/utils/open-cloud/open-cloud-client.js +49 -13
- package/dist/utils/open-cloud/open-cloud-client.js.map +1 -1
- package/dist/utils/open-cloud/open-cloud-client.test.js +125 -1
- package/dist/utils/open-cloud/open-cloud-client.test.js.map +1 -1
- package/dist/utils/open-cloud/open-cloud-logs.characterization.test.d.ts +22 -0
- package/dist/utils/open-cloud/open-cloud-logs.characterization.test.d.ts.map +1 -0
- package/dist/utils/open-cloud/open-cloud-logs.characterization.test.js +202 -0
- package/dist/utils/open-cloud/open-cloud-logs.characterization.test.js.map +1 -0
- package/dist/utils/package-manager-utils.d.ts.map +1 -1
- package/dist/utils/package-manager-utils.js +32 -1
- package/dist/utils/package-manager-utils.js.map +1 -1
- package/dist/utils/package-manager-utils.test.js +25 -0
- package/dist/utils/package-manager-utils.test.js.map +1 -1
- package/dist/utils/testing/batch-run.e2e.test.d.ts +17 -0
- package/dist/utils/testing/batch-run.e2e.test.d.ts.map +1 -0
- package/dist/utils/testing/batch-run.e2e.test.js +343 -0
- package/dist/utils/testing/batch-run.e2e.test.js.map +1 -0
- package/dist/utils/testing/batch-test-runner-template.test.d.ts +11 -0
- package/dist/utils/testing/batch-test-runner-template.test.d.ts.map +1 -0
- package/dist/utils/testing/batch-test-runner-template.test.js +38 -0
- package/dist/utils/testing/batch-test-runner-template.test.js.map +1 -0
- package/dist/utils/testing/log-fetch-stats.d.ts +40 -0
- package/dist/utils/testing/log-fetch-stats.d.ts.map +1 -0
- package/dist/utils/testing/log-fetch-stats.js +18 -0
- package/dist/utils/testing/log-fetch-stats.js.map +1 -0
- package/dist/utils/testing/log-fetch-stats.test.d.ts +2 -0
- package/dist/utils/testing/log-fetch-stats.test.d.ts.map +1 -0
- package/dist/utils/testing/log-fetch-stats.test.js +41 -0
- package/dist/utils/testing/log-fetch-stats.test.js.map +1 -0
- package/dist/utils/testing/parsers/batch-log-parser.d.ts +126 -1
- package/dist/utils/testing/parsers/batch-log-parser.d.ts.map +1 -1
- package/dist/utils/testing/parsers/batch-log-parser.js +440 -103
- package/dist/utils/testing/parsers/batch-log-parser.js.map +1 -1
- package/dist/utils/testing/parsers/batch-log-parser.test.js +494 -1
- package/dist/utils/testing/parsers/batch-log-parser.test.js.map +1 -1
- package/dist/utils/testing/parsers/batch-log-renderer.d.ts +79 -0
- package/dist/utils/testing/parsers/batch-log-renderer.d.ts.map +1 -0
- package/dist/utils/testing/parsers/batch-log-renderer.js +241 -0
- package/dist/utils/testing/parsers/batch-log-renderer.js.map +1 -0
- package/dist/utils/testing/parsers/batch-log-renderer.test.d.ts +2 -0
- package/dist/utils/testing/parsers/batch-log-renderer.test.d.ts.map +1 -0
- package/dist/utils/testing/parsers/batch-log-renderer.test.js +202 -0
- package/dist/utils/testing/parsers/batch-log-renderer.test.js.map +1 -0
- package/dist/utils/testing/runner/combined-project-generator.js +1 -1
- package/dist/utils/testing/runner/combined-project-generator.js.map +1 -1
- package/dist/utils/testing/runner/test-runner.d.ts +24 -0
- package/dist/utils/testing/runner/test-runner.d.ts.map +1 -1
- package/dist/utils/testing/runner/test-runner.js +166 -6
- package/dist/utils/testing/runner/test-runner.js.map +1 -1
- package/dist/utils/testing/runner/test-runner.test.js +228 -2
- package/dist/utils/testing/runner/test-runner.test.js.map +1 -1
- package/dist/utils/testing/structured-test-results.d.ts +77 -0
- package/dist/utils/testing/structured-test-results.d.ts.map +1 -0
- package/dist/utils/testing/structured-test-results.js +140 -0
- package/dist/utils/testing/structured-test-results.js.map +1 -0
- package/dist/utils/testing/structured-test-results.test.d.ts +7 -0
- package/dist/utils/testing/structured-test-results.test.d.ts.map +1 -0
- package/dist/utils/testing/structured-test-results.test.js +179 -0
- package/dist/utils/testing/structured-test-results.test.js.map +1 -0
- package/package.json +6 -6
- package/src/commands/batch-command/batch-deploy-command.ts +4 -4
- package/src/commands/batch-command/batch-test-command.ts +94 -19
- package/src/commands/deploy-command/deploy-init-utils.ts +7 -1
- package/src/commands/deploy-command/index.ts +1 -1
- package/src/utils/deploy/deploy-github-columns.ts +14 -4
- package/src/utils/deploy/deploy-metadata.ts +1 -1
- package/src/utils/job-context/base-job-context.ts +3 -2
- package/src/utils/job-context/batch-script-job-context.ts +98 -17
- package/src/utils/job-context/cloud-job-context.test.ts +112 -0
- package/src/utils/job-context/cloud-job-context.ts +12 -6
- package/src/utils/job-context/job-context.ts +71 -2
- package/src/utils/job-context/local-job-context.test.ts +89 -0
- package/src/utils/job-context/local-job-context.ts +7 -3
- package/src/utils/open-cloud/open-cloud-client.test.ts +155 -1
- package/src/utils/open-cloud/open-cloud-client.ts +94 -18
- package/src/utils/open-cloud/open-cloud-logs.characterization.test.ts +277 -0
- package/src/utils/package-manager-utils.test.ts +27 -0
- package/src/utils/package-manager-utils.ts +34 -1
- package/src/utils/testing/batch-run.e2e.test.ts +425 -0
- package/src/utils/testing/batch-test-runner-template.test.ts +62 -0
- package/src/utils/testing/log-fetch-stats.test.ts +48 -0
- package/src/utils/testing/log-fetch-stats.ts +55 -0
- package/src/utils/testing/parsers/batch-log-parser.test.ts +660 -1
- package/src/utils/testing/parsers/batch-log-parser.ts +721 -116
- package/src/utils/testing/parsers/batch-log-renderer.test.ts +330 -0
- package/src/utils/testing/parsers/batch-log-renderer.ts +329 -0
- package/src/utils/testing/runner/combined-project-generator.ts +1 -1
- package/src/utils/testing/runner/test-runner.test.ts +332 -2
- package/src/utils/testing/runner/test-runner.ts +231 -9
- package/src/utils/testing/structured-test-results.test.ts +267 -0
- package/src/utils/testing/structured-test-results.ts +199 -0
- package/templates/batch-test-runner.lua +242 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/templates/batch-test-runner.luau +0 -137
- /package/build-scripts/{smoke-test-server.luau → smoke-test-server.lua} +0 -0
- /package/build-scripts/{transform-rojo-merge-place.luau → transform-rojo-merge-place.lua} +0 -0
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
2
|
import * as fs from 'fs/promises';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
OpenCloudClient,
|
|
5
|
+
getTaskReturnValues,
|
|
6
|
+
type LuauTask,
|
|
7
|
+
} from './open-cloud-client.js';
|
|
4
8
|
import type { RateLimiter } from './rate-limiter.js';
|
|
5
9
|
|
|
6
10
|
vi.mock('@quenty/cli-output-helpers', () => ({
|
|
@@ -367,3 +371,153 @@ describe('OpenCloudClient.resolveLatestPlaceVersionAsync', () => {
|
|
|
367
371
|
).rejects.toThrowError(/unparseable version path/);
|
|
368
372
|
});
|
|
369
373
|
});
|
|
374
|
+
|
|
375
|
+
describe('getTaskReturnValues', () => {
|
|
376
|
+
function makeTask(overrides: Partial<LuauTask>): LuauTask {
|
|
377
|
+
return {
|
|
378
|
+
path: 'universes/1/places/2/versions/3/luau-execution-session-tasks/4',
|
|
379
|
+
createTime: '2026-01-01T00:00:00Z',
|
|
380
|
+
updateTime: '2026-01-01T00:01:00Z',
|
|
381
|
+
user: 'users/1',
|
|
382
|
+
state: 'COMPLETE',
|
|
383
|
+
script: 'return 1',
|
|
384
|
+
...overrides,
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
it('returns the values natively typed, one entry per returned value', () => {
|
|
389
|
+
// Roblox serializes the return value itself, so a returned table arrives as
|
|
390
|
+
// real nested JSON — there is no JSON string to parse a second time.
|
|
391
|
+
const task = makeTask({
|
|
392
|
+
output: {
|
|
393
|
+
results: [{ slug: 'maid', counts: { passed: 1014 } }, 'str', 42, true],
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
expect(getTaskReturnValues(task)).toEqual([
|
|
398
|
+
{ slug: 'maid', counts: { passed: 1014 } },
|
|
399
|
+
'str',
|
|
400
|
+
42,
|
|
401
|
+
true,
|
|
402
|
+
]);
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
it('reports an empty result when the task returned nothing', () => {
|
|
406
|
+
expect(getTaskReturnValues(makeTask({ output: {} }))).toEqual([]);
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
it('reports undefined when a failed task carried no output at all', () => {
|
|
410
|
+
// An oversize return value fails the task with no output and no error
|
|
411
|
+
// message, so "nothing came back to read" has to stay distinguishable from
|
|
412
|
+
// "the script returned nothing" — only the former can fall back to logs.
|
|
413
|
+
const task = makeTask({ state: 'FAILED', output: undefined });
|
|
414
|
+
|
|
415
|
+
expect(getTaskReturnValues(task)).toBeUndefined();
|
|
416
|
+
});
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
describe('OpenCloudClient.getRawTaskLogsAsync', () => {
|
|
420
|
+
afterEach(() => {
|
|
421
|
+
vi.restoreAllMocks();
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Page the logs endpoint the way the real one does: many messages inside each
|
|
426
|
+
* `luauExecutionSessionTaskLog`, and pagination over those entries rather than
|
|
427
|
+
* over messages.
|
|
428
|
+
*/
|
|
429
|
+
function makeLogsLimiter(
|
|
430
|
+
pages: Array<Array<string[]>>,
|
|
431
|
+
urls: string[] = []
|
|
432
|
+
): RateLimiter {
|
|
433
|
+
return {
|
|
434
|
+
fetchAsync: vi.fn(async (url: string | URL) => {
|
|
435
|
+
urls.push(String(url));
|
|
436
|
+
const page = pages[urls.length - 1] ?? [];
|
|
437
|
+
return new Response(
|
|
438
|
+
JSON.stringify({
|
|
439
|
+
luauExecutionSessionTaskLogs: page.map((messages) => ({
|
|
440
|
+
structuredMessages: messages.map((message) => ({
|
|
441
|
+
message,
|
|
442
|
+
createTime: '2026-08-27T00:00:00.38Z',
|
|
443
|
+
messageType: 'OUTPUT',
|
|
444
|
+
})),
|
|
445
|
+
})),
|
|
446
|
+
...(urls.length < pages.length
|
|
447
|
+
? { nextPageToken: `token-${urls.length}` }
|
|
448
|
+
: {}),
|
|
449
|
+
}),
|
|
450
|
+
{ status: 200 }
|
|
451
|
+
);
|
|
452
|
+
}),
|
|
453
|
+
} as unknown as RateLimiter;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
it('counts pages, entries and messages separately', async () => {
|
|
457
|
+
// Entries are not lines: the endpoint's page size applies to the log
|
|
458
|
+
// objects, each of which carries many messages. Conflating the two reads a
|
|
459
|
+
// 2-entry response as a 2-line run.
|
|
460
|
+
const client = new OpenCloudClient({
|
|
461
|
+
apiKey: 'test-key',
|
|
462
|
+
rateLimiter: makeLogsLimiter([[['a', 'b', 'c'], ['d']], [['e', 'f']]]),
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
const fetched = await client.getRawTaskLogsAsync('universes/1/tasks/2');
|
|
466
|
+
|
|
467
|
+
expect(fetched.text).toBe('a\nb\nc\nd\ne\nf');
|
|
468
|
+
expect(fetched.stats).toEqual({
|
|
469
|
+
requests: 2,
|
|
470
|
+
pages: 2,
|
|
471
|
+
entries: 3,
|
|
472
|
+
messages: 6,
|
|
473
|
+
chars: 11,
|
|
474
|
+
});
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
it('counts every attempt when empty logs were retried', async () => {
|
|
478
|
+
// The retry is invisible in the text it eventually returns, so a run whose
|
|
479
|
+
// logs only showed up on the third ask looks exactly like one that answered
|
|
480
|
+
// immediately. It is the difference between a slow API and a silent run.
|
|
481
|
+
vi.spyOn(global, 'setTimeout').mockImplementation((fn: () => void) => {
|
|
482
|
+
fn();
|
|
483
|
+
return 0 as unknown as NodeJS.Timeout;
|
|
484
|
+
});
|
|
485
|
+
|
|
486
|
+
// One unpaged response per attempt: two empty, then the logs. A limiter
|
|
487
|
+
// that pages would fold all three into a single attempt and prove nothing.
|
|
488
|
+
let attempt = 0;
|
|
489
|
+
const rateLimiter = {
|
|
490
|
+
fetchAsync: vi.fn(async () => {
|
|
491
|
+
attempt++;
|
|
492
|
+
return new Response(
|
|
493
|
+
JSON.stringify({
|
|
494
|
+
luauExecutionSessionTaskLogs:
|
|
495
|
+
attempt < 3
|
|
496
|
+
? []
|
|
497
|
+
: [
|
|
498
|
+
{
|
|
499
|
+
structuredMessages: [
|
|
500
|
+
{
|
|
501
|
+
message: 'late',
|
|
502
|
+
createTime: '2026-08-27T00:00:00.38Z',
|
|
503
|
+
messageType: 'OUTPUT',
|
|
504
|
+
},
|
|
505
|
+
],
|
|
506
|
+
},
|
|
507
|
+
],
|
|
508
|
+
}),
|
|
509
|
+
{ status: 200 }
|
|
510
|
+
);
|
|
511
|
+
}),
|
|
512
|
+
} as unknown as RateLimiter;
|
|
513
|
+
|
|
514
|
+
const client = new OpenCloudClient({ apiKey: 'test-key', rateLimiter });
|
|
515
|
+
|
|
516
|
+
const fetched = await client.getRawTaskLogsAsync('universes/1/tasks/2');
|
|
517
|
+
|
|
518
|
+
expect(fetched.text).toBe('late');
|
|
519
|
+
expect(fetched.stats.requests).toBe(3);
|
|
520
|
+
expect(fetched.stats.pages).toBe(1);
|
|
521
|
+
expect(fetched.stats.messages).toBe(1);
|
|
522
|
+
});
|
|
523
|
+
});
|
|
@@ -7,6 +7,30 @@ import {
|
|
|
7
7
|
parseTestLogs,
|
|
8
8
|
type ParsedTestLogs,
|
|
9
9
|
} from '../testing/test-log-parser.js';
|
|
10
|
+
import { type LogFetchStats } from '../testing/log-fetch-stats.js';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* One engine message, with the severity Open Cloud reported for it.
|
|
14
|
+
*
|
|
15
|
+
* A message is not a line: an error arrives with its whole traceback attached,
|
|
16
|
+
* so `message` can span several. `messageType` is OUTPUT, WARNING or ERROR.
|
|
17
|
+
*/
|
|
18
|
+
export interface TaskLogMessage {
|
|
19
|
+
message: string;
|
|
20
|
+
messageType?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** A task's log text together with what the fetch had to do to collect it. */
|
|
24
|
+
export interface TaskLogFetch {
|
|
25
|
+
text: string;
|
|
26
|
+
stats: LogFetchStats;
|
|
27
|
+
/**
|
|
28
|
+
* The same output as `text`, still split into typed messages. Kept because
|
|
29
|
+
* severity is the API's to state and cannot be recovered from the joined
|
|
30
|
+
* text: a traceback's continuation lines look like ordinary output.
|
|
31
|
+
*/
|
|
32
|
+
messages: TaskLogMessage[];
|
|
33
|
+
}
|
|
10
34
|
|
|
11
35
|
export interface LuauTask {
|
|
12
36
|
path: string;
|
|
@@ -22,12 +46,38 @@ export interface LuauTask {
|
|
|
22
46
|
| 'FAILED';
|
|
23
47
|
script: string;
|
|
24
48
|
timeout?: string;
|
|
25
|
-
/**
|
|
26
|
-
|
|
49
|
+
/**
|
|
50
|
+
* What the script returned, populated on COMPLETE. `results` is a flat array
|
|
51
|
+
* of the returned values — `return t, "s", 42` arrives as three entries —
|
|
52
|
+
* natively typed: Roblox serializes them itself, so a returned Lua table is
|
|
53
|
+
* real nested JSON here, not a JSON string. (Which is also why a script must
|
|
54
|
+
* not JSONEncode its result: that lands as a double-encoded string.)
|
|
55
|
+
*
|
|
56
|
+
* Absent whenever the task produced no result. A FAILED task carries no
|
|
57
|
+
* `output` at all, with no error message explaining why — and an oversize
|
|
58
|
+
* return value (~4MB observed; ~2MB still arrives complete) fails the task
|
|
59
|
+
* exactly that way rather than truncating the value.
|
|
60
|
+
*/
|
|
61
|
+
output?: { results?: unknown[] };
|
|
27
62
|
/** Error details (populated on FAILED). */
|
|
28
63
|
error?: { code?: string; message?: string };
|
|
29
64
|
}
|
|
30
65
|
|
|
66
|
+
/**
|
|
67
|
+
* The values a finished task's script returned, or `undefined` when the task
|
|
68
|
+
* reported no result at all.
|
|
69
|
+
*
|
|
70
|
+
* The distinction matters: `undefined` means nothing came back to read, so a
|
|
71
|
+
* caller can fall back to the task's logs, while `[]` means the task did report
|
|
72
|
+
* a result and the script returned nothing — no fallback will find more.
|
|
73
|
+
*/
|
|
74
|
+
export function getTaskReturnValues(task: LuauTask): unknown[] | undefined {
|
|
75
|
+
if (!task.output) {
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
return task.output.results ?? [];
|
|
79
|
+
}
|
|
80
|
+
|
|
31
81
|
export interface OpenCloudClientOptions {
|
|
32
82
|
apiKey: string | (() => Promise<string>);
|
|
33
83
|
rateLimiter: RateLimiter;
|
|
@@ -316,22 +366,32 @@ export class OpenCloudClient {
|
|
|
316
366
|
|
|
317
367
|
async getTaskLogsAsync(taskPath: string): Promise<ParsedTestLogs> {
|
|
318
368
|
const raw = await this.getRawTaskLogsAsync(taskPath);
|
|
319
|
-
return parseTestLogs(raw);
|
|
369
|
+
return parseTestLogs(raw.text);
|
|
320
370
|
}
|
|
321
371
|
|
|
322
372
|
/**
|
|
323
|
-
* Fetch raw log text from a completed Luau execution task
|
|
373
|
+
* Fetch raw log text from a completed Luau execution task, along with the
|
|
374
|
+
* shape of the fetch that produced it.
|
|
375
|
+
*
|
|
324
376
|
* Retries a few times if the API returns empty logs, since the test runner
|
|
325
|
-
* always produces at least some output.
|
|
377
|
+
* always produces at least some output. The reported `requests` count spans
|
|
378
|
+
* every attempt, so a fetch that only filled in on its third try says so.
|
|
326
379
|
*/
|
|
327
|
-
async getRawTaskLogsAsync(taskPath: string): Promise<
|
|
380
|
+
async getRawTaskLogsAsync(taskPath: string): Promise<TaskLogFetch> {
|
|
328
381
|
const maxAttempts = 3;
|
|
329
382
|
const retryDelayMs = 1000;
|
|
383
|
+
let requests = 0;
|
|
330
384
|
|
|
331
385
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
332
|
-
const
|
|
333
|
-
|
|
334
|
-
|
|
386
|
+
const fetched = await this._fetchRawLogsAsync(taskPath);
|
|
387
|
+
requests += fetched.stats.requests;
|
|
388
|
+
|
|
389
|
+
if (fetched.text || attempt === maxAttempts) {
|
|
390
|
+
return {
|
|
391
|
+
text: fetched.text,
|
|
392
|
+
stats: { ...fetched.stats, requests },
|
|
393
|
+
messages: fetched.messages,
|
|
394
|
+
};
|
|
335
395
|
}
|
|
336
396
|
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
|
|
337
397
|
}
|
|
@@ -340,12 +400,12 @@ export class OpenCloudClient {
|
|
|
340
400
|
return this._fetchRawLogsAsync(taskPath);
|
|
341
401
|
}
|
|
342
402
|
|
|
343
|
-
private async _fetchRawLogsAsync(taskPath: string): Promise<
|
|
403
|
+
private async _fetchRawLogsAsync(taskPath: string): Promise<TaskLogFetch> {
|
|
344
404
|
const apiKey = await this._resolveApiKeyAsync();
|
|
345
405
|
// Arrival order, deliberately. Do not sort by createTime: every print in a
|
|
346
406
|
// single frame shares one value, so it cannot order them anyway, and the
|
|
347
407
|
// fractional digit count varies, so comparing the strings reorders them.
|
|
348
|
-
const messages:
|
|
408
|
+
const messages: TaskLogMessage[] = [];
|
|
349
409
|
let pageToken: string | undefined;
|
|
350
410
|
let pagesFetched = 0;
|
|
351
411
|
let entriesFetched = 0;
|
|
@@ -383,9 +443,16 @@ export class OpenCloudClient {
|
|
|
383
443
|
|
|
384
444
|
for (const entry of data.luauExecutionSessionTaskLogs ?? []) {
|
|
385
445
|
if (entry.structuredMessages?.length) {
|
|
386
|
-
messages.push(
|
|
446
|
+
messages.push(
|
|
447
|
+
...entry.structuredMessages.map((msg) => ({
|
|
448
|
+
message: msg.message,
|
|
449
|
+
messageType: msg.messageType,
|
|
450
|
+
}))
|
|
451
|
+
);
|
|
387
452
|
} else if (entry.messages?.length) {
|
|
388
|
-
|
|
453
|
+
// The flat view carries no severity, which stays absent rather than
|
|
454
|
+
// being guessed at.
|
|
455
|
+
messages.push(...entry.messages.map((message) => ({ message })));
|
|
389
456
|
}
|
|
390
457
|
}
|
|
391
458
|
|
|
@@ -394,17 +461,26 @@ export class OpenCloudClient {
|
|
|
394
461
|
pageToken = data.nextPageToken || undefined;
|
|
395
462
|
} while (pageToken);
|
|
396
463
|
|
|
397
|
-
const text = messages.join('\n');
|
|
464
|
+
const text = messages.map((entry) => entry.message).join('\n');
|
|
465
|
+
|
|
466
|
+
const stats: LogFetchStats = {
|
|
467
|
+
requests: pagesFetched,
|
|
468
|
+
pages: pagesFetched,
|
|
469
|
+
entries: entriesFetched,
|
|
470
|
+
messages: messages.length,
|
|
471
|
+
chars: text.length,
|
|
472
|
+
};
|
|
398
473
|
|
|
399
474
|
// Log volume is the prime suspect when engine output goes missing between
|
|
400
475
|
// the task and the parser, and the parser cannot tell a short run from a
|
|
401
|
-
// truncated fetch.
|
|
476
|
+
// truncated fetch. Handed out with the text rather than only printed, so
|
|
477
|
+
// the diagnostic that finds the output missing can say what the fetch saw.
|
|
402
478
|
OutputHelper.verbose(
|
|
403
|
-
`[open-cloud] Task logs: ${
|
|
404
|
-
`${messages
|
|
479
|
+
`[open-cloud] Task logs: ${stats.pages} page(s), ${stats.entries} entries, ` +
|
|
480
|
+
`${stats.messages} messages, ${stats.chars} chars`
|
|
405
481
|
);
|
|
406
482
|
|
|
407
|
-
return text;
|
|
483
|
+
return { text, stats, messages };
|
|
408
484
|
}
|
|
409
485
|
|
|
410
486
|
/**
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Characterization tests for how Open Cloud actually delivers a Luau execution
|
|
3
|
+
* task's logs and return value. Nothing here tests our code: every assertion is
|
|
4
|
+
* a claim about Roblox's behaviour that our log handling is built on, written
|
|
5
|
+
* so the claim fails out loud if Roblox changes it.
|
|
6
|
+
*
|
|
7
|
+
* These claims decide real things — whether a truncated run is worth
|
|
8
|
+
* re-fetching, whether `maxPageSize` is worth sending, whether a batch package
|
|
9
|
+
* with no output was dropped or never ran — and none of them are documented by
|
|
10
|
+
* Roblox. Prose in a gotchas file cannot notice when it goes stale. This can.
|
|
11
|
+
*
|
|
12
|
+
* Disabled by default: it spends real Open Cloud quota. To run it:
|
|
13
|
+
*
|
|
14
|
+
* NEVERMORE_CHARACTERIZATION=1 npm test -- open-cloud-logs
|
|
15
|
+
*
|
|
16
|
+
* It reads the key through the same credential store the CLI uses, so
|
|
17
|
+
* `nevermore auth login` is the only setup. The universe and place default to
|
|
18
|
+
* the nevermore-test-runner test target and can be pointed elsewhere with
|
|
19
|
+
* NEVERMORE_CHARACTERIZATION_UNIVERSE / _PLACE.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { describe, it, expect, beforeAll } from 'vitest';
|
|
23
|
+
import { loadStoredApiKeyAsync } from '@quenty/nevermore-cli-helpers';
|
|
24
|
+
import {
|
|
25
|
+
OpenCloudClient,
|
|
26
|
+
getTaskReturnValues,
|
|
27
|
+
type LuauTask,
|
|
28
|
+
} from './open-cloud-client.js';
|
|
29
|
+
import { RateLimiter } from './rate-limiter.js';
|
|
30
|
+
|
|
31
|
+
const ENABLED = process.env.NEVERMORE_CHARACTERIZATION === '1';
|
|
32
|
+
|
|
33
|
+
/** The nevermore-test-runner test target, which exists to be run against. */
|
|
34
|
+
const UNIVERSE_ID = Number(
|
|
35
|
+
process.env.NEVERMORE_CHARACTERIZATION_UNIVERSE ?? 9716264427
|
|
36
|
+
);
|
|
37
|
+
const PLACE_ID = Number(
|
|
38
|
+
process.env.NEVERMORE_CHARACTERIZATION_PLACE ?? 138951894938783
|
|
39
|
+
);
|
|
40
|
+
|
|
41
|
+
const LINES_THAT_FIT = 5_000;
|
|
42
|
+
const LINES_AT_CAPACITY = 8_000;
|
|
43
|
+
const LINES_THAT_OVERFLOW_LESS = 10_000;
|
|
44
|
+
const LINES_THAT_OVERFLOW = 20_000;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Retention is bounded by memory, and each retained message costs this many
|
|
48
|
+
* bytes beyond its own text — timestamp, type and framing. Derived by holding
|
|
49
|
+
* the print count at 20,000 and varying only the payload width, then solving
|
|
50
|
+
* `chars + k * messages` for the k that makes that budget constant.
|
|
51
|
+
*/
|
|
52
|
+
const PER_MESSAGE_OVERHEAD_BYTES = 47;
|
|
53
|
+
|
|
54
|
+
/** A printed line that names its own position, so survivors identify themselves. */
|
|
55
|
+
const SEQ = new RegExp('^SEQ (' + String.raw`\d` + '+)');
|
|
56
|
+
|
|
57
|
+
describe.runIf(ENABLED)('Open Cloud task log delivery', () => {
|
|
58
|
+
let client: OpenCloudClient;
|
|
59
|
+
let placeVersion: number;
|
|
60
|
+
|
|
61
|
+
beforeAll(async () => {
|
|
62
|
+
const apiKey = await loadStoredApiKeyAsync();
|
|
63
|
+
if (!apiKey) {
|
|
64
|
+
throw new Error(
|
|
65
|
+
'No stored Open Cloud API key. Run `nevermore auth login` first.'
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
client = new OpenCloudClient({ apiKey, rateLimiter: new RateLimiter() });
|
|
70
|
+
placeVersion = await client.resolveLatestPlaceVersionAsync(
|
|
71
|
+
UNIVERSE_ID,
|
|
72
|
+
PLACE_ID,
|
|
73
|
+
'saved'
|
|
74
|
+
);
|
|
75
|
+
}, 120_000);
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Run a task printing `lines` numbered messages, each optionally padded to
|
|
79
|
+
* widen it, and report what came back.
|
|
80
|
+
*/
|
|
81
|
+
async function printLinesAsync(lines: number, padChars = 0) {
|
|
82
|
+
const script = padChars
|
|
83
|
+
? `local pad = string.rep("p", ${padChars}) for i = 1, ${lines} do print("SEQ " .. i .. " " .. pad) end`
|
|
84
|
+
: `for i = 1, ${lines} do print("SEQ " .. i) end`;
|
|
85
|
+
|
|
86
|
+
const task = await client.createExecutionTaskAsync(
|
|
87
|
+
UNIVERSE_ID,
|
|
88
|
+
PLACE_ID,
|
|
89
|
+
placeVersion,
|
|
90
|
+
script,
|
|
91
|
+
300_000
|
|
92
|
+
);
|
|
93
|
+
const completed = await client.pollTaskCompletionAsync(task.path);
|
|
94
|
+
expect(completed.state).toBe('COMPLETE');
|
|
95
|
+
|
|
96
|
+
const fetched = await client.getRawTaskLogsAsync(completed.path);
|
|
97
|
+
const seq = fetched.text
|
|
98
|
+
.split('\n')
|
|
99
|
+
.map((line) => SEQ.exec(line.trim())?.[1])
|
|
100
|
+
.filter((n): n is string => n !== undefined)
|
|
101
|
+
.map(Number);
|
|
102
|
+
|
|
103
|
+
return { task: completed, ...fetched, seq };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Bytes the buffer spent on what it kept, per-message overhead included. */
|
|
107
|
+
function retainedBudget(run: { stats: { chars: number; messages: number } }) {
|
|
108
|
+
return run.stats.chars + PER_MESSAGE_OVERHEAD_BYTES * run.stats.messages;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
it('delivers a small run complete and in order', async () => {
|
|
112
|
+
const run = await printLinesAsync(LINES_THAT_FIT);
|
|
113
|
+
|
|
114
|
+
expect(run.seq).toHaveLength(LINES_THAT_FIT);
|
|
115
|
+
expect(run.seq[0]).toBe(1);
|
|
116
|
+
expect(run.seq[run.seq.length - 1]).toBe(LINES_THAT_FIT);
|
|
117
|
+
// Arrival order is chronological, so no client-side sort is needed — and
|
|
118
|
+
// sorting by createTime would be wrong, since a whole frame shares one.
|
|
119
|
+
expect(run.seq).toEqual([...run.seq].sort((a, b) => a - b));
|
|
120
|
+
}, 300_000);
|
|
121
|
+
|
|
122
|
+
it('truncates a large run by dropping the head, keeping a contiguous tail', async () => {
|
|
123
|
+
// Says a short log is missing output rather than proof a run printed
|
|
124
|
+
// little, and rules out the loss being a page we failed to request — an
|
|
125
|
+
// unrequested page would have held the head. It is why the batch parser
|
|
126
|
+
// closes sections on their END marker.
|
|
127
|
+
const run = await printLinesAsync(LINES_THAT_OVERFLOW);
|
|
128
|
+
|
|
129
|
+
expect(run.seq.length).toBeLessThan(LINES_THAT_OVERFLOW);
|
|
130
|
+
expect(run.seq[run.seq.length - 1]).toBe(LINES_THAT_OVERFLOW);
|
|
131
|
+
expect(run.seq[0]).toBeGreaterThan(1);
|
|
132
|
+
expect(run.seq).toEqual(
|
|
133
|
+
Array.from({ length: run.seq.length }, (_, i) => run.seq[0] + i)
|
|
134
|
+
);
|
|
135
|
+
}, 300_000);
|
|
136
|
+
|
|
137
|
+
it('bounds retention by memory, not by number of print calls', async () => {
|
|
138
|
+
// Both runs make exactly the same number of print calls. If the buffer held
|
|
139
|
+
// a fixed number of messages they would retain the same count; the padded
|
|
140
|
+
// one keeps a small fraction of it, so the budget is bytes.
|
|
141
|
+
const narrow = await printLinesAsync(LINES_THAT_OVERFLOW);
|
|
142
|
+
const wide = await printLinesAsync(LINES_THAT_OVERFLOW, 500);
|
|
143
|
+
|
|
144
|
+
expect(wide.stats.messages).toBeLessThan(narrow.stats.messages / 5);
|
|
145
|
+
expect(wide.stats.chars).toBeGreaterThan(narrow.stats.chars);
|
|
146
|
+
|
|
147
|
+
// And it is one budget in both: bytes kept, plus a fixed cost per message,
|
|
148
|
+
// lands on the same number however wide the messages are. ~450 KB.
|
|
149
|
+
expect(retainedBudget(wide)).toBeGreaterThan(retainedBudget(narrow) * 0.95);
|
|
150
|
+
expect(retainedBudget(wide)).toBeLessThan(retainedBudget(narrow) * 1.05);
|
|
151
|
+
expect(retainedBudget(narrow)).toBeGreaterThan(400_000);
|
|
152
|
+
expect(retainedBudget(narrow)).toBeLessThan(500_000);
|
|
153
|
+
}, 300_000);
|
|
154
|
+
|
|
155
|
+
it('keeps the same amount however much is printed, at one message width', async () => {
|
|
156
|
+
// The corollary: at a fixed message size, printing twice as much does not
|
|
157
|
+
// get twice as much back. A long run cannot be made to report itself by
|
|
158
|
+
// printing more carefully, which is why counts travel in the return value.
|
|
159
|
+
const atCapacity = await printLinesAsync(LINES_AT_CAPACITY);
|
|
160
|
+
expect(atCapacity.seq).toHaveLength(LINES_AT_CAPACITY);
|
|
161
|
+
|
|
162
|
+
const over = await printLinesAsync(LINES_THAT_OVERFLOW_LESS);
|
|
163
|
+
const wayOver = await printLinesAsync(LINES_THAT_OVERFLOW);
|
|
164
|
+
|
|
165
|
+
expect(over.seq.length).toBeLessThan(LINES_THAT_OVERFLOW_LESS);
|
|
166
|
+
expect(wayOver.seq.length).toBeLessThan(LINES_THAT_OVERFLOW);
|
|
167
|
+
expect(wayOver.seq.length).toBeGreaterThan(over.seq.length * 0.9);
|
|
168
|
+
expect(wayOver.seq.length).toBeLessThan(over.seq.length * 1.1);
|
|
169
|
+
}, 300_000);
|
|
170
|
+
|
|
171
|
+
it('returns only what the task printed, with nothing of its own mixed in', async () => {
|
|
172
|
+
// Rules out engine or internal messages taking room in the buffer: every
|
|
173
|
+
// message that came back is one of ours, so a retained count short of a
|
|
174
|
+
// round number is the byte budget, not foreign output.
|
|
175
|
+
const run = await printLinesAsync(LINES_THAT_OVERFLOW);
|
|
176
|
+
|
|
177
|
+
expect(run.seq).toHaveLength(run.stats.messages);
|
|
178
|
+
}, 300_000);
|
|
179
|
+
|
|
180
|
+
it('delivers the same messages structured or flat, and never a second page', async () => {
|
|
181
|
+
// Why we send no maxPageSize, and why the view is a free choice: asking for
|
|
182
|
+
// 10 still returns the whole window in one page. A page size that counted
|
|
183
|
+
// messages would cap this at 10 and hand back a token for the rest.
|
|
184
|
+
const run = await printLinesAsync(LINES_THAT_OVERFLOW);
|
|
185
|
+
|
|
186
|
+
const structured = await fetchOnePageAsync(run.task.path, {
|
|
187
|
+
view: 'STRUCTURED',
|
|
188
|
+
});
|
|
189
|
+
const flat = await fetchOnePageAsync(run.task.path, {});
|
|
190
|
+
|
|
191
|
+
// Same messages in the same order — the view changes the envelope, not the
|
|
192
|
+
// delivery.
|
|
193
|
+
expect(flat.messages).toEqual(structured.messages);
|
|
194
|
+
expect(structured.messages).toHaveLength(run.stats.messages);
|
|
195
|
+
|
|
196
|
+
const variants: Array<Record<string, string>> = [
|
|
197
|
+
{ view: 'STRUCTURED', maxPageSize: '10' },
|
|
198
|
+
{ view: 'STRUCTURED', maxPageSize: '100' },
|
|
199
|
+
{ maxPageSize: '10' },
|
|
200
|
+
];
|
|
201
|
+
|
|
202
|
+
for (const params of variants) {
|
|
203
|
+
const page = await fetchOnePageAsync(run.task.path, params);
|
|
204
|
+
expect(page.nextPageToken).toBeUndefined();
|
|
205
|
+
expect(page.messages).toEqual(structured.messages);
|
|
206
|
+
}
|
|
207
|
+
}, 300_000);
|
|
208
|
+
|
|
209
|
+
it('delivers a large return value, but drops the task entirely past a limit', async () => {
|
|
210
|
+
// Why the returned failure list is capped rather than trusted to arrive: an
|
|
211
|
+
// oversize return value does not come back truncated, it takes the whole
|
|
212
|
+
// run with it — and such a task carries no output and no message saying so,
|
|
213
|
+
// which is why "nothing came back" must stay distinguishable from "the
|
|
214
|
+
// script returned nothing".
|
|
215
|
+
const twoMegabytes = await returnStringAsync(2 * 1024 * 1024);
|
|
216
|
+
expect(twoMegabytes.state).toBe('COMPLETE');
|
|
217
|
+
expect(getTaskReturnValues(twoMegabytes)).toHaveLength(1);
|
|
218
|
+
|
|
219
|
+
const fiveMegabytes = await returnStringAsync(5 * 1024 * 1024);
|
|
220
|
+
expect(fiveMegabytes.state).not.toBe('COMPLETE');
|
|
221
|
+
expect(getTaskReturnValues(fiveMegabytes)).toBeUndefined();
|
|
222
|
+
}, 300_000);
|
|
223
|
+
|
|
224
|
+
/** Run a task returning a string of `bytes`, and wait for whatever happens. */
|
|
225
|
+
async function returnStringAsync(bytes: number): Promise<LuauTask> {
|
|
226
|
+
const task = await client.createExecutionTaskAsync(
|
|
227
|
+
UNIVERSE_ID,
|
|
228
|
+
PLACE_ID,
|
|
229
|
+
placeVersion,
|
|
230
|
+
`return string.rep("x", ${bytes})`,
|
|
231
|
+
300_000
|
|
232
|
+
);
|
|
233
|
+
|
|
234
|
+
// Returns on any terminal state, failure included — which is one of the
|
|
235
|
+
// outcomes under test here, not an error.
|
|
236
|
+
return client.pollTaskCompletionAsync(task.path);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* One raw request, bypassing the client so the query can be varied. Returns
|
|
241
|
+
* only what the claims above are about.
|
|
242
|
+
*/
|
|
243
|
+
async function fetchOnePageAsync(
|
|
244
|
+
taskPath: string,
|
|
245
|
+
params: Record<string, string>
|
|
246
|
+
): Promise<{ nextPageToken?: string; messages: string[] }> {
|
|
247
|
+
const apiKey = (await loadStoredApiKeyAsync())!;
|
|
248
|
+
const url = new URL(`https://apis.roblox.com/cloud/v2/${taskPath}/logs`);
|
|
249
|
+
for (const [key, value] of Object.entries(params)) {
|
|
250
|
+
url.searchParams.set(key, value);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const response = await fetch(url.toString(), {
|
|
254
|
+
headers: { 'X-API-Key': apiKey },
|
|
255
|
+
});
|
|
256
|
+
expect(response.ok).toBe(true);
|
|
257
|
+
|
|
258
|
+
const data = (await response.json()) as {
|
|
259
|
+
luauExecutionSessionTaskLogs?: Array<{
|
|
260
|
+
messages?: string[];
|
|
261
|
+
structuredMessages?: Array<{ message: string }>;
|
|
262
|
+
}>;
|
|
263
|
+
nextPageToken?: string;
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
const messages: string[] = [];
|
|
267
|
+
for (const entry of data.luauExecutionSessionTaskLogs ?? []) {
|
|
268
|
+
if (entry.structuredMessages?.length) {
|
|
269
|
+
messages.push(...entry.structuredMessages.map((m) => m.message));
|
|
270
|
+
} else if (entry.messages?.length) {
|
|
271
|
+
messages.push(...entry.messages);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
return { nextPageToken: data.nextPageToken || undefined, messages };
|
|
276
|
+
}
|
|
277
|
+
});
|
|
@@ -106,6 +106,33 @@ describe('package-manager-utils', () => {
|
|
|
106
106
|
expect(await detectPackageManagerAsync(root)).toBe('pnpm');
|
|
107
107
|
});
|
|
108
108
|
|
|
109
|
+
it('ignores a lockfile sitting in the home directory', async () => {
|
|
110
|
+
// npm leaves a package-lock.json behind after one `npm install` run in a
|
|
111
|
+
// home directory, and walking to the filesystem root adopted it — every
|
|
112
|
+
// project below it then installed with the one tool that rejects this
|
|
113
|
+
// repo's `workspace:` ranges. This test was the symptom: the temp dir
|
|
114
|
+
// lives under the home directory on Windows, so "defaults to pnpm" here
|
|
115
|
+
// failed on a machine that had one and passed everywhere else.
|
|
116
|
+
const home = await fs.realpath(os.homedir());
|
|
117
|
+
const marker = path.join(home, 'package-lock.json');
|
|
118
|
+
const alreadyThere = await fs
|
|
119
|
+
.access(marker)
|
|
120
|
+
.then(() => true)
|
|
121
|
+
.catch(() => false);
|
|
122
|
+
|
|
123
|
+
if (!alreadyThere) {
|
|
124
|
+
await fs.writeFile(marker, '{}');
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
try {
|
|
128
|
+
expect(await detectPackageManagerAsync(root)).toBe('pnpm');
|
|
129
|
+
} finally {
|
|
130
|
+
if (!alreadyThere) {
|
|
131
|
+
await fs.rm(marker, { force: true });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
|
|
109
136
|
it('still honors an npm project', async () => {
|
|
110
137
|
await writeFileAsync('package.json', JSON.stringify({ name: 'game' }));
|
|
111
138
|
await writeFileAsync('package-lock.json', '{}');
|