@forgeax/engine-profiler 0.0.0-dev.8d955ade1c79

package/src/__tests__/cli-input-errors.test.ts

@@ -0,0 +1,171 @@
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import { readCliInput, runProfilerCli } from '../cli.js';
+
+function expectStructuredError(
+  result: { stdout: string; stderr: string; exitCode: number },
+  code: string,
+): void {
+  expect(result.exitCode).not.toBe(0);
+  expect(result.stdout).toBe('');
+  const parsed = JSON.parse(result.stderr) as {
+    error?: { code?: string; expected?: string; hint?: string; detail?: unknown };
+  };
+  expect(parsed).toEqual({
+    error: {
+      code,
+      expected: expect.any(String),
+      hint: expect.any(String),
+      detail: expect.anything(),
+    },
+  });
+}
+
+function expectStructuredErrorWithSide(
+  result: { stdout: string; stderr: string; exitCode: number },
+  code: string,
+  side: 'left' | 'right',
+): void {
+  expect(result.exitCode).not.toBe(0);
+  expect(result.stdout).toBe('');
+  expect(JSON.parse(result.stderr)).toEqual({
+    error: {
+      code,
+      expected: expect.any(String),
+      hint: expect.any(String),
+      detail: {
+        side,
+        ...(code === 'cli-input-file-read-failed' ? { path: expect.any(String) } : {}),
+        message: expect.any(String),
+      },
+    },
+  });
+}
+
+describe('profiler CLI input error contract', () => {
+  it('does not read stdin when a file input is selected', () => {
+    expect(
+      readCliInput(['summary', '--file', 'capture.json'], () => {
+        throw new Error('stdin read');
+      }),
+    ).toBe('');
+  });
+
+  it('reads stdin when no file input is selected', () => {
+    expect(readCliInput(['summary'], () => 'capture')).toBe('capture');
+  });
+
+  it('rejects empty stdin without producing a summary', () => {
+    expectStructuredError(runProfilerCli(['summary'], ''), 'cli-input-empty');
+  });
+
+  it('rejects malformed JSON from stdin', () => {
+    expectStructuredError(
+      runProfilerCli(['summary'], '{"schemaVersion":'),
+      'cli-input-invalid-json',
+    );
+  });
+
+  it('rejects a missing file before model analysis', () => {
+    expectStructuredError(
+      runProfilerCli(['summary', '--file', '/tmp/forgeax-profiler-missing-capture.json'], ''),
+      'cli-input-file-read-failed',
+    );
+  });
+
+  it('rejects schema-invalid JSON as a structured artifact error', () => {
+    expectStructuredError(
+      runProfilerCli(['summary'], '{"schemaVersion":"1.0"}'),
+      'profile-artifact-invalid',
+    );
+  });
+
+  it('rejects unsupported artifact versions without guessing fields', () => {
+    expectStructuredError(
+      runProfilerCli(
+        ['summary'],
+        JSON.stringify({
+          schemaVersion: '9.0',
+          captureId: 'capture-0001',
+          timeUnit: 'microseconds',
+          frameLimit: 1,
+          eventLimit: 1,
+          phaseCatalog: { app: [], render: [] },
+          records: [],
+          completeness: { status: 'partial', retainedEventCount: 0, droppedEventCount: 0 },
+        }),
+      ),
+      'profile-artifact-incompatible',
+    );
+  });
+
+  it('rejects non-positive and unsafe frame identifiers', () => {
+    const input = JSON.stringify({
+      schemaVersion: '1.0',
+      captureId: 'capture-0001',
+      timeUnit: 'microseconds',
+      frameLimit: 1,
+      eventLimit: 1,
+      phaseCatalog: { app: ['input'], render: [] },
+      records: [],
+      completeness: { status: 'partial', retainedEventCount: 0, droppedEventCount: 0 },
+    });
+
+    expectStructuredError(runProfilerCli(['frame', '--frame-id', '0'], input), 'cli-query-invalid');
+    expectStructuredError(
+      runProfilerCli(['frame', '--frame-id', '9007199254740992'], input),
+      'cli-query-invalid',
+    );
+  });
+
+  it('rejects unknown flags and missing phase selectors', () => {
+    const input = JSON.stringify({
+      schemaVersion: '1.0',
+      captureId: 'capture-0001',
+      timeUnit: 'microseconds',
+      frameLimit: 1,
+      eventLimit: 1,
+      phaseCatalog: { app: ['input'], render: [] },
+      records: [],
+      completeness: { status: 'partial', retainedEventCount: 0, droppedEventCount: 0 },
+    });
+
+    expectStructuredError(runProfilerCli(['summary', '--unknown'], input), 'cli-arguments-invalid');
+    expectStructuredError(
+      runProfilerCli(['phase', '--source', 'app'], input),
+      'cli-arguments-invalid',
+    );
+  });
+
+  it('requires two file inputs for compare and does not consume stdin', () => {
+    expect(
+      readCliInput(['compare', '--left-file', 'left.json', '--right-file', 'right.json'], () => {
+        throw new Error('stdin read');
+      }),
+    ).toBe('');
+    expectStructuredError(
+      runProfilerCli(['compare', '--left-file', 'left.json'], ''),
+      'cli-arguments-invalid',
+    );
+    expectStructuredError(
+      runProfilerCli(['summary', '--left-file', 'left.json', '--right-file', 'right.json'], '{}'),
+      'cli-arguments-invalid',
+    );
+  });
+
+  it('attributes unreadable compare files independently', () => {
+    const validPath = fileURLToPath(new URL('./fixtures/cli/valid-capture.json', import.meta.url));
+    const missingLeftPath = '/tmp/forgeax-profiler-missing-left-capture.json';
+    const missingRightPath = '/tmp/forgeax-profiler-missing-right-capture.json';
+    expectStructuredErrorWithSide(
+      runProfilerCli(['compare', '--left-file', missingLeftPath, '--right-file', validPath], ''),
+      'cli-input-file-read-failed',
+      'left',
+    );
+    expectStructuredErrorWithSide(
+      runProfilerCli(['compare', '--left-file', validPath, '--right-file', missingRightPath], ''),
+      'cli-input-file-read-failed',
+      'right',
+    );
+  });
+});

package/src/__tests__/cli-process.integration.test.ts

@@ -0,0 +1,127 @@
+import { spawnSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+
+const cliPath = fileURLToPath(new URL('../../dist/cli.mjs', import.meta.url));
+const fixturePath = fileURLToPath(new URL('./fixtures/cli/valid-capture.json', import.meta.url));
+const partialFixturePath = fileURLToPath(
+  new URL('./fixtures/cli/partial-capture.json', import.meta.url),
+);
+const invalidFixturePath = fileURLToPath(
+  new URL('./fixtures/cli/invalid-version.json', import.meta.url),
+);
+
+function readFixture(): string {
+  return readFileSync(fixturePath, 'utf8');
+}
+
+function runCli(args: readonly string[], input: string) {
+  return spawnSync(process.execPath, [cliPath, ...args], {
+    input,
+    encoding: 'utf8',
+  });
+}
+
+function parseObject(output: string): Record<string, unknown> {
+  return JSON.parse(output) as Record<string, unknown>;
+}
+
+describe('profiler CLI public process contract', () => {
+  it('prints deterministic summary, frame, and phase JSON through the built entry', () => {
+    const input = readFixture();
+    const summary = runCli(['summary'], input);
+    const summaryRepeat = runCli(['summary'], input);
+    const frame = runCli(['frame', '--frame-id', '2'], input);
+    const phase = runCli(['phase', '--source', 'render', '--phase', 'submit'], input);
+
+    expect(summary.status).toBe(0);
+    expect(summary.stderr).toBe('');
+    expect(parseObject(summary.stdout)).toEqual(parseObject(summaryRepeat.stdout));
+    expect(parseObject(summary.stdout)).toMatchObject({
+      query: 'summary',
+      captureId: 'capture-0042',
+      completeness: { status: 'complete' },
+    });
+    expect(frame.status).toBe(0);
+    expect(parseObject(frame.stdout)).toMatchObject({ query: 'frame', frameId: 2 });
+    expect(phase.status).toBe(0);
+    expect(parseObject(phase.stdout)).toMatchObject({
+      query: 'phase',
+      source: 'render',
+      phase: 'submit',
+    });
+  });
+
+  it('accepts file input and keeps invalid input on stderr as one error object', () => {
+    const fromFile = runCli(['summary', '--file', fixturePath], '');
+    const invalid = runCli(['summary'], '{"schemaVersion":');
+
+    expect(fromFile.status).toBe(0);
+    expect(fromFile.stderr).toBe('');
+    expect(parseObject(fromFile.stdout)).toMatchObject({ captureId: 'capture-0042' });
+    expect(invalid.status).not.toBe(0);
+    expect(invalid.stdout).toBe('');
+    expect(parseObject(invalid.stderr)).toEqual({
+      error: {
+        code: 'cli-input-invalid-json',
+        expected: expect.any(String),
+        hint: expect.any(String),
+        detail: expect.any(Object),
+      },
+    });
+  });
+
+  it('compares two file artifacts deterministically through the built entry', () => {
+    const args = ['compare', '--left-file', fixturePath, '--right-file', partialFixturePath];
+    const first = runCli(args, 'stdin must not be read');
+    const second = runCli(args, 'different stdin must not be read');
+    const output = parseObject(first.stdout);
+
+    expect(first.status).toBe(0);
+    expect(first.stderr).toBe('');
+    expect(first.stdout).toBe(second.stdout);
+    expect(output).toMatchObject({
+      query: 'compare',
+      left: {
+        summary: { captureId: 'capture-0042' },
+        completeness: { status: 'complete' },
+      },
+      right: {
+        summary: { captureId: 'capture-0043' },
+        completeness: { status: 'partial' },
+      },
+      phases: expect.any(Array),
+    });
+  });
+
+  it('attributes an invalid compare artifact to its input side', () => {
+    const leftInvalid = runCli(
+      ['compare', '--left-file', invalidFixturePath, '--right-file', fixturePath],
+      '',
+    );
+    const rightInvalid = runCli(
+      ['compare', '--left-file', fixturePath, '--right-file', invalidFixturePath],
+      '',
+    );
+
+    expect(leftInvalid.status).not.toBe(0);
+    expect(parseObject(leftInvalid.stderr)).toEqual({
+      error: {
+        code: 'profile-artifact-incompatible',
+        expected: expect.any(String),
+        hint: expect.any(String),
+        detail: { path: '/schemaVersion', message: expect.any(String), side: 'left' },
+      },
+    });
+    expect(rightInvalid.status).not.toBe(0);
+    expect(parseObject(rightInvalid.stderr)).toEqual({
+      error: {
+        code: 'profile-artifact-incompatible',
+        expected: expect.any(String),
+        hint: expect.any(String),
+        detail: { path: '/schemaVersion', message: expect.any(String), side: 'right' },
+      },
+    });
+  });
+});

package/src/__tests__/compare.test.ts

@@ -0,0 +1,223 @@
+import { readFileSync } from 'node:fs';
+import { describe, expect, it } from 'vitest';
+import { compareProfileCaptures } from '../compare.js';
+
+function readFixture(name: string): unknown {
+  return JSON.parse(
+    readFileSync(new URL(`./fixtures/cli/${name}`, import.meta.url), 'utf8'),
+  ) as unknown;
+}
+
+function capture(variant: 'left' | 'right'): Record<string, unknown> {
+  const right = variant === 'right';
+  return {
+    schemaVersion: '1.0',
+    captureId: right ? 'capture-0002' : 'capture-0001',
+    timeUnit: 'microseconds',
+    frameLimit: 4,
+    eventLimit: 16,
+    phaseCatalog: {
+      app: ['input', 'update'],
+      render: right ? ['submit', 'gpu'] : ['submit'],
+    },
+    records: right
+      ? [
+          {
+            kind: 'phase',
+            source: 'app',
+            frameId: 1,
+            phase: 'input',
+            startMicros: 1,
+            endMicros: 16,
+            durationMicros: 15,
+          },
+          {
+            kind: 'phase',
+            source: 'app',
+            frameId: 1,
+            phase: 'update',
+            parentSource: 'app',
+            parentPhase: 'input',
+            startMicros: 16,
+            endMicros: 41,
+            durationMicros: 25,
+          },
+          {
+            kind: 'phase',
+            source: 'render',
+            frameId: 1,
+            phase: 'submit',
+            startMicros: 41,
+            endMicros: 46,
+            durationMicros: 5,
+          },
+          {
+            kind: 'phase',
+            source: 'render',
+            frameId: 2,
+            phase: 'gpu',
+            startMicros: 50,
+            endMicros: 90,
+            durationMicros: 40,
+          },
+        ]
+      : [
+          {
+            kind: 'phase',
+            source: 'app',
+            frameId: 1,
+            phase: 'input',
+            startMicros: 1,
+            endMicros: 11,
+            durationMicros: 10,
+          },
+          {
+            kind: 'phase',
+            source: 'app',
+            frameId: 1,
+            phase: 'update',
+            parentSource: 'app',
+            parentPhase: 'input',
+            startMicros: 11,
+            endMicros: 31,
+            durationMicros: 20,
+          },
+          { kind: 'skip', source: 'render', frameId: 1, phase: 'submit', reason: 'not recorded' },
+        ],
+    completeness: right
+      ? {
+          status: 'overflow',
+          retainedEventCount: 4,
+          droppedEventCount: 2,
+          firstAffectedFrameId: 2,
+          lastAffectedFrameId: 2,
+        }
+      : { status: 'complete', retainedEventCount: 3, droppedEventCount: 0 },
+  };
+}
+
+describe('ProfileCapture comparison', () => {
+  it('preserves both summaries and compares a union of full phase identities', () => {
+    const result = compareProfileCaptures(capture('left'), capture('right'));
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) throw new Error(result.error.code);
+    expect(result.value.left).toMatchObject({
+      summary: {
+        captureId: 'capture-0001',
+        frameCount: 1,
+        recordCount: 3,
+        phaseCount: 2,
+        skipCount: 1,
+      },
+      completeness: { status: 'complete', retainedEventCount: 3, droppedEventCount: 0 },
+    });
+    expect(result.value.right).toMatchObject({
+      summary: { captureId: 'capture-0002' },
+      completeness: {
+        status: 'overflow',
+        retainedEventCount: 4,
+        droppedEventCount: 2,
+        firstAffectedFrameId: 2,
+        lastAffectedFrameId: 2,
+      },
+    });
+
+    const update = result.value.phases.find((phase) => phase.identity.phase === 'update');
+    expect(update).toEqual({
+      identity: { source: 'app', phase: 'update', parentSource: 'app', parentPhase: 'input' },
+      left: { count: 1, skipCount: 0, p95DurationMicros: 20 },
+      right: { count: 1, skipCount: 0, p95DurationMicros: 25 },
+      delta: { count: 0, skipCount: 0, p95DurationMicros: 5 },
+    });
+
+    const submit = result.value.phases.find(
+      (phase) => phase.identity.source === 'render' && phase.identity.phase === 'submit',
+    );
+    expect(submit).toEqual({
+      identity: { source: 'render', phase: 'submit' },
+      left: { count: 0, skipCount: 1, p95DurationMicros: null },
+      right: { count: 1, skipCount: 0, p95DurationMicros: 5 },
+      delta: { count: 1, skipCount: -1 },
+    });
+
+    const gpu = result.value.phases.find((phase) => phase.identity.phase === 'gpu');
+    expect(gpu).toEqual({
+      identity: { source: 'render', phase: 'gpu' },
+      right: { count: 1, skipCount: 0, p95DurationMicros: 40 },
+    });
+  });
+
+  it('keeps invalid input attributable to its side', () => {
+    const left = compareProfileCaptures({ schemaVersion: '9.0' }, capture('right'));
+    expect(left).toEqual({
+      ok: false,
+      error: {
+        code: 'profile-artifact-incompatible',
+        expected: expect.any(String),
+        hint: expect.any(String),
+        detail: {
+          path: '/schemaVersion',
+          message: expect.any(String),
+          side: 'left',
+        },
+      },
+    });
+
+    const right = compareProfileCaptures(capture('left'), { schemaVersion: '9.0' });
+    expect(right).toMatchObject({ ok: false, error: { detail: { side: 'right' } } });
+  });
+
+  it('does not mutate either artifact and produces stable ordering', () => {
+    const left = capture('left');
+    const right = capture('right');
+    const beforeLeft = JSON.stringify(left);
+    const beforeRight = JSON.stringify(right);
+    const first = compareProfileCaptures(left, right);
+    const second = compareProfileCaptures(left, right);
+
+    expect(first).toEqual(second);
+    expect(JSON.stringify(left)).toBe(beforeLeft);
+    expect(JSON.stringify(right)).toBe(beforeRight);
+  });
+
+  it('keeps an unavailable p95 unavailable instead of manufacturing a delta', () => {
+    const result = compareProfileCaptures(
+      readFixture('valid-capture.json'),
+      readFixture('partial-capture.json'),
+    );
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) throw new Error(result.error.code);
+    const input = result.value.phases.find(
+      (phase) => phase.identity.source === 'app' && phase.identity.phase === 'input',
+    );
+    expect(input?.left?.p95DurationMicros).toBe(10);
+    expect(input?.right?.p95DurationMicros).toBe(15);
+    expect(input?.delta?.p95DurationMicros).toBe(5);
+    const update = result.value.phases.find(
+      (phase) => phase.identity.source === 'app' && phase.identity.phase === 'update',
+    );
+    expect(update?.left?.p95DurationMicros).toBe(30);
+    expect(update?.right).toBeUndefined();
+    expect(update?.delta).toBeUndefined();
+  });
+
+  it('does not merge equal phase names that have different parent identities', () => {
+    const right = capture('right');
+    const rootUpdate = (right.records as Array<Record<string, unknown>>).map((record) => {
+      if (record.phase !== 'update') return record;
+      const { parentSource: _parentSource, parentPhase: _parentPhase, ...withoutParent } = record;
+      return withoutParent;
+    });
+    const result = compareProfileCaptures(capture('left'), { ...right, records: rootUpdate });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) throw new Error(result.error.code);
+    const updates = result.value.phases.filter((phase) => phase.identity.phase === 'update');
+    expect(updates).toHaveLength(2);
+    expect(updates.map((phase) => phase.identity.parentPhase)).toEqual(['input', undefined]);
+    expect(updates[0]?.delta).toBeUndefined();
+    expect(updates[1]?.delta).toBeUndefined();
+  });
+});

package/src/__tests__/consumer-smoke.integration.test.ts

@@ -0,0 +1,37 @@
+import { spawnSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+import { buildProfileModel } from '@forgeax/engine-profiler';
+import { describe, expect, it } from 'vitest';
+
+// @perf-budget-skip: intentional public-entry subprocess smoke gate.
+const packageRoot = fileURLToPath(new URL('../../', import.meta.url));
+const scriptPath = fileURLToPath(new URL('../../scripts/consume-smoke.mjs', import.meta.url));
+const fixturePath = fileURLToPath(
+  new URL('./fixtures/profile-capture/model-input.json', import.meta.url),
+);
+
+describe('consumer smoke', () => {
+  it('consumes a fixture through the public package entry', () => {
+    const result = spawnSync(process.execPath, [scriptPath], {
+      cwd: packageRoot,
+      encoding: 'utf8',
+    });
+
+    expect(result.status).toBe(0);
+    expect(result.stderr).toBe('');
+    expect(result.stdout.trim().split('\n')).toHaveLength(1);
+
+    const output = JSON.parse(result.stdout) as Record<string, unknown>;
+    const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as unknown;
+    const model = buildProfileModel(fixture);
+    expect(model.ok).toBe(true);
+    if (!model.ok) throw new Error(model.error.code);
+    expect(output).toEqual({
+      query: 'summary',
+      ...model.value.summary,
+      phases: model.value.phases,
+    });
+  });
+});

package/src/__tests__/fixtures/cli/invalid-version.json

@@ -0,0 +1,18 @@
+{
+  "schemaVersion": "2.0",
+  "captureId": "capture-0001",
+  "timeUnit": "microseconds",
+  "frameLimit": 2,
+  "eventLimit": 8,
+  "phaseCatalog": {
+    "app": ["input", "update"],
+    "render": ["prepare", "submit"]
+  },
+  "records": [],
+  "completeness": {
+    "status": "partial",
+    "retainedEventCount": 0,
+    "droppedEventCount": 0,
+    "incompleteReason": "unsupported-version"
+  }
+}

package/src/__tests__/fixtures/cli/partial-capture.json

@@ -0,0 +1,28 @@
+{
+  "schemaVersion": "1.0",
+  "captureId": "capture-0043",
+  "timeUnit": "microseconds",
+  "frameLimit": 4,
+  "eventLimit": 8,
+  "phaseCatalog": {
+    "app": ["input", "update"],
+    "render": ["prepare", "submit"]
+  },
+  "records": [
+    {
+      "kind": "phase",
+      "source": "app",
+      "frameId": 1,
+      "phase": "input",
+      "startMicros": 1000,
+      "endMicros": 1015,
+      "durationMicros": 15
+    }
+  ],
+  "completeness": {
+    "status": "partial",
+    "retainedEventCount": 1,
+    "droppedEventCount": 0,
+    "incompleteReason": "stopped-during-frame"
+  }
+}

package/src/__tests__/fixtures/cli/valid-capture.json

@@ -0,0 +1,43 @@
+{
+  "schemaVersion": "1.0",
+  "captureId": "capture-0042",
+  "timeUnit": "microseconds",
+  "frameLimit": 2,
+  "eventLimit": 8,
+  "phaseCatalog": {
+    "app": ["input", "update"],
+    "render": ["prepare", "submit"]
+  },
+  "records": [
+    {
+      "kind": "phase",
+      "source": "app",
+      "frameId": 1,
+      "phase": "input",
+      "startMicros": 1000,
+      "endMicros": 1010,
+      "durationMicros": 10
+    },
+    {
+      "kind": "phase",
+      "source": "app",
+      "frameId": 2,
+      "phase": "update",
+      "startMicros": 2000,
+      "endMicros": 2030,
+      "durationMicros": 30
+    },
+    {
+      "kind": "skip",
+      "source": "render",
+      "frameId": 2,
+      "phase": "submit",
+      "reason": "no-render"
+    }
+  ],
+  "completeness": {
+    "status": "complete",
+    "retainedEventCount": 3,
+    "droppedEventCount": 0
+  }
+}