@bldrs-ai/conway 1.559.1526 → 1.560.1539
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/compiled/examples/browser-bundled.cjs +1 -1
- package/compiled/examples/cli-bundled.cjs +1 -1
- package/compiled/examples/cli-step-bundled.cjs +1 -1
- package/compiled/examples/validator-bundled.cjs +1 -1
- package/compiled/src/core/retained_memory.d.ts +10 -2
- package/compiled/src/core/retained_memory.d.ts.map +1 -1
- package/compiled/src/core/retained_memory.js +10 -2
- package/compiled/src/ifc/ifc_regression_batch_main.js +12 -3
- package/compiled/src/scripts/bless_perf_snapshot.test.js +88 -1
- package/compiled/src/scripts/perf_ab_compare.test.d.ts +2 -0
- package/compiled/src/scripts/perf_ab_compare.test.d.ts.map +1 -0
- package/compiled/src/scripts/perf_ab_compare.test.js +218 -0
- package/compiled/src/version/version.js +1 -1
- package/compiled/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -30,7 +30,7 @@ var import_node_process = require("node:process");
|
|
|
30
30
|
var readline = __toESM(require("node:readline"), 1);
|
|
31
31
|
|
|
32
32
|
// compiled/src/version/version.js
|
|
33
|
-
var versionString = "Conway v1.
|
|
33
|
+
var versionString = "Conway v1.560.1539";
|
|
34
34
|
|
|
35
35
|
// compiled/dependencies/conway-geom/interface/conway_geometry.js
|
|
36
36
|
var wasmType = "";
|
|
@@ -14965,7 +14965,7 @@ ${t5.join("\n")}` : "";
|
|
|
14965
14965
|
var import_process = require("process");
|
|
14966
14966
|
|
|
14967
14967
|
// compiled/src/version/version.js
|
|
14968
|
-
var versionString = "Conway v1.
|
|
14968
|
+
var versionString = "Conway v1.560.1539";
|
|
14969
14969
|
|
|
14970
14970
|
// compiled/dependencies/conway-geom/interface/conway_geometry.js
|
|
14971
14971
|
function pThreadsAllowed() {
|
|
@@ -15943,7 +15943,7 @@ var ParsingBuffer = class {
|
|
|
15943
15943
|
};
|
|
15944
15944
|
|
|
15945
15945
|
// compiled/src/version/version.js
|
|
15946
|
-
var versionString = "Conway v1.
|
|
15946
|
+
var versionString = "Conway v1.560.1539";
|
|
15947
15947
|
|
|
15948
15948
|
// compiled/dependencies/conway-geom/interface/conway_geometry.js
|
|
15949
15949
|
function pThreadsAllowed() {
|
|
@@ -944,7 +944,7 @@ var EntityTypesIfcCount = 909;
|
|
|
944
944
|
var entity_types_ifc_gen_default = EntityTypesIfc;
|
|
945
945
|
|
|
946
946
|
// compiled/src/version/version.js
|
|
947
|
-
var versionString = "Conway v1.
|
|
947
|
+
var versionString = "Conway v1.560.1539";
|
|
948
948
|
|
|
949
949
|
// compiled/dependencies/conway-geom/interface/conway_geometry.js
|
|
950
950
|
var wasmType = "";
|
|
@@ -87,8 +87,16 @@ export declare function canSettleMemory(): boolean;
|
|
|
87
87
|
* produces `parseTimeMs` / `geometryTimeMs` / `totalTimeMs` — baseline before
|
|
88
88
|
* the load starts, retained sample after teardown. That property is what
|
|
89
89
|
* makes the retention columns free, and it is checkable by measurement: a run
|
|
90
|
-
* with `--expose-gc` and a run without share identical code, so
|
|
91
|
-
* columns
|
|
90
|
+
* with `--expose-gc` and a run without share identical code, so any movement
|
|
91
|
+
* in the timing columns between them is down to the settle. The SIGN says
|
|
92
|
+
* which way: gc-on slower is the settle leaking into the measured window,
|
|
93
|
+
* which is a bug; gc-on faster is the pre-load settle taking engine-init
|
|
94
|
+
* garbage OUT of that window. That is an absolute cost of about 10 ms per
|
|
95
|
+
* load, measured as 13-16% of `parseTimeMs` on models that parse in ~60 ms
|
|
96
|
+
* and unresolvable against one that parses in 578 ms, so read a ratio
|
|
97
|
+
* against the model's own parse time. `rc-regression.yml` runs both
|
|
98
|
+
* conditions in one job; see design/new/perf-measurement.md §"The settle
|
|
99
|
+
* also cleans the window".
|
|
92
100
|
*
|
|
93
101
|
* @return {Promise<SettledMemorySample | undefined>} The settled sample, or
|
|
94
102
|
* undefined where no collector is exposed — in which case the caller emits
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retained_memory.d.ts","sourceRoot":"","sources":["../../../src/core/retained_memory.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AASH;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAElC,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAA;IAEhB,8CAA8C;IAC9C,aAAa,EAAE,MAAM,CAAA;IAErB,uDAAuD;IACvD,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,8DAA8D;AAC9D,MAAM,WAAW,gBAAgB;IAE/B,2DAA2D;IAC3D,KAAK,EAAE,MAAM,CAAA;IAEb,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAA;IAElB,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAA;CACnB;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,IAAI,CAAE,MAAM,IAAI,CAAE,GAAG,SAAS,CAKtD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,IAAI,OAAO,CAKzC;AAED
|
|
1
|
+
{"version":3,"file":"retained_memory.d.ts","sourceRoot":"","sources":["../../../src/core/retained_memory.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AASH;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAElC,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAA;IAEhB,8CAA8C;IAC9C,aAAa,EAAE,MAAM,CAAA;IAErB,uDAAuD;IACvD,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,8DAA8D;AAC9D,MAAM,WAAW,gBAAgB;IAE/B,2DAA2D;IAC3D,KAAK,EAAE,MAAM,CAAA;IAEb,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAA;IAElB,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAA;CACnB;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,IAAI,CAAE,MAAM,IAAI,CAAE,GAAG,SAAS,CAKtD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,IAAI,OAAO,CAKzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAsB,qBAAqB,IACvC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAuB3C;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,4BAA4B,CAC9C,QAAQ,EAAE,MAAM,GAAI,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAG/D;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,gBAAgB,CAC5B,QAAQ,EAAE,mBAAmB,GAAG,SAAS,EACzC,QAAQ,EAAE,mBAAmB,GAAG,SAAS,GAAI,gBAAgB,GAAG,SAAS,CAa5E"}
|
|
@@ -72,8 +72,16 @@ export function canSettleMemory() {
|
|
|
72
72
|
* produces `parseTimeMs` / `geometryTimeMs` / `totalTimeMs` — baseline before
|
|
73
73
|
* the load starts, retained sample after teardown. That property is what
|
|
74
74
|
* makes the retention columns free, and it is checkable by measurement: a run
|
|
75
|
-
* with `--expose-gc` and a run without share identical code, so
|
|
76
|
-
* columns
|
|
75
|
+
* with `--expose-gc` and a run without share identical code, so any movement
|
|
76
|
+
* in the timing columns between them is down to the settle. The SIGN says
|
|
77
|
+
* which way: gc-on slower is the settle leaking into the measured window,
|
|
78
|
+
* which is a bug; gc-on faster is the pre-load settle taking engine-init
|
|
79
|
+
* garbage OUT of that window. That is an absolute cost of about 10 ms per
|
|
80
|
+
* load, measured as 13-16% of `parseTimeMs` on models that parse in ~60 ms
|
|
81
|
+
* and unresolvable against one that parses in 578 ms, so read a ratio
|
|
82
|
+
* against the model's own parse time. `rc-regression.yml` runs both
|
|
83
|
+
* conditions in one job; see design/new/perf-measurement.md §"The settle
|
|
84
|
+
* also cleans the window".
|
|
77
85
|
*
|
|
78
86
|
* @return {Promise<SettledMemorySample | undefined>} The settled sample, or
|
|
79
87
|
* undefined where no collector is exposed — in which case the caller emits
|
|
@@ -29,9 +29,18 @@ const CHILD_NODE_FLAGS = '--experimental-specifier-resolution=node';
|
|
|
29
29
|
* both samples sit outside the timed region — baseline before the load,
|
|
30
30
|
* retained sample after teardown — so a run with the flag and a run without
|
|
31
31
|
* should produce the same timing columns from identical code. Set the
|
|
32
|
-
* variable to `0`, `false` or `off` for the without side of that A/B
|
|
33
|
-
*
|
|
34
|
-
*
|
|
32
|
+
* variable to `0`, `false` or `off` for the without side of that A/B; the
|
|
33
|
+
* `rebless` job of `.github/workflows/rc-regression.yml` does exactly that
|
|
34
|
+
* for its second, control pass, because the two conditions have to share a
|
|
35
|
+
* runner for the comparison to mean anything (between two CI runs the timing
|
|
36
|
+
* columns carry a ~1.5x runner scale factor against a ~1% effect).
|
|
37
|
+
*
|
|
38
|
+
* Read any movement with its sign. Gc-on SLOWER means the settle is reaching
|
|
39
|
+
* the measured window, which is a bug rather than a tolerance. Gc-on FASTER
|
|
40
|
+
* means the opposite — the pre-load settle collects engine-init garbage that
|
|
41
|
+
* the flag-off run collects inside the timed region instead. That is about
|
|
42
|
+
* 10 ms per load: 13-16% of `parseTimeMs` on a model that parses in ~60 ms,
|
|
43
|
+
* and lost in the noise on one that parses in 578 ms.
|
|
35
44
|
*
|
|
36
45
|
* @return {boolean} True unless CONWAY_PERF_EXPOSE_GC disables it.
|
|
37
46
|
*/
|
|
@@ -16,7 +16,7 @@ import { createRequire } from 'module';
|
|
|
16
16
|
const require_ = createRequire(import.meta.url);
|
|
17
17
|
// Resolved from the repo root: the test runs from compiled/src/scripts, and
|
|
18
18
|
// scripts/ is not part of the tsc build. Jest's rootDir is the repo root.
|
|
19
|
-
const { DETAIL_COLUMNS, findPreviousSnapshot, isChronologicalDelta, removeStaleDeltas, writeDetailCsv, versionCompare, } = require_(path.resolve(process.cwd(), 'scripts/bless_perf_snapshot.cjs'));
|
|
19
|
+
const { DETAIL_COLUMNS, findPreviousSnapshot, isChronologicalDelta, removeStaleDeltas, renderReadme, writeDetailCsv, versionCompare, } = require_(path.resolve(process.cwd(), 'scripts/bless_perf_snapshot.cjs'));
|
|
20
20
|
const { parseCsv } = require_(path.resolve(process.cwd(), 'scripts/csv_rfc4180.cjs'));
|
|
21
21
|
let workDir;
|
|
22
22
|
beforeEach(() => {
|
|
@@ -328,3 +328,90 @@ describe('isChronologicalDelta', () => {
|
|
|
328
328
|
}
|
|
329
329
|
});
|
|
330
330
|
});
|
|
331
|
+
describe('renderReadme', () => {
|
|
332
|
+
const info = {
|
|
333
|
+
engine: 'conway1.560.1600-ci',
|
|
334
|
+
repoName: 'test-models',
|
|
335
|
+
modelCount: 97,
|
|
336
|
+
retentionCount: 97,
|
|
337
|
+
deltaName: 'conway1.451.1357-ci_1.560.1600_delta.csv',
|
|
338
|
+
previousName: 'conway1.451.1357-ci_test-models',
|
|
339
|
+
};
|
|
340
|
+
test('does not repeat the pre-#553 claim that geometryMemoryMb is N/A', () => {
|
|
341
|
+
// #553 restored geometryMemoryMb on the conway-native writer (SKYLARK250
|
|
342
|
+
// reads 185.22 against the 185.836 recorded at 1.451). The README shipped
|
|
343
|
+
// beside the 1.549 snapshot still says it is unmeasured, which is now the
|
|
344
|
+
// opposite of the truth for a column carrying real data; that text was
|
|
345
|
+
// removed from this template, and this keeps it out.
|
|
346
|
+
const text = renderReadme(info);
|
|
347
|
+
expect(text).not.toContain('`geometryMemoryMb` is absent here');
|
|
348
|
+
expect(text).toContain('`schemaVersion`,\n`preprocessorVersion` and `originatingSystem` are `N/A`');
|
|
349
|
+
// The N/A-is-not-a-zero framing is the point of #548 and must survive for
|
|
350
|
+
// the columns genuinely absent from an older snapshot.
|
|
351
|
+
expect(text).toContain('That is a missing\nmeasurement, not a zero');
|
|
352
|
+
});
|
|
353
|
+
test('says on its own face whether the settle ran', () => {
|
|
354
|
+
// A directory of N/A retention must explain itself without anyone having
|
|
355
|
+
// to find the workflow that produced it.
|
|
356
|
+
expect(renderReadme(info)).toContain('Retention is measured on 97 of 97');
|
|
357
|
+
expect(renderReadme({ ...info, retentionCount: 0 }))
|
|
358
|
+
.toContain('Retention is `N/A` on every row here');
|
|
359
|
+
});
|
|
360
|
+
test('names the blessed pass as the one this snapshot came from', () => {
|
|
361
|
+
const text = renderReadme(info);
|
|
362
|
+
expect(text).toContain('CONWAY_PERF_EXPOSE_GC=0');
|
|
363
|
+
expect(text).toContain('The control pass is never blessed');
|
|
364
|
+
});
|
|
365
|
+
test('the N/A inventory covers the retention columns, not just FAIL rows', () => {
|
|
366
|
+
// A settle-less snapshot used to assert both 'Every other column is
|
|
367
|
+
// measured - with the exception of a row whose loadStatus is not OK' and
|
|
368
|
+
// 'Retention is `N/A` on every row here', four paragraphs apart in one
|
|
369
|
+
// file. The retention columns read N/A on an OK row whenever the run had
|
|
370
|
+
// no --expose-gc, which is exactly the run this branch describes.
|
|
371
|
+
const text = renderReadme({ ...info, retentionCount: 0 });
|
|
372
|
+
expect(text).toContain('Retention is `N/A` on every row here');
|
|
373
|
+
expect(text).not.toContain('Every other column is measured — with the');
|
|
374
|
+
expect(text).toContain('the three retention columns carry `N/A`');
|
|
375
|
+
expect(text).toContain('on an `OK` row as\nmuch as a failed one');
|
|
376
|
+
});
|
|
377
|
+
test('attributes the geometryMemoryMb split to the writers #555 measured', () => {
|
|
378
|
+
// #555 measured 16.8 vs 22.3 MB between `ifc_command_line_main` and the
|
|
379
|
+
// IFC regression child - two IFC pipelines. MB-Khaya never reaches the
|
|
380
|
+
// AP214 child, so pinning that figure to the IFC-row-vs-STEP-row split
|
|
381
|
+
// this file mixes would cite evidence for a claim it does not support.
|
|
382
|
+
const text = renderReadme(info);
|
|
383
|
+
expect(text).toContain('The IFC **CLI** and the IFC\nregression child read 16.8 vs 22.3 MB');
|
|
384
|
+
expect(text).toContain('has not been measured to disagree');
|
|
385
|
+
});
|
|
386
|
+
test('describes the second-engine term as closed by #557, not as current', () => {
|
|
387
|
+
// conway#557 landed: both regression children now extract on the engine
|
|
388
|
+
// main() initialised. A README still saying an IFC row carries ~100 MB of
|
|
389
|
+
// second engine would be describing a world that no longer exists - and
|
|
390
|
+
// the boundary that DOES matter now is that pre-#557 snapshots carry the
|
|
391
|
+
// constant and this one does not.
|
|
392
|
+
const text = renderReadme(info);
|
|
393
|
+
expect(text).toContain('carried a second, unrelated split until conway#557');
|
|
394
|
+
expect(text).toContain('~55-60 MB constant on every IFC row');
|
|
395
|
+
expect(text).not.toContain('roughly 100 MB');
|
|
396
|
+
expect(text).toContain('A snapshot blessed\nbefore conway#557');
|
|
397
|
+
});
|
|
398
|
+
test('warns off the misreadings the columns invite', () => {
|
|
399
|
+
const text = renderReadme(info);
|
|
400
|
+
// Retention is live model + leak, not leak.
|
|
401
|
+
expect(text).toContain('still-live model plus anything genuinely leaked');
|
|
402
|
+
// The two columns that are pipeline-scoped, with their issues.
|
|
403
|
+
expect(text).toContain('conway/issues/555');
|
|
404
|
+
expect(text).toContain('conway/issues/557');
|
|
405
|
+
// The third native quantity, and the arrayBuffers/external subset rule.
|
|
406
|
+
expect(text).toContain('getAllocationSize');
|
|
407
|
+
expect(text).toContain('ArrayBuffer subset');
|
|
408
|
+
// Cross-run timing carries the runner scale factor.
|
|
409
|
+
expect(text).toContain('median 1.55x');
|
|
410
|
+
});
|
|
411
|
+
test('names the delta and its predecessor, or says there is none', () => {
|
|
412
|
+
expect(renderReadme(info)).toContain('`conway1.451.1357-ci_1.560.1600_delta.csv` diffs this run against ' +
|
|
413
|
+
'`../conway1.451.1357-ci_test-models/`.');
|
|
414
|
+
expect(renderReadme({ ...info, deltaName: '', previousName: '' }))
|
|
415
|
+
.toContain('no delta was produced');
|
|
416
|
+
});
|
|
417
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"perf_ab_compare.test.d.ts","sourceRoot":"","sources":["../../../src/scripts/perf_ab_compare.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import os from 'os';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { afterEach, beforeEach, describe, expect, test } from '@jest/globals';
|
|
5
|
+
import { createRequire } from 'module';
|
|
6
|
+
/**
|
|
7
|
+
* The rc-regression GC-settle A/B comparator (scripts/perf_ab_compare.cjs).
|
|
8
|
+
*
|
|
9
|
+
* The rc job runs the corpus twice in one job — the blessed pass in the
|
|
10
|
+
* shipped configuration, then a control pass with `CONWAY_PERF_EXPOSE_GC=0` —
|
|
11
|
+
* and differences them there, because between two separate CI runs the timing
|
|
12
|
+
* columns carry a ~1.5x runner scale factor two orders of magnitude larger
|
|
13
|
+
* than the ~1% effect under test (conway#554). This pins the three things
|
|
14
|
+
* that make the resulting number readable: that a control pass which measured
|
|
15
|
+
* retention is called out as an invalid A/B rather than reported as a result,
|
|
16
|
+
* that `N/A` propagates instead of being coerced to 0 (#548), and that
|
|
17
|
+
* `Date.now()`-quantised rows are excluded from the ratio statistics rather
|
|
18
|
+
* than allowed to dominate them.
|
|
19
|
+
*/
|
|
20
|
+
/* eslint-disable no-magic-numbers -- the literals here are the fixture
|
|
21
|
+
timings and percentile fractions the cases are about; naming a 0.1 or a
|
|
22
|
+
110 ms parse would hide the shape each case exists to express. */
|
|
23
|
+
const require_ = createRequire(import.meta.url);
|
|
24
|
+
const { RATIO_FLOOR_MS, checkSwitch, compareRuns, numeric, percentile, readPerfCsv, renderCsv, renderMarkdown, summariseColumn, } = require_(path.resolve(process.cwd(), 'scripts/perf_ab_compare.cjs'));
|
|
25
|
+
/**
|
|
26
|
+
* A perf.csv row with the columns this comparator reads, overridable per test.
|
|
27
|
+
*
|
|
28
|
+
* @param overrides Column values to set.
|
|
29
|
+
* @return {PerfRow} The row.
|
|
30
|
+
*/
|
|
31
|
+
function row(overrides) {
|
|
32
|
+
return {
|
|
33
|
+
status: 'OK',
|
|
34
|
+
parseTimeMs: '100',
|
|
35
|
+
geometryTimeMs: '1000',
|
|
36
|
+
totalTimeMs: '1100',
|
|
37
|
+
heapUsedMb: '50.00',
|
|
38
|
+
rssMb: '300.00',
|
|
39
|
+
peakRssMb: '310.00',
|
|
40
|
+
retainedRssMb: 'N/A',
|
|
41
|
+
retainedHeapUsedMb: 'N/A',
|
|
42
|
+
retainedExternalMb: 'N/A',
|
|
43
|
+
...overrides,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* A blessed-pass row: the settle ran, so retention is measured.
|
|
48
|
+
*
|
|
49
|
+
* @param overrides Column values to set.
|
|
50
|
+
* @return {PerfRow} The row.
|
|
51
|
+
*/
|
|
52
|
+
function blessedRow(overrides) {
|
|
53
|
+
return row({
|
|
54
|
+
retainedRssMb: '380.00',
|
|
55
|
+
retainedHeapUsedMb: '10.00',
|
|
56
|
+
retainedExternalMb: '47.00',
|
|
57
|
+
...overrides,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
describe('readPerfCsv', () => {
|
|
61
|
+
let workDir;
|
|
62
|
+
beforeEach(() => {
|
|
63
|
+
workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'perf-ab-'));
|
|
64
|
+
});
|
|
65
|
+
afterEach(() => {
|
|
66
|
+
fs.rmSync(workDir, { recursive: true, force: true });
|
|
67
|
+
});
|
|
68
|
+
test('reads the 16-column perf.csv both passes write', () => {
|
|
69
|
+
// The exact header ifc_regression_main.ts emits, with a control-pass row:
|
|
70
|
+
// the file this comparator is pointed at in CI, not a reduced stand-in.
|
|
71
|
+
const file = path.join(workDir, 'perf-nogc.csv');
|
|
72
|
+
fs.writeFileSync(file, 'file,status,parseTimeMs,geometryTimeMs,totalTimeMs,geometryMemoryMb,' +
|
|
73
|
+
'peakWasmHeapMb,rssMb,peakRssMb,heapUsedMb,heapTotalMb,externalMb,' +
|
|
74
|
+
'arrayBuffersMb,retainedRssMb,retainedHeapUsedMb,retainedExternalMb\n' +
|
|
75
|
+
'AC20-FZK-Haus.ifc,OK,76,436,512,1.69,35.13,289.39,289.52,54.40,84.34,' +
|
|
76
|
+
'9.98,7.93,N/A,N/A,N/A\n');
|
|
77
|
+
const rows = readPerfCsv(file);
|
|
78
|
+
expect(rows).toHaveLength(1);
|
|
79
|
+
expect(rows[0].file).toBe('AC20-FZK-Haus.ifc');
|
|
80
|
+
expect(rows[0].totalTimeMs).toBe('512');
|
|
81
|
+
expect(rows[0].retainedRssMb).toBe('N/A');
|
|
82
|
+
});
|
|
83
|
+
test('an empty file is no rows, not a throw', () => {
|
|
84
|
+
const file = path.join(workDir, 'empty.csv');
|
|
85
|
+
fs.writeFileSync(file, '');
|
|
86
|
+
expect(readPerfCsv(file)).toEqual([]);
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
describe('numeric', () => {
|
|
90
|
+
test('treats N/A and blank as absent rather than zero', () => {
|
|
91
|
+
// #548: parseValue returning 0.0 for a missing column produced a phantom
|
|
92
|
+
// -185.836 MB "win" for SKYLARK250. Nothing in this file may repeat it.
|
|
93
|
+
expect(numeric('N/A')).toBeNull();
|
|
94
|
+
expect(numeric('')).toBeNull();
|
|
95
|
+
expect(numeric(undefined)).toBeNull();
|
|
96
|
+
expect(numeric('0')).toBe(0);
|
|
97
|
+
expect(numeric('12.5')).toBe(12.5);
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
describe('percentile', () => {
|
|
101
|
+
test('interpolates between samples and handles the empty case', () => {
|
|
102
|
+
expect(percentile([1, 2, 3, 4], 0.5)).toBeCloseTo(2.5);
|
|
103
|
+
expect(percentile([1, 2, 3, 4, 5], 0.1)).toBeCloseTo(1.4);
|
|
104
|
+
expect(percentile([], 0.5)).toBeNull();
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
describe('checkSwitch', () => {
|
|
108
|
+
test('a control pass with no retention against a blessed pass with it is valid', () => {
|
|
109
|
+
const check = checkSwitch([blessedRow({ file: 'a.ifc' }), blessedRow({ file: 'b.ifc' })], [row({ file: 'a.ifc' }), row({ file: 'b.ifc' })]);
|
|
110
|
+
expect(check.valid).toBe(true);
|
|
111
|
+
expect(check.blessedMeasured).toBe(2);
|
|
112
|
+
expect(check.controlMeasured).toBe(0);
|
|
113
|
+
});
|
|
114
|
+
test('a control pass that measured retention is not an A/B at all', () => {
|
|
115
|
+
// The env switch never reached the children, so both passes ran the same
|
|
116
|
+
// configuration and every timing ratio below is noise against noise.
|
|
117
|
+
const check = checkSwitch([blessedRow({ file: 'a.ifc' })], [blessedRow({ file: 'a.ifc' })]);
|
|
118
|
+
expect(check.valid).toBe(false);
|
|
119
|
+
expect(check.controlMeasured).toBe(1);
|
|
120
|
+
});
|
|
121
|
+
test('a blessed pass with no retention is equally not an A/B', () => {
|
|
122
|
+
const check = checkSwitch([row({ file: 'a.ifc' })], [row({ file: 'a.ifc' })]);
|
|
123
|
+
expect(check.valid).toBe(false);
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
describe('summariseColumn', () => {
|
|
127
|
+
const pairs = (blessed, control) => blessed.map((value, index) => ({
|
|
128
|
+
file: `m${index}.ifc`,
|
|
129
|
+
blessed: row({ file: `m${index}.ifc`, ...value }),
|
|
130
|
+
control: row({ file: `m${index}.ifc`, ...control[index] }),
|
|
131
|
+
}));
|
|
132
|
+
test('ratio, sign count and median delta', () => {
|
|
133
|
+
const stat = summariseColumn(pairs([{ totalTimeMs: '110' }, { totalTimeMs: '90' }, { totalTimeMs: '100' }], [{ totalTimeMs: '100' }, { totalTimeMs: '100' }, { totalTimeMs: '100' }]), 'totalTimeMs', true);
|
|
134
|
+
expect(stat.n).toBe(3);
|
|
135
|
+
expect(stat.medianRatio).toBeCloseTo(1.0);
|
|
136
|
+
expect(stat.slower).toBe(1);
|
|
137
|
+
expect(stat.medianDelta).toBe(0);
|
|
138
|
+
});
|
|
139
|
+
test('rows under the quantisation floor are counted out, not counted in', () => {
|
|
140
|
+
// A 3 ms parse carries a +/-1 ms Date.now() tick — a 33% "ratio" that is
|
|
141
|
+
// pure rounding, and there are enough tiny models in the corpus to swamp
|
|
142
|
+
// the median with them.
|
|
143
|
+
const stat = summariseColumn(pairs([{ parseTimeMs: '3' }, { parseTimeMs: '4' }, { parseTimeMs: '200' }], [{ parseTimeMs: '1' }, { parseTimeMs: '2' }, { parseTimeMs: '100' }]), 'parseTimeMs', true);
|
|
144
|
+
expect(RATIO_FLOOR_MS).toBe(10);
|
|
145
|
+
expect(stat.floored).toBe(2);
|
|
146
|
+
expect(stat.n).toBe(1);
|
|
147
|
+
expect(stat.medianRatio).toBeCloseTo(2.0);
|
|
148
|
+
});
|
|
149
|
+
test('a zero on the control side is floored, not dropped uncounted', () => {
|
|
150
|
+
// 0 ms is BELOW the 10 ms floor, so it belongs in `floored`. Testing
|
|
151
|
+
// `off === 0` ahead of the floor dropped such a row out of both counts,
|
|
152
|
+
// leaving n + floored unable to account for every pair - which the
|
|
153
|
+
// module header promises they can. Reachable: box.ifc parses in 1 ms in
|
|
154
|
+
// the committed snapshots, one Date.now() tick from 0, and a
|
|
155
|
+
// zero-geometry model's geometryTimeMs is the same story.
|
|
156
|
+
const stat = summariseColumn(pairs([{ geometryTimeMs: '0' }, { geometryTimeMs: '200' }], [{ geometryTimeMs: '0' }, { geometryTimeMs: '100' }]), 'geometryTimeMs', true);
|
|
157
|
+
expect(stat.floored).toBe(1);
|
|
158
|
+
expect(stat.n).toBe(1);
|
|
159
|
+
expect(stat.floored + stat.n).toBe(2);
|
|
160
|
+
});
|
|
161
|
+
test('the floor is not applied to the memory columns', () => {
|
|
162
|
+
const stat = summariseColumn(pairs([{ heapUsedMb: '5.0' }], [{ heapUsedMb: '8.0' }]), 'heapUsedMb', false);
|
|
163
|
+
expect(stat.floored).toBe(0);
|
|
164
|
+
expect(stat.n).toBe(1);
|
|
165
|
+
});
|
|
166
|
+
test('an unmeasured cell on either side drops the pair, it does not read zero', () => {
|
|
167
|
+
const stat = summariseColumn(pairs([{ totalTimeMs: 'N/A' }, { totalTimeMs: '200' }], [{ totalTimeMs: '100' }, { totalTimeMs: 'N/A' }]), 'totalTimeMs', true);
|
|
168
|
+
expect(stat.n).toBe(0);
|
|
169
|
+
expect(stat.medianRatio).toBeNull();
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
describe('compareRuns', () => {
|
|
173
|
+
test('joins on file, and reports rows only one pass produced', () => {
|
|
174
|
+
const comparison = compareRuns([blessedRow({ file: 'a.ifc' }), blessedRow({ file: 'only-blessed.ifc' })], [row({ file: 'a.ifc' }), row({ file: 'only-control.ifc' })]);
|
|
175
|
+
expect(comparison.pairs.map((pair) => pair.file)).toEqual(['a.ifc']);
|
|
176
|
+
expect(comparison.blessedOnly).toEqual(['only-blessed.ifc']);
|
|
177
|
+
expect(comparison.controlOnly).toEqual(['only-control.ifc']);
|
|
178
|
+
});
|
|
179
|
+
test('a model that failed in either pass is not timed against one that did not', () => {
|
|
180
|
+
const comparison = compareRuns([blessedRow({ file: 'a.ifc', status: 'FAIL' })], [row({ file: 'a.ifc' })]);
|
|
181
|
+
expect(comparison.pairs).toHaveLength(0);
|
|
182
|
+
});
|
|
183
|
+
test('summarises the three timing columns and the memory columns', () => {
|
|
184
|
+
const comparison = compareRuns([blessedRow({ file: 'a.ifc' })], [row({ file: 'a.ifc' })]);
|
|
185
|
+
expect(comparison.timing.map((stat) => stat.column))
|
|
186
|
+
.toEqual(['parseTimeMs', 'geometryTimeMs', 'totalTimeMs']);
|
|
187
|
+
expect(comparison.memory.map((stat) => stat.column))
|
|
188
|
+
.toEqual(['heapUsedMb', 'rssMb', 'peakRssMb']);
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
describe('renderMarkdown', () => {
|
|
192
|
+
test('a valid A/B says so, and reports the sign rule in both directions', () => {
|
|
193
|
+
const text = renderMarkdown(compareRuns([blessedRow({ file: 'a.ifc' })], [row({ file: 'a.ifc' })]), 'bldrs-ai/test-models');
|
|
194
|
+
expect(text).toContain('**Switch check: OK.**');
|
|
195
|
+
expect(text).toContain('gc on SLOWER');
|
|
196
|
+
expect(text).toContain('gc on FASTER');
|
|
197
|
+
});
|
|
198
|
+
test('an invalid A/B is labelled INVALID rather than presented as a result', () => {
|
|
199
|
+
const text = renderMarkdown(compareRuns([blessedRow({ file: 'a.ifc' })], [blessedRow({ file: 'a.ifc' })]), 'bldrs-ai/test-models');
|
|
200
|
+
expect(text).toContain('INVALID');
|
|
201
|
+
expect(text).toContain('answers nothing about the settle');
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
describe('renderCsv', () => {
|
|
205
|
+
test('carries both passes, the ratio, and the blessed retention columns', () => {
|
|
206
|
+
const text = renderCsv(compareRuns([blessedRow({ file: 'a.ifc', totalTimeMs: '110' })], [row({ file: 'a.ifc', totalTimeMs: '100' })]));
|
|
207
|
+
const [header, dataRow] = text.trim().split('\n');
|
|
208
|
+
expect(header).toContain('totalTimeMs_gcOn,totalTimeMs_gcOff,totalTimeMs_ratio');
|
|
209
|
+
expect(header).toContain('retainedRssMb_gcOn');
|
|
210
|
+
expect(dataRow).toContain('110,100,1.1000');
|
|
211
|
+
expect(dataRow).toContain('380.00');
|
|
212
|
+
});
|
|
213
|
+
test('an unmeasured input yields N/A, never a fabricated ratio', () => {
|
|
214
|
+
const text = renderCsv(compareRuns([blessedRow({ file: 'a.ifc', totalTimeMs: 'N/A' })], [row({ file: 'a.ifc', totalTimeMs: '100' })]));
|
|
215
|
+
const dataRow = text.trim().split('\n')[1];
|
|
216
|
+
expect(dataRow).toContain('N/A,100,N/A');
|
|
217
|
+
});
|
|
218
|
+
});
|
|
@@ -5,5 +5,5 @@
|
|
|
5
5
|
// only the first segment (major) is meaningful and is the one CI carries forward.
|
|
6
6
|
// Must stay in `vN.N.N` shape: the CI stamp regex, scripts/updateVersion.mjs, and
|
|
7
7
|
// statistics.ts all match `v\d+\.\d+\.\d+`.
|
|
8
|
-
const versionString = 'Conway v1.
|
|
8
|
+
const versionString = 'Conway v1.560.1539';
|
|
9
9
|
export { versionString };
|