exifreader 4.39.1 → 4.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +324 -166
- package/bin/profile-diff.js +171 -0
- package/bin/profile.js +852 -0
- package/dist/exif-reader.js +1 -1
- package/dist/exif-reader.js.map +1 -1
- package/exif-reader.d.ts +53 -10
- package/package.json +7 -7
- package/src/exif-reader.js +36 -142
- package/src/file-loaders.js +308 -0
- package/src/image-header-avif.js +7 -4
- package/src/image-header-gif.js +21 -1
- package/src/image-header-heic.js +7 -4
- package/src/image-header-iso-bmff.js +169 -18
- package/src/image-header-jpeg.js +53 -1
- package/src/image-header-jxl.js +63 -12
- package/src/image-header-png.js +51 -5
- package/src/image-header-webp.js +32 -2
- package/src/image-header.js +32 -12
- package/src/load-auto.js +310 -0
- package/src/loadview-pipeline.js +66 -1
- package/src/xml.js +39 -1
package/bin/profile.js
ADDED
|
@@ -0,0 +1,852 @@
|
|
|
1
|
+
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
|
4
|
+
|
|
5
|
+
/* eslint-disable no-console */
|
|
6
|
+
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
const crypto = require('crypto');
|
|
10
|
+
const {parseArgs} = require('util');
|
|
11
|
+
const {performance} = require('perf_hooks');
|
|
12
|
+
|
|
13
|
+
const KIB = 1024;
|
|
14
|
+
const MIB = 1024 * 1024;
|
|
15
|
+
const GIB = 1024 * 1024 * 1024;
|
|
16
|
+
|
|
17
|
+
const EXIFREADER_ROOT_DIR = path.join(__dirname, '..');
|
|
18
|
+
const DEFAULT_DIR = path.join('test', 'fixtures', 'images');
|
|
19
|
+
const CACHE_FILE = path.join(EXIFREADER_ROOT_DIR, '.profile-cache.json');
|
|
20
|
+
const KNOWN_EXTENSIONS = new Set([
|
|
21
|
+
'jpg', 'jpeg', 'jpg_original',
|
|
22
|
+
'png',
|
|
23
|
+
'gif',
|
|
24
|
+
'tiff', 'tif',
|
|
25
|
+
'webp',
|
|
26
|
+
'heic', 'heif',
|
|
27
|
+
'avif',
|
|
28
|
+
'jxl'
|
|
29
|
+
]);
|
|
30
|
+
const TYPE_ALIASES = {
|
|
31
|
+
jpg: ['jpg', 'jpeg', 'jpg_original'],
|
|
32
|
+
jpeg: ['jpg', 'jpeg', 'jpg_original'],
|
|
33
|
+
tiff: ['tiff', 'tif'],
|
|
34
|
+
tif: ['tiff', 'tif'],
|
|
35
|
+
heic: ['heic', 'heif'],
|
|
36
|
+
heif: ['heic', 'heif']
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
main().catch((error) => {
|
|
40
|
+
console.error(error && error.stack || error);
|
|
41
|
+
process.exit(1);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
async function main() {
|
|
45
|
+
const options = parseCliArgs();
|
|
46
|
+
|
|
47
|
+
if (options.help) {
|
|
48
|
+
printHelp();
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
if (options.lengthAuto && !options.includeIo) {
|
|
53
|
+
console.error('--length-auto requires --include-io (it is an IO-loop feature).');
|
|
54
|
+
process.exit(1);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const ExifReader = loadExifReader();
|
|
58
|
+
const libraryHash = computeLibraryHash();
|
|
59
|
+
|
|
60
|
+
const targetDir = path.resolve(options.dir);
|
|
61
|
+
if (!fs.existsSync(targetDir) || !fs.statSync(targetDir).isDirectory()) {
|
|
62
|
+
console.error(`Directory not found: ${targetDir}`);
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const {files, skipped} = discoverFiles(targetDir, options.types);
|
|
67
|
+
if (files.length === 0) {
|
|
68
|
+
console.error(`No matching image files found in ${targetDir}`);
|
|
69
|
+
if (skipped > 0) {
|
|
70
|
+
console.error(`(${skipped} files skipped due to unknown extension or filter)`);
|
|
71
|
+
}
|
|
72
|
+
process.exit(1);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (!options.json) {
|
|
76
|
+
printHeader(options, targetDir, files.length, skipped);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const results = await runBenchmark(ExifReader, files, options);
|
|
80
|
+
|
|
81
|
+
if (options.probeBytes) {
|
|
82
|
+
await runProbeBytes(ExifReader, files, results, options, libraryHash);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (options.json) {
|
|
86
|
+
process.stdout.write(JSON.stringify(buildJsonReport(results, options, targetDir), null, 2) + '\n');
|
|
87
|
+
} else {
|
|
88
|
+
printReport(results, options);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function parseCliArgs() {
|
|
93
|
+
const {values} = parseArgs({
|
|
94
|
+
options: {
|
|
95
|
+
dir: {type: 'string', default: DEFAULT_DIR},
|
|
96
|
+
iterations: {type: 'string', short: 'n', default: '1'},
|
|
97
|
+
warmup: {type: 'string', default: '0'},
|
|
98
|
+
type: {type: 'string', multiple: true, default: []},
|
|
99
|
+
top: {type: 'string', default: '10'},
|
|
100
|
+
json: {type: 'boolean', default: false},
|
|
101
|
+
'include-io': {type: 'boolean', default: false},
|
|
102
|
+
'length-auto': {type: 'boolean', default: false},
|
|
103
|
+
'exclude-mpf': {type: 'boolean', default: false},
|
|
104
|
+
'probe-bytes': {type: 'boolean', default: false},
|
|
105
|
+
'probe-equality': {type: 'string', default: 'keys'},
|
|
106
|
+
help: {type: 'boolean', short: 'h', default: false}
|
|
107
|
+
},
|
|
108
|
+
allowPositionals: false
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
const iterations = parsePositiveInt(values.iterations, 'iterations');
|
|
112
|
+
const warmup = parseNonNegativeInt(values.warmup, 'warmup');
|
|
113
|
+
const top = parseNonNegativeInt(values.top, 'top');
|
|
114
|
+
|
|
115
|
+
if (values['probe-equality'] !== 'keys' && values['probe-equality'] !== 'strict') {
|
|
116
|
+
console.error(`Invalid --probe-equality value: ${values['probe-equality']} (expected "keys" or "strict")`);
|
|
117
|
+
process.exit(1);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const types = normalizeTypes(values.type);
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
dir: values.dir,
|
|
124
|
+
iterations,
|
|
125
|
+
warmup,
|
|
126
|
+
types,
|
|
127
|
+
top,
|
|
128
|
+
json: values.json,
|
|
129
|
+
includeIo: values['include-io'],
|
|
130
|
+
lengthAuto: values['length-auto'],
|
|
131
|
+
excludeMpf: values['exclude-mpf'],
|
|
132
|
+
probeBytes: values['probe-bytes'],
|
|
133
|
+
probeEquality: values['probe-equality'],
|
|
134
|
+
help: values.help
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function parsePositiveInt(value, name) {
|
|
139
|
+
const n = Number.parseInt(value, 10);
|
|
140
|
+
if (!Number.isFinite(n) || n < 1) {
|
|
141
|
+
console.error(`Invalid --${name} value: ${value} (expected positive integer)`);
|
|
142
|
+
process.exit(1);
|
|
143
|
+
}
|
|
144
|
+
return n;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function parseNonNegativeInt(value, name) {
|
|
148
|
+
const n = Number.parseInt(value, 10);
|
|
149
|
+
if (!Number.isFinite(n) || n < 0) {
|
|
150
|
+
console.error(`Invalid --${name} value: ${value} (expected non-negative integer)`);
|
|
151
|
+
process.exit(1);
|
|
152
|
+
}
|
|
153
|
+
return n;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function normalizeTypes(rawTypes) {
|
|
157
|
+
if (!rawTypes || rawTypes.length === 0) {
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
const expanded = new Set();
|
|
161
|
+
for (const raw of rawTypes) {
|
|
162
|
+
const t = raw.toLowerCase().replace(/^\./, '');
|
|
163
|
+
const aliases = TYPE_ALIASES[t] || [t];
|
|
164
|
+
for (const a of aliases) {
|
|
165
|
+
expanded.add(a);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return expanded;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function printHelp() {
|
|
172
|
+
console.log(`Usage: node bin/profile.js [options]
|
|
173
|
+
|
|
174
|
+
Profiles ExifReader against a folder of image files.
|
|
175
|
+
|
|
176
|
+
Options:
|
|
177
|
+
--dir <path> Folder to scan (default: ${DEFAULT_DIR})
|
|
178
|
+
-n, --iterations <n> Times to parse each file (default: 1)
|
|
179
|
+
--warmup <k> Warmup iterations to discard (default: 0)
|
|
180
|
+
--type <ext> Filter by extension; repeat for multiple
|
|
181
|
+
(aliases: jpg → jpg/jpeg/jpg_original,
|
|
182
|
+
tiff → tiff/tif, heic → heic/heif)
|
|
183
|
+
--top <n> Show N slowest files (default: 10, 0 to hide)
|
|
184
|
+
--include-io Time fs read + parse instead of parse-only
|
|
185
|
+
--length-auto Use length: 'auto' (adaptive minimal-fetch).
|
|
186
|
+
Requires --include-io; reports bytes fetched
|
|
187
|
+
and number of IO requests per file.
|
|
188
|
+
--exclude-mpf Pass excludeTags: {mpf: true} to load(). Useful
|
|
189
|
+
with --length-auto to skip MPF sub-image blocks
|
|
190
|
+
that push the metadata range near EOF.
|
|
191
|
+
|
|
192
|
+
All runs pass expanded:true + includeOffsets:true to load() so
|
|
193
|
+
length:'auto' and non-auto runs are directly comparable. The overhead
|
|
194
|
+
versus a default flat load() is in the ~5-10% range.
|
|
195
|
+
--probe-bytes Bisect minimum prefix length needed to parse
|
|
196
|
+
equal to the full file (use for I/O optimization)
|
|
197
|
+
--probe-equality <m> "keys" (default) or "strict"
|
|
198
|
+
--json Emit machine-readable JSON
|
|
199
|
+
-h, --help Show this help
|
|
200
|
+
|
|
201
|
+
Examples:
|
|
202
|
+
node bin/profile.js
|
|
203
|
+
node bin/profile.js -n 10 --warmup 2
|
|
204
|
+
node bin/profile.js --type jpg --type png
|
|
205
|
+
node bin/profile.js --dir /path/to/photos --probe-bytes
|
|
206
|
+
`);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function loadExifReader() {
|
|
210
|
+
const distPath = path.join(EXIFREADER_ROOT_DIR, 'dist', 'exif-reader.js');
|
|
211
|
+
if (!fs.existsSync(distPath)) {
|
|
212
|
+
console.error('dist/exif-reader.js not found. Run "npm run build" first.');
|
|
213
|
+
process.exit(1);
|
|
214
|
+
}
|
|
215
|
+
return require(distPath);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function computeLibraryHash() {
|
|
219
|
+
const distPath = path.join(EXIFREADER_ROOT_DIR, 'dist', 'exif-reader.js');
|
|
220
|
+
const stat = fs.statSync(distPath);
|
|
221
|
+
const hash = crypto.createHash('sha1');
|
|
222
|
+
hash.update(String(stat.size));
|
|
223
|
+
hash.update(':');
|
|
224
|
+
hash.update(String(stat.mtimeMs));
|
|
225
|
+
return hash.digest('hex').slice(0, 16);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function discoverFiles(dir, types) {
|
|
229
|
+
const entries = fs.readdirSync(dir, {withFileTypes: true});
|
|
230
|
+
const files = [];
|
|
231
|
+
let skipped = 0;
|
|
232
|
+
for (const entry of entries) {
|
|
233
|
+
if (!entry.isFile()) {
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
const ext = path.extname(entry.name).slice(1).toLowerCase();
|
|
237
|
+
if (!KNOWN_EXTENSIONS.has(ext)) {
|
|
238
|
+
skipped++;
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
if (types && !types.has(ext)) {
|
|
242
|
+
skipped++;
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
files.push({
|
|
246
|
+
name: entry.name,
|
|
247
|
+
path: path.join(dir, entry.name),
|
|
248
|
+
ext,
|
|
249
|
+
format: canonicalFormat(ext)
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
files.sort((a, b) => a.name.localeCompare(b.name));
|
|
253
|
+
return {files, skipped};
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function canonicalFormat(ext) {
|
|
257
|
+
if (ext === 'jpeg' || ext === 'jpg_original') {
|
|
258
|
+
return 'jpg';
|
|
259
|
+
}
|
|
260
|
+
if (ext === 'tif') {
|
|
261
|
+
return 'tiff';
|
|
262
|
+
}
|
|
263
|
+
if (ext === 'heif') {
|
|
264
|
+
return 'heic';
|
|
265
|
+
}
|
|
266
|
+
return ext;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function printHeader(options, targetDir, fileCount, skipped) {
|
|
270
|
+
const distPath = path.relative(EXIFREADER_ROOT_DIR, path.join(EXIFREADER_ROOT_DIR, 'dist', 'exif-reader.js'));
|
|
271
|
+
let mode = options.includeIo ? 'parse+IO' : 'parse-only';
|
|
272
|
+
const extras = [];
|
|
273
|
+
if (options.lengthAuto) {
|
|
274
|
+
extras.push('length:auto');
|
|
275
|
+
}
|
|
276
|
+
if (options.excludeMpf) {
|
|
277
|
+
extras.push('no-mpf');
|
|
278
|
+
}
|
|
279
|
+
if (extras.length > 0) {
|
|
280
|
+
mode += ' (' + extras.join(', ') + ')';
|
|
281
|
+
}
|
|
282
|
+
console.log(`ExifReader profiler - entry=${distPath} mode=${mode}`);
|
|
283
|
+
console.log(`dir=${targetDir} iterations=${options.iterations} warmup=${options.warmup}` + (options.probeBytes ? ` probe=${options.probeEquality}` : ''));
|
|
284
|
+
console.log(`Scanned ${fileCount + skipped} files, ${fileCount} matched, ${skipped} skipped`);
|
|
285
|
+
console.log('');
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
async function runBenchmark(ExifReader, files, options) {
|
|
289
|
+
const fileResults = [];
|
|
290
|
+
let preloadedBytes = 0;
|
|
291
|
+
|
|
292
|
+
for (const file of files) {
|
|
293
|
+
const stat = fs.statSync(file.path);
|
|
294
|
+
let buffer = null;
|
|
295
|
+
if (!options.includeIo) {
|
|
296
|
+
buffer = fs.readFileSync(file.path);
|
|
297
|
+
preloadedBytes += buffer.length;
|
|
298
|
+
}
|
|
299
|
+
fileResults.push({
|
|
300
|
+
file,
|
|
301
|
+
sizeBytes: stat.size,
|
|
302
|
+
buffer,
|
|
303
|
+
iterations: [],
|
|
304
|
+
error: null
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
if (!options.json && !options.includeIo) {
|
|
309
|
+
console.log(`Preloaded ${formatBytes(preloadedBytes)} into memory.`);
|
|
310
|
+
console.log('');
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const total = files.length;
|
|
314
|
+
let parsedCount = 0;
|
|
315
|
+
|
|
316
|
+
for (const result of fileResults) {
|
|
317
|
+
for (let i = 0; i < options.warmup; i++) {
|
|
318
|
+
try {
|
|
319
|
+
await parseFile(ExifReader, result, options);
|
|
320
|
+
} catch (_warmupError) { // eslint-disable-line no-unused-vars
|
|
321
|
+
// Warmup errors are discarded; the real run will surface them below.
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
for (let i = 0; i < options.iterations; i++) {
|
|
326
|
+
const start = performance.now();
|
|
327
|
+
try {
|
|
328
|
+
await parseFile(ExifReader, result, options);
|
|
329
|
+
result.iterations.push(performance.now() - start);
|
|
330
|
+
} catch (error) {
|
|
331
|
+
result.error = error && error.message || String(error);
|
|
332
|
+
break;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
parsedCount++;
|
|
337
|
+
if (!options.json && (parsedCount % 25 === 0 || parsedCount === total)) {
|
|
338
|
+
process.stderr.write(` parsed ${parsedCount}/${total}\r`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
if (!options.json) {
|
|
343
|
+
process.stderr.write('\n');
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
return fileResults;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
async function parseFile(ExifReader, result, options) {
|
|
350
|
+
const loadOpts = {expanded: true, includeOffsets: true};
|
|
351
|
+
if (options.lengthAuto) {
|
|
352
|
+
loadOpts.length = 'auto';
|
|
353
|
+
}
|
|
354
|
+
if (options.excludeMpf) {
|
|
355
|
+
loadOpts.excludeTags = {mpf: true};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const target = options.includeIo ? result.file.path : result.buffer;
|
|
359
|
+
const out = ExifReader.load(target, loadOpts);
|
|
360
|
+
const tags = isThenable(out) ? await out : out;
|
|
361
|
+
|
|
362
|
+
if (options.lengthAuto && tags && tags.metadataRange) {
|
|
363
|
+
result.lastFetched = tags.metadataRange.fetched;
|
|
364
|
+
result.lastRequests = tags.metadataRange.requests;
|
|
365
|
+
result.lastEnd = tags.metadataRange.end;
|
|
366
|
+
}
|
|
367
|
+
return tags;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function isThenable(value) {
|
|
371
|
+
return value && typeof value.then === 'function';
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
async function runProbeBytes(ExifReader, files, results, options, libraryHash) {
|
|
375
|
+
const cache = loadCache();
|
|
376
|
+
let updated = false;
|
|
377
|
+
const total = files.length;
|
|
378
|
+
let done = 0;
|
|
379
|
+
|
|
380
|
+
if (!options.json) {
|
|
381
|
+
console.log('');
|
|
382
|
+
console.log(`Probing minimum bytes (equality=${options.probeEquality})...`);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
for (const result of results) {
|
|
386
|
+
if (result.error) {
|
|
387
|
+
result.minBytes = null;
|
|
388
|
+
done++;
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
const stat = fs.statSync(result.file.path);
|
|
392
|
+
const cacheKey = `${result.file.path}|${stat.size}|${stat.mtimeMs}|${libraryHash}|${options.probeEquality}`;
|
|
393
|
+
|
|
394
|
+
if (cache[cacheKey] !== undefined) {
|
|
395
|
+
result.minBytes = cache[cacheKey];
|
|
396
|
+
} else {
|
|
397
|
+
const buffer = result.buffer || fs.readFileSync(result.file.path);
|
|
398
|
+
try {
|
|
399
|
+
result.minBytes = await bisectMinBytes(ExifReader, buffer, options.probeEquality);
|
|
400
|
+
} catch (_probeError) { // eslint-disable-line no-unused-vars
|
|
401
|
+
result.minBytes = null;
|
|
402
|
+
}
|
|
403
|
+
cache[cacheKey] = result.minBytes;
|
|
404
|
+
updated = true;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
done++;
|
|
408
|
+
if (!options.json && (done % 10 === 0 || done === total)) {
|
|
409
|
+
process.stderr.write(` probed ${done}/${total}\r`);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
if (!options.json) {
|
|
414
|
+
process.stderr.write('\n');
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
if (updated) {
|
|
418
|
+
saveCache(cache);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
async function bisectMinBytes(ExifReader, buffer, equalityMode) {
|
|
423
|
+
const full = await parseQuietly(ExifReader, buffer);
|
|
424
|
+
if (full === null) {
|
|
425
|
+
return null;
|
|
426
|
+
}
|
|
427
|
+
const fingerprint = computeFingerprint(full, equalityMode);
|
|
428
|
+
|
|
429
|
+
let lo = 1;
|
|
430
|
+
let hi = buffer.length;
|
|
431
|
+
|
|
432
|
+
while (lo < hi) {
|
|
433
|
+
const mid = Math.floor((lo + hi) / 2);
|
|
434
|
+
const slice = buffer.subarray(0, mid);
|
|
435
|
+
const candidate = await parseQuietly(ExifReader, slice);
|
|
436
|
+
if (candidate !== null && computeFingerprint(candidate, equalityMode) === fingerprint) {
|
|
437
|
+
hi = mid;
|
|
438
|
+
} else {
|
|
439
|
+
lo = mid + 1;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
return lo;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
async function parseQuietly(ExifReader, buffer) {
|
|
446
|
+
try {
|
|
447
|
+
const out = ExifReader.load(buffer);
|
|
448
|
+
return isThenable(out) ? await out : out;
|
|
449
|
+
} catch (_error) { // eslint-disable-line no-unused-vars
|
|
450
|
+
return null;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function computeFingerprint(result, equalityMode) {
|
|
455
|
+
if (equalityMode === 'keys') {
|
|
456
|
+
const keys = Object.keys(result).sort();
|
|
457
|
+
return JSON.stringify(keys);
|
|
458
|
+
}
|
|
459
|
+
return JSON.stringify(normalizeForCompare(result));
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function normalizeForCompare(value) {
|
|
463
|
+
if (value === null || value === undefined) {
|
|
464
|
+
return value;
|
|
465
|
+
}
|
|
466
|
+
if (Array.isArray(value)) {
|
|
467
|
+
return value.map(normalizeForCompare);
|
|
468
|
+
}
|
|
469
|
+
if (Buffer.isBuffer(value)) {
|
|
470
|
+
const hash = crypto.createHash('sha1').update(value).digest('hex');
|
|
471
|
+
return `buf:${value.length}:${hash}`;
|
|
472
|
+
}
|
|
473
|
+
if (typeof value === 'object') {
|
|
474
|
+
const out = {};
|
|
475
|
+
for (const key of Object.keys(value).sort()) {
|
|
476
|
+
out[key] = normalizeForCompare(value[key]);
|
|
477
|
+
}
|
|
478
|
+
return out;
|
|
479
|
+
}
|
|
480
|
+
if (typeof value === 'bigint') {
|
|
481
|
+
return `bigint:${value.toString()}`;
|
|
482
|
+
}
|
|
483
|
+
return value;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
function loadCache() {
|
|
487
|
+
try {
|
|
488
|
+
if (fs.existsSync(CACHE_FILE)) {
|
|
489
|
+
return JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'));
|
|
490
|
+
}
|
|
491
|
+
} catch (_error) { // eslint-disable-line no-unused-vars
|
|
492
|
+
// Corrupt cache: ignore and rebuild.
|
|
493
|
+
}
|
|
494
|
+
return {};
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
function saveCache(cache) {
|
|
498
|
+
fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2) + '\n');
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
function printReport(results, options) {
|
|
502
|
+
const stats = aggregate(results);
|
|
503
|
+
|
|
504
|
+
console.log('');
|
|
505
|
+
printFormatTable(stats, options);
|
|
506
|
+
|
|
507
|
+
if (options.top > 0) {
|
|
508
|
+
console.log('');
|
|
509
|
+
printSlowestFiles(results, options);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
console.log('');
|
|
513
|
+
printOverall(stats, options);
|
|
514
|
+
|
|
515
|
+
const failed = results.filter((r) => r.error);
|
|
516
|
+
if (failed.length > 0) {
|
|
517
|
+
console.log('');
|
|
518
|
+
console.log(`Errors (${failed.length}):`);
|
|
519
|
+
for (const r of failed) {
|
|
520
|
+
console.log(` ${r.file.name}: ${r.error}`);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function aggregate(results) {
|
|
526
|
+
const byFormat = new Map();
|
|
527
|
+
let totalBytes = 0;
|
|
528
|
+
let totalTime = 0;
|
|
529
|
+
let totalParses = 0;
|
|
530
|
+
let totalFiles = 0;
|
|
531
|
+
let totalErrors = 0;
|
|
532
|
+
let totalMinBytes = 0;
|
|
533
|
+
let totalMinBytesFiles = 0;
|
|
534
|
+
let totalFetched = 0;
|
|
535
|
+
let totalFetchedFiles = 0;
|
|
536
|
+
let totalRequests = 0;
|
|
537
|
+
|
|
538
|
+
for (const r of results) {
|
|
539
|
+
const fmt = r.file.format;
|
|
540
|
+
if (!byFormat.has(fmt)) {
|
|
541
|
+
byFormat.set(fmt, {
|
|
542
|
+
format: fmt,
|
|
543
|
+
count: 0,
|
|
544
|
+
bytes: 0,
|
|
545
|
+
medians: [],
|
|
546
|
+
errors: 0,
|
|
547
|
+
minBytesValues: [],
|
|
548
|
+
minBytesRatios: [],
|
|
549
|
+
fetchedValues: [],
|
|
550
|
+
fetchedRatios: [],
|
|
551
|
+
requestsValues: []
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
const bucket = byFormat.get(fmt);
|
|
555
|
+
bucket.count++;
|
|
556
|
+
bucket.bytes += r.sizeBytes;
|
|
557
|
+
totalBytes += r.sizeBytes;
|
|
558
|
+
totalFiles++;
|
|
559
|
+
if (r.error) {
|
|
560
|
+
bucket.errors++;
|
|
561
|
+
totalErrors++;
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
const median = percentile(r.iterations, 0.5);
|
|
565
|
+
const sum = r.iterations.reduce((a, b) => a + b, 0);
|
|
566
|
+
r.median = median;
|
|
567
|
+
r.sumIterations = sum;
|
|
568
|
+
bucket.medians.push(median);
|
|
569
|
+
totalTime += sum;
|
|
570
|
+
totalParses += r.iterations.length;
|
|
571
|
+
|
|
572
|
+
if (typeof r.minBytes === 'number') {
|
|
573
|
+
bucket.minBytesValues.push(r.minBytes);
|
|
574
|
+
bucket.minBytesRatios.push(r.minBytes / r.sizeBytes);
|
|
575
|
+
totalMinBytes += r.minBytes;
|
|
576
|
+
totalMinBytesFiles++;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
if (typeof r.lastFetched === 'number') {
|
|
580
|
+
bucket.fetchedValues.push(r.lastFetched);
|
|
581
|
+
bucket.fetchedRatios.push(r.lastFetched / r.sizeBytes);
|
|
582
|
+
totalFetched += r.lastFetched;
|
|
583
|
+
totalFetchedFiles++;
|
|
584
|
+
}
|
|
585
|
+
if (typeof r.lastRequests === 'number') {
|
|
586
|
+
bucket.requestsValues.push(r.lastRequests);
|
|
587
|
+
totalRequests += r.lastRequests;
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
const formatStats = [];
|
|
592
|
+
for (const bucket of byFormat.values()) {
|
|
593
|
+
const medians = bucket.medians;
|
|
594
|
+
const totalMs = medians.reduce((a, b) => a + b, 0);
|
|
595
|
+
const meanMs = medians.length > 0 ? totalMs / medians.length : 0;
|
|
596
|
+
const p50 = percentile(medians, 0.5);
|
|
597
|
+
const p95 = percentile(medians, 0.95);
|
|
598
|
+
const throughputMiBps = totalMs > 0 ? (bucket.bytes / MIB) / (totalMs / 1000) : 0;
|
|
599
|
+
const bytesP50 = percentile(bucket.minBytesValues, 0.5);
|
|
600
|
+
const bytesP95 = percentile(bucket.minBytesValues, 0.95);
|
|
601
|
+
const meanRatio = meanOf(bucket.minBytesRatios);
|
|
602
|
+
const fetchedP50 = percentile(bucket.fetchedValues, 0.5);
|
|
603
|
+
const fetchedP95 = percentile(bucket.fetchedValues, 0.95);
|
|
604
|
+
const fetchedRatio = meanOf(bucket.fetchedRatios);
|
|
605
|
+
const requestsMean = meanOf(bucket.requestsValues);
|
|
606
|
+
const requestsP95 = percentile(bucket.requestsValues, 0.95);
|
|
607
|
+
formatStats.push({
|
|
608
|
+
format: bucket.format,
|
|
609
|
+
count: bucket.count,
|
|
610
|
+
bytes: bucket.bytes,
|
|
611
|
+
totalMs,
|
|
612
|
+
meanMs,
|
|
613
|
+
p50Ms: p50,
|
|
614
|
+
p95Ms: p95,
|
|
615
|
+
throughputMiBps,
|
|
616
|
+
errors: bucket.errors,
|
|
617
|
+
bytesP50,
|
|
618
|
+
bytesP95,
|
|
619
|
+
meanRatio,
|
|
620
|
+
fetchedP50,
|
|
621
|
+
fetchedP95,
|
|
622
|
+
fetchedRatio,
|
|
623
|
+
requestsMean,
|
|
624
|
+
requestsP95
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
formatStats.sort((a, b) => b.totalMs - a.totalMs);
|
|
629
|
+
|
|
630
|
+
return {
|
|
631
|
+
formats: formatStats,
|
|
632
|
+
overall: {
|
|
633
|
+
totalFiles,
|
|
634
|
+
totalErrors,
|
|
635
|
+
totalParses,
|
|
636
|
+
totalBytes,
|
|
637
|
+
totalTime,
|
|
638
|
+
throughputMiBps: totalTime > 0 ? (totalBytes / MIB) / (totalTime / 1000) : 0,
|
|
639
|
+
minBytesTotal: totalMinBytesFiles > 0 ? totalMinBytes : null,
|
|
640
|
+
minBytesFiles: totalMinBytesFiles,
|
|
641
|
+
minBytesRatio: totalMinBytesFiles > 0 && totalBytes > 0 ? totalMinBytes / totalBytes : null,
|
|
642
|
+
fetchedTotal: totalFetchedFiles > 0 ? totalFetched : null,
|
|
643
|
+
fetchedFiles: totalFetchedFiles,
|
|
644
|
+
fetchedRatio: totalFetchedFiles > 0 && totalBytes > 0 ? totalFetched / totalBytes : null,
|
|
645
|
+
requestsTotal: totalFetchedFiles > 0 ? totalRequests : null,
|
|
646
|
+
requestsMean: totalFetchedFiles > 0 ? totalRequests / totalFetchedFiles : null
|
|
647
|
+
}
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
function meanOf(values) {
|
|
652
|
+
if (!values || values.length === 0) {
|
|
653
|
+
return null;
|
|
654
|
+
}
|
|
655
|
+
let sum = 0;
|
|
656
|
+
for (const v of values) {
|
|
657
|
+
sum += v;
|
|
658
|
+
}
|
|
659
|
+
return sum / values.length;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
function percentile(values, p) {
|
|
663
|
+
if (!values || values.length === 0) {
|
|
664
|
+
return null;
|
|
665
|
+
}
|
|
666
|
+
const sorted = values.slice().sort((a, b) => a - b);
|
|
667
|
+
const idx = Math.min(sorted.length - 1, Math.floor(p * sorted.length));
|
|
668
|
+
return sorted[idx];
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
function printFormatTable(stats, options) {
|
|
672
|
+
const columns = [
|
|
673
|
+
{key: 'format', label: 'format', align: 'left'},
|
|
674
|
+
{key: 'count', label: 'count', align: 'right', format: (v) => String(v)},
|
|
675
|
+
{key: 'totalMiB', label: 'total MiB', align: 'right', format: (v) => v.toFixed(1)},
|
|
676
|
+
{key: 'meanMs', label: 'mean ms', align: 'right', format: (v) => v.toFixed(2)},
|
|
677
|
+
{key: 'p50Ms', label: 'p50 ms', align: 'right', format: (v) => v.toFixed(2)},
|
|
678
|
+
{key: 'p95Ms', label: 'p95 ms', align: 'right', format: (v) => v.toFixed(2)},
|
|
679
|
+
{key: 'mibps', label: 'MiB/s', align: 'right', format: (v) => v.toFixed(1)},
|
|
680
|
+
{key: 'errors', label: 'errors', align: 'right', format: (v) => String(v)}
|
|
681
|
+
];
|
|
682
|
+
if (options.probeBytes) {
|
|
683
|
+
columns.push(
|
|
684
|
+
{key: 'bytesP50', label: 'bytes-p50', align: 'right', format: formatBytesCompact},
|
|
685
|
+
{key: 'bytesP95', label: 'bytes-p95', align: 'right', format: formatBytesCompact},
|
|
686
|
+
{key: 'needed', label: 'needed/total', align: 'right', format: (v) => v === null ? '-' : (v * 100).toFixed(1) + '%'}
|
|
687
|
+
);
|
|
688
|
+
}
|
|
689
|
+
if (options.lengthAuto) {
|
|
690
|
+
columns.push(
|
|
691
|
+
{key: 'fetchedP50', label: 'fetched-p50', align: 'right', format: formatBytesCompact},
|
|
692
|
+
{key: 'fetchedP95', label: 'fetched-p95', align: 'right', format: formatBytesCompact},
|
|
693
|
+
{key: 'fetchedPct', label: 'fetched/total', align: 'right', format: (v) => v === null ? '-' : (v * 100).toFixed(1) + '%'},
|
|
694
|
+
{key: 'reqsMean', label: 'reqs-mean', align: 'right', format: (v) => v === null ? '-' : v.toFixed(2)}
|
|
695
|
+
);
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
const rows = stats.formats.map((f) => ({
|
|
699
|
+
format: f.format,
|
|
700
|
+
count: f.count,
|
|
701
|
+
totalMiB: f.bytes / MIB,
|
|
702
|
+
meanMs: f.meanMs,
|
|
703
|
+
p50Ms: f.p50Ms === null ? 0 : f.p50Ms,
|
|
704
|
+
p95Ms: f.p95Ms === null ? 0 : f.p95Ms,
|
|
705
|
+
mibps: f.throughputMiBps,
|
|
706
|
+
errors: f.errors,
|
|
707
|
+
bytesP50: f.bytesP50,
|
|
708
|
+
bytesP95: f.bytesP95,
|
|
709
|
+
needed: f.meanRatio,
|
|
710
|
+
fetchedP50: f.fetchedP50,
|
|
711
|
+
fetchedP95: f.fetchedP95,
|
|
712
|
+
fetchedPct: f.fetchedRatio,
|
|
713
|
+
reqsMean: f.requestsMean
|
|
714
|
+
}));
|
|
715
|
+
|
|
716
|
+
console.log('Per-format (sorted by total time):');
|
|
717
|
+
printTable(columns, rows);
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
function printSlowestFiles(results, options) {
|
|
721
|
+
const ranked = results
|
|
722
|
+
.filter((r) => !r.error && typeof r.median === 'number')
|
|
723
|
+
.sort((a, b) => b.median - a.median)
|
|
724
|
+
.slice(0, options.top);
|
|
725
|
+
if (ranked.length === 0) {
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
728
|
+
console.log(`Top ${ranked.length} slowest files (by per-file p50 across iterations):`);
|
|
729
|
+
for (const r of ranked) {
|
|
730
|
+
const time = r.median.toFixed(2).padStart(8) + ' ms';
|
|
731
|
+
const size = formatBytesCompact(r.sizeBytes).padStart(9);
|
|
732
|
+
let minBytes = '';
|
|
733
|
+
if (typeof r.minBytes === 'number') {
|
|
734
|
+
const ratio = ((r.minBytes / r.sizeBytes) * 100).toFixed(1);
|
|
735
|
+
minBytes = ' needs ' + formatBytesCompact(r.minBytes).padStart(9) + ` (${ratio}%)`;
|
|
736
|
+
}
|
|
737
|
+
console.log(` ${time} ${size} ${r.file.name}${minBytes}`);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
function printOverall(stats, options) {
|
|
742
|
+
const o = stats.overall;
|
|
743
|
+
const successFiles = o.totalFiles - o.totalErrors;
|
|
744
|
+
console.log(`Overall: ${successFiles} files parsed, ${formatBytes(o.totalBytes)} total, ${o.totalTime.toFixed(0)} ms total parse time, ${o.throughputMiBps.toFixed(1)} MiB/s`);
|
|
745
|
+
if (options.probeBytes && o.minBytesFiles > 0) {
|
|
746
|
+
console.log(`Probe: ${o.minBytesFiles} files bisected, needed ${formatBytes(o.minBytesTotal)} of ${formatBytes(o.totalBytes)} (${(o.minBytesRatio * 100).toFixed(1)}% of total bytes)`);
|
|
747
|
+
}
|
|
748
|
+
if (options.lengthAuto && o.fetchedFiles > 0) {
|
|
749
|
+
console.log(`Auto: ${o.fetchedFiles} files, fetched ${formatBytes(o.fetchedTotal)} of ${formatBytes(o.totalBytes)} (${(o.fetchedRatio * 100).toFixed(1)}% of total bytes), ${o.requestsMean.toFixed(2)} requests/file mean`);
|
|
750
|
+
}
|
|
751
|
+
if (o.totalErrors > 0) {
|
|
752
|
+
console.log(`Errors: ${o.totalErrors}`);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function printTable(columns, rows) {
|
|
757
|
+
const widths = columns.map((col) => {
|
|
758
|
+
const headerLen = col.label.length;
|
|
759
|
+
const maxData = rows.reduce((max, row) => {
|
|
760
|
+
const formatted = col.format ? col.format(row[col.key]) : String(row[col.key] === null || row[col.key] === undefined ? '-' : row[col.key]);
|
|
761
|
+
return Math.max(max, formatted.length);
|
|
762
|
+
}, 0);
|
|
763
|
+
return Math.max(headerLen, maxData);
|
|
764
|
+
});
|
|
765
|
+
|
|
766
|
+
const header = columns.map((col, i) => align(col.label, widths[i], col.align)).join(' ');
|
|
767
|
+
console.log(' ' + header);
|
|
768
|
+
|
|
769
|
+
for (const row of rows) {
|
|
770
|
+
const line = columns.map((col, i) => align(renderCell(col, row[col.key]), widths[i], col.align)).join(' ');
|
|
771
|
+
console.log(' ' + line);
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
function renderCell(col, value) {
|
|
776
|
+
if (value === null || value === undefined) {
|
|
777
|
+
return '-';
|
|
778
|
+
}
|
|
779
|
+
if (col.format) {
|
|
780
|
+
return col.format(value);
|
|
781
|
+
}
|
|
782
|
+
return String(value);
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
function align(text, width, alignment) {
|
|
786
|
+
if (alignment === 'right') {
|
|
787
|
+
return text.padStart(width);
|
|
788
|
+
}
|
|
789
|
+
return text.padEnd(width);
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
function formatBytes(n) {
|
|
793
|
+
if (n === null || n === undefined) {
|
|
794
|
+
return '-';
|
|
795
|
+
}
|
|
796
|
+
if (n >= GIB) {
|
|
797
|
+
return (n / GIB).toFixed(2) + ' GiB';
|
|
798
|
+
}
|
|
799
|
+
if (n >= MIB) {
|
|
800
|
+
return (n / MIB).toFixed(2) + ' MiB';
|
|
801
|
+
}
|
|
802
|
+
if (n >= KIB) {
|
|
803
|
+
return (n / KIB).toFixed(1) + ' KiB';
|
|
804
|
+
}
|
|
805
|
+
return n + ' B';
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
function formatBytesCompact(n) {
|
|
809
|
+
if (n === null || n === undefined) {
|
|
810
|
+
return '-';
|
|
811
|
+
}
|
|
812
|
+
if (n >= GIB) {
|
|
813
|
+
return (n / GIB).toFixed(1) + 'Gi';
|
|
814
|
+
}
|
|
815
|
+
if (n >= MIB) {
|
|
816
|
+
return (n / MIB).toFixed(1) + 'Mi';
|
|
817
|
+
}
|
|
818
|
+
if (n >= KIB) {
|
|
819
|
+
return (n / KIB).toFixed(0) + 'Ki';
|
|
820
|
+
}
|
|
821
|
+
return n + 'B';
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
function buildJsonReport(results, options, targetDir) {
|
|
825
|
+
const stats = aggregate(results);
|
|
826
|
+
return {
|
|
827
|
+
dir: targetDir,
|
|
828
|
+
options: {
|
|
829
|
+
iterations: options.iterations,
|
|
830
|
+
warmup: options.warmup,
|
|
831
|
+
includeIo: options.includeIo,
|
|
832
|
+
lengthAuto: options.lengthAuto,
|
|
833
|
+
excludeMpf: options.excludeMpf,
|
|
834
|
+
probeBytes: options.probeBytes,
|
|
835
|
+
probeEquality: options.probeBytes ? options.probeEquality : null,
|
|
836
|
+
types: options.types ? Array.from(options.types) : null
|
|
837
|
+
},
|
|
838
|
+
formats: stats.formats,
|
|
839
|
+
overall: stats.overall,
|
|
840
|
+
files: results.map((r) => ({
|
|
841
|
+
name: r.file.name,
|
|
842
|
+
format: r.file.format,
|
|
843
|
+
sizeBytes: r.sizeBytes,
|
|
844
|
+
iterations: r.iterations,
|
|
845
|
+
medianMs: typeof r.median === 'number' ? r.median : null,
|
|
846
|
+
minBytes: typeof r.minBytes === 'number' ? r.minBytes : null,
|
|
847
|
+
fetched: typeof r.lastFetched === 'number' ? r.lastFetched : null,
|
|
848
|
+
requests: typeof r.lastRequests === 'number' ? r.lastRequests : null,
|
|
849
|
+
error: r.error
|
|
850
|
+
}))
|
|
851
|
+
};
|
|
852
|
+
}
|