@rubensworks/process-sparql-benchmark-results 1.3.5 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/psbr.js +4 -1
- package/bin/psbr.js.map +1 -1
- package/index.js +5 -1
- package/index.js.map +1 -1
- package/lib/TemplateUtils.js +23 -10
- package/lib/TemplateUtils.js.map +1 -1
- package/lib/cli/CliHelpers.js +27 -14
- package/lib/cli/CliHelpers.js.map +1 -1
- package/lib/cli/CliRunner.d.ts +1 -1
- package/lib/cli/CliRunner.js +5 -5
- package/lib/cli/CliRunner.js.map +1 -1
- package/lib/cli/ITaskContext.js.map +1 -1
- package/lib/cli/commands/CommandCsv.d.ts +1 -1
- package/lib/cli/commands/CommandCsv.js +7 -5
- package/lib/cli/commands/CommandCsv.js.map +1 -1
- package/lib/cli/commands/CommandStats.d.ts +1 -1
- package/lib/cli/commands/CommandStats.js +7 -5
- package/lib/cli/commands/CommandStats.js.map +1 -1
- package/lib/cli/commands/CommandTex.d.ts +1 -1
- package/lib/cli/commands/CommandTex.js +7 -5
- package/lib/cli/commands/CommandTex.js.map +1 -1
- package/lib/cli/commands/csv/CommandBsbm.d.ts +2 -2
- package/lib/cli/commands/csv/CommandBsbm.js +103 -85
- package/lib/cli/commands/csv/CommandBsbm.js.map +1 -1
- package/lib/cli/commands/csv/CommandGithubActionsBenchmark.d.ts +2 -2
- package/lib/cli/commands/csv/CommandGithubActionsBenchmark.js +149 -102
- package/lib/cli/commands/csv/CommandGithubActionsBenchmark.js.map +1 -1
- package/lib/cli/commands/csv/CommandQuery.d.ts +2 -2
- package/lib/cli/commands/csv/CommandQuery.js +111 -93
- package/lib/cli/commands/csv/CommandQuery.js.map +1 -1
- package/lib/cli/commands/csv/CommandSummary.d.ts +2 -2
- package/lib/cli/commands/csv/CommandSummary.js +230 -212
- package/lib/cli/commands/csv/CommandSummary.js.map +1 -1
- package/lib/cli/commands/csv/CommandWins.d.ts +2 -2
- package/lib/cli/commands/csv/CommandWins.js +134 -119
- package/lib/cli/commands/csv/CommandWins.js.map +1 -1
- package/lib/cli/commands/csv/TableSerializerCsv.d.ts +0 -1
- package/lib/cli/commands/csv/TableSerializerMarkdown.d.ts +0 -1
- package/lib/cli/commands/csv/TableSerializerMarkdown.js.map +1 -1
- package/lib/cli/commands/stats/CommandDocker.d.ts +2 -2
- package/lib/cli/commands/stats/CommandDocker.js +66 -48
- package/lib/cli/commands/stats/CommandDocker.js.map +1 -1
- package/lib/cli/commands/tex/CommandQuery.d.ts +2 -2
- package/lib/cli/commands/tex/CommandQuery.js +241 -223
- package/lib/cli/commands/tex/CommandQuery.js.map +1 -1
- package/lib/cli/commands/tex/CommandQueryTimes.d.ts +2 -2
- package/lib/cli/commands/tex/CommandQueryTimes.js +141 -123
- package/lib/cli/commands/tex/CommandQueryTimes.js.map +1 -1
- package/lib/cli/commands/tex/TexUtils.js +38 -20
- package/lib/cli/commands/tex/TexUtils.js.map +1 -1
- package/lib/correctness/CorrectnessCheckerUtils.js +2 -3
- package/lib/correctness/CorrectnessCheckerUtils.js.map +1 -1
- package/package.json +43 -52
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Argv } from 'yargs';
|
|
2
2
|
export declare const command = "summary <experiment-dir...>";
|
|
3
3
|
export declare const desc = "Summarize query results from the given experiments in a table";
|
|
4
|
-
export declare
|
|
5
|
-
export declare
|
|
4
|
+
export declare function builder(yargs: Argv<any>): Argv<any>;
|
|
5
|
+
export declare function handler(argv: Record<string, any>): Promise<void>;
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
3
|
if (k2 === undefined) k2 = k;
|
|
4
|
-
Object.
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
5
9
|
}) : (function(o, m, k, k2) {
|
|
6
10
|
if (k2 === undefined) k2 = k;
|
|
7
11
|
o[k2] = m[k];
|
|
@@ -11,18 +15,30 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
|
|
|
11
15
|
}) : function(o, v) {
|
|
12
16
|
o["default"] = v;
|
|
13
17
|
});
|
|
14
|
-
var __importStar = (this && this.__importStar) || function (
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
};
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
21
35
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
22
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
23
37
|
};
|
|
24
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
-
exports.
|
|
39
|
+
exports.desc = exports.command = void 0;
|
|
40
|
+
exports.builder = builder;
|
|
41
|
+
exports.handler = handler;
|
|
26
42
|
const fs = __importStar(require("fs"));
|
|
27
43
|
const path_1 = __importDefault(require("path"));
|
|
28
44
|
const CorrectnessCheckerUtils_1 = require("../../../correctness/CorrectnessCheckerUtils");
|
|
@@ -32,215 +48,217 @@ const TableSerializerCsv_1 = require("./TableSerializerCsv");
|
|
|
32
48
|
const TableSerializerMarkdown_1 = require("./TableSerializerMarkdown");
|
|
33
49
|
exports.command = 'summary <experiment-dir...>';
|
|
34
50
|
exports.desc = 'Summarize query results from the given experiments in a table';
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
serializer.
|
|
102
|
-
'',
|
|
103
|
-
'$$\\overline{t}$$',
|
|
104
|
-
'$$\\tilde{t}$$',
|
|
105
|
-
'$$\\overline{t}_1$$',
|
|
106
|
-
'$$\\tilde{t}_1$$',
|
|
107
|
-
'$$\\overline{req}$$',
|
|
108
|
-
'$$\\sum ans$$',
|
|
109
|
-
...correctnessReference ? ['$$\\overline{acc}$$'] : [],
|
|
110
|
-
'$$\\sum to$$',
|
|
111
|
-
], {
|
|
112
|
-
align: [
|
|
113
|
-
'left',
|
|
114
|
-
'right',
|
|
115
|
-
'right',
|
|
116
|
-
'right',
|
|
117
|
-
'right',
|
|
118
|
-
'right',
|
|
119
|
-
'right',
|
|
120
|
-
'right',
|
|
121
|
-
'right',
|
|
122
|
-
],
|
|
123
|
-
});
|
|
124
|
-
}
|
|
125
|
-
else {
|
|
126
|
-
serializer.writeHeader([
|
|
127
|
-
'Experiment',
|
|
128
|
-
'Query',
|
|
129
|
-
'Time',
|
|
130
|
-
'Requests',
|
|
131
|
-
'Results',
|
|
132
|
-
...correctnessReference ? ['Correctness'] : [],
|
|
133
|
-
'Timeout',
|
|
134
|
-
], {
|
|
135
|
-
align: [
|
|
136
|
-
'left',
|
|
137
|
-
'right',
|
|
138
|
-
'right',
|
|
139
|
-
'right',
|
|
140
|
-
'right',
|
|
141
|
-
'right',
|
|
142
|
-
'right',
|
|
143
|
-
],
|
|
144
|
-
});
|
|
145
|
-
}
|
|
146
|
-
// Collect data from all experiments
|
|
147
|
-
for (const [experimentId, experimentDirectory] of experimentDirectories.entries()) {
|
|
148
|
-
// Read CSV file
|
|
149
|
-
const timesTotal = {};
|
|
150
|
-
const timesFirstTotal = {};
|
|
151
|
-
const requestsTotal = {};
|
|
152
|
-
let results = {};
|
|
153
|
-
let timeout = {};
|
|
154
|
-
await TexUtils_1.handleCsvFile(experimentDirectory, argv, data => {
|
|
155
|
-
if (!queryRegex || queryRegex.test(data.name)) {
|
|
156
|
-
if (!(data.name in timesTotal)) {
|
|
157
|
-
timesTotal[data.name] = [];
|
|
158
|
-
}
|
|
159
|
-
timesTotal[data.name].push(Number.parseInt(data.time, 10));
|
|
160
|
-
if (!(data.name in requestsTotal)) {
|
|
161
|
-
requestsTotal[data.name] = [];
|
|
162
|
-
}
|
|
163
|
-
requestsTotal[data.name].push(Number
|
|
164
|
-
.parseInt(data.error === 'false' && data.httpRequests ? data.httpRequests : '0', 10));
|
|
165
|
-
results[data.name] = Number.parseInt(data.results, 10);
|
|
166
|
-
timeout[data.name] = data.error === 'true';
|
|
167
|
-
if (data.timestamps) {
|
|
168
|
-
if (!(data.name in timesFirstTotal)) {
|
|
169
|
-
timesFirstTotal[data.name] = [];
|
|
170
|
-
}
|
|
171
|
-
timesFirstTotal[data.name].push(Number.parseInt(data.timestamps.split(' ')[0], 10));
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
});
|
|
175
|
-
// Calculate average
|
|
176
|
-
let timesAverage = {};
|
|
177
|
-
let timesAll = [];
|
|
178
|
-
for (const [query, times] of Object.entries(timesTotal)) {
|
|
179
|
-
timesAverage[query] = TexUtils_1.calcAverage(times);
|
|
180
|
-
timesAll = [...timesAll, ...times];
|
|
181
|
-
}
|
|
182
|
-
let timesFirstAverage = {};
|
|
183
|
-
let timesFirstAll = [];
|
|
184
|
-
for (const [query, times] of Object.entries(timesFirstTotal)) {
|
|
185
|
-
timesFirstAverage[query] = TexUtils_1.calcAverage(times);
|
|
186
|
-
timesFirstAll = [...timesFirstAll, ...times];
|
|
187
|
-
}
|
|
188
|
-
let requestsSum = {};
|
|
189
|
-
for (const [query, requests] of Object.entries(requestsTotal)) {
|
|
190
|
-
requestsSum[query] = TexUtils_1.calcAverage(requests);
|
|
191
|
-
}
|
|
192
|
-
// Calculate correctness
|
|
193
|
-
let correctness;
|
|
194
|
-
if (correctnessReference) {
|
|
195
|
-
correctness = Object.fromEntries(Object.entries(results)
|
|
196
|
-
.map(([key, value]) => [key, correctnessReference.getCorrectness(experimentId, key, value)]));
|
|
197
|
-
}
|
|
198
|
-
// Determine query names
|
|
199
|
-
const queryNames = TexUtils_1.getQueryNames(Object.keys(results), argv);
|
|
200
|
-
timesAverage = TexUtils_1.relabelQueryNames(timesAverage, queryNames);
|
|
201
|
-
timesFirstAverage = TexUtils_1.relabelQueryNames(timesFirstAverage, queryNames);
|
|
202
|
-
requestsSum = TexUtils_1.relabelQueryNames(requestsSum, queryNames);
|
|
203
|
-
results = TexUtils_1.relabelQueryNames(results, queryNames);
|
|
204
|
-
timeout = TexUtils_1.relabelQueryNames(timeout, queryNames);
|
|
205
|
-
// Write rows in output file
|
|
51
|
+
function builder(yargs) {
|
|
52
|
+
return yargs
|
|
53
|
+
.options({
|
|
54
|
+
queryRegex: {
|
|
55
|
+
type: 'string',
|
|
56
|
+
alias: 'q',
|
|
57
|
+
describe: 'Regex for queries to include (before any label overrides). Examples: \'^C\', \'^[^C]\', ...',
|
|
58
|
+
},
|
|
59
|
+
name: {
|
|
60
|
+
type: 'string',
|
|
61
|
+
alias: 'n',
|
|
62
|
+
describe: 'Custom output file name',
|
|
63
|
+
default: 'data_summary.csv',
|
|
64
|
+
},
|
|
65
|
+
inputName: {
|
|
66
|
+
type: 'string',
|
|
67
|
+
describe: 'Custom input file name per experiment',
|
|
68
|
+
default: 'query-times.csv',
|
|
69
|
+
},
|
|
70
|
+
inputDelimiter: {
|
|
71
|
+
type: 'string',
|
|
72
|
+
describe: 'Delimiter for the input CSV file',
|
|
73
|
+
default: ';',
|
|
74
|
+
},
|
|
75
|
+
overrideCombinationLabels: {
|
|
76
|
+
type: 'string',
|
|
77
|
+
describe: 'Comma-separated list of combination labels to use',
|
|
78
|
+
},
|
|
79
|
+
overrideQueryLabels: {
|
|
80
|
+
type: 'string',
|
|
81
|
+
describe: 'Comma-separated list of query labels to use',
|
|
82
|
+
},
|
|
83
|
+
markdown: {
|
|
84
|
+
type: 'boolean',
|
|
85
|
+
describe: 'If the output should be serialized as markdown',
|
|
86
|
+
default: false,
|
|
87
|
+
},
|
|
88
|
+
queryAverage: {
|
|
89
|
+
type: 'boolean',
|
|
90
|
+
describe: 'If queries will be averaged over',
|
|
91
|
+
default: false,
|
|
92
|
+
},
|
|
93
|
+
correctnessReference: {
|
|
94
|
+
type: 'string',
|
|
95
|
+
describe: 'Path to a JSON file mapping queries to expected cardinality',
|
|
96
|
+
default: '',
|
|
97
|
+
},
|
|
98
|
+
markRows: {
|
|
99
|
+
type: 'string',
|
|
100
|
+
describe: 'Comma-separated list of row id\'s to mark (markdown-only)',
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
function handler(argv) {
|
|
105
|
+
return (0, CliHelpers_1.wrapCommandHandler)(argv, async (context) => (0, CliHelpers_1.wrapVisualProgress)('Summarizing data', async () => {
|
|
106
|
+
// Load options
|
|
107
|
+
const { experimentDirectories, experimentNames } = (0, TexUtils_1.getExperimentNames)(argv);
|
|
108
|
+
const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;
|
|
109
|
+
const correctnessReference = argv.correctnessReference ?
|
|
110
|
+
(0, CorrectnessCheckerUtils_1.constructCorrectnessChecker)(argv.correctnessReference) :
|
|
111
|
+
undefined;
|
|
112
|
+
const markRows = argv.markRows ?
|
|
113
|
+
argv.markRows.split(',').map((value) => Number.parseInt(value, 10)) :
|
|
114
|
+
[];
|
|
115
|
+
// Prepare output CSV file
|
|
116
|
+
const os = fs.createWriteStream(path_1.default.join(context.cwd, `${argv.name}`));
|
|
117
|
+
const serializer = argv.markdown ? new TableSerializerMarkdown_1.TableSerializerMarkdown(os) : new TableSerializerCsv_1.TableSerializerCsv(os);
|
|
206
118
|
if (argv.queryAverage) {
|
|
207
|
-
|
|
208
|
-
'
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
'
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
119
|
+
serializer.writeHeader([
|
|
120
|
+
'',
|
|
121
|
+
'$$\\overline{t}$$',
|
|
122
|
+
'$$\\tilde{t}$$',
|
|
123
|
+
'$$\\overline{t}_1$$',
|
|
124
|
+
'$$\\tilde{t}_1$$',
|
|
125
|
+
'$$\\overline{req}$$',
|
|
126
|
+
'$$\\sum ans$$',
|
|
127
|
+
...correctnessReference ? ['$$\\overline{acc}$$'] : [],
|
|
128
|
+
'$$\\sum to$$',
|
|
129
|
+
], {
|
|
130
|
+
align: [
|
|
131
|
+
'left',
|
|
132
|
+
'right',
|
|
133
|
+
'right',
|
|
134
|
+
'right',
|
|
135
|
+
'right',
|
|
136
|
+
'right',
|
|
137
|
+
'right',
|
|
138
|
+
'right',
|
|
139
|
+
'right',
|
|
140
|
+
],
|
|
141
|
+
});
|
|
227
142
|
}
|
|
228
143
|
else {
|
|
229
|
-
|
|
144
|
+
serializer.writeHeader([
|
|
145
|
+
'Experiment',
|
|
146
|
+
'Query',
|
|
147
|
+
'Time',
|
|
148
|
+
'Requests',
|
|
149
|
+
'Results',
|
|
150
|
+
...correctnessReference ? ['Correctness'] : [],
|
|
151
|
+
'Timeout',
|
|
152
|
+
], {
|
|
153
|
+
align: [
|
|
154
|
+
'left',
|
|
155
|
+
'right',
|
|
156
|
+
'right',
|
|
157
|
+
'right',
|
|
158
|
+
'right',
|
|
159
|
+
'right',
|
|
160
|
+
'right',
|
|
161
|
+
],
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
// Collect data from all experiments
|
|
165
|
+
for (const [experimentId, experimentDirectory] of experimentDirectories.entries()) {
|
|
166
|
+
// Read CSV file
|
|
167
|
+
const timesTotal = {};
|
|
168
|
+
const timesFirstTotal = {};
|
|
169
|
+
const requestsTotal = {};
|
|
170
|
+
let results = {};
|
|
171
|
+
let timeout = {};
|
|
172
|
+
await (0, TexUtils_1.handleCsvFile)(experimentDirectory, argv, (data) => {
|
|
173
|
+
if (!queryRegex || queryRegex.test(data.name)) {
|
|
174
|
+
if (!(data.name in timesTotal)) {
|
|
175
|
+
timesTotal[data.name] = [];
|
|
176
|
+
}
|
|
177
|
+
timesTotal[data.name].push(Number.parseFloat(data.time));
|
|
178
|
+
if (!(data.name in requestsTotal)) {
|
|
179
|
+
requestsTotal[data.name] = [];
|
|
180
|
+
}
|
|
181
|
+
requestsTotal[data.name].push(Number
|
|
182
|
+
.parseInt(data.error === 'false' && data.httpRequests ? data.httpRequests : '0', 10));
|
|
183
|
+
results[data.name] = Number.parseInt(data.results, 10);
|
|
184
|
+
timeout[data.name] = data.error === 'true';
|
|
185
|
+
if (data.timestamps) {
|
|
186
|
+
if (!(data.name in timesFirstTotal)) {
|
|
187
|
+
timesFirstTotal[data.name] = [];
|
|
188
|
+
}
|
|
189
|
+
timesFirstTotal[data.name].push(Number.parseFloat(data.timestamps.split(' ')[0]));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
});
|
|
193
|
+
// Calculate average
|
|
194
|
+
let timesAverage = {};
|
|
195
|
+
let timesAll = [];
|
|
196
|
+
for (const [query, times] of Object.entries(timesTotal)) {
|
|
197
|
+
timesAverage[query] = (0, TexUtils_1.calcAverage)(times);
|
|
198
|
+
timesAll = [...timesAll, ...times];
|
|
199
|
+
}
|
|
200
|
+
let timesFirstAverage = {};
|
|
201
|
+
let timesFirstAll = [];
|
|
202
|
+
for (const [query, times] of Object.entries(timesFirstTotal)) {
|
|
203
|
+
timesFirstAverage[query] = (0, TexUtils_1.calcAverage)(times);
|
|
204
|
+
timesFirstAll = [...timesFirstAll, ...times];
|
|
205
|
+
}
|
|
206
|
+
let requestsSum = {};
|
|
207
|
+
for (const [query, requests] of Object.entries(requestsTotal)) {
|
|
208
|
+
requestsSum[query] = (0, TexUtils_1.calcAverage)(requests);
|
|
209
|
+
}
|
|
210
|
+
// Calculate correctness
|
|
211
|
+
let correctness;
|
|
212
|
+
if (correctnessReference) {
|
|
213
|
+
correctness = Object.fromEntries(Object.entries(results)
|
|
214
|
+
.map(([key, value]) => [key, correctnessReference.getCorrectness(experimentId, key, value)]));
|
|
215
|
+
}
|
|
216
|
+
// Determine query names
|
|
217
|
+
const queryNames = (0, TexUtils_1.getQueryNames)(Object.keys(results), argv);
|
|
218
|
+
timesAverage = (0, TexUtils_1.relabelQueryNames)(timesAverage, queryNames);
|
|
219
|
+
timesFirstAverage = (0, TexUtils_1.relabelQueryNames)(timesFirstAverage, queryNames);
|
|
220
|
+
requestsSum = (0, TexUtils_1.relabelQueryNames)(requestsSum, queryNames);
|
|
221
|
+
results = (0, TexUtils_1.relabelQueryNames)(results, queryNames);
|
|
222
|
+
timeout = (0, TexUtils_1.relabelQueryNames)(timeout, queryNames);
|
|
223
|
+
// Write rows in output file
|
|
224
|
+
if (argv.queryAverage) {
|
|
225
|
+
const valueTimeFirstAverage = Object.values(timesFirstAverage).length === 0 ?
|
|
226
|
+
'N/A' :
|
|
227
|
+
(0, TexUtils_1.calcAverage)(Object.values(timesFirstAverage))
|
|
228
|
+
.toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });
|
|
229
|
+
const valueTimeFirstMedian = timesFirstAll.length === 0 ?
|
|
230
|
+
'N/A' :
|
|
231
|
+
(0, TexUtils_1.calcMedian)(timesFirstAll)
|
|
232
|
+
.toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });
|
|
233
|
+
// Average across queries
|
|
230
234
|
serializer.writeRow([
|
|
231
235
|
experimentNames[experimentId],
|
|
232
|
-
|
|
233
|
-
`${
|
|
234
|
-
`${
|
|
235
|
-
`${
|
|
236
|
-
|
|
237
|
-
`${
|
|
236
|
+
`${(0, TexUtils_1.calcAverage)(Object.values(timesAverage)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,
|
|
237
|
+
`${(0, TexUtils_1.calcMedian)(timesAll).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,
|
|
238
|
+
`${valueTimeFirstAverage}`,
|
|
239
|
+
`${valueTimeFirstMedian}`,
|
|
240
|
+
`${(0, TexUtils_1.calcSum)(Object.values(requestsSum)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,
|
|
241
|
+
`${(0, TexUtils_1.calcAverage)(Object.values(results)).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`,
|
|
242
|
+
...correctness ? [`${((0, TexUtils_1.calcAverage)(Object.values(correctness)) * 100).toFixed(2)}%`] : [],
|
|
243
|
+
`${(0, TexUtils_1.calcSum)(Object.values(timeout).map(value => value ? 1 : 0))}`,
|
|
238
244
|
], { mark: markRows.includes(experimentId) });
|
|
239
245
|
}
|
|
246
|
+
else {
|
|
247
|
+
for (const [query, time] of Object.entries(timesAverage)) {
|
|
248
|
+
serializer.writeRow([
|
|
249
|
+
experimentNames[experimentId],
|
|
250
|
+
query,
|
|
251
|
+
`${time}`,
|
|
252
|
+
`${requestsSum[query]}`,
|
|
253
|
+
`${results[query]}`,
|
|
254
|
+
...correctness ? [`${(correctness[query] * 100).toFixed(2)}%`] : [],
|
|
255
|
+
`${timeout[query]}`,
|
|
256
|
+
], { mark: markRows.includes(experimentId) });
|
|
257
|
+
}
|
|
258
|
+
}
|
|
240
259
|
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
}
|
|
245
|
-
exports.handler = handler;
|
|
260
|
+
// Close output CSV file
|
|
261
|
+
serializer.close();
|
|
262
|
+
}));
|
|
263
|
+
}
|
|
246
264
|
//# sourceMappingURL=CommandSummary.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CommandSummary.js","sourceRoot":"","sources":["CommandSummary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,gDAAwB;AAExB,0FAA2F;AAC3F,iDAA0E;AAE1E,8CAOyB;AACzB,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,6BAA6B,CAAC;AACxC,QAAA,IAAI,GAAG,+DAA+D,CAAC;AAC7E,MAAM,OAAO,GAAG,CAAC,KAAgB,EAAa,EAAE,CACrD,KAAK;KACF,OAAO,CAAC;IACP,UAAU,EAAE;QACV,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,6FAA6F;KACxG;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,yBAAyB;QACnC,OAAO,EAAE,kBAAkB;KAC5B;IACD,SAAS,EAAE;QACT,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,uCAAuC;QACjD,OAAO,EAAE,iBAAiB;KAC3B;IACD,cAAc,EAAE;QACd,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,kCAAkC;QAC5C,OAAO,EAAE,GAAG;KACb;IACD,yBAAyB,EAAE;QACzB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,mDAAmD;KAC9D;IACD,mBAAmB,EAAE;QACnB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,6CAA6C;KACxD;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,gDAAgD;QAC1D,OAAO,EAAE,KAAK;KACf;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,kCAAkC;QAC5C,OAAO,EAAE,KAAK;KACf;IACD,oBAAoB,EAAE;QACpB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,6DAA6D;QACvE,OAAO,EAAE,EAAE;KACZ;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,2DAA2D;KACtE;CACF,CAAC,CAAC;AAnDM,QAAA,OAAO,WAmDb;AACA,MAAM,OAAO,GAAG,CAAC,IAAyB,EAAiB,EAAE,CAAC,+BAAkB,CAAC,IAAI,EAC1F,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,+BAAkB,CAAC,kBAAkB,EAAE,KAAK,IAAG,EAAE;IAC/E,eAAe;IACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,6BAAkB,CAAC,IAAI,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACtD,qDAA2B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACxD,SAAS,CAAC;IACZ,MAAM,QAAQ,GAAa,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,EAAE,CAAC;IAEL,0BAA0B;IAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;IAChG,IAAI,IAAI,CAAC,YAAY,EAAE;QACrB,UAAU,CAAC,WAAW,CAAC;YACrB,EAAE;YACF,mBAAmB;YACnB,gBAAgB;YAChB,qBAAqB;YACrB,kBAAkB;YAClB,qBAAqB;YACrB,eAAe;YACf,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,qBAAqB,CAAE,CAAC,CAAC,CAAC,EAAE;YACxD,cAAc;SACf,EAAE;YACD,KAAK,EAAE;gBACL,MAAM;gBACN,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;aACR;SACF,CAAC,CAAC;KACJ;SAAM;QACL,UAAU,CAAC,WAAW,CAAC;YACrB,YAAY;YACZ,OAAO;YACP,MAAM;YACN,UAAU;YACV,SAAS;YACT,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,aAAa,CAAE,CAAC,CAAC,CAAC,EAAE;YAChD,SAAS;SACV,EAAE;YACD,KAAK,EAAE;gBACL,MAAM;gBACN,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;aACR;SACF,CAAC,CAAC;KACJ;IAED,oCAAoC;IACpC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE;QACnF,gBAAgB;QAChB,MAAM,UAAU,GAA6B,EAAE,CAAC;QAChD,MAAM,eAAe,GAA6B,EAAE,CAAC;QACrD,MAAM,aAAa,GAA6B,EAAE,CAAC;QACnD,IAAI,OAAO,GAA2B,EAAE,CAAC;QACzC,IAAI,OAAO,GAA4B,EAAE,CAAC;QAC1C,MAAM,wBAAa,CAAC,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE;YACpD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7C,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,EAAE;oBAC9B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;iBAC5B;gBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;gBAE3D,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,aAAa,CAAC,EAAE;oBACjC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;iBAC/B;gBACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM;qBACjC,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;gBAExF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACvD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC;gBAE3C,IAAI,IAAI,CAAC,UAAU,EAAE;oBACnB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE;wBACnC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;qBACjC;oBACD,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;iBACrF;aACF;QACH,CAAC,CAAC,CAAC;QAEH,oBAAoB;QACpB,IAAI,YAAY,GAA2B,EAAE,CAAC;QAC9C,IAAI,QAAQ,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YACzD,YAAY,CAAC,KAAK,CAAC,GAAG,sBAAW,CAAC,KAAK,CAAC,CAAC;YACzC,QAAQ,GAAG,CAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAE,CAAC;SACtC;QACD,IAAI,iBAAiB,GAA2B,EAAE,CAAC;QACnD,IAAI,aAAa,GAAa,EAAE,CAAC;QACjC,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;YAC9D,iBAAiB,CAAC,KAAK,CAAC,GAAG,sBAAW,CAAC,KAAK,CAAC,CAAC;YAC9C,aAAa,GAAG,CAAE,GAAG,aAAa,EAAE,GAAG,KAAK,CAAE,CAAC;SAChD;QACD,IAAI,WAAW,GAA2B,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAE,KAAK,EAAE,QAAQ,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;YAC/D,WAAW,CAAC,KAAK,CAAC,GAAG,sBAAW,CAAC,QAAQ,CAAC,CAAC;SAC5C;QAED,wBAAwB;QACxB,IAAI,WAAW,CAAC;QAChB,IAAI,oBAAoB,EAAE;YACxB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;iBACrD,GAAG,CAAC,CAAC,CAAE,GAAG,EAAE,KAAK,CAAE,EAAE,EAAE,CAAC,CAAE,GAAG,EAAE,oBAAoB,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAE,CAAC,CAAC,CAAC;SACrG;QAED,wBAAwB;QACxB,MAAM,UAAU,GAAG,wBAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QAC7D,YAAY,GAAG,4BAAiB,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QAC3D,iBAAiB,GAAG,4BAAiB,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QACrE,WAAW,GAAG,4BAAiB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACzD,OAAO,GAAG,4BAAiB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACjD,OAAO,GAAG,4BAAiB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAEjD,4BAA4B;QAC5B,IAAI,IAAI,CAAC,YAAY,EAAE;YACrB,MAAM,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;gBAC3E,KAAK,CAAC,CAAC;gBACP,sBAAW,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;qBAC1C,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;YACrF,MAAM,oBAAoB,GAAG,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;gBACvD,KAAK,CAAC,CAAC;gBACP,qBAAU,CAAC,aAAa,CAAC;qBACtB,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;YAErF,yBAAyB;YACzB,UAAU,CAAC,QAAQ,CAAC;gBAClB,eAAe,CAAC,YAAY,CAAC;gBAC7B,GAAG,sBAAW,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;gBAC7H,GAAG,qBAAU,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;gBACzG,GAAG,qBAAqB,EAAE;gBAC1B,GAAG,oBAAoB,EAAE;gBACzB,GAAG,kBAAO,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;gBACxH,GAAG,sBAAW,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;gBACxH,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,sBAAW,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;gBAC1F,GAAG,kBAAO,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;aACjE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;SAC/C;aAAM;YACL,KAAK,MAAM,CAAE,KAAK,EAAE,IAAI,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE;gBAC1D,UAAU,CAAC,QAAQ,CAAC;oBAClB,eAAe,CAAC,YAAY,CAAC;oBAC7B,KAAK;oBACL,GAAG,IAAI,EAAE;oBACT,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE;oBACvB,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;oBACnB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;oBACrE,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;iBACpB,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;aAC/C;SACF;KACF;IAED,wBAAwB;IACxB,UAAU,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC,CAAC,CAAC,CAAC;AAvKO,QAAA,OAAO,WAuKd","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { constructCorrectnessChecker } from '../../../correctness/CorrectnessCheckerUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n calcAverage, calcMedian,\n calcSum,\n getExperimentNames,\n getQueryNames,\n handleCsvFile,\n relabelQueryNames,\n} from '../tex/TexUtils';\nimport { TableSerializerCsv } from './TableSerializerCsv';\nimport { TableSerializerMarkdown } from './TableSerializerMarkdown';\n\nexport const command = 'summary <experiment-dir...>';\nexport const desc = 'Summarize query results from the given experiments in a table';\nexport const builder = (yargs: Argv<any>): Argv<any> =>\n yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'data_summary.csv',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n overrideQueryLabels: {\n type: 'string',\n describe: 'Comma-separated list of query labels to use',\n },\n markdown: {\n type: 'boolean',\n describe: 'If the output should be serialized as markdown',\n default: false,\n },\n queryAverage: {\n type: 'boolean',\n describe: 'If queries will be averaged over',\n default: false,\n },\n correctnessReference: {\n type: 'string',\n describe: 'Path to a JSON file mapping queries to expected cardinality',\n default: '',\n },\n markRows: {\n type: 'string',\n describe: 'Comma-separated list of row id\\'s to mark (markdown-only)',\n },\n });\nexport const handler = (argv: Record<string, any>): Promise<void> => wrapCommandHandler(argv,\n async(context: ITaskContext) => wrapVisualProgress('Summarizing data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const correctnessReference = argv.correctnessReference ?\n constructCorrectnessChecker(argv.correctnessReference) :\n undefined;\n const markRows: number[] = argv.markRows ?\n argv.markRows.split(',').map((value: string) => Number.parseInt(value, 10)) :\n [];\n\n // Prepare output CSV file\n const os = fs.createWriteStream(Path.join(context.cwd, `${argv.name}`));\n const serializer = argv.markdown ? new TableSerializerMarkdown(os) : new TableSerializerCsv(os);\n if (argv.queryAverage) {\n serializer.writeHeader([\n '',\n '$$\\\\overline{t}$$',\n '$$\\\\tilde{t}$$',\n '$$\\\\overline{t}_1$$',\n '$$\\\\tilde{t}_1$$',\n '$$\\\\overline{req}$$',\n '$$\\\\sum ans$$',\n ...correctnessReference ? [ '$$\\\\overline{acc}$$' ] : [],\n '$$\\\\sum to$$',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n } else {\n serializer.writeHeader([\n 'Experiment',\n 'Query',\n 'Time',\n 'Requests',\n 'Results',\n ...correctnessReference ? [ 'Correctness' ] : [],\n 'Timeout',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n }\n\n // Collect data from all experiments\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n // Read CSV file\n const timesTotal: Record<string, number[]> = {};\n const timesFirstTotal: Record<string, number[]> = {};\n const requestsTotal: Record<string, number[]> = {};\n let results: Record<string, number> = {};\n let timeout: Record<string, boolean> = {};\n await handleCsvFile(experimentDirectory, argv, data => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (!(data.name in timesTotal)) {\n timesTotal[data.name] = [];\n }\n timesTotal[data.name].push(Number.parseInt(data.time, 10));\n\n if (!(data.name in requestsTotal)) {\n requestsTotal[data.name] = [];\n }\n requestsTotal[data.name].push(Number\n .parseInt(data.error === 'false' && data.httpRequests ? data.httpRequests : '0', 10));\n\n results[data.name] = Number.parseInt(data.results, 10);\n timeout[data.name] = data.error === 'true';\n\n if (data.timestamps) {\n if (!(data.name in timesFirstTotal)) {\n timesFirstTotal[data.name] = [];\n }\n timesFirstTotal[data.name].push(Number.parseInt(data.timestamps.split(' ')[0], 10));\n }\n }\n });\n\n // Calculate average\n let timesAverage: Record<string, number> = {};\n let timesAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesTotal)) {\n timesAverage[query] = calcAverage(times);\n timesAll = [ ...timesAll, ...times ];\n }\n let timesFirstAverage: Record<string, number> = {};\n let timesFirstAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesFirstTotal)) {\n timesFirstAverage[query] = calcAverage(times);\n timesFirstAll = [ ...timesFirstAll, ...times ];\n }\n let requestsSum: Record<string, number> = {};\n for (const [ query, requests ] of Object.entries(requestsTotal)) {\n requestsSum[query] = calcAverage(requests);\n }\n\n // Calculate correctness\n let correctness;\n if (correctnessReference) {\n correctness = Object.fromEntries(Object.entries(results)\n .map(([ key, value ]) => [ key, correctnessReference.getCorrectness(experimentId, key, value) ]));\n }\n\n // Determine query names\n const queryNames = getQueryNames(Object.keys(results), argv);\n timesAverage = relabelQueryNames(timesAverage, queryNames);\n timesFirstAverage = relabelQueryNames(timesFirstAverage, queryNames);\n requestsSum = relabelQueryNames(requestsSum, queryNames);\n results = relabelQueryNames(results, queryNames);\n timeout = relabelQueryNames(timeout, queryNames);\n\n // Write rows in output file\n if (argv.queryAverage) {\n const valueTimeFirstAverage = Object.values(timesFirstAverage).length === 0 ?\n 'N/A' :\n calcAverage(Object.values(timesFirstAverage))\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n const valueTimeFirstMedian = timesFirstAll.length === 0 ?\n 'N/A' :\n calcMedian(timesFirstAll)\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n\n // Average across queries\n serializer.writeRow([\n experimentNames[experimentId],\n `${calcAverage(Object.values(timesAverage)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcMedian(timesAll).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${valueTimeFirstAverage}`,\n `${valueTimeFirstMedian}`,\n `${calcSum(Object.values(requestsSum)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcAverage(Object.values(results)).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`,\n ...correctness ? [ `${(calcAverage(Object.values(correctness)) * 100).toFixed(2)}%` ] : [],\n `${calcSum(Object.values(timeout).map(value => value ? 1 : 0))}`,\n ], { mark: markRows.includes(experimentId) });\n } else {\n for (const [ query, time ] of Object.entries(timesAverage)) {\n serializer.writeRow([\n experimentNames[experimentId],\n query,\n `${time}`,\n `${requestsSum[query]}`,\n `${results[query]}`,\n ...correctness ? [ `${(correctness[query] * 100).toFixed(2)}%` ] : [],\n `${timeout[query]}`,\n ], { mark: markRows.includes(experimentId) });\n }\n }\n }\n\n // Close output CSV file\n serializer.close();\n }));\n"]}
|
|
1
|
+
{"version":3,"file":"CommandSummary.js","sourceRoot":"","sources":["CommandSummary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,0BAoDC;AACD,0BAwKC;AAjPD,uCAAyB;AACzB,gDAAwB;AAExB,0FAA2F;AAC3F,iDAA0E;AAE1E,8CAQyB;AACzB,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,6BAA6B,CAAC;AACxC,QAAA,IAAI,GAAG,+DAA+D,CAAC;AACpF,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6FAA6F;SACxG;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,kBAAkB;SAC5B;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,mBAAmB,EAAE;YACnB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6CAA6C;SACxD;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gDAAgD;YAC1D,OAAO,EAAE,KAAK;SACf;QACD,YAAY,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,KAAK;SACf;QACD,oBAAoB,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6DAA6D;YACvE,OAAO,EAAE,EAAE;SACZ;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,2DAA2D;SACtE;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EAAC,IAAI,EAAE,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,kBAAkB,EAAE,KAAK,IAAG,EAAE;QAC/G,eAAe;QACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACtD,IAAA,qDAA2B,EAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YACxD,SAAS,CAAC;QACZ,MAAM,QAAQ,GAAa,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3E,EAAE,CAAC;QAEP,0BAA0B;QAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;QAChG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,UAAU,CAAC,WAAW,CAAC;gBACrB,EAAE;gBACF,mBAAmB;gBACnB,gBAAgB;gBAChB,qBAAqB;gBACrB,kBAAkB;gBAClB,qBAAqB;gBACrB,eAAe;gBACf,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,qBAAqB,CAAE,CAAC,CAAC,CAAC,EAAE;gBACxD,cAAc;aACf,EAAE;gBACD,KAAK,EAAE;oBACL,MAAM;oBACN,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;iBACR;aACF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,WAAW,CAAC;gBACrB,YAAY;gBACZ,OAAO;gBACP,MAAM;gBACN,UAAU;gBACV,SAAS;gBACT,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,aAAa,CAAE,CAAC,CAAC,CAAC,EAAE;gBAChD,SAAS;aACV,EAAE;gBACD,KAAK,EAAE;oBACL,MAAM;oBACN,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;iBACR;aACF,CAAC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE,CAAC;YACpF,gBAAgB;YAChB,MAAM,UAAU,GAA6B,EAAE,CAAC;YAChD,MAAM,eAAe,GAA6B,EAAE,CAAC;YACrD,MAAM,aAAa,GAA6B,EAAE,CAAC;YACnD,IAAI,OAAO,GAA2B,EAAE,CAAC;YACzC,IAAI,OAAO,GAA4B,EAAE,CAAC;YAC1C,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,EAAE,CAAC;wBAC/B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC7B,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBAEzD,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,aAAa,CAAC,EAAE,CAAC;wBAClC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAChC,CAAC;oBACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM;yBACjC,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;oBAExF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC;oBAE3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE,CAAC;4BACpC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;wBAClC,CAAC;wBACD,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACpF,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,oBAAoB;YACpB,IAAI,YAAY,GAA2B,EAAE,CAAC;YAC9C,IAAI,QAAQ,GAAa,EAAE,CAAC;YAC5B,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1D,YAAY,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBACzC,QAAQ,GAAG,CAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAE,CAAC;YACvC,CAAC;YACD,IAAI,iBAAiB,GAA2B,EAAE,CAAC;YACnD,IAAI,aAAa,GAAa,EAAE,CAAC;YACjC,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC/D,iBAAiB,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC9C,aAAa,GAAG,CAAE,GAAG,aAAa,EAAE,GAAG,KAAK,CAAE,CAAC;YACjD,CAAC;YACD,IAAI,WAAW,GAA2B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAE,KAAK,EAAE,QAAQ,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;gBAChE,WAAW,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,QAAQ,CAAC,CAAC;YAC7C,CAAC;YAED,wBAAwB;YACxB,IAAI,WAAW,CAAC;YAChB,IAAI,oBAAoB,EAAE,CAAC;gBACzB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;qBACrD,GAAG,CAAC,CAAC,CAAE,GAAG,EAAE,KAAK,CAAE,EAAE,EAAE,CAAC,CAAE,GAAG,EAAE,oBAAoB,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAE,CAAC,CAAC,CAAC;YACtG,CAAC;YAED,wBAAwB;YACxB,MAAM,UAAU,GAAG,IAAA,wBAAa,EAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;YAC7D,YAAY,GAAG,IAAA,4BAAiB,EAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAC3D,iBAAiB,GAAG,IAAA,4BAAiB,EAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;YACrE,WAAW,GAAG,IAAA,4BAAiB,EAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YACzD,OAAO,GAAG,IAAA,4BAAiB,EAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YACjD,OAAO,GAAG,IAAA,4BAAiB,EAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YAEjD,4BAA4B;YAC5B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;oBAC3E,KAAK,CAAC,CAAC;oBACP,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;yBAC1C,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrF,MAAM,oBAAoB,GAAG,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;oBACvD,KAAK,CAAC,CAAC;oBACP,IAAA,qBAAU,EAAC,aAAa,CAAC;yBACtB,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;gBAErF,yBAAyB;gBACzB,UAAU,CAAC,QAAQ,CAAC;oBAClB,eAAe,CAAC,YAAY,CAAC;oBAC7B,GAAG,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBAC7H,GAAG,IAAA,qBAAU,EAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACzG,GAAG,qBAAqB,EAAE;oBAC1B,GAAG,oBAAoB,EAAE;oBACzB,GAAG,IAAA,kBAAO,EAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACxH,GAAG,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACxH,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;oBAC1F,GAAG,IAAA,kBAAO,EAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;iBACjE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,CAAE,KAAK,EAAE,IAAI,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;oBAC3D,UAAU,CAAC,QAAQ,CAAC;wBAClB,eAAe,CAAC,YAAY,CAAC;wBAC7B,KAAK;wBACL,GAAG,IAAI,EAAE;wBACT,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE;wBACvB,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;wBACnB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;wBACrE,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;qBACpB,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC,CAAC;AACN,CAAC","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { constructCorrectnessChecker } from '../../../correctness/CorrectnessCheckerUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n calcAverage,\n calcMedian,\n calcSum,\n getExperimentNames,\n getQueryNames,\n handleCsvFile,\n relabelQueryNames,\n} from '../tex/TexUtils';\nimport { TableSerializerCsv } from './TableSerializerCsv';\nimport { TableSerializerMarkdown } from './TableSerializerMarkdown';\n\nexport const command = 'summary <experiment-dir...>';\nexport const desc = 'Summarize query results from the given experiments in a table';\nexport function builder(yargs: Argv<any>): Argv<any> {\n return yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'data_summary.csv',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n overrideQueryLabels: {\n type: 'string',\n describe: 'Comma-separated list of query labels to use',\n },\n markdown: {\n type: 'boolean',\n describe: 'If the output should be serialized as markdown',\n default: false,\n },\n queryAverage: {\n type: 'boolean',\n describe: 'If queries will be averaged over',\n default: false,\n },\n correctnessReference: {\n type: 'string',\n describe: 'Path to a JSON file mapping queries to expected cardinality',\n default: '',\n },\n markRows: {\n type: 'string',\n describe: 'Comma-separated list of row id\\'s to mark (markdown-only)',\n },\n });\n}\nexport function handler(argv: Record<string, any>): Promise<void> {\n return wrapCommandHandler(argv, async(context: ITaskContext) => wrapVisualProgress('Summarizing data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const correctnessReference = argv.correctnessReference ?\n constructCorrectnessChecker(argv.correctnessReference) :\n undefined;\n const markRows: number[] = argv.markRows ?\n argv.markRows.split(',').map((value: string) => Number.parseInt(value, 10)) :\n [];\n\n // Prepare output CSV file\n const os = fs.createWriteStream(Path.join(context.cwd, `${argv.name}`));\n const serializer = argv.markdown ? new TableSerializerMarkdown(os) : new TableSerializerCsv(os);\n if (argv.queryAverage) {\n serializer.writeHeader([\n '',\n '$$\\\\overline{t}$$',\n '$$\\\\tilde{t}$$',\n '$$\\\\overline{t}_1$$',\n '$$\\\\tilde{t}_1$$',\n '$$\\\\overline{req}$$',\n '$$\\\\sum ans$$',\n ...correctnessReference ? [ '$$\\\\overline{acc}$$' ] : [],\n '$$\\\\sum to$$',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n } else {\n serializer.writeHeader([\n 'Experiment',\n 'Query',\n 'Time',\n 'Requests',\n 'Results',\n ...correctnessReference ? [ 'Correctness' ] : [],\n 'Timeout',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n }\n\n // Collect data from all experiments\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n // Read CSV file\n const timesTotal: Record<string, number[]> = {};\n const timesFirstTotal: Record<string, number[]> = {};\n const requestsTotal: Record<string, number[]> = {};\n let results: Record<string, number> = {};\n let timeout: Record<string, boolean> = {};\n await handleCsvFile(experimentDirectory, argv, (data) => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (!(data.name in timesTotal)) {\n timesTotal[data.name] = [];\n }\n timesTotal[data.name].push(Number.parseFloat(data.time));\n\n if (!(data.name in requestsTotal)) {\n requestsTotal[data.name] = [];\n }\n requestsTotal[data.name].push(Number\n .parseInt(data.error === 'false' && data.httpRequests ? data.httpRequests : '0', 10));\n\n results[data.name] = Number.parseInt(data.results, 10);\n timeout[data.name] = data.error === 'true';\n\n if (data.timestamps) {\n if (!(data.name in timesFirstTotal)) {\n timesFirstTotal[data.name] = [];\n }\n timesFirstTotal[data.name].push(Number.parseFloat(data.timestamps.split(' ')[0]));\n }\n }\n });\n\n // Calculate average\n let timesAverage: Record<string, number> = {};\n let timesAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesTotal)) {\n timesAverage[query] = calcAverage(times);\n timesAll = [ ...timesAll, ...times ];\n }\n let timesFirstAverage: Record<string, number> = {};\n let timesFirstAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesFirstTotal)) {\n timesFirstAverage[query] = calcAverage(times);\n timesFirstAll = [ ...timesFirstAll, ...times ];\n }\n let requestsSum: Record<string, number> = {};\n for (const [ query, requests ] of Object.entries(requestsTotal)) {\n requestsSum[query] = calcAverage(requests);\n }\n\n // Calculate correctness\n let correctness;\n if (correctnessReference) {\n correctness = Object.fromEntries(Object.entries(results)\n .map(([ key, value ]) => [ key, correctnessReference.getCorrectness(experimentId, key, value) ]));\n }\n\n // Determine query names\n const queryNames = getQueryNames(Object.keys(results), argv);\n timesAverage = relabelQueryNames(timesAverage, queryNames);\n timesFirstAverage = relabelQueryNames(timesFirstAverage, queryNames);\n requestsSum = relabelQueryNames(requestsSum, queryNames);\n results = relabelQueryNames(results, queryNames);\n timeout = relabelQueryNames(timeout, queryNames);\n\n // Write rows in output file\n if (argv.queryAverage) {\n const valueTimeFirstAverage = Object.values(timesFirstAverage).length === 0 ?\n 'N/A' :\n calcAverage(Object.values(timesFirstAverage))\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n const valueTimeFirstMedian = timesFirstAll.length === 0 ?\n 'N/A' :\n calcMedian(timesFirstAll)\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n\n // Average across queries\n serializer.writeRow([\n experimentNames[experimentId],\n `${calcAverage(Object.values(timesAverage)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcMedian(timesAll).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${valueTimeFirstAverage}`,\n `${valueTimeFirstMedian}`,\n `${calcSum(Object.values(requestsSum)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcAverage(Object.values(results)).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`,\n ...correctness ? [ `${(calcAverage(Object.values(correctness)) * 100).toFixed(2)}%` ] : [],\n `${calcSum(Object.values(timeout).map(value => value ? 1 : 0))}`,\n ], { mark: markRows.includes(experimentId) });\n } else {\n for (const [ query, time ] of Object.entries(timesAverage)) {\n serializer.writeRow([\n experimentNames[experimentId],\n query,\n `${time}`,\n `${requestsSum[query]}`,\n `${results[query]}`,\n ...correctness ? [ `${(correctness[query] * 100).toFixed(2)}%` ] : [],\n `${timeout[query]}`,\n ], { mark: markRows.includes(experimentId) });\n }\n }\n }\n\n // Close output CSV file\n serializer.close();\n }));\n}\n"]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Argv } from 'yargs';
|
|
2
2
|
export declare const command = "wins <experiment-dir...>";
|
|
3
3
|
export declare const desc = "Show distribution of query wins from the given experiments in a table";
|
|
4
|
-
export declare
|
|
5
|
-
export declare
|
|
4
|
+
export declare function builder(yargs: Argv<any>): Argv<any>;
|
|
5
|
+
export declare function handler(argv: Record<string, any>): Promise<void>;
|