@rubensworks/process-sparql-benchmark-results 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/LICENSE.txt +21 -0
  2. package/README.md +285 -0
  3. package/bin/psbr +2 -0
  4. package/bin/psbr.d.ts +2 -0
  5. package/bin/psbr.js +6 -0
  6. package/bin/psbr.js.map +1 -0
  7. package/index.d.ts +3 -0
  8. package/index.js +16 -0
  9. package/index.js.map +1 -0
  10. package/index.ts +3 -0
  11. package/lib/TemplateUtils.d.ts +1 -0
  12. package/lib/TemplateUtils.js +35 -0
  13. package/lib/TemplateUtils.js.map +1 -0
  14. package/lib/cli/CliHelpers.d.ts +5 -0
  15. package/lib/cli/CliHelpers.js +87 -0
  16. package/lib/cli/CliHelpers.js.map +1 -0
  17. package/lib/cli/CliRunner.d.ts +1 -0
  18. package/lib/cli/CliRunner.js +24 -0
  19. package/lib/cli/CliRunner.js.map +1 -0
  20. package/lib/cli/ErrorHandled.d.ts +8 -0
  21. package/lib/cli/ErrorHandled.js +15 -0
  22. package/lib/cli/ErrorHandled.js.map +1 -0
  23. package/lib/cli/ITaskContext.d.ts +10 -0
  24. package/lib/cli/ITaskContext.js +3 -0
  25. package/lib/cli/ITaskContext.js.map +1 -0
  26. package/lib/cli/commands/CommandCsv.d.ts +4 -0
  27. package/lib/cli/commands/CommandCsv.js +10 -0
  28. package/lib/cli/commands/CommandCsv.js.map +1 -0
  29. package/lib/cli/commands/CommandStats.d.ts +4 -0
  30. package/lib/cli/commands/CommandStats.js +10 -0
  31. package/lib/cli/commands/CommandStats.js.map +1 -0
  32. package/lib/cli/commands/CommandTex.d.ts +4 -0
  33. package/lib/cli/commands/CommandTex.js +10 -0
  34. package/lib/cli/commands/CommandTex.js.map +1 -0
  35. package/lib/cli/commands/csv/CommandQuery.d.ts +5 -0
  36. package/lib/cli/commands/csv/CommandQuery.js +120 -0
  37. package/lib/cli/commands/csv/CommandQuery.js.map +1 -0
  38. package/lib/cli/commands/csv/CommandSummary.d.ts +5 -0
  39. package/lib/cli/commands/csv/CommandSummary.js +246 -0
  40. package/lib/cli/commands/csv/CommandSummary.js.map +1 -0
  41. package/lib/cli/commands/csv/CommandWins.d.ts +5 -0
  42. package/lib/cli/commands/csv/CommandWins.js +151 -0
  43. package/lib/cli/commands/csv/CommandWins.js.map +1 -0
  44. package/lib/cli/commands/csv/ITableSerializer.d.ts +9 -0
  45. package/lib/cli/commands/csv/ITableSerializer.js +3 -0
  46. package/lib/cli/commands/csv/ITableSerializer.js.map +1 -0
  47. package/lib/cli/commands/csv/TableSerializerCsv.d.ts +10 -0
  48. package/lib/cli/commands/csv/TableSerializerCsv.js +19 -0
  49. package/lib/cli/commands/csv/TableSerializerCsv.js.map +1 -0
  50. package/lib/cli/commands/csv/TableSerializerMarkdown.d.ts +18 -0
  51. package/lib/cli/commands/csv/TableSerializerMarkdown.js +34 -0
  52. package/lib/cli/commands/csv/TableSerializerMarkdown.js.map +1 -0
  53. package/lib/cli/commands/stats/CommandDocker.d.ts +7 -0
  54. package/lib/cli/commands/stats/CommandDocker.js +79 -0
  55. package/lib/cli/commands/stats/CommandDocker.js.map +1 -0
  56. package/lib/cli/commands/tex/CommandQuery.d.ts +5 -0
  57. package/lib/cli/commands/tex/CommandQuery.js +253 -0
  58. package/lib/cli/commands/tex/CommandQuery.js.map +1 -0
  59. package/lib/cli/commands/tex/CommandQueryTimes.d.ts +5 -0
  60. package/lib/cli/commands/tex/CommandQueryTimes.js +153 -0
  61. package/lib/cli/commands/tex/CommandQueryTimes.js.map +1 -0
  62. package/lib/cli/commands/tex/TexUtils.d.ts +34 -0
  63. package/lib/cli/commands/tex/TexUtils.js +149 -0
  64. package/lib/cli/commands/tex/TexUtils.js.map +1 -0
  65. package/lib/correctness/CorrectnessCheckerQueries.d.ts +6 -0
  66. package/lib/correctness/CorrectnessCheckerQueries.js +13 -0
  67. package/lib/correctness/CorrectnessCheckerQueries.js.map +1 -0
  68. package/lib/correctness/CorrectnessCheckerUtils.d.ts +2 -0
  69. package/lib/correctness/CorrectnessCheckerUtils.js +14 -0
  70. package/lib/correctness/CorrectnessCheckerUtils.js.map +1 -0
  71. package/lib/correctness/ICorrectnessChecker.d.ts +3 -0
  72. package/lib/correctness/ICorrectnessChecker.js +3 -0
  73. package/lib/correctness/ICorrectnessChecker.js.map +1 -0
  74. package/lib/templates/tex/plot_query_data.tex +94 -0
  75. package/lib/templates/tex/plot_query_times.tex +75 -0
  76. package/package.json +100 -0
@@ -0,0 +1,10 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.builder = exports.desc = exports.command = void 0;
4
+ exports.command = 'tex';
5
+ exports.desc = 'Creates a LaTeX TikZ plot file';
6
+ const builder = (yargs) => yargs
7
+ .commandDir('tex')
8
+ .demandCommand();
9
+ exports.builder = builder;
10
+ //# sourceMappingURL=CommandTex.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CommandTex.js","sourceRoot":"","sources":["CommandTex.ts"],"names":[],"mappings":";;;AAEa,QAAA,OAAO,GAAG,KAAK,CAAC;AAChB,QAAA,IAAI,GAAG,gCAAgC,CAAC;AAC9C,MAAM,OAAO,GAAG,CAAC,KAAgB,EAAa,EAAE,CACrD,KAAK;KACF,UAAU,CAAC,KAAK,CAAC;KACjB,aAAa,EAAE,CAAC;AAHR,QAAA,OAAO,WAGC","sourcesContent":["import type { Argv } from 'yargs';\n\nexport const command = 'tex';\nexport const desc = 'Creates a LaTeX TikZ plot file';\nexport const builder = (yargs: Argv<any>): Argv<any> =>\n yargs\n .commandDir('tex')\n .demandCommand();\n"]}
@@ -0,0 +1,5 @@
1
+ import type { Argv } from 'yargs';
2
+ export declare const command = "query <experiment-dir...>";
3
+ export declare const desc = "List all query execution times from the given experiments";
4
+ export declare const builder: (yargs: Argv<any>) => Argv<any>;
5
+ export declare const handler: (argv: Record<string, any>) => Promise<void>;
@@ -0,0 +1,120 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
5
+ }) : (function(o, m, k, k2) {
6
+ if (k2 === undefined) k2 = k;
7
+ o[k2] = m[k];
8
+ }));
9
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
10
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
11
+ }) : function(o, v) {
12
+ o["default"] = v;
13
+ });
14
+ var __importStar = (this && this.__importStar) || function (mod) {
15
+ if (mod && mod.__esModule) return mod;
16
+ var result = {};
17
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
18
+ __setModuleDefault(result, mod);
19
+ return result;
20
+ };
21
+ var __importDefault = (this && this.__importDefault) || function (mod) {
22
+ return (mod && mod.__esModule) ? mod : { "default": mod };
23
+ };
24
+ Object.defineProperty(exports, "__esModule", { value: true });
25
+ exports.handler = exports.builder = exports.desc = exports.command = void 0;
26
+ const fs = __importStar(require("fs"));
27
+ const path_1 = __importDefault(require("path"));
28
+ const csv_parse_1 = __importDefault(require("csv-parse"));
29
+ const CorrectnessCheckerUtils_1 = require("../../../correctness/CorrectnessCheckerUtils");
30
+ const CliHelpers_1 = require("../../CliHelpers");
31
+ const TexUtils_1 = require("../tex/TexUtils");
32
+ const TableSerializerCsv_1 = require("./TableSerializerCsv");
33
+ const TableSerializerMarkdown_1 = require("./TableSerializerMarkdown");
34
+ exports.command = 'query <experiment-dir...>';
35
+ exports.desc = 'List all query execution times from the given experiments';
36
+ const builder = (yargs) => yargs
37
+ .options({
38
+ queryRegex: {
39
+ type: 'string',
40
+ alias: 'q',
41
+ describe: 'Regex for queries to include (before any label overrides). Examples: \'^C\', \'^[^C]\', ...',
42
+ },
43
+ name: {
44
+ type: 'string',
45
+ alias: 'n',
46
+ describe: 'Custom output file name',
47
+ default: 'data_all.csv',
48
+ },
49
+ inputName: {
50
+ type: 'string',
51
+ describe: 'Custom input file name per experiment',
52
+ default: 'query-times.csv',
53
+ },
54
+ inputDelimiter: {
55
+ type: 'string',
56
+ describe: 'Delimiter for the input CSV file',
57
+ default: ';',
58
+ },
59
+ overrideCombinationLabels: {
60
+ type: 'string',
61
+ describe: 'Comma-separated list of combination labels to use',
62
+ },
63
+ markdown: {
64
+ type: 'boolean',
65
+ describe: 'If the output should be serialized as markdown',
66
+ default: false,
67
+ },
68
+ correctnessReference: {
69
+ type: 'string',
70
+ describe: 'Path to a JSON file mapping queries to expected cardinality',
71
+ default: '',
72
+ },
73
+ });
74
+ exports.builder = builder;
75
+ const handler = (argv) => CliHelpers_1.wrapCommandHandler(argv, async (context) => CliHelpers_1.wrapVisualProgress('Listing data', async () => {
76
+ // Load options
77
+ const { experimentDirectories, experimentNames } = TexUtils_1.getExperimentNames(argv);
78
+ const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;
79
+ const correctnessReference = argv.correctnessReference ?
80
+ CorrectnessCheckerUtils_1.constructCorrectnessChecker(argv.correctnessReference) :
81
+ undefined;
82
+ // Prepare output CSV file
83
+ const os = fs.createWriteStream(path_1.default.join(context.cwd, `${argv.name}`));
84
+ const serializer = argv.markdown ? new TableSerializerMarkdown_1.TableSerializerMarkdown(os) : new TableSerializerCsv_1.TableSerializerCsv(os);
85
+ serializer.writeHeader([
86
+ 'experiment',
87
+ 'time',
88
+ 'requests',
89
+ 'results',
90
+ ...correctnessReference ? ['correctness'] : [],
91
+ ]);
92
+ // Read CSV files
93
+ for (const [experimentId, experimentDirectory] of experimentDirectories.entries()) {
94
+ // Read CSV file
95
+ const csvFile = path_1.default.join(experimentDirectory, argv.inputName);
96
+ await new Promise((resolve, reject) => {
97
+ const parser = csv_parse_1.default({ delimiter: argv.inputDelimiter, columns: true });
98
+ parser.on('data', data => {
99
+ if (!queryRegex || queryRegex.test(data.name)) {
100
+ serializer.writeRow([
101
+ experimentNames[experimentId],
102
+ data.time,
103
+ data.httpRequests || '0',
104
+ data.results,
105
+ ...correctnessReference ?
106
+ [correctnessReference.getCorrectness(experimentId, data.name, Number.parseInt(data.results, 10))] :
107
+ [],
108
+ ]);
109
+ }
110
+ });
111
+ parser.on('error', reject);
112
+ parser.on('end', resolve);
113
+ fs.createReadStream(csvFile).pipe(parser);
114
+ });
115
+ }
116
+ // Close output CSV file
117
+ serializer.close();
118
+ }));
119
+ exports.handler = handler;
120
+ //# sourceMappingURL=CommandQuery.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CommandQuery.js","sourceRoot":"","sources":["CommandQuery.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,gDAAwB;AAExB,0DAA8B;AAG9B,0FAA2F;AAC3F,iDAA0E;AAG1E,8CAAqD;AACrD,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,2BAA2B,CAAC;AACtC,QAAA,IAAI,GAAG,2DAA2D,CAAC;AACzE,MAAM,OAAO,GAAG,CAAC,KAAgB,EAAa,EAAE,CACrD,KAAK;KACF,OAAO,CAAC;IACP,UAAU,EAAE;QACV,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,6FAA6F;KACxG;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,yBAAyB;QACnC,OAAO,EAAE,cAAc;KACxB;IACD,SAAS,EAAE;QACT,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,uCAAuC;QACjD,OAAO,EAAE,iBAAiB;KAC3B;IACD,cAAc,EAAE;QACd,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,kCAAkC;QAC5C,OAAO,EAAE,GAAG;KACb;IACD,yBAAyB,EAAE;QACzB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,mDAAmD;KAC9D;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,gDAAgD;QAC1D,OAAO,EAAE,KAAK;KACf;IACD,oBAAoB,EAAE;QACpB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,6DAA6D;QACvE,OAAO,EAAE,EAAE;KACZ;CACF,CAAC,CAAC;AAtCM,QAAA,OAAO,WAsCb;AACA,MAAM,OAAO,GAAG,CAAC,IAAyB,EAAiB,EAAE,CAAC,+BAAkB,CAAC,IAAI,EAC1F,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,+BAAkB,CAAC,cAAc,EAAE,KAAK,IAAG,EAAE;IAC3E,eAAe;IACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,6BAAkB,CAAC,IAAI,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACtD,qDAA2B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACxD,SAAS,CAAC;IAEZ,0BAA0B;IAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;IAChG,UAAU,CAAC,WAAW,CAAC;QACrB,YAAY;QACZ,MAAM;QACN,UAAU;QACV,SAAS;QACT,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,aAAa,CAAE,CAAC,CAAC,CAAC,EAAE;KACjD,CAAC,CAAC;IAEH,iBAAiB;IACjB,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE;QACnF,gBAAgB;QAChB,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACpC,MAAM,MAAM,GAAG,mBAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;gBACvB,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;oBAC7C,UAAU,CAAC,QAAQ,CAAC;wBAClB,eAAe,CAAC,YAAY,CAAC;wBAC7B,IAAI,CAAC,IAAI;wBACT,IAAI,CAAC,YAAY,IAAI,GAAG;wBACxB,IAAI,CAAC,OAAO;wBACZ,GAAG,oBAAoB,CAAC,CAAC;4BACvB,CAAE,oBAAoB,CAAC,cAAc,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC;4BACrG,EAAE;qBACL,CAAC,CAAC;iBACJ;YACH,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAE1B,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;KACJ;IAED,wBAAwB;IACxB,UAAU,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC,CAAC,CAAC,CAAC;AAhDO,QAAA,OAAO,WAgDd","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\n\nimport parse from 'csv-parse';\nimport type { Argv } from 'yargs';\n\nimport { constructCorrectnessChecker } from '../../../correctness/CorrectnessCheckerUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\n\nimport type { ITaskContext } from '../../ITaskContext';\nimport { getExperimentNames } from '../tex/TexUtils';\nimport { TableSerializerCsv } from './TableSerializerCsv';\nimport { TableSerializerMarkdown } from './TableSerializerMarkdown';\n\nexport const command = 'query <experiment-dir...>';\nexport const desc = 'List all query execution times from the given experiments';\nexport const builder = (yargs: Argv<any>): Argv<any> =>\n yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'data_all.csv',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n markdown: {\n type: 'boolean',\n describe: 'If the output should be serialized as markdown',\n default: false,\n },\n correctnessReference: {\n type: 'string',\n describe: 'Path to a JSON file mapping queries to expected cardinality',\n default: '',\n },\n });\nexport const handler = (argv: Record<string, any>): Promise<void> => wrapCommandHandler(argv,\n async(context: ITaskContext) => wrapVisualProgress('Listing data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const correctnessReference = argv.correctnessReference ?\n constructCorrectnessChecker(argv.correctnessReference) :\n undefined;\n\n // Prepare output CSV file\n const os = fs.createWriteStream(Path.join(context.cwd, `${argv.name}`));\n const serializer = argv.markdown ? new TableSerializerMarkdown(os) : new TableSerializerCsv(os);\n serializer.writeHeader([\n 'experiment',\n 'time',\n 'requests',\n 'results',\n ...correctnessReference ? [ 'correctness' ] : [],\n ]);\n\n // Read CSV files\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n // Read CSV file\n const csvFile = Path.join(experimentDirectory, argv.inputName);\n await new Promise((resolve, reject) => {\n const parser = parse({ delimiter: argv.inputDelimiter, columns: true });\n parser.on('data', data => {\n if (!queryRegex || queryRegex.test(data.name)) {\n serializer.writeRow([\n experimentNames[experimentId],\n data.time,\n data.httpRequests || '0',\n data.results,\n ...correctnessReference ?\n [ correctnessReference.getCorrectness(experimentId, data.name, Number.parseInt(data.results, 10)) ] :\n [],\n ]);\n }\n });\n parser.on('error', reject);\n parser.on('end', resolve);\n\n fs.createReadStream(csvFile).pipe(parser);\n });\n }\n\n // Close output CSV file\n serializer.close();\n }));\n"]}
@@ -0,0 +1,5 @@
1
+ import type { Argv } from 'yargs';
2
+ export declare const command = "summary <experiment-dir...>";
3
+ export declare const desc = "Summarize query results from the given experiments in a table";
4
+ export declare const builder: (yargs: Argv<any>) => Argv<any>;
5
+ export declare const handler: (argv: Record<string, any>) => Promise<void>;
@@ -0,0 +1,246 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
5
+ }) : (function(o, m, k, k2) {
6
+ if (k2 === undefined) k2 = k;
7
+ o[k2] = m[k];
8
+ }));
9
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
10
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
11
+ }) : function(o, v) {
12
+ o["default"] = v;
13
+ });
14
+ var __importStar = (this && this.__importStar) || function (mod) {
15
+ if (mod && mod.__esModule) return mod;
16
+ var result = {};
17
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
18
+ __setModuleDefault(result, mod);
19
+ return result;
20
+ };
21
+ var __importDefault = (this && this.__importDefault) || function (mod) {
22
+ return (mod && mod.__esModule) ? mod : { "default": mod };
23
+ };
24
+ Object.defineProperty(exports, "__esModule", { value: true });
25
+ exports.handler = exports.builder = exports.desc = exports.command = void 0;
26
+ const fs = __importStar(require("fs"));
27
+ const path_1 = __importDefault(require("path"));
28
+ const CorrectnessCheckerUtils_1 = require("../../../correctness/CorrectnessCheckerUtils");
29
+ const CliHelpers_1 = require("../../CliHelpers");
30
+ const TexUtils_1 = require("../tex/TexUtils");
31
+ const TableSerializerCsv_1 = require("./TableSerializerCsv");
32
+ const TableSerializerMarkdown_1 = require("./TableSerializerMarkdown");
33
+ exports.command = 'summary <experiment-dir...>';
34
+ exports.desc = 'Summarize query results from the given experiments in a table';
35
+ const builder = (yargs) => yargs
36
+ .options({
37
+ queryRegex: {
38
+ type: 'string',
39
+ alias: 'q',
40
+ describe: 'Regex for queries to include (before any label overrides). Examples: \'^C\', \'^[^C]\', ...',
41
+ },
42
+ name: {
43
+ type: 'string',
44
+ alias: 'n',
45
+ describe: 'Custom output file name',
46
+ default: 'data_summary.csv',
47
+ },
48
+ inputName: {
49
+ type: 'string',
50
+ describe: 'Custom input file name per experiment',
51
+ default: 'query-times.csv',
52
+ },
53
+ inputDelimiter: {
54
+ type: 'string',
55
+ describe: 'Delimiter for the input CSV file',
56
+ default: ';',
57
+ },
58
+ overrideCombinationLabels: {
59
+ type: 'string',
60
+ describe: 'Comma-separated list of combination labels to use',
61
+ },
62
+ overrideQueryLabels: {
63
+ type: 'string',
64
+ describe: 'Comma-separated list of query labels to use',
65
+ },
66
+ markdown: {
67
+ type: 'boolean',
68
+ describe: 'If the output should be serialized as markdown',
69
+ default: false,
70
+ },
71
+ queryAverage: {
72
+ type: 'boolean',
73
+ describe: 'If queries will be averaged over',
74
+ default: false,
75
+ },
76
+ correctnessReference: {
77
+ type: 'string',
78
+ describe: 'Path to a JSON file mapping queries to expected cardinality',
79
+ default: '',
80
+ },
81
+ markRows: {
82
+ type: 'string',
83
+ describe: 'Comma-separated list of row id\'s to mark (markdown-only)',
84
+ },
85
+ });
86
+ exports.builder = builder;
87
+ const handler = (argv) => CliHelpers_1.wrapCommandHandler(argv, async (context) => CliHelpers_1.wrapVisualProgress('Summarizing data', async () => {
88
+ // Load options
89
+ const { experimentDirectories, experimentNames } = TexUtils_1.getExperimentNames(argv);
90
+ const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;
91
+ const correctnessReference = argv.correctnessReference ?
92
+ CorrectnessCheckerUtils_1.constructCorrectnessChecker(argv.correctnessReference) :
93
+ undefined;
94
+ const markRows = argv.markRows ?
95
+ argv.markRows.split(',').map((value) => Number.parseInt(value, 10)) :
96
+ [];
97
+ // Prepare output CSV file
98
+ const os = fs.createWriteStream(path_1.default.join(context.cwd, `${argv.name}`));
99
+ const serializer = argv.markdown ? new TableSerializerMarkdown_1.TableSerializerMarkdown(os) : new TableSerializerCsv_1.TableSerializerCsv(os);
100
+ if (argv.queryAverage) {
101
+ serializer.writeHeader([
102
+ '',
103
+ '$$\\overline{t}$$',
104
+ '$$\\tilde{t}$$',
105
+ '$$\\overline{t}_1$$',
106
+ '$$\\tilde{t}_1$$',
107
+ '$$\\overline{req}$$',
108
+ '$$\\sum ans$$',
109
+ ...correctnessReference ? ['$$\\overline{cor}$$'] : [],
110
+ '$$\\sum to$$',
111
+ ], {
112
+ align: [
113
+ 'left',
114
+ 'right',
115
+ 'right',
116
+ 'right',
117
+ 'right',
118
+ 'right',
119
+ 'right',
120
+ 'right',
121
+ 'right',
122
+ ],
123
+ });
124
+ }
125
+ else {
126
+ serializer.writeHeader([
127
+ 'Experiment',
128
+ 'Query',
129
+ 'Time',
130
+ 'Requests',
131
+ 'Results',
132
+ ...correctnessReference ? ['Correctness'] : [],
133
+ 'Timeout',
134
+ ], {
135
+ align: [
136
+ 'left',
137
+ 'right',
138
+ 'right',
139
+ 'right',
140
+ 'right',
141
+ 'right',
142
+ 'right',
143
+ ],
144
+ });
145
+ }
146
+ // Collect data from all experiments
147
+ for (const [experimentId, experimentDirectory] of experimentDirectories.entries()) {
148
+ // Read CSV file
149
+ const timesTotal = {};
150
+ const timesFirstTotal = {};
151
+ const requestsTotal = {};
152
+ let results = {};
153
+ let timeout = {};
154
+ await TexUtils_1.handleCsvFile(experimentDirectory, argv, data => {
155
+ if (!queryRegex || queryRegex.test(data.name)) {
156
+ if (!(data.name in timesTotal)) {
157
+ timesTotal[data.name] = [];
158
+ }
159
+ timesTotal[data.name].push(Number.parseInt(data.time, 10));
160
+ if (!(data.name in requestsTotal)) {
161
+ requestsTotal[data.name] = [];
162
+ }
163
+ requestsTotal[data.name].push(Number
164
+ .parseInt(data.error === 'false' && data.httpRequests ? data.httpRequests : '0', 10));
165
+ results[data.name] = Number.parseInt(data.results, 10);
166
+ timeout[data.name] = data.error === 'true';
167
+ if (data.timestamps) {
168
+ if (!(data.name in timesFirstTotal)) {
169
+ timesFirstTotal[data.name] = [];
170
+ }
171
+ timesFirstTotal[data.name].push(Number.parseInt(data.timestamps.split(' ')[0], 10));
172
+ }
173
+ }
174
+ });
175
+ // Calculate average
176
+ let timesAverage = {};
177
+ let timesAll = [];
178
+ for (const [query, times] of Object.entries(timesTotal)) {
179
+ timesAverage[query] = TexUtils_1.calcAverage(times);
180
+ timesAll = [...timesAll, ...times];
181
+ }
182
+ let timesFirstAverage = {};
183
+ let timesFirstAll = [];
184
+ for (const [query, times] of Object.entries(timesFirstTotal)) {
185
+ timesFirstAverage[query] = TexUtils_1.calcAverage(times);
186
+ timesFirstAll = [...timesFirstAll, ...times];
187
+ }
188
+ let requestsSum = {};
189
+ for (const [query, requests] of Object.entries(requestsTotal)) {
190
+ requestsSum[query] = TexUtils_1.calcAverage(requests);
191
+ }
192
+ // Calculate correctness
193
+ let correctness;
194
+ if (correctnessReference) {
195
+ correctness = Object.fromEntries(Object.entries(results)
196
+ .map(([key, value]) => [key, correctnessReference.getCorrectness(experimentId, key, value)]));
197
+ }
198
+ // Determine query names
199
+ const queryNames = TexUtils_1.getQueryNames(Object.keys(results), argv);
200
+ timesAverage = TexUtils_1.relabelQueryNames(timesAverage, queryNames);
201
+ timesFirstAverage = TexUtils_1.relabelQueryNames(timesFirstAverage, queryNames);
202
+ requestsSum = TexUtils_1.relabelQueryNames(requestsSum, queryNames);
203
+ results = TexUtils_1.relabelQueryNames(results, queryNames);
204
+ timeout = TexUtils_1.relabelQueryNames(timeout, queryNames);
205
+ // Write rows in output file
206
+ if (argv.queryAverage) {
207
+ const valueTimeFirstAverage = Object.values(timesFirstAverage).length === 0 ?
208
+ 'N/A' :
209
+ TexUtils_1.calcAverage(Object.values(timesFirstAverage))
210
+ .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });
211
+ const valueTimeFirstMedian = timesFirstAll.length === 0 ?
212
+ 'N/A' :
213
+ TexUtils_1.calcMedian(timesFirstAll)
214
+ .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });
215
+ // Average across queries
216
+ serializer.writeRow([
217
+ experimentNames[experimentId],
218
+ `${TexUtils_1.calcAverage(Object.values(timesAverage)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,
219
+ `${TexUtils_1.calcMedian(timesAll).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,
220
+ `${valueTimeFirstAverage}`,
221
+ `${valueTimeFirstMedian}`,
222
+ `${TexUtils_1.calcSum(Object.values(requestsSum)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,
223
+ `${TexUtils_1.calcAverage(Object.values(results)).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`,
224
+ ...correctness ? [`${(TexUtils_1.calcAverage(Object.values(correctness)) * 100).toFixed(2)}%`] : [],
225
+ `${TexUtils_1.calcSum(Object.values(timeout).map(value => value ? 1 : 0))}`,
226
+ ], { mark: markRows.includes(experimentId) });
227
+ }
228
+ else {
229
+ for (const [query, time] of Object.entries(timesAverage)) {
230
+ serializer.writeRow([
231
+ experimentNames[experimentId],
232
+ query,
233
+ `${time}`,
234
+ `${requestsSum[query]}`,
235
+ `${results[query]}`,
236
+ ...correctness ? [`${(correctness[query] * 100).toFixed(2)}%`] : [],
237
+ `${timeout[query]}`,
238
+ ], { mark: markRows.includes(experimentId) });
239
+ }
240
+ }
241
+ }
242
+ // Close output CSV file
243
+ serializer.close();
244
+ }));
245
+ exports.handler = handler;
246
+ //# sourceMappingURL=CommandSummary.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CommandSummary.js","sourceRoot":"","sources":["CommandSummary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,gDAAwB;AAExB,0FAA2F;AAC3F,iDAA0E;AAE1E,8CAOyB;AACzB,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,6BAA6B,CAAC;AACxC,QAAA,IAAI,GAAG,+DAA+D,CAAC;AAC7E,MAAM,OAAO,GAAG,CAAC,KAAgB,EAAa,EAAE,CACrD,KAAK;KACF,OAAO,CAAC;IACP,UAAU,EAAE;QACV,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,6FAA6F;KACxG;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,yBAAyB;QACnC,OAAO,EAAE,kBAAkB;KAC5B;IACD,SAAS,EAAE;QACT,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,uCAAuC;QACjD,OAAO,EAAE,iBAAiB;KAC3B;IACD,cAAc,EAAE;QACd,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,kCAAkC;QAC5C,OAAO,EAAE,GAAG;KACb;IACD,yBAAyB,EAAE;QACzB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,mDAAmD;KAC9D;IACD,mBAAmB,EAAE;QACnB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,6CAA6C;KACxD;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,gDAAgD;QAC1D,OAAO,EAAE,KAAK;KACf;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,kCAAkC;QAC5C,OAAO,EAAE,KAAK;KACf;IACD,oBAAoB,EAAE;QACpB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,6DAA6D;QACvE,OAAO,EAAE,EAAE;KACZ;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,2DAA2D;KACtE;CACF,CAAC,CAAC;AAnDM,QAAA,OAAO,WAmDb;AACA,MAAM,OAAO,GAAG,CAAC,IAAyB,EAAiB,EAAE,CAAC,+BAAkB,CAAC,IAAI,EAC1F,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,+BAAkB,CAAC,kBAAkB,EAAE,KAAK,IAAG,EAAE;IAC/E,eAAe;IACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,6BAAkB,CAAC,IAAI,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACtD,qDAA2B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACxD,SAAS,CAAC;IACZ,MAAM,QAAQ,GAAa,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,EAAE,CAAC;IAEL,0BAA0B;IAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;IAChG,IAAI,IAAI,CAAC,YAAY,EAAE;QACrB,UAAU,CAAC,WAAW,CAAC;YACrB,EAAE;YACF,mBAAmB;YACnB,gBAAgB;YAChB,qBAAqB;YACrB,kBAAkB;YAClB,qBAAqB;YACrB,eAAe;YACf,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,qBAAqB,CAAE,CAAC,CAAC,CAAC,EAAE;YACxD,cAAc;SACf,EAAE;YACD,KAAK,EAAE;gBACL,MAAM;gBACN,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;aACR;SACF,CAAC,CAAC;KACJ;SAAM;QACL,UAAU,CAAC,WAAW,CAAC;YACrB,YAAY;YACZ,OAAO;YACP,MAAM;YACN,UAAU;YACV,SAAS;YACT,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,aAAa,CAAE,CAAC,CAAC,CAAC,EAAE;YAChD,SAAS;SACV,EAAE;YACD,KAAK,EAAE;gBACL,MAAM;gBACN,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO;aACR;SACF,CAAC,CAAC;KACJ;IAED,oCAAoC;IACpC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE;QACnF,gBAAgB;QAChB,MAAM,UAAU,GAA6B,EAAE,CAAC;QAChD,MAAM,eAAe,GAA6B,EAAE,CAAC;QACrD,MAAM,aAAa,GAA6B,EAAE,CAAC;QACnD,IAAI,OAAO,GAA2B,EAAE,CAAC;QACzC,IAAI,OAAO,GAA4B,EAAE,CAAC;QAC1C,MAAM,wBAAa,CAAC,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE;YACpD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7C,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,EAAE;oBAC9B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;iBAC5B;gBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;gBAE3D,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,aAAa,CAAC,EAAE;oBACjC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;iBAC/B;gBACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM;qBACjC,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;gBAExF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACvD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC;gBAE3C,IAAI,IAAI,CAAC,UAAU,EAAE;oBACnB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE;wBACnC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;qBACjC;oBACD,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;iBACrF;aACF;QACH,CAAC,CAAC,CAAC;QAEH,oBAAoB;QACpB,IAAI,YAAY,GAA2B,EAAE,CAAC;QAC9C,IAAI,QAAQ,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YACzD,YAAY,CAAC,KAAK,CAAC,GAAG,sBAAW,CAAC,KAAK,CAAC,CAAC;YACzC,QAAQ,GAAG,CAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAE,CAAC;SACtC;QACD,IAAI,iBAAiB,GAA2B,EAAE,CAAC;QACnD,IAAI,aAAa,GAAa,EAAE,CAAC;QACjC,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;YAC9D,iBAAiB,CAAC,KAAK,CAAC,GAAG,sBAAW,CAAC,KAAK,CAAC,CAAC;YAC9C,aAAa,GAAG,CAAE,GAAG,aAAa,EAAE,GAAG,KAAK,CAAE,CAAC;SAChD;QACD,IAAI,WAAW,GAA2B,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAE,KAAK,EAAE,QAAQ,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;YAC/D,WAAW,CAAC,KAAK,CAAC,GAAG,sBAAW,CAAC,QAAQ,CAAC,CAAC;SAC5C;QAED,wBAAwB;QACxB,IAAI,WAAW,CAAC;QAChB,IAAI,oBAAoB,EAAE;YACxB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;iBACrD,GAAG,CAAC,CAAC,CAAE,GAAG,EAAE,KAAK,CAAE,EAAE,EAAE,CAAC,CAAE,GAAG,EAAE,oBAAoB,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAE,CAAC,CAAC,CAAC;SACrG;QAED,wBAAwB;QACxB,MAAM,UAAU,GAAG,wBAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QAC7D,YAAY,GAAG,4BAAiB,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QAC3D,iBAAiB,GAAG,4BAAiB,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QACrE,WAAW,GAAG,4BAAiB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACzD,OAAO,GAAG,4BAAiB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACjD,OAAO,GAAG,4BAAiB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAEjD,4BAA4B;QAC5B,IAAI,IAAI,CAAC,YAAY,EAAE;YACrB,MAAM,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;gBAC3E,KAAK,CAAC,CAAC;gBACP,sBAAW,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;qBAC1C,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;YACrF,MAAM,oBAAoB,GAAG,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;gBACvD,KAAK,CAAC,CAAC;gBACP,qBAAU,CAAC,aAAa,CAAC;qBACtB,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;YAErF,yBAAyB;YACzB,UAAU,CAAC,QAAQ,CAAC;gBAClB,eAAe,CAAC,YAAY,CAAC;gBAC7B,GAAG,sBAAW,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;gBAC7H,GAAG,qBAAU,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;gBACzG,GAAG,qBAAqB,EAAE;gBAC1B,GAAG,oBAAoB,EAAE;gBACzB,GAAG,kBAAO,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;gBACxH,GAAG,sBAAW,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;gBACxH,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,sBAAW,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;gBAC1F,GAAG,kBAAO,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;aACjE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;SAC/C;aAAM;YACL,KAAK,MAAM,CAAE,KAAK,EAAE,IAAI,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE;gBAC1D,UAAU,CAAC,QAAQ,CAAC;oBAClB,eAAe,CAAC,YAAY,CAAC;oBAC7B,KAAK;oBACL,GAAG,IAAI,EAAE;oBACT,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE;oBACvB,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;oBACnB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;oBACrE,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;iBACpB,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;aAC/C;SACF;KACF;IAED,wBAAwB;IACxB,UAAU,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC,CAAC,CAAC,CAAC;AAvKO,QAAA,OAAO,WAuKd","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { constructCorrectnessChecker } from '../../../correctness/CorrectnessCheckerUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n calcAverage, calcMedian,\n calcSum,\n getExperimentNames,\n getQueryNames,\n handleCsvFile,\n relabelQueryNames,\n} from '../tex/TexUtils';\nimport { TableSerializerCsv } from './TableSerializerCsv';\nimport { TableSerializerMarkdown } from './TableSerializerMarkdown';\n\nexport const command = 'summary <experiment-dir...>';\nexport const desc = 'Summarize query results from the given experiments in a table';\nexport const builder = (yargs: Argv<any>): Argv<any> =>\n yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'data_summary.csv',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n overrideQueryLabels: {\n type: 'string',\n describe: 'Comma-separated list of query labels to use',\n },\n markdown: {\n type: 'boolean',\n describe: 'If the output should be serialized as markdown',\n default: false,\n },\n queryAverage: {\n type: 'boolean',\n describe: 'If queries will be averaged over',\n default: false,\n },\n correctnessReference: {\n type: 'string',\n describe: 'Path to a JSON file mapping queries to expected cardinality',\n default: '',\n },\n markRows: {\n type: 'string',\n describe: 'Comma-separated list of row id\\'s to mark (markdown-only)',\n },\n });\nexport const handler = (argv: Record<string, any>): Promise<void> => wrapCommandHandler(argv,\n async(context: ITaskContext) => wrapVisualProgress('Summarizing data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const correctnessReference = argv.correctnessReference ?\n constructCorrectnessChecker(argv.correctnessReference) :\n undefined;\n const markRows: number[] = argv.markRows ?\n argv.markRows.split(',').map((value: string) => Number.parseInt(value, 10)) :\n [];\n\n // Prepare output CSV file\n const os = fs.createWriteStream(Path.join(context.cwd, `${argv.name}`));\n const serializer = argv.markdown ? new TableSerializerMarkdown(os) : new TableSerializerCsv(os);\n if (argv.queryAverage) {\n serializer.writeHeader([\n '',\n '$$\\\\overline{t}$$',\n '$$\\\\tilde{t}$$',\n '$$\\\\overline{t}_1$$',\n '$$\\\\tilde{t}_1$$',\n '$$\\\\overline{req}$$',\n '$$\\\\sum ans$$',\n ...correctnessReference ? [ '$$\\\\overline{cor}$$' ] : [],\n '$$\\\\sum to$$',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n } else {\n serializer.writeHeader([\n 'Experiment',\n 'Query',\n 'Time',\n 'Requests',\n 'Results',\n ...correctnessReference ? [ 'Correctness' ] : [],\n 'Timeout',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n }\n\n // Collect data from all experiments\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n // Read CSV file\n const timesTotal: Record<string, number[]> = {};\n const timesFirstTotal: Record<string, number[]> = {};\n const requestsTotal: Record<string, number[]> = {};\n let results: Record<string, number> = {};\n let timeout: Record<string, boolean> = {};\n await handleCsvFile(experimentDirectory, argv, data => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (!(data.name in timesTotal)) {\n timesTotal[data.name] = [];\n }\n timesTotal[data.name].push(Number.parseInt(data.time, 10));\n\n if (!(data.name in requestsTotal)) {\n requestsTotal[data.name] = [];\n }\n requestsTotal[data.name].push(Number\n .parseInt(data.error === 'false' && data.httpRequests ? data.httpRequests : '0', 10));\n\n results[data.name] = Number.parseInt(data.results, 10);\n timeout[data.name] = data.error === 'true';\n\n if (data.timestamps) {\n if (!(data.name in timesFirstTotal)) {\n timesFirstTotal[data.name] = [];\n }\n timesFirstTotal[data.name].push(Number.parseInt(data.timestamps.split(' ')[0], 10));\n }\n }\n });\n\n // Calculate average\n let timesAverage: Record<string, number> = {};\n let timesAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesTotal)) {\n timesAverage[query] = calcAverage(times);\n timesAll = [ ...timesAll, ...times ];\n }\n let timesFirstAverage: Record<string, number> = {};\n let timesFirstAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesFirstTotal)) {\n timesFirstAverage[query] = calcAverage(times);\n timesFirstAll = [ ...timesFirstAll, ...times ];\n }\n let requestsSum: Record<string, number> = {};\n for (const [ query, requests ] of Object.entries(requestsTotal)) {\n requestsSum[query] = calcAverage(requests);\n }\n\n // Calculate correctness\n let correctness;\n if (correctnessReference) {\n correctness = Object.fromEntries(Object.entries(results)\n .map(([ key, value ]) => [ key, correctnessReference.getCorrectness(experimentId, key, value) ]));\n }\n\n // Determine query names\n const queryNames = getQueryNames(Object.keys(results), argv);\n timesAverage = relabelQueryNames(timesAverage, queryNames);\n timesFirstAverage = relabelQueryNames(timesFirstAverage, queryNames);\n requestsSum = relabelQueryNames(requestsSum, queryNames);\n results = relabelQueryNames(results, queryNames);\n timeout = relabelQueryNames(timeout, queryNames);\n\n // Write rows in output file\n if (argv.queryAverage) {\n const valueTimeFirstAverage = Object.values(timesFirstAverage).length === 0 ?\n 'N/A' :\n calcAverage(Object.values(timesFirstAverage))\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n const valueTimeFirstMedian = timesFirstAll.length === 0 ?\n 'N/A' :\n calcMedian(timesFirstAll)\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n\n // Average across queries\n serializer.writeRow([\n experimentNames[experimentId],\n `${calcAverage(Object.values(timesAverage)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcMedian(timesAll).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${valueTimeFirstAverage}`,\n `${valueTimeFirstMedian}`,\n `${calcSum(Object.values(requestsSum)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcAverage(Object.values(results)).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`,\n ...correctness ? [ `${(calcAverage(Object.values(correctness)) * 100).toFixed(2)}%` ] : [],\n `${calcSum(Object.values(timeout).map(value => value ? 1 : 0))}`,\n ], { mark: markRows.includes(experimentId) });\n } else {\n for (const [ query, time ] of Object.entries(timesAverage)) {\n serializer.writeRow([\n experimentNames[experimentId],\n query,\n `${time}`,\n `${requestsSum[query]}`,\n `${results[query]}`,\n ...correctness ? [ `${(correctness[query] * 100).toFixed(2)}%` ] : [],\n `${timeout[query]}`,\n ], { mark: markRows.includes(experimentId) });\n }\n }\n }\n\n // Close output CSV file\n serializer.close();\n }));\n"]}
@@ -0,0 +1,5 @@
1
+ import type { Argv } from 'yargs';
2
+ export declare const command = "wins <experiment-dir...>";
3
+ export declare const desc = "Show distribution of query wins from the given experiments in a table";
4
+ export declare const builder: (yargs: Argv<any>) => Argv<any>;
5
+ export declare const handler: (argv: Record<string, any>) => Promise<void>;
@@ -0,0 +1,151 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
5
+ }) : (function(o, m, k, k2) {
6
+ if (k2 === undefined) k2 = k;
7
+ o[k2] = m[k];
8
+ }));
9
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
10
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
11
+ }) : function(o, v) {
12
+ o["default"] = v;
13
+ });
14
+ var __importStar = (this && this.__importStar) || function (mod) {
15
+ if (mod && mod.__esModule) return mod;
16
+ var result = {};
17
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
18
+ __setModuleDefault(result, mod);
19
+ return result;
20
+ };
21
+ var __importDefault = (this && this.__importDefault) || function (mod) {
22
+ return (mod && mod.__esModule) ? mod : { "default": mod };
23
+ };
24
+ Object.defineProperty(exports, "__esModule", { value: true });
25
+ exports.handler = exports.builder = exports.desc = exports.command = void 0;
26
+ const fs = __importStar(require("fs"));
27
+ const path_1 = __importDefault(require("path"));
28
+ const CorrectnessCheckerUtils_1 = require("../../../correctness/CorrectnessCheckerUtils");
29
+ const CliHelpers_1 = require("../../CliHelpers");
30
+ const TexUtils_1 = require("../tex/TexUtils");
31
+ const TableSerializerCsv_1 = require("./TableSerializerCsv");
32
+ const TableSerializerMarkdown_1 = require("./TableSerializerMarkdown");
33
+ exports.command = 'wins <experiment-dir...>';
34
+ exports.desc = 'Show distribution of query wins from the given experiments in a table';
35
+ const builder = (yargs) => yargs
36
+ .options({
37
+ queryRegex: {
38
+ type: 'string',
39
+ alias: 'q',
40
+ describe: 'Regex for queries to include (before any label overrides). Examples: \'^C\', \'^[^C]\', ...',
41
+ },
42
+ name: {
43
+ type: 'string',
44
+ alias: 'n',
45
+ describe: 'Custom output file name',
46
+ default: 'data_wins.csv',
47
+ },
48
+ inputName: {
49
+ type: 'string',
50
+ describe: 'Custom input file name per experiment',
51
+ default: 'query-times.csv',
52
+ },
53
+ inputDelimiter: {
54
+ type: 'string',
55
+ describe: 'Delimiter for the input CSV file',
56
+ default: ';',
57
+ },
58
+ overrideCombinationLabels: {
59
+ type: 'string',
60
+ describe: 'Comma-separated list of combination labels to use',
61
+ },
62
+ markdown: {
63
+ type: 'boolean',
64
+ describe: 'If the output should be serialized as markdown',
65
+ default: false,
66
+ },
67
+ correctnessReference: {
68
+ type: 'string',
69
+ describe: 'Path to a JSON file mapping queries to expected cardinality',
70
+ default: '',
71
+ },
72
+ markRows: {
73
+ type: 'string',
74
+ describe: 'Comma-separated list of row id\'s to mark (markdown-only)',
75
+ },
76
+ });
77
+ exports.builder = builder;
78
+ const handler = (argv) => CliHelpers_1.wrapCommandHandler(argv, async (context) => CliHelpers_1.wrapVisualProgress('Collecting win data', async () => {
79
+ // Load options
80
+ const { experimentDirectories, experimentNames } = TexUtils_1.getExperimentNames(argv);
81
+ const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;
82
+ const correctnessReference = argv.correctnessReference ?
83
+ CorrectnessCheckerUtils_1.constructCorrectnessChecker(argv.correctnessReference) :
84
+ undefined;
85
+ const markRows = argv.markRows ?
86
+ argv.markRows.split(',').map((value) => Number.parseInt(value, 10)) :
87
+ [];
88
+ // Prepare output CSV file
89
+ const os = fs.createWriteStream(path_1.default.join(context.cwd, `${argv.name}`));
90
+ const serializer = argv.markdown ? new TableSerializerMarkdown_1.TableSerializerMarkdown(os) : new TableSerializerCsv_1.TableSerializerCsv(os);
91
+ // Collect timings
92
+ const times = {};
93
+ const queries = [];
94
+ const experimentIds = [];
95
+ for (const [experimentId, experimentDirectory] of experimentDirectories.entries()) {
96
+ const results = {};
97
+ experimentIds.push(experimentId);
98
+ times[experimentId] = {};
99
+ await TexUtils_1.handleCsvFile(experimentDirectory, argv, data => {
100
+ if (!queryRegex || queryRegex.test(data.name)) {
101
+ if (experimentId === 0) {
102
+ queries.push(data.name);
103
+ }
104
+ times[experimentId][data.name] = Number.parseInt(data.time, 10);
105
+ results[data.name] = Number.parseInt(data.results, 10);
106
+ }
107
+ });
108
+ // Calculate correctness
109
+ if (correctnessReference) {
110
+ for (const [query, cardinality] of Object.entries(results)) {
111
+ if (correctnessReference.getCorrectness(experimentId, query, cardinality) !== 1) {
112
+ times[experimentId][query] = Number.POSITIVE_INFINITY;
113
+ }
114
+ }
115
+ }
116
+ }
117
+ // Calculate wins
118
+ const experimentWins = {};
119
+ for (const query of queries) {
120
+ let bestExperiment = -1;
121
+ let lowestTime = Number.POSITIVE_INFINITY;
122
+ for (const experimentId of experimentIds) {
123
+ if (times[experimentId][query] < lowestTime) {
124
+ lowestTime = times[experimentId][query];
125
+ bestExperiment = experimentId;
126
+ }
127
+ }
128
+ if (!(bestExperiment in experimentWins)) {
129
+ experimentWins[bestExperiment] = 0;
130
+ }
131
+ experimentWins[bestExperiment]++;
132
+ }
133
+ // Serialize wins
134
+ serializer.writeHeader([
135
+ '',
136
+ ...experimentNames,
137
+ ], {
138
+ align: [
139
+ 'left',
140
+ ...experimentNames.map(value => 'right'),
141
+ ],
142
+ });
143
+ serializer.writeRow([
144
+ 'Wins',
145
+ ...experimentIds.map(experimentId => `${experimentWins[experimentId] || 0}`),
146
+ ]);
147
+ // Close output CSV file
148
+ serializer.close();
149
+ }));
150
+ exports.handler = handler;
151
+ //# sourceMappingURL=CommandWins.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CommandWins.js","sourceRoot":"","sources":["CommandWins.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,gDAAwB;AAExB,0FAA2F;AAC3F,iDAA0E;AAE1E,8CAKyB;AACzB,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,0BAA0B,CAAC;AACrC,QAAA,IAAI,GAAG,uEAAuE,CAAC;AACrF,MAAM,OAAO,GAAG,CAAC,KAAgB,EAAa,EAAE,CACrD,KAAK;KACF,OAAO,CAAC;IACP,UAAU,EAAE;QACV,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,6FAA6F;KACxG;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,yBAAyB;QACnC,OAAO,EAAE,eAAe;KACzB;IACD,SAAS,EAAE;QACT,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,uCAAuC;QACjD,OAAO,EAAE,iBAAiB;KAC3B;IACD,cAAc,EAAE;QACd,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,kCAAkC;QAC5C,OAAO,EAAE,GAAG;KACb;IACD,yBAAyB,EAAE;QACzB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,mDAAmD;KAC9D;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,gDAAgD;QAC1D,OAAO,EAAE,KAAK;KACf;IACD,oBAAoB,EAAE;QACpB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,6DAA6D;QACvE,OAAO,EAAE,EAAE;KACZ;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,2DAA2D;KACtE;CACF,CAAC,CAAC;AA1CM,QAAA,OAAO,WA0Cb;AACA,MAAM,OAAO,GAAG,CAAC,IAAyB,EAAiB,EAAE,CAAC,+BAAkB,CAAC,IAAI,EAC1F,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,+BAAkB,CAAC,qBAAqB,EAAE,KAAK,IAAG,EAAE;IAClF,eAAe;IACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,6BAAkB,CAAC,IAAI,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACtD,qDAA2B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACxD,SAAS,CAAC;IACZ,MAAM,QAAQ,GAAa,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,EAAE,CAAC;IAEL,0BAA0B;IAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;IAEhG,kBAAkB;IAClB,MAAM,KAAK,GAA2C,EAAE,CAAC;IACzD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE;QACnF,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACjC,KAAK,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;QAEzB,MAAM,wBAAa,CAAC,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE;YACpD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7C,IAAI,YAAY,KAAK,CAAC,EAAE;oBACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACzB;gBAED,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;aACxD;QACH,CAAC,CAAC,CAAC;QAEH,wBAAwB;QACxB,IAAI,oBAAoB,EAAE;YACxB,KAAK,MAAM,CAAE,KAAK,EAAE,WAAW,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBAC5D,IAAI,oBAAoB,CAAC,cAAc,CAAC,YAAY,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE;oBAC/E,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;iBACvD;aACF;SACF;KACF;IAED,iBAAiB;IACjB,MAAM,cAAc,GAA2B,EAAE,CAAC;IAClD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;QAC3B,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC;QACxB,IAAI,UAAU,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAC1C,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE;YACxC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,UAAU,EAAE;gBAC3C,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC;gBACxC,cAAc,GAAG,YAAY,CAAC;aAC/B;SACF;QAED,IAAI,CAAC,CAAC,cAAc,IAAI,cAAc,CAAC,EAAE;YACvC,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;SACpC;QACD,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC;KAClC;IAED,iBAAiB;IACjB,UAAU,CAAC,WAAW,CAAC;QACrB,EAAE;QACF,GAAG,eAAe;KACnB,EAAE;QACD,KAAK,EAAE;YACL,MAAM;YACN,GAAc,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC;SACpD;KACF,CAAC,CAAC;IACH,UAAU,CAAC,QAAQ,CAAC;QAClB,MAAM;QACN,GAAG,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;KAC7E,CAAC,CAAC;IAEH,wBAAwB;IACxB,UAAU,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC,CAAC,CAAC,CAAC;AAjFO,QAAA,OAAO,WAiFd","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { constructCorrectnessChecker } from '../../../correctness/CorrectnessCheckerUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n\n getExperimentNames,\n\n handleCsvFile,\n} from '../tex/TexUtils';\nimport { TableSerializerCsv } from './TableSerializerCsv';\nimport { TableSerializerMarkdown } from './TableSerializerMarkdown';\n\nexport const command = 'wins <experiment-dir...>';\nexport const desc = 'Show distribution of query wins from the given experiments in a table';\nexport const builder = (yargs: Argv<any>): Argv<any> =>\n yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'data_wins.csv',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n markdown: {\n type: 'boolean',\n describe: 'If the output should be serialized as markdown',\n default: false,\n },\n correctnessReference: {\n type: 'string',\n describe: 'Path to a JSON file mapping queries to expected cardinality',\n default: '',\n },\n markRows: {\n type: 'string',\n describe: 'Comma-separated list of row id\\'s to mark (markdown-only)',\n },\n });\nexport const handler = (argv: Record<string, any>): Promise<void> => wrapCommandHandler(argv,\n async(context: ITaskContext) => wrapVisualProgress('Collecting win data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const correctnessReference = argv.correctnessReference ?\n constructCorrectnessChecker(argv.correctnessReference) :\n undefined;\n const markRows: number[] = argv.markRows ?\n argv.markRows.split(',').map((value: string) => Number.parseInt(value, 10)) :\n [];\n\n // Prepare output CSV file\n const os = fs.createWriteStream(Path.join(context.cwd, `${argv.name}`));\n const serializer = argv.markdown ? new TableSerializerMarkdown(os) : new TableSerializerCsv(os);\n\n // Collect timings\n const times: Record<string, Record<string, number>> = {};\n const queries: string[] = [];\n const experimentIds: number[] = [];\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n const results: Record<string, number> = {};\n experimentIds.push(experimentId);\n times[experimentId] = {};\n\n await handleCsvFile(experimentDirectory, argv, data => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (experimentId === 0) {\n queries.push(data.name);\n }\n\n times[experimentId][data.name] = Number.parseInt(data.time, 10);\n results[data.name] = Number.parseInt(data.results, 10);\n }\n });\n\n // Calculate correctness\n if (correctnessReference) {\n for (const [ query, cardinality ] of Object.entries(results)) {\n if (correctnessReference.getCorrectness(experimentId, query, cardinality) !== 1) {\n times[experimentId][query] = Number.POSITIVE_INFINITY;\n }\n }\n }\n }\n\n // Calculate wins\n const experimentWins: Record<number, number> = {};\n for (const query of queries) {\n let bestExperiment = -1;\n let lowestTime = Number.POSITIVE_INFINITY;\n for (const experimentId of experimentIds) {\n if (times[experimentId][query] < lowestTime) {\n lowestTime = times[experimentId][query];\n bestExperiment = experimentId;\n }\n }\n\n if (!(bestExperiment in experimentWins)) {\n experimentWins[bestExperiment] = 0;\n }\n experimentWins[bestExperiment]++;\n }\n\n // Serialize wins\n serializer.writeHeader([\n '',\n ...experimentNames,\n ], {\n align: [\n 'left',\n ...<'right'[]>experimentNames.map(value => 'right'),\n ],\n });\n serializer.writeRow([\n 'Wins',\n ...experimentIds.map(experimentId => `${experimentWins[experimentId] || 0}`),\n ]);\n\n // Close output CSV file\n serializer.close();\n }));\n"]}
@@ -0,0 +1,9 @@
1
+ export interface ITableSerializer {
2
+ writeHeader: (columns: string[], options?: {
3
+ align?: ('left' | 'right')[];
4
+ }) => void;
5
+ writeRow: (columns: string[], options?: {
6
+ mark?: boolean;
7
+ }) => void;
8
+ close: () => void;
9
+ }
@@ -0,0 +1,3 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ //# sourceMappingURL=ITableSerializer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ITableSerializer.js","sourceRoot":"","sources":["ITableSerializer.ts"],"names":[],"mappings":"","sourcesContent":["export interface ITableSerializer {\n writeHeader: (columns: string[], options?: { align?: ('left' | 'right')[] }) => void;\n writeRow: (columns: string[], options?: { mark?: boolean }) => void;\n close: () => void;\n}\n"]}