@rubensworks/process-sparql-benchmark-results 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/commands/csv/CommandGithubActionsBenchmark.js +1 -1
- package/lib/cli/commands/csv/CommandGithubActionsBenchmark.js.map +1 -1
- package/lib/cli/commands/csv/CommandSummary.js +2 -2
- package/lib/cli/commands/csv/CommandSummary.js.map +1 -1
- package/lib/cli/commands/csv/CommandWins.js +1 -1
- package/lib/cli/commands/csv/CommandWins.js.map +1 -1
- package/lib/cli/commands/tex/CommandQuery.js +13 -2
- package/lib/cli/commands/tex/CommandQuery.js.map +1 -1
- package/lib/templates/tex/plot_query_data.tex +1 -1
- package/package.json +3 -8
|
@@ -102,7 +102,7 @@ function handler(argv) {
|
|
|
102
102
|
const ghbenchDataRaw = {};
|
|
103
103
|
await (0, TexUtils_1.handleCsvFile)(experimentDirectory, argv, (data) => {
|
|
104
104
|
if (!queryRegex || queryRegex.test(data.name)) {
|
|
105
|
-
const value = Number.
|
|
105
|
+
const value = Number.parseFloat(data.time);
|
|
106
106
|
if (!ghbenchDataRaw[data.name]) {
|
|
107
107
|
ghbenchDataRaw[data.name] = {
|
|
108
108
|
name: `${experimentNames[experimentId]} - ${data.name}`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CommandGithubActionsBenchmark.js","sourceRoot":"","sources":["CommandGithubActionsBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAaA,0BA6CC;AACD,0BA8FC;AAzJD,uCAAyB;AACzB,gDAAwB;AAExB,iDAA0E;AAE1E,8CAIyB;AAEZ,QAAA,OAAO,GAAG,6BAA6B,CAAC;AACxC,QAAA,IAAI,GAAG,gEAAgE,CAAC;AACrF,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6DAA6D;gBACrE,kCAAkC;SACrC;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,cAAc;SACxB;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,KAAK,EAAE;YACL,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gEAAgE;YAC1E,OAAO,EAAE,KAAK;SACf;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,8EAA8E;YACxF,OAAO,EAAE,IAAI;SACd;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,iFAAiF;YAC3F,OAAO,EAAE,KAAK;SACf;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EACvB,IAAI,EACJ,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,yBAAyB,EAAE,KAAK,IAAG,EAAE;QACxF,eAAe;QACb,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAElF,kBAAkB;QAClB,MAAM,WAAW,GAAgB,EAAE,CAAC;QACpC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE,CAAC;YACpF,MAAM,cAAc,GAAmC,EAAE,CAAC;YAE1D,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAC7C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC/B,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;4BAC1B,IAAI,EAAE,GAAG,eAAe,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE;4BACvD,IAAI,EAAE,IAAI;4BACV,MAAM,EAAE,CAAE,KAAK,CAAE;4BACjB,KAAK,EAAE;gCACL,OAAO,EAAE,CAAE,IAAI,CAAC,OAAO,CAAE;gCACzB,KAAK,EAAE,CAAE,IAAI,CAAC,KAAK,CAAE;gCACrB,YAAY,EAAE,CAAE,IAAI,CAAC,YAAY,CAAE;6BACpC;yBACF,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC7C,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC3D,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACvD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACvE,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,qBAAqB;YACrB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClD,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,CAAC;gBAC/F,6FAA6F;gBAC7F,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,GAAG,IAAA,qBAAU,EAAC,aAAa,CAAC,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzB,KAAK,IAAI,KAAK,CAAC;gBACjB,CAAC;gBAED,kBAAkB;gBAClB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClB,WAAW,CAAC,IAAI,CAAC;wBACf,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK;wBACL,KAAK,EAAE,aAAa,KAAK,CAAC,KAAK,CAAC,OAAO,cAAc,KAAK,CAAC,KAAK,CAAC,KAAK,sBAAsB,KAAK,CAAC,KAAK,CAAC,YAAY,GAAG;qBACxH,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,GAAG,eAAe,CAAC,YAAY,CAAC,EAAE;oBACxC,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,KAAK;iBACb,CAAC,CAAC;YACL,CAAC;YAED,4DAA4D;YAC5D,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC;qBACjD,GAAG,CAAC,CAAC,CAAE,IAAI,EAAE,KAAK,CAAE,EAAE,EAAE,CAAC,CAAC;oBACzB,IAAI;oBACJ,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,MAAM;oBAClE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM;iBAChC,CAAC,CAAC;qBACF,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACrC,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,GAAG,eAAe,CAAC,YAAY,CAAC,mBAAmB;oBACzD,IAAI,EAAE,uBAAuB;oBAC7B,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;oBAClE,KAAK,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAC/B,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;wBAC/F,MAAM;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,eAAe;QACf,mCAAmC;QACnC,EAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IAC5G,CAAC,CAAC,CACH,CAAC;AACJ,CAAC","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n calcMedian,\n getExperimentNames,\n handleCsvFile,\n} from '../tex/TexUtils';\n\nexport const command = 'ghbench <experiment-dir...>';\nexport const desc = 'Generates a JSON file for usage in the Benchmark Github Action';\nexport function builder(yargs: Argv<any>): Argv<any> {\n return yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). ' +\n 'Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'ghbench.json',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n total: {\n type: 'boolean',\n describe: 'If total execution time across the experiment must be reported',\n default: false,\n },\n detailed: {\n type: 'boolean',\n describe: 'If separate execution time for each query in the experiment must be reported',\n default: true,\n },\n failures: {\n type: 'boolean',\n describe: 'If the number of failed query instantiations in the experiment must be reported',\n default: false,\n },\n });\n}\nexport function handler(argv: Record<string, any>): Promise<void> {\n return wrapCommandHandler(\n argv,\n async(context: ITaskContext) => wrapVisualProgress('Collecting ghbench data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n\n // Collect timings\n const ghbenchData: GhbenchData = [];\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n const ghbenchDataRaw: Record<string, GhbenchDataRaw> = {};\n\n await handleCsvFile(experimentDirectory, argv, (data) => {\n if (!queryRegex || queryRegex.test(data.name)) {\n const value = Number.parseInt(data.time, 10);\n if (!ghbenchDataRaw[data.name]) {\n ghbenchDataRaw[data.name] = {\n name: `${experimentNames[experimentId]} - ${data.name}`,\n unit: 'ms',\n values: [ value ],\n extra: {\n results: [ data.results ],\n error: [ data.error ],\n httpRequests: [ data.httpRequests ],\n },\n };\n } else {\n ghbenchDataRaw[data.name].values.push(value);\n ghbenchDataRaw[data.name].extra.results.push(data.results);\n ghbenchDataRaw[data.name].extra.error.push(data.error);\n ghbenchDataRaw[data.name].extra.httpRequests.push(data.httpRequests);\n }\n }\n });\n\n // Calculate averages\n let total = 0;\n for (const entry of Object.values(ghbenchDataRaw)) {\n const successValues = entry.values.filter((val, index) => entry.extra.error[index] !== 'true');\n // Queries that failed for all instantiations have no time, so they must not be reported as 0\n if (successValues.length === 0) {\n continue;\n }\n const value = calcMedian(successValues);\n if (!Number.isNaN(value)) {\n total += value;\n }\n\n // Output detailed\n if (argv.detailed) {\n ghbenchData.push({\n name: entry.name,\n unit: entry.unit,\n value,\n extra: `Results: [${entry.extra.results}]; Error: [${entry.extra.error}]; HTTP Requests: [${entry.extra.httpRequests}]`,\n });\n }\n }\n\n // Output totals\n if (argv.total) {\n ghbenchData.push({\n name: `${experimentNames[experimentId]}`,\n unit: 'ms',\n value: total,\n });\n }\n\n // Output failures, as failed queries have no time to report\n if (argv.failures) {\n const failedQueries = Object.entries(ghbenchDataRaw)\n .map(([ name, entry ]) => ({\n name,\n failed: entry.extra.error.filter(error => error === 'true').length,\n count: entry.extra.error.length,\n }))\n .filter(entry => entry.failed > 0);\n ghbenchData.push({\n name: `${experimentNames[experimentId]} - failed queries`,\n unit: 'failed instantiations',\n value: failedQueries.reduce((sum, entry) => sum + entry.failed, 0),\n extra: failedQueries.length > 0 ?\n failedQueries.map(entry => `${entry.name}: ${entry.failed}/${entry.count} failed`).join('; ') :\n 'None',\n });\n }\n }\n\n // Write output\n // eslint-disable-next-line no-sync\n fs.writeFileSync(Path.join(context.cwd, `${argv.name}`), JSON.stringify(ghbenchData, null, ' '), 'utf8');\n }),\n );\n}\n\ninterface GhbenchDataRaw {\n name: string;\n unit: string;\n values: number[];\n range?: string;\n extra: { results: number[]; error: string[]; httpRequests: number[] };\n}\n\ntype GhbenchData = {\n name: string;\n unit: string;\n value: number;\n range?: string;\n extra?: string;\n}[];\n"]}
|
|
1
|
+
{"version":3,"file":"CommandGithubActionsBenchmark.js","sourceRoot":"","sources":["CommandGithubActionsBenchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAaA,0BA6CC;AACD,0BA8FC;AAzJD,uCAAyB;AACzB,gDAAwB;AAExB,iDAA0E;AAE1E,8CAIyB;AAEZ,QAAA,OAAO,GAAG,6BAA6B,CAAC;AACxC,QAAA,IAAI,GAAG,gEAAgE,CAAC;AACrF,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6DAA6D;gBACrE,kCAAkC;SACrC;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,cAAc;SACxB;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,KAAK,EAAE;YACL,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gEAAgE;YAC1E,OAAO,EAAE,KAAK;SACf;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,8EAA8E;YACxF,OAAO,EAAE,IAAI;SACd;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,iFAAiF;YAC3F,OAAO,EAAE,KAAK;SACf;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EACvB,IAAI,EACJ,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,yBAAyB,EAAE,KAAK,IAAG,EAAE;QACxF,eAAe;QACb,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAElF,kBAAkB;QAClB,MAAM,WAAW,GAAgB,EAAE,CAAC;QACpC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE,CAAC;YACpF,MAAM,cAAc,GAAmC,EAAE,CAAC;YAE1D,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC3C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC/B,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;4BAC1B,IAAI,EAAE,GAAG,eAAe,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE;4BACvD,IAAI,EAAE,IAAI;4BACV,MAAM,EAAE,CAAE,KAAK,CAAE;4BACjB,KAAK,EAAE;gCACL,OAAO,EAAE,CAAE,IAAI,CAAC,OAAO,CAAE;gCACzB,KAAK,EAAE,CAAE,IAAI,CAAC,KAAK,CAAE;gCACrB,YAAY,EAAE,CAAE,IAAI,CAAC,YAAY,CAAE;6BACpC;yBACF,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC7C,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC3D,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACvD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACvE,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,qBAAqB;YACrB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClD,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,CAAC;gBAC/F,6FAA6F;gBAC7F,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,GAAG,IAAA,qBAAU,EAAC,aAAa,CAAC,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzB,KAAK,IAAI,KAAK,CAAC;gBACjB,CAAC;gBAED,kBAAkB;gBAClB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClB,WAAW,CAAC,IAAI,CAAC;wBACf,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK;wBACL,KAAK,EAAE,aAAa,KAAK,CAAC,KAAK,CAAC,OAAO,cAAc,KAAK,CAAC,KAAK,CAAC,KAAK,sBAAsB,KAAK,CAAC,KAAK,CAAC,YAAY,GAAG;qBACxH,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,GAAG,eAAe,CAAC,YAAY,CAAC,EAAE;oBACxC,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,KAAK;iBACb,CAAC,CAAC;YACL,CAAC;YAED,4DAA4D;YAC5D,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC;qBACjD,GAAG,CAAC,CAAC,CAAE,IAAI,EAAE,KAAK,CAAE,EAAE,EAAE,CAAC,CAAC;oBACzB,IAAI;oBACJ,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,MAAM;oBAClE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM;iBAChC,CAAC,CAAC;qBACF,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACrC,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,GAAG,eAAe,CAAC,YAAY,CAAC,mBAAmB;oBACzD,IAAI,EAAE,uBAAuB;oBAC7B,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;oBAClE,KAAK,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAC/B,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;wBAC/F,MAAM;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,eAAe;QACf,mCAAmC;QACnC,EAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IAC5G,CAAC,CAAC,CACH,CAAC;AACJ,CAAC","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n calcMedian,\n getExperimentNames,\n handleCsvFile,\n} from '../tex/TexUtils';\n\nexport const command = 'ghbench <experiment-dir...>';\nexport const desc = 'Generates a JSON file for usage in the Benchmark Github Action';\nexport function builder(yargs: Argv<any>): Argv<any> {\n return yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). ' +\n 'Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'ghbench.json',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n total: {\n type: 'boolean',\n describe: 'If total execution time across the experiment must be reported',\n default: false,\n },\n detailed: {\n type: 'boolean',\n describe: 'If separate execution time for each query in the experiment must be reported',\n default: true,\n },\n failures: {\n type: 'boolean',\n describe: 'If the number of failed query instantiations in the experiment must be reported',\n default: false,\n },\n });\n}\nexport function handler(argv: Record<string, any>): Promise<void> {\n return wrapCommandHandler(\n argv,\n async(context: ITaskContext) => wrapVisualProgress('Collecting ghbench data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n\n // Collect timings\n const ghbenchData: GhbenchData = [];\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n const ghbenchDataRaw: Record<string, GhbenchDataRaw> = {};\n\n await handleCsvFile(experimentDirectory, argv, (data) => {\n if (!queryRegex || queryRegex.test(data.name)) {\n const value = Number.parseFloat(data.time);\n if (!ghbenchDataRaw[data.name]) {\n ghbenchDataRaw[data.name] = {\n name: `${experimentNames[experimentId]} - ${data.name}`,\n unit: 'ms',\n values: [ value ],\n extra: {\n results: [ data.results ],\n error: [ data.error ],\n httpRequests: [ data.httpRequests ],\n },\n };\n } else {\n ghbenchDataRaw[data.name].values.push(value);\n ghbenchDataRaw[data.name].extra.results.push(data.results);\n ghbenchDataRaw[data.name].extra.error.push(data.error);\n ghbenchDataRaw[data.name].extra.httpRequests.push(data.httpRequests);\n }\n }\n });\n\n // Calculate averages\n let total = 0;\n for (const entry of Object.values(ghbenchDataRaw)) {\n const successValues = entry.values.filter((val, index) => entry.extra.error[index] !== 'true');\n // Queries that failed for all instantiations have no time, so they must not be reported as 0\n if (successValues.length === 0) {\n continue;\n }\n const value = calcMedian(successValues);\n if (!Number.isNaN(value)) {\n total += value;\n }\n\n // Output detailed\n if (argv.detailed) {\n ghbenchData.push({\n name: entry.name,\n unit: entry.unit,\n value,\n extra: `Results: [${entry.extra.results}]; Error: [${entry.extra.error}]; HTTP Requests: [${entry.extra.httpRequests}]`,\n });\n }\n }\n\n // Output totals\n if (argv.total) {\n ghbenchData.push({\n name: `${experimentNames[experimentId]}`,\n unit: 'ms',\n value: total,\n });\n }\n\n // Output failures, as failed queries have no time to report\n if (argv.failures) {\n const failedQueries = Object.entries(ghbenchDataRaw)\n .map(([ name, entry ]) => ({\n name,\n failed: entry.extra.error.filter(error => error === 'true').length,\n count: entry.extra.error.length,\n }))\n .filter(entry => entry.failed > 0);\n ghbenchData.push({\n name: `${experimentNames[experimentId]} - failed queries`,\n unit: 'failed instantiations',\n value: failedQueries.reduce((sum, entry) => sum + entry.failed, 0),\n extra: failedQueries.length > 0 ?\n failedQueries.map(entry => `${entry.name}: ${entry.failed}/${entry.count} failed`).join('; ') :\n 'None',\n });\n }\n }\n\n // Write output\n // eslint-disable-next-line no-sync\n fs.writeFileSync(Path.join(context.cwd, `${argv.name}`), JSON.stringify(ghbenchData, null, ' '), 'utf8');\n }),\n );\n}\n\ninterface GhbenchDataRaw {\n name: string;\n unit: string;\n values: number[];\n range?: string;\n extra: { results: number[]; error: string[]; httpRequests: number[] };\n}\n\ntype GhbenchData = {\n name: string;\n unit: string;\n value: number;\n range?: string;\n extra?: string;\n}[];\n"]}
|
|
@@ -174,7 +174,7 @@ function handler(argv) {
|
|
|
174
174
|
if (!(data.name in timesTotal)) {
|
|
175
175
|
timesTotal[data.name] = [];
|
|
176
176
|
}
|
|
177
|
-
timesTotal[data.name].push(Number.
|
|
177
|
+
timesTotal[data.name].push(Number.parseFloat(data.time));
|
|
178
178
|
if (!(data.name in requestsTotal)) {
|
|
179
179
|
requestsTotal[data.name] = [];
|
|
180
180
|
}
|
|
@@ -186,7 +186,7 @@ function handler(argv) {
|
|
|
186
186
|
if (!(data.name in timesFirstTotal)) {
|
|
187
187
|
timesFirstTotal[data.name] = [];
|
|
188
188
|
}
|
|
189
|
-
timesFirstTotal[data.name].push(Number.
|
|
189
|
+
timesFirstTotal[data.name].push(Number.parseFloat(data.timestamps.split(' ')[0]));
|
|
190
190
|
}
|
|
191
191
|
}
|
|
192
192
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CommandSummary.js","sourceRoot":"","sources":["CommandSummary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,0BAoDC;AACD,0BAwKC;AAjPD,uCAAyB;AACzB,gDAAwB;AAExB,0FAA2F;AAC3F,iDAA0E;AAE1E,8CAQyB;AACzB,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,6BAA6B,CAAC;AACxC,QAAA,IAAI,GAAG,+DAA+D,CAAC;AACpF,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6FAA6F;SACxG;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,kBAAkB;SAC5B;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,mBAAmB,EAAE;YACnB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6CAA6C;SACxD;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gDAAgD;YAC1D,OAAO,EAAE,KAAK;SACf;QACD,YAAY,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,KAAK;SACf;QACD,oBAAoB,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6DAA6D;YACvE,OAAO,EAAE,EAAE;SACZ;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,2DAA2D;SACtE;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EAAC,IAAI,EAAE,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,kBAAkB,EAAE,KAAK,IAAG,EAAE;QAC/G,eAAe;QACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACtD,IAAA,qDAA2B,EAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YACxD,SAAS,CAAC;QACZ,MAAM,QAAQ,GAAa,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3E,EAAE,CAAC;QAEP,0BAA0B;QAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;QAChG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,UAAU,CAAC,WAAW,CAAC;gBACrB,EAAE;gBACF,mBAAmB;gBACnB,gBAAgB;gBAChB,qBAAqB;gBACrB,kBAAkB;gBAClB,qBAAqB;gBACrB,eAAe;gBACf,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,qBAAqB,CAAE,CAAC,CAAC,CAAC,EAAE;gBACxD,cAAc;aACf,EAAE;gBACD,KAAK,EAAE;oBACL,MAAM;oBACN,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;iBACR;aACF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,WAAW,CAAC;gBACrB,YAAY;gBACZ,OAAO;gBACP,MAAM;gBACN,UAAU;gBACV,SAAS;gBACT,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,aAAa,CAAE,CAAC,CAAC,CAAC,EAAE;gBAChD,SAAS;aACV,EAAE;gBACD,KAAK,EAAE;oBACL,MAAM;oBACN,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;iBACR;aACF,CAAC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE,CAAC;YACpF,gBAAgB;YAChB,MAAM,UAAU,GAA6B,EAAE,CAAC;YAChD,MAAM,eAAe,GAA6B,EAAE,CAAC;YACrD,MAAM,aAAa,GAA6B,EAAE,CAAC;YACnD,IAAI,OAAO,GAA2B,EAAE,CAAC;YACzC,IAAI,OAAO,GAA4B,EAAE,CAAC;YAC1C,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,EAAE,CAAC;wBAC/B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC7B,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;oBAE3D,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,aAAa,CAAC,EAAE,CAAC;wBAClC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAChC,CAAC;oBACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM;yBACjC,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;oBAExF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC;oBAE3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE,CAAC;4BACpC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;wBAClC,CAAC;wBACD,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;oBACtF,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,oBAAoB;YACpB,IAAI,YAAY,GAA2B,EAAE,CAAC;YAC9C,IAAI,QAAQ,GAAa,EAAE,CAAC;YAC5B,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1D,YAAY,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBACzC,QAAQ,GAAG,CAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAE,CAAC;YACvC,CAAC;YACD,IAAI,iBAAiB,GAA2B,EAAE,CAAC;YACnD,IAAI,aAAa,GAAa,EAAE,CAAC;YACjC,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC/D,iBAAiB,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC9C,aAAa,GAAG,CAAE,GAAG,aAAa,EAAE,GAAG,KAAK,CAAE,CAAC;YACjD,CAAC;YACD,IAAI,WAAW,GAA2B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAE,KAAK,EAAE,QAAQ,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;gBAChE,WAAW,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,QAAQ,CAAC,CAAC;YAC7C,CAAC;YAED,wBAAwB;YACxB,IAAI,WAAW,CAAC;YAChB,IAAI,oBAAoB,EAAE,CAAC;gBACzB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;qBACrD,GAAG,CAAC,CAAC,CAAE,GAAG,EAAE,KAAK,CAAE,EAAE,EAAE,CAAC,CAAE,GAAG,EAAE,oBAAoB,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAE,CAAC,CAAC,CAAC;YACtG,CAAC;YAED,wBAAwB;YACxB,MAAM,UAAU,GAAG,IAAA,wBAAa,EAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;YAC7D,YAAY,GAAG,IAAA,4BAAiB,EAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAC3D,iBAAiB,GAAG,IAAA,4BAAiB,EAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;YACrE,WAAW,GAAG,IAAA,4BAAiB,EAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YACzD,OAAO,GAAG,IAAA,4BAAiB,EAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YACjD,OAAO,GAAG,IAAA,4BAAiB,EAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YAEjD,4BAA4B;YAC5B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;oBAC3E,KAAK,CAAC,CAAC;oBACP,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;yBAC1C,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrF,MAAM,oBAAoB,GAAG,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;oBACvD,KAAK,CAAC,CAAC;oBACP,IAAA,qBAAU,EAAC,aAAa,CAAC;yBACtB,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;gBAErF,yBAAyB;gBACzB,UAAU,CAAC,QAAQ,CAAC;oBAClB,eAAe,CAAC,YAAY,CAAC;oBAC7B,GAAG,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBAC7H,GAAG,IAAA,qBAAU,EAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACzG,GAAG,qBAAqB,EAAE;oBAC1B,GAAG,oBAAoB,EAAE;oBACzB,GAAG,IAAA,kBAAO,EAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACxH,GAAG,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACxH,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;oBAC1F,GAAG,IAAA,kBAAO,EAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;iBACjE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,CAAE,KAAK,EAAE,IAAI,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;oBAC3D,UAAU,CAAC,QAAQ,CAAC;wBAClB,eAAe,CAAC,YAAY,CAAC;wBAC7B,KAAK;wBACL,GAAG,IAAI,EAAE;wBACT,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE;wBACvB,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;wBACnB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;wBACrE,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;qBACpB,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC,CAAC;AACN,CAAC","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { constructCorrectnessChecker } from '../../../correctness/CorrectnessCheckerUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n calcAverage,\n calcMedian,\n calcSum,\n getExperimentNames,\n getQueryNames,\n handleCsvFile,\n relabelQueryNames,\n} from '../tex/TexUtils';\nimport { TableSerializerCsv } from './TableSerializerCsv';\nimport { TableSerializerMarkdown } from './TableSerializerMarkdown';\n\nexport const command = 'summary <experiment-dir...>';\nexport const desc = 'Summarize query results from the given experiments in a table';\nexport function builder(yargs: Argv<any>): Argv<any> {\n return yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'data_summary.csv',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n overrideQueryLabels: {\n type: 'string',\n describe: 'Comma-separated list of query labels to use',\n },\n markdown: {\n type: 'boolean',\n describe: 'If the output should be serialized as markdown',\n default: false,\n },\n queryAverage: {\n type: 'boolean',\n describe: 'If queries will be averaged over',\n default: false,\n },\n correctnessReference: {\n type: 'string',\n describe: 'Path to a JSON file mapping queries to expected cardinality',\n default: '',\n },\n markRows: {\n type: 'string',\n describe: 'Comma-separated list of row id\\'s to mark (markdown-only)',\n },\n });\n}\nexport function handler(argv: Record<string, any>): Promise<void> {\n return wrapCommandHandler(argv, async(context: ITaskContext) => wrapVisualProgress('Summarizing data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const correctnessReference = argv.correctnessReference ?\n constructCorrectnessChecker(argv.correctnessReference) :\n undefined;\n const markRows: number[] = argv.markRows ?\n argv.markRows.split(',').map((value: string) => Number.parseInt(value, 10)) :\n [];\n\n // Prepare output CSV file\n const os = fs.createWriteStream(Path.join(context.cwd, `${argv.name}`));\n const serializer = argv.markdown ? new TableSerializerMarkdown(os) : new TableSerializerCsv(os);\n if (argv.queryAverage) {\n serializer.writeHeader([\n '',\n '$$\\\\overline{t}$$',\n '$$\\\\tilde{t}$$',\n '$$\\\\overline{t}_1$$',\n '$$\\\\tilde{t}_1$$',\n '$$\\\\overline{req}$$',\n '$$\\\\sum ans$$',\n ...correctnessReference ? [ '$$\\\\overline{acc}$$' ] : [],\n '$$\\\\sum to$$',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n } else {\n serializer.writeHeader([\n 'Experiment',\n 'Query',\n 'Time',\n 'Requests',\n 'Results',\n ...correctnessReference ? [ 'Correctness' ] : [],\n 'Timeout',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n }\n\n // Collect data from all experiments\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n // Read CSV file\n const timesTotal: Record<string, number[]> = {};\n const timesFirstTotal: Record<string, number[]> = {};\n const requestsTotal: Record<string, number[]> = {};\n let results: Record<string, number> = {};\n let timeout: Record<string, boolean> = {};\n await handleCsvFile(experimentDirectory, argv, (data) => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (!(data.name in timesTotal)) {\n timesTotal[data.name] = [];\n }\n timesTotal[data.name].push(Number.parseInt(data.time, 10));\n\n if (!(data.name in requestsTotal)) {\n requestsTotal[data.name] = [];\n }\n requestsTotal[data.name].push(Number\n .parseInt(data.error === 'false' && data.httpRequests ? data.httpRequests : '0', 10));\n\n results[data.name] = Number.parseInt(data.results, 10);\n timeout[data.name] = data.error === 'true';\n\n if (data.timestamps) {\n if (!(data.name in timesFirstTotal)) {\n timesFirstTotal[data.name] = [];\n }\n timesFirstTotal[data.name].push(Number.parseInt(data.timestamps.split(' ')[0], 10));\n }\n }\n });\n\n // Calculate average\n let timesAverage: Record<string, number> = {};\n let timesAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesTotal)) {\n timesAverage[query] = calcAverage(times);\n timesAll = [ ...timesAll, ...times ];\n }\n let timesFirstAverage: Record<string, number> = {};\n let timesFirstAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesFirstTotal)) {\n timesFirstAverage[query] = calcAverage(times);\n timesFirstAll = [ ...timesFirstAll, ...times ];\n }\n let requestsSum: Record<string, number> = {};\n for (const [ query, requests ] of Object.entries(requestsTotal)) {\n requestsSum[query] = calcAverage(requests);\n }\n\n // Calculate correctness\n let correctness;\n if (correctnessReference) {\n correctness = Object.fromEntries(Object.entries(results)\n .map(([ key, value ]) => [ key, correctnessReference.getCorrectness(experimentId, key, value) ]));\n }\n\n // Determine query names\n const queryNames = getQueryNames(Object.keys(results), argv);\n timesAverage = relabelQueryNames(timesAverage, queryNames);\n timesFirstAverage = relabelQueryNames(timesFirstAverage, queryNames);\n requestsSum = relabelQueryNames(requestsSum, queryNames);\n results = relabelQueryNames(results, queryNames);\n timeout = relabelQueryNames(timeout, queryNames);\n\n // Write rows in output file\n if (argv.queryAverage) {\n const valueTimeFirstAverage = Object.values(timesFirstAverage).length === 0 ?\n 'N/A' :\n calcAverage(Object.values(timesFirstAverage))\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n const valueTimeFirstMedian = timesFirstAll.length === 0 ?\n 'N/A' :\n calcMedian(timesFirstAll)\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n\n // Average across queries\n serializer.writeRow([\n experimentNames[experimentId],\n `${calcAverage(Object.values(timesAverage)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcMedian(timesAll).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${valueTimeFirstAverage}`,\n `${valueTimeFirstMedian}`,\n `${calcSum(Object.values(requestsSum)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcAverage(Object.values(results)).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`,\n ...correctness ? [ `${(calcAverage(Object.values(correctness)) * 100).toFixed(2)}%` ] : [],\n `${calcSum(Object.values(timeout).map(value => value ? 1 : 0))}`,\n ], { mark: markRows.includes(experimentId) });\n } else {\n for (const [ query, time ] of Object.entries(timesAverage)) {\n serializer.writeRow([\n experimentNames[experimentId],\n query,\n `${time}`,\n `${requestsSum[query]}`,\n `${results[query]}`,\n ...correctness ? [ `${(correctness[query] * 100).toFixed(2)}%` ] : [],\n `${timeout[query]}`,\n ], { mark: markRows.includes(experimentId) });\n }\n }\n }\n\n // Close output CSV file\n serializer.close();\n }));\n}\n"]}
|
|
1
|
+
{"version":3,"file":"CommandSummary.js","sourceRoot":"","sources":["CommandSummary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,0BAoDC;AACD,0BAwKC;AAjPD,uCAAyB;AACzB,gDAAwB;AAExB,0FAA2F;AAC3F,iDAA0E;AAE1E,8CAQyB;AACzB,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,6BAA6B,CAAC;AACxC,QAAA,IAAI,GAAG,+DAA+D,CAAC;AACpF,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6FAA6F;SACxG;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,kBAAkB;SAC5B;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,mBAAmB,EAAE;YACnB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6CAA6C;SACxD;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gDAAgD;YAC1D,OAAO,EAAE,KAAK;SACf;QACD,YAAY,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,KAAK;SACf;QACD,oBAAoB,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6DAA6D;YACvE,OAAO,EAAE,EAAE;SACZ;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,2DAA2D;SACtE;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EAAC,IAAI,EAAE,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,kBAAkB,EAAE,KAAK,IAAG,EAAE;QAC/G,eAAe;QACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACtD,IAAA,qDAA2B,EAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YACxD,SAAS,CAAC;QACZ,MAAM,QAAQ,GAAa,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3E,EAAE,CAAC;QAEP,0BAA0B;QAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;QAChG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,UAAU,CAAC,WAAW,CAAC;gBACrB,EAAE;gBACF,mBAAmB;gBACnB,gBAAgB;gBAChB,qBAAqB;gBACrB,kBAAkB;gBAClB,qBAAqB;gBACrB,eAAe;gBACf,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,qBAAqB,CAAE,CAAC,CAAC,CAAC,EAAE;gBACxD,cAAc;aACf,EAAE;gBACD,KAAK,EAAE;oBACL,MAAM;oBACN,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;iBACR;aACF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,WAAW,CAAC;gBACrB,YAAY;gBACZ,OAAO;gBACP,MAAM;gBACN,UAAU;gBACV,SAAS;gBACT,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAE,aAAa,CAAE,CAAC,CAAC,CAAC,EAAE;gBAChD,SAAS;aACV,EAAE;gBACD,KAAK,EAAE;oBACL,MAAM;oBACN,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;oBACP,OAAO;iBACR;aACF,CAAC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE,CAAC;YACpF,gBAAgB;YAChB,MAAM,UAAU,GAA6B,EAAE,CAAC;YAChD,MAAM,eAAe,GAA6B,EAAE,CAAC;YACrD,MAAM,aAAa,GAA6B,EAAE,CAAC;YACnD,IAAI,OAAO,GAA2B,EAAE,CAAC;YACzC,IAAI,OAAO,GAA4B,EAAE,CAAC;YAC1C,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,EAAE,CAAC;wBAC/B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC7B,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBAEzD,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,aAAa,CAAC,EAAE,CAAC;wBAClC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBAChC,CAAC;oBACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM;yBACjC,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;oBAExF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC;oBAE3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE,CAAC;4BACpC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;wBAClC,CAAC;wBACD,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACpF,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,oBAAoB;YACpB,IAAI,YAAY,GAA2B,EAAE,CAAC;YAC9C,IAAI,QAAQ,GAAa,EAAE,CAAC;YAC5B,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1D,YAAY,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBACzC,QAAQ,GAAG,CAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAE,CAAC;YACvC,CAAC;YACD,IAAI,iBAAiB,GAA2B,EAAE,CAAC;YACnD,IAAI,aAAa,GAAa,EAAE,CAAC;YACjC,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC/D,iBAAiB,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC9C,aAAa,GAAG,CAAE,GAAG,aAAa,EAAE,GAAG,KAAK,CAAE,CAAC;YACjD,CAAC;YACD,IAAI,WAAW,GAA2B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAE,KAAK,EAAE,QAAQ,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;gBAChE,WAAW,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAW,EAAC,QAAQ,CAAC,CAAC;YAC7C,CAAC;YAED,wBAAwB;YACxB,IAAI,WAAW,CAAC;YAChB,IAAI,oBAAoB,EAAE,CAAC;gBACzB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;qBACrD,GAAG,CAAC,CAAC,CAAE,GAAG,EAAE,KAAK,CAAE,EAAE,EAAE,CAAC,CAAE,GAAG,EAAE,oBAAoB,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAE,CAAC,CAAC,CAAC;YACtG,CAAC;YAED,wBAAwB;YACxB,MAAM,UAAU,GAAG,IAAA,wBAAa,EAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;YAC7D,YAAY,GAAG,IAAA,4BAAiB,EAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAC3D,iBAAiB,GAAG,IAAA,4BAAiB,EAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;YACrE,WAAW,GAAG,IAAA,4BAAiB,EAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YACzD,OAAO,GAAG,IAAA,4BAAiB,EAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YACjD,OAAO,GAAG,IAAA,4BAAiB,EAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YAEjD,4BAA4B;YAC5B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;oBAC3E,KAAK,CAAC,CAAC;oBACP,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;yBAC1C,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrF,MAAM,oBAAoB,GAAG,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;oBACvD,KAAK,CAAC,CAAC;oBACP,IAAA,qBAAU,EAAC,aAAa,CAAC;yBACtB,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;gBAErF,yBAAyB;gBACzB,UAAU,CAAC,QAAQ,CAAC;oBAClB,eAAe,CAAC,YAAY,CAAC;oBAC7B,GAAG,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBAC7H,GAAG,IAAA,qBAAU,EAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACzG,GAAG,qBAAqB,EAAE;oBAC1B,GAAG,oBAAoB,EAAE;oBACzB,GAAG,IAAA,kBAAO,EAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACxH,GAAG,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,EAAE;oBACxH,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,IAAA,sBAAW,EAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;oBAC1F,GAAG,IAAA,kBAAO,EAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;iBACjE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,CAAE,KAAK,EAAE,IAAI,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;oBAC3D,UAAU,CAAC,QAAQ,CAAC;wBAClB,eAAe,CAAC,YAAY,CAAC;wBAC7B,KAAK;wBACL,GAAG,IAAI,EAAE;wBACT,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE;wBACvB,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;wBACnB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,EAAE;wBACrE,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;qBACpB,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC,CAAC;AACN,CAAC","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { constructCorrectnessChecker } from '../../../correctness/CorrectnessCheckerUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n calcAverage,\n calcMedian,\n calcSum,\n getExperimentNames,\n getQueryNames,\n handleCsvFile,\n relabelQueryNames,\n} from '../tex/TexUtils';\nimport { TableSerializerCsv } from './TableSerializerCsv';\nimport { TableSerializerMarkdown } from './TableSerializerMarkdown';\n\nexport const command = 'summary <experiment-dir...>';\nexport const desc = 'Summarize query results from the given experiments in a table';\nexport function builder(yargs: Argv<any>): Argv<any> {\n return yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'data_summary.csv',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n overrideQueryLabels: {\n type: 'string',\n describe: 'Comma-separated list of query labels to use',\n },\n markdown: {\n type: 'boolean',\n describe: 'If the output should be serialized as markdown',\n default: false,\n },\n queryAverage: {\n type: 'boolean',\n describe: 'If queries will be averaged over',\n default: false,\n },\n correctnessReference: {\n type: 'string',\n describe: 'Path to a JSON file mapping queries to expected cardinality',\n default: '',\n },\n markRows: {\n type: 'string',\n describe: 'Comma-separated list of row id\\'s to mark (markdown-only)',\n },\n });\n}\nexport function handler(argv: Record<string, any>): Promise<void> {\n return wrapCommandHandler(argv, async(context: ITaskContext) => wrapVisualProgress('Summarizing data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const correctnessReference = argv.correctnessReference ?\n constructCorrectnessChecker(argv.correctnessReference) :\n undefined;\n const markRows: number[] = argv.markRows ?\n argv.markRows.split(',').map((value: string) => Number.parseInt(value, 10)) :\n [];\n\n // Prepare output CSV file\n const os = fs.createWriteStream(Path.join(context.cwd, `${argv.name}`));\n const serializer = argv.markdown ? new TableSerializerMarkdown(os) : new TableSerializerCsv(os);\n if (argv.queryAverage) {\n serializer.writeHeader([\n '',\n '$$\\\\overline{t}$$',\n '$$\\\\tilde{t}$$',\n '$$\\\\overline{t}_1$$',\n '$$\\\\tilde{t}_1$$',\n '$$\\\\overline{req}$$',\n '$$\\\\sum ans$$',\n ...correctnessReference ? [ '$$\\\\overline{acc}$$' ] : [],\n '$$\\\\sum to$$',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n } else {\n serializer.writeHeader([\n 'Experiment',\n 'Query',\n 'Time',\n 'Requests',\n 'Results',\n ...correctnessReference ? [ 'Correctness' ] : [],\n 'Timeout',\n ], {\n align: [\n 'left',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n 'right',\n ],\n });\n }\n\n // Collect data from all experiments\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n // Read CSV file\n const timesTotal: Record<string, number[]> = {};\n const timesFirstTotal: Record<string, number[]> = {};\n const requestsTotal: Record<string, number[]> = {};\n let results: Record<string, number> = {};\n let timeout: Record<string, boolean> = {};\n await handleCsvFile(experimentDirectory, argv, (data) => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (!(data.name in timesTotal)) {\n timesTotal[data.name] = [];\n }\n timesTotal[data.name].push(Number.parseFloat(data.time));\n\n if (!(data.name in requestsTotal)) {\n requestsTotal[data.name] = [];\n }\n requestsTotal[data.name].push(Number\n .parseInt(data.error === 'false' && data.httpRequests ? data.httpRequests : '0', 10));\n\n results[data.name] = Number.parseInt(data.results, 10);\n timeout[data.name] = data.error === 'true';\n\n if (data.timestamps) {\n if (!(data.name in timesFirstTotal)) {\n timesFirstTotal[data.name] = [];\n }\n timesFirstTotal[data.name].push(Number.parseFloat(data.timestamps.split(' ')[0]));\n }\n }\n });\n\n // Calculate average\n let timesAverage: Record<string, number> = {};\n let timesAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesTotal)) {\n timesAverage[query] = calcAverage(times);\n timesAll = [ ...timesAll, ...times ];\n }\n let timesFirstAverage: Record<string, number> = {};\n let timesFirstAll: number[] = [];\n for (const [ query, times ] of Object.entries(timesFirstTotal)) {\n timesFirstAverage[query] = calcAverage(times);\n timesFirstAll = [ ...timesFirstAll, ...times ];\n }\n let requestsSum: Record<string, number> = {};\n for (const [ query, requests ] of Object.entries(requestsTotal)) {\n requestsSum[query] = calcAverage(requests);\n }\n\n // Calculate correctness\n let correctness;\n if (correctnessReference) {\n correctness = Object.fromEntries(Object.entries(results)\n .map(([ key, value ]) => [ key, correctnessReference.getCorrectness(experimentId, key, value) ]));\n }\n\n // Determine query names\n const queryNames = getQueryNames(Object.keys(results), argv);\n timesAverage = relabelQueryNames(timesAverage, queryNames);\n timesFirstAverage = relabelQueryNames(timesFirstAverage, queryNames);\n requestsSum = relabelQueryNames(requestsSum, queryNames);\n results = relabelQueryNames(results, queryNames);\n timeout = relabelQueryNames(timeout, queryNames);\n\n // Write rows in output file\n if (argv.queryAverage) {\n const valueTimeFirstAverage = Object.values(timesFirstAverage).length === 0 ?\n 'N/A' :\n calcAverage(Object.values(timesFirstAverage))\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n const valueTimeFirstMedian = timesFirstAll.length === 0 ?\n 'N/A' :\n calcMedian(timesFirstAll)\n .toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 });\n\n // Average across queries\n serializer.writeRow([\n experimentNames[experimentId],\n `${calcAverage(Object.values(timesAverage)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcMedian(timesAll).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${valueTimeFirstAverage}`,\n `${valueTimeFirstMedian}`,\n `${calcSum(Object.values(requestsSum)).toLocaleString('en-US', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}`,\n `${calcAverage(Object.values(results)).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`,\n ...correctness ? [ `${(calcAverage(Object.values(correctness)) * 100).toFixed(2)}%` ] : [],\n `${calcSum(Object.values(timeout).map(value => value ? 1 : 0))}`,\n ], { mark: markRows.includes(experimentId) });\n } else {\n for (const [ query, time ] of Object.entries(timesAverage)) {\n serializer.writeRow([\n experimentNames[experimentId],\n query,\n `${time}`,\n `${requestsSum[query]}`,\n `${results[query]}`,\n ...correctness ? [ `${(correctness[query] * 100).toFixed(2)}%` ] : [],\n `${timeout[query]}`,\n ], { mark: markRows.includes(experimentId) });\n }\n }\n }\n\n // Close output CSV file\n serializer.close();\n }));\n}\n"]}
|
|
@@ -116,7 +116,7 @@ function handler(argv) {
|
|
|
116
116
|
if (experimentId === 0) {
|
|
117
117
|
queries.push(data.name);
|
|
118
118
|
}
|
|
119
|
-
times[experimentId][data.name] = Number.
|
|
119
|
+
times[experimentId][data.name] = Number.parseFloat(data.time);
|
|
120
120
|
results[data.name] = Number.parseInt(data.results, 10);
|
|
121
121
|
}
|
|
122
122
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CommandWins.js","sourceRoot":"","sources":["CommandWins.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiBA,0BA2CC;AACD,0BA8EC;AA3ID,uCAAyB;AACzB,gDAAwB;AAExB,0FAA2F;AAC3F,iDAA0E;AAE1E,8CAKyB;AACzB,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,0BAA0B,CAAC;AACrC,QAAA,IAAI,GAAG,uEAAuE,CAAC;AAC5F,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6FAA6F;SACxG;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,eAAe;SACzB;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gDAAgD;YAC1D,OAAO,EAAE,KAAK;SACf;QACD,oBAAoB,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6DAA6D;YACvE,OAAO,EAAE,EAAE;SACZ;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,2DAA2D;SACtE;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EAAC,IAAI,EAAE,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,qBAAqB,EAAE,KAAK,IAAG,EAAE;QAClH,eAAe;QACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACtD,IAAA,qDAA2B,EAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YACxD,SAAS,CAAC;QACZ,0BAA0B;QAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;QAEhG,kBAAkB;QAClB,MAAM,KAAK,GAA2C,EAAE,CAAC;QACzD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE,CAAC;YACpF,MAAM,OAAO,GAA2B,EAAE,CAAC;YAC3C,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjC,KAAK,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;wBACvB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC1B,CAAC;oBAED,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"CommandWins.js","sourceRoot":"","sources":["CommandWins.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiBA,0BA2CC;AACD,0BA8EC;AA3ID,uCAAyB;AACzB,gDAAwB;AAExB,0FAA2F;AAC3F,iDAA0E;AAE1E,8CAKyB;AACzB,6DAA0D;AAC1D,uEAAoE;AAEvD,QAAA,OAAO,GAAG,0BAA0B,CAAC;AACrC,QAAA,IAAI,GAAG,uEAAuE,CAAC;AAC5F,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6FAA6F;SACxG;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,eAAe;SACzB;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gDAAgD;YAC1D,OAAO,EAAE,KAAK;SACf;QACD,oBAAoB,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6DAA6D;YACvE,OAAO,EAAE,EAAE;SACZ;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,2DAA2D;SACtE;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EAAC,IAAI,EAAE,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,qBAAqB,EAAE,KAAK,IAAG,EAAE;QAClH,eAAe;QACf,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACtD,IAAA,qDAA2B,EAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YACxD,SAAS,CAAC;QACZ,0BAA0B;QAC1B,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,iDAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,uCAAkB,CAAC,EAAE,CAAC,CAAC;QAEhG,kBAAkB;QAClB,MAAM,KAAK,GAA2C,EAAE,CAAC;QACzD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,KAAK,MAAM,CAAE,YAAY,EAAE,mBAAmB,CAAE,IAAI,qBAAqB,CAAC,OAAO,EAAE,EAAE,CAAC;YACpF,MAAM,OAAO,GAA2B,EAAE,CAAC;YAC3C,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjC,KAAK,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;wBACvB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC1B,CAAC;oBAED,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC9D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACzD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,wBAAwB;YACxB,IAAI,oBAAoB,EAAE,CAAC;gBACzB,KAAK,MAAM,CAAE,KAAK,EAAE,WAAW,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7D,IAAI,oBAAoB,CAAC,cAAc,CAAC,YAAY,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;wBAChF,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;oBACxD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,iBAAiB;QACjB,MAAM,cAAc,GAA2B,EAAE,CAAC;QAClD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC;YACxB,IAAI,UAAU,GAAG,MAAM,CAAC,iBAAiB,CAAC;YAC1C,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;gBACzC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,UAAU,EAAE,CAAC;oBAC5C,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC;oBACxC,cAAc,GAAG,YAAY,CAAC;gBAChC,CAAC;YACH,CAAC;YAED,IAAI,CAAC,CAAC,cAAc,IAAI,cAAc,CAAC,EAAE,CAAC;gBACxC,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YACrC,CAAC;YACD,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC;QACnC,CAAC;QAED,iBAAiB;QACjB,UAAU,CAAC,WAAW,CAAC;YACrB,EAAE;YACF,GAAG,eAAe;SACnB,EAAE;YACD,KAAK,EAAE;gBACL,MAAM;gBACN,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,EAAE,EAAE,GAAY,EAAE,CAAC,OAAO,CAAC;aAC1E;SACF,CAAC,CAAC;QACH,UAAU,CAAC,QAAQ,CAAC;YAClB,MAAM;YACN,GAAG,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAC;QAEH,wBAAwB;QACxB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC,CAAC;AACN,CAAC","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\nimport type { Argv } from 'yargs';\nimport { constructCorrectnessChecker } from '../../../correctness/CorrectnessCheckerUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n\n getExperimentNames,\n\n handleCsvFile,\n} from '../tex/TexUtils';\nimport { TableSerializerCsv } from './TableSerializerCsv';\nimport { TableSerializerMarkdown } from './TableSerializerMarkdown';\n\nexport const command = 'wins <experiment-dir...>';\nexport const desc = 'Show distribution of query wins from the given experiments in a table';\nexport function builder(yargs: Argv<any>): Argv<any> {\n return yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'data_wins.csv',\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n markdown: {\n type: 'boolean',\n describe: 'If the output should be serialized as markdown',\n default: false,\n },\n correctnessReference: {\n type: 'string',\n describe: 'Path to a JSON file mapping queries to expected cardinality',\n default: '',\n },\n markRows: {\n type: 'string',\n describe: 'Comma-separated list of row id\\'s to mark (markdown-only)',\n },\n });\n}\nexport function handler(argv: Record<string, any>): Promise<void> {\n return wrapCommandHandler(argv, async(context: ITaskContext) => wrapVisualProgress('Collecting win data', async() => {\n // Load options\n const { experimentDirectories, experimentNames } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const correctnessReference = argv.correctnessReference ?\n constructCorrectnessChecker(argv.correctnessReference) :\n undefined;\n // Prepare output CSV file\n const os = fs.createWriteStream(Path.join(context.cwd, `${argv.name}`));\n const serializer = argv.markdown ? new TableSerializerMarkdown(os) : new TableSerializerCsv(os);\n\n // Collect timings\n const times: Record<string, Record<string, number>> = {};\n const queries: string[] = [];\n const experimentIds: number[] = [];\n for (const [ experimentId, experimentDirectory ] of experimentDirectories.entries()) {\n const results: Record<string, number> = {};\n experimentIds.push(experimentId);\n times[experimentId] = {};\n\n await handleCsvFile(experimentDirectory, argv, (data) => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (experimentId === 0) {\n queries.push(data.name);\n }\n\n times[experimentId][data.name] = Number.parseFloat(data.time);\n results[data.name] = Number.parseInt(data.results, 10);\n }\n });\n\n // Calculate correctness\n if (correctnessReference) {\n for (const [ query, cardinality ] of Object.entries(results)) {\n if (correctnessReference.getCorrectness(experimentId, query, cardinality) !== 1) {\n times[experimentId][query] = Number.POSITIVE_INFINITY;\n }\n }\n }\n }\n\n // Calculate wins\n const experimentWins: Record<number, number> = {};\n for (const query of queries) {\n let bestExperiment = -1;\n let lowestTime = Number.POSITIVE_INFINITY;\n for (const experimentId of experimentIds) {\n if (times[experimentId][query] < lowestTime) {\n lowestTime = times[experimentId][query];\n bestExperiment = experimentId;\n }\n }\n\n if (!(bestExperiment in experimentWins)) {\n experimentWins[bestExperiment] = 0;\n }\n experimentWins[bestExperiment]++;\n }\n\n // Serialize wins\n serializer.writeHeader([\n '',\n ...experimentNames,\n ], {\n align: [\n 'left',\n ...Array.from({ length: experimentNames.length }, (): 'right' => 'right'),\n ],\n });\n serializer.writeRow([\n 'Wins',\n ...experimentIds.map(experimentId => `${experimentWins[experimentId] || 0}`),\n ]);\n\n // Close output CSV file\n serializer.close();\n }));\n}\n"]}
|
|
@@ -155,14 +155,18 @@ function handler(argv) {
|
|
|
155
155
|
if (!(data.name in totals)) {
|
|
156
156
|
totals[data.name] = [];
|
|
157
157
|
}
|
|
158
|
-
|
|
158
|
+
// Failed queries have no execution time, so they are left out instead of counting as zero.
|
|
159
|
+
if (data.error === 'true') {
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
let value = Number.parseFloat(data[metric] || '0');
|
|
159
163
|
value = Math.max(argv.zeroReplacement, value);
|
|
160
164
|
totals[data.name].push(value);
|
|
161
165
|
if (data.timestamps) {
|
|
162
166
|
if (!(data.name in totalsFirst)) {
|
|
163
167
|
totalsFirst[data.name] = [];
|
|
164
168
|
}
|
|
165
|
-
totalsFirst[data.name].push(Number.
|
|
169
|
+
totalsFirst[data.name].push(Number.parseFloat(data.timestamps.split(' ')[0]));
|
|
166
170
|
}
|
|
167
171
|
}
|
|
168
172
|
});
|
|
@@ -171,6 +175,13 @@ function handler(argv) {
|
|
|
171
175
|
const averageMinus = {};
|
|
172
176
|
const averagePlus = {};
|
|
173
177
|
for (const [query, times] of Object.entries(totals)) {
|
|
178
|
+
// Without successful executions, there is nothing to plot for this query.
|
|
179
|
+
if (times.length === 0) {
|
|
180
|
+
averages[query] = Number.NaN;
|
|
181
|
+
averageMinus[query] = Number.NaN;
|
|
182
|
+
averagePlus[query] = Number.NaN;
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
174
185
|
const average = times.reduce((sum, current) => sum + current) / times.length;
|
|
175
186
|
averages[query] = average;
|
|
176
187
|
averageMinus[query] = average - times.reduce((minV, value) => Math.min(value, minV));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CommandQuery.js","sourceRoot":"","sources":["CommandQuery.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,0BAwFC;AACD,0BAkLC;AA/RD,uCAAyB;AACzB,gDAAwB;AAGxB,0DAA6D;AAC7D,iDAA0E;AAG1E,yCAQoB;AAEP,QAAA,OAAO,GAAG,2BAA2B,CAAC;AACtC,QAAA,IAAI,GAAG,2DAA2D,CAAC;AAChF,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6FAA6F;SACxG;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,mBAAmB;SAC7B;QACD,KAAK,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yCAAyC;SACpD;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,gDAAgD;YAC1D,OAAO,EAAE,CAAC;SACX;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,4DAA4D;SACvE;QACD,MAAM,EAAE;YACN,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gCAAgC;YAC1C,OAAO,EAAE,IAAI;SACd;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6CAA6C;YACvD,OAAO,EAAE,SAAS;SACnB;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,qCAAqC;YAC/C,OAAO,EAAE,KAAK;SACf;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,mBAAmB,EAAE;YACnB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6CAA6C;SACxD;QACD,eAAe,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,gEAAgE;YAC1E,OAAO,EAAE,CAAC;SACX;QACD,GAAG,EAAE;YACH,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,oEAAoE;YAC9E,OAAO,EAAE,KAAK;SACf;QACD,MAAM,EAAE;YACN,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,oBAAoB;YAC9B,OAAO,EAAE,CAAE,MAAM,EAAE,cAAc,CAAE;YACnC,OAAO,EAAE,MAAM;SAChB;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gCAAgC;YAC1C,OAAO,EAAE,IAAI;SACd;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gGAAgG;YAC1G,OAAO,EAAE,KAAK;SACf;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EAAC,IAAI,EAAE,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,eAAe,EAAE,KAAK,IAAG,EAAE;QAC5G,gBAAgB;QAChB,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC3F,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,MAAM,WAAW,GAAG,IAAA,yBAAc,EAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,uDAAuD;QACvD,IAAI,UAAgC,CAAC;QACrC,IAAI,gBAAgB,GAA6B,EAAE,CAAC;QACpD,MAAM,cAAc,GAA2B,EAAE,CAAC;QAClD,KAAK,MAAM,mBAAmB,IAAI,qBAAqB,EAAE,CAAC;YACxD,gBAAgB;YAChB,MAAM,MAAM,GAA6B,EAAE,CAAC;YAC5C,MAAM,WAAW,GAA6B,EAAE,CAAC;YACjD,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACzB,CAAC;oBAED,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;oBACrD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;oBAE9C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAE9B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE,CAAC;4BAChC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC9B,CAAC;wBACD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;oBAClF,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,oBAAoB;YACpB,MAAM,QAAQ,GAA2B,EAAE,CAAC;YAC5C,MAAM,YAAY,GAA2B,EAAE,CAAC;YAChD,MAAM,WAAW,GAA2B,EAAE,CAAC;YAC/C,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;gBAC7E,QAAQ,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;gBAC1B,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBACrF,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBACtE,WAAW,CAAC,KAAK,CAAC,GAAG,QAAQ,GAAG,OAAO,CAAC;gBAExC,IAAI,CAAC,CAAC,KAAK,IAAI,cAAc,CAAC,EAAE,CAAC;oBAC/B,cAAc,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACN,cAAc,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;YACD,MAAM,aAAa,GAA2B,EAAE,CAAC;YACjD,MAAM,iBAAiB,GAA2B,EAAE,CAAC;YACrD,MAAM,gBAAgB,GAA2B,EAAE,CAAC;YACpD,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3D,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;gBAC7E,aAAa,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;gBAC/B,iBAAiB,CAAC,KAAK,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1F,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBACtE,gBAAgB,CAAC,KAAK,CAAC,GAAG,QAAQ,GAAG,OAAO,CAAC;YAC/C,CAAC;YAED,4BAA4B;YAC5B,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACnC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;oBACnC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;gBACnC,CAAC;YACH,CAAC;iBAAM,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACzE,CAAC;YAED,sCAAsC;YACtC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;gBACnC,gBAAgB,CAAC,SAAS,CAAC,CAAC,IAAI,CAC9B,QAAQ,CAAC,SAAS,CAAC,EACnB,YAAY,CAAC,SAAS,CAAC,EACvB,WAAW,CAAC,SAAS,CAAC,CACvB,CAAC;gBACF,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;oBACtB,gBAAgB,CAAC,SAAS,CAAC,CAAC,IAAI,CAC9B,aAAa,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,GAAG,EACtC,iBAAiB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,GAAG,EAC1C,gBAAgB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,GAAG,CAC1C,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,iCAAiC;YACjC,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACtD,gBAAgB,CAAC,SAAS,CAAC,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;YAC5G,CAAC;QACH,CAAC;QACD,yBAAyB;QACzB,UAAU,GAAG,IAAA,wBAAa,EAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC7C,gBAAgB,GAAG,IAAA,4BAAiB,EAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;QAEnE,wBAAwB;QACxB,MAAM,eAAe,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC;QACzF,eAAe,CAAC,KAAK,CAAC,SAAS,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACtD,IAAI,KAAK,GAAG,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC;YAChD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,GAAG,GAAG,KAAK,IAAI,EAAE,eAAe,EAAE,gBAAgB,EAAE,aAAa,CAAC;YACzE,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClB,KAAK,MAAM,CAAE,GAAG,EAAE,OAAO,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAChE,eAAe,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;QACD,eAAe,CAAC,KAAK,EAAE,CAAC;QAExB,2CAA2C;QAC3C,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,kCAAkC,IAAI,CAAC,cAAc,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAEnH,oBAAoB;QACpB,IAAI,QAAQ,GAAG,gBAAgB,GAAG,eAAe;aAC9C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE;YAChB,MAAM,MAAM,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChE,MAAM,SAAS,GAAG,MAAM,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,kCAAkC,EAAE,SAAS,SAAS,oCAAoC,EAAE,UAAU,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACzJ,OAAO,2BAA2B,MAAM,sCAAsC,CAAC,MAAM,0CAA0C,EAAE,SAAS,SAAS,IAAI,QAAQ,yBAAyB,IAAI,CAAC,IAAI,SAAS,CAAC;QAC7M,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,MAAM,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,aAAa,GAAG,eAAe;iBAClC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE;gBAChB,MAAM,MAAM,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;gBAChE,MAAM,SAAS,GAAG,MAAM,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvE,OAAO,gCAAgC,MAAM,uFAAuF,EAAE,eAAe,SAAS,0BAA0B,IAAI,CAAC,IAAI,SAAS,CAAC;YAC7M,CAAC,CAAC;iBACD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,QAAQ,GAAG,GAAG,QAAQ,KAAK,gBAAgB,GAAG,aAAa,EAAE,CAAC;QAChE,CAAC;QAED,uBAAuB;QACvB,MAAM,IAAA,mCAAmB,EACvB,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,KAAK,EAAE,qBAAqB,CAAC,EAC9D,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,MAAM,CAAC,kBAExC,QAAQ,EAAE,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAC1C,KAAK,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,EAC3D,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAC7B,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EACzE,IAAI,EAAE,QAAQ,EACd,YAAY,EAAE,WAAW,EACzB,UAAU,EAAE,IAAI,CAAC,SAAS,IACvB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAErD,CAAC,QAAQ,EAAE,EAAE;YACX,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,QAAQ,GAAG,QAAQ;qBAChB,OAAO,CAAC,UAAU,EAAE,4DAA4D,CAAC;qBACjF,OAAO,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;YAC9C,CAAC;YACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,YAAY,CAAC,CAAC;YACtE,CAAC;YACD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;gBAC9B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,yBAAyB,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CACF,CAAC;QAEF,sBAAsB;QACtB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,MAAM,IAAA,gBAAK,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC,CAAC,CAAC,CAAC;AACN,CAAC","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\n\nimport type { Argv } from 'yargs';\nimport { instantiateTemplate } from '../../../TemplateUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\n\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n\n getColorScheme,\n getExperimentNames,\n getQueryNames,\n handleCsvFile,\n relabelQueryNames,\n toSvg,\n} from './TexUtils';\n\nexport const command = 'query <experiment-dir...>';\nexport const desc = 'Plot the query execution times from the given experiments';\nexport function builder(yargs: Argv<any>): Argv<any> {\n return yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'plot_queries_data',\n },\n color: {\n type: 'string',\n alias: 'c',\n describe: 'Color scheme name from colorbrewer2.org',\n },\n shiftColorList: {\n type: 'number',\n describe: 'How many color list elements should be skipped',\n default: 0,\n },\n maxY: {\n type: 'number',\n describe: 'The upper limit of the Y-axis. Defaults to maximum Y value',\n },\n legend: {\n type: 'boolean',\n describe: 'If a legend should be included',\n default: true,\n },\n legendPos: {\n type: 'string',\n describe: 'The legend position X,Y (anchor north-east)',\n default: '1.0,1.0',\n },\n logY: {\n type: 'boolean',\n describe: 'If the Y-Axis must have a log scale',\n default: false,\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n overrideQueryLabels: {\n type: 'string',\n describe: 'Comma-separated list of query labels to use',\n },\n zeroReplacement: {\n type: 'number',\n describe: 'If zero values occur, what values they should be replaced with',\n default: 0,\n },\n svg: {\n type: 'boolean',\n describe: 'If the tex file should be converted to svg via the tex2svg command',\n default: false,\n },\n metric: {\n type: 'string',\n describe: 'The metric to plot',\n choices: [ 'time', 'httpRequests' ],\n default: 'time',\n },\n outliers: {\n type: 'boolean',\n describe: 'If outliers should be included',\n default: true,\n },\n relative: {\n type: 'boolean',\n describe: 'If the maximum value per query should be set to 1, and all other values made relative to that.',\n default: false,\n },\n });\n}\nexport function handler(argv: Record<string, any>): Promise<void> {\n return wrapCommandHandler(argv, async(context: ITaskContext) => wrapVisualProgress('Plotting data', async() => {\n // Load CLI args\n const { experimentDirectories, experimentNames, experimentIds } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const colorScheme = getColorScheme(argv, experimentDirectories);\n const metric = argv.metric;\n\n // Prepare query CSV file with averages per query group\n let queryNames: string[] | undefined;\n let outputCsvEntries: Record<string, number[]> = {};\n const maxQueryValues: Record<string, number> = {};\n for (const experimentDirectory of experimentDirectories) {\n // Read CSV file\n const totals: Record<string, number[]> = {};\n const totalsFirst: Record<string, number[]> = {};\n await handleCsvFile(experimentDirectory, argv, (data) => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (!(data.name in totals)) {\n totals[data.name] = [];\n }\n\n let value = Number.parseInt(data[metric] || '0', 10);\n value = Math.max(argv.zeroReplacement, value);\n\n totals[data.name].push(value);\n\n if (data.timestamps) {\n if (!(data.name in totalsFirst)) {\n totalsFirst[data.name] = [];\n }\n totalsFirst[data.name].push(Number.parseInt(data.timestamps.split(' ')[0], 10));\n }\n }\n });\n\n // Calculate average\n const averages: Record<string, number> = {};\n const averageMinus: Record<string, number> = {};\n const averagePlus: Record<string, number> = {};\n for (const [ query, times ] of Object.entries(totals)) {\n const average = times.reduce((sum, current) => sum + current) / times.length;\n averages[query] = average;\n averageMinus[query] = average - times.reduce((minV, value) => Math.min(value, minV));\n const maxValue = times.reduce((maxV, value) => Math.max(value, maxV));\n averagePlus[query] = maxValue - average;\n\n if (!(query in maxQueryValues)) {\n maxQueryValues[query] = maxValue;\n } else {\n maxQueryValues[query] = Math.max(maxQueryValues[query], maxValue);\n }\n }\n const averagesFirst: Record<string, number> = {};\n const averageFirstMinus: Record<string, number> = {};\n const averageFirstPlus: Record<string, number> = {};\n for (const [ query, times ] of Object.entries(totalsFirst)) {\n const average = times.reduce((sum, current) => sum + current) / times.length;\n averagesFirst[query] = average;\n averageFirstMinus[query] = average - times.reduce((minV, value) => Math.min(value, minV));\n const maxValue = times.reduce((maxV, value) => Math.max(value, maxV));\n averageFirstPlus[query] = maxValue - average;\n }\n\n // Set query names and count\n if (!queryNames) {\n queryNames = Object.keys(averages);\n for (const queryName of queryNames) {\n outputCsvEntries[queryName] = [];\n }\n } else if (queryNames.join(',') !== Object.keys(averages).join(',')) {\n throw new Error(`Tried to plot experiments with different query sets`);\n }\n\n // Append average, min, and max result\n for (const queryName of queryNames) {\n outputCsvEntries[queryName].push(\n averages[queryName],\n averageMinus[queryName],\n averagePlus[queryName],\n );\n if (metric === 'time') {\n outputCsvEntries[queryName].push(\n averagesFirst[queryName] || Number.NaN,\n averageFirstMinus[queryName] || Number.NaN,\n averageFirstPlus[queryName] || Number.NaN,\n );\n }\n }\n }\n if (!queryNames) {\n throw new Error(`No queries could be found`);\n }\n if (argv.relative) {\n // Make values relative per query\n for (const queryName of Object.keys(outputCsvEntries)) {\n outputCsvEntries[queryName] = outputCsvEntries[queryName].map(value => value / maxQueryValues[queryName]);\n }\n }\n // Determine query labels\n queryNames = getQueryNames(queryNames, argv);\n outputCsvEntries = relabelQueryNames(outputCsvEntries, queryNames);\n\n // Write output CSV file\n const csvOutputStream = fs.createWriteStream(Path.join(context.cwd, `${argv.name}.csv`));\n csvOutputStream.write(`query;${experimentIds.map((id) => {\n let value = `${id}-mean;${id}-minus;${id}-plus`;\n if (metric === 'time') {\n value = `${value};${id}-first-mean;${id}-first-minus;${id}-first-plus`;\n }\n return value;\n }).join(';')}\\n`);\n for (const [ key, columns ] of Object.entries(outputCsvEntries)) {\n csvOutputStream.write(`${key};${columns.join(';')}\\n`);\n }\n csvOutputStream.close();\n\n // Shift color list by a number of elements\n const prependPlotLines = argv.shiftColorList > 0 ? `\\\\pgfplotsset{cycle list shift=${argv.shiftColorList}}\\n` : '';\n\n // Prepare bar lines\n let barLines = prependPlotLines + experimentNames\n .map((name, id) => {\n const offset = (id - ((experimentNames.length - 1) / 2)) * 2.75;\n const yModifier = metric === 'time' && !argv.relative ? ' / 1000' : '';\n const outliers = argv.outliers ? `, y error plus expr=(\\\\thisrow{${id}-plus}${yModifier}), y error minus expr=(\\\\thisrow{${id}-minus}${yModifier})` : '';\n return `\\\\addplot+[ybar, xshift=${offset}pt,legend image post style={xshift=${-offset}pt}] table [x=query, y expr=(\\\\thisrow{${id}-mean}${yModifier})${outliers}, col sep=semicolon]{\"${argv.name}.csv\"};`;\n })\n .join('\\n');\n if (metric === 'time' && argv.outliers) {\n const extraBarLines = experimentNames\n .map((name, id) => {\n const offset = (id - ((experimentNames.length - 1) / 2)) * 2.75;\n const yModifier = metric === 'time' && !argv.relative ? ' / 1000' : '';\n return `\\\\addplot+[only marks,xshift=${offset}pt,mark=star,mark options={color=gray,scale=0.5}] table [x=query, y expr=(\\\\thisrow{${id}-first-mean}${yModifier}), col sep=semicolon]{\"${argv.name}.csv\"};`;\n })\n .join('\\n');\n barLines = `${barLines}\\n${prependPlotLines}${extraBarLines}`;\n }\n\n // Instantiate template\n await instantiateTemplate(\n Path.join(context.templatesRoot, 'tex', 'plot_query_data.tex'),\n Path.join(context.cwd, `${argv.name}.tex`),\n {\n X_LIMITS: experimentDirectories.length * 2,\n WIDTH: queryNames.length * (experimentNames.length + 1) * 4,\n QUERIES: queryNames.join(','),\n LEGEND: experimentNames.map(name => name.replace(/_/gu, '\\\\_')).join(','),\n BARS: barLines,\n COLOR_SCHEME: colorScheme,\n LEGEND_POS: argv.legendPos,\n ...argv.maxY ? { Y_MAX: `ymax=${argv.maxY},` } : {},\n },\n (contents) => {\n if (!argv.legend) {\n contents = contents.replace(/\\\\legend\\{.*\\}/ug, '');\n }\n if (argv.logY) {\n contents = contents\n .replace(/ymin=0,/u, 'ymin=0.000001,ymode=log,log origin=infty,log basis y={10},')\n .replace(/ \\/ 1000\\)/ug, ' / 1000)+1e-5');\n }\n if (argv.relative) {\n contents = contents.replace('ylabel={Duration (s)},', 'ylabel={},');\n }\n if (metric === 'httpRequests') {\n contents = contents.replace('ylabel={Duration (s)},', 'ylabel={HTTP Requests},');\n }\n return contents;\n },\n );\n\n // Render CSV from TeX\n if (argv.svg) {\n await toSvg(argv, context);\n }\n }));\n}\n"]}
|
|
1
|
+
{"version":3,"file":"CommandQuery.js","sourceRoot":"","sources":["CommandQuery.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,0BAwFC;AACD,0BA6LC;AA1SD,uCAAyB;AACzB,gDAAwB;AAGxB,0DAA6D;AAC7D,iDAA0E;AAG1E,yCAQoB;AAEP,QAAA,OAAO,GAAG,2BAA2B,CAAC;AACtC,QAAA,IAAI,GAAG,2DAA2D,CAAC;AAChF,SAAgB,OAAO,CAAC,KAAgB;IACtC,OAAO,KAAK;SACT,OAAO,CAAC;QACP,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,6FAA6F;SACxG;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yBAAyB;YACnC,OAAO,EAAE,mBAAmB;SAC7B;QACD,KAAK,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,yCAAyC;SACpD;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,gDAAgD;YAC1D,OAAO,EAAE,CAAC;SACX;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,4DAA4D;SACvE;QACD,MAAM,EAAE;YACN,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gCAAgC;YAC1C,OAAO,EAAE,IAAI;SACd;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6CAA6C;YACvD,OAAO,EAAE,SAAS;SACnB;QACD,IAAI,EAAE;YACJ,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,qCAAqC;YAC/C,OAAO,EAAE,KAAK;SACf;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,uCAAuC;YACjD,OAAO,EAAE,iBAAiB;SAC3B;QACD,cAAc,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,kCAAkC;YAC5C,OAAO,EAAE,GAAG;SACb;QACD,yBAAyB,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,mDAAmD;SAC9D;QACD,mBAAmB,EAAE;YACnB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,6CAA6C;SACxD;QACD,eAAe,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,gEAAgE;YAC1E,OAAO,EAAE,CAAC;SACX;QACD,GAAG,EAAE;YACH,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,oEAAoE;YAC9E,OAAO,EAAE,KAAK;SACf;QACD,MAAM,EAAE;YACN,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,oBAAoB;YAC9B,OAAO,EAAE,CAAE,MAAM,EAAE,cAAc,CAAE;YACnC,OAAO,EAAE,MAAM;SAChB;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gCAAgC;YAC1C,OAAO,EAAE,IAAI;SACd;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,gGAAgG;YAC1G,OAAO,EAAE,KAAK;SACf;KACF,CAAC,CAAC;AACP,CAAC;AACD,SAAgB,OAAO,CAAC,IAAyB;IAC/C,OAAO,IAAA,+BAAkB,EAAC,IAAI,EAAE,KAAK,EAAC,OAAqB,EAAE,EAAE,CAAC,IAAA,+BAAkB,EAAC,eAAe,EAAE,KAAK,IAAG,EAAE;QAC5G,gBAAgB;QAChB,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC;QAC3F,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,MAAM,WAAW,GAAG,IAAA,yBAAc,EAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,uDAAuD;QACvD,IAAI,UAAgC,CAAC;QACrC,IAAI,gBAAgB,GAA6B,EAAE,CAAC;QACpD,MAAM,cAAc,GAA2B,EAAE,CAAC;QAClD,KAAK,MAAM,mBAAmB,IAAI,qBAAqB,EAAE,CAAC;YACxD,gBAAgB;YAChB,MAAM,MAAM,GAA6B,EAAE,CAAC;YAC5C,MAAM,WAAW,GAA6B,EAAE,CAAC;YACjD,MAAM,IAAA,wBAAa,EAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACtD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;wBAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACzB,CAAC;oBACD,2FAA2F;oBAC3F,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;wBAC1B,OAAO;oBACT,CAAC;oBAED,IAAI,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;oBACnD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;oBAE9C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAE9B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE,CAAC;4BAChC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC9B,CAAC;wBACD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAChF,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,oBAAoB;YACpB,MAAM,QAAQ,GAA2B,EAAE,CAAC;YAC5C,MAAM,YAAY,GAA2B,EAAE,CAAC;YAChD,MAAM,WAAW,GAA2B,EAAE,CAAC;YAC/C,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtD,0EAA0E;gBAC1E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACvB,QAAQ,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;oBAC7B,YAAY,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;oBACjC,WAAW,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;oBAChC,SAAS;gBACX,CAAC;gBACD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;gBAC7E,QAAQ,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;gBAC1B,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBACrF,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBACtE,WAAW,CAAC,KAAK,CAAC,GAAG,QAAQ,GAAG,OAAO,CAAC;gBAExC,IAAI,CAAC,CAAC,KAAK,IAAI,cAAc,CAAC,EAAE,CAAC;oBAC/B,cAAc,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACN,cAAc,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;YACD,MAAM,aAAa,GAA2B,EAAE,CAAC;YACjD,MAAM,iBAAiB,GAA2B,EAAE,CAAC;YACrD,MAAM,gBAAgB,GAA2B,EAAE,CAAC;YACpD,KAAK,MAAM,CAAE,KAAK,EAAE,KAAK,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3D,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;gBAC7E,aAAa,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;gBAC/B,iBAAiB,CAAC,KAAK,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1F,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBACtE,gBAAgB,CAAC,KAAK,CAAC,GAAG,QAAQ,GAAG,OAAO,CAAC;YAC/C,CAAC;YAED,4BAA4B;YAC5B,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACnC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;oBACnC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;gBACnC,CAAC;YACH,CAAC;iBAAM,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACzE,CAAC;YAED,sCAAsC;YACtC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;gBACnC,gBAAgB,CAAC,SAAS,CAAC,CAAC,IAAI,CAC9B,QAAQ,CAAC,SAAS,CAAC,EACnB,YAAY,CAAC,SAAS,CAAC,EACvB,WAAW,CAAC,SAAS,CAAC,CACvB,CAAC;gBACF,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;oBACtB,gBAAgB,CAAC,SAAS,CAAC,CAAC,IAAI,CAC9B,aAAa,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,GAAG,EACtC,iBAAiB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,GAAG,EAC1C,gBAAgB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,GAAG,CAC1C,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,iCAAiC;YACjC,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACtD,gBAAgB,CAAC,SAAS,CAAC,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;YAC5G,CAAC;QACH,CAAC;QACD,yBAAyB;QACzB,UAAU,GAAG,IAAA,wBAAa,EAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC7C,gBAAgB,GAAG,IAAA,4BAAiB,EAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;QAEnE,wBAAwB;QACxB,MAAM,eAAe,GAAG,EAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC;QACzF,eAAe,CAAC,KAAK,CAAC,SAAS,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACtD,IAAI,KAAK,GAAG,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC;YAChD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,GAAG,GAAG,KAAK,IAAI,EAAE,eAAe,EAAE,gBAAgB,EAAE,aAAa,CAAC;YACzE,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClB,KAAK,MAAM,CAAE,GAAG,EAAE,OAAO,CAAE,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAChE,eAAe,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;QACD,eAAe,CAAC,KAAK,EAAE,CAAC;QAExB,2CAA2C;QAC3C,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,kCAAkC,IAAI,CAAC,cAAc,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAEnH,oBAAoB;QACpB,IAAI,QAAQ,GAAG,gBAAgB,GAAG,eAAe;aAC9C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE;YAChB,MAAM,MAAM,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChE,MAAM,SAAS,GAAG,MAAM,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,kCAAkC,EAAE,SAAS,SAAS,oCAAoC,EAAE,UAAU,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACzJ,OAAO,2BAA2B,MAAM,sCAAsC,CAAC,MAAM,0CAA0C,EAAE,SAAS,SAAS,IAAI,QAAQ,yBAAyB,IAAI,CAAC,IAAI,SAAS,CAAC;QAC7M,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,MAAM,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,aAAa,GAAG,eAAe;iBAClC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE;gBAChB,MAAM,MAAM,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;gBAChE,MAAM,SAAS,GAAG,MAAM,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvE,OAAO,gCAAgC,MAAM,uFAAuF,EAAE,eAAe,SAAS,0BAA0B,IAAI,CAAC,IAAI,SAAS,CAAC;YAC7M,CAAC,CAAC;iBACD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,QAAQ,GAAG,GAAG,QAAQ,KAAK,gBAAgB,GAAG,aAAa,EAAE,CAAC;QAChE,CAAC;QAED,uBAAuB;QACvB,MAAM,IAAA,mCAAmB,EACvB,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,KAAK,EAAE,qBAAqB,CAAC,EAC9D,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,MAAM,CAAC,kBAExC,QAAQ,EAAE,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAC1C,KAAK,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,EAC3D,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAC7B,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EACzE,IAAI,EAAE,QAAQ,EACd,YAAY,EAAE,WAAW,EACzB,UAAU,EAAE,IAAI,CAAC,SAAS,IACvB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAErD,CAAC,QAAQ,EAAE,EAAE;YACX,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,QAAQ,GAAG,QAAQ;qBAChB,OAAO,CAAC,UAAU,EAAE,4DAA4D,CAAC;qBACjF,OAAO,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;YAC9C,CAAC;YACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,YAAY,CAAC,CAAC;YACtE,CAAC;YACD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;gBAC9B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,yBAAyB,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CACF,CAAC;QAEF,sBAAsB;QACtB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,MAAM,IAAA,gBAAK,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC,CAAC,CAAC,CAAC;AACN,CAAC","sourcesContent":["import * as fs from 'fs';\nimport Path from 'path';\n\nimport type { Argv } from 'yargs';\nimport { instantiateTemplate } from '../../../TemplateUtils';\nimport { wrapCommandHandler, wrapVisualProgress } from '../../CliHelpers';\n\nimport type { ITaskContext } from '../../ITaskContext';\nimport {\n\n getColorScheme,\n getExperimentNames,\n getQueryNames,\n handleCsvFile,\n relabelQueryNames,\n toSvg,\n} from './TexUtils';\n\nexport const command = 'query <experiment-dir...>';\nexport const desc = 'Plot the query execution times from the given experiments';\nexport function builder(yargs: Argv<any>): Argv<any> {\n return yargs\n .options({\n queryRegex: {\n type: 'string',\n alias: 'q',\n describe: 'Regex for queries to include (before any label overrides). Examples: \\'^C\\', \\'^[^C]\\', ...',\n },\n name: {\n type: 'string',\n alias: 'n',\n describe: 'Custom output file name',\n default: 'plot_queries_data',\n },\n color: {\n type: 'string',\n alias: 'c',\n describe: 'Color scheme name from colorbrewer2.org',\n },\n shiftColorList: {\n type: 'number',\n describe: 'How many color list elements should be skipped',\n default: 0,\n },\n maxY: {\n type: 'number',\n describe: 'The upper limit of the Y-axis. Defaults to maximum Y value',\n },\n legend: {\n type: 'boolean',\n describe: 'If a legend should be included',\n default: true,\n },\n legendPos: {\n type: 'string',\n describe: 'The legend position X,Y (anchor north-east)',\n default: '1.0,1.0',\n },\n logY: {\n type: 'boolean',\n describe: 'If the Y-Axis must have a log scale',\n default: false,\n },\n inputName: {\n type: 'string',\n describe: 'Custom input file name per experiment',\n default: 'query-times.csv',\n },\n inputDelimiter: {\n type: 'string',\n describe: 'Delimiter for the input CSV file',\n default: ';',\n },\n overrideCombinationLabels: {\n type: 'string',\n describe: 'Comma-separated list of combination labels to use',\n },\n overrideQueryLabels: {\n type: 'string',\n describe: 'Comma-separated list of query labels to use',\n },\n zeroReplacement: {\n type: 'number',\n describe: 'If zero values occur, what values they should be replaced with',\n default: 0,\n },\n svg: {\n type: 'boolean',\n describe: 'If the tex file should be converted to svg via the tex2svg command',\n default: false,\n },\n metric: {\n type: 'string',\n describe: 'The metric to plot',\n choices: [ 'time', 'httpRequests' ],\n default: 'time',\n },\n outliers: {\n type: 'boolean',\n describe: 'If outliers should be included',\n default: true,\n },\n relative: {\n type: 'boolean',\n describe: 'If the maximum value per query should be set to 1, and all other values made relative to that.',\n default: false,\n },\n });\n}\nexport function handler(argv: Record<string, any>): Promise<void> {\n return wrapCommandHandler(argv, async(context: ITaskContext) => wrapVisualProgress('Plotting data', async() => {\n // Load CLI args\n const { experimentDirectories, experimentNames, experimentIds } = getExperimentNames(argv);\n const queryRegex = argv.queryRegex ? new RegExp(argv.queryRegex, 'u') : undefined;\n const colorScheme = getColorScheme(argv, experimentDirectories);\n const metric = argv.metric;\n\n // Prepare query CSV file with averages per query group\n let queryNames: string[] | undefined;\n let outputCsvEntries: Record<string, number[]> = {};\n const maxQueryValues: Record<string, number> = {};\n for (const experimentDirectory of experimentDirectories) {\n // Read CSV file\n const totals: Record<string, number[]> = {};\n const totalsFirst: Record<string, number[]> = {};\n await handleCsvFile(experimentDirectory, argv, (data) => {\n if (!queryRegex || queryRegex.test(data.name)) {\n if (!(data.name in totals)) {\n totals[data.name] = [];\n }\n // Failed queries have no execution time, so they are left out instead of counting as zero.\n if (data.error === 'true') {\n return;\n }\n\n let value = Number.parseFloat(data[metric] || '0');\n value = Math.max(argv.zeroReplacement, value);\n\n totals[data.name].push(value);\n\n if (data.timestamps) {\n if (!(data.name in totalsFirst)) {\n totalsFirst[data.name] = [];\n }\n totalsFirst[data.name].push(Number.parseFloat(data.timestamps.split(' ')[0]));\n }\n }\n });\n\n // Calculate average\n const averages: Record<string, number> = {};\n const averageMinus: Record<string, number> = {};\n const averagePlus: Record<string, number> = {};\n for (const [ query, times ] of Object.entries(totals)) {\n // Without successful executions, there is nothing to plot for this query.\n if (times.length === 0) {\n averages[query] = Number.NaN;\n averageMinus[query] = Number.NaN;\n averagePlus[query] = Number.NaN;\n continue;\n }\n const average = times.reduce((sum, current) => sum + current) / times.length;\n averages[query] = average;\n averageMinus[query] = average - times.reduce((minV, value) => Math.min(value, minV));\n const maxValue = times.reduce((maxV, value) => Math.max(value, maxV));\n averagePlus[query] = maxValue - average;\n\n if (!(query in maxQueryValues)) {\n maxQueryValues[query] = maxValue;\n } else {\n maxQueryValues[query] = Math.max(maxQueryValues[query], maxValue);\n }\n }\n const averagesFirst: Record<string, number> = {};\n const averageFirstMinus: Record<string, number> = {};\n const averageFirstPlus: Record<string, number> = {};\n for (const [ query, times ] of Object.entries(totalsFirst)) {\n const average = times.reduce((sum, current) => sum + current) / times.length;\n averagesFirst[query] = average;\n averageFirstMinus[query] = average - times.reduce((minV, value) => Math.min(value, minV));\n const maxValue = times.reduce((maxV, value) => Math.max(value, maxV));\n averageFirstPlus[query] = maxValue - average;\n }\n\n // Set query names and count\n if (!queryNames) {\n queryNames = Object.keys(averages);\n for (const queryName of queryNames) {\n outputCsvEntries[queryName] = [];\n }\n } else if (queryNames.join(',') !== Object.keys(averages).join(',')) {\n throw new Error(`Tried to plot experiments with different query sets`);\n }\n\n // Append average, min, and max result\n for (const queryName of queryNames) {\n outputCsvEntries[queryName].push(\n averages[queryName],\n averageMinus[queryName],\n averagePlus[queryName],\n );\n if (metric === 'time') {\n outputCsvEntries[queryName].push(\n averagesFirst[queryName] || Number.NaN,\n averageFirstMinus[queryName] || Number.NaN,\n averageFirstPlus[queryName] || Number.NaN,\n );\n }\n }\n }\n if (!queryNames) {\n throw new Error(`No queries could be found`);\n }\n if (argv.relative) {\n // Make values relative per query\n for (const queryName of Object.keys(outputCsvEntries)) {\n outputCsvEntries[queryName] = outputCsvEntries[queryName].map(value => value / maxQueryValues[queryName]);\n }\n }\n // Determine query labels\n queryNames = getQueryNames(queryNames, argv);\n outputCsvEntries = relabelQueryNames(outputCsvEntries, queryNames);\n\n // Write output CSV file\n const csvOutputStream = fs.createWriteStream(Path.join(context.cwd, `${argv.name}.csv`));\n csvOutputStream.write(`query;${experimentIds.map((id) => {\n let value = `${id}-mean;${id}-minus;${id}-plus`;\n if (metric === 'time') {\n value = `${value};${id}-first-mean;${id}-first-minus;${id}-first-plus`;\n }\n return value;\n }).join(';')}\\n`);\n for (const [ key, columns ] of Object.entries(outputCsvEntries)) {\n csvOutputStream.write(`${key};${columns.join(';')}\\n`);\n }\n csvOutputStream.close();\n\n // Shift color list by a number of elements\n const prependPlotLines = argv.shiftColorList > 0 ? `\\\\pgfplotsset{cycle list shift=${argv.shiftColorList}}\\n` : '';\n\n // Prepare bar lines\n let barLines = prependPlotLines + experimentNames\n .map((name, id) => {\n const offset = (id - ((experimentNames.length - 1) / 2)) * 2.75;\n const yModifier = metric === 'time' && !argv.relative ? ' / 1000' : '';\n const outliers = argv.outliers ? `, y error plus expr=(\\\\thisrow{${id}-plus}${yModifier}), y error minus expr=(\\\\thisrow{${id}-minus}${yModifier})` : '';\n return `\\\\addplot+[ybar, xshift=${offset}pt,legend image post style={xshift=${-offset}pt}] table [x=query, y expr=(\\\\thisrow{${id}-mean}${yModifier})${outliers}, col sep=semicolon]{\"${argv.name}.csv\"};`;\n })\n .join('\\n');\n if (metric === 'time' && argv.outliers) {\n const extraBarLines = experimentNames\n .map((name, id) => {\n const offset = (id - ((experimentNames.length - 1) / 2)) * 2.75;\n const yModifier = metric === 'time' && !argv.relative ? ' / 1000' : '';\n return `\\\\addplot+[only marks,xshift=${offset}pt,mark=star,mark options={color=gray,scale=0.5}] table [x=query, y expr=(\\\\thisrow{${id}-first-mean}${yModifier}), col sep=semicolon]{\"${argv.name}.csv\"};`;\n })\n .join('\\n');\n barLines = `${barLines}\\n${prependPlotLines}${extraBarLines}`;\n }\n\n // Instantiate template\n await instantiateTemplate(\n Path.join(context.templatesRoot, 'tex', 'plot_query_data.tex'),\n Path.join(context.cwd, `${argv.name}.tex`),\n {\n X_LIMITS: experimentDirectories.length * 2,\n WIDTH: queryNames.length * (experimentNames.length + 1) * 4,\n QUERIES: queryNames.join(','),\n LEGEND: experimentNames.map(name => name.replace(/_/gu, '\\\\_')).join(','),\n BARS: barLines,\n COLOR_SCHEME: colorScheme,\n LEGEND_POS: argv.legendPos,\n ...argv.maxY ? { Y_MAX: `ymax=${argv.maxY},` } : {},\n },\n (contents) => {\n if (!argv.legend) {\n contents = contents.replace(/\\\\legend\\{.*\\}/ug, '');\n }\n if (argv.logY) {\n contents = contents\n .replace(/ymin=0,/u, 'ymin=0.000001,ymode=log,log origin=infty,log basis y={10},')\n .replace(/ \\/ 1000\\)/ug, ' / 1000)+1e-5');\n }\n if (argv.relative) {\n contents = contents.replace('ylabel={Duration (s)},', 'ylabel={},');\n }\n if (metric === 'httpRequests') {\n contents = contents.replace('ylabel={Duration (s)},', 'ylabel={HTTP Requests},');\n }\n return contents;\n },\n );\n\n // Render CSV from TeX\n if (argv.svg) {\n await toSvg(argv, context);\n }\n }));\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rubensworks/process-sparql-benchmark-results",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.2",
|
|
4
4
|
"description": "Personal tools to post-process query results from SPARQL benchmarks",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://github.com/rubensworks/process-sparql-benchmark-results.js#readme",
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"test": "jest ${1}",
|
|
38
38
|
"coveralls": "jest --coverage && cat ./coverage/lcov.info | coveralls",
|
|
39
39
|
"lint": "eslint . --cache",
|
|
40
|
-
"prepare": "npm run build",
|
|
40
|
+
"prepare": "husky && npm run build",
|
|
41
41
|
"version": "manual-git-changelog onversion"
|
|
42
42
|
},
|
|
43
43
|
"jest": {
|
|
@@ -79,7 +79,7 @@
|
|
|
79
79
|
"devDependencies": {
|
|
80
80
|
"@rubensworks/eslint-config": "^3",
|
|
81
81
|
"@types/jest": "^30.0.0",
|
|
82
|
-
"@types/node": "^
|
|
82
|
+
"@types/node": "^26.0.0",
|
|
83
83
|
"coveralls": "^3.0.0",
|
|
84
84
|
"eslint": "^8.57.1",
|
|
85
85
|
"husky": "^9.0.0",
|
|
@@ -89,10 +89,5 @@
|
|
|
89
89
|
"streamify-array": "^2.0.0",
|
|
90
90
|
"ts-jest": "^29.0.0",
|
|
91
91
|
"typescript": "^5.0.0"
|
|
92
|
-
},
|
|
93
|
-
"husky": {
|
|
94
|
-
"hooks": {
|
|
95
|
-
"pre-commit": "npm run build && npm run lint && npm run test"
|
|
96
|
-
}
|
|
97
92
|
}
|
|
98
93
|
}
|