@eagleoutice/flowr 2.0.1 → 2.0.3

package/benchmark/summarizer/first-phase/input.js

@@ -3,27 +3,24 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.processSummarizedFileMeasurement = exports.processRunMeasurement = void 0;
+exports.processSummarizedRunMeasurement = exports.processRunMeasurement = void 0;
 const fs_1 = __importDefault(require("fs"));
 const process_1 = require("../second-phase/process");
 const process_2 = require("./process");
 const assert_1 = require("../../../util/assert");
 const ansi_1 = require("../../../util/ansi");
 const json_1 = require("../../../util/json");
-const files_1 = require("../../../util/files");
 const print_1 = require("../../stats/print");
-async function processRunMeasurement(line, fileNum, lineNum, summarizedText, outputPath) {
+async function processRunMeasurement(line, fileNum, lineNum, textOutputAppendPath, rawOutputPath) {
     let got = JSON.parse(line.toString());
     console.log(`[file ${fileNum}, line ${lineNum}] Summarize for ${got.filename}`);
     // now we have to recover the maps and bigints :C
     got = {
-        filename: got.filename,
-        'file-id': got['file-id'],
-        'run-num': got['run-num'],
+        ...got,
         stats: {
-            input: got.stats.input,
-            request: got.stats.request,
-            dataflow: got.stats.dataflow,
+            ...got.stats,
+            memory: new Map(got.stats.memory
+                .map(([k, v]) => [k, v])),
             commonMeasurements: new Map(got.stats.commonMeasurements
                 .map(([k, v]) => {
                 (0, assert_1.guard)(v.endsWith('n'), 'Expected a bigint');
@@ -38,28 +35,31 @@ async function processRunMeasurement(line, fileNum, lineNum, summarizedText, out
     let atSliceNumber = 0;
     const summarized = await (0, process_2.summarizeSlicerStats)(got.stats, (criterion, stats) => {
         console.log(`${ansi_1.escape}1F${ansi_1.escape}1G${ansi_1.escape}2K    [${++atSliceNumber}/${totalSlices}] Summarizing ${JSON.stringify(criterion)} (reconstructed has ${stats.reconstructedCode.code.length} characters)`);
+        if (stats.reconstructedCode.code.length < 50) {
+            console.log(`Reconstructed code: ${stats.reconstructedCode.code}`);
+        }
     });
-    console.log(`    - Append raw summary to ${outputPath}`);
-    fs_1.default.appendFileSync(outputPath, `${JSON.stringify({
+    console.log(`    - Write raw summary to ${rawOutputPath}`);
+    fs_1.default.writeFileSync(rawOutputPath, `${JSON.stringify({
         filename: got.filename,
         'file-id': got['file-id'],
         'run-num': got['run-num'],
         summarize: summarized
     }, json_1.jsonReplacer)}\n`);
-    console.log(`    - Append textual summary to ${summarizedText}`);
-    fs_1.default.appendFileSync(summarizedText, `${(0, print_1.stats2string)(summarized)}\n`);
+    console.log(`    - Append textual summary to ${textOutputAppendPath}`);
+    fs_1.default.appendFileSync(textOutputAppendPath, `${(0, print_1.stats2string)(summarized)}\n`);
 }
 exports.processRunMeasurement = processRunMeasurement;
-function processSummarizedFileMeasurement(file, summariesFile, outputPath) {
-    console.log(`Summarize all runs for ${file}`);
+function processSummarizedRunMeasurement(runNum, summarizedFiles, appendPath) {
+    console.log(`Summarizing all file statistics for run ${runNum}`);
     const summaries = [];
-    (0, files_1.readLineByLineSync)(summariesFile, l => (0, process_1.processNextSummary)(l, summaries));
-    fs_1.default.appendFileSync(outputPath, `${JSON.stringify({
-        filename: file,
-        summarize: (0, process_1.summarizeAllSummarizedStats)(summaries)
-    }, json_1.jsonReplacer)}\n`);
+    for (const file of summarizedFiles) {
+        (0, process_1.processNextSummary)(fs_1.default.readFileSync(file), summaries);
+    }
+    fs_1.default.appendFileSync(appendPath, `${JSON.stringify((0, process_1.summarizeAllSummarizedStats)(summaries), json_1.jsonReplacer)}\n`);
+    console.log(`Appended summary of run ${runNum} to ${appendPath}`);
 }
-exports.processSummarizedFileMeasurement = processSummarizedFileMeasurement;
+exports.processSummarizedRunMeasurement = processSummarizedRunMeasurement;
 function mapPerSliceStats(k, v) {
     return [k, {
             reconstructedCode: v.reconstructedCode,

package/benchmark/summarizer/first-phase/process.d.ts

@@ -1,4 +1,4 @@
-import type { Reduction, SummarizedSlicerStats } from '../data';
+import type { Reduction, SummarizedSlicerStats, TimePerToken } from '../data';
 import type { SummarizedMeasurement } from '../../../util/summarizer';
 import type { PerSliceStats, SlicerStats } from '../../stats/stats';
 import type { SlicingCriteria } from '../../../slicing/criterion/parse';
@@ -8,4 +8,6 @@ import type { SlicingCriteria } from '../../../slicing/criterion/parse';
  */
 export declare function summarizeSlicerStats(stats: SlicerStats, report?: (criteria: SlicingCriteria, stats: PerSliceStats) => void): Promise<Readonly<SummarizedSlicerStats>>;
 export declare function summarizeSummarizedMeasurement(data: SummarizedMeasurement[]): SummarizedMeasurement;
-export declare function summarizeReductions(reductions: Reduction<SummarizedMeasurement>[]): Reduction<SummarizedMeasurement>;
+export declare function summarizeSummarizedReductions(reductions: Reduction<SummarizedMeasurement>[]): Reduction<SummarizedMeasurement>;
+export declare function summarizeSummarizedTimePerToken(times: TimePerToken[]): TimePerToken;
+export declare function summarizeTimePerToken(times: TimePerToken<number>[]): TimePerToken;

package/benchmark/summarizer/first-phase/process.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.summarizeReductions = exports.summarizeSummarizedMeasurement = exports.summarizeSlicerStats = void 0;
+exports.summarizeTimePerToken = exports.summarizeSummarizedTimePerToken = exports.summarizeSummarizedReductions = exports.summarizeSummarizedMeasurement = exports.summarizeSlicerStats = void 0;
 const tmp = __importStar(require("tmp"));
 const fs_1 = __importDefault(require("fs"));
 const defaultmap_1 = require("../../../util/defaultmap");
@@ -65,14 +65,24 @@ function safeDivPercentage(a, b) {
         }
     }
 }
-function calculateReductionForSlice(input, dataflow, perSlice) {
+function calculateReductionForSlice(input, dataflow, perSlice, ignoreFluff) {
+    const perSliceLines = ignoreFluff ? perSlice.nonEmptyLines : perSlice.lines;
+    const inputLines = ignoreFluff ? input.numberOfNonEmptyLines : input.numberOfLines;
     return {
-        numberOfLines: safeDivPercentage(perSlice.lines, input.numberOfLines),
-        numberOfLinesNoAutoSelection: safeDivPercentage(perSlice.lines - perSlice.autoSelected, input.numberOfLines),
-        numberOfCharacters: safeDivPercentage(perSlice.characters, input.numberOfCharacters),
-        numberOfNonWhitespaceCharacters: safeDivPercentage(perSlice.nonWhitespaceCharacters, input.numberOfNonWhitespaceCharacters),
-        numberOfRTokens: safeDivPercentage(perSlice.tokens, input.numberOfRTokens),
-        numberOfNormalizedTokens: safeDivPercentage(perSlice.normalizedTokens, input.numberOfNormalizedTokens),
+        numberOfLines: safeDivPercentage(perSliceLines, inputLines),
+        numberOfLinesNoAutoSelection: safeDivPercentage(perSliceLines - perSlice.linesWithAutoSelected, inputLines),
+        numberOfCharacters: ignoreFluff ?
+            safeDivPercentage(perSlice.charactersNoComments, input.numberOfCharactersNoComments) :
+            safeDivPercentage(perSlice.characters, input.numberOfCharacters),
+        numberOfNonWhitespaceCharacters: ignoreFluff ?
+            safeDivPercentage(perSlice.nonWhitespaceCharactersNoComments, input.numberOfNonWhitespaceCharactersNoComments) :
+            safeDivPercentage(perSlice.nonWhitespaceCharacters, input.numberOfNonWhitespaceCharacters),
+        numberOfRTokens: ignoreFluff ?
+            safeDivPercentage(perSlice.tokensNoComments, input.numberOfRTokensNoComments) :
+            safeDivPercentage(perSlice.tokens, input.numberOfRTokens),
+        numberOfNormalizedTokens: ignoreFluff ?
+            safeDivPercentage(perSlice.normalizedTokensNoComments, input.numberOfNormalizedTokensNoComments) :
+            safeDivPercentage(perSlice.normalizedTokens, input.numberOfNormalizedTokens),
         numberOfDataflowNodes: safeDivPercentage(perSlice.dataflowNodes, dataflow.numberOfNodes)
     };
 }
@@ -82,33 +92,44 @@ function calculateReductionForSlice(input, dataflow, perSlice) {
  */
 async function summarizeSlicerStats(stats, report = () => {
 }) {
-    const perSliceStats = stats.perSliceMeasurements;
     const collect = new defaultmap_1.DefaultMap(() => []);
     const sizeOfSliceCriteria = [];
     const reParseShellSession = new shell_1.RShell();
+    const sliceTimes = [];
+    const reconstructTimes = [];
+    const totalTimes = [];
     const reductions = [];
+    const reductionsNoFluff = [];
     let failedOutputs = 0;
     const sliceSize = {
         lines: [],
-        autoSelected: [],
+        nonEmptyLines: [],
+        linesWithAutoSelected: [],
         characters: [],
+        charactersNoComments: [],
         nonWhitespaceCharacters: [],
+        nonWhitespaceCharactersNoComments: [],
         tokens: [],
+        tokensNoComments: [],
         normalizedTokens: [],
+        normalizedTokensNoComments: [],
         dataflowNodes: []
     };
     let timesHitThreshold = 0;
-    for (const [criteria, perSliceStat] of perSliceStats) {
+    for (const [criteria, perSliceStat] of stats.perSliceMeasurements) {
         report(criteria, perSliceStat);
         for (const measure of perSliceStat.measurements) {
             collect.get(measure[0]).push(Number(measure[1]));
         }
         sizeOfSliceCriteria.push(perSliceStat.slicingCriteria.length);
         timesHitThreshold += perSliceStat.timesHitThreshold > 0 ? 1 : 0;
-        const { code: output, autoSelected } = perSliceStat.reconstructedCode;
-        sliceSize.autoSelected.push(autoSelected);
-        const lines = output.split('\n').length;
+        const { code: output, linesWithAutoSelected } = perSliceStat.reconstructedCode;
+        sliceSize.linesWithAutoSelected.push(linesWithAutoSelected);
+        const split = output.split('\n');
+        const lines = split.length;
+        const nonEmptyLines = split.filter(l => l.trim().length > 0).length;
         sliceSize.lines.push(lines);
+        sliceSize.nonEmptyLines.push(nonEmptyLines);
         sliceSize.characters.push(output.length);
         const nonWhitespace = (0, strings_1.withoutWhitespace)(output).length;
         sliceSize.nonWhitespaceCharacters.push(nonWhitespace);
@@ -118,22 +139,60 @@ async function summarizeSlicerStats(stats, report = () => {
             fs_1.default.writeFileSync(tempfile().name, output);
             const reParsed = await (0, retriever_1.retrieveNormalizedAstFromRCode)({ request: 'file', content: tempfile().name }, reParseShellSession);
             let numberOfNormalizedTokens = 0;
-            (0, visitor_1.visitAst)(reParsed.ast, _ => {
+            let numberOfNormalizedTokensNoComments = 0;
+            let commentChars = 0;
+            let commentCharsNoWhitespace = 0;
+            (0, visitor_1.visitAst)(reParsed.ast, t => {
                 numberOfNormalizedTokens++;
+                const comments = t.info.additionalTokens?.filter(t => t.type === "RComment" /* RType.Comment */);
+                if (comments && comments.length > 0) {
+                    const content = comments.map(c => c.lexeme ?? '').join('');
+                    commentChars += content.length;
+                    commentCharsNoWhitespace += (0, strings_1.withoutWhitespace)(content).length;
+                }
+                else {
+                    numberOfNormalizedTokensNoComments++;
+                }
                 return false;
             });
             sliceSize.normalizedTokens.push(numberOfNormalizedTokens);
+            sliceSize.normalizedTokensNoComments.push(numberOfNormalizedTokensNoComments);
+            sliceSize.charactersNoComments.push(output.length - commentChars);
+            sliceSize.nonWhitespaceCharactersNoComments.push(nonWhitespace - commentCharsNoWhitespace);
             const numberOfRTokens = await (0, retriever_1.retrieveNumberOfRTokensOfLastParse)(reParseShellSession);
             sliceSize.tokens.push(numberOfRTokens);
-            reductions.push(calculateReductionForSlice(stats.input, stats.dataflow, {
+            const numberOfRTokensNoComments = await (0, retriever_1.retrieveNumberOfRTokensOfLastParse)(reParseShellSession, true);
+            sliceSize.tokensNoComments.push(numberOfRTokensNoComments);
+            const perSlice = {
                 lines: lines,
+                nonEmptyLines: nonEmptyLines,
                 characters: output.length,
+                charactersNoComments: output.length - commentChars,
                 nonWhitespaceCharacters: nonWhitespace,
-                autoSelected: autoSelected,
+                nonWhitespaceCharactersNoComments: nonWhitespace - commentCharsNoWhitespace,
+                linesWithAutoSelected: linesWithAutoSelected,
                 tokens: numberOfRTokens,
+                tokensNoComments: numberOfRTokensNoComments,
                 normalizedTokens: numberOfNormalizedTokens,
+                normalizedTokensNoComments: numberOfNormalizedTokensNoComments,
                 dataflowNodes: perSliceStat.numberOfDataflowNodesSliced
-            }));
+            };
+            reductions.push(calculateReductionForSlice(stats.input, stats.dataflow, perSlice, false));
+            reductionsNoFluff.push(calculateReductionForSlice(stats.input, stats.dataflow, perSlice, true));
+            const sliceTime = Number(perSliceStat.measurements.get('static slicing'));
+            const reconstructTime = Number(perSliceStat.measurements.get('reconstruct code'));
+            sliceTimes.push({
+                raw: sliceTime / numberOfRTokens,
+                normalized: sliceTime / numberOfNormalizedTokens
+            });
+            reconstructTimes.push({
+                raw: reconstructTime / numberOfRTokens,
+                normalized: reconstructTime / numberOfNormalizedTokens
+            });
+            totalTimes.push({
+                raw: (sliceTime + reconstructTime) / numberOfRTokens,
+                normalized: (sliceTime + reconstructTime) / numberOfNormalizedTokens
+            });
         }
         catch (e) {
             console.error(`    ! Failed to re-parse the output of the slicer for ${JSON.stringify(criteria)}`); //, e
@@ -151,27 +210,28 @@ async function summarizeSlicerStats(stats, report = () => {
     return {
         ...stats,
         perSliceMeasurements: {
-            numberOfSlices: perSliceStats.size,
+            numberOfSlices: stats.perSliceMeasurements.size,
             sliceCriteriaSizes: (0, summarizer_1.summarizeMeasurement)(sizeOfSliceCriteria),
             measurements: summarized,
             failedToRepParse: failedOutputs,
             timesHitThreshold,
-            reduction: {
-                numberOfLines: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLines).filter(assert_1.isNotUndefined)),
-                numberOfLinesNoAutoSelection: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLinesNoAutoSelection).filter(assert_1.isNotUndefined)),
-                numberOfCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfCharacters).filter(assert_1.isNotUndefined)),
-                numberOfNonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNonWhitespaceCharacters).filter(assert_1.isNotUndefined)),
-                numberOfRTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfRTokens).filter(assert_1.isNotUndefined)),
-                numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNormalizedTokens).filter(assert_1.isNotUndefined)),
-                numberOfDataflowNodes: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfDataflowNodes).filter(assert_1.isNotUndefined))
-            },
+            reduction: summarizeReductions(reductions),
+            reductionNoFluff: summarizeReductions(reductionsNoFluff),
+            sliceTimePerToken: summarizeTimePerToken(sliceTimes),
+            reconstructTimePerToken: summarizeTimePerToken(reconstructTimes),
+            totalPerSliceTimePerToken: summarizeTimePerToken(totalTimes),
             sliceSize: {
                 lines: (0, summarizer_1.summarizeMeasurement)(sliceSize.lines),
+                nonEmptyLines: (0, summarizer_1.summarizeMeasurement)(sliceSize.nonEmptyLines),
                 characters: (0, summarizer_1.summarizeMeasurement)(sliceSize.characters),
+                charactersNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.charactersNoComments),
                 nonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(sliceSize.nonWhitespaceCharacters),
-                autoSelected: (0, summarizer_1.summarizeMeasurement)(sliceSize.autoSelected),
+                nonWhitespaceCharactersNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.nonWhitespaceCharactersNoComments),
+                linesWithAutoSelected: (0, summarizer_1.summarizeMeasurement)(sliceSize.linesWithAutoSelected),
                 tokens: (0, summarizer_1.summarizeMeasurement)(sliceSize.tokens),
+                tokensNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.tokensNoComments),
                 normalizedTokens: (0, summarizer_1.summarizeMeasurement)(sliceSize.normalizedTokens),
+                normalizedTokensNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.normalizedTokensNoComments),
                 dataflowNodes: (0, summarizer_1.summarizeMeasurement)(sliceSize.dataflowNodes)
             }
         }
@@ -179,10 +239,12 @@ async function summarizeSlicerStats(stats, report = () => {
 }
 exports.summarizeSlicerStats = summarizeSlicerStats;
 function summarizeSummarizedMeasurement(data) {
+    data = data.filter(assert_1.isNotUndefined);
     const min = data.map(d => d.min).filter(assert_1.isNotUndefined).reduce((a, b) => Math.min(a, b), Infinity);
     const max = data.map(d => d.max).filter(assert_1.isNotUndefined).reduce((a, b) => Math.max(a, b), -Infinity);
-    // get most average
-    const median = data.map(d => d.median).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
+    // calculate median of medians (don't just average the median!)
+    const medians = data.map(d => d.median).filter(assert_1.isNotUndefined).sort((a, b) => a - b);
+    const median = medians[Math.floor(medians.length / 2)];
     const mean = data.map(d => d.mean).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
     // Method 1 of https://www.statology.org/averaging-standard-deviations/
     const std = Math.sqrt(data.map(d => d.std ** 2).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length);
@@ -190,7 +252,7 @@ function summarizeSummarizedMeasurement(data) {
     return { min, max, median, mean, std, total };
 }
 exports.summarizeSummarizedMeasurement = summarizeSummarizedMeasurement;
-function summarizeReductions(reductions) {
+function summarizeSummarizedReductions(reductions) {
     return {
         numberOfDataflowNodes: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfDataflowNodes)),
         numberOfLines: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfLines)),
@@ -201,5 +263,30 @@ function summarizeReductions(reductions) {
         numberOfRTokens: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfRTokens))
     };
 }
-exports.summarizeReductions = summarizeReductions;
+exports.summarizeSummarizedReductions = summarizeSummarizedReductions;
+function summarizeReductions(reductions) {
+    return {
+        numberOfLines: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLines).filter(assert_1.isNotUndefined)),
+        numberOfLinesNoAutoSelection: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLinesNoAutoSelection).filter(assert_1.isNotUndefined)),
+        numberOfCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfCharacters).filter(assert_1.isNotUndefined)),
+        numberOfNonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNonWhitespaceCharacters).filter(assert_1.isNotUndefined)),
+        numberOfRTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfRTokens).filter(assert_1.isNotUndefined)),
+        numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNormalizedTokens).filter(assert_1.isNotUndefined)),
+        numberOfDataflowNodes: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfDataflowNodes).filter(assert_1.isNotUndefined))
+    };
+}
+function summarizeSummarizedTimePerToken(times) {
+    return {
+        raw: summarizeSummarizedMeasurement(times.map(t => t.raw)),
+        normalized: summarizeSummarizedMeasurement(times.map(t => t.normalized)),
+    };
+}
+exports.summarizeSummarizedTimePerToken = summarizeSummarizedTimePerToken;
+function summarizeTimePerToken(times) {
+    return {
+        raw: (0, summarizer_1.summarizeMeasurement)(times.map(t => t.raw)),
+        normalized: (0, summarizer_1.summarizeMeasurement)(times.map(t => t.normalized)),
+    };
+}
+exports.summarizeTimePerToken = summarizeTimePerToken;
 //# sourceMappingURL=process.js.map

package/benchmark/summarizer/second-phase/graph.js

@@ -47,6 +47,13 @@ function writeGraphOutput(ultimate, outputGraphPath) {
         value: ultimate.reduction.numberOfNormalizedTokens.mean,
         extra: `std: ${ultimate.reduction.numberOfNormalizedTokens.std}`
     });
+    data.push({
+        name: 'memory (df-graph)',
+        unit: 'Bytes',
+        value: Number(ultimate.dataflow.sizeOfObject.mean),
+        range: Number(ultimate.dataflow.sizeOfObject.std),
+        extra: `median: ${(ultimate.dataflow.sizeOfObject.median).toFixed(2)}`
+    });
     // write the output file
     fs_1.default.writeFileSync(outputGraphPath, JSON.stringify(data, json_1.jsonReplacer));
 }

package/benchmark/summarizer/second-phase/process.js

@@ -9,7 +9,16 @@ const stats_1 = require("../../stats/stats");
 function summarizeAllSummarizedStats(stats) {
     const commonMeasurements = new defaultmap_1.DefaultMap(() => []);
     const perSliceMeasurements = new defaultmap_1.DefaultMap(() => []);
+    const sliceTimesPerToken = [];
+    const reconstructTimesPerToken = [];
+    const totalPerSliceTimesPerToken = [];
+    const retrieveTimesPerToken = [];
+    const normalizeTimesPerToken = [];
+    const dataflowTimesPerToken = [];
+    const totalCommonTimesPerToken = [];
+    const memory = new defaultmap_1.DefaultMap(() => []);
     const reductions = [];
+    const reductionsNoFluff = [];
     const inputs = [];
     const dataflows = [];
     let failedToRepParse = 0;
@@ -22,7 +31,18 @@ function summarizeAllSummarizedStats(stats) {
         for (const [k, v] of stat.perSliceMeasurements.measurements) {
             perSliceMeasurements.get(k).push(v);
         }
+        sliceTimesPerToken.push(stat.perSliceMeasurements.sliceTimePerToken);
+        reconstructTimesPerToken.push(stat.perSliceMeasurements.reconstructTimePerToken);
+        totalPerSliceTimesPerToken.push(stat.perSliceMeasurements.totalPerSliceTimePerToken);
+        retrieveTimesPerToken.push(stat.retrieveTimePerToken);
+        normalizeTimesPerToken.push(stat.normalizeTimePerToken);
+        dataflowTimesPerToken.push(stat.dataflowTimePerToken);
+        totalCommonTimesPerToken.push(stat.totalCommonTimePerToken);
+        for (const [k, v] of stat.memory) {
+            memory.get(k).push(v);
+        }
         reductions.push(stat.perSliceMeasurements.reduction);
+        reductionsNoFluff.push(stat.perSliceMeasurements.reductionNoFluff);
         inputs.push(stat.input);
         dataflows.push(stat.dataflow);
         failedToRepParse += stat.perSliceMeasurements.failedToRepParse;
@@ -34,21 +54,35 @@ function summarizeAllSummarizedStats(stats) {
         totalSlices: totalSlices,
         commonMeasurements: new Map([...commonMeasurements.entries()].map(([k, v]) => [k, (0, summarizer_1.summarizeMeasurement)(v)])),
         perSliceMeasurements: new Map([...perSliceMeasurements.entries()].map(([k, v]) => [k, (0, process_1.summarizeSummarizedMeasurement)(v)])),
+        sliceTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(sliceTimesPerToken),
+        reconstructTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(reconstructTimesPerToken),
+        totalPerSliceTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(totalPerSliceTimesPerToken),
+        retrieveTimePerToken: (0, process_1.summarizeTimePerToken)(retrieveTimesPerToken),
+        normalizeTimePerToken: (0, process_1.summarizeTimePerToken)(normalizeTimesPerToken),
+        dataflowTimePerToken: (0, process_1.summarizeTimePerToken)(dataflowTimesPerToken),
+        totalCommonTimePerToken: (0, process_1.summarizeTimePerToken)(totalCommonTimesPerToken),
         failedToRepParse,
         timesHitThreshold,
-        reduction: (0, process_1.summarizeReductions)(reductions),
+        reduction: (0, process_1.summarizeSummarizedReductions)(reductions),
+        reductionNoFluff: (0, process_1.summarizeSummarizedReductions)(reductionsNoFluff),
         input: {
             numberOfLines: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfLines)),
+            numberOfNonEmptyLines: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNonEmptyLines)),
             numberOfCharacters: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfCharacters)),
+            numberOfCharactersNoComments: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfCharactersNoComments)),
             numberOfNonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNonWhitespaceCharacters)),
+            numberOfNonWhitespaceCharactersNoComments: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNonWhitespaceCharactersNoComments)),
             numberOfRTokens: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfRTokens)),
-            numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNormalizedTokens))
+            numberOfRTokensNoComments: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfRTokensNoComments)),
+            numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNormalizedTokens)),
+            numberOfNormalizedTokensNoComments: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNormalizedTokensNoComments))
         },
         dataflow: {
             numberOfNodes: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfNodes)),
             numberOfFunctionDefinitions: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfFunctionDefinitions)),
             numberOfCalls: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfCalls)),
-            numberOfEdges: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfEdges))
+            numberOfEdges: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfEdges)),
+            sizeOfObject: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.sizeOfObject))
         }
     };
 }
@@ -63,19 +97,33 @@ function summarizeAllUltimateStats(stats) {
         // average out / summarize other measurements
         commonMeasurements: new Map(stats_1.CommonSlicerMeasurements.map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.commonMeasurements.get(m)))])),
         perSliceMeasurements: new Map(stats_1.PerSliceMeasurements.map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.perSliceMeasurements.get(m)))])),
-        reduction: (0, process_1.summarizeReductions)(stats.map(s => s.reduction)),
+        sliceTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.sliceTimePerToken)),
+        reconstructTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.reconstructTimePerToken)),
+        totalPerSliceTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.totalPerSliceTimePerToken)),
+        retrieveTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.retrieveTimePerToken)),
+        normalizeTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.normalizeTimePerToken)),
+        dataflowTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.dataflowTimePerToken)),
+        totalCommonTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.totalCommonTimePerToken)),
+        reduction: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reduction)),
+        reductionNoFluff: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reductionNoFluff)),
         input: {
             numberOfLines: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfLines)),
+            numberOfNonEmptyLines: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNonEmptyLines)),
             numberOfCharacters: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfCharacters)),
+            numberOfCharactersNoComments: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfCharactersNoComments)),
             numberOfNonWhitespaceCharacters: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNonWhitespaceCharacters)),
+            numberOfNonWhitespaceCharactersNoComments: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNonWhitespaceCharactersNoComments)),
             numberOfRTokens: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfRTokens)),
-            numberOfNormalizedTokens: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNormalizedTokens))
+            numberOfRTokensNoComments: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfRTokensNoComments)),
+            numberOfNormalizedTokens: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNormalizedTokens)),
+            numberOfNormalizedTokensNoComments: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNormalizedTokensNoComments))
         },
         dataflow: {
             numberOfNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfNodes)),
             numberOfFunctionDefinitions: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfFunctionDefinitions)),
             numberOfCalls: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfCalls)),
-            numberOfEdges: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfEdges))
+            numberOfEdges: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfEdges)),
+            sizeOfObject: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.sizeOfObject))
         }
     };
 }
@@ -84,22 +132,19 @@ function processNextSummary(line, allSummarized) {
     let got = JSON.parse(line.toString());
     got = {
         summarize: {
-            input: got.summarize.input,
-            request: got.summarize.request,
-            dataflow: got.summarize.dataflow,
+            ...got.summarize,
+            // restore maps
+            memory: new Map(got.summarize.memory
+                .map(([k, v]) => [k, v])),
             commonMeasurements: new Map(got.summarize.commonMeasurements
                 .map(([k, v]) => {
                 (0, assert_1.guard)(v.endsWith('n'), 'Expected a bigint');
                 return [k, BigInt(v.slice(0, -1))];
             })),
             perSliceMeasurements: {
-                numberOfSlices: got.summarize.perSliceMeasurements.numberOfSlices,
-                sliceCriteriaSizes: got.summarize.perSliceMeasurements.sliceCriteriaSizes,
+                ...got.summarize.perSliceMeasurements,
+                // restore maps
                 measurements: new Map(got.summarize.perSliceMeasurements.measurements),
-                reduction: got.summarize.perSliceMeasurements.reduction,
-                timesHitThreshold: got.summarize.perSliceMeasurements.timesHitThreshold,
-                failedToRepParse: got.summarize.perSliceMeasurements.failedToRepParse,
-                sliceSize: got.summarize.perSliceMeasurements.sliceSize
             }
         }
     };
@@ -109,19 +154,12 @@ exports.processNextSummary = processNextSummary;
 function processNextUltimateSummary(line, allSummarized) {
     let got = JSON.parse(line.toString());
     got = {
-        summarize: {
-            totalRequests: got.summarize.totalRequests,
-            totalSlices: got.summarize.totalSlices,
-            commonMeasurements: new Map(got.summarize.commonMeasurements),
-            perSliceMeasurements: new Map(got.summarize.perSliceMeasurements),
-            failedToRepParse: got.summarize.failedToRepParse,
-            timesHitThreshold: got.summarize.timesHitThreshold,
-            reduction: got.summarize.reduction,
-            input: got.summarize.input,
-            dataflow: got.summarize.dataflow,
-        }
+        ...got,
+        // restore maps
+        commonMeasurements: new Map(got.commonMeasurements),
+        perSliceMeasurements: new Map(got.perSliceMeasurements),
     };
-    allSummarized.push(got.summarize);
+    allSummarized.push(got);
 }
 exports.processNextUltimateSummary = processNextUltimateSummary;
 //# sourceMappingURL=process.js.map

package/benchmark/summarizer/summarizer.d.ts

@@ -28,4 +28,5 @@ export declare class BenchmarkSummarizer extends Summarizer<UltimateSlicerStats,
     preparationPhase(): Promise<void>;
     summarizePhase(): Promise<UltimateSlicerStats>;
     private removeIfExists;
+    private summaryFile;
 }

package/benchmark/summarizer/summarizer.js

@@ -13,31 +13,41 @@ const summarizer_1 = require("../../util/summarizer");
 const files_1 = require("../../util/files");
 const json_1 = require("../../util/json");
 const print_1 = require("../stats/print");
+const defaultmap_1 = require("../../util/defaultmap");
 class BenchmarkSummarizer extends summarizer_1.Summarizer {
     constructor(config) {
         super(config);
     }
     async preparationPhase() {
-        this.removeIfExists(`${this.config.intermediateOutputPath}.json`);
+        this.removeIfExists(this.summaryFile());
         this.removeIfExists(this.config.intermediateOutputPath);
         fs_1.default.mkdirSync(this.config.intermediateOutputPath);
-        const dirContent = fs_1.default.readdirSync(this.config.inputPath);
-        for (let i = 0; i < dirContent.length; i++) {
-            const filePath = path_1.default.join(this.config.inputPath, dirContent[i]);
-            const outputPath = path_1.default.join(this.config.intermediateOutputPath, dirContent[i]);
+        const filesToSummarize = fs_1.default.readdirSync(this.config.inputPath);
+        const outputPathsPerRun = new defaultmap_1.DefaultMap(() => []);
+        for (let i = 0; i < filesToSummarize.length; i++) {
+            const fileInputPath = path_1.default.join(this.config.inputPath, filesToSummarize[i]);
+            const outputDir = path_1.default.join(this.config.intermediateOutputPath, path_1.default.parse(filesToSummarize[i]).name);
+            fs_1.default.mkdirSync(outputDir);
+            const textOutputPath = path_1.default.join(outputDir, 'summary.log');
             // generate measurements for each run
-            await (0, files_1.readLineByLine)(filePath, (line, lineNumber) => (0, input_1.processRunMeasurement)(line, i, lineNumber, `${outputPath}.log`, outputPath));
-            // generate combined measurements for the file
-            (0, input_1.processSummarizedFileMeasurement)(filePath, outputPath, `${this.config.intermediateOutputPath}.json`);
+            await (0, files_1.readLineByLine)(fileInputPath, (line, lineNumber) => {
+                const runOutputPath = path_1.default.join(outputDir, `run-${lineNumber}.json`);
+                outputPathsPerRun.get(lineNumber).push(runOutputPath);
+                return (0, input_1.processRunMeasurement)(line, i, lineNumber, textOutputPath, runOutputPath);
+            });
+        }
+        // generate combined measurements for each file per run
+        for (const [run, paths] of outputPathsPerRun.entries()) {
+            (0, input_1.processSummarizedRunMeasurement)(run, paths, this.summaryFile());
         }
         this.log('Done summarizing');
     }
     // eslint-disable-next-line @typescript-eslint/require-await -- just to obey the structure
     async summarizePhase() {
-        this.log(`Summarizing all summaries from ${this.config.inputPath}...`);
+        this.log(`Summarizing all summaries from ${this.summaryFile()}...`);
         this.removeIfExists(this.config.outputPath);
         const summaries = [];
-        (0, files_1.readLineByLineSync)(`${this.config.intermediateOutputPath}.json`, (l) => (0, process_1.processNextUltimateSummary)(l, summaries));
+        (0, files_1.readLineByLineSync)(this.summaryFile(), (l) => (0, process_1.processNextUltimateSummary)(l, summaries));
         const ultimate = (0, process_1.summarizeAllUltimateStats)(summaries);
         this.log(`Writing ultimate summary to ${this.config.outputPath}`);
         fs_1.default.writeFileSync(this.config.outputPath, JSON.stringify(ultimate, json_1.jsonReplacer));
@@ -53,6 +63,9 @@ class BenchmarkSummarizer extends summarizer_1.Summarizer {
             fs_1.default.rmSync(path, { recursive: true });
         }
     }
+    summaryFile() {
+        return `${this.config.intermediateOutputPath}.json`;
+    }
 }
 exports.BenchmarkSummarizer = BenchmarkSummarizer;
 //# sourceMappingURL=summarizer.js.map

package/cli/repl/commands/commands.js

@@ -61,6 +61,15 @@ const _commands = {
     'controlflow*': cfg_1.controlflowStarCommand
 };
 let commandsInitialized = false;
+function hasModule(path) {
+    try {
+        require.resolve(path);
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+}
 function commands() {
     if (commandsInitialized) {
         return _commands;
@@ -74,7 +83,16 @@ function commands() {
                 script: true,
                 usageExample: `:${script} --help`,
                 fn: async (output, _s, remainingLine) => {
-                    await (0, execute_1.waitOnScript)(`${__dirname}/../../${target}`, (0, args_1.splitAtEscapeSensitive)(remainingLine), stdio => (0, execute_1.stdioCaptureProcessor)(stdio, msg => output.stdout(msg), msg => output.stderr(msg)));
+                    // check if the target *module* exists in the current directory, else try two dirs up, otherwise, fail with a message
+                    let path = `${__dirname}/${target}`;
+                    if (!hasModule(path)) {
+                        path = `${__dirname}/../../${target}`;
+                        if (!hasModule(path)) {
+                            output.stderr(`Could not find the target script ${target} in the current directory or two directories up.`);
+                            return;
+                        }
+                    }
+                    await (0, execute_1.waitOnScript)(path, (0, args_1.splitAtEscapeSensitive)(remainingLine), stdio => (0, execute_1.stdioCaptureProcessor)(stdio, msg => output.stdout(msg), msg => output.stderr(msg)));
                 }
             };
         }