@eagleoutice/flowr 2.0.0 → 2.0.2

package/benchmark/slicer.js

@@ -17,8 +17,8 @@ const strings_1 = require("../util/strings");
 const shell_1 = require("../r-bridge/shell");
 const default_pipelines_1 = require("../core/steps/pipeline/default-pipelines");
 const retriever_1 = require("../r-bridge/retriever");
-const collect_1 = require("../r-bridge/lang-4.x/ast/model/collect");
 const collect_all_1 = require("../slicing/criterion/collect-all");
+const visitor_1 = require("../r-bridge/lang-4.x/ast/model/processing/visitor");
 exports.benchmarkLogger = log_1.log.getSubLogger({ name: 'benchmark' });
 /**
  * A slicer that can be used to slice exactly one file (multiple times).
@@ -67,6 +67,7 @@ class BenchmarkSlicer {
         const loadedContent = request.request === 'text' ? request.content : fs_1.default.readFileSync(request.content, 'utf-8');
         // retrieve number of R tokens - flowr_parsed should still contain the last parsed code
         const numberOfRTokens = await (0, retriever_1.retrieveNumberOfRTokensOfLastParse)(this.shell);
+        const numberOfRTokensNoComments = await (0, retriever_1.retrieveNumberOfRTokensOfLastParse)(this.shell, true);
         (0, assert_1.guard)(this.normalizedAst !== undefined, 'normalizedAst should be defined after initialization');
         (0, assert_1.guard)(this.dataflow !== undefined, 'dataflow should be defined after initialization');
         // collect dataflow graph size
@@ -84,16 +85,40 @@ class BenchmarkSlicer {
                 numberOfDefinitions++;
             }
         }
+        let nodes = 0;
+        let nodesNoComments = 0;
+        let commentChars = 0;
+        let commentCharsNoWhitespace = 0;
+        (0, visitor_1.visitAst)(this.normalizedAst.ast, t => {
+            nodes++;
+            const comments = t.info.additionalTokens?.filter(t => t.type === "RComment" /* RType.Comment */);
+            if (comments && comments.length > 0) {
+                const content = comments.map(c => c.lexeme ?? '').join('');
+                commentChars += content.length;
+                commentCharsNoWhitespace += (0, strings_1.withoutWhitespace)(content).length;
+            }
+            else {
+                nodesNoComments++;
+            }
+            return false;
+        });
+        const split = loadedContent.split('\n');
+        const nonWhitespace = (0, strings_1.withoutWhitespace)(loadedContent).length;
         this.stats = {
             commonMeasurements: new Map(),
             perSliceMeasurements: this.perSliceMeasurements,
             request,
             input: {
-                numberOfLines: loadedContent.split('\n').length,
+                numberOfLines: split.length,
+                numberOfNonEmptyLines: split.filter(l => l.trim().length > 0).length,
                 numberOfCharacters: loadedContent.length,
-                numberOfNonWhitespaceCharacters: (0, strings_1.withoutWhitespace)(loadedContent).length,
+                numberOfCharactersNoComments: loadedContent.length - commentChars,
+                numberOfNonWhitespaceCharacters: nonWhitespace,
+                numberOfNonWhitespaceCharactersNoComments: nonWhitespace - commentCharsNoWhitespace,
                 numberOfRTokens: numberOfRTokens,
-                numberOfNormalizedTokens: [...(0, collect_1.collectAllIds)(this.normalizedAst.ast)].length
+                numberOfRTokensNoComments: numberOfRTokensNoComments,
+                numberOfNormalizedTokens: nodes,
+                numberOfNormalizedTokensNoComments: nodesNoComments
             },
             dataflow: {
                 numberOfNodes: [...this.dataflow.graph.vertices(true)].length,
@@ -122,7 +147,7 @@ class BenchmarkSlicer {
             timesHitThreshold: 0,
             reconstructedCode: {
                 code: '',
-                autoSelected: 0
+                linesWithAutoSelected: 0
             }
         };
         this.perSliceMeasurements.set(slicingCriteria, stats);

package/benchmark/stats/print.js

@@ -40,6 +40,7 @@ function asPercentage(num) {
     if (isNaN(num)) {
         return '??%';
     }
+    (0, assert_1.guard)(num >= 0 && num <= 1, `Percentage ${num} should be between 0 and 1`);
     return pad(`${roundTo(num * 100, 3)}%`);
 }
 function asFloat(num) {
@@ -81,11 +82,14 @@ Slicing summary for ${stats.perSliceMeasurements.numberOfSlices} slice${stats.pe
   Used Slice Criteria Sizes:  ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceCriteriaSizes)}
   Result Slice Sizes:   
     Number of lines:                     ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.lines)}
+    Number of non-empty lines:           ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.nonEmptyLines)}
     Number of characters:                ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.characters)}
     Number of non whitespace characters: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.nonWhitespaceCharacters)}
-    Number of auto selected:             ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.autoSelected)}
+    Number of auto selected lines:       ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.linesWithAutoSelected)}
     Number of R tokens:                  ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.tokens)}
+    Number of R tokens (w/o comments):   ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.tokensNoComments)}
     Normalized R tokens:                 ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.normalizedTokens)}
+    Normalized R tokens (w/o comments):  ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.normalizedTokensNoComments)}
     Number of dataflow nodes:            ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.dataflowNodes)}
 `;
     }
@@ -95,10 +99,15 @@ Total:                        ${print(stats.commonMeasurements, 'total')}
 
 Input:
   Number of lines:                     ${pad(stats.input.numberOfLines)}
+  Number of non empty lines:           ${pad(stats.input.numberOfNonEmptyLines)}
   Number of characters:                ${pad(stats.input.numberOfCharacters)}
+  Number of characters (w/o comments): ${pad(stats.input.numberOfCharactersNoComments)}
   Number of non whitespace characters: ${pad(stats.input.numberOfNonWhitespaceCharacters)}
+  Number of n. w. c. (w/o comments):   ${pad(stats.input.numberOfNonWhitespaceCharactersNoComments)}
   Number of tokens:                    ${pad(stats.input.numberOfRTokens)}
+  Number of tokens (w/o comments):     ${pad(stats.input.numberOfRTokensNoComments)}
   Normalized R tokens:                 ${pad(stats.input.numberOfNormalizedTokens)}
+  Normalized R tokens (w/o comments):  ${pad(stats.input.numberOfNormalizedTokensNoComments)}
 
 Dataflow:
   Number of nodes:            ${pad(stats.dataflow.numberOfNodes)}
@@ -122,24 +131,23 @@ Slice summary for:
   Reconstruction:             ${formatSummarizedTimeMeasure(stats.perSliceMeasurements.get('reconstruct code'))}
   Failed to Re-Parse:         ${pad(stats.failedToRepParse)}/${stats.totalSlices}
   Times hit Threshold:        ${pad(stats.timesHitThreshold)}/${stats.totalSlices} 
-  Reductions (reduced by x%):   
-    Number of lines:                     ${formatSummarizedMeasure(stats.reduction.numberOfLines, asPercentage)}
-    Number of lines no auto:             ${formatSummarizedMeasure(stats.reduction.numberOfLinesNoAutoSelection, asPercentage)}
-    Number of characters:                ${formatSummarizedMeasure(stats.reduction.numberOfCharacters, asPercentage)}
-    Number of non whitespace characters: ${formatSummarizedMeasure(stats.reduction.numberOfNonWhitespaceCharacters, asPercentage)}
-    Number of R tokens:                  ${formatSummarizedMeasure(stats.reduction.numberOfRTokens, asPercentage)}
-    Normalized R tokens:                 ${formatSummarizedMeasure(stats.reduction.numberOfNormalizedTokens, asPercentage)}
-    Number of dataflow nodes:            ${formatSummarizedMeasure(stats.reduction.numberOfDataflowNodes, asPercentage)}
+${reduction2String('Reductions', stats.reduction)}
+${reduction2String('Reductions without comments and empty lines', stats.reductionNoFluff)}
 
 Shell close:                  ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('close R session'))}
 Total:                        ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('total'))}
 
 Input:
   Number of lines:                     ${formatSummarizedMeasure(stats.input.numberOfLines)}
+  Number of non empty lines:           ${formatSummarizedMeasure(stats.input.numberOfNonEmptyLines)}
   Number of characters:                ${formatSummarizedMeasure(stats.input.numberOfCharacters)}
+  Number of characters (w/o comments): ${formatSummarizedMeasure(stats.input.numberOfCharactersNoComments)}
   Number of non whitespace characters: ${formatSummarizedMeasure(stats.input.numberOfNonWhitespaceCharacters)}
+  Number of n. w. c. (w/o comments):   ${formatSummarizedMeasure(stats.input.numberOfNonWhitespaceCharactersNoComments)}
   Number of tokens:                    ${formatSummarizedMeasure(stats.input.numberOfRTokens)}
+  Number of tokens (w/o comments):     ${formatSummarizedMeasure(stats.input.numberOfRTokensNoComments)}
   Normalized R tokens:                 ${formatSummarizedMeasure(stats.input.numberOfNormalizedTokens)}
+  Normalized R tokens (w/o comments):  ${formatSummarizedMeasure(stats.input.numberOfNormalizedTokensNoComments)}
 
 Dataflow:
   Number of nodes:            ${formatSummarizedMeasure(stats.dataflow.numberOfNodes)}
@@ -148,4 +156,15 @@ Dataflow:
   Number of function defs:    ${formatSummarizedMeasure(stats.dataflow.numberOfFunctionDefinitions)}`;
 }
 exports.ultimateStats2String = ultimateStats2String;
+function reduction2String(title, reduction) {
+    return `
+  ${title} (reduced by x%):   
+    Number of lines:                     ${formatSummarizedMeasure(reduction.numberOfLines, asPercentage)}
+    Number of lines no auto:             ${formatSummarizedMeasure(reduction.numberOfLinesNoAutoSelection, asPercentage)}
+    Number of characters:                ${formatSummarizedMeasure(reduction.numberOfCharacters, asPercentage)}
+    Number of non whitespace characters: ${formatSummarizedMeasure(reduction.numberOfNonWhitespaceCharacters, asPercentage)}
+    Number of R tokens:                  ${formatSummarizedMeasure(reduction.numberOfRTokens, asPercentage)}
+    Normalized R tokens:                 ${formatSummarizedMeasure(reduction.numberOfNormalizedTokens, asPercentage)}
+    Number of dataflow nodes:            ${formatSummarizedMeasure(reduction.numberOfDataflowNodes, asPercentage)}`;
+}
 //# sourceMappingURL=print.js.map

package/benchmark/stats/stats.d.ts

@@ -19,10 +19,15 @@ export interface PerSliceStats {
 }
 export interface SlicerStatsInput<T = number> {
     numberOfLines: T;
+    numberOfNonEmptyLines: T;
     numberOfCharacters: T;
+    numberOfCharactersNoComments: T;
     numberOfNonWhitespaceCharacters: T;
+    numberOfNonWhitespaceCharactersNoComments: T;
     numberOfRTokens: T;
+    numberOfRTokensNoComments: T;
     numberOfNormalizedTokens: T;
+    numberOfNormalizedTokensNoComments: T;
 }
 export interface SlicerStatsDataflow<T = number> {
     numberOfNodes: T;

package/benchmark/summarizer/data.d.ts

@@ -2,13 +2,18 @@ import type { SummarizedMeasurement } from '../../util/summarizer';
 import type { CommonSlicerMeasurements, PerSliceMeasurements, SlicerStats, SlicerStatsDataflow, SlicerStatsInput } from '../stats/stats';
 export interface SliceSizeCollection {
     lines: number[];
+    nonEmptyLines: number[];
     characters: number[];
+    charactersNoComments: number[];
     nonWhitespaceCharacters: number[];
+    nonWhitespaceCharactersNoComments: number[];
     /** like library statements during reconstruction */
-    autoSelected: number[];
+    linesWithAutoSelected: number[];
     dataflowNodes: number[];
     tokens: number[];
+    tokensNoComments: number[];
     normalizedTokens: number[];
+    normalizedTokensNoComments: number[];
 }
 /**
  * @see SlicerStats
@@ -33,6 +38,8 @@ export interface SummarizedPerSliceStats {
     sliceCriteriaSizes: SummarizedMeasurement;
     measurements: Map<PerSliceMeasurements, SummarizedMeasurement>;
     reduction: Reduction<SummarizedMeasurement>;
+    /** reduction, but without taking into account comments and empty lines */
+    reductionNoFluff: Reduction<SummarizedMeasurement>;
     failedToRepParse: number;
     timesHitThreshold: number;
     sliceSize: {
@@ -49,6 +56,8 @@ export interface UltimateSlicerStats {
     /** sum */
     timesHitThreshold: number;
     reduction: Reduction<SummarizedMeasurement>;
+    /** reduction, but without taking into account comments and empty lines */
+    reductionNoFluff: Reduction<SummarizedMeasurement>;
     input: SlicerStatsInput<SummarizedMeasurement>;
     dataflow: SlicerStatsDataflow<SummarizedMeasurement>;
 }

package/benchmark/summarizer/first-phase/process.d.ts

@@ -8,4 +8,4 @@ import type { SlicingCriteria } from '../../../slicing/criterion/parse';
  */
 export declare function summarizeSlicerStats(stats: SlicerStats, report?: (criteria: SlicingCriteria, stats: PerSliceStats) => void): Promise<Readonly<SummarizedSlicerStats>>;
 export declare function summarizeSummarizedMeasurement(data: SummarizedMeasurement[]): SummarizedMeasurement;
-export declare function summarizeReductions(reductions: Reduction<SummarizedMeasurement>[]): Reduction<SummarizedMeasurement>;
+export declare function summarizeSummarizedReductions(reductions: Reduction<SummarizedMeasurement>[]): Reduction<SummarizedMeasurement>;

package/benchmark/summarizer/first-phase/process.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.summarizeReductions = exports.summarizeSummarizedMeasurement = exports.summarizeSlicerStats = void 0;
+exports.summarizeSummarizedReductions = exports.summarizeSummarizedMeasurement = exports.summarizeSlicerStats = void 0;
 const tmp = __importStar(require("tmp"));
 const fs_1 = __importDefault(require("fs"));
 const defaultmap_1 = require("../../../util/defaultmap");
@@ -65,14 +65,24 @@ function safeDivPercentage(a, b) {
         }
     }
 }
-function calculateReductionForSlice(input, dataflow, perSlice) {
+function calculateReductionForSlice(input, dataflow, perSlice, ignoreFluff) {
+    const perSliceLines = ignoreFluff ? perSlice.nonEmptyLines : perSlice.lines;
+    const inputLines = ignoreFluff ? input.numberOfNonEmptyLines : input.numberOfLines;
     return {
-        numberOfLines: safeDivPercentage(perSlice.lines, input.numberOfLines),
-        numberOfLinesNoAutoSelection: safeDivPercentage(perSlice.lines - perSlice.autoSelected, input.numberOfLines),
-        numberOfCharacters: safeDivPercentage(perSlice.characters, input.numberOfCharacters),
-        numberOfNonWhitespaceCharacters: safeDivPercentage(perSlice.nonWhitespaceCharacters, input.numberOfNonWhitespaceCharacters),
-        numberOfRTokens: safeDivPercentage(perSlice.tokens, input.numberOfRTokens),
-        numberOfNormalizedTokens: safeDivPercentage(perSlice.normalizedTokens, input.numberOfNormalizedTokens),
+        numberOfLines: safeDivPercentage(perSliceLines, inputLines),
+        numberOfLinesNoAutoSelection: safeDivPercentage(perSliceLines - perSlice.linesWithAutoSelected, inputLines),
+        numberOfCharacters: ignoreFluff ?
+            safeDivPercentage(perSlice.charactersNoComments, input.numberOfCharactersNoComments) :
+            safeDivPercentage(perSlice.characters, input.numberOfCharacters),
+        numberOfNonWhitespaceCharacters: ignoreFluff ?
+            safeDivPercentage(perSlice.nonWhitespaceCharactersNoComments, input.numberOfNonWhitespaceCharactersNoComments) :
+            safeDivPercentage(perSlice.nonWhitespaceCharacters, input.numberOfNonWhitespaceCharacters),
+        numberOfRTokens: ignoreFluff ?
+            safeDivPercentage(perSlice.tokensNoComments, input.numberOfRTokensNoComments) :
+            safeDivPercentage(perSlice.tokens, input.numberOfRTokens),
+        numberOfNormalizedTokens: ignoreFluff ?
+            safeDivPercentage(perSlice.normalizedTokensNoComments, input.numberOfNormalizedTokensNoComments) :
+            safeDivPercentage(perSlice.normalizedTokens, input.numberOfNormalizedTokens),
         numberOfDataflowNodes: safeDivPercentage(perSlice.dataflowNodes, dataflow.numberOfNodes)
     };
 }
@@ -87,14 +97,20 @@ async function summarizeSlicerStats(stats, report = () => {
     const sizeOfSliceCriteria = [];
     const reParseShellSession = new shell_1.RShell();
     const reductions = [];
+    const reductionsNoFluff = [];
     let failedOutputs = 0;
     const sliceSize = {
         lines: [],
-        autoSelected: [],
+        nonEmptyLines: [],
+        linesWithAutoSelected: [],
         characters: [],
+        charactersNoComments: [],
         nonWhitespaceCharacters: [],
+        nonWhitespaceCharactersNoComments: [],
         tokens: [],
+        tokensNoComments: [],
         normalizedTokens: [],
+        normalizedTokensNoComments: [],
         dataflowNodes: []
     };
     let timesHitThreshold = 0;
@@ -105,10 +121,13 @@ async function summarizeSlicerStats(stats, report = () => {
         }
         sizeOfSliceCriteria.push(perSliceStat.slicingCriteria.length);
         timesHitThreshold += perSliceStat.timesHitThreshold > 0 ? 1 : 0;
-        const { code: output, autoSelected } = perSliceStat.reconstructedCode;
-        sliceSize.autoSelected.push(autoSelected);
-        const lines = output.split('\n').length;
+        const { code: output, linesWithAutoSelected } = perSliceStat.reconstructedCode;
+        sliceSize.linesWithAutoSelected.push(linesWithAutoSelected);
+        const split = output.split('\n');
+        const lines = split.length;
+        const nonEmptyLines = split.filter(l => l.trim().length > 0).length;
         sliceSize.lines.push(lines);
+        sliceSize.nonEmptyLines.push(nonEmptyLines);
         sliceSize.characters.push(output.length);
         const nonWhitespace = (0, strings_1.withoutWhitespace)(output).length;
         sliceSize.nonWhitespaceCharacters.push(nonWhitespace);
@@ -118,22 +137,46 @@ async function summarizeSlicerStats(stats, report = () => {
             fs_1.default.writeFileSync(tempfile().name, output);
             const reParsed = await (0, retriever_1.retrieveNormalizedAstFromRCode)({ request: 'file', content: tempfile().name }, reParseShellSession);
             let numberOfNormalizedTokens = 0;
-            (0, visitor_1.visitAst)(reParsed.ast, _ => {
+            let numberOfNormalizedTokensNoComments = 0;
+            let commentChars = 0;
+            let commentCharsNoWhitespace = 0;
+            (0, visitor_1.visitAst)(reParsed.ast, t => {
                 numberOfNormalizedTokens++;
+                const comments = t.info.additionalTokens?.filter(t => t.type === "RComment" /* RType.Comment */);
+                if (comments && comments.length > 0) {
+                    const content = comments.map(c => c.lexeme ?? '').join('');
+                    commentChars += content.length;
+                    commentCharsNoWhitespace += (0, strings_1.withoutWhitespace)(content).length;
+                }
+                else {
+                    numberOfNormalizedTokensNoComments++;
+                }
                 return false;
             });
             sliceSize.normalizedTokens.push(numberOfNormalizedTokens);
+            sliceSize.normalizedTokensNoComments.push(numberOfNormalizedTokensNoComments);
+            sliceSize.charactersNoComments.push(output.length - commentChars);
+            sliceSize.nonWhitespaceCharactersNoComments.push(nonWhitespace - commentCharsNoWhitespace);
             const numberOfRTokens = await (0, retriever_1.retrieveNumberOfRTokensOfLastParse)(reParseShellSession);
             sliceSize.tokens.push(numberOfRTokens);
-            reductions.push(calculateReductionForSlice(stats.input, stats.dataflow, {
+            const numberOfRTokensNoComments = await (0, retriever_1.retrieveNumberOfRTokensOfLastParse)(reParseShellSession, true);
+            sliceSize.tokensNoComments.push(numberOfRTokensNoComments);
+            const perSlice = {
                 lines: lines,
+                nonEmptyLines: nonEmptyLines,
                 characters: output.length,
+                charactersNoComments: output.length - commentChars,
                 nonWhitespaceCharacters: nonWhitespace,
-                autoSelected: autoSelected,
+                nonWhitespaceCharactersNoComments: nonWhitespace - commentCharsNoWhitespace,
+                linesWithAutoSelected: linesWithAutoSelected,
                 tokens: numberOfRTokens,
+                tokensNoComments: numberOfRTokensNoComments,
                 normalizedTokens: numberOfNormalizedTokens,
+                normalizedTokensNoComments: numberOfNormalizedTokensNoComments,
                 dataflowNodes: perSliceStat.numberOfDataflowNodesSliced
-            }));
+            };
+            reductions.push(calculateReductionForSlice(stats.input, stats.dataflow, perSlice, false));
+            reductionsNoFluff.push(calculateReductionForSlice(stats.input, stats.dataflow, perSlice, true));
         }
         catch (e) {
             console.error(`    ! Failed to re-parse the output of the slicer for ${JSON.stringify(criteria)}`); //, e
@@ -156,22 +199,20 @@ async function summarizeSlicerStats(stats, report = () => {
             measurements: summarized,
             failedToRepParse: failedOutputs,
             timesHitThreshold,
-            reduction: {
-                numberOfLines: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLines).filter(assert_1.isNotUndefined)),
-                numberOfLinesNoAutoSelection: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLinesNoAutoSelection).filter(assert_1.isNotUndefined)),
-                numberOfCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfCharacters).filter(assert_1.isNotUndefined)),
-                numberOfNonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNonWhitespaceCharacters).filter(assert_1.isNotUndefined)),
-                numberOfRTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfRTokens).filter(assert_1.isNotUndefined)),
-                numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNormalizedTokens).filter(assert_1.isNotUndefined)),
-                numberOfDataflowNodes: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfDataflowNodes).filter(assert_1.isNotUndefined))
-            },
+            reduction: summarizeReductions(reductions),
+            reductionNoFluff: summarizeReductions(reductionsNoFluff),
             sliceSize: {
                 lines: (0, summarizer_1.summarizeMeasurement)(sliceSize.lines),
+                nonEmptyLines: (0, summarizer_1.summarizeMeasurement)(sliceSize.nonEmptyLines),
                 characters: (0, summarizer_1.summarizeMeasurement)(sliceSize.characters),
+                charactersNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.charactersNoComments),
                 nonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(sliceSize.nonWhitespaceCharacters),
-                autoSelected: (0, summarizer_1.summarizeMeasurement)(sliceSize.autoSelected),
+                nonWhitespaceCharactersNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.nonWhitespaceCharactersNoComments),
+                linesWithAutoSelected: (0, summarizer_1.summarizeMeasurement)(sliceSize.linesWithAutoSelected),
                 tokens: (0, summarizer_1.summarizeMeasurement)(sliceSize.tokens),
+                tokensNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.tokensNoComments),
                 normalizedTokens: (0, summarizer_1.summarizeMeasurement)(sliceSize.normalizedTokens),
+                normalizedTokensNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.normalizedTokensNoComments),
                 dataflowNodes: (0, summarizer_1.summarizeMeasurement)(sliceSize.dataflowNodes)
             }
         }
@@ -181,8 +222,9 @@ exports.summarizeSlicerStats = summarizeSlicerStats;
 function summarizeSummarizedMeasurement(data) {
     const min = data.map(d => d.min).filter(assert_1.isNotUndefined).reduce((a, b) => Math.min(a, b), Infinity);
     const max = data.map(d => d.max).filter(assert_1.isNotUndefined).reduce((a, b) => Math.max(a, b), -Infinity);
-    // get most average
-    const median = data.map(d => d.median).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
+    // calculate median of medians (don't just average the median!)
+    const medians = data.map(d => d.median).filter(assert_1.isNotUndefined).sort((a, b) => a - b);
+    const median = medians[Math.floor(medians.length / 2)];
     const mean = data.map(d => d.mean).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
     // Method 1 of https://www.statology.org/averaging-standard-deviations/
     const std = Math.sqrt(data.map(d => d.std ** 2).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length);
@@ -190,7 +232,7 @@ function summarizeSummarizedMeasurement(data) {
     return { min, max, median, mean, std, total };
 }
 exports.summarizeSummarizedMeasurement = summarizeSummarizedMeasurement;
-function summarizeReductions(reductions) {
+function summarizeSummarizedReductions(reductions) {
     return {
         numberOfDataflowNodes: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfDataflowNodes)),
         numberOfLines: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfLines)),
@@ -201,5 +243,16 @@ function summarizeReductions(reductions) {
         numberOfRTokens: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfRTokens))
     };
 }
-exports.summarizeReductions = summarizeReductions;
+exports.summarizeSummarizedReductions = summarizeSummarizedReductions;
+function summarizeReductions(reductions) {
+    return {
+        numberOfLines: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLines).filter(assert_1.isNotUndefined)),
+        numberOfLinesNoAutoSelection: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLinesNoAutoSelection).filter(assert_1.isNotUndefined)),
+        numberOfCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfCharacters).filter(assert_1.isNotUndefined)),
+        numberOfNonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNonWhitespaceCharacters).filter(assert_1.isNotUndefined)),
+        numberOfRTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfRTokens).filter(assert_1.isNotUndefined)),
+        numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNormalizedTokens).filter(assert_1.isNotUndefined)),
+        numberOfDataflowNodes: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfDataflowNodes).filter(assert_1.isNotUndefined))
+    };
+}
 //# sourceMappingURL=process.js.map

package/benchmark/summarizer/second-phase/process.js

@@ -10,6 +10,7 @@ function summarizeAllSummarizedStats(stats) {
     const commonMeasurements = new defaultmap_1.DefaultMap(() => []);
     const perSliceMeasurements = new defaultmap_1.DefaultMap(() => []);
     const reductions = [];
+    const reductionsNoFluff = [];
     const inputs = [];
     const dataflows = [];
     let failedToRepParse = 0;
@@ -23,6 +24,7 @@ function summarizeAllSummarizedStats(stats) {
             perSliceMeasurements.get(k).push(v);
         }
         reductions.push(stat.perSliceMeasurements.reduction);
+        reductionsNoFluff.push(stat.perSliceMeasurements.reductionNoFluff);
         inputs.push(stat.input);
         dataflows.push(stat.dataflow);
         failedToRepParse += stat.perSliceMeasurements.failedToRepParse;
@@ -36,13 +38,19 @@ function summarizeAllSummarizedStats(stats) {
         perSliceMeasurements: new Map([...perSliceMeasurements.entries()].map(([k, v]) => [k, (0, process_1.summarizeSummarizedMeasurement)(v)])),
         failedToRepParse,
         timesHitThreshold,
-        reduction: (0, process_1.summarizeReductions)(reductions),
+        reduction: (0, process_1.summarizeSummarizedReductions)(reductions),
+        reductionNoFluff: (0, process_1.summarizeSummarizedReductions)(reductionsNoFluff),
         input: {
             numberOfLines: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfLines)),
+            numberOfNonEmptyLines: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNonEmptyLines)),
             numberOfCharacters: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfCharacters)),
+            numberOfCharactersNoComments: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfCharactersNoComments)),
             numberOfNonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNonWhitespaceCharacters)),
+            numberOfNonWhitespaceCharactersNoComments: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNonWhitespaceCharactersNoComments)),
             numberOfRTokens: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfRTokens)),
-            numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNormalizedTokens))
+            numberOfRTokensNoComments: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfRTokensNoComments)),
+            numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNormalizedTokens)),
+            numberOfNormalizedTokensNoComments: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNormalizedTokensNoComments))
         },
         dataflow: {
             numberOfNodes: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfNodes)),
@@ -63,13 +71,19 @@ function summarizeAllUltimateStats(stats) {
         // average out / summarize other measurements
         commonMeasurements: new Map(stats_1.CommonSlicerMeasurements.map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.commonMeasurements.get(m)))])),
         perSliceMeasurements: new Map(stats_1.PerSliceMeasurements.map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.perSliceMeasurements.get(m)))])),
-        reduction: (0, process_1.summarizeReductions)(stats.map(s => s.reduction)),
+        reduction: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reduction)),
+        reductionNoFluff: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reductionNoFluff)),
         input: {
             numberOfLines: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfLines)),
+            numberOfNonEmptyLines: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNonEmptyLines)),
             numberOfCharacters: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfCharacters)),
+            numberOfCharactersNoComments: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfCharactersNoComments)),
             numberOfNonWhitespaceCharacters: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNonWhitespaceCharacters)),
+            numberOfNonWhitespaceCharactersNoComments: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNonWhitespaceCharactersNoComments)),
             numberOfRTokens: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfRTokens)),
-            numberOfNormalizedTokens: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNormalizedTokens))
+            numberOfRTokensNoComments: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfRTokensNoComments)),
+            numberOfNormalizedTokens: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNormalizedTokens)),
+            numberOfNormalizedTokensNoComments: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNormalizedTokensNoComments))
         },
         dataflow: {
             numberOfNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfNodes)),
@@ -97,6 +111,7 @@ function processNextSummary(line, allSummarized) {
                 sliceCriteriaSizes: got.summarize.perSliceMeasurements.sliceCriteriaSizes,
                 measurements: new Map(got.summarize.perSliceMeasurements.measurements),
                 reduction: got.summarize.perSliceMeasurements.reduction,
+                reductionNoFluff: got.summarize.perSliceMeasurements.reductionNoFluff,
                 timesHitThreshold: got.summarize.perSliceMeasurements.timesHitThreshold,
                 failedToRepParse: got.summarize.perSliceMeasurements.failedToRepParse,
                 sliceSize: got.summarize.perSliceMeasurements.sliceSize
@@ -117,6 +132,7 @@ function processNextUltimateSummary(line, allSummarized) {
             failedToRepParse: got.summarize.failedToRepParse,
             timesHitThreshold: got.summarize.timesHitThreshold,
             reduction: got.summarize.reduction,
+            reductionNoFluff: got.summarize.reductionNoFluff,
             input: got.summarize.input,
             dataflow: got.summarize.dataflow,
         }

package/cli/repl/commands/commands.js

@@ -61,6 +61,15 @@ const _commands = {
     'controlflow*': cfg_1.controlflowStarCommand
 };
 let commandsInitialized = false;
+function hasModule(path) {
+    try {
+        require.resolve(path);
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+}
 function commands() {
     if (commandsInitialized) {
         return _commands;
@@ -74,7 +83,16 @@ function commands() {
                 script: true,
                 usageExample: `:${script} --help`,
                 fn: async (output, _s, remainingLine) => {
-                    await (0, execute_1.waitOnScript)(`${__dirname}/../../${target}`, (0, args_1.splitAtEscapeSensitive)(remainingLine), stdio => (0, execute_1.stdioCaptureProcessor)(stdio, msg => output.stdout(msg), msg => output.stderr(msg)));
+                    // check if the target *module* exists in the current directory, else try two dirs up, otherwise, fail with a message
+                    let path = `${__dirname}/${target}`;
+                    if (!hasModule(path)) {
+                        path = `${__dirname}/../../${target}`;
+                        if (!hasModule(path)) {
+                            output.stderr(`Could not find the target script ${target} in the current directory or two directories up.`);
+                            return;
+                        }
+                    }
+                    await (0, execute_1.waitOnScript)(path, (0, args_1.splitAtEscapeSensitive)(remainingLine), stdio => (0, execute_1.stdioCaptureProcessor)(stdio, msg => output.stdout(msg), msg => output.stderr(msg)));
                 }
             };
         }

package/cli/slicer-app.js

@@ -40,7 +40,7 @@ async function getSlice() {
             reconstruct = reconstructedCode;
             if (options.output) {
                 console.log('Written reconstructed code to', options.output);
-                console.log(`Automatically selected ${reconstructedCode.autoSelected} statements`);
+                console.log(`Automatically selected ${reconstructedCode.linesWithAutoSelected} lines`);
                 fs_1.default.writeFileSync(options.output, reconstructedCode.code);
             }
             else if (!options.api && !options.diff) {

package/dataflow/environments/append.js

@@ -15,7 +15,6 @@ function uniqueMergeValues(old, value) {
 }
 function appendIEnvironmentWith(base, next) {
     (0, assert_1.guard)(base !== undefined && next !== undefined, 'can not append environments with undefined');
-    (0, assert_1.guard)(base.name === next.name, 'cannot overwrite environments with different names');
     const map = new Map(base.memory);
     for (const [key, value] of next.memory) {
         const old = map.get(key);
@@ -27,7 +26,7 @@ function appendIEnvironmentWith(base, next) {
         }
     }
     const parent = base.parent === environment_1.BuiltInEnvironment ? environment_1.BuiltInEnvironment : appendIEnvironmentWith(base.parent, next.parent);
-    const out = new environment_1.Environment(base.name, parent);
+    const out = new environment_1.Environment(parent);
     out.memory = map;
     return out;
 }

package/dataflow/environments/built-in.js

@@ -111,7 +111,8 @@ registerBuiltInFunctions(['delayedAssign'], built_in_assignment_1.processAssignm
 registerBuiltInFunctions(['<<-'], built_in_assignment_1.processAssignment, { superAssignment: true, canBeReplacement: true });
 registerBuiltInFunctions(['->'], built_in_assignment_1.processAssignment, { swapSourceAndTarget: true, canBeReplacement: true });
 registerBuiltInFunctions(['->>'], built_in_assignment_1.processAssignment, { superAssignment: true, swapSourceAndTarget: true, canBeReplacement: true });
-registerBuiltInFunctions(['&&', '||', '&', '|'], built_in_logical_bin_op_1.processSpecialBinOp, { lazy: true });
+registerBuiltInFunctions(['&&', '&'], built_in_logical_bin_op_1.processSpecialBinOp, { lazy: true, evalRhsWhen: true });
+registerBuiltInFunctions(['||', '|'], built_in_logical_bin_op_1.processSpecialBinOp, { lazy: true, evalRhsWhen: false });
 registerBuiltInFunctions(['|>', '%>%'], built_in_pipe_1.processPipe, {});
 registerBuiltInFunctions(['function', '\\'], built_in_function_definition_1.processFunctionDefinition, {});
 registerBuiltInFunctions(['quote', 'substitute', 'bquote'], built_in_quote_1.processQuote, { quoteArgumentsWithIndex: 0 });

package/dataflow/environments/clone.js

@@ -9,7 +9,7 @@ function cloneEnvironment(environment, recurseParents) {
     else if (environment.id === environment_1.BuiltInEnvironment.id) {
         return environment_1.BuiltInEnvironment;
     }
-    const clone = new environment_1.Environment(environment.name, recurseParents ? cloneEnvironment(environment.parent, recurseParents) : environment.parent);
+    const clone = new environment_1.Environment(recurseParents ? cloneEnvironment(environment.parent, recurseParents) : environment.parent);
     clone.memory = new Map(JSON.parse(JSON.stringify([...environment.memory])));
     return clone;
 }

package/dataflow/environments/diff.d.ts

@@ -2,5 +2,5 @@ import type { GenericDifferenceInformation, WriteableDifferenceReport } from '..
 import type { IEnvironment, REnvironmentInformation } from './environment';
 import type { IdentifierReference } from './identifier';
 export declare function diffIdentifierReferences<Report extends WriteableDifferenceReport>(a: IdentifierReference | undefined, b: IdentifierReference | undefined, info: GenericDifferenceInformation<Report>): void;
-export declare function diffEnvironment<Report extends WriteableDifferenceReport>(a: IEnvironment | undefined, b: IEnvironment | undefined, info: GenericDifferenceInformation<Report>): void;
+export declare function diffEnvironment<Report extends WriteableDifferenceReport>(a: IEnvironment | undefined, b: IEnvironment | undefined, info: GenericDifferenceInformation<Report>, depth: number): void;
 export declare function diffEnvironmentInformation<Report extends WriteableDifferenceReport>(a: REnvironmentInformation | undefined, b: REnvironmentInformation | undefined, info: GenericDifferenceInformation<Report>): void;