@eagleoutice/flowr 1.4.2 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/core/print/slice-diff-ansi.js +7 -7
  2. package/core/steps.d.ts +352 -11
  3. package/core/steps.js +4 -4
  4. package/index.d.ts +0 -2
  5. package/index.js +0 -2
  6. package/package.json +65 -215
  7. package/util/{summarizer/summarizer.d.ts → summarizer.d.ts} +15 -1
  8. package/util/summarizer.js +37 -0
  9. package/util/version.d.ts +2 -0
  10. package/util/version.js +10 -0
  11. package/benchmark/index.d.ts +0 -3
  12. package/benchmark/index.js +0 -20
  13. package/benchmark/slicer.d.ts +0 -98
  14. package/benchmark/slicer.js +0 -213
  15. package/benchmark/stats/index.d.ts +0 -10
  16. package/benchmark/stats/index.js +0 -27
  17. package/benchmark/stats/print.d.ts +0 -7
  18. package/benchmark/stats/print.js +0 -155
  19. package/benchmark/stats/stats.d.ts +0 -40
  20. package/benchmark/stats/stats.js +0 -6
  21. package/benchmark/stopwatch.d.ts +0 -35
  22. package/benchmark/stopwatch.js +0 -79
  23. package/cli/benchmark-app.d.ts +0 -10
  24. package/cli/benchmark-app.js +0 -67
  25. package/cli/benchmark-helper-app.d.ts +0 -9
  26. package/cli/benchmark-helper-app.js +0 -69
  27. package/cli/common/features.d.ts +0 -3
  28. package/cli/common/features.js +0 -30
  29. package/cli/common/index.d.ts +0 -2
  30. package/cli/common/index.js +0 -19
  31. package/cli/common/options.d.ts +0 -20
  32. package/cli/common/options.js +0 -85
  33. package/cli/common/script.d.ts +0 -21
  34. package/cli/common/script.js +0 -61
  35. package/cli/common/scripts-info.d.ts +0 -25
  36. package/cli/common/scripts-info.js +0 -83
  37. package/cli/export-quads-app.d.ts +0 -7
  38. package/cli/export-quads-app.js +0 -42
  39. package/cli/repl/commands/cfg.d.ts +0 -3
  40. package/cli/repl/commands/cfg.js +0 -37
  41. package/cli/repl/commands/commands.d.ts +0 -13
  42. package/cli/repl/commands/commands.js +0 -142
  43. package/cli/repl/commands/dataflow.d.ts +0 -3
  44. package/cli/repl/commands/dataflow.js +0 -34
  45. package/cli/repl/commands/execute.d.ts +0 -4
  46. package/cli/repl/commands/execute.js +0 -27
  47. package/cli/repl/commands/index.d.ts +0 -2
  48. package/cli/repl/commands/index.js +0 -19
  49. package/cli/repl/commands/main.d.ts +0 -39
  50. package/cli/repl/commands/main.js +0 -14
  51. package/cli/repl/commands/normalize.d.ts +0 -3
  52. package/cli/repl/commands/normalize.js +0 -34
  53. package/cli/repl/commands/parse.d.ts +0 -2
  54. package/cli/repl/commands/parse.js +0 -109
  55. package/cli/repl/commands/quit.d.ts +0 -2
  56. package/cli/repl/commands/quit.js +0 -15
  57. package/cli/repl/commands/version.d.ts +0 -16
  58. package/cli/repl/commands/version.js +0 -33
  59. package/cli/repl/core.d.ts +0 -41
  60. package/cli/repl/core.js +0 -172
  61. package/cli/repl/execute.d.ts +0 -28
  62. package/cli/repl/execute.js +0 -79
  63. package/cli/repl/index.d.ts +0 -5
  64. package/cli/repl/index.js +0 -22
  65. package/cli/repl/prompt.d.ts +0 -2
  66. package/cli/repl/prompt.js +0 -9
  67. package/cli/repl/server/connection.d.ts +0 -21
  68. package/cli/repl/server/connection.js +0 -213
  69. package/cli/repl/server/messages/analysis.d.ts +0 -71
  70. package/cli/repl/server/messages/analysis.js +0 -21
  71. package/cli/repl/server/messages/error.d.ts +0 -11
  72. package/cli/repl/server/messages/error.js +0 -3
  73. package/cli/repl/server/messages/hello.d.ts +0 -20
  74. package/cli/repl/server/messages/hello.js +0 -3
  75. package/cli/repl/server/messages/index.d.ts +0 -1
  76. package/cli/repl/server/messages/index.js +0 -3
  77. package/cli/repl/server/messages/messages.d.ts +0 -35
  78. package/cli/repl/server/messages/messages.js +0 -40
  79. package/cli/repl/server/messages/repl.d.ts +0 -33
  80. package/cli/repl/server/messages/repl.js +0 -37
  81. package/cli/repl/server/messages/slice.d.ts +0 -25
  82. package/cli/repl/server/messages/slice.js +0 -37
  83. package/cli/repl/server/net.d.ts +0 -49
  84. package/cli/repl/server/net.js +0 -63
  85. package/cli/repl/server/send.d.ts +0 -4
  86. package/cli/repl/server/send.js +0 -18
  87. package/cli/repl/server/server.d.ts +0 -20
  88. package/cli/repl/server/server.js +0 -66
  89. package/cli/repl/server/validate.d.ts +0 -15
  90. package/cli/repl/server/validate.js +0 -34
  91. package/cli/slicer-app.d.ts +0 -11
  92. package/cli/slicer-app.js +0 -81
  93. package/cli/statistics-app.d.ts +0 -11
  94. package/cli/statistics-app.js +0 -98
  95. package/cli/statistics-helper-app.d.ts +0 -11
  96. package/cli/statistics-helper-app.js +0 -82
  97. package/cli/summarizer-app.d.ts +0 -18
  98. package/cli/summarizer-app.js +0 -66
  99. package/flowr.d.ts +0 -29
  100. package/flowr.js +0 -142
  101. package/statistics/features/common-syntax-probability.d.ts +0 -31
  102. package/statistics/features/common-syntax-probability.js +0 -156
  103. package/statistics/features/feature.d.ts +0 -175
  104. package/statistics/features/feature.js +0 -30
  105. package/statistics/features/index.d.ts +0 -1
  106. package/statistics/features/index.js +0 -18
  107. package/statistics/features/post-processing.d.ts +0 -12
  108. package/statistics/features/post-processing.js +0 -21
  109. package/statistics/features/supported/assignments/assignments.d.ts +0 -11
  110. package/statistics/features/supported/assignments/assignments.js +0 -53
  111. package/statistics/features/supported/assignments/index.d.ts +0 -1
  112. package/statistics/features/supported/assignments/index.js +0 -6
  113. package/statistics/features/supported/assignments/post-process.d.ts +0 -3
  114. package/statistics/features/supported/assignments/post-process.js +0 -125
  115. package/statistics/features/supported/comments/comments.d.ts +0 -18
  116. package/statistics/features/supported/comments/comments.js +0 -133
  117. package/statistics/features/supported/comments/index.d.ts +0 -1
  118. package/statistics/features/supported/comments/index.js +0 -6
  119. package/statistics/features/supported/comments/post-process.d.ts +0 -3
  120. package/statistics/features/supported/comments/post-process.js +0 -50
  121. package/statistics/features/supported/control-flow/control-flow.d.ts +0 -17
  122. package/statistics/features/supported/control-flow/control-flow.js +0 -67
  123. package/statistics/features/supported/control-flow/index.d.ts +0 -1
  124. package/statistics/features/supported/control-flow/index.js +0 -6
  125. package/statistics/features/supported/control-flow/post-process.d.ts +0 -3
  126. package/statistics/features/supported/control-flow/post-process.js +0 -65
  127. package/statistics/features/supported/data-access/data-access.d.ts +0 -15
  128. package/statistics/features/supported/data-access/data-access.js +0 -118
  129. package/statistics/features/supported/data-access/index.d.ts +0 -1
  130. package/statistics/features/supported/data-access/index.js +0 -6
  131. package/statistics/features/supported/data-access/post-process.d.ts +0 -3
  132. package/statistics/features/supported/data-access/post-process.js +0 -107
  133. package/statistics/features/supported/defined-functions/defined-functions.d.ts +0 -35
  134. package/statistics/features/supported/defined-functions/defined-functions.js +0 -139
  135. package/statistics/features/supported/defined-functions/index.d.ts +0 -1
  136. package/statistics/features/supported/defined-functions/index.js +0 -6
  137. package/statistics/features/supported/defined-functions/post-process.d.ts +0 -6
  138. package/statistics/features/supported/defined-functions/post-process.js +0 -177
  139. package/statistics/features/supported/expression-list/expression-list.d.ts +0 -9
  140. package/statistics/features/supported/expression-list/expression-list.js +0 -36
  141. package/statistics/features/supported/expression-list/index.d.ts +0 -1
  142. package/statistics/features/supported/expression-list/index.js +0 -6
  143. package/statistics/features/supported/expression-list/post-process.d.ts +0 -3
  144. package/statistics/features/supported/expression-list/post-process.js +0 -44
  145. package/statistics/features/supported/index.d.ts +0 -10
  146. package/statistics/features/supported/index.js +0 -27
  147. package/statistics/features/supported/loops/index.d.ts +0 -1
  148. package/statistics/features/supported/loops/index.js +0 -6
  149. package/statistics/features/supported/loops/loops.d.ts +0 -20
  150. package/statistics/features/supported/loops/loops.js +0 -79
  151. package/statistics/features/supported/loops/post-process.d.ts +0 -3
  152. package/statistics/features/supported/loops/post-process.js +0 -72
  153. package/statistics/features/supported/used-functions/index.d.ts +0 -1
  154. package/statistics/features/supported/used-functions/index.js +0 -6
  155. package/statistics/features/supported/used-functions/post-process.d.ts +0 -6
  156. package/statistics/features/supported/used-functions/post-process.js +0 -179
  157. package/statistics/features/supported/used-functions/used-functions.d.ts +0 -24
  158. package/statistics/features/supported/used-functions/used-functions.js +0 -95
  159. package/statistics/features/supported/used-packages/index.d.ts +0 -1
  160. package/statistics/features/supported/used-packages/index.js +0 -6
  161. package/statistics/features/supported/used-packages/post-process.d.ts +0 -3
  162. package/statistics/features/supported/used-packages/post-process.js +0 -121
  163. package/statistics/features/supported/used-packages/used-packages.d.ts +0 -16
  164. package/statistics/features/supported/used-packages/used-packages.js +0 -130
  165. package/statistics/features/supported/values/index.d.ts +0 -1
  166. package/statistics/features/supported/values/index.js +0 -6
  167. package/statistics/features/supported/values/post-process.d.ts +0 -3
  168. package/statistics/features/supported/values/post-process.js +0 -72
  169. package/statistics/features/supported/values/values.d.ts +0 -14
  170. package/statistics/features/supported/values/values.js +0 -101
  171. package/statistics/features/supported/variables/index.d.ts +0 -1
  172. package/statistics/features/supported/variables/index.js +0 -6
  173. package/statistics/features/supported/variables/post-process.d.ts +0 -9
  174. package/statistics/features/supported/variables/post-process.js +0 -122
  175. package/statistics/features/supported/variables/variables.d.ts +0 -15
  176. package/statistics/features/supported/variables/variables.js +0 -70
  177. package/statistics/index.d.ts +0 -6
  178. package/statistics/index.js +0 -24
  179. package/statistics/meta-statistics.d.ts +0 -33
  180. package/statistics/meta-statistics.js +0 -17
  181. package/statistics/output/file-provider.d.ts +0 -37
  182. package/statistics/output/file-provider.js +0 -97
  183. package/statistics/output/index.d.ts +0 -4
  184. package/statistics/output/index.js +0 -21
  185. package/statistics/output/print-stats.d.ts +0 -17
  186. package/statistics/output/print-stats.js +0 -69
  187. package/statistics/output/statistics-file.d.ts +0 -37
  188. package/statistics/output/statistics-file.js +0 -69
  189. package/statistics/statistics.d.ts +0 -24
  190. package/statistics/statistics.js +0 -108
  191. package/util/summarizer/auto-detect.d.ts +0 -2
  192. package/util/summarizer/auto-detect.js +0 -32
  193. package/util/summarizer/benchmark/data.d.ts +0 -66
  194. package/util/summarizer/benchmark/data.js +0 -13
  195. package/util/summarizer/benchmark/first-phase/input.d.ts +0 -3
  196. package/util/summarizer/benchmark/first-phase/input.js +0 -75
  197. package/util/summarizer/benchmark/first-phase/process.d.ts +0 -11
  198. package/util/summarizer/benchmark/first-phase/process.js +0 -217
  199. package/util/summarizer/benchmark/second-phase/graph.d.ts +0 -2
  200. package/util/summarizer/benchmark/second-phase/graph.js +0 -54
  201. package/util/summarizer/benchmark/second-phase/process.d.ts +0 -6
  202. package/util/summarizer/benchmark/second-phase/process.js +0 -126
  203. package/util/summarizer/benchmark/summarizer.d.ts +0 -31
  204. package/util/summarizer/benchmark/summarizer.js +0 -58
  205. package/util/summarizer/statistics/first-phase/process.d.ts +0 -6
  206. package/util/summarizer/statistics/first-phase/process.js +0 -81
  207. package/util/summarizer/statistics/post-process/clusterer.d.ts +0 -26
  208. package/util/summarizer/statistics/post-process/clusterer.js +0 -43
  209. package/util/summarizer/statistics/post-process/file-based-count.d.ts +0 -17
  210. package/util/summarizer/statistics/post-process/file-based-count.js +0 -49
  211. package/util/summarizer/statistics/post-process/histogram.d.ts +0 -59
  212. package/util/summarizer/statistics/post-process/histogram.js +0 -128
  213. package/util/summarizer/statistics/post-process/index.d.ts +0 -4
  214. package/util/summarizer/statistics/post-process/index.js +0 -21
  215. package/util/summarizer/statistics/post-process/post-process-output.d.ts +0 -16
  216. package/util/summarizer/statistics/post-process/post-process-output.js +0 -103
  217. package/util/summarizer/statistics/second-phase/process.d.ts +0 -11
  218. package/util/summarizer/statistics/second-phase/process.js +0 -117
  219. package/util/summarizer/statistics/summarizer.d.ts +0 -35
  220. package/util/summarizer/statistics/summarizer.js +0 -135
  221. package/util/summarizer/summarizer.js +0 -13
  222. /package/{statistics/output → util}/ansi.d.ts +0 -0
  223. /package/{statistics/output → util}/ansi.js +0 -0
@@ -1,179 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.postProcess = void 0;
7
- const data_1 = require("../../../../util/summarizer/benchmark/data");
8
- const common_syntax_probability_1 = require("../../common-syntax-probability");
9
- const process_1 = require("../../../../util/summarizer/benchmark/first-phase/process");
10
- const files_1 = require("../../../../util/files");
11
- const path_1 = __importDefault(require("path"));
12
- const fs_1 = __importDefault(require("fs"));
13
- const time_1 = require("../../../../util/time");
14
- const used_functions_1 = require("./used-functions");
15
- const numbers_1 = require("../../../../util/numbers");
16
- function retrieveFunctionCallInformation(featureRoot, info, config, outputPath) {
17
- // each number[][] contains a 'number[]' per file
18
- /**
19
- * maps fn-name (including namespace) to number of arguments and their location (the number of elements in the array give the number of total call)
20
- * we use tuples to reduce the memory!
21
- * A function that is defined within the file is _always_ decorated with the filename (as second array element)!
22
- */
23
- const functionsPerFile = new Map();
24
- const importantFunctions = new Map(['parse',
25
- 'eval',
26
- 'deparse',
27
- 'quote',
28
- 'body',
29
- 'formals',
30
- 'body',
31
- 'environment',
32
- 'new.env',
33
- 'assign',
34
- 'get',
35
- 'setGeneric',
36
- 'R6Class'].map(name => [name, fs_1.default.createWriteStream(path_1.default.join(outputPath, `${name}.csv`))]));
37
- for (const [, value] of importantFunctions) {
38
- value.write('filepath,location,namespace,inspected by,classification,notes\n');
39
- }
40
- // we collect only `all-calls`
41
- (0, files_1.readLineByLineSync)(path_1.default.join(featureRoot, `${used_functions_1.AllCallsFileBase}.txt`), (line, lineNumber) => processNextLine(functionsPerFile, lineNumber, info, JSON.parse(String(line)), config, importantFunctions));
42
- for (const [, value] of importantFunctions) {
43
- value.close();
44
- }
45
- importantFunctions.clear();
46
- console.log(` [${(0, time_1.date2string)(new Date())}] Used functions process completed, start to write out function info`);
47
- const fnOutStream = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'function-calls.csv'));
48
- const prefixes = ['total', 'args', 'line-frac'];
49
- const others = prefixes.flatMap(data_1.summarizedMeasurement2CsvHeader).join(',');
50
- fnOutStream.write(`function,unique-projects,unique-files,${others}\n`);
51
- for (const [key, [uniqueProjects, uniqueFiles, total, args, lineFrac]] of functionsPerFile.entries()) {
52
- const totalSum = (0, process_1.summarizeMeasurement)(total.flat(), info.size);
53
- const argsSum = (0, process_1.summarizeMeasurement)(args.flat(), info.size);
54
- const lineFracSum = (0, process_1.summarizeMeasurement)(lineFrac.flat());
55
- // we write in csv style :), we escape the key in case it contains commas (with filenames)etc.
56
- fnOutStream.write(`${JSON.stringify(key ?? 'unknown')},${uniqueProjects.size},${uniqueFiles.size},${(0, data_1.summarizedMeasurement2Csv)(totalSum)},${(0, data_1.summarizedMeasurement2Csv)(argsSum)},${(0, data_1.summarizedMeasurement2Csv)(lineFracSum)}\n`);
57
- }
58
- fnOutStream.close();
59
- }
60
- function writeFunctionCallsMetaInformationToCsv(outputPath, data) {
61
- const out = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'function-calls-meta.csv'));
62
- out.write(`kind,unique-projects,unique-files,${(0, data_1.summarizedMeasurement2CsvHeader)()}\n`);
63
- out.write(`average-call,${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(data.averageCall.flat()))}\n`);
64
- out.write(`nested-calls,${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(data.nestedCalls.flat()))}\n`);
65
- out.write(`deepest-nesting,${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(data.deepestNesting.flat()))}\n`);
66
- out.write(`empty-args,${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(data.emptyArgs.flat()))}\n`);
67
- out.write(`unnamed-calls,${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(data.unnamedCalls.flat()))}\n`);
68
- out.close();
69
- }
70
- function retrieveFunctionCallMetaInformation(info, outputPath) {
71
- const data = {
72
- averageCall: [],
73
- nestedCalls: [],
74
- deepestNesting: [],
75
- emptyArgs: [],
76
- unnamedCalls: [],
77
- args: []
78
- };
79
- for (const meta of info.values()) {
80
- const us = meta.usedFunctions;
81
- data.averageCall.push([us.allFunctionCalls]);
82
- data.nestedCalls.push([us.nestedFunctionCalls]);
83
- data.deepestNesting.push([us.deepestNesting]);
84
- data.emptyArgs.push([(0, numbers_1.bigint2number)(us.args[0])]);
85
- data.unnamedCalls.push([us.unnamedCalls]);
86
- for (const [i, val] of Object.entries(us.args)) {
87
- if (Number(i) !== 0) {
88
- let get = data.args[Number(i)];
89
- if (!get) {
90
- get = (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(() => []);
91
- data.args[Number(i)] = get;
92
- }
93
- (0, common_syntax_probability_1.appendCommonSyntaxTypeCounter)(get, val);
94
- }
95
- }
96
- }
97
- console.log(` [${(0, time_1.date2string)(new Date())}] Used functions metadata reading completed, summarizing and writing to file`);
98
- writeFunctionCallsMetaInformationToCsv(outputPath, data);
99
- for (const [index, arg] of data.args.entries()) {
100
- // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
101
- if (!arg) {
102
- // we treat the first/0-argument entry separate for legacy reasons
103
- continue;
104
- }
105
- const out = fs_1.default.createWriteStream(path_1.default.join(outputPath, `function-calls-arg-${index}.csv`));
106
- out.write(`kind,name,${(0, data_1.summarizedMeasurement2CsvHeader)()}\n`);
107
- for (const [name, vals] of Object.entries(arg)) {
108
- if (Array.isArray(vals)) {
109
- out.write(`${JSON.stringify(name)},"",${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(vals.flat()))}\n`);
110
- }
111
- else {
112
- for (const [keyName, keyValue] of Object.entries(vals)) {
113
- out.write(`${JSON.stringify(name)},${JSON.stringify(keyName)},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(keyValue.flat()))}\n`);
114
- }
115
- }
116
- }
117
- out.close();
118
- }
119
- }
120
- /**
121
- * Note: the summary does not contain a 0 for each function that is _not_ called by a file. Hence, the minimum can not be 0 (division for mean etc. will still be performed on total file count)
122
- */
123
- function postProcess(featureRoot, info, outputPath, config) {
124
- retrieveFunctionCallInformation(featureRoot, info, config, outputPath);
125
- console.log(` [${(0, time_1.date2string)(new Date())}] Used functions reading completed, summarizing info...`);
126
- retrieveFunctionCallMetaInformation(info, outputPath);
127
- }
128
- exports.postProcess = postProcess;
129
- function processNextLine(data, lineNumber, info, line, config, importants) {
130
- if (lineNumber % 2_500 === 0) {
131
- console.log(` [${(0, time_1.date2string)(new Date())}] Used functions processed ${lineNumber} lines`);
132
- }
133
- const [hits, context] = line;
134
- // group hits by fullname
135
- const groupedByFunctionName = new Map();
136
- for (const [name, loc, args, ns, known] of hits) {
137
- const importantWrite = name && importants.get(name);
138
- if (importantWrite) {
139
- importantWrite.write(`${JSON.stringify(context)},${loc?.[0] ?? '?'}:${loc?.[1] ?? '?'},${ns ?? '""'},,,\n`);
140
- }
141
- const fullname = ns && ns !== '' ? `${ns}::${name ?? ''}` : name;
142
- const key = (fullname ?? '') + (known === 1 ? '-' + (context ?? '') : '');
143
- const stats = info.get(context ?? '')?.stats.lines[0].length;
144
- let get = groupedByFunctionName.get(key);
145
- if (!get) {
146
- get = [new Set(), new Set(), [], [], []];
147
- groupedByFunctionName.set(key, get);
148
- }
149
- // we retrieve the first component fo the path
150
- const projectName = context?.split(path_1.default.sep)[config.projectSkip];
151
- get[0].add(projectName ?? '');
152
- get[1].add(context ?? '');
153
- get[2].push(1);
154
- get[3].push(args);
155
- if (loc && stats) {
156
- // we reduce by 1 to get flat 0% if it is the first line
157
- get[4].push(stats === 1 ? 1 : (loc[0] - 1) / (stats - 1));
158
- }
159
- }
160
- for (const [key, info] of groupedByFunctionName.entries()) {
161
- let get = data.get(key);
162
- if (!get) {
163
- get = [new Set(), new Set(), [], [], []];
164
- // an amazing empty structure :D
165
- data.set(key, get);
166
- }
167
- // for total, we only need the number of elements as it will always be one :D
168
- for (const context of info[0]) {
169
- get[0].add(context);
170
- }
171
- for (const context of info[1]) {
172
- get[1].add(context);
173
- }
174
- get[2].push([info[2].length]);
175
- get[3].push(info[3]);
176
- get[4].push(info[4]);
177
- }
178
- }
179
- //# sourceMappingURL=post-process.js.map
@@ -1,24 +0,0 @@
1
- import type { Feature } from '../../feature';
2
- import type { Writable } from 'ts-essentials';
3
- import type { CommonSyntaxTypeCounts } from '../../common-syntax-probability';
4
- declare const initialFunctionUsageInfo: {
5
- allFunctionCalls: number;
6
- args: Record<number, bigint | CommonSyntaxTypeCounts<bigint>>;
7
- /** `a(b(), c(3, d()))` has 3 (`b`, `c`, `d`) */
8
- nestedFunctionCalls: number;
9
- deepestNesting: number;
10
- unnamedCalls: number;
11
- };
12
- export type FunctionUsageInfo = Writable<typeof initialFunctionUsageInfo>;
13
- export declare const AllCallsFileBase = "all-calls";
14
- export declare const usedFunctions: Feature<FunctionUsageInfo>;
15
- export type FunctionCallInformation = [
16
- /** the name of the called function, or undefined if this was an unnamed function call */
17
- name: string | undefined,
18
- location: [line: number, character: number] | undefined,
19
- numberOfArguments: number,
20
- /** whether this was called from a namespace, like `a::b()` */
21
- namespace: string | undefined,
22
- knownDefinitionInFile: 0 | 1
23
- ];
24
- export {};
@@ -1,95 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.usedFunctions = exports.AllCallsFileBase = void 0;
4
- const output_1 = require("../../../output");
5
- const r_bridge_1 = require("../../../../r-bridge");
6
- const dataflow_1 = require("../../../../dataflow");
7
- const common_syntax_probability_1 = require("../../common-syntax-probability");
8
- const post_process_1 = require("./post-process");
9
- const initialFunctionUsageInfo = {
10
- allFunctionCalls: 0,
11
- args: {
12
- // only if called without arguments
13
- 0: 0n,
14
- 1: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)()
15
- },
16
- /** `a(b(), c(3, d()))` has 3 (`b`, `c`, `d`) */
17
- nestedFunctionCalls: 0,
18
- deepestNesting: 0,
19
- unnamedCalls: 0
20
- };
21
- exports.AllCallsFileBase = 'all-calls';
22
- exports.usedFunctions = {
23
- name: 'Used Functions',
24
- description: 'All functions called, split into various sub-categories',
25
- process(existing, input) {
26
- visitCalls(existing, input);
27
- return existing;
28
- },
29
- initialValue: initialFunctionUsageInfo,
30
- postProcess: post_process_1.postProcess
31
- };
32
- function classifyArguments(args, existing) {
33
- if (args.length === 0) {
34
- existing[0]++;
35
- return;
36
- }
37
- let i = 1;
38
- for (const arg of args) {
39
- if (arg === undefined) {
40
- existing[0]++;
41
- continue;
42
- }
43
- existing[i] = (0, common_syntax_probability_1.updateCommonSyntaxTypeCounts)(existing[i] ?? (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(), arg);
44
- i++;
45
- }
46
- }
47
- function visitCalls(info, input) {
48
- const calls = [];
49
- const allCalls = [];
50
- (0, r_bridge_1.visitAst)(input.normalizedRAst.ast, node => {
51
- if (node.type !== "RFunctionCall" /* RType.FunctionCall */) {
52
- return;
53
- }
54
- if (calls.length > 0) {
55
- info.nestedFunctionCalls++;
56
- (0, output_1.appendStatisticsFile)(exports.usedFunctions.name, 'nested-calls', [node.lexeme], input.filepath);
57
- info.deepestNesting = Math.max(info.deepestNesting, calls.length);
58
- }
59
- const dataflowNode = input.dataflow.graph.get(node.info.id);
60
- let hasCallsEdge = false;
61
- if (dataflowNode) {
62
- hasCallsEdge = [...dataflowNode[1].values()].some(e => e.types.has(dataflow_1.EdgeType.Calls));
63
- }
64
- if (node.flavor === 'unnamed') {
65
- info.unnamedCalls++;
66
- (0, output_1.appendStatisticsFile)(exports.usedFunctions.name, 'unnamed-calls', [node.lexeme], input.filepath);
67
- allCalls.push([
68
- undefined,
69
- [node.location.start.line, node.location.start.column],
70
- node.arguments.length,
71
- '',
72
- hasCallsEdge ? 1 : 0
73
- ]);
74
- }
75
- else {
76
- allCalls.push([
77
- node.functionName.lexeme,
78
- [node.location.start.line, node.location.start.column],
79
- node.arguments.length,
80
- node.functionName.namespace ?? '',
81
- hasCallsEdge ? 1 : 0
82
- ]);
83
- }
84
- classifyArguments(node.arguments, info.args);
85
- calls.push(node);
86
- }, node => {
87
- // drop again :D
88
- if (node.type === "RFunctionCall" /* RType.FunctionCall */) {
89
- calls.pop();
90
- }
91
- });
92
- info.allFunctionCalls += allCalls.length;
93
- (0, output_1.appendStatisticsFile)(exports.usedFunctions.name, exports.AllCallsFileBase, allCalls, input.filepath);
94
- }
95
- //# sourceMappingURL=used-functions.js.map
@@ -1 +0,0 @@
1
- export { usedPackages } from './used-packages';
@@ -1,6 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.usedPackages = void 0;
4
- var used_packages_1 = require("./used-packages");
5
- Object.defineProperty(exports, "usedPackages", { enumerable: true, get: function () { return used_packages_1.usedPackages; } });
6
- //# sourceMappingURL=index.js.map
@@ -1,3 +0,0 @@
1
- import type { FeatureStatisticsWithMeta } from '../../feature';
2
- import type { StatisticsSummarizerConfiguration } from '../../../../util/summarizer/statistics/summarizer';
3
- export declare function postProcess(featureRoot: string, info: Map<string, FeatureStatisticsWithMeta>, outputPath: string, config: StatisticsSummarizerConfiguration): void;
@@ -1,121 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.postProcess = void 0;
7
- const post_processing_1 = require("../../post-processing");
8
- const used_packages_1 = require("./used-packages");
9
- const fs_1 = __importDefault(require("fs"));
10
- const path_1 = __importDefault(require("path"));
11
- const data_1 = require("../../../../util/summarizer/benchmark/data");
12
- const process_1 = require("../../../../util/summarizer/benchmark/first-phase/process");
13
- const files_1 = require("../../../../util/files");
14
- const arrays_1 = require("../../../../util/arrays");
15
- const strings_1 = require("../../../../util/strings");
16
- function postProcess(featureRoot, info, outputPath, config) {
17
- const collected = {};
18
- for (const [filepath, data] of info.entries()) {
19
- const value = data.usedPackages;
20
- for (const [key, val] of Object.entries(value)) {
21
- let get = collected[key];
22
- if (!get) {
23
- get = (0, post_processing_1.emptySummarizedWithProject)();
24
- collected[key] = get;
25
- }
26
- get.count.push(val);
27
- if (val > 0) {
28
- (0, post_processing_1.recordFilePath)(get, filepath, config);
29
- }
30
- }
31
- }
32
- const variablesOutStream = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'loading-functions.csv'));
33
- variablesOutStream.write(`kind,unique-projects,unique-files,${(0, data_1.summarizedMeasurement2CsvHeader)()}\n`);
34
- for (const [key, val] of Object.entries(collected)) {
35
- const data = val;
36
- const sum = (0, process_1.summarizeMeasurement)(data.count);
37
- variablesOutStream.write(`${JSON.stringify(key)},${data.uniqueProjects.size},${data.uniqueFiles.size},${(0, data_1.summarizedMeasurement2Csv)(sum)}\n`);
38
- }
39
- variablesOutStream.close();
40
- // now we want to collect the names of the loaded libraries,
41
- // we collect and store them separately (per kind) but also, we want store the summarized results in the end!
42
- const loadedLibraries = new Map();
43
- for (const key of Object.keys(used_packages_1.initialUsedPackageInfos)) {
44
- const data = retrieveDataForLoad(key, featureRoot, outputPath, config);
45
- for (const [name, val] of data.entries()) {
46
- let get = loadedLibraries.get(name);
47
- if (!get) {
48
- get = (0, post_processing_1.emptySummarizedWithProject)();
49
- loadedLibraries.set(name, get);
50
- }
51
- get.count.push(...val.count);
52
- for (const uniqueFile of val.uniqueFiles) {
53
- get.uniqueFiles.add(uniqueFile);
54
- }
55
- for (const uniqueProject of val.uniqueProjects) {
56
- get.uniqueProjects.add(uniqueProject);
57
- }
58
- }
59
- }
60
- const output = path_1.default.join(outputPath, 'all-operators.csv');
61
- const out = fs_1.default.createWriteStream(output);
62
- out.write(`kind,unique-projects,unique-files,${(0, data_1.summarizedMeasurement2CsvHeader)()}\n`);
63
- for (const [key, val] of loadedLibraries.entries()) {
64
- const { count, uniqueProjects, uniqueFiles } = val;
65
- const sum = (0, process_1.summarizeMeasurement)(count);
66
- out.write(`${JSON.stringify(key)},${uniqueProjects.size},${uniqueFiles.size},${(0, data_1.summarizedMeasurement2Csv)(sum)}\n`);
67
- }
68
- out.close();
69
- }
70
- exports.postProcess = postProcess;
71
- // directly writes the results to the output path
72
- function retrieveDataForLoad(operator, readFromPath, outputPath, config) {
73
- const input = path_1.default.join(readFromPath, `${operator}.txt`);
74
- if (!fs_1.default.existsSync(input)) {
75
- // if there is nothing with this, just return :)
76
- return new Map();
77
- }
78
- const collected = new Map();
79
- (0, files_1.readLineByLineSync)(input, (line, lineNumber) => {
80
- if (line.length === 0) {
81
- return;
82
- }
83
- if (lineNumber % 2_500 === 0) {
84
- console.log(` Processing line ${lineNumber} from ${input}`);
85
- }
86
- const [packages, context] = JSON.parse(line.toString());
87
- // first we have to collect what this file gives us
88
- // we normalize surrounding quotation marks
89
- const bag = (0, arrays_1.array2bag)(packages.map(p => {
90
- if ((0, strings_1.startAndEndsWith)(p, '"') || (0, strings_1.startAndEndsWith)(p, "'") || (0, strings_1.startAndEndsWith)(p, '`')) {
91
- return p.slice(1, -1);
92
- }
93
- else {
94
- return p;
95
- }
96
- }));
97
- // now we merge it into the global map (oh gosh this is so horrible
98
- for (const [name, count] of bag.entries()) {
99
- let get = collected.get(name);
100
- if (!get) {
101
- get = (0, post_processing_1.emptySummarizedWithProject)();
102
- collected.set(name, get);
103
- }
104
- get.count.push(count);
105
- if (count > 0) {
106
- (0, post_processing_1.recordFilePath)(get, context, config);
107
- }
108
- }
109
- });
110
- const output = path_1.default.join(outputPath, `${operator}.csv`);
111
- const out = fs_1.default.createWriteStream(output);
112
- out.write(`kind,unique-projects,unique-files,${(0, data_1.summarizedMeasurement2CsvHeader)()}\n`);
113
- for (const [key, val] of collected.entries()) {
114
- const { count, uniqueProjects, uniqueFiles } = val;
115
- const sum = (0, process_1.summarizeMeasurement)(count);
116
- out.write(`${JSON.stringify(key)},${uniqueProjects.size},${uniqueFiles.size},${(0, data_1.summarizedMeasurement2Csv)(sum)}\n`);
117
- }
118
- out.close();
119
- return collected;
120
- }
121
- //# sourceMappingURL=post-process.js.map
@@ -1,16 +0,0 @@
1
- import type { Feature } from '../../feature';
2
- import type { Writable } from 'ts-essentials';
3
- export declare const initialUsedPackageInfos: {
4
- library: number;
5
- require: number;
6
- loadNamespace: number;
7
- requireNamespace: number;
8
- attachNamespace: number;
9
- withinApply: number;
10
- '::': number;
11
- ':::': number;
12
- /** just contains all occurrences where it is impossible to statically determine which package is loaded */
13
- '<loadedByVariable>': number;
14
- };
15
- export type UsedPackageInfo = Writable<typeof initialUsedPackageInfos>;
16
- export declare const usedPackages: Feature<UsedPackageInfo>;
@@ -1,130 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.usedPackages = exports.initialUsedPackageInfos = void 0;
27
- const xpath = __importStar(require("xpath-ts2"));
28
- const output_1 = require("../../../output");
29
- const post_process_1 = require("./post-process");
30
- exports.initialUsedPackageInfos = {
31
- library: 0,
32
- require: 0,
33
- loadNamespace: 0,
34
- requireNamespace: 0,
35
- attachNamespace: 0,
36
- withinApply: 0,
37
- '::': 0,
38
- ':::': 0,
39
- /** just contains all occurrences where it is impossible to statically determine which package is loaded */
40
- '<loadedByVariable>': 0
41
- };
42
- // based on the extraction routine of lintr search for function calls which are not character-loads (we can not trace those...)
43
- const withinApply = xpath.parse(`
44
- //SYMBOL_FUNCTION_CALL[contains(.,"apply")]/../..
45
- //SYMBOL[text()='require' or text()='library' or text()='loadNamespace' or text()='requireNamespace' or text()='attachNamespace']
46
- /../..
47
- `);
48
- // horrible ways I found exploratively like loading within `sapply`
49
- const libraryOrRequire = xpath.parse(`
50
- //SYMBOL_FUNCTION_CALL[text() = $variable]
51
- /parent::expr
52
- /parent::expr[
53
- expr[2][STR_CONST]
54
- or (
55
- expr[2][SYMBOL]
56
- and not(
57
- SYMBOL_SUB[text() = 'character.only']
58
- /following-sibling::expr[1]
59
- /NUM_CONST[text() = 'TRUE' or text() = 'T']
60
- )
61
- )
62
- ]/OP-LEFT-PAREN[1]/following-sibling::expr[1][SYMBOL | STR_CONST]
63
- `);
64
- // there is no except in xpath 1.0?
65
- const packageLoadedWithVariableLoadRequire = xpath.parse(`
66
- //SYMBOL_FUNCTION_CALL[text() = 'library' or text() = 'require']
67
- /parent::expr
68
- /parent::expr[
69
- expr[2][SYMBOL]
70
- and (
71
- SYMBOL_SUB[text() = 'character.only']
72
- /following-sibling::expr[1]
73
- /NUM_CONST[text() = 'TRUE' or text() = 'T']
74
- )
75
- ]/OP-LEFT-PAREN[1]/following-sibling::expr[1][SYMBOL | STR_CONST]
76
- `);
77
- const packageLoadedWithVariableNamespaces = xpath.parse(`
78
- //SYMBOL_FUNCTION_CALL[text() = 'loadNamespace' or text() = 'requireNamespace' or text() = 'attachNamespace']/../following-sibling::expr[1][SYMBOL]
79
- `);
80
- const queryForFunctionCall = xpath.parse(`
81
- //SYMBOL_FUNCTION_CALL[text() = $variable]/../following-sibling::expr[1][STR_CONST]
82
- `);
83
- // otherwise, the parser seems to fail
84
- const queryForNsAccess = xpath.parse(`
85
- //NS_GET[text() = $variable]/../SYMBOL_PACKAGE[1]
86
- |
87
- //NS_GET_INT[text() = $variable]/../SYMBOL_PACKAGE[1]
88
- `);
89
- const queries = [
90
- {
91
- types: ['library', 'require'],
92
- query: libraryOrRequire
93
- },
94
- {
95
- types: ['loadNamespace', 'requireNamespace', 'attachNamespace'],
96
- query: queryForFunctionCall
97
- },
98
- {
99
- types: ['::', ':::'],
100
- query: queryForNsAccess
101
- }
102
- ];
103
- exports.usedPackages = {
104
- name: 'Used Packages',
105
- description: 'All the packages used in the code',
106
- process(existing, input) {
107
- // we will unify in the end, so we can count, group etc. but we do not re-count multiple packages in the same file
108
- for (const q of queries) {
109
- for (const fn of q.types) {
110
- const nodes = q.query.select({ node: input.parsedRAst, variables: { variable: fn } });
111
- existing[fn] += nodes.length;
112
- (0, output_1.appendStatisticsFile)(this.name, fn, nodes, input.filepath, true);
113
- }
114
- }
115
- const nodesForVariableLoad = [
116
- ...packageLoadedWithVariableLoadRequire.select({ node: input.parsedRAst }),
117
- ...packageLoadedWithVariableNamespaces.select({ node: input.parsedRAst })
118
- ];
119
- existing['<loadedByVariable>'] += nodesForVariableLoad.length;
120
- // should not be unique as variables may be repeated, and we have no idea
121
- (0, output_1.appendStatisticsFile)(this.name, '<loadedByVariable>', nodesForVariableLoad, input.filepath);
122
- const withinApplyNodes = withinApply.select({ node: input.parsedRAst });
123
- existing.withinApply += withinApplyNodes.length;
124
- (0, output_1.appendStatisticsFile)(this.name, 'withinApply', withinApplyNodes, input.filepath);
125
- return existing;
126
- },
127
- initialValue: exports.initialUsedPackageInfos,
128
- postProcess: post_process_1.postProcess
129
- };
130
- //# sourceMappingURL=used-packages.js.map
@@ -1 +0,0 @@
1
- export { values } from './values';
@@ -1,6 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.values = void 0;
4
- var values_1 = require("./values");
5
- Object.defineProperty(exports, "values", { enumerable: true, get: function () { return values_1.values; } });
6
- //# sourceMappingURL=index.js.map
@@ -1,3 +0,0 @@
1
- import type { FeatureStatisticsWithMeta } from '../../feature';
2
- import type { StatisticsSummarizerConfiguration } from '../../../../util/summarizer/statistics/summarizer';
3
- export declare function postProcess(featureRoot: string, info: Map<string, FeatureStatisticsWithMeta>, outputPath: string, config: StatisticsSummarizerConfiguration): void;