@flisk/analyze-tracking 0.7.2 → 0.7.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +4 -0
  2. package/bin/cli.js +30 -2
  3. package/package.json +12 -8
  4. package/src/analyze/go/astTraversal.js +121 -0
  5. package/src/analyze/go/constants.js +20 -0
  6. package/src/analyze/go/eventDeduplicator.js +47 -0
  7. package/src/analyze/go/eventExtractor.js +156 -0
  8. package/src/analyze/go/goAstParser/constants.js +39 -0
  9. package/src/analyze/go/goAstParser/expressionParser.js +281 -0
  10. package/src/analyze/go/goAstParser/index.js +52 -0
  11. package/src/analyze/go/goAstParser/statementParser.js +387 -0
  12. package/src/analyze/go/goAstParser/tokenizer.js +196 -0
  13. package/src/analyze/go/goAstParser/typeParser.js +202 -0
  14. package/src/analyze/go/goAstParser/utils.js +99 -0
  15. package/src/analyze/go/index.js +55 -0
  16. package/src/analyze/go/propertyExtractor.js +670 -0
  17. package/src/analyze/go/trackingDetector.js +71 -0
  18. package/src/analyze/go/trackingExtractor.js +54 -0
  19. package/src/analyze/go/typeContext.js +88 -0
  20. package/src/analyze/go/utils.js +215 -0
  21. package/src/analyze/index.js +11 -6
  22. package/src/analyze/javascript/constants.js +115 -0
  23. package/src/analyze/javascript/detectors/analytics-source.js +119 -0
  24. package/src/analyze/javascript/detectors/index.js +10 -0
  25. package/src/analyze/javascript/extractors/event-extractor.js +179 -0
  26. package/src/analyze/javascript/extractors/index.js +13 -0
  27. package/src/analyze/javascript/extractors/property-extractor.js +172 -0
  28. package/src/analyze/javascript/index.js +38 -0
  29. package/src/analyze/javascript/parser.js +126 -0
  30. package/src/analyze/javascript/utils/function-finder.js +123 -0
  31. package/src/analyze/python/index.js +111 -0
  32. package/src/analyze/python/pythonTrackingAnalyzer.py +814 -0
  33. package/src/analyze/ruby/detectors.js +46 -0
  34. package/src/analyze/ruby/extractors.js +258 -0
  35. package/src/analyze/ruby/index.js +51 -0
  36. package/src/analyze/ruby/traversal.js +123 -0
  37. package/src/analyze/ruby/types.js +30 -0
  38. package/src/analyze/ruby/visitor.js +66 -0
  39. package/src/analyze/typescript/constants.js +109 -0
  40. package/src/analyze/typescript/detectors/analytics-source.js +125 -0
  41. package/src/analyze/typescript/detectors/index.js +10 -0
  42. package/src/analyze/typescript/extractors/event-extractor.js +269 -0
  43. package/src/analyze/typescript/extractors/index.js +14 -0
  44. package/src/analyze/typescript/extractors/property-extractor.js +427 -0
  45. package/src/analyze/typescript/index.js +48 -0
  46. package/src/analyze/typescript/parser.js +131 -0
  47. package/src/analyze/typescript/utils/function-finder.js +139 -0
  48. package/src/analyze/typescript/utils/type-resolver.js +208 -0
  49. package/src/generateDescriptions/index.js +81 -0
  50. package/src/generateDescriptions/llmUtils.js +33 -0
  51. package/src/generateDescriptions/promptUtils.js +62 -0
  52. package/src/generateDescriptions/schemaUtils.js +61 -0
  53. package/src/index.js +13 -4
  54. package/src/{fileProcessor.js → utils/fileProcessor.js} +5 -0
  55. package/src/{repoDetails.js → utils/repoDetails.js} +5 -0
  56. package/src/utils/yamlGenerator.js +47 -0
  57. package/src/analyze/analyzeGoFile.js +0 -1164
  58. package/src/analyze/analyzeJsFile.js +0 -87
  59. package/src/analyze/analyzePythonFile.js +0 -42
  60. package/src/analyze/analyzeRubyFile.js +0 -419
  61. package/src/analyze/analyzeTsFile.js +0 -192
  62. package/src/analyze/go2json.js +0 -1069
  63. package/src/analyze/helpers.js +0 -656
  64. package/src/analyze/pythonTrackingAnalyzer.py +0 -541
  65. package/src/generateDescriptions.js +0 -196
  66. package/src/yamlGenerator.js +0 -23
@@ -1,1069 +0,0 @@
1
- const fs = require("fs");
2
-
3
- function go2json() {
4
- let that = this;
5
-
6
- const SIGIL = [
7
- /*TRIPLE*/ "<<=",">>=",
8
- /*DOUBLE*/ "+=","-=","*=","/=","%=","++","--",":=","==","&&","||",">=","<=","<<",">>","&=","^=","|=","!=","<-",
9
- /*SINGLE*/ "=","+","-","*","/","%","{","}","[","]","(",")",",","&","|","!","<",">","^",";",":"
10
- ];
11
- const OPERATOR = SIGIL.filter(x=>!["{","}","[","]",";",":=","="].includes(x));
12
- const DOT = ".";
13
- const WHITESPACE = " \t";
14
- const NEWLINE = "\n\r";
15
- const NUMBER = "01234567890";
16
- const QUOTE = "\"'`";
17
- const PRIMTYPES = ["int","byte","bool","float32","float64","int8","int32","int16","int64","uint8","uint32","uint16","uint64","rune","string"];
18
-
19
- function go2tokens(src){
20
- let ident = "";
21
- let isNum = false;
22
- let tokens = [];
23
- let i = 0;
24
- let line = 1;
25
- let col = 1;
26
- let sigilset = Array.from(new Set(SIGIL.join("")));
27
-
28
- function newlineMaybe(){
29
- if (tokens.length && tokens[tokens.length-1].tag != "newline"){
30
- tokens.push({tag:"newline",value:"\n", line: line, col: col});
31
- }
32
- }
33
- function pushIdent(){
34
- if (ident.length){
35
- tokens.push({tag:isNum?"number":"ident",value:ident, line: line, col: col - ident.length});
36
- ident="";
37
- isNum = false;
38
- }
39
- }
40
-
41
- function advancePosition(char) {
42
- if (NEWLINE.includes(char)) {
43
- line++;
44
- col = 1;
45
- } else {
46
- col++;
47
- }
48
- }
49
- while (i < src.length){
50
- // console.log(i,src[i])
51
- if (WHITESPACE.includes(src[i])){
52
- pushIdent();
53
- advancePosition(src[i]);
54
- i++;
55
- }else if (NEWLINE.includes(src[i])){
56
- pushIdent();
57
- newlineMaybe();
58
- advancePosition(src[i]);
59
- i++;
60
- }else if (src[i]=="/" && src[i+1]=="/"){
61
- var cmt = "";
62
- while (src[i] != "\n" && i < src.length){
63
- cmt += src[i];
64
- advancePosition(src[i]);
65
- i++;
66
- }
67
- // tokens.push({tag:"comment",value:cmt});
68
- newlineMaybe();
69
- if (i < src.length) {
70
- advancePosition(src[i]);
71
- i++;
72
- }
73
- }else if (src[i] == "/" && src[i+1] == "*"){
74
- advancePosition(src[i]);
75
- advancePosition(src[i+1]);
76
- i+=2;
77
- let lvl = 0;
78
- while (true){
79
- if (i > src.length*2){
80
- throw "Unexpected EOF"
81
- }
82
- if (src[i-1]=="/" && src[i] == "*"){
83
- lvl ++;
84
- }
85
- if (src[i-1] == "*" && src[i] == "/"){
86
- if (!lvl){
87
- advancePosition(src[i]);
88
- i++;
89
- break;
90
- }
91
- lvl --;
92
- }
93
- advancePosition(src[i]);
94
- i++;
95
- }
96
- }else if (QUOTE.includes(src[i])){
97
- let startLine = line;
98
- let startCol = col;
99
- let j = i+1;
100
- advancePosition(src[i]); // advance for opening quote
101
- while (true){
102
- if (src[j] == "\\"){
103
- advancePosition(src[j]);
104
- j++;
105
- if (j < src.length) {
106
- advancePosition(src[j]);
107
- j++;
108
- }
109
- }else if (src[j] == src[i]){
110
- advancePosition(src[j]); // advance for closing quote
111
- break;
112
- } else {
113
- advancePosition(src[j]);
114
- j++;
115
- }
116
- }
117
- j++;
118
- tokens.push({tag:src[i]=="'"?"char":"string",value:src.slice(i,j), line: startLine, col: startCol});
119
- i=j;
120
- }else if (src[i] == "." && src[i+1] == "." && src[i+2] == "."){
121
- pushIdent();
122
- tokens.push({tag:"sigil",value:"...", line: line, col: col});
123
- advancePosition(src[i]);
124
- advancePosition(src[i+1]);
125
- advancePosition(src[i+2]);
126
- i+=3;
127
- }else if (sigilset.includes(src[i])){
128
- if (src[i] == "-" || src[i] == "+"){ // e.g. 1e+8 1E-9
129
- if (isNum && ident[ident.length-1]=="e"||ident[ident.length-1]=="E"){
130
- ident += src[i];
131
- advancePosition(src[i]);
132
- i++;
133
- continue;
134
- }
135
- }
136
- pushIdent();
137
- let done = false;
138
- for (var j = 0; j < SIGIL.length; j++){
139
- let l = SIGIL[j].length;
140
- let ok = true;
141
- for (var k = 0; k < l; k++){
142
- if (src[i+k] != SIGIL[j][k]){
143
- ok = false;
144
- break;
145
- }
146
- }
147
- if (ok){
148
- tokens.push({tag:"sigil",value:SIGIL[j], line: line, col: col});
149
- for (let k = 0; k < l; k++) {
150
- advancePosition(src[i+k]);
151
- }
152
- i += l;
153
- done = true;
154
- break;
155
- }
156
- }
157
- }else if (DOT.includes(src[i])){
158
- if (isNum){
159
- ident += src[i];
160
- advancePosition(src[i]);
161
- i++;
162
- }else{
163
- pushIdent();
164
- tokens.push({tag:"sigil",value:DOT, line: line, col: col});
165
- advancePosition(src[i]);
166
- i++;
167
- }
168
- }else if (NUMBER.includes(src[i])){
169
- if (ident.length == 0){
170
- isNum = true;
171
-
172
- }
173
- ident += src[i];
174
- advancePosition(src[i]);
175
- i++;
176
- }else{
177
- ident += src[i];
178
- advancePosition(src[i]);
179
- i++;
180
- }
181
- }
182
- pushIdent();
183
- newlineMaybe();
184
- return tokens;
185
- }
186
-
187
-
188
- function tokens2ast(tokens){
189
- function gtok(i){
190
- return tokens[i]||{};
191
- }
192
-
193
-
194
- function toksHasVal(toks,val){
195
- for (var i = 0; i < toks.length; i++){
196
- if (toks[i].value == val){
197
- return true;
198
- }
199
- }
200
- return false;
201
- }
202
-
203
- function tillNestEndImpl(toks,i,l,r){
204
- let tk = [];
205
- let lvl = 0;
206
-
207
- while (true){
208
- if (i >= toks.length){
209
- return [i,tk]
210
- }
211
- if (toks[i].value == l){
212
- lvl ++;
213
- }else if (toks[i].value == r){
214
- if (lvl == 0){
215
- return [i,tk];
216
- }
217
- lvl --;
218
- }
219
- tk.push(toks[i]);
220
- i++;
221
- }
222
- return [i,tk];
223
- }
224
-
225
-
226
- function parseExpr(toks){
227
-
228
-
229
- let i;
230
- let lvl = 0;
231
- for (i = 0; i < toks.length; i++){
232
- if ("{[(".includes(toks[i].value)){
233
- lvl ++;
234
- }
235
- if (")]}".includes(toks[i].value)){
236
- lvl --;
237
- }
238
- if (toks[i].value == ":="){
239
- if (lvl != 0){
240
- continue;
241
- }
242
- return {
243
- tag:"declare",
244
- names:splitToksBy(toks.slice(0,i),",").map(x=>x[0].value),
245
- value:parseExpr(toks.slice(i+1)),
246
- type:{tag:"auto"},
247
- isconst:false,
248
- }
249
- }
250
- }
251
- lvl = 0;
252
-
253
- for (i = 0; i < toks.length; i++){
254
- if ("{[(".includes(toks[i].value)){
255
- lvl ++;
256
- }
257
- if (")]}".includes(toks[i].value)){
258
- lvl --;
259
- }
260
- if (toks[i].value == "="){
261
- if (lvl != 0){
262
- continue;
263
- }
264
- return {
265
- tag:"assign",
266
- lhs:parseExpr(toks.slice(0,i)),
267
- rhs:parseExpr(toks.slice(i+1)),
268
- }
269
- }
270
- }
271
-
272
- // console.log(splitToksBy(toks,","))
273
- let groups = splitToksBy(toks,",");
274
- if (groups.length > 1){
275
- return {
276
- tag:"tuple",
277
- items:groups.map(parseExpr),
278
- }
279
- }
280
-
281
- let body = [];
282
-
283
- i = 0;
284
- while (i < toks.length){
285
- function tillNestEnd(l,r){
286
- let [j,tk] = tillNestEndImpl(toks,i,l,r);
287
- i = j;
288
- return tk;
289
- }
290
-
291
- if (toks[i].value== "("){
292
-
293
- if (i > 0 && (toks[i-1].tag == "ident") && (toks[i-1].value == "make") && (body.length<2 || body[body.length-2].tag != "access") ){
294
-
295
- i++;
296
- body.pop();
297
- let args = splitToksBy(tillNestEnd("(",")"),",");
298
-
299
- body.push({
300
- tag:"alloc",
301
- type:parseType(args[0]),
302
- size:args[1]?parseExpr(args[1]):null,
303
- });
304
- i++;
305
- }else if (i > 0 && (toks[i-1].tag == "ident" || toks[i-1].value == "]" || toks[i-1].value == "}" || toks[i-1].value == ")")){
306
- let fun = body.pop();
307
- if (fun.tag == "ident" && PRIMTYPES.includes(fun.value)){
308
- i++;
309
- body.push({
310
- tag:"cast",
311
- type:{tag:fun.value},
312
- value:parseExpr(tillNestEnd("(",")"))
313
- })
314
- i++;
315
- }else{
316
- i++;
317
- let startToken = fun.line ? fun : (toks[i-1] || {});
318
- body.push({
319
- tag:"call",
320
- func:fun,
321
- args:splitToksBy(tillNestEnd("(",")"),",").map(parseExpr),
322
- line: startToken.line,
323
- col: startToken.col
324
- });
325
- i++;
326
-
327
- }
328
- }else{
329
- i++;
330
- body.push(parseExpr(tillNestEnd("(",")")));
331
- i++;
332
- }
333
- }else if (toks[i].value == "."){
334
- i++;
335
- // console.log(toks[i])
336
- body.push({
337
- tag:"access",
338
- struct:body.pop(),
339
- member:toks[i].value,
340
- })
341
- i++;
342
- }else if (toks[i].value == "["){
343
- if (i > 0 && (toks[i-1].tag == "ident" || toks[i-1].value == "]")){
344
- i++;
345
- let idx = tillNestEnd("[","]");
346
- let idc = splitToksBy(idx,":");
347
- // console.log(idc);
348
- if (idc.length == 1){
349
- body.push({
350
- tag:"index",
351
- container:body.pop(),
352
- index:parseExpr(idx),
353
- });
354
- }else{
355
- body.push({
356
- tag:"slice",
357
- container:body.pop(),
358
- lo:parseExpr(idc[0]),
359
- hi:parseExpr(idc[1]),
360
- });
361
- }
362
- i++;
363
- }else{
364
- i++;
365
- let size = tillNestEnd("[","]");
366
- let mode = 0;
367
- for (var j = 0; j < toks.length; j++){
368
- if (toks[j].value == "{"){
369
- mode = 0;
370
- break;
371
- }else if (toks[j].value == "("){
372
- mode = 1;
373
- break;
374
- }
375
- }
376
- if (mode==0){
377
- let lit = {
378
- tag:"arraylit",
379
- size:parseExpr(size),
380
- }
381
- i++;
382
- lit.type = parseType(tillNestEnd("}","{"));
383
- i++;
384
- lit.items = splitToksBy(tillNestEnd("{","}"),",");
385
- body.push(lit);
386
- i++;
387
- }else{
388
- i++;
389
- let type = tillNestEnd(")","(");
390
- type = parseType(type);
391
- i++;
392
- let val =parseExpr(tillNestEnd("(",")"));
393
- body.push({
394
- tag:"cast",
395
- type:{tag:"array",size:parseExpr(size),item:type},
396
- value:val
397
- })
398
- i++;
399
-
400
- }
401
- }
402
-
403
- }else if (toks[i].value == "{"){
404
- // console.log(toks,i)
405
- i++;
406
- let cont = tillNestEnd("{","}");
407
- cont = splitToksBy(cont,",")
408
- let fields = [];
409
- for (let j= 0; j < cont.length; j++){
410
- let pair = splitToksBy(cont[j],",");
411
- let lhs = pair[0];
412
- let rhs = pair[1];
413
- if (!rhs){
414
- fields.push({name:null,value:parseExpr(lhs)});
415
- }else{
416
- fields.push({name:lhs[0].value,value:parseExpr(rhs)});
417
- }
418
- }
419
-
420
- let structType = body.pop();
421
- let startToken = structType.line ? structType : (toks[i-1] || {});
422
- body.push({
423
- tag:"structlit",
424
- struct:structType,
425
- fields,
426
- line: startToken.line,
427
- col: startToken.col
428
- });
429
- i++;
430
-
431
- }else if (toks[i].tag == "sigil" && OPERATOR.includes(toks[i].value)){
432
- body.push({tag:"op",value:toks[i].value});
433
- i++;
434
- }else if (toks[i].tag == "newline"){
435
- i++;
436
- }else if (toks[i].value == "func"){
437
- let stmt = {tag:"lambda"};
438
- i++;
439
- i++;
440
- let args = tillNestEnd("(",")");
441
-
442
- args = parseArgs(args);
443
-
444
- stmt.args = args;
445
-
446
- i++;
447
- stmt.returns = [];
448
- if (toks[i].value != "{"){
449
- if (toks[i].value != "("){
450
- stmt.returns.push({name:null,type:parseType(tillNestEnd("}","{"))});
451
- }else{
452
- i++;
453
- stmt.returns = parseRetTypes(tillNestEnd("(",")"));
454
- i++;
455
- }
456
- }
457
- i++;
458
- stmt.body = tokens2ast(tillNestEnd("{","}"));
459
- body.push(stmt);
460
- i++;
461
- }else{
462
- body.push(toks[i]);
463
- i++;
464
- }
465
- }
466
- return {tag:"expr",body};
467
- }
468
- function parseArgs(toks){
469
-
470
- let args = [];
471
-
472
- let i = 0;
473
-
474
- let lvl = 0;
475
- while (i < toks.length){
476
-
477
-
478
- let arg = {};
479
- arg.name = toks[i].value;
480
- i++;
481
- let typ = []
482
- let lvl = 0;
483
- while (i < toks.length){
484
- if (toks[i].value == "("){
485
- lvl ++;
486
- }else if (toks[i].value == ")"){
487
- lvl --;
488
- }else if (toks[i].value == ","){
489
- if (lvl == 0){
490
- break;
491
- }
492
- }
493
- typ.push(toks[i]);
494
- i++;
495
- }
496
- // console.log(typ);
497
- arg.type = parseType(typ);
498
- i++;
499
- args.push(arg);
500
-
501
-
502
-
503
- }
504
- // console.log(args);
505
-
506
- for (i = args.length-1; i >= 0; i--){
507
- if (args[i].type == undefined){
508
- args[i].type = args[i+1].type;
509
- }
510
-
511
- }
512
- return args;
513
- }
514
-
515
- function splitToksBy(toks,delim,delim2){
516
- let groups = [];
517
- let gp = [];
518
- let lvl = 0;
519
- for (let i = 0; i < toks.length; i++){
520
- if (toks[i].value == "{" || toks[i].value == "(" || toks[i].value == "["){
521
- lvl ++;
522
- gp.push(toks[i])
523
- }else if (toks[i].value == "}" || toks[i].value == ")" || toks[i].value == "]"){
524
- lvl --;
525
- gp.push(toks[i])
526
- }else if (toks[i].value == delim && lvl == 0){
527
- groups.push(gp);
528
- gp = [];
529
- }else if (delim2 != undefined && toks[i].value == delim2 && lvl == 0){
530
- groups.push(gp);
531
- gp = [];
532
- }else{
533
- gp.push(toks[i])
534
- }
535
- }
536
- if (groups.length || gp.length){
537
- groups.push(gp);
538
- }
539
- return groups;
540
- }
541
-
542
- function parseRetTypes(toks){
543
- let items = splitToksBy(toks,",");
544
- let simple = true;
545
- for (let j = 0; j < items.length; j++){
546
- if (items[j].length != 1){
547
- if (items[j][0].value != "map" && items[j][0].value != "[" && items[j][0].value != "*"){
548
- simple = false;
549
- }
550
- break;
551
- }
552
- }
553
- if (simple){
554
- return items.map(x=>({name:null,type:parseType(x)}));
555
- }
556
- let ret = items.map(x=>({}));
557
-
558
- for (let j = items.length-1; j >= 0 ; j--){
559
- let name = items[j][0].value;
560
- let type = items[j].slice(1);
561
- if (!type.length){
562
- type = ret[j+1].type;
563
- }else{
564
- type = parseType(type);
565
- }
566
- ret[j].name = name;
567
- ret[j].type = type;
568
- }
569
- return ret;
570
- }
571
-
572
-
573
- function parseType(toks){
574
-
575
- // return toks.map(x=>x.value).join("");
576
- if (toks.length == 1){
577
- return {tag:toks[0].value};
578
- }
579
- let i = 0;
580
- while (i < toks.length){
581
-
582
- function tillNestEnd(l,r){
583
- let [j,tk] = tillNestEndImpl(toks,i,l,r);
584
- i = j;
585
- return tk;
586
- }
587
-
588
- if (toks[i].value == "["){
589
-
590
- let typ = {tag:"array",size:null,item:null};
591
- i++;
592
- typ.size = parseExpr(tillNestEnd("[","]"));
593
- i++;
594
- typ.item = parseType(toks.slice(i));
595
- return typ;
596
-
597
- }else if (toks[i].value == "..."){
598
-
599
- let typ = {tag:"rest",item:null};
600
- i++;
601
- typ.item = parseType(toks.slice(i));
602
- return typ;
603
-
604
- }else if (toks[i].value == "*"){
605
- return {tag:"ptr",item:parseType(toks.slice(i+1))};
606
-
607
- }else if (toks[i].value == "map"){
608
-
609
- let typ = {tag:"map",key:null,value:null};
610
- i+=2;
611
-
612
- let te = tillNestEnd("[","]");
613
-
614
- typ.key = parseType(te);
615
-
616
- i++;
617
- typ.value = parseType(toks.slice(i));
618
-
619
- return typ;
620
- }else if (toks[i].value == "func"){
621
- return {tag:"lambda",...parseFuncSig(toks.slice(i+1))};
622
- }else if (toks[i].value == "interface"){
623
- return {tag:"interface"};
624
- }else if (toks[i].value == "<-" && toks[i+1].value == "chan"){
625
- return {tag:"chan", item:parseType(toks.slice(i+2)), mode:'i'}
626
- }else if (toks[i].value == "chan" && toks[i+1].value == "<-"){
627
- return {tag:"chan", item:parseType(toks.slice(i+2)), mode:'o'}
628
- }else if (toks[i].value == "chan"){
629
- return {tag:"chan", item:parseType(toks.slice(i+1)), mode:'io'}
630
-
631
- }else if (toks[i+1] && toks[i+1].value == "."){
632
-
633
- return {tag:"namespaced",namespace:toks[i].value,item:parseType(toks.slice(i+2))}
634
- }
635
- }
636
- }
637
-
638
- function parseFuncSig(toks){
639
- let lvl = 0;
640
- let k;
641
- for (k = 1; k < toks.length; k++){
642
-
643
- if (toks[k].value == "("){
644
- lvl ++;
645
- }else if (toks[k].value == ")"){
646
- if (lvl == 0){
647
- break;
648
- }
649
- lvl --;
650
- }
651
-
652
- }
653
-
654
- let args = toks.slice(1,k);
655
-
656
- args = parseRetTypes(args);
657
- let rets = toks.slice(k+1);
658
- if (rets.length){
659
- while(rets[0].value == "("){
660
- rets = rets.slice(1,-1);
661
- }
662
- rets = parseRetTypes(rets);
663
- }else{
664
- rets = [];
665
- }
666
- return {args:args,returns:rets}
667
- }
668
-
669
-
670
- let tree = [];
671
- let i = 0;
672
- while (i < tokens.length){
673
- // console.log(i,tokens[i])
674
- function tillStmtEnd(){
675
- let toks = [];
676
- let lvl = 0;
677
- while (true){
678
-
679
- if (i >= tokens.length){
680
- return toks;
681
- }
682
- // if (gtok(i).tag == undefined){
683
- // // process.exit();
684
- // }
685
- if (gtok(i).tag == "sigil" && gtok(i).value == ";"){
686
- break;
687
- }
688
- if ("[{(".includes(gtok(i).value)){
689
- lvl++;
690
- }else if (")}]".includes(gtok(i).value)){
691
- lvl--;
692
- }else if (gtok(i).tag == "newline"){
693
- if (lvl == 0){
694
- if (gtok(i-1).tag != "sigil" ||
695
- gtok(i-1).value == ";" ||
696
- gtok(i-1).value == "++" ||
697
- gtok(i-1).value == "--" ||
698
- "}])".includes(gtok(i-1).value)){
699
- break;
700
- }
701
- }
702
- }
703
- toks.push(tokens[i]);
704
- i++;
705
- }
706
- return toks;
707
- }
708
-
709
- function tillNestEnd(l,r){
710
- let [j,tk] = tillNestEndImpl(tokens,i,l,r);
711
- i = j;
712
- return tk;
713
- }
714
-
715
- if (gtok(i).tag == "ident" && gtok(i).value == "package"){
716
- tree.push({tag:"package",name:gtok(i+1).value});
717
- i+=2;
718
- }else if (gtok(i).tag == "ident" && gtok(i).value == "type"){
719
- if (gtok(i+2).value == "struct"){
720
- let stmt = {tag:"typedef",name:gtok(i+1).value,fields:[],embeds:[]};
721
- i += 4;
722
-
723
- let lns = splitToksBy(tillNestEnd("{","}"),"\n",";");
724
- for (let j = 0; j < lns.length; j++){
725
- let names = splitToksBy(lns[j],",");
726
- if (!names.length){
727
- continue;
728
- }
729
- if (names.length == 1 && names[0].length == 1){
730
- stmt.embeds.push(names[0][0].value);
731
-
732
- }else{
733
- let type = names[names.length-1].slice(1);
734
-
735
- type = parseType(type);
736
-
737
- for (let k = 0; k < names.length; k++){
738
- stmt.fields.push({name:names[k][0].value,type});
739
- }
740
- }
741
-
742
- }
743
- i++;
744
-
745
- tree.push(stmt);
746
- }else if (gtok(i+2).value == "interface"){
747
- let stmt = {tag:"interface",name:gtok(i+1).value,methods:[]};
748
- i += 4;
749
-
750
- let lns = splitToksBy(tillNestEnd("{","}"),"\n",";");
751
-
752
- for (let j = 0; j < lns.length; j++){
753
-
754
- let name = lns[j][0];
755
- if (!name){
756
- continue;
757
- }
758
- name = name.value;
759
- let sig = Object.assign({name},parseFuncSig(lns[j].slice(1)));
760
-
761
- stmt.methods.push(sig)
762
-
763
- }
764
- tree.push(stmt);
765
-
766
- i++;
767
- }else{
768
- let stmt = {tag:"typealias",name:gtok(i+1).value};
769
- i += 2;
770
- let typ = tillStmtEnd();
771
- stmt.value = parseType(typ);
772
- tree.push(stmt);
773
- }
774
- }else if (gtok(i).tag == "ident" && gtok(i).value == "func"){
775
- let stmt = {tag:"func",receiver:null};
776
- if (gtok(i+1).value == "("){
777
- stmt.receiver = {name:gtok(i+2).value};
778
- i += 3;
779
- stmt.receiver.type=parseType(tillNestEnd("(",")"));
780
- }
781
-
782
- i++;
783
- stmt.name = gtok(i).value;
784
- i+=2;
785
- let args = tillNestEnd("(",")");
786
-
787
- args = parseArgs(args);
788
-
789
- stmt.args = args;
790
-
791
- i++;
792
- stmt.returns = [];
793
- if (gtok(i).value != "{"){
794
- if (gtok(i).value != "("){
795
- stmt.returns.push({name:null,type:parseType(tillNestEnd("}","{"))});
796
- }else{
797
- i++;
798
- stmt.returns = parseRetTypes(tillNestEnd("(",")"));
799
- i++;
800
- }
801
- }
802
- i++;
803
- stmt.body = tokens2ast(tillNestEnd("{","}"));
804
- tree.push(stmt);
805
- i++;
806
- }else if (gtok(i).tag == "ident" && gtok(i).value == "if"){
807
- let stmt = {tag:"if"}
808
- i+=1;
809
- let cond = tillNestEnd("}","{");
810
- let conds = splitToksBy(cond,";");
811
- stmt.prepare = conds.slice(0,-1).map(parseExpr);
812
- stmt.condition = parseExpr(conds[conds.length-1]);
813
- i++;
814
- stmt.body = tokens2ast(tillNestEnd("{","}"));
815
- tree.push(stmt);
816
- i++;
817
-
818
- }else if (gtok(i).tag == "ident" && gtok(i).value == "else"){
819
- if (gtok(i+1).tag == "ident" && gtok(i+1).value == "if"){
820
- let stmt = {tag:"elseif"};
821
- i += 2;
822
- let cond = tillNestEnd("}","{");
823
- stmt.condition = parseExpr(cond);
824
- i++;
825
- stmt.body = tokens2ast(tillNestEnd("{","}"));
826
- tree.push(stmt);
827
- i++;
828
- }else{
829
- let stmt = {tag:"else"};
830
- i ++;
831
- i++;
832
- stmt.body = tokens2ast(tillNestEnd("{","}"));
833
- tree.push(stmt);
834
- i++;
835
- }
836
-
837
- }else if (gtok(i).tag == "ident" && gtok(i).value == "for"){
838
- let stmt = {tag:"for"}
839
- i+=1;
840
- let head = tillNestEnd("}","{");
841
-
842
- if (toksHasVal(head,"range")){
843
- stmt.tag = "foreach";
844
- let [lhs,rhs] = splitToksBy(head,":=");
845
-
846
- stmt.names = splitToksBy(lhs,",").map(x=>x[0].value);
847
- stmt.container = parseExpr(rhs.slice(1));
848
- i++;
849
- }else{
850
- stmt.headers = splitToksBy(head,";").map(parseExpr);
851
- i++;
852
- }
853
- stmt.body = tokens2ast(tillNestEnd("{","}"));
854
- tree.push(stmt);
855
- i++;
856
-
857
- }else if (gtok(i).tag == "ident" && (gtok(i).value == "var" || gtok(i).value == "const")){
858
- let isconst = (gtok(i).value == "const");
859
-
860
- if (gtok(i+1).value == "("){
861
- i++;
862
- let toks = tillStmtEnd().slice(1,-1);
863
- let lns = splitToksBy(toks,"\n",";").filter(x=>x.length);
864
-
865
- let lastval = null;
866
- let iota = 0;
867
- function inferval(rhs){
868
- if (rhs == null){
869
- if (lastval){
870
- rhs = lastval.slice();
871
- }else{
872
- return null; //probably invalid go, but whatever
873
- }
874
- }
875
- lastval = rhs.slice();
876
- let diduseiota = false;
877
- for (let j = 0; j < rhs.length; j++){
878
- if (rhs[j].value == "iota"){
879
- rhs[j] = {tag:"number",value:`${iota}`};
880
- diduseiota = true;
881
- }
882
- }
883
- if (diduseiota){
884
- iota ++;
885
- }
886
- return parseExpr(rhs);
887
- }
888
- for (let j = 0; j < lns.length; j++){
889
- let sides = splitToksBy(lns[j],"=");
890
-
891
- let lhs = sides[0];
892
- let type = lhs.slice(1);
893
-
894
- if (type.length){
895
- type = parseType(type);
896
- }else{
897
- type = null;
898
- }
899
- let rhs = sides[1];
900
- let stmt = {
901
- tag:"declare",
902
- names:[lhs[0].value],
903
- value:inferval(rhs),
904
- type:type||{tag:"auto"},
905
- isconst,
906
- };
907
- tree.push(stmt);
908
- }
909
- i++;
910
-
911
- }else{
912
-
913
- i++;
914
- let toks = tillStmtEnd();
915
- let sides = splitToksBy(toks,"=");
916
- let lhs = sides[0];
917
- let rhs = sides[1];
918
-
919
- let names = splitToksBy(lhs,",");
920
-
921
-
922
- let type = names[names.length-1].slice(1);
923
-
924
- if (type){
925
- type = parseType(type);
926
- }
927
-
928
- names = names.map(x=>x[0].value);
929
- let stmt = {
930
- tag:"declare",
931
- names,
932
- value:(!rhs)?null:parseExpr(rhs),
933
- type:type||{tag:"auto"},
934
- isconst,
935
- };
936
-
937
- tree.push(stmt);
938
- i++;
939
- }
940
- }else if (gtok(i).tag == "ident" && (gtok(i).value == "switch")){
941
- let stmt = {tag:"switch"}
942
- i+=1;
943
- let cond = tillNestEnd("}","{");
944
- stmt.condition = parseExpr(cond);
945
- i++;
946
- let body = tillNestEnd("{","}");
947
-
948
- stmt.cases = [];
949
- let j = 0;
950
- let s0=null;
951
- let s1=null;
952
- while (j < body.length){
953
- if (body[j].value=="case" || body[j].value == "default"){
954
- if (s0 == null){
955
- s1 = null;
956
- s0 = [];
957
- }else{
958
- if (s0.length){
959
- stmt.cases.push({
960
- tag:"case",
961
- condition:parseExpr(s0),
962
- body:tokens2ast(s1),
963
- })
964
- }else{
965
- stmt.cases.push({
966
- tag:"default",
967
- body:tokens2ast(s1),
968
- })
969
- }
970
- s0 = [];
971
- s1 = null;
972
- }
973
- }else if (body[j].value==":"){
974
- s1 = [];
975
- }else{
976
- if (s1 != null){
977
- s1.push(body[j]);
978
- }else if (s0 != null){
979
- s0.push(body[j]);
980
- }
981
- }
982
- j++;
983
- }
984
- if (s0 != null){
985
- if (s0.length){
986
- stmt.cases.push({
987
- tag:"case",
988
- condition:parseExpr(s0),
989
- body:tokens2ast(s1),
990
- })
991
- }else{
992
- stmt.cases.push({
993
- tag:"default",
994
- body:tokens2ast(s1),
995
- })
996
- }
997
- }
998
-
999
- tree.push(stmt);
1000
- i++;
1001
-
1002
- }else if (gtok(i).tag == "newline"){
1003
- i++;
1004
- }else if (gtok(i).tag == "comment"){
1005
- tree.push({tag:"comment",text:gtok(i).value});
1006
- i++;
1007
- }else if (gtok(i).value == "return"){
1008
- i++;
1009
- tree.push({tag:"return",value:parseExpr(tillStmtEnd())})
1010
- i++;
1011
- }else if (gtok(i).value == "import"){
1012
- i++;
1013
- let imps = tillStmtEnd();
1014
- // imps = imps.filter(x=>x.tag!="newline");
1015
-
1016
- for (let j = 0; j < imps.length-1; j++){
1017
- if (imps[j].tag == "string"){
1018
- tree.push({tag:"import",value:imps[j].value})
1019
- }
1020
- }
1021
- i++;
1022
-
1023
- }else if (gtok(i).value == "go"){
1024
- i++;
1025
- let e = tillStmtEnd();
1026
- tree.push({tag:"invoke",func:parseExpr(e)})
1027
- i++;
1028
- }else if (gtok(i).value == "defer"){
1029
- i++;
1030
- let e = tillStmtEnd();
1031
- tree.push({tag:"defer",expr:parseExpr(e)})
1032
- i++;
1033
- }else{
1034
- let toks = tillStmtEnd();
1035
- // console.log(toks);
1036
- let stmt = parseExpr(toks);
1037
- if (stmt.tag == "expr"){
1038
- stmt = {
1039
- tag:"exec",
1040
- expr:stmt,
1041
- };
1042
- }
1043
- tree.push(stmt);
1044
- i++;
1045
- }
1046
- }
1047
- return tree;
1048
- }
1049
-
1050
- function go2ast(src){
1051
- var tokens = go2tokens(src);
1052
- // tokens.map(x=>console.log(x));
1053
- var tree = tokens2ast(tokens);
1054
- return tree;
1055
- }
1056
-
1057
- this.go2ast = go2ast;
1058
- this.go2tokens = go2tokens;
1059
- this.tokens2ast = tokens2ast;
1060
- }
1061
-
1062
- function extractGoAST(src) {
1063
- const parser = new go2json();
1064
- const tokens = parser.go2tokens(src);
1065
- const tree = parser.tokens2ast(tokens);
1066
- return tree;
1067
- }
1068
-
1069
- module.exports = { extractGoAST };