@appthreat/atom-parsetools 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/astgen.js +297 -57
- package/package.json +8 -6
- package/plugins/composer/installed.php +6 -6
- package/plugins/rubyastgen/bundle/ruby/4.0.0/extensions/x86_64-linux/4.0.0/prism-1.9.0/gem_make.out +5 -5
- package/plugins/rubyastgen/bundle/ruby/4.0.0/extensions/x86_64-linux/4.0.0/racc-1.8.1/gem_make.out +5 -5
package/README.md
CHANGED
|
@@ -7,6 +7,10 @@ This package hosts a collection of parsing tools that complement the `@appthreat
|
|
|
7
7
|
- rbastgen - Generates AST for Ruby projects using the AppThreat's `ruby_ast_gen` gem
|
|
8
8
|
- scalasem - Generates a custom semantics slice for Scala Projects by utilising scalac command.
|
|
9
9
|
|
|
10
|
+
## Runtime support
|
|
11
|
+
|
|
12
|
+
These tools run on both [Node.js](https://nodejs.org) (>= 22, required by `@babel/parser` 8) and [Bun](https://bun.sh). All commands and the accompanying regression test-suite are exercised under both runtimes in CI, so the commands below can be invoked with either `node` or `bun` interchangeably (for example `bun astgen.js -i .`).
|
|
13
|
+
|
|
10
14
|
## Command usages
|
|
11
15
|
|
|
12
16
|
### astgen
|
|
@@ -25,6 +29,16 @@ Options:
|
|
|
25
29
|
-h Show help [boolean]
|
|
26
30
|
```
|
|
27
31
|
|
|
32
|
+
#### Environment variables
|
|
33
|
+
|
|
34
|
+
| Variable | Default | Purpose |
|
|
35
|
+
| --------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
36
|
+
| `ASTGEN_TYPE_WORKERS` | `1` (off) | Number of worker threads for the TypeScript type-generation phase, or `auto` to derive it from the available CPUs. The TypeScript checker is single-threaded, so parallelism comes from sharding files across workers, each building its own program. **Opt-in:** sharding changes TypeScript's internal type-id ordering, which reorders the members of a small number of inferred union types (e.g. `A \| B` → `B \| A`; semantically identical). Leave unset for byte-identical output; set it (e.g. `auto` or `8`) to trade that cosmetic reordering for a large speedup on big projects. |
|
|
37
|
+
| `ASTGEN_INCLUDE_TEST_FILES` | `false` | When `true`, do not exclude test files (`*.poku.js`, `*.test.*`, `*.spec.*`, `*.e2e.*`, `__tests__/`, `__mocks__/`) from AST and type generation. They are excluded by default because they are typically the heaviest, lowest-value inputs for type generation. |
|
|
38
|
+
| `ASTGEN_CONCURRENCY` | `10` | Chunk size for the in-thread file loop (bounds peak memory between `gc()` passes). |
|
|
39
|
+
| `ASTGEN_INCLUDE_NODE_MODULES_BUNDLES` | `false` | When `true`, also parse bundled entrypoints inside `node_modules` (files matching `*.(bundle\|dist\|index\|min\|app).(js\|cjs\|mjs)`). Off by default; `node_modules` is otherwise skipped entirely. |
|
|
40
|
+
| `ASTGEN_IGNORE_DIRS` | unset | Comma/space-separated list of directories to ignore. As a side effect, when it is set and does **not** contain `node_modules`, the `node_modules` bundle entrypoints above are included (equivalent to `ASTGEN_INCLUDE_NODE_MODULES_BUNDLES=true`). |
|
|
41
|
+
|
|
28
42
|
### phpastgen
|
|
29
43
|
|
|
30
44
|
```text
|
package/astgen.js
CHANGED
|
@@ -4,8 +4,15 @@ import { join, dirname, relative, resolve, basename } from "path";
|
|
|
4
4
|
import { fileURLToPath } from "url";
|
|
5
5
|
import { parse } from "@babel/parser";
|
|
6
6
|
import { parse as parseHermes } from "hermes-parser";
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
// TypeScript 7.0 (the native Go port) ships without a programmatic compiler
|
|
8
|
+
// API, which astgen relies on heavily (createProgram, getTypeChecker,
|
|
9
|
+
// forEachChild, SyntaxKind, TypeFormatFlags, ...). The officially supported
|
|
10
|
+
// bridge until the API returns in TS 7.1 is the @typescript/typescript6
|
|
11
|
+
// package, which is the same TS 6 engine and therefore keeps AST shapes and
|
|
12
|
+
// type-inference accuracy identical.
|
|
13
|
+
import tsc from "@typescript/typescript6";
|
|
14
|
+
import { tmpdir, cpus, availableParallelism } from "os";
|
|
15
|
+
import { Worker, isMainThread, parentPort, workerData } from "worker_threads";
|
|
9
16
|
import {
|
|
10
17
|
readFileSync,
|
|
11
18
|
mkdirSync,
|
|
@@ -18,7 +25,11 @@ import {
|
|
|
18
25
|
} from "fs";
|
|
19
26
|
import { getAllFiles } from "@appthreat/atom-common";
|
|
20
27
|
|
|
21
|
-
|
|
28
|
+
// Printed by `astgen --version`. Downstream frontends (e.g. chen's jssrc2cpg)
|
|
29
|
+
// fold this into their parse-cache fingerprint, so it MUST be bumped whenever
|
|
30
|
+
// the emitted AST/type shape changes — otherwise stale cached parses from an
|
|
31
|
+
// older astgen are silently reused. Bumped for the Babel 8 AST-shape change.
|
|
32
|
+
const ASTGEN_VERSION = "4.1.0";
|
|
22
33
|
|
|
23
34
|
const HELP_TEXT = `Options:
|
|
24
35
|
-i, --src Source directory [default: "."]
|
|
@@ -381,6 +392,25 @@ const babelSafeFlowParserOptions = {
|
|
|
381
392
|
]
|
|
382
393
|
};
|
|
383
394
|
|
|
395
|
+
// Test files (e.g. *.poku.js, *.test.ts, *.spec.js, __tests__/*) are
|
|
396
|
+
// test-runner artifacts that are typically the heaviest, lowest-value inputs
|
|
397
|
+
// for type generation (each is often a full twin of a source module wrapped in
|
|
398
|
+
// test scaffolding). They are excluded by default to keep the type-generation
|
|
399
|
+
// phase scalable. Set ASTGEN_INCLUDE_TEST_FILES=true to restore them (e.g. when
|
|
400
|
+
// a downstream consumer wants test files analysed).
|
|
401
|
+
const shouldIncludeTestFiles =
|
|
402
|
+
process.env?.ASTGEN_INCLUDE_TEST_FILES === "true";
|
|
403
|
+
|
|
404
|
+
const TEST_FILE_EXT = "(?:js|jsx|cjs|mjs|ts|tsx|mts|cts)";
|
|
405
|
+
const TEST_FILE_PATTERN = new RegExp(
|
|
406
|
+
`(?:\\.(?:poku|test|spec|e2e|integration|it)\\.${TEST_FILE_EXT}$` +
|
|
407
|
+
`|[\\\\/]__(?:tests|mocks)__[\\\\/])`,
|
|
408
|
+
"i"
|
|
409
|
+
);
|
|
410
|
+
|
|
411
|
+
const isExcludedTestFile = (file) =>
|
|
412
|
+
!shouldIncludeTestFiles && TEST_FILE_PATTERN.test(file);
|
|
413
|
+
|
|
384
414
|
const shouldIncludeNodeModulesBundles =
|
|
385
415
|
process.env?.ASTGEN_INCLUDE_NODE_MODULES_BUNDLES === "true" ||
|
|
386
416
|
(process.env?.ASTGEN_IGNORE_DIRS &&
|
|
@@ -426,7 +456,9 @@ const getAllSrcJSAndTSFiles = (src) => {
|
|
|
426
456
|
// Step 2: Combine both lists
|
|
427
457
|
return Promise.all([allFilesPromise, bundledFilesPromise]).then(
|
|
428
458
|
([allFiles, bundledFiles]) =>
|
|
429
|
-
[...new Set([...allFiles, ...bundledFiles])]
|
|
459
|
+
[...new Set([...allFiles, ...bundledFiles])]
|
|
460
|
+
.filter((file) => !isExcludedTestFile(file))
|
|
461
|
+
.sort()
|
|
430
462
|
);
|
|
431
463
|
};
|
|
432
464
|
|
|
@@ -1087,6 +1119,33 @@ function createTsc(srcFiles, src) {
|
|
|
1087
1119
|
const typeChecker = program.getTypeChecker();
|
|
1088
1120
|
const seenTypes = new Map();
|
|
1089
1121
|
|
|
1122
|
+
// Return-type inference below can re-walk the body of the *same* function
|
|
1123
|
+
// declaration once per referencing call site / initializer (e.g. a helper
|
|
1124
|
+
// called 50 times triggers 50 identical body walks). Since the checker
|
|
1125
|
+
// state is fixed for the lifetime of this program, the inferred string for
|
|
1126
|
+
// a given node is deterministic, so we memoize by node to remove the
|
|
1127
|
+
// redundant re-walks without changing any output. A sentinel distinguishes
|
|
1128
|
+
// "computed and produced undefined" from "not yet computed".
|
|
1129
|
+
const RETURN_TYPE_NOT_COMPUTED = Symbol("returnTypeNotComputed");
|
|
1130
|
+
const memoizeByNode = (fn) => {
|
|
1131
|
+
const cache = new WeakMap();
|
|
1132
|
+
return (node, ...rest) => {
|
|
1133
|
+
if (!node || typeof node !== "object" || rest.length > 0) {
|
|
1134
|
+
return fn(node, ...rest);
|
|
1135
|
+
}
|
|
1136
|
+
const cached = cache.get(node);
|
|
1137
|
+
if (cached !== undefined) {
|
|
1138
|
+
return cached === RETURN_TYPE_NOT_COMPUTED ? undefined : cached;
|
|
1139
|
+
}
|
|
1140
|
+
const result = fn(node);
|
|
1141
|
+
cache.set(
|
|
1142
|
+
node,
|
|
1143
|
+
result === undefined ? RETURN_TYPE_NOT_COMPUTED : result
|
|
1144
|
+
);
|
|
1145
|
+
return result;
|
|
1146
|
+
};
|
|
1147
|
+
};
|
|
1148
|
+
|
|
1090
1149
|
const safeTypeToString = (type, context) => {
|
|
1091
1150
|
try {
|
|
1092
1151
|
return normalizeTypeString(
|
|
@@ -1218,7 +1277,7 @@ function createTsc(srcFiles, src) {
|
|
|
1218
1277
|
return collectedTypes;
|
|
1219
1278
|
};
|
|
1220
1279
|
|
|
1221
|
-
const
|
|
1280
|
+
const inferAsyncReturnTypeFromBodyImpl = (node) => {
|
|
1222
1281
|
if (!node.body) {
|
|
1223
1282
|
return undefined;
|
|
1224
1283
|
}
|
|
@@ -1240,6 +1299,9 @@ function createTsc(srcFiles, src) {
|
|
|
1240
1299
|
? unionType
|
|
1241
1300
|
: `Promise<${unionType}>`;
|
|
1242
1301
|
};
|
|
1302
|
+
const inferAsyncReturnTypeFromBody = memoizeByNode(
|
|
1303
|
+
inferAsyncReturnTypeFromBodyImpl
|
|
1304
|
+
);
|
|
1243
1305
|
|
|
1244
1306
|
const buildFunctionSignatureType = (node, returnTypeStr) => {
|
|
1245
1307
|
if (!node.parameters) {
|
|
@@ -1295,7 +1357,7 @@ function createTsc(srcFiles, src) {
|
|
|
1295
1357
|
return undefined;
|
|
1296
1358
|
};
|
|
1297
1359
|
|
|
1298
|
-
const
|
|
1360
|
+
const inferFunctionDeclarationReturnTypeImpl = (declaration) => {
|
|
1299
1361
|
const signature = typeChecker.getSignatureFromDeclaration(declaration);
|
|
1300
1362
|
if (!signature) {
|
|
1301
1363
|
return undefined;
|
|
@@ -1317,8 +1379,11 @@ function createTsc(srcFiles, src) {
|
|
|
1317
1379
|
}
|
|
1318
1380
|
return inferredType;
|
|
1319
1381
|
};
|
|
1382
|
+
const inferFunctionDeclarationReturnType = memoizeByNode(
|
|
1383
|
+
inferFunctionDeclarationReturnTypeImpl
|
|
1384
|
+
);
|
|
1320
1385
|
|
|
1321
|
-
const
|
|
1386
|
+
const inferAsyncReturnTypeFromSyntaxBodyImpl = (node) => {
|
|
1322
1387
|
if (!node?.body) {
|
|
1323
1388
|
return undefined;
|
|
1324
1389
|
}
|
|
@@ -1361,6 +1426,9 @@ function createTsc(srcFiles, src) {
|
|
|
1361
1426
|
? unionType
|
|
1362
1427
|
: `Promise<${unionType}>`;
|
|
1363
1428
|
};
|
|
1429
|
+
const inferAsyncReturnTypeFromSyntaxBody = memoizeByNode(
|
|
1430
|
+
inferAsyncReturnTypeFromSyntaxBodyImpl
|
|
1431
|
+
);
|
|
1364
1432
|
|
|
1365
1433
|
const addType = (node, currentSeenTypes = seenTypes) => {
|
|
1366
1434
|
// STRUCTURAL/CONTAINER NODES
|
|
@@ -1761,64 +1829,209 @@ function createTsc(srcFiles, src) {
|
|
|
1761
1829
|
}
|
|
1762
1830
|
}
|
|
1763
1831
|
|
|
1832
|
+
/**
|
|
1833
|
+
* Expand the set of output files to include the source files that tsconfig
|
|
1834
|
+
* pulls in (that live under the source root), mirroring the program root names.
|
|
1835
|
+
* Computed from the parsed tsconfig alone so no TypeScript program has to be
|
|
1836
|
+
* built on the main thread.
|
|
1837
|
+
*/
|
|
1838
|
+
const expandSrcFilesWithRootNames = (srcFiles, rootNames, src) => {
|
|
1839
|
+
if (!rootNames) {
|
|
1840
|
+
return srcFiles;
|
|
1841
|
+
}
|
|
1842
|
+
const srcRoot = resolve(src);
|
|
1843
|
+
const srcFileByResolvedPath = new Map(
|
|
1844
|
+
srcFiles.map((file) => [resolve(file), file])
|
|
1845
|
+
);
|
|
1846
|
+
for (const file of rootNames) {
|
|
1847
|
+
const resolvedFile = resolve(file);
|
|
1848
|
+
if (
|
|
1849
|
+
resolvedFile.startsWith(srcRoot) &&
|
|
1850
|
+
/\.(?:js|jsx|cjs|mjs|ts|tsx|mts|cts)$/.test(file) &&
|
|
1851
|
+
!isExcludedTestFile(file) &&
|
|
1852
|
+
!srcFileByResolvedPath.has(resolvedFile)
|
|
1853
|
+
) {
|
|
1854
|
+
srcFileByResolvedPath.set(
|
|
1855
|
+
resolvedFile,
|
|
1856
|
+
join(src, relative(srcRoot, resolvedFile))
|
|
1857
|
+
);
|
|
1858
|
+
}
|
|
1859
|
+
}
|
|
1860
|
+
return [...srcFileByResolvedPath.values()].sort();
|
|
1861
|
+
};
|
|
1862
|
+
|
|
1863
|
+
const runGc = () => {
|
|
1864
|
+
if (typeof globalThis.gc === "function") {
|
|
1865
|
+
try {
|
|
1866
|
+
globalThis.gc();
|
|
1867
|
+
} catch (e) {
|
|
1868
|
+
// ignore
|
|
1869
|
+
}
|
|
1870
|
+
} else if (typeof Bun !== "undefined" && typeof Bun.gc === "function") {
|
|
1871
|
+
try {
|
|
1872
|
+
Bun.gc(true);
|
|
1873
|
+
} catch (e) {
|
|
1874
|
+
// ignore
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
};
|
|
1878
|
+
|
|
1879
|
+
/**
|
|
1880
|
+
* Process a list of files (AST + optional type generation) on the current
|
|
1881
|
+
* thread, in memory-bounded chunks. `ts` is the shared TypeScript program
|
|
1882
|
+
* instance (or undefined when type generation is disabled).
|
|
1883
|
+
*/
|
|
1884
|
+
const processFilesInline = async (srcFiles, options, ts) => {
|
|
1885
|
+
const CONCURRENCY_LIMIT = Math.max(
|
|
1886
|
+
1,
|
|
1887
|
+
Number.parseInt(process.env.ASTGEN_CONCURRENCY || "10", 10) || 10
|
|
1888
|
+
);
|
|
1889
|
+
for (let i = 0; i < srcFiles.length; i += CONCURRENCY_LIMIT) {
|
|
1890
|
+
const chunk = srcFiles.slice(i, i + CONCURRENCY_LIMIT);
|
|
1891
|
+
await Promise.all(chunk.map((file) => processFile(file, options, ts)));
|
|
1892
|
+
runGc();
|
|
1893
|
+
}
|
|
1894
|
+
};
|
|
1895
|
+
|
|
1896
|
+
/**
|
|
1897
|
+
* Decide how many worker threads to use for the type-generation phase.
|
|
1898
|
+
* The TypeScript checker is single-threaded and CPU-bound, so real speedups
|
|
1899
|
+
* only come from running independent programs on separate threads. Each worker
|
|
1900
|
+
* builds its own full program (needed for cross-file type resolution).
|
|
1901
|
+
*
|
|
1902
|
+
* IMPORTANT: parallelism is OPT-IN. When files are sharded across workers,
|
|
1903
|
+
* TypeScript's per-program type-id assignment order changes, which reorders the
|
|
1904
|
+
* members of a small number of inferred union types (e.g. `A | B` -> `B | A`;
|
|
1905
|
+
* semantically identical, textually different). To keep the default output
|
|
1906
|
+
* byte-identical for downstream consumers, workers are only used when
|
|
1907
|
+
* ASTGEN_TYPE_WORKERS is set explicitly. Set it to the desired worker count
|
|
1908
|
+
* (e.g. number of cores) to trade that cosmetic reordering for speed; "auto"
|
|
1909
|
+
* derives the count from the available CPUs. Unset or 1 => single-threaded.
|
|
1910
|
+
*/
|
|
1911
|
+
const resolveTypeWorkerCount = (fileCount) => {
|
|
1912
|
+
const envValue = process.env.ASTGEN_TYPE_WORKERS;
|
|
1913
|
+
if (envValue === undefined || envValue === "") {
|
|
1914
|
+
return 1;
|
|
1915
|
+
}
|
|
1916
|
+
if (envValue.toLowerCase() === "auto") {
|
|
1917
|
+
let cores = 1;
|
|
1918
|
+
try {
|
|
1919
|
+
cores =
|
|
1920
|
+
typeof availableParallelism === "function"
|
|
1921
|
+
? availableParallelism()
|
|
1922
|
+
: cpus().length;
|
|
1923
|
+
} catch (e) {
|
|
1924
|
+
cores = 1;
|
|
1925
|
+
}
|
|
1926
|
+
const autoCount = Math.max(cores - 1, 1);
|
|
1927
|
+
return Math.min(autoCount, Math.max(1, fileCount));
|
|
1928
|
+
}
|
|
1929
|
+
const requested = Number.parseInt(envValue, 10);
|
|
1930
|
+
if (!Number.isFinite(requested) || requested < 1) {
|
|
1931
|
+
return 1;
|
|
1932
|
+
}
|
|
1933
|
+
return Math.min(requested, Math.max(1, fileCount));
|
|
1934
|
+
};
|
|
1935
|
+
|
|
1936
|
+
/**
|
|
1937
|
+
* Run the type-generation phase across worker threads. Files are sharded
|
|
1938
|
+
* round-robin so heavy files spread across workers. Each worker builds its own
|
|
1939
|
+
* program over `projectFiles`; the per-file AST and type set are identical to
|
|
1940
|
+
* the single-threaded path, except that a small number of inferred union types
|
|
1941
|
+
* may have their members printed in a different order (see resolveTypeWorkerCount).
|
|
1942
|
+
* Returns true only if every worker completed cleanly; on any failure the
|
|
1943
|
+
* caller falls back to inline processing (writes are idempotent, so
|
|
1944
|
+
* re-processing is safe).
|
|
1945
|
+
*/
|
|
1946
|
+
const runTypeGenerationInWorkers = (
|
|
1947
|
+
srcFiles,
|
|
1948
|
+
projectFiles,
|
|
1949
|
+
options,
|
|
1950
|
+
workerCount
|
|
1951
|
+
) => {
|
|
1952
|
+
const shards = Array.from({ length: workerCount }, () => []);
|
|
1953
|
+
srcFiles.forEach((file, index) => shards[index % workerCount].push(file));
|
|
1954
|
+
const workerEntry = fileURLToPath(import.meta.url);
|
|
1955
|
+
return Promise.all(
|
|
1956
|
+
shards.map((shard, index) => {
|
|
1957
|
+
if (shard.length === 0) {
|
|
1958
|
+
return Promise.resolve(true);
|
|
1959
|
+
}
|
|
1960
|
+
return new Promise((resolvePromise) => {
|
|
1961
|
+
let settled = false;
|
|
1962
|
+
const settle = (value) => {
|
|
1963
|
+
if (!settled) {
|
|
1964
|
+
settled = true;
|
|
1965
|
+
resolvePromise(value);
|
|
1966
|
+
}
|
|
1967
|
+
};
|
|
1968
|
+
try {
|
|
1969
|
+
const worker = new Worker(workerEntry, {
|
|
1970
|
+
workerData: {
|
|
1971
|
+
kind: "astgen-typegen",
|
|
1972
|
+
shard,
|
|
1973
|
+
projectFiles,
|
|
1974
|
+
options,
|
|
1975
|
+
index
|
|
1976
|
+
}
|
|
1977
|
+
});
|
|
1978
|
+
worker.on("error", (err) => {
|
|
1979
|
+
console.error("astgen type worker failed:", err?.message || err);
|
|
1980
|
+
settle(false);
|
|
1981
|
+
});
|
|
1982
|
+
worker.on("exit", (code) => settle(code === 0));
|
|
1983
|
+
} catch (err) {
|
|
1984
|
+
console.error("Unable to start astgen type worker:", err?.message);
|
|
1985
|
+
settle(false);
|
|
1986
|
+
}
|
|
1987
|
+
});
|
|
1988
|
+
})
|
|
1989
|
+
).then((results) => results.every(Boolean));
|
|
1990
|
+
};
|
|
1991
|
+
|
|
1764
1992
|
/**
|
|
1765
1993
|
* Generate AST for JavaScript or TypeScript
|
|
1766
1994
|
*/
|
|
1767
1995
|
const createJSAst = async (options) => {
|
|
1768
1996
|
try {
|
|
1769
|
-
const
|
|
1770
|
-
let srcFiles =
|
|
1771
|
-
|
|
1772
|
-
if (options.tsTypes) {
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
: srcFiles;
|
|
1776
|
-
ts = createTsc(projectFiles, options.src);
|
|
1777
|
-
if (ts?.rootNames) {
|
|
1778
|
-
const srcRoot = resolve(options.src);
|
|
1779
|
-
const srcFileByResolvedPath = new Map(
|
|
1780
|
-
srcFiles.map((file) => [resolve(file), file])
|
|
1781
|
-
);
|
|
1782
|
-
for (const file of ts.rootNames) {
|
|
1783
|
-
const resolvedFile = resolve(file);
|
|
1784
|
-
if (
|
|
1785
|
-
resolvedFile.startsWith(srcRoot) &&
|
|
1786
|
-
/\.(?:js|jsx|cjs|mjs|ts|tsx|mts|cts)$/.test(file) &&
|
|
1787
|
-
!srcFileByResolvedPath.has(resolvedFile)
|
|
1788
|
-
) {
|
|
1789
|
-
srcFileByResolvedPath.set(
|
|
1790
|
-
resolvedFile,
|
|
1791
|
-
join(options.src, relative(srcRoot, resolvedFile))
|
|
1792
|
-
);
|
|
1793
|
-
}
|
|
1794
|
-
}
|
|
1795
|
-
srcFiles = [...srcFileByResolvedPath.values()].sort();
|
|
1796
|
-
}
|
|
1997
|
+
const discovered = await getAllSrcJSAndTSFiles(options.src);
|
|
1998
|
+
let srcFiles = [...discovered].sort();
|
|
1999
|
+
|
|
2000
|
+
if (!options.tsTypes) {
|
|
2001
|
+
await processFilesInline(srcFiles, options, undefined);
|
|
2002
|
+
return;
|
|
1797
2003
|
}
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
2004
|
+
|
|
2005
|
+
const projectFiles = !shouldIncludeNodeModulesBundles
|
|
2006
|
+
? srcFiles.filter((file) => !file.includes("node_modules"))
|
|
2007
|
+
: srcFiles;
|
|
2008
|
+
// Compute the program root names from the parsed tsconfig without building
|
|
2009
|
+
// a program on the main thread, then expand the output file set to match.
|
|
2010
|
+
const tscConfig = createTscProgramConfig(projectFiles, options.src);
|
|
2011
|
+
srcFiles = expandSrcFilesWithRootNames(
|
|
2012
|
+
srcFiles,
|
|
2013
|
+
tscConfig?.rootNames,
|
|
2014
|
+
options.src
|
|
1801
2015
|
);
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
}
|
|
1814
|
-
} else if (typeof Bun !== "undefined" && typeof Bun.gc === "function") {
|
|
1815
|
-
try {
|
|
1816
|
-
Bun.gc(true);
|
|
1817
|
-
} catch (e) {
|
|
1818
|
-
// ignore
|
|
1819
|
-
}
|
|
2016
|
+
|
|
2017
|
+
const workerCount = resolveTypeWorkerCount(srcFiles.length);
|
|
2018
|
+
if (workerCount > 1) {
|
|
2019
|
+
const ranInParallel = await runTypeGenerationInWorkers(
|
|
2020
|
+
srcFiles,
|
|
2021
|
+
projectFiles,
|
|
2022
|
+
options,
|
|
2023
|
+
workerCount
|
|
2024
|
+
);
|
|
2025
|
+
if (ranInParallel) {
|
|
2026
|
+
return;
|
|
1820
2027
|
}
|
|
2028
|
+
console.error(
|
|
2029
|
+
"Falling back to single-threaded type generation after worker failure."
|
|
2030
|
+
);
|
|
1821
2031
|
}
|
|
2032
|
+
|
|
2033
|
+
const ts = createTsc(projectFiles, options.src);
|
|
2034
|
+
await processFilesInline(srcFiles, options, ts);
|
|
1822
2035
|
} catch (err) {
|
|
1823
2036
|
console.error(err);
|
|
1824
2037
|
}
|
|
@@ -1861,6 +2074,12 @@ const createVueAst = async (options) => {
|
|
|
1861
2074
|
const getCircularReplacer = () => {
|
|
1862
2075
|
const seen = new WeakSet();
|
|
1863
2076
|
return (key, value) => {
|
|
2077
|
+
// Babel 8 emits BigIntLiteral/`extra.value` as a native bigint, which
|
|
2078
|
+
// JSON.stringify cannot serialize. Emit it as a string, which also matches
|
|
2079
|
+
// the Babel 7 shape (BigIntLiteral.value was already a string there).
|
|
2080
|
+
if (typeof value === "bigint") {
|
|
2081
|
+
return value.toString();
|
|
2082
|
+
}
|
|
1864
2083
|
if (typeof value === "object" && value !== null) {
|
|
1865
2084
|
if (seen.has(value)) {
|
|
1866
2085
|
return;
|
|
@@ -1969,4 +2188,25 @@ async function main(argvs) {
|
|
|
1969
2188
|
}
|
|
1970
2189
|
}
|
|
1971
2190
|
|
|
1972
|
-
|
|
2191
|
+
/**
|
|
2192
|
+
* Type-generation worker entry point. Runs in a worker thread spawned by
|
|
2193
|
+
* runTypeGenerationInWorkers: builds its own TypeScript program over the full
|
|
2194
|
+
* project file set and processes only its assigned shard, so cross-file type
|
|
2195
|
+
* resolution is identical to the single-threaded path.
|
|
2196
|
+
*/
|
|
2197
|
+
const runTypeGenWorker = async ({ shard, projectFiles, options }) => {
|
|
2198
|
+
try {
|
|
2199
|
+
const ts = createTsc(projectFiles, options.src);
|
|
2200
|
+
await processFilesInline(shard, options, ts);
|
|
2201
|
+
parentPort?.postMessage({ done: true });
|
|
2202
|
+
} catch (err) {
|
|
2203
|
+
console.error(err);
|
|
2204
|
+
process.exit(1);
|
|
2205
|
+
}
|
|
2206
|
+
};
|
|
2207
|
+
|
|
2208
|
+
if (isMainThread) {
|
|
2209
|
+
main(process.argv);
|
|
2210
|
+
} else if (workerData?.kind === "astgen-typegen") {
|
|
2211
|
+
runTypeGenWorker(workerData);
|
|
2212
|
+
}
|
package/package.json
CHANGED
|
@@ -1,22 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@appthreat/atom-parsetools",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "Parsing tools that complement the @appthreat/atom project.",
|
|
5
5
|
"main": "./index.js",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"pretty": "prettier --write *.js --trailing-comma=none",
|
|
9
|
-
"test": "node test-fixtures/astgen-type-regression.js && node test-fixtures/astgen-json-regression.js && node test-fixtures/astgen-vue-regression.js && node test-fixtures/evaluate-astgen.js",
|
|
9
|
+
"test": "node test-fixtures/astgen-type-regression.js && node test-fixtures/astgen-json-regression.js && node test-fixtures/astgen-vue-regression.js && node test-fixtures/astgen-shape-snapshot.js && node test-fixtures/evaluate-astgen.js",
|
|
10
10
|
"test:evaluate": "node test-fixtures/evaluate-astgen.js",
|
|
11
|
+
"test:shape": "node test-fixtures/astgen-shape-snapshot.js",
|
|
12
|
+
"test:shape:update": "UPDATE_SHAPE_SNAPSHOT=1 node test-fixtures/astgen-shape-snapshot.js",
|
|
11
13
|
"test:json": "node test-fixtures/astgen-json-regression.js",
|
|
12
14
|
"test:fixtures": "node test-fixtures/test-suite.js",
|
|
13
15
|
"test:vue": "node test-fixtures/astgen-vue-regression.js"
|
|
14
16
|
},
|
|
15
17
|
"dependencies": {
|
|
16
18
|
"@appthreat/atom-common": "^1.1.0",
|
|
17
|
-
"@babel/parser": "^
|
|
18
|
-
"
|
|
19
|
-
"
|
|
19
|
+
"@babel/parser": "^8.0.4",
|
|
20
|
+
"@typescript/typescript6": "^6.0.2",
|
|
21
|
+
"hermes-parser": "^0.37.0"
|
|
20
22
|
},
|
|
21
23
|
"bin": {
|
|
22
24
|
"astgen": "astgen.js",
|
|
@@ -25,7 +27,7 @@
|
|
|
25
27
|
"scalasem": "scalasem.js"
|
|
26
28
|
},
|
|
27
29
|
"engines": {
|
|
28
|
-
"node": ">=
|
|
30
|
+
"node": ">=22.0.0"
|
|
29
31
|
},
|
|
30
32
|
"repository": {
|
|
31
33
|
"type": "git",
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
<?php return array(
|
|
2
2
|
'root' => array(
|
|
3
3
|
'name' => '__root__',
|
|
4
|
-
'pretty_version' => 'v1.
|
|
5
|
-
'version' => '1.
|
|
6
|
-
'reference' => '
|
|
4
|
+
'pretty_version' => 'v1.3.0',
|
|
5
|
+
'version' => '1.3.0.0',
|
|
6
|
+
'reference' => '65d96106e47fe9607cec2ba1be69da7c9c4d574c',
|
|
7
7
|
'type' => 'library',
|
|
8
8
|
'install_path' => __DIR__ . '/../../',
|
|
9
9
|
'aliases' => array(),
|
|
@@ -11,9 +11,9 @@
|
|
|
11
11
|
),
|
|
12
12
|
'versions' => array(
|
|
13
13
|
'__root__' => array(
|
|
14
|
-
'pretty_version' => 'v1.
|
|
15
|
-
'version' => '1.
|
|
16
|
-
'reference' => '
|
|
14
|
+
'pretty_version' => 'v1.3.0',
|
|
15
|
+
'version' => '1.3.0.0',
|
|
16
|
+
'reference' => '65d96106e47fe9607cec2ba1be69da7c9c4d574c',
|
|
17
17
|
'type' => 'library',
|
|
18
18
|
'install_path' => __DIR__ . '/../../',
|
|
19
19
|
'aliases' => array(),
|
package/plugins/rubyastgen/bundle/ruby/4.0.0/extensions/x86_64-linux/4.0.0/prism-1.9.0/gem_make.out
CHANGED
|
@@ -6,10 +6,10 @@ checking for whether -fvisibility=hidden is accepted as CFLAGS... yes
|
|
|
6
6
|
creating Makefile
|
|
7
7
|
|
|
8
8
|
current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rubyastgen/bundle/ruby/4.0.0/gems/prism-1.9.0/ext/prism
|
|
9
|
-
make -j5 DESTDIR\= sitearchdir\=./.gem.
|
|
9
|
+
make -j5 DESTDIR\= sitearchdir\=./.gem.20260807-2447-y3xzkz sitelibdir\=./.gem.20260807-2447-y3xzkz clean
|
|
10
10
|
|
|
11
11
|
current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rubyastgen/bundle/ruby/4.0.0/gems/prism-1.9.0/ext/prism
|
|
12
|
-
make -j5 DESTDIR\= sitearchdir\=./.gem.
|
|
12
|
+
make -j5 DESTDIR\= sitearchdir\=./.gem.20260807-2447-y3xzkz sitelibdir\=./.gem.20260807-2447-y3xzkz
|
|
13
13
|
compiling api_node.c
|
|
14
14
|
compiling api_pack.c
|
|
15
15
|
compiling extension.c
|
|
@@ -37,8 +37,8 @@ compiling ./../../src/util/pm_strpbrk.c
|
|
|
37
37
|
linking shared-object prism/prism.so
|
|
38
38
|
|
|
39
39
|
current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rubyastgen/bundle/ruby/4.0.0/gems/prism-1.9.0/ext/prism
|
|
40
|
-
make -j5 DESTDIR\= sitearchdir\=./.gem.
|
|
41
|
-
/usr/bin/install -c -m 0755 prism.so ./.gem.
|
|
40
|
+
make -j5 DESTDIR\= sitearchdir\=./.gem.20260807-2447-y3xzkz sitelibdir\=./.gem.20260807-2447-y3xzkz install
|
|
41
|
+
/usr/bin/install -c -m 0755 prism.so ./.gem.20260807-2447-y3xzkz/prism
|
|
42
42
|
|
|
43
43
|
current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rubyastgen/bundle/ruby/4.0.0/gems/prism-1.9.0/ext/prism
|
|
44
|
-
make DESTDIR\= sitearchdir\=./.gem.
|
|
44
|
+
make DESTDIR\= sitearchdir\=./.gem.20260807-2447-y3xzkz sitelibdir\=./.gem.20260807-2447-y3xzkz clean
|
package/plugins/rubyastgen/bundle/ruby/4.0.0/extensions/x86_64-linux/4.0.0/racc-1.8.1/gem_make.out
CHANGED
|
@@ -3,16 +3,16 @@ current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rub
|
|
|
3
3
|
creating Makefile
|
|
4
4
|
|
|
5
5
|
current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rubyastgen/bundle/ruby/4.0.0/gems/racc-1.8.1/ext/racc/cparse
|
|
6
|
-
make -j5 DESTDIR\= sitearchdir\=./.gem.
|
|
6
|
+
make -j5 DESTDIR\= sitearchdir\=./.gem.20260807-2447-krkev6 sitelibdir\=./.gem.20260807-2447-krkev6 clean
|
|
7
7
|
|
|
8
8
|
current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rubyastgen/bundle/ruby/4.0.0/gems/racc-1.8.1/ext/racc/cparse
|
|
9
|
-
make -j5 DESTDIR\= sitearchdir\=./.gem.
|
|
9
|
+
make -j5 DESTDIR\= sitearchdir\=./.gem.20260807-2447-krkev6 sitelibdir\=./.gem.20260807-2447-krkev6
|
|
10
10
|
compiling cparse.c
|
|
11
11
|
linking shared-object racc/cparse.so
|
|
12
12
|
|
|
13
13
|
current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rubyastgen/bundle/ruby/4.0.0/gems/racc-1.8.1/ext/racc/cparse
|
|
14
|
-
make -j5 DESTDIR\= sitearchdir\=./.gem.
|
|
15
|
-
/usr/bin/install -c -m 0755 cparse.so ./.gem.
|
|
14
|
+
make -j5 DESTDIR\= sitearchdir\=./.gem.20260807-2447-krkev6 sitelibdir\=./.gem.20260807-2447-krkev6 install
|
|
15
|
+
/usr/bin/install -c -m 0755 cparse.so ./.gem.20260807-2447-krkev6/racc
|
|
16
16
|
|
|
17
17
|
current directory: /home/runner/work/atom-parsetools/atom-parsetools/plugins/rubyastgen/bundle/ruby/4.0.0/gems/racc-1.8.1/ext/racc/cparse
|
|
18
|
-
make DESTDIR\= sitearchdir\=./.gem.
|
|
18
|
+
make DESTDIR\= sitearchdir\=./.gem.20260807-2447-krkev6 sitelibdir\=./.gem.20260807-2447-krkev6 clean
|