c-next 0.2.17 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.js +7645 -5941
- package/dist/index.js.map +4 -4
- package/grammar/CNext.g4 +8 -0
- package/package.json +1 -3
- package/src/transpiler/Transpiler.ts +286 -26
- package/src/transpiler/__tests__/compileCommandsDiscovery.integration.test.ts +94 -0
- package/src/transpiler/__tests__/externalSymbolRecovery.integration.test.ts +215 -0
- package/src/transpiler/logic/__tests__/detectAssemblySyntax.test.ts +116 -0
- package/src/transpiler/logic/analysis/FunctionCallAnalyzer.ts +8 -0
- package/src/transpiler/logic/analysis/MixedTypeCategoryAnalyzer.ts +408 -0
- package/src/transpiler/logic/analysis/PassByValueAnalyzer.ts +16 -97
- package/src/transpiler/logic/analysis/ReturnPathAnalyzer.ts +171 -0
- package/src/transpiler/logic/analysis/__tests__/MixedTypeCategoryAnalyzer.test.ts +346 -0
- package/src/transpiler/logic/analysis/__tests__/ReturnPathAnalyzer.test.ts +232 -0
- package/src/transpiler/logic/analysis/runAnalyzers.ts +23 -2
- package/src/transpiler/logic/analysis/types/IMixedTypeCategoryError.ts +17 -0
- package/src/transpiler/logic/analysis/types/IReturnPathError.ts +12 -0
- package/src/transpiler/logic/detectAssemblySyntax.ts +80 -0
- package/src/transpiler/logic/parser/grammar/CNext.interp +4 -1
- package/src/transpiler/logic/parser/grammar/CNext.tokens +169 -167
- package/src/transpiler/logic/parser/grammar/CNextLexer.interp +4 -1
- package/src/transpiler/logic/parser/grammar/CNextLexer.tokens +169 -167
- package/src/transpiler/logic/parser/grammar/CNextLexer.ts +545 -541
- package/src/transpiler/logic/parser/grammar/CNextListener.ts +11 -0
- package/src/transpiler/logic/parser/grammar/CNextParser.ts +1257 -1185
- package/src/transpiler/logic/parser/grammar/CNextVisitor.ts +7 -0
- package/src/transpiler/logic/preprocessor/CompileCommandsReader.ts +332 -0
- package/src/transpiler/logic/preprocessor/ExternalDeclarationOracle.ts +202 -0
- package/src/transpiler/logic/preprocessor/Preprocessor.ts +24 -6
- package/src/transpiler/logic/preprocessor/ToolchainDetector.ts +42 -1
- package/src/transpiler/logic/preprocessor/__tests__/CompileCommandsReader.test.ts +216 -0
- package/src/transpiler/logic/preprocessor/__tests__/ExternalDeclarationOracle.test.ts +153 -0
- package/src/transpiler/logic/preprocessor/__tests__/Preprocessor.test.ts +43 -18
- package/src/transpiler/logic/preprocessor/__tests__/ToolchainDetector.crossCompiler.test.ts +75 -0
- package/src/transpiler/logic/preprocessor/__tests__/fixtures/oracle/dependent.h +7 -0
- package/src/transpiler/logic/preprocessor/__tests__/fixtures/oracle/predecessor.h +5 -0
- package/src/transpiler/logic/preprocessor/types/ICompileCommandsResult.ts +14 -0
- package/src/transpiler/logic/preprocessor/types/IPreprocessOptions.ts +17 -0
- package/src/transpiler/logic/symbols/SymbolTable.ts +45 -0
- package/src/transpiler/logic/symbols/__tests__/SymbolTable.test.ts +49 -0
- package/src/transpiler/output/MisraSuppressionUtils.ts +52 -0
- package/src/transpiler/output/__tests__/MisraSuppressionUtils.test.ts +67 -0
- package/src/transpiler/output/codegen/CodeGenerator.ts +45 -4
- package/src/transpiler/output/codegen/TypeResolver.ts +148 -0
- package/src/transpiler/output/codegen/TypeValidator.ts +179 -81
- package/src/transpiler/output/codegen/__tests__/CodeGenerator.test.ts +13 -1
- package/src/transpiler/output/codegen/__tests__/TypeResolver.test.ts +93 -0
- package/src/transpiler/output/codegen/__tests__/TypeValidator.alwaysTrueLoop.test.ts +91 -0
- package/src/transpiler/output/codegen/__tests__/TypeValidator.test.ts +86 -14
- package/src/transpiler/output/codegen/assignment/AssignmentClassifier.ts +13 -0
- package/src/transpiler/output/codegen/assignment/AssignmentKind.ts +1 -1
- package/src/transpiler/output/codegen/assignment/handlers/AccessPatternHandlers.ts +20 -12
- package/src/transpiler/output/codegen/assignment/handlers/ArrayHandlers.ts +424 -22
- package/src/transpiler/output/codegen/assignment/handlers/BitAccessHandlers.ts +31 -18
- package/src/transpiler/output/codegen/assignment/handlers/BitmapHandlers.ts +7 -8
- package/src/transpiler/output/codegen/assignment/handlers/RegisterHandlers.ts +6 -8
- package/src/transpiler/output/codegen/assignment/handlers/RegisterUtils.ts +12 -10
- package/src/transpiler/output/codegen/assignment/handlers/SimpleHandler.ts +3 -7
- package/src/transpiler/output/codegen/assignment/handlers/SpecialHandlers.ts +7 -9
- package/src/transpiler/output/codegen/assignment/handlers/StringHandlers.ts +12 -10
- package/src/transpiler/output/codegen/assignment/handlers/__tests__/ArrayHandlers.test.ts +479 -11
- package/src/transpiler/output/codegen/generators/IOrchestrator.ts +3 -0
- package/src/transpiler/output/codegen/generators/expressions/CallExprGenerator.ts +28 -15
- package/src/transpiler/output/codegen/generators/expressions/PostfixExpressionGenerator.ts +26 -13
- package/src/transpiler/output/codegen/generators/expressions/__tests__/PostfixExpressionGenerator.test.ts +129 -1
- package/src/transpiler/output/codegen/generators/statements/ControlFlowGenerator.ts +64 -8
- package/src/transpiler/output/codegen/generators/statements/__tests__/ControlFlowGenerator.test.ts +8 -5
- package/src/transpiler/output/codegen/helpers/AssignmentExpectedTypeResolver.ts +30 -2
- package/src/transpiler/output/codegen/helpers/StringDeclHelper.ts +16 -13
- package/src/transpiler/output/codegen/helpers/__tests__/AssignmentExpectedTypeResolver.test.ts +19 -0
- package/src/transpiler/output/codegen/helpers/__tests__/StringDeclHelper.test.ts +57 -11
- package/src/transpiler/output/codegen/subscript/SubscriptClassifier.ts +30 -11
- package/src/transpiler/output/codegen/subscript/TSubscriptKind.ts +1 -1
- package/src/transpiler/output/codegen/subscript/__tests__/SubscriptClassifier.test.ts +38 -6
- package/src/transpiler/output/headers/HeaderGeneratorUtils.ts +65 -24
- package/src/transpiler/state/CodeGenState.ts +20 -16
- package/src/transpiler/types/ICachedFileEntry.ts +7 -0
- package/src/utils/cache/CacheManager.ts +13 -2
- package/src/utils/cache/__tests__/CacheManager.test.ts +18 -1
- package/src/utils/constants/TypeConstants.ts +13 -0
|
@@ -45,6 +45,7 @@ import { IfStatementContext } from "./CNextParser.js";
|
|
|
45
45
|
import { WhileStatementContext } from "./CNextParser.js";
|
|
46
46
|
import { DoWhileStatementContext } from "./CNextParser.js";
|
|
47
47
|
import { ForStatementContext } from "./CNextParser.js";
|
|
48
|
+
import { ForeverStatementContext } from "./CNextParser.js";
|
|
48
49
|
import { ForInitContext } from "./CNextParser.js";
|
|
49
50
|
import { ForVarDeclContext } from "./CNextParser.js";
|
|
50
51
|
import { ForAssignmentContext } from "./CNextParser.js";
|
|
@@ -353,6 +354,12 @@ export class CNextVisitor<Result> extends AbstractParseTreeVisitor<Result> {
|
|
|
353
354
|
* @return the visitor result
|
|
354
355
|
*/
|
|
355
356
|
visitForStatement?: (ctx: ForStatementContext) => Result;
|
|
357
|
+
/**
|
|
358
|
+
* Visit a parse tree produced by `CNextParser.foreverStatement`.
|
|
359
|
+
* @param ctx the parse tree
|
|
360
|
+
* @return the visitor result
|
|
361
|
+
*/
|
|
362
|
+
visitForeverStatement?: (ctx: ForeverStatementContext) => Result;
|
|
356
363
|
/**
|
|
357
364
|
* Visit a parse tree produced by `CNextParser.forInit`.
|
|
358
365
|
* @param ctx the parse tree
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CompileCommandsReader — derive the compiler's own view (include search path,
|
|
3
|
+
* preprocessor defines, compiler binary) from a `compile_commands.json`
|
|
4
|
+
* compilation database.
|
|
5
|
+
*
|
|
6
|
+
* cnext is a source consumer that must resolve external C/C++ headers exactly as
|
|
7
|
+
* the compiler will. Rather than couple to any one build system's path logic (or
|
|
8
|
+
* hand-mirror it in cnext.config.json), it reads the build-system-agnostic
|
|
9
|
+
* contract every build system converges on: the compiler command line, serialized
|
|
10
|
+
* per translation unit in `compile_commands.json` (the LLVM compilation database
|
|
11
|
+
* that clangd/clang-tidy also consume). Framework include paths are project-global
|
|
12
|
+
* — identical across translation units — so the union of every entry's `-I` set is
|
|
13
|
+
* the search path cnext needs; defines and the compiler binary come along for free.
|
|
14
|
+
*
|
|
15
|
+
* This is the pure parse layer: a database string in, `{ includePaths, defines,
|
|
16
|
+
* compiler }` out. No filesystem access — the loader layer reads the file.
|
|
17
|
+
*/
|
|
18
|
+
import { readFileSync } from "node:fs";
|
|
19
|
+
import { resolve, isAbsolute } from "node:path";
|
|
20
|
+
import ICompileCommandsResult from "./types/ICompileCommandsResult";
|
|
21
|
+
|
|
22
|
+
/** One compile_commands.json entry (clang spec: `command` OR `arguments`). */
|
|
23
|
+
interface ICompileCommandsEntry {
|
|
24
|
+
directory?: string;
|
|
25
|
+
file?: string;
|
|
26
|
+
command?: string;
|
|
27
|
+
arguments?: string[];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Mutable state threaded through the POSIX word-splitter. */
|
|
31
|
+
interface ITokenizerState {
|
|
32
|
+
tokens: string[];
|
|
33
|
+
current: string;
|
|
34
|
+
/** Whether a token is in progress (so empty quotes still emit ""). */
|
|
35
|
+
started: boolean;
|
|
36
|
+
mode: "normal" | "single" | "double";
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Mutable state accumulated across every compile-DB entry while parsing. */
|
|
40
|
+
interface ICompileCommandsAccumulator {
|
|
41
|
+
/** Include search paths in first-seen order, de-duplicated via `seen`. */
|
|
42
|
+
includePaths: string[];
|
|
43
|
+
/** Dedup guard for `includePaths`. */
|
|
44
|
+
seen: Set<string>;
|
|
45
|
+
/** Preprocessor defines (last write wins across entries). */
|
|
46
|
+
defines: Record<string, string | boolean>;
|
|
47
|
+
/** Compiler binary — the first entry-with-args argv[0] wins. */
|
|
48
|
+
compiler: string | null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
class CompileCommandsReader {
|
|
52
|
+
/**
|
|
53
|
+
* Include-search flags whose operand is a directory. Each accepts an operand
|
|
54
|
+
* attached to the flag or as the following token. `-I` is case-sensitive, so
|
|
55
|
+
* it never swallows the lowercase `-isystem`/`-iquote`/`-idirafter` flags.
|
|
56
|
+
*/
|
|
57
|
+
private static readonly INCLUDE_FLAGS = [
|
|
58
|
+
"-I",
|
|
59
|
+
"-isystem",
|
|
60
|
+
"-iquote",
|
|
61
|
+
"-idirafter",
|
|
62
|
+
];
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Read and parse a `compile_commands.json` from disk. Returns null if the file
|
|
66
|
+
* is missing or malformed, so callers can fall back to configured includes
|
|
67
|
+
* rather than fail — a missing compile database is a normal, non-fatal state.
|
|
68
|
+
*/
|
|
69
|
+
static load(path: string): ICompileCommandsResult | null {
|
|
70
|
+
try {
|
|
71
|
+
return CompileCommandsReader.parse(readFileSync(path, "utf8"));
|
|
72
|
+
} catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Parse a `compile_commands.json` string into the compiler's include search
|
|
79
|
+
* path, defines, and compiler binary (combined across all entries).
|
|
80
|
+
*/
|
|
81
|
+
static parse(jsonContent: string): ICompileCommandsResult {
|
|
82
|
+
const accumulator: ICompileCommandsAccumulator = {
|
|
83
|
+
includePaths: [],
|
|
84
|
+
seen: new Set<string>(),
|
|
85
|
+
defines: {},
|
|
86
|
+
compiler: null,
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const entries = JSON.parse(jsonContent) as ICompileCommandsEntry[];
|
|
90
|
+
for (const entry of entries) {
|
|
91
|
+
const args = CompileCommandsReader.entryArgs(entry);
|
|
92
|
+
if (accumulator.compiler === null && args.length > 0) {
|
|
93
|
+
accumulator.compiler = args[0]; // argv[0] is the compiler binary
|
|
94
|
+
}
|
|
95
|
+
CompileCommandsReader.processArgs(
|
|
96
|
+
accumulator,
|
|
97
|
+
args,
|
|
98
|
+
entry.directory ?? "",
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return {
|
|
103
|
+
includePaths: accumulator.includePaths,
|
|
104
|
+
defines: accumulator.defines,
|
|
105
|
+
compiler: accumulator.compiler,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Resolve one entry's argv: explicit `arguments` if present, else the
|
|
111
|
+
* word-split `command` string, else none. `arguments` is used when truthy so a
|
|
112
|
+
* present-but-empty `[]` is kept (an entry with no compile args) — matching the
|
|
113
|
+
* clang spec's `command`-OR-`arguments` shape.
|
|
114
|
+
*/
|
|
115
|
+
private static entryArgs(entry: ICompileCommandsEntry): string[] {
|
|
116
|
+
if (entry.arguments) return entry.arguments;
|
|
117
|
+
return entry.command ? CompileCommandsReader.tokenize(entry.command) : [];
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Collect include paths and defines from one entry's argv. */
|
|
121
|
+
private static processArgs(
|
|
122
|
+
accumulator: ICompileCommandsAccumulator,
|
|
123
|
+
args: string[],
|
|
124
|
+
directory: string,
|
|
125
|
+
): void {
|
|
126
|
+
for (let i = 0; i < args.length; i++) {
|
|
127
|
+
i += CompileCommandsReader.processArg(accumulator, args, i, directory);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Classify one arg — an include flag or a `-D` define — and collect its
|
|
133
|
+
* operand. Returns the number of ADDITIONAL tokens consumed (0, or 1 for a
|
|
134
|
+
* space-separated operand) so the caller advances past it.
|
|
135
|
+
*/
|
|
136
|
+
private static processArg(
|
|
137
|
+
accumulator: ICompileCommandsAccumulator,
|
|
138
|
+
args: string[],
|
|
139
|
+
index: number,
|
|
140
|
+
directory: string,
|
|
141
|
+
): number {
|
|
142
|
+
const includeFlag = CompileCommandsReader.matchIncludeFlag(args[index]);
|
|
143
|
+
if (includeFlag) {
|
|
144
|
+
const operand = CompileCommandsReader.resolveOperand(
|
|
145
|
+
args,
|
|
146
|
+
index,
|
|
147
|
+
includeFlag.length,
|
|
148
|
+
);
|
|
149
|
+
CompileCommandsReader.collectIncludePath(
|
|
150
|
+
accumulator,
|
|
151
|
+
directory,
|
|
152
|
+
operand.value,
|
|
153
|
+
);
|
|
154
|
+
return operand.extraConsumed;
|
|
155
|
+
}
|
|
156
|
+
if (args[index].startsWith("-D")) {
|
|
157
|
+
const operand = CompileCommandsReader.resolveOperand(args, index, 2);
|
|
158
|
+
CompileCommandsReader.collectDefine(accumulator, operand.value);
|
|
159
|
+
return operand.extraConsumed;
|
|
160
|
+
}
|
|
161
|
+
return 0;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Resolve a flag's operand: the text attached directly to the flag, or — if
|
|
166
|
+
* that is empty — the next token (the space-separated `-I foo` form), then
|
|
167
|
+
* consumed. The next-token form is taken unconditionally when the attached text
|
|
168
|
+
* is empty, even at the end of argv, so index advancement stays in step.
|
|
169
|
+
*/
|
|
170
|
+
private static resolveOperand(
|
|
171
|
+
args: string[],
|
|
172
|
+
index: number,
|
|
173
|
+
prefixLength: number,
|
|
174
|
+
): { value: string; extraConsumed: number } {
|
|
175
|
+
const attached = args[index].slice(prefixLength);
|
|
176
|
+
if (attached === "") {
|
|
177
|
+
return { value: args[index + 1] ?? "", extraConsumed: 1 };
|
|
178
|
+
}
|
|
179
|
+
return { value: attached, extraConsumed: 0 };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Collect an include directory (absolute kept verbatim; relative resolved
|
|
184
|
+
* against the entry `directory`), de-duplicated in first-seen order. An empty
|
|
185
|
+
* operand (a flag with no directory) is ignored.
|
|
186
|
+
*/
|
|
187
|
+
private static collectIncludePath(
|
|
188
|
+
accumulator: ICompileCommandsAccumulator,
|
|
189
|
+
directory: string,
|
|
190
|
+
raw: string,
|
|
191
|
+
): void {
|
|
192
|
+
if (raw === "") return;
|
|
193
|
+
const path = isAbsolute(raw) ? raw : resolve(directory, raw);
|
|
194
|
+
if (!accumulator.seen.has(path)) {
|
|
195
|
+
accumulator.seen.add(path);
|
|
196
|
+
accumulator.includePaths.push(path);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Collect a `-D` define, split on the FIRST `=`: `KEY=VAL` -> `VAL` (a trailing
|
|
202
|
+
* `KEY=` keeps the empty value), bare `KEY` -> `true`. An empty operand is
|
|
203
|
+
* ignored.
|
|
204
|
+
*/
|
|
205
|
+
private static collectDefine(
|
|
206
|
+
accumulator: ICompileCommandsAccumulator,
|
|
207
|
+
raw: string,
|
|
208
|
+
): void {
|
|
209
|
+
if (raw === "") return;
|
|
210
|
+
const eq = raw.indexOf("=");
|
|
211
|
+
if (eq === -1) {
|
|
212
|
+
accumulator.defines[raw] = true;
|
|
213
|
+
} else {
|
|
214
|
+
accumulator.defines[raw.slice(0, eq)] = raw.slice(eq + 1);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Return the include flag `arg` begins with, or null if it is not one. */
|
|
219
|
+
private static matchIncludeFlag(arg: string): string | null {
|
|
220
|
+
return (
|
|
221
|
+
CompileCommandsReader.INCLUDE_FLAGS.find((flag) =>
|
|
222
|
+
arg.startsWith(flag),
|
|
223
|
+
) ?? null
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Split a `command` string into argv, matching POSIX shell word-splitting (as
|
|
229
|
+
* `shlex`/`/bin/sh` do) so tokens equal what the compiler was actually invoked
|
|
230
|
+
* with. Real quotes are syntax (consumed); backslash-escaped quotes are literal
|
|
231
|
+
* (kept); whitespace inside quotes is preserved. Only the cases that occur in
|
|
232
|
+
* real compile databases are handled — no `$`/backtick expansion.
|
|
233
|
+
*/
|
|
234
|
+
private static tokenize(command: string): string[] {
|
|
235
|
+
const state: ITokenizerState = {
|
|
236
|
+
tokens: [],
|
|
237
|
+
current: "",
|
|
238
|
+
started: false,
|
|
239
|
+
mode: "normal",
|
|
240
|
+
};
|
|
241
|
+
for (let i = 0; i < command.length; i++) {
|
|
242
|
+
i += CompileCommandsReader.tokenizeChar(state, command, i);
|
|
243
|
+
}
|
|
244
|
+
if (state.started) state.tokens.push(state.current);
|
|
245
|
+
return state.tokens;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Consume one character at `command[i]` in the current tokenizer mode.
|
|
250
|
+
* Returns the number of ADDITIONAL characters consumed (0 or 1) so the caller
|
|
251
|
+
* can advance past an escape's target.
|
|
252
|
+
*/
|
|
253
|
+
private static tokenizeChar(
|
|
254
|
+
state: ITokenizerState,
|
|
255
|
+
command: string,
|
|
256
|
+
i: number,
|
|
257
|
+
): number {
|
|
258
|
+
const ch = command[i];
|
|
259
|
+
if (state.mode === "single") {
|
|
260
|
+
if (ch === "'") state.mode = "normal";
|
|
261
|
+
else state.current += ch;
|
|
262
|
+
return 0;
|
|
263
|
+
}
|
|
264
|
+
if (state.mode === "double") {
|
|
265
|
+
return CompileCommandsReader.tokenizeDoubleQuoted(state, command, i);
|
|
266
|
+
}
|
|
267
|
+
return CompileCommandsReader.tokenizeNormal(state, command, i);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Handle one character outside quotes; returns extra chars consumed. */
|
|
271
|
+
private static tokenizeNormal(
|
|
272
|
+
state: ITokenizerState,
|
|
273
|
+
command: string,
|
|
274
|
+
i: number,
|
|
275
|
+
): number {
|
|
276
|
+
const ch = command[i];
|
|
277
|
+
if (ch === "\\") {
|
|
278
|
+
const next = command[i + 1];
|
|
279
|
+
if (next === undefined) return 0; // trailing backslash: nothing follows
|
|
280
|
+
state.current += next; // outside quotes, backslash escapes any next char
|
|
281
|
+
state.started = true;
|
|
282
|
+
return 1;
|
|
283
|
+
}
|
|
284
|
+
if (ch === "'" || ch === '"') {
|
|
285
|
+
state.mode = ch === "'" ? "single" : "double";
|
|
286
|
+
state.started = true;
|
|
287
|
+
return 0;
|
|
288
|
+
}
|
|
289
|
+
if (ch === " " || ch === "\t" || ch === "\n" || ch === "\r") {
|
|
290
|
+
if (state.started) {
|
|
291
|
+
state.tokens.push(state.current);
|
|
292
|
+
state.current = "";
|
|
293
|
+
state.started = false;
|
|
294
|
+
}
|
|
295
|
+
return 0;
|
|
296
|
+
}
|
|
297
|
+
state.current += ch;
|
|
298
|
+
state.started = true;
|
|
299
|
+
return 0;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** Handle one character inside double quotes; returns extra chars consumed. */
|
|
303
|
+
private static tokenizeDoubleQuoted(
|
|
304
|
+
state: ITokenizerState,
|
|
305
|
+
command: string,
|
|
306
|
+
i: number,
|
|
307
|
+
): number {
|
|
308
|
+
const ch = command[i];
|
|
309
|
+
if (ch === '"') {
|
|
310
|
+
state.mode = "normal";
|
|
311
|
+
return 0;
|
|
312
|
+
}
|
|
313
|
+
if (ch === "\\") {
|
|
314
|
+
const next = command[i + 1];
|
|
315
|
+
// Match POSIX `shlex` (the reference this tokenizer is validated against):
|
|
316
|
+
// inside double quotes only `\"` and `\\` are escapes. Before anything else
|
|
317
|
+
// — `$`, a backtick, a newline — the backslash is a literal character.
|
|
318
|
+
// shlex and bash disagree there; we pin to shlex for verifiable parity, and
|
|
319
|
+
// real compile databases never quote those characters anyway.
|
|
320
|
+
if (next === '"' || next === "\\") {
|
|
321
|
+
state.current += next;
|
|
322
|
+
return 1;
|
|
323
|
+
}
|
|
324
|
+
state.current += ch;
|
|
325
|
+
return 0;
|
|
326
|
+
}
|
|
327
|
+
state.current += ch;
|
|
328
|
+
return 0;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
export default CompileCommandsReader;
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ExternalDeclarationOracle — recover FULL external symbols (function signatures,
|
|
3
|
+
* typedefs, opaque structs) from a set of C/C++ headers that cnext's normal,
|
|
4
|
+
* per-header-standalone symbol collection could not resolve.
|
|
5
|
+
*
|
|
6
|
+
* Two things defeat standalone collection of framework headers:
|
|
7
|
+
* 1. Include-order guards — FreeRTOS `task.h` opens with
|
|
8
|
+
* `#ifndef INC_FREERTOS_H / #error`, so it refuses to preprocess unless
|
|
9
|
+
* `FreeRTOS.h` ran first.
|
|
10
|
+
* 2. The parser's twin failure modes on a single header:
|
|
11
|
+
* - RAW (unpreprocessed) text still contains function-attribute macros like
|
|
12
|
+
* FreeRTOS `PRIVILEGED_FUNCTION` (`void vTaskDelay(...) PRIVILEGED_FUNCTION;`)
|
|
13
|
+
* — the trailing token breaks the declaration, so `vTaskDelay` is lost.
|
|
14
|
+
* - FULLY preprocessed text inlines the header's entire transitive tree
|
|
15
|
+
* (e.g. `driver/twai.h` -> ~100KB of xtensa HAL); ANTLR error-recovery then
|
|
16
|
+
* skips tokens in the blob and silently drops later declarations.
|
|
17
|
+
*
|
|
18
|
+
* The fix is to preprocess the include set AS A TRANSLATION UNIT (predecessors
|
|
19
|
+
* first) while KEEPING `#line` directives, then bucket the output BY SOURCE FILE.
|
|
20
|
+
* Each header's own slice is macro-expanded (PRIVILEGED_FUNCTION is gone) yet
|
|
21
|
+
* small (no inlined tree), so parsing it robustly yields full signatures. The
|
|
22
|
+
* caller parses each slice with cnext's real header parser, so recovered symbols
|
|
23
|
+
* carry their full types — codegen needs these to pass structs by address
|
|
24
|
+
* (`twai_driver_install(&cfg)`) and treat opaque types as pointers
|
|
25
|
+
* (`lv_obj_t` -> `lv_obj_t *`).
|
|
26
|
+
*
|
|
27
|
+
* Function-like macros (e.g. `pdMS_TO_TICKS`) have no declaration to parse, so
|
|
28
|
+
* their NAMES are collected separately (`-dM`) for the undeclared-call check
|
|
29
|
+
* only — a by-value macro invocation is already correct.
|
|
30
|
+
*
|
|
31
|
+
* Requires a toolchain that can preprocess the target's headers; for cross
|
|
32
|
+
* targets (ESP32/xtensa) set CNEXT_CROSS_COMPILER.
|
|
33
|
+
*/
|
|
34
|
+
import Preprocessor from "./Preprocessor";
|
|
35
|
+
import IPreprocessOptions from "./types/IPreprocessOptions";
|
|
36
|
+
|
|
37
|
+
// Function-like macro definitions from a `-dM` dump: `#define pdMS_TO_TICKS(`.
|
|
38
|
+
const FUNCTION_MACRO = /^#define\s+([A-Za-z_]\w*)\(/gm;
|
|
39
|
+
// gcc/clang line marker: `# 958 "/path/to/task.h" 2`.
|
|
40
|
+
const LINE_MARKER = /^#\s+\d+\s+"([^"]+)"/;
|
|
41
|
+
// Synthetic TU filename (appears in preprocessor errors, keyed on to locate a
|
|
42
|
+
// failing include by its line number).
|
|
43
|
+
const TU_NAME = "cnext-external-decl-oracle.c";
|
|
44
|
+
|
|
45
|
+
interface IExternalRecovery {
|
|
46
|
+
/**
|
|
47
|
+
* Map of source header path -> that header's OWN preprocessed text (its
|
|
48
|
+
* macro-expanded declarations, without the transitively-inlined tree). Parse
|
|
49
|
+
* each with the real header parser to register full external symbols.
|
|
50
|
+
*/
|
|
51
|
+
perFileContent: Map<string, string>;
|
|
52
|
+
/** Names of function-like macros — no declaration exists to parse. */
|
|
53
|
+
macroNames: Set<string>;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
class ExternalDeclarationOracle {
|
|
57
|
+
/**
|
|
58
|
+
* Preprocess `includeDirectives` (e.g. `"<freertos/task.h>"`) as one TU and
|
|
59
|
+
* return each header's own preprocessed slice plus function-like-macro names.
|
|
60
|
+
*
|
|
61
|
+
* @param includeDirectives include specs, in source order (angle- or
|
|
62
|
+
* quote-form, e.g. `<Arduino.h>` or `"foo.h"`)
|
|
63
|
+
* @param preprocessor the shared Preprocessor (uses CNEXT_CROSS_COMPILER)
|
|
64
|
+
* @param options include paths + defines for the TU
|
|
65
|
+
*/
|
|
66
|
+
static async recover(
|
|
67
|
+
includeDirectives: readonly string[],
|
|
68
|
+
preprocessor: Preprocessor,
|
|
69
|
+
options: IPreprocessOptions,
|
|
70
|
+
): Promise<IExternalRecovery | null> {
|
|
71
|
+
if (includeDirectives.length === 0 || !preprocessor.isAvailable()) {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// KEEP line directives so the output can be split back into per-file slices.
|
|
76
|
+
const base: IPreprocessOptions = { ...options, keepLineDirectives: true };
|
|
77
|
+
|
|
78
|
+
// Build the largest subset that preprocesses cleanly (drop a header that
|
|
79
|
+
// can't preprocess — e.g. a fragile lvgl.h — instead of losing everything).
|
|
80
|
+
const working = await ExternalDeclarationOracle.preprocessLargestWorkingTu(
|
|
81
|
+
[...includeDirectives],
|
|
82
|
+
preprocessor,
|
|
83
|
+
base,
|
|
84
|
+
);
|
|
85
|
+
if (!working) return null;
|
|
86
|
+
|
|
87
|
+
const perFileContent = ExternalDeclarationOracle.splitByFile(
|
|
88
|
+
working.content,
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
// Function-like macros the plain preprocess would have consumed at use.
|
|
92
|
+
const macroNames = new Set<string>();
|
|
93
|
+
const macros = await preprocessor.preprocessString(
|
|
94
|
+
ExternalDeclarationOracle.buildTu(working.directives),
|
|
95
|
+
TU_NAME,
|
|
96
|
+
{ ...options, keepLineDirectives: false, dumpMacros: true },
|
|
97
|
+
);
|
|
98
|
+
if (macros.success) {
|
|
99
|
+
for (const match of macros.content.matchAll(FUNCTION_MACRO)) {
|
|
100
|
+
macroNames.add(match[1]);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return { perFileContent, macroNames };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Bucket preprocessed output by originating source file using its `#line`
|
|
109
|
+
* markers. Synthetic units (`<built-in>`, `<command-line>`, the TU itself) are
|
|
110
|
+
* skipped. Each real header maps to the concatenation of its own emitted lines.
|
|
111
|
+
*/
|
|
112
|
+
private static splitByFile(content: string): Map<string, string> {
|
|
113
|
+
const buckets = new Map<string, string[]>();
|
|
114
|
+
let current = "";
|
|
115
|
+
for (const line of content.split("\n")) {
|
|
116
|
+
const marker = LINE_MARKER.exec(line);
|
|
117
|
+
if (marker) {
|
|
118
|
+
current = marker[1];
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
if (!current || current.startsWith("<") || current.endsWith(TU_NAME)) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
let bucket = buckets.get(current);
|
|
125
|
+
if (!bucket) {
|
|
126
|
+
bucket = [];
|
|
127
|
+
buckets.set(current, bucket);
|
|
128
|
+
}
|
|
129
|
+
bucket.push(line);
|
|
130
|
+
}
|
|
131
|
+
const perFile = new Map<string, string>();
|
|
132
|
+
for (const [file, lines] of buckets) {
|
|
133
|
+
perFile.set(file, lines.join("\n"));
|
|
134
|
+
}
|
|
135
|
+
return perFile;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
private static buildTu(directives: readonly string[]): string {
|
|
139
|
+
return directives.map((d) => `#include ${d}`).join("\n") + "\n";
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Preprocess the directives as a TU; on failure, drop the single include that
|
|
144
|
+
* caused it (identified by its line in the synthetic TU) and retry. Returns
|
|
145
|
+
* the preprocessed content and the surviving directive list, or null.
|
|
146
|
+
*/
|
|
147
|
+
private static async preprocessLargestWorkingTu(
|
|
148
|
+
directives: string[],
|
|
149
|
+
preprocessor: Preprocessor,
|
|
150
|
+
base: IPreprocessOptions,
|
|
151
|
+
): Promise<{ content: string; directives: string[] } | null> {
|
|
152
|
+
const remaining = directives;
|
|
153
|
+
// At most one drop per iteration; bound iterations by the ORIGINAL directive
|
|
154
|
+
// count, captured once. `remaining` aliases `directives` and shrinks via
|
|
155
|
+
// splice() below, so re-reading `directives.length` in the bound would halve
|
|
156
|
+
// the allowed attempts and bail before trying the last surviving includes.
|
|
157
|
+
const initialCount = directives.length;
|
|
158
|
+
for (
|
|
159
|
+
let attempt = 0;
|
|
160
|
+
remaining.length > 0 && attempt <= initialCount;
|
|
161
|
+
attempt++
|
|
162
|
+
) {
|
|
163
|
+
const result = await preprocessor.preprocessString(
|
|
164
|
+
ExternalDeclarationOracle.buildTu(remaining),
|
|
165
|
+
TU_NAME,
|
|
166
|
+
base,
|
|
167
|
+
);
|
|
168
|
+
if (result.success) {
|
|
169
|
+
return { content: result.content, directives: remaining };
|
|
170
|
+
}
|
|
171
|
+
const failedIndex = ExternalDeclarationOracle.findFailingIncludeIndex(
|
|
172
|
+
result.error,
|
|
173
|
+
remaining.length,
|
|
174
|
+
);
|
|
175
|
+
if (failedIndex < 0) return null; // can't localize — give up
|
|
176
|
+
remaining.splice(failedIndex, 1);
|
|
177
|
+
}
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* From a preprocessor error, find the index of the top-level `#include` that
|
|
183
|
+
* failed, via its line number in the synthetic TU (directive i is at line
|
|
184
|
+
* i+1). Returns -1 if it can't be localized.
|
|
185
|
+
*/
|
|
186
|
+
private static findFailingIncludeIndex(
|
|
187
|
+
error: string | undefined,
|
|
188
|
+
count: number,
|
|
189
|
+
): number {
|
|
190
|
+
if (!error) return -1;
|
|
191
|
+
const escapedTu = TU_NAME.replaceAll(".", String.raw`\.`);
|
|
192
|
+
const pattern = new RegExp(escapedTu + String.raw`:(\d+)`, "g");
|
|
193
|
+
let match: RegExpExecArray | null;
|
|
194
|
+
while ((match = pattern.exec(error)) !== null) {
|
|
195
|
+
const line = Number.parseInt(match[1], 10);
|
|
196
|
+
if (line >= 1 && line <= count) return line - 1;
|
|
197
|
+
}
|
|
198
|
+
return -1;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export default ExternalDeclarationOracle;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Runs the system preprocessor on C/C++ files before parsing
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import { execFile } from "node:child_process";
|
|
7
7
|
import { promisify } from "node:util";
|
|
8
8
|
import { writeFile, mkdtemp, rm } from "node:fs/promises";
|
|
9
9
|
import { tmpdir } from "node:os";
|
|
@@ -14,7 +14,7 @@ import ISourceMapping from "./types/ISourceMapping";
|
|
|
14
14
|
import IPreprocessOptions from "./types/IPreprocessOptions";
|
|
15
15
|
import ToolchainDetector from "./ToolchainDetector";
|
|
16
16
|
|
|
17
|
-
const
|
|
17
|
+
const execFileAsync = promisify(execFile);
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* Handles preprocessing of C/C++ files
|
|
@@ -153,6 +153,12 @@ class Preprocessor {
|
|
|
153
153
|
args.pop(); // Remove -P
|
|
154
154
|
}
|
|
155
155
|
|
|
156
|
+
// Dump macro definitions (#define list) instead of preprocessed source,
|
|
157
|
+
// to discover function-like macros the normal preprocess would consume.
|
|
158
|
+
if (options.dumpMacros) {
|
|
159
|
+
args.push("-dM");
|
|
160
|
+
}
|
|
161
|
+
|
|
156
162
|
// Add include paths
|
|
157
163
|
const includePaths = [
|
|
158
164
|
...this.defaultIncludePaths,
|
|
@@ -175,14 +181,26 @@ class Preprocessor {
|
|
|
175
181
|
}
|
|
176
182
|
}
|
|
177
183
|
|
|
184
|
+
// Import predecessor macros (gcc/clang -imacros) so include-order-dependent
|
|
185
|
+
// headers get the guards/attribute macros their includer would have defined
|
|
186
|
+
// first. -imacros keeps only the macros, not the predecessors' declarations,
|
|
187
|
+
// so the output stays scoped to the target file.
|
|
188
|
+
if (options.imacros) {
|
|
189
|
+
for (const macroHeader of options.imacros) {
|
|
190
|
+
args.push("-imacros", macroHeader);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
178
194
|
// Add the input file
|
|
179
195
|
args.push(filePath);
|
|
180
196
|
|
|
181
|
-
//
|
|
182
|
-
|
|
183
|
-
|
|
197
|
+
// Invoke the preprocessor via argv (execFile, NOT a shell). A shell would
|
|
198
|
+
// re-parse -D values that legitimately contain spaces / parentheses (e.g.
|
|
199
|
+
// -DARDUINO_BOARD="Espressif ... (8 MB QD, No PSRAM)"), breaking on the
|
|
200
|
+
// metacharacters. Passing args directly mirrors how the real compiler is
|
|
201
|
+
// invoked and is safe for any value the compiler accepts.
|
|
184
202
|
try {
|
|
185
|
-
const { stdout, stderr } = await
|
|
203
|
+
const { stdout, stderr } = await execFileAsync(toolchain.cpp, args, {
|
|
186
204
|
maxBuffer: 50 * 1024 * 1024, // 50MB buffer for large headers
|
|
187
205
|
});
|
|
188
206
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { execSync } from "node:child_process";
|
|
7
7
|
import { existsSync } from "node:fs";
|
|
8
|
-
import { join } from "node:path";
|
|
8
|
+
import { basename, join } from "node:path";
|
|
9
9
|
import IToolchain from "./types/IToolchain";
|
|
10
10
|
|
|
11
11
|
/**
|
|
@@ -17,6 +17,16 @@ class ToolchainDetector {
|
|
|
17
17
|
* Priority: ARM cross-compiler > clang > gcc
|
|
18
18
|
*/
|
|
19
19
|
static detect(): IToolchain | null {
|
|
20
|
+
// Explicit override: a project can name the compiler that owns its target
|
|
21
|
+
// headers (e.g. a cross-compiler such as xtensa-esp32s3-elf-gcc) via
|
|
22
|
+
// CNEXT_CROSS_COMPILER. Host gcc/clang lack a cross target's system headers
|
|
23
|
+
// and predefined macros, so their preprocessing of target headers fails.
|
|
24
|
+
const override = process.env.CNEXT_CROSS_COMPILER;
|
|
25
|
+
if (override) {
|
|
26
|
+
const overridden = ToolchainDetector.fromPath(override);
|
|
27
|
+
if (overridden) return overridden;
|
|
28
|
+
}
|
|
29
|
+
|
|
20
30
|
// Try ARM cross-compiler first (for embedded)
|
|
21
31
|
const arm = this.detectArmToolchain();
|
|
22
32
|
if (arm) return arm;
|
|
@@ -50,6 +60,37 @@ class ToolchainDetector {
|
|
|
50
60
|
return toolchains;
|
|
51
61
|
}
|
|
52
62
|
|
|
63
|
+
/**
|
|
64
|
+
* Build a toolchain from an explicit compiler path or executable name (e.g. a
|
|
65
|
+
* target cross-compiler such as xtensa-esp32s3-elf-gcc). Used by the
|
|
66
|
+
* CNEXT_CROSS_COMPILER override so a project can preprocess its target headers with the compiler
|
|
67
|
+
* that owns them.
|
|
68
|
+
*/
|
|
69
|
+
static fromPath(compiler: string): IToolchain | null {
|
|
70
|
+
let cc: string | null;
|
|
71
|
+
if (compiler.includes("/")) {
|
|
72
|
+
cc = existsSync(compiler) ? compiler : null;
|
|
73
|
+
} else {
|
|
74
|
+
cc = this.findExecutable(compiler);
|
|
75
|
+
}
|
|
76
|
+
if (!cc) return null;
|
|
77
|
+
|
|
78
|
+
// Best-effort C++ driver alongside the C driver (only cpp is used for
|
|
79
|
+
// preprocessing; cxx is derived for completeness).
|
|
80
|
+
const cxxCandidate = cc.replace(/gcc(\.exe)?$/, "g++$1");
|
|
81
|
+
const cxx =
|
|
82
|
+
cxxCandidate !== cc && existsSync(cxxCandidate) ? cxxCandidate : cc;
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
name: basename(cc),
|
|
86
|
+
cc,
|
|
87
|
+
cxx,
|
|
88
|
+
cpp: cc,
|
|
89
|
+
version: this.getVersion(cc),
|
|
90
|
+
isCrossCompiler: true,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
53
94
|
/**
|
|
54
95
|
* Detect ARM cross-compiler (arm-none-eabi-gcc)
|
|
55
96
|
*/
|