c-next 0.2.17 → 0.2.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +2 -2
  2. package/dist/index.js +7645 -5941
  3. package/dist/index.js.map +4 -4
  4. package/grammar/CNext.g4 +8 -0
  5. package/package.json +1 -3
  6. package/src/transpiler/Transpiler.ts +286 -26
  7. package/src/transpiler/__tests__/compileCommandsDiscovery.integration.test.ts +94 -0
  8. package/src/transpiler/__tests__/externalSymbolRecovery.integration.test.ts +215 -0
  9. package/src/transpiler/logic/__tests__/detectAssemblySyntax.test.ts +116 -0
  10. package/src/transpiler/logic/analysis/FunctionCallAnalyzer.ts +8 -0
  11. package/src/transpiler/logic/analysis/MixedTypeCategoryAnalyzer.ts +408 -0
  12. package/src/transpiler/logic/analysis/PassByValueAnalyzer.ts +16 -97
  13. package/src/transpiler/logic/analysis/ReturnPathAnalyzer.ts +171 -0
  14. package/src/transpiler/logic/analysis/__tests__/MixedTypeCategoryAnalyzer.test.ts +346 -0
  15. package/src/transpiler/logic/analysis/__tests__/ReturnPathAnalyzer.test.ts +232 -0
  16. package/src/transpiler/logic/analysis/runAnalyzers.ts +23 -2
  17. package/src/transpiler/logic/analysis/types/IMixedTypeCategoryError.ts +17 -0
  18. package/src/transpiler/logic/analysis/types/IReturnPathError.ts +12 -0
  19. package/src/transpiler/logic/detectAssemblySyntax.ts +80 -0
  20. package/src/transpiler/logic/parser/grammar/CNext.interp +4 -1
  21. package/src/transpiler/logic/parser/grammar/CNext.tokens +169 -167
  22. package/src/transpiler/logic/parser/grammar/CNextLexer.interp +4 -1
  23. package/src/transpiler/logic/parser/grammar/CNextLexer.tokens +169 -167
  24. package/src/transpiler/logic/parser/grammar/CNextLexer.ts +545 -541
  25. package/src/transpiler/logic/parser/grammar/CNextListener.ts +11 -0
  26. package/src/transpiler/logic/parser/grammar/CNextParser.ts +1257 -1185
  27. package/src/transpiler/logic/parser/grammar/CNextVisitor.ts +7 -0
  28. package/src/transpiler/logic/preprocessor/CompileCommandsReader.ts +332 -0
  29. package/src/transpiler/logic/preprocessor/ExternalDeclarationOracle.ts +202 -0
  30. package/src/transpiler/logic/preprocessor/Preprocessor.ts +24 -6
  31. package/src/transpiler/logic/preprocessor/ToolchainDetector.ts +42 -1
  32. package/src/transpiler/logic/preprocessor/__tests__/CompileCommandsReader.test.ts +216 -0
  33. package/src/transpiler/logic/preprocessor/__tests__/ExternalDeclarationOracle.test.ts +153 -0
  34. package/src/transpiler/logic/preprocessor/__tests__/Preprocessor.test.ts +43 -18
  35. package/src/transpiler/logic/preprocessor/__tests__/ToolchainDetector.crossCompiler.test.ts +75 -0
  36. package/src/transpiler/logic/preprocessor/__tests__/fixtures/oracle/dependent.h +7 -0
  37. package/src/transpiler/logic/preprocessor/__tests__/fixtures/oracle/predecessor.h +5 -0
  38. package/src/transpiler/logic/preprocessor/types/ICompileCommandsResult.ts +14 -0
  39. package/src/transpiler/logic/preprocessor/types/IPreprocessOptions.ts +17 -0
  40. package/src/transpiler/logic/symbols/SymbolTable.ts +45 -0
  41. package/src/transpiler/logic/symbols/__tests__/SymbolTable.test.ts +49 -0
  42. package/src/transpiler/output/MisraSuppressionUtils.ts +52 -0
  43. package/src/transpiler/output/__tests__/MisraSuppressionUtils.test.ts +67 -0
  44. package/src/transpiler/output/codegen/CodeGenerator.ts +45 -4
  45. package/src/transpiler/output/codegen/TypeResolver.ts +148 -0
  46. package/src/transpiler/output/codegen/TypeValidator.ts +179 -81
  47. package/src/transpiler/output/codegen/__tests__/CodeGenerator.test.ts +13 -1
  48. package/src/transpiler/output/codegen/__tests__/TypeResolver.test.ts +93 -0
  49. package/src/transpiler/output/codegen/__tests__/TypeValidator.alwaysTrueLoop.test.ts +91 -0
  50. package/src/transpiler/output/codegen/__tests__/TypeValidator.test.ts +86 -14
  51. package/src/transpiler/output/codegen/assignment/AssignmentClassifier.ts +13 -0
  52. package/src/transpiler/output/codegen/assignment/AssignmentKind.ts +1 -1
  53. package/src/transpiler/output/codegen/assignment/handlers/AccessPatternHandlers.ts +20 -12
  54. package/src/transpiler/output/codegen/assignment/handlers/ArrayHandlers.ts +424 -22
  55. package/src/transpiler/output/codegen/assignment/handlers/BitAccessHandlers.ts +31 -18
  56. package/src/transpiler/output/codegen/assignment/handlers/BitmapHandlers.ts +7 -8
  57. package/src/transpiler/output/codegen/assignment/handlers/RegisterHandlers.ts +6 -8
  58. package/src/transpiler/output/codegen/assignment/handlers/RegisterUtils.ts +12 -10
  59. package/src/transpiler/output/codegen/assignment/handlers/SimpleHandler.ts +3 -7
  60. package/src/transpiler/output/codegen/assignment/handlers/SpecialHandlers.ts +7 -9
  61. package/src/transpiler/output/codegen/assignment/handlers/StringHandlers.ts +12 -10
  62. package/src/transpiler/output/codegen/assignment/handlers/__tests__/ArrayHandlers.test.ts +479 -11
  63. package/src/transpiler/output/codegen/generators/IOrchestrator.ts +3 -0
  64. package/src/transpiler/output/codegen/generators/expressions/CallExprGenerator.ts +28 -15
  65. package/src/transpiler/output/codegen/generators/expressions/PostfixExpressionGenerator.ts +26 -13
  66. package/src/transpiler/output/codegen/generators/expressions/__tests__/PostfixExpressionGenerator.test.ts +129 -1
  67. package/src/transpiler/output/codegen/generators/statements/ControlFlowGenerator.ts +64 -8
  68. package/src/transpiler/output/codegen/generators/statements/__tests__/ControlFlowGenerator.test.ts +8 -5
  69. package/src/transpiler/output/codegen/helpers/AssignmentExpectedTypeResolver.ts +30 -2
  70. package/src/transpiler/output/codegen/helpers/StringDeclHelper.ts +16 -13
  71. package/src/transpiler/output/codegen/helpers/__tests__/AssignmentExpectedTypeResolver.test.ts +19 -0
  72. package/src/transpiler/output/codegen/helpers/__tests__/StringDeclHelper.test.ts +57 -11
  73. package/src/transpiler/output/codegen/subscript/SubscriptClassifier.ts +30 -11
  74. package/src/transpiler/output/codegen/subscript/TSubscriptKind.ts +1 -1
  75. package/src/transpiler/output/codegen/subscript/__tests__/SubscriptClassifier.test.ts +38 -6
  76. package/src/transpiler/output/headers/HeaderGeneratorUtils.ts +65 -24
  77. package/src/transpiler/state/CodeGenState.ts +20 -16
  78. package/src/transpiler/types/ICachedFileEntry.ts +7 -0
  79. package/src/utils/cache/CacheManager.ts +13 -2
  80. package/src/utils/cache/__tests__/CacheManager.test.ts +18 -1
  81. package/src/utils/constants/TypeConstants.ts +13 -0
@@ -45,6 +45,7 @@ import { IfStatementContext } from "./CNextParser.js";
45
45
  import { WhileStatementContext } from "./CNextParser.js";
46
46
  import { DoWhileStatementContext } from "./CNextParser.js";
47
47
  import { ForStatementContext } from "./CNextParser.js";
48
+ import { ForeverStatementContext } from "./CNextParser.js";
48
49
  import { ForInitContext } from "./CNextParser.js";
49
50
  import { ForVarDeclContext } from "./CNextParser.js";
50
51
  import { ForAssignmentContext } from "./CNextParser.js";
@@ -353,6 +354,12 @@ export class CNextVisitor<Result> extends AbstractParseTreeVisitor<Result> {
353
354
  * @return the visitor result
354
355
  */
355
356
  visitForStatement?: (ctx: ForStatementContext) => Result;
357
+ /**
358
+ * Visit a parse tree produced by `CNextParser.foreverStatement`.
359
+ * @param ctx the parse tree
360
+ * @return the visitor result
361
+ */
362
+ visitForeverStatement?: (ctx: ForeverStatementContext) => Result;
356
363
  /**
357
364
  * Visit a parse tree produced by `CNextParser.forInit`.
358
365
  * @param ctx the parse tree
@@ -0,0 +1,332 @@
1
+ /**
2
+ * CompileCommandsReader — derive the compiler's own view (include search path,
3
+ * preprocessor defines, compiler binary) from a `compile_commands.json`
4
+ * compilation database.
5
+ *
6
+ * cnext is a source consumer that must resolve external C/C++ headers exactly as
7
+ * the compiler will. Rather than couple to any one build system's path logic (or
8
+ * hand-mirror it in cnext.config.json), it reads the build-system-agnostic
9
+ * contract every build system converges on: the compiler command line, serialized
10
+ * per translation unit in `compile_commands.json` (the LLVM compilation database
11
+ * that clangd/clang-tidy also consume). Framework include paths are project-global
12
+ * — identical across translation units — so the union of every entry's `-I` set is
13
+ * the search path cnext needs; defines and the compiler binary come along for free.
14
+ *
15
+ * This is the pure parse layer: a database string in, `{ includePaths, defines,
16
+ * compiler }` out. No filesystem access — the loader layer reads the file.
17
+ */
18
+ import { readFileSync } from "node:fs";
19
+ import { resolve, isAbsolute } from "node:path";
20
+ import ICompileCommandsResult from "./types/ICompileCommandsResult";
21
+
22
+ /** One compile_commands.json entry (clang spec: `command` OR `arguments`). */
23
+ interface ICompileCommandsEntry {
24
+ directory?: string;
25
+ file?: string;
26
+ command?: string;
27
+ arguments?: string[];
28
+ }
29
+
30
+ /** Mutable state threaded through the POSIX word-splitter. */
31
+ interface ITokenizerState {
32
+ tokens: string[];
33
+ current: string;
34
+ /** Whether a token is in progress (so empty quotes still emit ""). */
35
+ started: boolean;
36
+ mode: "normal" | "single" | "double";
37
+ }
38
+
39
+ /** Mutable state accumulated across every compile-DB entry while parsing. */
40
+ interface ICompileCommandsAccumulator {
41
+ /** Include search paths in first-seen order, de-duplicated via `seen`. */
42
+ includePaths: string[];
43
+ /** Dedup guard for `includePaths`. */
44
+ seen: Set<string>;
45
+ /** Preprocessor defines (last write wins across entries). */
46
+ defines: Record<string, string | boolean>;
47
+ /** Compiler binary — the first entry-with-args argv[0] wins. */
48
+ compiler: string | null;
49
+ }
50
+
51
+ class CompileCommandsReader {
52
+ /**
53
+ * Include-search flags whose operand is a directory. Each accepts an operand
54
+ * attached to the flag or as the following token. `-I` is case-sensitive, so
55
+ * it never swallows the lowercase `-isystem`/`-iquote`/`-idirafter` flags.
56
+ */
57
+ private static readonly INCLUDE_FLAGS = [
58
+ "-I",
59
+ "-isystem",
60
+ "-iquote",
61
+ "-idirafter",
62
+ ];
63
+
64
+ /**
65
+ * Read and parse a `compile_commands.json` from disk. Returns null if the file
66
+ * is missing or malformed, so callers can fall back to configured includes
67
+ * rather than fail — a missing compile database is a normal, non-fatal state.
68
+ */
69
+ static load(path: string): ICompileCommandsResult | null {
70
+ try {
71
+ return CompileCommandsReader.parse(readFileSync(path, "utf8"));
72
+ } catch {
73
+ return null;
74
+ }
75
+ }
76
+
77
+ /**
78
+ * Parse a `compile_commands.json` string into the compiler's include search
79
+ * path, defines, and compiler binary (combined across all entries).
80
+ */
81
+ static parse(jsonContent: string): ICompileCommandsResult {
82
+ const accumulator: ICompileCommandsAccumulator = {
83
+ includePaths: [],
84
+ seen: new Set<string>(),
85
+ defines: {},
86
+ compiler: null,
87
+ };
88
+
89
+ const entries = JSON.parse(jsonContent) as ICompileCommandsEntry[];
90
+ for (const entry of entries) {
91
+ const args = CompileCommandsReader.entryArgs(entry);
92
+ if (accumulator.compiler === null && args.length > 0) {
93
+ accumulator.compiler = args[0]; // argv[0] is the compiler binary
94
+ }
95
+ CompileCommandsReader.processArgs(
96
+ accumulator,
97
+ args,
98
+ entry.directory ?? "",
99
+ );
100
+ }
101
+
102
+ return {
103
+ includePaths: accumulator.includePaths,
104
+ defines: accumulator.defines,
105
+ compiler: accumulator.compiler,
106
+ };
107
+ }
108
+
109
+ /**
110
+ * Resolve one entry's argv: explicit `arguments` if present, else the
111
+ * word-split `command` string, else none. `arguments` is used when truthy so a
112
+ * present-but-empty `[]` is kept (an entry with no compile args) — matching the
113
+ * clang spec's `command`-OR-`arguments` shape.
114
+ */
115
+ private static entryArgs(entry: ICompileCommandsEntry): string[] {
116
+ if (entry.arguments) return entry.arguments;
117
+ return entry.command ? CompileCommandsReader.tokenize(entry.command) : [];
118
+ }
119
+
120
+ /** Collect include paths and defines from one entry's argv. */
121
+ private static processArgs(
122
+ accumulator: ICompileCommandsAccumulator,
123
+ args: string[],
124
+ directory: string,
125
+ ): void {
126
+ for (let i = 0; i < args.length; i++) {
127
+ i += CompileCommandsReader.processArg(accumulator, args, i, directory);
128
+ }
129
+ }
130
+
131
+ /**
132
+ * Classify one arg — an include flag or a `-D` define — and collect its
133
+ * operand. Returns the number of ADDITIONAL tokens consumed (0, or 1 for a
134
+ * space-separated operand) so the caller advances past it.
135
+ */
136
+ private static processArg(
137
+ accumulator: ICompileCommandsAccumulator,
138
+ args: string[],
139
+ index: number,
140
+ directory: string,
141
+ ): number {
142
+ const includeFlag = CompileCommandsReader.matchIncludeFlag(args[index]);
143
+ if (includeFlag) {
144
+ const operand = CompileCommandsReader.resolveOperand(
145
+ args,
146
+ index,
147
+ includeFlag.length,
148
+ );
149
+ CompileCommandsReader.collectIncludePath(
150
+ accumulator,
151
+ directory,
152
+ operand.value,
153
+ );
154
+ return operand.extraConsumed;
155
+ }
156
+ if (args[index].startsWith("-D")) {
157
+ const operand = CompileCommandsReader.resolveOperand(args, index, 2);
158
+ CompileCommandsReader.collectDefine(accumulator, operand.value);
159
+ return operand.extraConsumed;
160
+ }
161
+ return 0;
162
+ }
163
+
164
+ /**
165
+ * Resolve a flag's operand: the text attached directly to the flag, or — if
166
+ * that is empty — the next token (the space-separated `-I foo` form), then
167
+ * consumed. The next-token form is taken unconditionally when the attached text
168
+ * is empty, even at the end of argv, so index advancement stays in step.
169
+ */
170
+ private static resolveOperand(
171
+ args: string[],
172
+ index: number,
173
+ prefixLength: number,
174
+ ): { value: string; extraConsumed: number } {
175
+ const attached = args[index].slice(prefixLength);
176
+ if (attached === "") {
177
+ return { value: args[index + 1] ?? "", extraConsumed: 1 };
178
+ }
179
+ return { value: attached, extraConsumed: 0 };
180
+ }
181
+
182
+ /**
183
+ * Collect an include directory (absolute kept verbatim; relative resolved
184
+ * against the entry `directory`), de-duplicated in first-seen order. An empty
185
+ * operand (a flag with no directory) is ignored.
186
+ */
187
+ private static collectIncludePath(
188
+ accumulator: ICompileCommandsAccumulator,
189
+ directory: string,
190
+ raw: string,
191
+ ): void {
192
+ if (raw === "") return;
193
+ const path = isAbsolute(raw) ? raw : resolve(directory, raw);
194
+ if (!accumulator.seen.has(path)) {
195
+ accumulator.seen.add(path);
196
+ accumulator.includePaths.push(path);
197
+ }
198
+ }
199
+
200
+ /**
201
+ * Collect a `-D` define, split on the FIRST `=`: `KEY=VAL` -> `VAL` (a trailing
202
+ * `KEY=` keeps the empty value), bare `KEY` -> `true`. An empty operand is
203
+ * ignored.
204
+ */
205
+ private static collectDefine(
206
+ accumulator: ICompileCommandsAccumulator,
207
+ raw: string,
208
+ ): void {
209
+ if (raw === "") return;
210
+ const eq = raw.indexOf("=");
211
+ if (eq === -1) {
212
+ accumulator.defines[raw] = true;
213
+ } else {
214
+ accumulator.defines[raw.slice(0, eq)] = raw.slice(eq + 1);
215
+ }
216
+ }
217
+
218
+ /** Return the include flag `arg` begins with, or null if it is not one. */
219
+ private static matchIncludeFlag(arg: string): string | null {
220
+ return (
221
+ CompileCommandsReader.INCLUDE_FLAGS.find((flag) =>
222
+ arg.startsWith(flag),
223
+ ) ?? null
224
+ );
225
+ }
226
+
227
+ /**
228
+ * Split a `command` string into argv, matching POSIX shell word-splitting (as
229
+ * `shlex`/`/bin/sh` do) so tokens equal what the compiler was actually invoked
230
+ * with. Real quotes are syntax (consumed); backslash-escaped quotes are literal
231
+ * (kept); whitespace inside quotes is preserved. Only the cases that occur in
232
+ * real compile databases are handled — no `$`/backtick expansion.
233
+ */
234
+ private static tokenize(command: string): string[] {
235
+ const state: ITokenizerState = {
236
+ tokens: [],
237
+ current: "",
238
+ started: false,
239
+ mode: "normal",
240
+ };
241
+ for (let i = 0; i < command.length; i++) {
242
+ i += CompileCommandsReader.tokenizeChar(state, command, i);
243
+ }
244
+ if (state.started) state.tokens.push(state.current);
245
+ return state.tokens;
246
+ }
247
+
248
+ /**
249
+ * Consume one character at `command[i]` in the current tokenizer mode.
250
+ * Returns the number of ADDITIONAL characters consumed (0 or 1) so the caller
251
+ * can advance past an escape's target.
252
+ */
253
+ private static tokenizeChar(
254
+ state: ITokenizerState,
255
+ command: string,
256
+ i: number,
257
+ ): number {
258
+ const ch = command[i];
259
+ if (state.mode === "single") {
260
+ if (ch === "'") state.mode = "normal";
261
+ else state.current += ch;
262
+ return 0;
263
+ }
264
+ if (state.mode === "double") {
265
+ return CompileCommandsReader.tokenizeDoubleQuoted(state, command, i);
266
+ }
267
+ return CompileCommandsReader.tokenizeNormal(state, command, i);
268
+ }
269
+
270
+ /** Handle one character outside quotes; returns extra chars consumed. */
271
+ private static tokenizeNormal(
272
+ state: ITokenizerState,
273
+ command: string,
274
+ i: number,
275
+ ): number {
276
+ const ch = command[i];
277
+ if (ch === "\\") {
278
+ const next = command[i + 1];
279
+ if (next === undefined) return 0; // trailing backslash: nothing follows
280
+ state.current += next; // outside quotes, backslash escapes any next char
281
+ state.started = true;
282
+ return 1;
283
+ }
284
+ if (ch === "'" || ch === '"') {
285
+ state.mode = ch === "'" ? "single" : "double";
286
+ state.started = true;
287
+ return 0;
288
+ }
289
+ if (ch === " " || ch === "\t" || ch === "\n" || ch === "\r") {
290
+ if (state.started) {
291
+ state.tokens.push(state.current);
292
+ state.current = "";
293
+ state.started = false;
294
+ }
295
+ return 0;
296
+ }
297
+ state.current += ch;
298
+ state.started = true;
299
+ return 0;
300
+ }
301
+
302
+ /** Handle one character inside double quotes; returns extra chars consumed. */
303
+ private static tokenizeDoubleQuoted(
304
+ state: ITokenizerState,
305
+ command: string,
306
+ i: number,
307
+ ): number {
308
+ const ch = command[i];
309
+ if (ch === '"') {
310
+ state.mode = "normal";
311
+ return 0;
312
+ }
313
+ if (ch === "\\") {
314
+ const next = command[i + 1];
315
+ // Match POSIX `shlex` (the reference this tokenizer is validated against):
316
+ // inside double quotes only `\"` and `\\` are escapes. Before anything else
317
+ // — `$`, a backtick, a newline — the backslash is a literal character.
318
+ // shlex and bash disagree there; we pin to shlex for verifiable parity, and
319
+ // real compile databases never quote those characters anyway.
320
+ if (next === '"' || next === "\\") {
321
+ state.current += next;
322
+ return 1;
323
+ }
324
+ state.current += ch;
325
+ return 0;
326
+ }
327
+ state.current += ch;
328
+ return 0;
329
+ }
330
+ }
331
+
332
+ export default CompileCommandsReader;
@@ -0,0 +1,202 @@
1
+ /**
2
+ * ExternalDeclarationOracle — recover FULL external symbols (function signatures,
3
+ * typedefs, opaque structs) from a set of C/C++ headers that cnext's normal,
4
+ * per-header-standalone symbol collection could not resolve.
5
+ *
6
+ * Two things defeat standalone collection of framework headers:
7
+ * 1. Include-order guards — FreeRTOS `task.h` opens with
8
+ * `#ifndef INC_FREERTOS_H / #error`, so it refuses to preprocess unless
9
+ * `FreeRTOS.h` ran first.
10
+ * 2. The parser's twin failure modes on a single header:
11
+ * - RAW (unpreprocessed) text still contains function-attribute macros like
12
+ * FreeRTOS `PRIVILEGED_FUNCTION` (`void vTaskDelay(...) PRIVILEGED_FUNCTION;`)
13
+ * — the trailing token breaks the declaration, so `vTaskDelay` is lost.
14
+ * - FULLY preprocessed text inlines the header's entire transitive tree
15
+ * (e.g. `driver/twai.h` -> ~100KB of xtensa HAL); ANTLR error-recovery then
16
+ * skips tokens in the blob and silently drops later declarations.
17
+ *
18
+ * The fix is to preprocess the include set AS A TRANSLATION UNIT (predecessors
19
+ * first) while KEEPING `#line` directives, then bucket the output BY SOURCE FILE.
20
+ * Each header's own slice is macro-expanded (PRIVILEGED_FUNCTION is gone) yet
21
+ * small (no inlined tree), so parsing it robustly yields full signatures. The
22
+ * caller parses each slice with cnext's real header parser, so recovered symbols
23
+ * carry their full types — codegen needs these to pass structs by address
24
+ * (`twai_driver_install(&cfg)`) and treat opaque types as pointers
25
+ * (`lv_obj_t` -> `lv_obj_t *`).
26
+ *
27
+ * Function-like macros (e.g. `pdMS_TO_TICKS`) have no declaration to parse, so
28
+ * their NAMES are collected separately (`-dM`) for the undeclared-call check
29
+ * only — a by-value macro invocation is already correct.
30
+ *
31
+ * Requires a toolchain that can preprocess the target's headers; for cross
32
+ * targets (ESP32/xtensa) set CNEXT_CROSS_COMPILER.
33
+ */
34
+ import Preprocessor from "./Preprocessor";
35
+ import IPreprocessOptions from "./types/IPreprocessOptions";
36
+
37
+ // Function-like macro definitions from a `-dM` dump: `#define pdMS_TO_TICKS(`.
38
+ const FUNCTION_MACRO = /^#define\s+([A-Za-z_]\w*)\(/gm;
39
+ // gcc/clang line marker: `# 958 "/path/to/task.h" 2`.
40
+ const LINE_MARKER = /^#\s+\d+\s+"([^"]+)"/;
41
+ // Synthetic TU filename (appears in preprocessor errors, keyed on to locate a
42
+ // failing include by its line number).
43
+ const TU_NAME = "cnext-external-decl-oracle.c";
44
+
45
+ interface IExternalRecovery {
46
+ /**
47
+ * Map of source header path -> that header's OWN preprocessed text (its
48
+ * macro-expanded declarations, without the transitively-inlined tree). Parse
49
+ * each with the real header parser to register full external symbols.
50
+ */
51
+ perFileContent: Map<string, string>;
52
+ /** Names of function-like macros — no declaration exists to parse. */
53
+ macroNames: Set<string>;
54
+ }
55
+
56
+ class ExternalDeclarationOracle {
57
+ /**
58
+ * Preprocess `includeDirectives` (e.g. `"<freertos/task.h>"`) as one TU and
59
+ * return each header's own preprocessed slice plus function-like-macro names.
60
+ *
61
+ * @param includeDirectives include specs, in source order (angle- or
62
+ * quote-form, e.g. `<Arduino.h>` or `"foo.h"`)
63
+ * @param preprocessor the shared Preprocessor (uses CNEXT_CROSS_COMPILER)
64
+ * @param options include paths + defines for the TU
65
+ */
66
+ static async recover(
67
+ includeDirectives: readonly string[],
68
+ preprocessor: Preprocessor,
69
+ options: IPreprocessOptions,
70
+ ): Promise<IExternalRecovery | null> {
71
+ if (includeDirectives.length === 0 || !preprocessor.isAvailable()) {
72
+ return null;
73
+ }
74
+
75
+ // KEEP line directives so the output can be split back into per-file slices.
76
+ const base: IPreprocessOptions = { ...options, keepLineDirectives: true };
77
+
78
+ // Build the largest subset that preprocesses cleanly (drop a header that
79
+ // can't preprocess — e.g. a fragile lvgl.h — instead of losing everything).
80
+ const working = await ExternalDeclarationOracle.preprocessLargestWorkingTu(
81
+ [...includeDirectives],
82
+ preprocessor,
83
+ base,
84
+ );
85
+ if (!working) return null;
86
+
87
+ const perFileContent = ExternalDeclarationOracle.splitByFile(
88
+ working.content,
89
+ );
90
+
91
+ // Function-like macros the plain preprocess would have consumed at use.
92
+ const macroNames = new Set<string>();
93
+ const macros = await preprocessor.preprocessString(
94
+ ExternalDeclarationOracle.buildTu(working.directives),
95
+ TU_NAME,
96
+ { ...options, keepLineDirectives: false, dumpMacros: true },
97
+ );
98
+ if (macros.success) {
99
+ for (const match of macros.content.matchAll(FUNCTION_MACRO)) {
100
+ macroNames.add(match[1]);
101
+ }
102
+ }
103
+
104
+ return { perFileContent, macroNames };
105
+ }
106
+
107
+ /**
108
+ * Bucket preprocessed output by originating source file using its `#line`
109
+ * markers. Synthetic units (`<built-in>`, `<command-line>`, the TU itself) are
110
+ * skipped. Each real header maps to the concatenation of its own emitted lines.
111
+ */
112
+ private static splitByFile(content: string): Map<string, string> {
113
+ const buckets = new Map<string, string[]>();
114
+ let current = "";
115
+ for (const line of content.split("\n")) {
116
+ const marker = LINE_MARKER.exec(line);
117
+ if (marker) {
118
+ current = marker[1];
119
+ continue;
120
+ }
121
+ if (!current || current.startsWith("<") || current.endsWith(TU_NAME)) {
122
+ continue;
123
+ }
124
+ let bucket = buckets.get(current);
125
+ if (!bucket) {
126
+ bucket = [];
127
+ buckets.set(current, bucket);
128
+ }
129
+ bucket.push(line);
130
+ }
131
+ const perFile = new Map<string, string>();
132
+ for (const [file, lines] of buckets) {
133
+ perFile.set(file, lines.join("\n"));
134
+ }
135
+ return perFile;
136
+ }
137
+
138
+ private static buildTu(directives: readonly string[]): string {
139
+ return directives.map((d) => `#include ${d}`).join("\n") + "\n";
140
+ }
141
+
142
+ /**
143
+ * Preprocess the directives as a TU; on failure, drop the single include that
144
+ * caused it (identified by its line in the synthetic TU) and retry. Returns
145
+ * the preprocessed content and the surviving directive list, or null.
146
+ */
147
+ private static async preprocessLargestWorkingTu(
148
+ directives: string[],
149
+ preprocessor: Preprocessor,
150
+ base: IPreprocessOptions,
151
+ ): Promise<{ content: string; directives: string[] } | null> {
152
+ const remaining = directives;
153
+ // At most one drop per iteration; bound iterations by the ORIGINAL directive
154
+ // count, captured once. `remaining` aliases `directives` and shrinks via
155
+ // splice() below, so re-reading `directives.length` in the bound would halve
156
+ // the allowed attempts and bail before trying the last surviving includes.
157
+ const initialCount = directives.length;
158
+ for (
159
+ let attempt = 0;
160
+ remaining.length > 0 && attempt <= initialCount;
161
+ attempt++
162
+ ) {
163
+ const result = await preprocessor.preprocessString(
164
+ ExternalDeclarationOracle.buildTu(remaining),
165
+ TU_NAME,
166
+ base,
167
+ );
168
+ if (result.success) {
169
+ return { content: result.content, directives: remaining };
170
+ }
171
+ const failedIndex = ExternalDeclarationOracle.findFailingIncludeIndex(
172
+ result.error,
173
+ remaining.length,
174
+ );
175
+ if (failedIndex < 0) return null; // can't localize — give up
176
+ remaining.splice(failedIndex, 1);
177
+ }
178
+ return null;
179
+ }
180
+
181
+ /**
182
+ * From a preprocessor error, find the index of the top-level `#include` that
183
+ * failed, via its line number in the synthetic TU (directive i is at line
184
+ * i+1). Returns -1 if it can't be localized.
185
+ */
186
+ private static findFailingIncludeIndex(
187
+ error: string | undefined,
188
+ count: number,
189
+ ): number {
190
+ if (!error) return -1;
191
+ const escapedTu = TU_NAME.replaceAll(".", String.raw`\.`);
192
+ const pattern = new RegExp(escapedTu + String.raw`:(\d+)`, "g");
193
+ let match: RegExpExecArray | null;
194
+ while ((match = pattern.exec(error)) !== null) {
195
+ const line = Number.parseInt(match[1], 10);
196
+ if (line >= 1 && line <= count) return line - 1;
197
+ }
198
+ return -1;
199
+ }
200
+ }
201
+
202
+ export default ExternalDeclarationOracle;
@@ -3,7 +3,7 @@
3
3
  * Runs the system preprocessor on C/C++ files before parsing
4
4
  */
5
5
 
6
- import { exec } from "node:child_process";
6
+ import { execFile } from "node:child_process";
7
7
  import { promisify } from "node:util";
8
8
  import { writeFile, mkdtemp, rm } from "node:fs/promises";
9
9
  import { tmpdir } from "node:os";
@@ -14,7 +14,7 @@ import ISourceMapping from "./types/ISourceMapping";
14
14
  import IPreprocessOptions from "./types/IPreprocessOptions";
15
15
  import ToolchainDetector from "./ToolchainDetector";
16
16
 
17
- const execAsync = promisify(exec);
17
+ const execFileAsync = promisify(execFile);
18
18
 
19
19
  /**
20
20
  * Handles preprocessing of C/C++ files
@@ -153,6 +153,12 @@ class Preprocessor {
153
153
  args.pop(); // Remove -P
154
154
  }
155
155
 
156
+ // Dump macro definitions (#define list) instead of preprocessed source,
157
+ // to discover function-like macros the normal preprocess would consume.
158
+ if (options.dumpMacros) {
159
+ args.push("-dM");
160
+ }
161
+
156
162
  // Add include paths
157
163
  const includePaths = [
158
164
  ...this.defaultIncludePaths,
@@ -175,14 +181,26 @@ class Preprocessor {
175
181
  }
176
182
  }
177
183
 
184
+ // Import predecessor macros (gcc/clang -imacros) so include-order-dependent
185
+ // headers get the guards/attribute macros their includer would have defined
186
+ // first. -imacros keeps only the macros, not the predecessors' declarations,
187
+ // so the output stays scoped to the target file.
188
+ if (options.imacros) {
189
+ for (const macroHeader of options.imacros) {
190
+ args.push("-imacros", macroHeader);
191
+ }
192
+ }
193
+
178
194
  // Add the input file
179
195
  args.push(filePath);
180
196
 
181
- // Build command
182
- const command = `${toolchain.cpp} ${args.join(" ")}`;
183
-
197
+ // Invoke the preprocessor via argv (execFile, NOT a shell). A shell would
198
+ // re-parse -D values that legitimately contain spaces / parentheses (e.g.
199
+ // -DARDUINO_BOARD="Espressif ... (8 MB QD, No PSRAM)"), breaking on the
200
+ // metacharacters. Passing args directly mirrors how the real compiler is
201
+ // invoked and is safe for any value the compiler accepts.
184
202
  try {
185
- const { stdout, stderr } = await execAsync(command, {
203
+ const { stdout, stderr } = await execFileAsync(toolchain.cpp, args, {
186
204
  maxBuffer: 50 * 1024 * 1024, // 50MB buffer for large headers
187
205
  });
188
206
 
@@ -5,7 +5,7 @@
5
5
 
6
6
  import { execSync } from "node:child_process";
7
7
  import { existsSync } from "node:fs";
8
- import { join } from "node:path";
8
+ import { basename, join } from "node:path";
9
9
  import IToolchain from "./types/IToolchain";
10
10
 
11
11
  /**
@@ -17,6 +17,16 @@ class ToolchainDetector {
17
17
  * Priority: ARM cross-compiler > clang > gcc
18
18
  */
19
19
  static detect(): IToolchain | null {
20
+ // Explicit override: a project can name the compiler that owns its target
21
+ // headers (e.g. a cross-compiler such as xtensa-esp32s3-elf-gcc) via
22
+ // CNEXT_CROSS_COMPILER. Host gcc/clang lack a cross target's system headers
23
+ // and predefined macros, so their preprocessing of target headers fails.
24
+ const override = process.env.CNEXT_CROSS_COMPILER;
25
+ if (override) {
26
+ const overridden = ToolchainDetector.fromPath(override);
27
+ if (overridden) return overridden;
28
+ }
29
+
20
30
  // Try ARM cross-compiler first (for embedded)
21
31
  const arm = this.detectArmToolchain();
22
32
  if (arm) return arm;
@@ -50,6 +60,37 @@ class ToolchainDetector {
50
60
  return toolchains;
51
61
  }
52
62
 
63
+ /**
64
+ * Build a toolchain from an explicit compiler path or executable name (e.g. a
65
+ * target cross-compiler such as xtensa-esp32s3-elf-gcc). Used by the
66
+ * CNEXT_CROSS_COMPILER override so a project can preprocess its target headers with the compiler
67
+ * that owns them.
68
+ */
69
+ static fromPath(compiler: string): IToolchain | null {
70
+ let cc: string | null;
71
+ if (compiler.includes("/")) {
72
+ cc = existsSync(compiler) ? compiler : null;
73
+ } else {
74
+ cc = this.findExecutable(compiler);
75
+ }
76
+ if (!cc) return null;
77
+
78
+ // Best-effort C++ driver alongside the C driver (only cpp is used for
79
+ // preprocessing; cxx is derived for completeness).
80
+ const cxxCandidate = cc.replace(/gcc(\.exe)?$/, "g++$1");
81
+ const cxx =
82
+ cxxCandidate !== cc && existsSync(cxxCandidate) ? cxxCandidate : cc;
83
+
84
+ return {
85
+ name: basename(cc),
86
+ cc,
87
+ cxx,
88
+ cpp: cc,
89
+ version: this.getVersion(cc),
90
+ isCrossCompiler: true,
91
+ };
92
+ }
93
+
53
94
  /**
54
95
  * Detect ARM cross-compiler (arm-none-eabi-gcc)
55
96
  */