@shapeshift-labs/frontier-lang-compiler 0.2.9 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -2
- package/bench/smoke.mjs +13 -0
- package/dist/index.d.ts +144 -0
- package/dist/index.js +680 -27
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -50,6 +50,8 @@ console.log(summary.categories);
|
|
|
50
50
|
console.log(readiness.readiness);
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
+
The loss taxonomy separates broad scanner limits from specific round-trip risks such as conditional compilation, reflection, overload/type-inference gaps, comments/trivia preservation, source-map approximation, parser diagnostics, and target projection loss. These records are evidence labels for merge admission; they are not claims that the lightweight scanner expanded macros, evaluated inactive branches, resolved overloads, or ran a type checker.
|
|
54
|
+
|
|
53
55
|
Ask the compiler what is actually covered before sending native imports into a merge queue:
|
|
54
56
|
|
|
55
57
|
```js
|
|
@@ -71,6 +73,28 @@ console.log(python.imports.readiness); // scanner imports are intentionally revi
|
|
|
71
73
|
console.log(python.parserAdapters); // host-owned exact parsers such as LibCST can be injected
|
|
72
74
|
```
|
|
73
75
|
|
|
76
|
+
Preserve exact native source text, token/trivia hashes, comments, whitespace, and source directives as evidence. This does not claim full semantic understanding; it keeps round-trip material available while exact parser adapters catch up:
|
|
77
|
+
|
|
78
|
+
```js
|
|
79
|
+
import {
|
|
80
|
+
createNativeSourcePreservation,
|
|
81
|
+
importNativeSource
|
|
82
|
+
} from '@shapeshift-labs/frontier-lang-compiler';
|
|
83
|
+
|
|
84
|
+
const sourceText = '// kept\nexport function step(frame) { return frame + 1; }\n';
|
|
85
|
+
const preservation = createNativeSourcePreservation({
|
|
86
|
+
language: 'javascript',
|
|
87
|
+
sourcePath: 'src/runtime.js',
|
|
88
|
+
sourceText
|
|
89
|
+
});
|
|
90
|
+
const imported = importNativeSource({ language: 'javascript', sourcePath: 'src/runtime.js', sourceText });
|
|
91
|
+
|
|
92
|
+
console.log(preservation.summary.comments); // comments and whitespace are tracked
|
|
93
|
+
console.log(imported.metadata.sourcePreservation.sourceHash);
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
When `sourceText` is present, hashes are computed from the actual text. Caller-provided hashes are recorded as declared metadata and cannot make stale text project as exact source. Use `includeTokens`, `includeTrivia`, `includeDirectives`, and `max*` options to keep preservation records compact for large files.
|
|
97
|
+
|
|
74
98
|
Create a compact semantic sidecar for swarm merge admission. This is the artifact a coordinator can index instead of reading a worker directory by hand:
|
|
75
99
|
|
|
76
100
|
```js
|
|
@@ -96,7 +120,7 @@ console.log(sidecar.ownershipRegions[0].key); // source#src/runtime.ts#class#Run
|
|
|
96
120
|
console.log(sidecar.patchHints[0].supportedOperations); // source-region patch operations
|
|
97
121
|
```
|
|
98
122
|
|
|
99
|
-
Project a native import back to source. Exact source is preserved
|
|
123
|
+
Project a native import back to source. Exact source is preserved when the import carries matching source-preservation evidence or when supplied text matches the import hash; otherwise the compiler emits declaration stubs with review-required loss evidence:
|
|
100
124
|
|
|
101
125
|
```js
|
|
102
126
|
import {
|
|
@@ -111,7 +135,7 @@ const imported = importNativeSource({
|
|
|
111
135
|
sourceText
|
|
112
136
|
});
|
|
113
137
|
|
|
114
|
-
const projection = projectNativeImportToSource(imported
|
|
138
|
+
const projection = projectNativeImportToSource(imported);
|
|
115
139
|
|
|
116
140
|
console.log(projection.mode); // "preserved-source"
|
|
117
141
|
console.log(projection.readiness.readiness); // "ready"
|
|
@@ -146,6 +170,9 @@ const project = await importNativeProject({
|
|
|
146
170
|
|
|
147
171
|
console.log(imported.universalAst.sourceMaps.length);
|
|
148
172
|
console.log(project.semanticIndex.symbols.length);
|
|
173
|
+
console.log(imported.adapter.coverage.exactness);
|
|
174
|
+
console.log(imported.adapter.coverage.semanticCoverage.level);
|
|
175
|
+
console.log(project.metadata.sourcePreservationSummary.total);
|
|
149
176
|
```
|
|
150
177
|
|
|
151
178
|
The built-in adapter factories are dependency-light wrappers for caller-owned parsers or ASTs:
|
|
@@ -155,6 +182,8 @@ The built-in adapter factories are dependency-light wrappers for caller-owned pa
|
|
|
155
182
|
- `createTypeScriptCompilerNativeImporterAdapter`
|
|
156
183
|
- `createTreeSitterNativeImporterAdapter`
|
|
157
184
|
|
|
185
|
+
Adapter summaries include a structured `coverage` record so merge queues can distinguish exact parser AST imports from declaration scans. The record declares exactness, parser token/trivia support, diagnostics support, source-range and generated-range support, and semantic coverage. Built-in wrappers normalize native AST/CST nodes and declaration-level semantic indexes; they do not claim resolved references, types, control flow, generated ranges, or token/trivia fidelity unless the host adapter supplies that evidence.
|
|
186
|
+
|
|
158
187
|
## Related Packages
|
|
159
188
|
|
|
160
189
|
The published Frontier package family is generated from one shared package catalog so READMEs stay in sync across packages:
|
package/bench/smoke.mjs
CHANGED
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
compileFrontierSource,
|
|
4
4
|
createEstreeNativeImporterAdapter,
|
|
5
5
|
createNativeImportCoverageMatrix,
|
|
6
|
+
createNativeSourcePreservation,
|
|
6
7
|
createSemanticImportSidecar,
|
|
7
8
|
importNativeSource,
|
|
8
9
|
projectNativeImportToSource,
|
|
@@ -67,6 +68,15 @@ const matrixStart = performance.now();
|
|
|
67
68
|
const coverageMatrix = createNativeImportCoverageMatrix({ imports: nativeImportResults });
|
|
68
69
|
const matrixDurationMs = performance.now() - matrixStart;
|
|
69
70
|
|
|
71
|
+
const preservationStart = performance.now();
|
|
72
|
+
const preservationRecords = nativeImportResults.map((imported) => imported.metadata.sourcePreservation ?? createNativeSourcePreservation({
|
|
73
|
+
language: imported.language,
|
|
74
|
+
sourcePath: imported.sourcePath,
|
|
75
|
+
sourceText: imported.metadata.sourcePreservation?.sourceText ?? ''
|
|
76
|
+
}));
|
|
77
|
+
const preservationDurationMs = performance.now() - preservationStart;
|
|
78
|
+
const preservationTokens = preservationRecords.reduce((sum, record) => sum + record.tokens.length + record.trivia.length, 0);
|
|
79
|
+
|
|
70
80
|
const sidecarStart = performance.now();
|
|
71
81
|
const semanticSidecars = nativeImportResults.map((imported) => createSemanticImportSidecar(imported));
|
|
72
82
|
const sidecarDurationMs = performance.now() - sidecarStart;
|
|
@@ -87,6 +97,9 @@ console.log(JSON.stringify({
|
|
|
87
97
|
coverageMatrixLanguages: coverageMatrix.summary.languages,
|
|
88
98
|
coverageMatrixImports: coverageMatrix.summary.imports,
|
|
89
99
|
coverageMatrixDurationMs: Number(matrixDurationMs.toFixed(2)),
|
|
100
|
+
sourcePreservationRecords: preservationRecords.length,
|
|
101
|
+
sourcePreservationTokens: preservationTokens,
|
|
102
|
+
sourcePreservationDurationMs: Number(preservationDurationMs.toFixed(2)),
|
|
90
103
|
semanticSidecars: semanticSidecars.length,
|
|
91
104
|
sidecarOwnershipRegions,
|
|
92
105
|
sidecarDurationMs: Number(sidecarDurationMs.toFixed(2)),
|
package/dist/index.d.ts
CHANGED
|
@@ -79,27 +79,42 @@ export type NativeImportTaxonomyKind =
|
|
|
79
79
|
| 'opaqueBodies'
|
|
80
80
|
| 'macroExpansion'
|
|
81
81
|
| 'preprocessor'
|
|
82
|
+
| 'conditionalCompilation'
|
|
82
83
|
| 'metaprogramming'
|
|
84
|
+
| 'reflection'
|
|
83
85
|
| 'generatedCode'
|
|
86
|
+
| 'overloadTypeInference'
|
|
84
87
|
| 'sourcePreservation'
|
|
88
|
+
| 'commentsTrivia'
|
|
85
89
|
| 'parserDiagnostics'
|
|
86
90
|
| 'unsupportedSyntax'
|
|
87
91
|
| 'partialSemanticIndex'
|
|
88
92
|
| 'sourceMapApproximation'
|
|
93
|
+
| 'targetProjectionLoss'
|
|
89
94
|
| string;
|
|
90
95
|
|
|
91
96
|
export type NativeImportKnownLossKind =
|
|
92
97
|
| 'declarationOnlyCoverage'
|
|
93
98
|
| 'opaqueNative'
|
|
94
99
|
| 'macroExpansion'
|
|
100
|
+
| 'macroHygiene'
|
|
95
101
|
| 'preprocessor'
|
|
102
|
+
| 'conditionalCompilation'
|
|
96
103
|
| 'metaprogramming'
|
|
104
|
+
| 'reflection'
|
|
105
|
+
| 'dynamicRuntime'
|
|
106
|
+
| 'dynamicDispatch'
|
|
97
107
|
| 'generatedCode'
|
|
108
|
+
| 'overloadResolution'
|
|
109
|
+
| 'typeInference'
|
|
98
110
|
| 'sourcePreservation'
|
|
111
|
+
| 'commentsTrivia'
|
|
99
112
|
| 'parserDiagnostic'
|
|
100
113
|
| 'unsupportedSyntax'
|
|
114
|
+
| 'unsupportedSemantic'
|
|
101
115
|
| 'partialSemanticIndex'
|
|
102
116
|
| 'sourceMapApproximation'
|
|
117
|
+
| 'targetProjectionLoss'
|
|
103
118
|
| string;
|
|
104
119
|
|
|
105
120
|
export interface NativeImportLossSummaryOptions {
|
|
@@ -201,6 +216,82 @@ export interface NativeImportCoverageMatrixOptions {
|
|
|
201
216
|
readonly generatedAt?: number;
|
|
202
217
|
}
|
|
203
218
|
|
|
219
|
+
export type NativeSourceTokenKind =
|
|
220
|
+
| 'identifier'
|
|
221
|
+
| 'keyword'
|
|
222
|
+
| 'number'
|
|
223
|
+
| 'string'
|
|
224
|
+
| 'operator'
|
|
225
|
+
| 'punctuation'
|
|
226
|
+
| 'comment'
|
|
227
|
+
| 'whitespace'
|
|
228
|
+
| 'newline'
|
|
229
|
+
| 'directive'
|
|
230
|
+
| 'unknown'
|
|
231
|
+
| string;
|
|
232
|
+
|
|
233
|
+
export interface NativeSourcePreservedToken {
|
|
234
|
+
readonly id: string;
|
|
235
|
+
readonly kind: NativeSourceTokenKind;
|
|
236
|
+
readonly text?: string;
|
|
237
|
+
readonly textHash: string;
|
|
238
|
+
readonly span: SourceSpan;
|
|
239
|
+
readonly metadata?: Record<string, unknown>;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export interface NativeSourcePreservedDirective {
|
|
243
|
+
readonly id: string;
|
|
244
|
+
readonly kind: string;
|
|
245
|
+
readonly text?: string;
|
|
246
|
+
readonly textHash: string;
|
|
247
|
+
readonly span: SourceSpan;
|
|
248
|
+
readonly metadata?: Record<string, unknown>;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export interface NativeSourcePreservation {
|
|
252
|
+
readonly kind: 'frontier.lang.nativeSourcePreservation';
|
|
253
|
+
readonly version: 1;
|
|
254
|
+
readonly id: string;
|
|
255
|
+
readonly language: FrontierSourceLanguage | string;
|
|
256
|
+
readonly sourcePath?: string;
|
|
257
|
+
readonly sourceHash: string;
|
|
258
|
+
readonly sourceBytes: number;
|
|
259
|
+
readonly lineCount: number;
|
|
260
|
+
readonly newline: 'lf' | 'crlf' | 'mixed' | 'none';
|
|
261
|
+
readonly encoding: string;
|
|
262
|
+
readonly sourceText?: string;
|
|
263
|
+
readonly tokens: readonly NativeSourcePreservedToken[];
|
|
264
|
+
readonly trivia: readonly NativeSourcePreservedToken[];
|
|
265
|
+
readonly directives: readonly NativeSourcePreservedDirective[];
|
|
266
|
+
readonly summary: {
|
|
267
|
+
readonly tokens: number;
|
|
268
|
+
readonly trivia: number;
|
|
269
|
+
readonly directives: number;
|
|
270
|
+
readonly comments: number;
|
|
271
|
+
readonly whitespace: number;
|
|
272
|
+
readonly exactSourceAvailable: boolean;
|
|
273
|
+
readonly truncated: boolean;
|
|
274
|
+
};
|
|
275
|
+
readonly metadata?: Record<string, unknown>;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
export interface CreateNativeSourcePreservationOptions {
|
|
279
|
+
readonly id?: string;
|
|
280
|
+
readonly language?: FrontierSourceLanguage | string;
|
|
281
|
+
readonly sourcePath?: string;
|
|
282
|
+
readonly sourceHash?: string;
|
|
283
|
+
readonly sourceText: string;
|
|
284
|
+
readonly encoding?: string;
|
|
285
|
+
readonly includeSourceText?: boolean;
|
|
286
|
+
readonly includeTokens?: boolean;
|
|
287
|
+
readonly includeTrivia?: boolean;
|
|
288
|
+
readonly includeDirectives?: boolean;
|
|
289
|
+
readonly maxTokens?: number;
|
|
290
|
+
readonly maxTrivia?: number;
|
|
291
|
+
readonly maxDirectives?: number;
|
|
292
|
+
readonly metadata?: Record<string, unknown>;
|
|
293
|
+
}
|
|
294
|
+
|
|
204
295
|
export interface SemanticImportOwnershipRegion {
|
|
205
296
|
readonly id: string;
|
|
206
297
|
readonly key: string;
|
|
@@ -322,6 +413,52 @@ export interface SemanticImportSidecarOptions {
|
|
|
322
413
|
readonly metadata?: Record<string, unknown>;
|
|
323
414
|
}
|
|
324
415
|
|
|
416
|
+
export type NativeImporterAdapterExactness =
|
|
417
|
+
| 'exact-parser-ast'
|
|
418
|
+
| 'parser-tree'
|
|
419
|
+
| 'adapter-reported-native-ast'
|
|
420
|
+
| 'loss-aware-native-ast'
|
|
421
|
+
| 'unknown'
|
|
422
|
+
| string;
|
|
423
|
+
|
|
424
|
+
export interface NativeImporterAdapterSemanticCoverage {
|
|
425
|
+
readonly level: 'native-ast' | 'declaration-index' | 'semantic-index' | string;
|
|
426
|
+
readonly declarations: boolean;
|
|
427
|
+
readonly symbols: boolean;
|
|
428
|
+
readonly references: boolean;
|
|
429
|
+
readonly types: boolean;
|
|
430
|
+
readonly controlFlow: boolean;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
export interface NativeImporterAdapterCoverageObserved {
|
|
434
|
+
readonly diagnostics: number;
|
|
435
|
+
readonly losses: number;
|
|
436
|
+
readonly nativeAstNodes: number;
|
|
437
|
+
readonly semanticSymbols: number;
|
|
438
|
+
readonly sourceMapMappings: number;
|
|
439
|
+
readonly sourceRanges: boolean;
|
|
440
|
+
readonly generatedRanges: boolean;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
export interface NativeImporterAdapterCoverageSummary {
|
|
444
|
+
readonly exactness: NativeImporterAdapterExactness;
|
|
445
|
+
readonly exactAst: boolean;
|
|
446
|
+
readonly tokens: boolean;
|
|
447
|
+
readonly trivia: boolean;
|
|
448
|
+
readonly diagnostics: boolean;
|
|
449
|
+
readonly sourceRanges: boolean;
|
|
450
|
+
readonly generatedRanges: boolean;
|
|
451
|
+
readonly semanticCoverage: NativeImporterAdapterSemanticCoverage;
|
|
452
|
+
readonly notes: readonly string[];
|
|
453
|
+
readonly observed?: NativeImporterAdapterCoverageObserved;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
export type NativeImporterAdapterCoverageInput =
|
|
457
|
+
Omit<Partial<NativeImporterAdapterCoverageSummary>, 'semanticCoverage' | 'observed'> & {
|
|
458
|
+
readonly semanticCoverage?: Partial<NativeImporterAdapterSemanticCoverage>;
|
|
459
|
+
readonly observed?: Partial<NativeImporterAdapterCoverageObserved>;
|
|
460
|
+
};
|
|
461
|
+
|
|
325
462
|
export interface NativeImporterAdapterDiagnostic {
|
|
326
463
|
readonly id?: string;
|
|
327
464
|
readonly severity?: 'info' | 'warning' | 'error';
|
|
@@ -362,6 +499,7 @@ export interface ImportNativeSourceOptions {
|
|
|
362
499
|
readonly losses?: readonly NativeAstLossRecord[];
|
|
363
500
|
readonly evidence?: readonly EvidenceRecord[];
|
|
364
501
|
readonly evidenceId?: string;
|
|
502
|
+
readonly sourcePreservation?: NativeSourcePreservation;
|
|
365
503
|
readonly patch?: SemanticPatchBundle;
|
|
366
504
|
readonly patchId?: string;
|
|
367
505
|
readonly author?: string;
|
|
@@ -405,6 +543,7 @@ export interface NativeImporterAdapter {
|
|
|
405
543
|
readonly parser: string;
|
|
406
544
|
readonly version?: string;
|
|
407
545
|
readonly capabilities?: readonly string[];
|
|
546
|
+
readonly coverage?: NativeImporterAdapterCoverageInput;
|
|
408
547
|
readonly supportedExtensions?: readonly string[];
|
|
409
548
|
readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
|
|
410
549
|
readonly parse: (input: NativeImporterAdapterParseInput) => NativeImporterAdapterParseResult | Promise<NativeImporterAdapterParseResult>;
|
|
@@ -416,6 +555,7 @@ export interface JavaScriptNativeImporterAdapterOptions {
|
|
|
416
555
|
readonly parser?: string;
|
|
417
556
|
readonly version?: string;
|
|
418
557
|
readonly capabilities?: readonly string[];
|
|
558
|
+
readonly coverage?: NativeImporterAdapterCoverageInput;
|
|
419
559
|
readonly supportedExtensions?: readonly string[];
|
|
420
560
|
readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
|
|
421
561
|
readonly ast?: unknown;
|
|
@@ -432,6 +572,7 @@ export interface TypeScriptCompilerNativeImporterAdapterOptions {
|
|
|
432
572
|
readonly parser?: string;
|
|
433
573
|
readonly version?: string;
|
|
434
574
|
readonly capabilities?: readonly string[];
|
|
575
|
+
readonly coverage?: NativeImporterAdapterCoverageInput;
|
|
435
576
|
readonly supportedExtensions?: readonly string[];
|
|
436
577
|
readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
|
|
437
578
|
readonly typescript?: unknown;
|
|
@@ -451,6 +592,7 @@ export interface TreeSitterNativeImporterAdapterOptions {
|
|
|
451
592
|
readonly parserName?: string;
|
|
452
593
|
readonly version?: string;
|
|
453
594
|
readonly capabilities?: readonly string[];
|
|
595
|
+
readonly coverage?: NativeImporterAdapterCoverageInput;
|
|
454
596
|
readonly supportedExtensions?: readonly string[];
|
|
455
597
|
readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
|
|
456
598
|
readonly parserInstance?: { readonly parse: (sourceText: string) => unknown };
|
|
@@ -466,6 +608,7 @@ export interface NativeImporterAdapterSummary {
|
|
|
466
608
|
readonly parser: string;
|
|
467
609
|
readonly version?: string;
|
|
468
610
|
readonly capabilities: readonly string[];
|
|
611
|
+
readonly coverage: NativeImporterAdapterCoverageSummary;
|
|
469
612
|
readonly supportedExtensions: readonly string[];
|
|
470
613
|
readonly diagnostics: readonly NativeImporterAdapterDiagnostic[];
|
|
471
614
|
}
|
|
@@ -595,6 +738,7 @@ export declare function resolveCapabilityAdapters(document: FrontierLangDocument
|
|
|
595
738
|
export declare function summarizeNativeImportLosses(losses?: readonly NativeAstLossRecord[], options?: NativeImportLossSummaryOptions): NativeImportLossSummary;
|
|
596
739
|
export declare function classifyNativeImportReadiness(losses?: readonly NativeAstLossRecord[], options?: NativeImportLossSummaryOptions): NativeImportReadinessClassification;
|
|
597
740
|
export declare function createNativeImportCoverageMatrix(options?: NativeImportCoverageMatrixOptions): NativeImportCoverageMatrix;
|
|
741
|
+
export declare function createNativeSourcePreservation(options: CreateNativeSourcePreservationOptions): NativeSourcePreservation;
|
|
598
742
|
export declare function createSemanticImportSidecar(importResult: NativeSourceImportResult | NativeProjectImportResult, options?: SemanticImportSidecarOptions): SemanticImportSidecar;
|
|
599
743
|
export declare function createEstreeNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
600
744
|
export declare function createBabelNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
|
package/dist/index.js
CHANGED
|
@@ -73,27 +73,42 @@ export const NativeImportTaxonomyKinds = Object.freeze([
|
|
|
73
73
|
'opaqueBodies',
|
|
74
74
|
'macroExpansion',
|
|
75
75
|
'preprocessor',
|
|
76
|
+
'conditionalCompilation',
|
|
76
77
|
'metaprogramming',
|
|
78
|
+
'reflection',
|
|
77
79
|
'generatedCode',
|
|
80
|
+
'overloadTypeInference',
|
|
78
81
|
'sourcePreservation',
|
|
82
|
+
'commentsTrivia',
|
|
79
83
|
'parserDiagnostics',
|
|
80
84
|
'unsupportedSyntax',
|
|
81
85
|
'partialSemanticIndex',
|
|
82
|
-
'sourceMapApproximation'
|
|
86
|
+
'sourceMapApproximation',
|
|
87
|
+
'targetProjectionLoss'
|
|
83
88
|
]);
|
|
84
89
|
|
|
85
90
|
export const NativeImportLossKinds = Object.freeze([
|
|
86
91
|
'declarationOnlyCoverage',
|
|
87
92
|
'opaqueNative',
|
|
88
93
|
'macroExpansion',
|
|
94
|
+
'macroHygiene',
|
|
89
95
|
'preprocessor',
|
|
96
|
+
'conditionalCompilation',
|
|
90
97
|
'metaprogramming',
|
|
98
|
+
'reflection',
|
|
99
|
+
'dynamicRuntime',
|
|
100
|
+
'dynamicDispatch',
|
|
91
101
|
'generatedCode',
|
|
102
|
+
'overloadResolution',
|
|
103
|
+
'typeInference',
|
|
92
104
|
'sourcePreservation',
|
|
105
|
+
'commentsTrivia',
|
|
93
106
|
'parserDiagnostic',
|
|
94
107
|
'unsupportedSyntax',
|
|
108
|
+
'unsupportedSemantic',
|
|
95
109
|
'partialSemanticIndex',
|
|
96
|
-
'sourceMapApproximation'
|
|
110
|
+
'sourceMapApproximation',
|
|
111
|
+
'targetProjectionLoss'
|
|
97
112
|
]);
|
|
98
113
|
|
|
99
114
|
export const NativeImportReadinessBySeverity = Object.freeze({
|
|
@@ -356,6 +371,70 @@ export function createNativeImportCoverageMatrix(input = {}) {
|
|
|
356
371
|
};
|
|
357
372
|
}
|
|
358
373
|
|
|
374
|
+
export function createNativeSourcePreservation(options) {
|
|
375
|
+
if (!options || typeof options.sourceText !== 'string') {
|
|
376
|
+
throw new Error('createNativeSourcePreservation requires sourceText');
|
|
377
|
+
}
|
|
378
|
+
const language = options.language ?? 'source';
|
|
379
|
+
const sourceText = options.sourceText;
|
|
380
|
+
const computedSourceHash = hashSemanticValue(sourceText);
|
|
381
|
+
const declaredSourceHash = options.sourceHash;
|
|
382
|
+
const sourceHash = computedSourceHash;
|
|
383
|
+
const tokensAndTrivia = scanPreservedSourceTokens(sourceText, {
|
|
384
|
+
language,
|
|
385
|
+
sourcePath: options.sourcePath,
|
|
386
|
+
sourceHash,
|
|
387
|
+
includeTokens: options.includeTokens !== false,
|
|
388
|
+
includeTrivia: options.includeTrivia !== false,
|
|
389
|
+
maxTokens: options.maxTokens,
|
|
390
|
+
maxTrivia: options.maxTrivia
|
|
391
|
+
});
|
|
392
|
+
const directiveScan = options.includeDirectives === false
|
|
393
|
+
? { directives: [], truncated: false }
|
|
394
|
+
: scanPreservedSourceDirectives(sourceText, {
|
|
395
|
+
language,
|
|
396
|
+
sourcePath: options.sourcePath,
|
|
397
|
+
sourceHash,
|
|
398
|
+
maxDirectives: options.maxDirectives
|
|
399
|
+
});
|
|
400
|
+
const directives = directiveScan.directives;
|
|
401
|
+
const newline = detectNewlineStyle(sourceText);
|
|
402
|
+
return {
|
|
403
|
+
kind: 'frontier.lang.nativeSourcePreservation',
|
|
404
|
+
version: 1,
|
|
405
|
+
id: options.id ?? `native_source_preservation_${idFragment(options.sourcePath ?? language)}_${idFragment(sourceHash)}`,
|
|
406
|
+
language,
|
|
407
|
+
sourcePath: options.sourcePath,
|
|
408
|
+
sourceHash,
|
|
409
|
+
sourceBytes: Buffer.byteLength(sourceText, options.encoding ?? 'utf8'),
|
|
410
|
+
lineCount: sourceText.length ? sourceText.split(/\r\n|\r|\n/).length : 0,
|
|
411
|
+
newline,
|
|
412
|
+
encoding: options.encoding ?? 'utf8',
|
|
413
|
+
...(options.includeSourceText === false ? {} : { sourceText }),
|
|
414
|
+
tokens: tokensAndTrivia.tokens,
|
|
415
|
+
trivia: tokensAndTrivia.trivia,
|
|
416
|
+
directives,
|
|
417
|
+
summary: {
|
|
418
|
+
tokens: tokensAndTrivia.tokens.length,
|
|
419
|
+
trivia: tokensAndTrivia.trivia.length,
|
|
420
|
+
directives: directives.length,
|
|
421
|
+
comments: tokensAndTrivia.trivia.filter((entry) => entry.kind === 'comment').length,
|
|
422
|
+
whitespace: tokensAndTrivia.trivia.filter((entry) => entry.kind === 'whitespace' || entry.kind === 'newline').length,
|
|
423
|
+
exactSourceAvailable: options.includeSourceText !== false,
|
|
424
|
+
truncated: tokensAndTrivia.truncated || directiveScan.truncated
|
|
425
|
+
},
|
|
426
|
+
metadata: {
|
|
427
|
+
preservation: 'source-text-token-trivia-directive-evidence',
|
|
428
|
+
tokenization: 'frontier-lightweight-lexical-scan',
|
|
429
|
+
...(declaredSourceHash ? {
|
|
430
|
+
declaredSourceHash,
|
|
431
|
+
sourceHashVerified: declaredSourceHash === computedSourceHash
|
|
432
|
+
} : {}),
|
|
433
|
+
...options.metadata
|
|
434
|
+
}
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
359
438
|
export function createSemanticImportSidecar(importResult, options = {}) {
|
|
360
439
|
const imports = Array.isArray(importResult?.imports) ? importResult.imports : [importResult].filter(Boolean);
|
|
361
440
|
const importEntries = imports.map((imported, index) => semanticImportSidecarEntry(imported, index, options));
|
|
@@ -458,6 +537,20 @@ export function createTypeScriptCompilerNativeImporterAdapter(options = {}) {
|
|
|
458
537
|
parser: options.parser ?? 'typescript-compiler-api',
|
|
459
538
|
version: options.version,
|
|
460
539
|
capabilities: uniqueStrings(['nativeAst', 'semanticIndex', 'sourceMaps', 'diagnostics', ...(options.capabilities ?? [])]),
|
|
540
|
+
coverage: nativeImporterAdapterCoverage({
|
|
541
|
+
exactness: 'exact-parser-ast',
|
|
542
|
+
exactAst: true,
|
|
543
|
+
tokens: false,
|
|
544
|
+
trivia: false,
|
|
545
|
+
diagnostics: true,
|
|
546
|
+
sourceRanges: true,
|
|
547
|
+
generatedRanges: false,
|
|
548
|
+
semanticCoverage: declarationSemanticCoverage(),
|
|
549
|
+
notes: [
|
|
550
|
+
'Normalizes a caller-owned TypeScript SourceFile into native AST nodes and declaration-level semantic index records.',
|
|
551
|
+
'Type resolution, reference resolution, control flow, generated ranges, and parser token/trivia streams require host-supplied adapter evidence.'
|
|
552
|
+
]
|
|
553
|
+
}, options.coverage),
|
|
461
554
|
supportedExtensions: options.supportedExtensions ?? ['.ts', '.tsx', '.js', '.jsx'],
|
|
462
555
|
diagnostics: options.diagnostics,
|
|
463
556
|
parse(input) {
|
|
@@ -488,6 +581,20 @@ export function createTreeSitterNativeImporterAdapter(options = {}) {
|
|
|
488
581
|
parser: options.parserName ?? options.parser ?? 'tree-sitter',
|
|
489
582
|
version: options.version,
|
|
490
583
|
capabilities: uniqueStrings(['nativeAst', 'semanticIndex', 'sourceMaps', 'diagnostics', ...(options.capabilities ?? [])]),
|
|
584
|
+
coverage: nativeImporterAdapterCoverage({
|
|
585
|
+
exactness: 'parser-tree',
|
|
586
|
+
exactAst: true,
|
|
587
|
+
tokens: false,
|
|
588
|
+
trivia: false,
|
|
589
|
+
diagnostics: true,
|
|
590
|
+
sourceRanges: true,
|
|
591
|
+
generatedRanges: false,
|
|
592
|
+
semanticCoverage: declarationSemanticCoverage(),
|
|
593
|
+
notes: [
|
|
594
|
+
'Normalizes a caller-owned tree-sitter tree into native AST nodes and declaration-level semantic index records.',
|
|
595
|
+
'The built-in wrapper walks named syntax nodes; exact token/trivia streams and generated ranges require adapter-specific evidence.'
|
|
596
|
+
]
|
|
597
|
+
}, options.coverage),
|
|
491
598
|
supportedExtensions: options.supportedExtensions ?? [],
|
|
492
599
|
diagnostics: options.diagnostics,
|
|
493
600
|
parse(input) {
|
|
@@ -592,7 +699,14 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
|
|
|
592
699
|
parseResult.losses,
|
|
593
700
|
diagnostics.map((diagnostic, index) => adapterDiagnosticToLoss(diagnostic, index, summary, parseInput))
|
|
594
701
|
);
|
|
595
|
-
const
|
|
702
|
+
const adapterSummary = {
|
|
703
|
+
...summary,
|
|
704
|
+
coverage: observeNativeImporterAdapterCoverage(summary.coverage, parseResult, {
|
|
705
|
+
diagnostics,
|
|
706
|
+
losses
|
|
707
|
+
})
|
|
708
|
+
};
|
|
709
|
+
const sourceEvidence = adapterDiagnosticsEvidence(adapterSummary, diagnostics, {
|
|
596
710
|
language,
|
|
597
711
|
parser,
|
|
598
712
|
parserVersion,
|
|
@@ -614,8 +728,9 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
|
|
|
614
728
|
metadata: {
|
|
615
729
|
adapterId: summary.id,
|
|
616
730
|
adapterVersion: summary.version,
|
|
617
|
-
adapterCapabilities:
|
|
618
|
-
|
|
731
|
+
adapterCapabilities: adapterSummary.capabilities,
|
|
732
|
+
adapterCoverage: adapterSummary.coverage,
|
|
733
|
+
supportedExtensions: adapterSummary.supportedExtensions,
|
|
619
734
|
diagnostics: diagnostics.map(serializableDiagnostic),
|
|
620
735
|
...input.metadata,
|
|
621
736
|
...parseResult.metadata
|
|
@@ -624,6 +739,7 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
|
|
|
624
739
|
adapterId: summary.id,
|
|
625
740
|
adapterVersion: summary.version,
|
|
626
741
|
parser,
|
|
742
|
+
adapterCoverage: adapterSummary.coverage,
|
|
627
743
|
...input.nativeAstMetadata,
|
|
628
744
|
...parseResult.nativeAstMetadata
|
|
629
745
|
},
|
|
@@ -631,6 +747,7 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
|
|
|
631
747
|
adapterId: summary.id,
|
|
632
748
|
adapterVersion: summary.version,
|
|
633
749
|
parser,
|
|
750
|
+
adapterCoverage: adapterSummary.coverage,
|
|
634
751
|
...input.nativeSourceMetadata,
|
|
635
752
|
...parseResult.nativeSourceMetadata
|
|
636
753
|
},
|
|
@@ -649,7 +766,7 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
|
|
|
649
766
|
};
|
|
650
767
|
return {
|
|
651
768
|
...importNativeSource(importInput),
|
|
652
|
-
adapter:
|
|
769
|
+
adapter: adapterSummary,
|
|
653
770
|
diagnostics
|
|
654
771
|
};
|
|
655
772
|
}
|
|
@@ -681,6 +798,7 @@ export function projectNativeImportToSource(importResult, options = {}) {
|
|
|
681
798
|
sourcePath: context.sourcePath,
|
|
682
799
|
expectedSourceHash: context.sourceHash,
|
|
683
800
|
providedSourceHash: candidateSource?.sourceHash,
|
|
801
|
+
sourcePreservationId: candidateSource?.sourcePreservationId,
|
|
684
802
|
sourceHashVerified: candidateSource?.hashVerified ?? false,
|
|
685
803
|
declarationCount: declarations.length
|
|
686
804
|
}
|
|
@@ -725,6 +843,7 @@ export function projectNativeImportToSource(importResult, options = {}) {
|
|
|
725
843
|
universalAstId: importResult.universalAst?.id,
|
|
726
844
|
exactSourceAvailable: candidateSource?.exact === true,
|
|
727
845
|
sourceTextAvailable: typeof candidateSource?.sourceText === 'string',
|
|
846
|
+
sourcePreservationId: candidateSource?.sourcePreservationId,
|
|
728
847
|
sourceHashVerified: candidateSource?.hashVerified ?? false,
|
|
729
848
|
nativeImportLossSummary,
|
|
730
849
|
...options.metadata
|
|
@@ -736,17 +855,30 @@ export function importNativeSource(input) {
|
|
|
736
855
|
const language = input.language ?? input.nativeAst?.language;
|
|
737
856
|
if (!language) throw new Error('importNativeSource requires a language or nativeAst.language');
|
|
738
857
|
const sourcePath = input.sourcePath ?? input.nativeAst?.sourcePath;
|
|
739
|
-
const
|
|
858
|
+
const declaredSourceHash = input.sourceHash ?? input.nativeAst?.sourceHash;
|
|
859
|
+
const sourceHash = typeof input.sourceText === 'string'
|
|
860
|
+
? hashSemanticValue(input.sourceText)
|
|
861
|
+
: declaredSourceHash ?? hashSemanticValue(input.nativeAst?.nodes ?? input.nativeAst ?? {});
|
|
740
862
|
const targetPath = input.targetPath ?? input.target?.emitPath;
|
|
741
863
|
const targetHash = input.targetHash;
|
|
742
864
|
const importIdPart = idFragment(input.id ?? input.nativeSourceId ?? sourcePath ?? language);
|
|
865
|
+
const sourcePreservation = input.sourcePreservation ?? (typeof input.sourceText === 'string'
|
|
866
|
+
? createNativeSourcePreservation({
|
|
867
|
+
language,
|
|
868
|
+
sourcePath,
|
|
869
|
+
sourceHash: declaredSourceHash,
|
|
870
|
+
sourceText: input.sourceText,
|
|
871
|
+
metadata: { importIdPart }
|
|
872
|
+
})
|
|
873
|
+
: undefined);
|
|
743
874
|
const lightweight = !input.nativeAst && !input.nodes && input.sourceText
|
|
744
875
|
? createLightweightNativeImport({
|
|
745
876
|
language,
|
|
746
877
|
sourceText: input.sourceText,
|
|
747
878
|
sourcePath,
|
|
748
879
|
sourceHash,
|
|
749
|
-
parser: input.parser
|
|
880
|
+
parser: input.parser,
|
|
881
|
+
sourcePreservation
|
|
750
882
|
})
|
|
751
883
|
: undefined;
|
|
752
884
|
const nativeAst = input.nativeAst ?? createNativeAstRecord({
|
|
@@ -769,6 +901,15 @@ export function importNativeSource(input) {
|
|
|
769
901
|
losses: input.losses ?? lightweight?.losses,
|
|
770
902
|
metadata: {
|
|
771
903
|
...(input.sourceText ? { sourceBytes: input.sourceText.length } : {}),
|
|
904
|
+
...(sourcePreservation ? {
|
|
905
|
+
sourcePreservationId: sourcePreservation.id,
|
|
906
|
+
sourcePreservationSummary: sourcePreservation.summary,
|
|
907
|
+
sourcePreservation
|
|
908
|
+
} : {}),
|
|
909
|
+
...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
|
|
910
|
+
declaredSourceHash,
|
|
911
|
+
sourceHashVerified: false
|
|
912
|
+
} : {}),
|
|
772
913
|
...lightweight?.metadata,
|
|
773
914
|
...input.nativeAstMetadata
|
|
774
915
|
}
|
|
@@ -793,6 +934,14 @@ export function importNativeSource(input) {
|
|
|
793
934
|
metadata: {
|
|
794
935
|
semanticStatus,
|
|
795
936
|
mappings: input.mappings ?? [],
|
|
937
|
+
...(sourcePreservation ? {
|
|
938
|
+
sourcePreservationId: sourcePreservation.id,
|
|
939
|
+
sourcePreservation
|
|
940
|
+
} : {}),
|
|
941
|
+
...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
|
|
942
|
+
declaredSourceHash,
|
|
943
|
+
sourceHashVerified: false
|
|
944
|
+
} : {}),
|
|
796
945
|
...input.nativeSourceMetadata
|
|
797
946
|
}
|
|
798
947
|
});
|
|
@@ -816,7 +965,16 @@ export function importNativeSource(input) {
|
|
|
816
965
|
metadata: {
|
|
817
966
|
parser: nativeAst.parser,
|
|
818
967
|
sourcePath,
|
|
819
|
-
semanticStatus
|
|
968
|
+
semanticStatus,
|
|
969
|
+
...(sourcePreservation ? {
|
|
970
|
+
sourcePreservationId: sourcePreservation.id,
|
|
971
|
+
sourcePreservationSummary: sourcePreservation.summary
|
|
972
|
+
} : {})
|
|
973
|
+
,
|
|
974
|
+
...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
|
|
975
|
+
declaredSourceHash,
|
|
976
|
+
sourceHashVerified: false
|
|
977
|
+
} : {})
|
|
820
978
|
}
|
|
821
979
|
}];
|
|
822
980
|
const lossSummary = summarizeNativeImportLosses(losses, {
|
|
@@ -896,6 +1054,14 @@ export function importNativeSource(input) {
|
|
|
896
1054
|
sourcePath,
|
|
897
1055
|
semanticStatus,
|
|
898
1056
|
nativeImportLossSummary: lossSummary,
|
|
1057
|
+
...(sourcePreservation ? {
|
|
1058
|
+
sourcePreservationId: sourcePreservation.id,
|
|
1059
|
+
sourcePreservation
|
|
1060
|
+
} : {}),
|
|
1061
|
+
...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
|
|
1062
|
+
declaredSourceHash,
|
|
1063
|
+
sourceHashVerified: false
|
|
1064
|
+
} : {}),
|
|
899
1065
|
...input.universalAstMetadata
|
|
900
1066
|
}
|
|
901
1067
|
});
|
|
@@ -915,6 +1081,14 @@ export function importNativeSource(input) {
|
|
|
915
1081
|
semanticIndexId: semanticIndex?.id,
|
|
916
1082
|
universalAstId: universalAst.id,
|
|
917
1083
|
sourceMapIds: sourceMaps.map((sourceMap) => sourceMap.id),
|
|
1084
|
+
...(sourcePreservation ? {
|
|
1085
|
+
sourcePreservationId: sourcePreservation.id,
|
|
1086
|
+
sourcePreservationSummary: sourcePreservation.summary
|
|
1087
|
+
} : {}),
|
|
1088
|
+
...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
|
|
1089
|
+
declaredSourceHash,
|
|
1090
|
+
sourceHashVerified: false
|
|
1091
|
+
} : {}),
|
|
918
1092
|
nativeImportLossSummary: lossSummary
|
|
919
1093
|
}
|
|
920
1094
|
});
|
|
@@ -937,6 +1111,14 @@ export function importNativeSource(input) {
|
|
|
937
1111
|
sourceMapIds: sourceMaps.map((sourceMap) => sourceMap.id),
|
|
938
1112
|
semanticStatus,
|
|
939
1113
|
mappings: resultSourceMapMappings,
|
|
1114
|
+
...(sourcePreservation ? {
|
|
1115
|
+
sourcePreservationId: sourcePreservation.id,
|
|
1116
|
+
sourcePreservation
|
|
1117
|
+
} : {}),
|
|
1118
|
+
...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
|
|
1119
|
+
declaredSourceHash,
|
|
1120
|
+
sourceHashVerified: false
|
|
1121
|
+
} : {}),
|
|
940
1122
|
nativeImportLossSummary: lossSummary,
|
|
941
1123
|
...input.metadata
|
|
942
1124
|
}
|
|
@@ -1040,7 +1222,7 @@ function createLightweightNativeImport(input) {
|
|
|
1040
1222
|
}
|
|
1041
1223
|
if (declaration.loss) losses.push(declaration.loss);
|
|
1042
1224
|
}
|
|
1043
|
-
losses.push(...lightweightCoverageLosses(input, declarations));
|
|
1225
|
+
losses.push(...lightweightCoverageLosses(input, declarations, input.sourcePreservation));
|
|
1044
1226
|
|
|
1045
1227
|
const semanticIndex = createSemanticIndexRecord({
|
|
1046
1228
|
id: `index_${idFragment(input.sourcePath ?? input.language)}`,
|
|
@@ -1076,7 +1258,15 @@ function createLightweightNativeImport(input) {
|
|
|
1076
1258
|
losses,
|
|
1077
1259
|
semanticIndex,
|
|
1078
1260
|
mappings,
|
|
1079
|
-
metadata: {
|
|
1261
|
+
metadata: {
|
|
1262
|
+
parser,
|
|
1263
|
+
scanKind: 'lightweight-declaration-scan',
|
|
1264
|
+
declarationCount: declarations.length,
|
|
1265
|
+
...(input.sourcePreservation ? {
|
|
1266
|
+
sourcePreservationId: input.sourcePreservation.id,
|
|
1267
|
+
sourcePreservationSummary: input.sourcePreservation.summary
|
|
1268
|
+
} : {})
|
|
1269
|
+
}
|
|
1080
1270
|
};
|
|
1081
1271
|
}
|
|
1082
1272
|
|
|
@@ -1121,20 +1311,32 @@ function nativeImportProjectionContext(importResult, options) {
|
|
|
1121
1311
|
}
|
|
1122
1312
|
|
|
1123
1313
|
function nativeProjectionSourceCandidate(context, options) {
|
|
1124
|
-
const
|
|
1314
|
+
const preservation = sourcePreservationFromProjectionContext(context);
|
|
1315
|
+
const explicitSourceText = options.sourceText ?? options.preservedSourceText ?? options.exactSourceText;
|
|
1316
|
+
const sourceText = explicitSourceText ?? preservation?.sourceText;
|
|
1125
1317
|
if (typeof sourceText !== 'string') return undefined;
|
|
1126
|
-
const
|
|
1318
|
+
const computedSourceHash = hashSemanticValue(sourceText);
|
|
1319
|
+
const declaredSourceHash = options.sourceHash ?? (explicitSourceText === undefined ? preservation?.sourceHash : undefined);
|
|
1320
|
+
const sourceHash = computedSourceHash;
|
|
1127
1321
|
const hashVerified = Boolean(context.sourceHash);
|
|
1128
1322
|
const exact = !context.sourceHash || sourceHash === context.sourceHash || options.verifySourceHash === false;
|
|
1129
1323
|
return {
|
|
1130
1324
|
sourceText,
|
|
1131
1325
|
sourceHash,
|
|
1326
|
+
declaredSourceHash,
|
|
1132
1327
|
hashVerified,
|
|
1133
1328
|
exact,
|
|
1134
|
-
mismatch: hashVerified && sourceHash !== context.sourceHash && options.verifySourceHash !== false
|
|
1329
|
+
mismatch: hashVerified && sourceHash !== context.sourceHash && options.verifySourceHash !== false,
|
|
1330
|
+
sourcePreservationId: preservation?.id
|
|
1135
1331
|
};
|
|
1136
1332
|
}
|
|
1137
1333
|
|
|
1334
|
+
function sourcePreservationFromProjectionContext(context) {
|
|
1335
|
+
return context.nativeSource?.metadata?.sourcePreservation
|
|
1336
|
+
?? context.nativeAst?.metadata?.sourcePreservation
|
|
1337
|
+
?? context.nativeSource?.ast?.metadata?.sourcePreservation;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1138
1340
|
function nativeProjectionDeclarations(importResult, context) {
|
|
1139
1341
|
const semanticIndex = context.semanticIndex;
|
|
1140
1342
|
const occurrencesBySymbol = new Map();
|
|
@@ -1238,14 +1440,15 @@ function nativeProjectionStubLosses(context, candidateSource, declarations, opti
|
|
|
1238
1440
|
: 'Exact native source text was not provided; emitted declaration stubs reconstructed from import metadata.';
|
|
1239
1441
|
const losses = [nativeProjectionLoss(context, {
|
|
1240
1442
|
id: `loss_${context.idPart}_native_source_stub`,
|
|
1241
|
-
kind: 'sourcePreservation',
|
|
1443
|
+
kind: candidateSource?.mismatch ? 'sourcePreservation' : 'targetProjectionLoss',
|
|
1242
1444
|
severity: 'warning',
|
|
1243
1445
|
message,
|
|
1244
1446
|
metadata: {
|
|
1245
1447
|
reason,
|
|
1246
1448
|
projectionMode: 'native-source-stubs',
|
|
1247
1449
|
expectedSourceHash: context.sourceHash,
|
|
1248
|
-
providedSourceHash: candidateSource?.sourceHash
|
|
1450
|
+
providedSourceHash: candidateSource?.sourceHash,
|
|
1451
|
+
declaredSourceHash: candidateSource?.declaredSourceHash
|
|
1249
1452
|
}
|
|
1250
1453
|
})];
|
|
1251
1454
|
if (!declarations.length) {
|
|
@@ -2249,7 +2452,7 @@ function opaqueBodyLoss(input, lineNumber, nodeId, name) {
|
|
|
2249
2452
|
};
|
|
2250
2453
|
}
|
|
2251
2454
|
|
|
2252
|
-
function lightweightCoverageLosses(input, declarations) {
|
|
2455
|
+
function lightweightCoverageLosses(input, declarations, sourcePreservation) {
|
|
2253
2456
|
const id = idFragment(input.sourcePath ?? input.language);
|
|
2254
2457
|
const span = declarations[0]?.span ?? {
|
|
2255
2458
|
sourceId: input.sourceHash,
|
|
@@ -2291,12 +2494,275 @@ function lightweightCoverageLosses(input, declarations) {
|
|
|
2291
2494
|
phase: 'read',
|
|
2292
2495
|
sourceFormat: input.language,
|
|
2293
2496
|
kind: 'sourcePreservation',
|
|
2294
|
-
message:
|
|
2295
|
-
|
|
2497
|
+
message: sourcePreservation
|
|
2498
|
+
? 'Comments, whitespace, token order, directives, and formatting are preserved as opaque native source evidence; exact structural edits still require a parser adapter.'
|
|
2499
|
+
: 'Comments, whitespace, token order, directives, and formatting are not preserved by the lightweight importer.',
|
|
2500
|
+
span,
|
|
2501
|
+
metadata: sourcePreservation ? {
|
|
2502
|
+
sourcePreservationId: sourcePreservation.id,
|
|
2503
|
+
sourcePreservationSummary: sourcePreservation.summary
|
|
2504
|
+
} : undefined
|
|
2296
2505
|
}
|
|
2297
2506
|
];
|
|
2298
2507
|
}
|
|
2299
2508
|
|
|
2509
|
+
function scanPreservedSourceTokens(sourceText, input) {
|
|
2510
|
+
const tokens = [];
|
|
2511
|
+
const trivia = [];
|
|
2512
|
+
const includeTokens = input.includeTokens !== false;
|
|
2513
|
+
const includeTrivia = input.includeTrivia !== false;
|
|
2514
|
+
const maxTokens = Number.isFinite(input.maxTokens) ? Math.max(0, input.maxTokens) : 20000;
|
|
2515
|
+
const maxTrivia = Number.isFinite(input.maxTrivia) ? Math.max(0, input.maxTrivia) : 20000;
|
|
2516
|
+
let offset = 0;
|
|
2517
|
+
let line = 1;
|
|
2518
|
+
let column = 1;
|
|
2519
|
+
let truncated = false;
|
|
2520
|
+
const push = (target, kind, text, start) => {
|
|
2521
|
+
if ((target === tokens && !includeTokens) || (target === trivia && !includeTrivia)) return;
|
|
2522
|
+
const max = target === tokens ? maxTokens : maxTrivia;
|
|
2523
|
+
if (target.length >= max) {
|
|
2524
|
+
truncated = true;
|
|
2525
|
+
return;
|
|
2526
|
+
}
|
|
2527
|
+
target.push(preservedSourceSegment({
|
|
2528
|
+
index: target.length,
|
|
2529
|
+
kind,
|
|
2530
|
+
text,
|
|
2531
|
+
start,
|
|
2532
|
+
end: { offset, line, column },
|
|
2533
|
+
sourceHash: input.sourceHash,
|
|
2534
|
+
sourcePath: input.sourcePath
|
|
2535
|
+
}));
|
|
2536
|
+
};
|
|
2537
|
+
while (offset < sourceText.length) {
|
|
2538
|
+
const start = { offset, line, column };
|
|
2539
|
+
const char = sourceText[offset];
|
|
2540
|
+
const next = sourceText[offset + 1];
|
|
2541
|
+
if (char === '\r' || char === '\n') {
|
|
2542
|
+
const text = char === '\r' && next === '\n' ? '\r\n' : char;
|
|
2543
|
+
offset += text.length;
|
|
2544
|
+
line += 1;
|
|
2545
|
+
column = 1;
|
|
2546
|
+
push(trivia, 'newline', text, start);
|
|
2547
|
+
continue;
|
|
2548
|
+
}
|
|
2549
|
+
if (char === ' ' || char === '\t' || char === '\v' || char === '\f') {
|
|
2550
|
+
let text = '';
|
|
2551
|
+
while (offset < sourceText.length && /[ \t\v\f]/.test(sourceText[offset])) {
|
|
2552
|
+
text += sourceText[offset];
|
|
2553
|
+
offset += 1;
|
|
2554
|
+
column += 1;
|
|
2555
|
+
}
|
|
2556
|
+
push(trivia, 'whitespace', text, start);
|
|
2557
|
+
continue;
|
|
2558
|
+
}
|
|
2559
|
+
if (char === '/' && next === '/') {
|
|
2560
|
+
let text = '';
|
|
2561
|
+
while (offset < sourceText.length && sourceText[offset] !== '\n' && sourceText[offset] !== '\r') {
|
|
2562
|
+
text += sourceText[offset];
|
|
2563
|
+
offset += 1;
|
|
2564
|
+
column += 1;
|
|
2565
|
+
}
|
|
2566
|
+
push(trivia, 'comment', text, start);
|
|
2567
|
+
continue;
|
|
2568
|
+
}
|
|
2569
|
+
if (char === '/' && next === '*') {
|
|
2570
|
+
let text = '';
|
|
2571
|
+
while (offset < sourceText.length) {
|
|
2572
|
+
const current = sourceText[offset];
|
|
2573
|
+
text += current;
|
|
2574
|
+
offset += 1;
|
|
2575
|
+
if (current === '\n') {
|
|
2576
|
+
line += 1;
|
|
2577
|
+
column = 1;
|
|
2578
|
+
} else {
|
|
2579
|
+
column += 1;
|
|
2580
|
+
}
|
|
2581
|
+
if (current === '*' && sourceText[offset] === '/') {
|
|
2582
|
+
text += '/';
|
|
2583
|
+
offset += 1;
|
|
2584
|
+
column += 1;
|
|
2585
|
+
break;
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
push(trivia, 'comment', text, start);
|
|
2589
|
+
continue;
|
|
2590
|
+
}
|
|
2591
|
+
if (char === '#' && isHashCommentLanguage(input.language)) {
|
|
2592
|
+
let text = '';
|
|
2593
|
+
while (offset < sourceText.length && sourceText[offset] !== '\n' && sourceText[offset] !== '\r') {
|
|
2594
|
+
text += sourceText[offset];
|
|
2595
|
+
offset += 1;
|
|
2596
|
+
column += 1;
|
|
2597
|
+
}
|
|
2598
|
+
push(trivia, preservedHashLineKind(text), text, start);
|
|
2599
|
+
continue;
|
|
2600
|
+
}
|
|
2601
|
+
if (char === '"' || char === '\'' || char === '`') {
|
|
2602
|
+
const quote = char;
|
|
2603
|
+
let text = char;
|
|
2604
|
+
offset += 1;
|
|
2605
|
+
column += 1;
|
|
2606
|
+
let escaped = false;
|
|
2607
|
+
while (offset < sourceText.length) {
|
|
2608
|
+
const current = sourceText[offset];
|
|
2609
|
+
text += current;
|
|
2610
|
+
offset += 1;
|
|
2611
|
+
if (current === '\n') {
|
|
2612
|
+
line += 1;
|
|
2613
|
+
column = 1;
|
|
2614
|
+
} else {
|
|
2615
|
+
column += 1;
|
|
2616
|
+
}
|
|
2617
|
+
if (escaped) {
|
|
2618
|
+
escaped = false;
|
|
2619
|
+
} else if (current === '\\') {
|
|
2620
|
+
escaped = true;
|
|
2621
|
+
} else if (current === quote) {
|
|
2622
|
+
break;
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
push(tokens, 'string', text, start);
|
|
2626
|
+
continue;
|
|
2627
|
+
}
|
|
2628
|
+
if (/[0-9]/.test(char)) {
|
|
2629
|
+
let text = '';
|
|
2630
|
+
while (offset < sourceText.length && /[0-9a-fA-F_xXoObBeE.+-]/.test(sourceText[offset])) {
|
|
2631
|
+
text += sourceText[offset];
|
|
2632
|
+
offset += 1;
|
|
2633
|
+
column += 1;
|
|
2634
|
+
}
|
|
2635
|
+
push(tokens, 'number', text, start);
|
|
2636
|
+
continue;
|
|
2637
|
+
}
|
|
2638
|
+
if (isIdentifierStart(char)) {
|
|
2639
|
+
let text = '';
|
|
2640
|
+
while (offset < sourceText.length && isIdentifierPart(sourceText[offset])) {
|
|
2641
|
+
text += sourceText[offset];
|
|
2642
|
+
offset += 1;
|
|
2643
|
+
column += 1;
|
|
2644
|
+
}
|
|
2645
|
+
push(tokens, preservedKeywordSet.has(text) ? 'keyword' : 'identifier', text, start);
|
|
2646
|
+
continue;
|
|
2647
|
+
}
|
|
2648
|
+
let text = char;
|
|
2649
|
+
if (/[=+\-*/%&|^!<>?:.]/.test(char)) {
|
|
2650
|
+
while (offset + text.length < sourceText.length && /[=+\-*/%&|^!<>?:.]/.test(sourceText[offset + text.length])) text += sourceText[offset + text.length];
|
|
2651
|
+
offset += text.length;
|
|
2652
|
+
column += text.length;
|
|
2653
|
+
push(tokens, 'operator', text, start);
|
|
2654
|
+
} else {
|
|
2655
|
+
offset += 1;
|
|
2656
|
+
column += 1;
|
|
2657
|
+
push(tokens, /[()[\]{};,]/.test(char) ? 'punctuation' : 'unknown', text, start);
|
|
2658
|
+
}
|
|
2659
|
+
}
|
|
2660
|
+
return { tokens, trivia, truncated };
|
|
2661
|
+
}
|
|
2662
|
+
|
|
2663
|
+
function scanPreservedSourceDirectives(sourceText, input) {
|
|
2664
|
+
const directives = [];
|
|
2665
|
+
const maxDirectives = Number.isFinite(input.maxDirectives) ? Math.max(0, input.maxDirectives) : 20000;
|
|
2666
|
+
let truncated = false;
|
|
2667
|
+
let offset = 0;
|
|
2668
|
+
for (const { line, number } of sourceLines(sourceText)) {
|
|
2669
|
+
const trimmed = line.trim();
|
|
2670
|
+
const directiveKind = preservedDirectiveKind(trimmed, input.language);
|
|
2671
|
+
if (directiveKind) {
|
|
2672
|
+
if (directives.length >= maxDirectives) {
|
|
2673
|
+
truncated = true;
|
|
2674
|
+
offset += line.length + 1;
|
|
2675
|
+
continue;
|
|
2676
|
+
}
|
|
2677
|
+
const startColumn = Math.max(1, line.indexOf(trimmed) + 1);
|
|
2678
|
+
directives.push({
|
|
2679
|
+
id: `directive_${idFragment(input.sourcePath ?? input.language)}_${directives.length + 1}`,
|
|
2680
|
+
kind: directiveKind,
|
|
2681
|
+
text: trimmed,
|
|
2682
|
+
textHash: hashSemanticValue(trimmed),
|
|
2683
|
+
span: {
|
|
2684
|
+
sourceId: input.sourceHash,
|
|
2685
|
+
path: input.sourcePath,
|
|
2686
|
+
start: offset + startColumn - 1,
|
|
2687
|
+
end: offset + startColumn - 1 + trimmed.length,
|
|
2688
|
+
startLine: number,
|
|
2689
|
+
startColumn,
|
|
2690
|
+
endLine: number,
|
|
2691
|
+
endColumn: startColumn + trimmed.length
|
|
2692
|
+
},
|
|
2693
|
+
metadata: { language: input.language }
|
|
2694
|
+
});
|
|
2695
|
+
}
|
|
2696
|
+
offset += line.length + 1;
|
|
2697
|
+
}
|
|
2698
|
+
return { directives, truncated };
|
|
2699
|
+
}
|
|
2700
|
+
|
|
2701
|
+
function preservedSourceSegment(input) {
|
|
2702
|
+
const id = `${input.kind}_${input.index + 1}_${idFragment(input.start.offset)}`;
|
|
2703
|
+
return {
|
|
2704
|
+
id,
|
|
2705
|
+
kind: input.kind,
|
|
2706
|
+
text: input.text,
|
|
2707
|
+
textHash: hashSemanticValue(input.text),
|
|
2708
|
+
span: {
|
|
2709
|
+
sourceId: input.sourceHash,
|
|
2710
|
+
path: input.sourcePath,
|
|
2711
|
+
start: input.start.offset,
|
|
2712
|
+
end: input.end.offset,
|
|
2713
|
+
startLine: input.start.line,
|
|
2714
|
+
startColumn: input.start.column,
|
|
2715
|
+
endLine: input.end.line,
|
|
2716
|
+
endColumn: input.end.column
|
|
2717
|
+
}
|
|
2718
|
+
};
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2721
|
+
function preservedDirectiveKind(trimmed, language) {
|
|
2722
|
+
if (!trimmed) return undefined;
|
|
2723
|
+
if (/^#\s*(include|define|if|ifdef|ifndef|elif|else|endif|pragma)\b/.test(trimmed)) return 'preprocessor';
|
|
2724
|
+
if (/^#!\s*/.test(trimmed)) return 'shebang';
|
|
2725
|
+
if (/^['"]use strict['"];?$/.test(trimmed)) return 'runtime-directive';
|
|
2726
|
+
if (/^(import|export|package|module|namespace|use|using|from|require)\b/.test(trimmed)) return 'module-directive';
|
|
2727
|
+
if (normalizeNativeLanguageId(language) === 'python' && /^from\s+\S+\s+import\b/.test(trimmed)) return 'module-directive';
|
|
2728
|
+
return undefined;
|
|
2729
|
+
}
|
|
2730
|
+
|
|
2731
|
+
function preservedHashLineKind(text) {
|
|
2732
|
+
return preservedDirectiveKind(String(text).trim(), 'c') ? 'directive' : 'comment';
|
|
2733
|
+
}
|
|
2734
|
+
|
|
2735
|
+
function isHashCommentLanguage(language) {
|
|
2736
|
+
return ['python', 'ruby', 'shell', 'bash', 'zsh', 'r', 'perl', 'yaml', 'toml'].includes(normalizeNativeLanguageId(language));
|
|
2737
|
+
}
|
|
2738
|
+
|
|
2739
|
+
function detectNewlineStyle(sourceText) {
|
|
2740
|
+
const crlf = (sourceText.match(/\r\n/g) ?? []).length;
|
|
2741
|
+
const normalized = sourceText.replace(/\r\n/g, '');
|
|
2742
|
+
const lf = (normalized.match(/\n/g) ?? []).length;
|
|
2743
|
+
const cr = (normalized.match(/\r/g) ?? []).length;
|
|
2744
|
+
const kinds = [crlf ? 'crlf' : undefined, lf ? 'lf' : undefined, cr ? 'cr' : undefined].filter(Boolean);
|
|
2745
|
+
if (!kinds.length) return 'none';
|
|
2746
|
+
if (kinds.length > 1 || cr) return 'mixed';
|
|
2747
|
+
return kinds[0];
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
const preservedKeywordSet = new Set([
|
|
2751
|
+
'abstract', 'as', 'async', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue', 'def', 'defer',
|
|
2752
|
+
'do', 'else', 'enum', 'export', 'extends', 'extern', 'false', 'final', 'fn', 'for', 'from', 'func', 'function',
|
|
2753
|
+
'if', 'impl', 'import', 'in', 'interface', 'let', 'match', 'mod', 'module', 'mut', 'namespace', 'new', 'nil',
|
|
2754
|
+
'none', 'null', 'package', 'private', 'protected', 'pub', 'public', 'return', 'self', 'static', 'struct',
|
|
2755
|
+
'switch', 'this', 'throw', 'trait', 'true', 'try', 'type', 'use', 'using', 'var', 'while', 'yield'
|
|
2756
|
+
]);
|
|
2757
|
+
|
|
2758
|
+
function isIdentifierStart(char) {
|
|
2759
|
+
return /[A-Za-z_$]/.test(char ?? '');
|
|
2760
|
+
}
|
|
2761
|
+
|
|
2762
|
+
function isIdentifierPart(char) {
|
|
2763
|
+
return /[A-Za-z0-9_$]/.test(char ?? '');
|
|
2764
|
+
}
|
|
2765
|
+
|
|
2300
2766
|
function sourceLines(sourceText) {
|
|
2301
2767
|
return String(sourceText ?? '').split(/\r?\n/).map((line, index) => ({ line, number: index + 1 }));
|
|
2302
2768
|
}
|
|
@@ -2636,11 +3102,13 @@ function nativeImportCategoryForLossKind(kind) {
|
|
|
2636
3102
|
if (kind === 'preprocessor' || kind === 'conditionalCompilation' || kind === 'macroHygiene') return 'preprocessor';
|
|
2637
3103
|
if (kind === 'metaprogramming' || kind === 'reflection' || kind === 'dynamicDispatch' || kind === 'dynamicRuntime') return 'metaprogramming';
|
|
2638
3104
|
if (kind === 'generatedCode') return 'generatedCode';
|
|
2639
|
-
if (kind === '
|
|
3105
|
+
if (kind === 'overloadResolution' || kind === 'typeInference') return 'overloadTypeInference';
|
|
3106
|
+
if (kind === 'sourcePreservation' || kind === 'commentsTrivia' || kind === 'nonRoundTrippable') return 'sourcePreservation';
|
|
2640
3107
|
if (kind === 'parserDiagnostic') return 'parserDiagnostics';
|
|
2641
3108
|
if (kind === 'unsupportedSyntax' || kind === 'unsupportedSemantic') return 'unsupportedSyntax';
|
|
2642
3109
|
if (kind === 'partialSemanticIndex') return 'partialSemanticIndex';
|
|
2643
3110
|
if (kind === 'sourceMapApproximation') return 'sourceMapApproximation';
|
|
3111
|
+
if (kind === 'targetProjectionLoss') return 'targetProjectionLoss';
|
|
2644
3112
|
return String(kind ?? 'opaqueNative');
|
|
2645
3113
|
}
|
|
2646
3114
|
|
|
@@ -3004,6 +3472,20 @@ function createJavaScriptSyntaxImporterAdapter(options) {
|
|
|
3004
3472
|
parser: options.parser,
|
|
3005
3473
|
version: options.version,
|
|
3006
3474
|
capabilities: uniqueStrings(['nativeAst', 'semanticIndex', 'sourceMaps', 'diagnostics', ...(options.capabilities ?? [])]),
|
|
3475
|
+
coverage: nativeImporterAdapterCoverage({
|
|
3476
|
+
exactness: 'exact-parser-ast',
|
|
3477
|
+
exactAst: true,
|
|
3478
|
+
tokens: false,
|
|
3479
|
+
trivia: false,
|
|
3480
|
+
diagnostics: true,
|
|
3481
|
+
sourceRanges: true,
|
|
3482
|
+
generatedRanges: false,
|
|
3483
|
+
semanticCoverage: declarationSemanticCoverage(),
|
|
3484
|
+
notes: [
|
|
3485
|
+
'Normalizes a caller-owned ESTree/Babel-compatible AST into native AST nodes and declaration-level semantic index records.',
|
|
3486
|
+
'The wrapper ignores parser token/trivia/comment arrays unless a host adapter explicitly maps them into preservation evidence.'
|
|
3487
|
+
]
|
|
3488
|
+
}, options.coverage),
|
|
3007
3489
|
supportedExtensions: options.supportedExtensions,
|
|
3008
3490
|
diagnostics: options.diagnostics,
|
|
3009
3491
|
parse(input) {
|
|
@@ -3454,6 +3936,14 @@ function createNativeProjectImportResult(input, imports) {
|
|
|
3454
3936
|
mergeCandidates.push(...(result.mergeCandidates ?? []));
|
|
3455
3937
|
operations.push(...(result.patch?.operations ?? []));
|
|
3456
3938
|
}
|
|
3939
|
+
const uniqueLosses = uniqueByLossId(losses);
|
|
3940
|
+
const uniqueEvidence = uniqueByEvidenceId(evidence);
|
|
3941
|
+
const nativeImportLossSummary = summarizeNativeImportLosses(uniqueLosses, {
|
|
3942
|
+
evidence: uniqueEvidence,
|
|
3943
|
+
scanKind: 'native-project-import',
|
|
3944
|
+
semanticStatus: uniqueLosses.some((loss) => loss.severity === 'error') ? 'partial' : 'mapped'
|
|
3945
|
+
});
|
|
3946
|
+
const sourcePreservationSummary = summarizeProjectSourcePreservation(imports);
|
|
3457
3947
|
const document = createDocument({
|
|
3458
3948
|
id: input.documentId ?? `document_${idPart}`,
|
|
3459
3949
|
name: input.documentName ?? input.name ?? 'NativeProject',
|
|
@@ -3464,6 +3954,8 @@ function createNativeProjectImportResult(input, imports) {
|
|
|
3464
3954
|
semanticStatus: losses.some((loss) => loss.severity === 'error') ? 'partial' : 'mapped',
|
|
3465
3955
|
projectRoot: input.projectRoot,
|
|
3466
3956
|
sourceCount: imports.length,
|
|
3957
|
+
nativeImportLossSummary,
|
|
3958
|
+
sourcePreservationSummary,
|
|
3467
3959
|
...input.documentMetadata
|
|
3468
3960
|
}
|
|
3469
3961
|
});
|
|
@@ -3473,12 +3965,14 @@ function createNativeProjectImportResult(input, imports) {
|
|
|
3473
3965
|
nativeSources,
|
|
3474
3966
|
semanticIndex,
|
|
3475
3967
|
sourceMaps,
|
|
3476
|
-
losses:
|
|
3477
|
-
evidence:
|
|
3968
|
+
losses: uniqueLosses,
|
|
3969
|
+
evidence: uniqueEvidence,
|
|
3478
3970
|
metadata: {
|
|
3479
3971
|
sourceLanguage: input.language ?? 'mixed',
|
|
3480
3972
|
projectRoot: input.projectRoot,
|
|
3481
3973
|
sourceCount: imports.length,
|
|
3974
|
+
nativeImportLossSummary,
|
|
3975
|
+
sourcePreservationSummary,
|
|
3482
3976
|
...input.universalAstMetadata
|
|
3483
3977
|
}
|
|
3484
3978
|
});
|
|
@@ -3487,12 +3981,14 @@ function createNativeProjectImportResult(input, imports) {
|
|
|
3487
3981
|
author: input.author ?? '@shapeshift-labs/frontier-lang-compiler/importNativeProject',
|
|
3488
3982
|
risk: losses.some((loss) => loss.severity === 'error') ? 'high' : losses.some((loss) => loss.severity === 'warning') ? 'medium' : 'low',
|
|
3489
3983
|
operations,
|
|
3490
|
-
evidence:
|
|
3984
|
+
evidence: uniqueEvidence,
|
|
3491
3985
|
metadata: {
|
|
3492
3986
|
semanticIndexId: semanticIndex?.id,
|
|
3493
3987
|
universalAstId: universalAst.id,
|
|
3494
3988
|
sourceMapIds: sourceMaps.map((sourceMap) => sourceMap.id),
|
|
3495
|
-
sourceCount: imports.length
|
|
3989
|
+
sourceCount: imports.length,
|
|
3990
|
+
nativeImportLossSummary,
|
|
3991
|
+
sourcePreservationSummary
|
|
3496
3992
|
}
|
|
3497
3993
|
});
|
|
3498
3994
|
return {
|
|
@@ -3508,17 +4004,35 @@ function createNativeProjectImportResult(input, imports) {
|
|
|
3508
4004
|
semanticIndex,
|
|
3509
4005
|
universalAst,
|
|
3510
4006
|
sourceMaps,
|
|
3511
|
-
losses:
|
|
3512
|
-
evidence:
|
|
4007
|
+
losses: uniqueLosses,
|
|
4008
|
+
evidence: uniqueEvidence,
|
|
3513
4009
|
mergeCandidates,
|
|
3514
4010
|
metadata: {
|
|
3515
4011
|
sourceCount: imports.length,
|
|
3516
4012
|
sourcePaths: imports.map((result) => result.sourcePath).filter(Boolean),
|
|
4013
|
+
nativeImportLossSummary,
|
|
4014
|
+
sourcePreservationSummary,
|
|
3517
4015
|
...input.metadata
|
|
3518
4016
|
}
|
|
3519
4017
|
};
|
|
3520
4018
|
}
|
|
3521
4019
|
|
|
4020
|
+
function summarizeProjectSourcePreservation(imports) {
|
|
4021
|
+
const records = imports
|
|
4022
|
+
.map((result) => result.metadata?.sourcePreservation ?? result.nativeSource?.metadata?.sourcePreservation ?? result.nativeAst?.metadata?.sourcePreservation)
|
|
4023
|
+
.filter(Boolean);
|
|
4024
|
+
return {
|
|
4025
|
+
total: records.length,
|
|
4026
|
+
exactSourceAvailable: records.filter((record) => record.summary?.exactSourceAvailable).length,
|
|
4027
|
+
sourceBytes: records.reduce((sum, record) => sum + (record.sourceBytes ?? 0), 0),
|
|
4028
|
+
tokens: records.reduce((sum, record) => sum + (record.summary?.tokens ?? record.tokens?.length ?? 0), 0),
|
|
4029
|
+
trivia: records.reduce((sum, record) => sum + (record.summary?.trivia ?? record.trivia?.length ?? 0), 0),
|
|
4030
|
+
directives: records.reduce((sum, record) => sum + (record.summary?.directives ?? record.directives?.length ?? 0), 0),
|
|
4031
|
+
truncated: records.some((record) => record.summary?.truncated === true),
|
|
4032
|
+
ids: records.map((record) => record.id).filter(Boolean)
|
|
4033
|
+
};
|
|
4034
|
+
}
|
|
4035
|
+
|
|
3522
4036
|
function mergeSemanticIndexes(imports, input, idPart) {
|
|
3523
4037
|
const indexes = imports.map((result) => result.semanticIndex ?? result.universalAst?.semanticIndex).filter(Boolean);
|
|
3524
4038
|
if (!indexes.length) return undefined;
|
|
@@ -3564,9 +4078,15 @@ function normalizeNativeImporterAdapter(adapter) {
|
|
|
3564
4078
|
parser: String(adapter.parser),
|
|
3565
4079
|
version: adapter.version === undefined ? undefined : String(adapter.version)
|
|
3566
4080
|
};
|
|
4081
|
+
const capabilities = normalizeStringList(adapter.capabilities);
|
|
3567
4082
|
return Object.freeze({
|
|
3568
4083
|
...summaryInput,
|
|
3569
|
-
capabilities
|
|
4084
|
+
capabilities,
|
|
4085
|
+
coverage: normalizeNativeImporterAdapterCoverage(adapter.coverage, {
|
|
4086
|
+
capabilities,
|
|
4087
|
+
language: adapter.language,
|
|
4088
|
+
parser: String(adapter.parser)
|
|
4089
|
+
}),
|
|
3570
4090
|
supportedExtensions: normalizeStringList(adapter.supportedExtensions).map((extension) => extension.startsWith('.') ? extension.toLowerCase() : `.${extension.toLowerCase()}`),
|
|
3571
4091
|
diagnostics: normalizeAdapterDiagnostics(adapter.diagnostics, summaryInput, {
|
|
3572
4092
|
language: adapter.language,
|
|
@@ -3576,6 +4096,138 @@ function normalizeNativeImporterAdapter(adapter) {
|
|
|
3576
4096
|
});
|
|
3577
4097
|
}
|
|
3578
4098
|
|
|
4099
|
+
function nativeImporterAdapterCoverage(defaults = {}, overrides = {}) {
|
|
4100
|
+
return {
|
|
4101
|
+
...defaults,
|
|
4102
|
+
...overrides,
|
|
4103
|
+
semanticCoverage: {
|
|
4104
|
+
...(defaults.semanticCoverage ?? {}),
|
|
4105
|
+
...(overrides.semanticCoverage ?? {})
|
|
4106
|
+
},
|
|
4107
|
+
notes: uniqueStrings([...(defaults.notes ?? []), ...(overrides.notes ?? [])])
|
|
4108
|
+
};
|
|
4109
|
+
}
|
|
4110
|
+
|
|
4111
|
+
function normalizeNativeImporterAdapterCoverage(value = {}, context = {}) {
|
|
4112
|
+
const capabilities = new Set(normalizeStringList(context.capabilities).map((capability) => capability.toLowerCase()));
|
|
4113
|
+
const hasCapability = (...names) => names.some((name) => capabilities.has(String(name).toLowerCase()));
|
|
4114
|
+
const exactAst = Boolean(value.exactAst ?? hasCapability('exactAst', 'exactAstImport'));
|
|
4115
|
+
const sourceRanges = Boolean(value.sourceRanges ?? hasCapability('sourceRanges', 'sourceRange', 'ranges', 'sourceMaps'));
|
|
4116
|
+
const generatedRanges = Boolean(value.generatedRanges ?? hasCapability('generatedRanges', 'generatedRange', 'generatedSourceMaps'));
|
|
4117
|
+
const diagnostics = Boolean(value.diagnostics ?? hasCapability('diagnostics', 'parserDiagnostics'));
|
|
4118
|
+
return Object.freeze({
|
|
4119
|
+
exactness: String(value.exactness ?? inferredAdapterExactness(exactAst, capabilities)),
|
|
4120
|
+
exactAst,
|
|
4121
|
+
tokens: Boolean(value.tokens ?? hasCapability('tokens', 'tokenStream')),
|
|
4122
|
+
trivia: Boolean(value.trivia ?? hasCapability('trivia', 'comments', 'formatting')),
|
|
4123
|
+
diagnostics,
|
|
4124
|
+
sourceRanges,
|
|
4125
|
+
generatedRanges,
|
|
4126
|
+
semanticCoverage: normalizeNativeImporterSemanticCoverage(value.semanticCoverage, {
|
|
4127
|
+
capabilities,
|
|
4128
|
+
sourceRanges,
|
|
4129
|
+
generatedRanges
|
|
4130
|
+
}),
|
|
4131
|
+
notes: uniqueStrings(value.notes ?? inferredAdapterCoverageNotes(context, {
|
|
4132
|
+
exactAst,
|
|
4133
|
+
sourceRanges,
|
|
4134
|
+
generatedRanges,
|
|
4135
|
+
diagnostics
|
|
4136
|
+
}))
|
|
4137
|
+
});
|
|
4138
|
+
}
|
|
4139
|
+
|
|
4140
|
+
function observeNativeImporterAdapterCoverage(coverage, parseResult = {}, context = {}) {
|
|
4141
|
+
const nodes = parseResult.nativeAst?.nodes ?? parseResult.nodes ?? {};
|
|
4142
|
+
const nodeList = Object.values(nodes);
|
|
4143
|
+
const sourceMapMappings = parseResult.sourceMaps?.flatMap((sourceMap) => sourceMap.mappings ?? []) ?? parseResult.mappings ?? [];
|
|
4144
|
+
const semanticIndex = parseResult.semanticIndex;
|
|
4145
|
+
const semanticSymbols = semanticIndex?.symbols?.length ?? 0;
|
|
4146
|
+
const observedSourceRanges = nodeList.some((node) => Boolean(node?.span)) || sourceMapMappings.some((mapping) => Boolean(mapping?.sourceSpan));
|
|
4147
|
+
const observedGeneratedRanges = sourceMapMappings.some((mapping) => Boolean(mapping?.generatedSpan));
|
|
4148
|
+
const observedSemanticCoverage = normalizeNativeImporterSemanticCoverage({
|
|
4149
|
+
...coverage.semanticCoverage,
|
|
4150
|
+
level: semanticSymbols
|
|
4151
|
+
? maxSemanticCoverageLevel(coverage.semanticCoverage?.level, 'declaration-index')
|
|
4152
|
+
: coverage.semanticCoverage?.level,
|
|
4153
|
+
declarations: coverage.semanticCoverage?.declarations || semanticSymbols > 0,
|
|
4154
|
+
symbols: coverage.semanticCoverage?.symbols || semanticSymbols > 0
|
|
4155
|
+
}, {});
|
|
4156
|
+
return Object.freeze({
|
|
4157
|
+
...coverage,
|
|
4158
|
+
diagnostics: coverage.diagnostics || (context.diagnostics?.length ?? 0) > 0,
|
|
4159
|
+
sourceRanges: coverage.sourceRanges || observedSourceRanges,
|
|
4160
|
+
generatedRanges: coverage.generatedRanges || observedGeneratedRanges,
|
|
4161
|
+
semanticCoverage: observedSemanticCoverage,
|
|
4162
|
+
observed: {
|
|
4163
|
+
diagnostics: context.diagnostics?.length ?? 0,
|
|
4164
|
+
losses: context.losses?.length ?? 0,
|
|
4165
|
+
nativeAstNodes: nodeList.length,
|
|
4166
|
+
semanticSymbols,
|
|
4167
|
+
sourceMapMappings: sourceMapMappings.length,
|
|
4168
|
+
sourceRanges: observedSourceRanges,
|
|
4169
|
+
generatedRanges: observedGeneratedRanges
|
|
4170
|
+
}
|
|
4171
|
+
});
|
|
4172
|
+
}
|
|
4173
|
+
|
|
4174
|
+
function declarationSemanticCoverage() {
|
|
4175
|
+
return {
|
|
4176
|
+
level: 'declaration-index',
|
|
4177
|
+
declarations: true,
|
|
4178
|
+
symbols: true,
|
|
4179
|
+
references: false,
|
|
4180
|
+
types: false,
|
|
4181
|
+
controlFlow: false
|
|
4182
|
+
};
|
|
4183
|
+
}
|
|
4184
|
+
|
|
4185
|
+
function normalizeNativeImporterSemanticCoverage(value = {}, context = {}) {
|
|
4186
|
+
const capabilities = context.capabilities ?? new Set();
|
|
4187
|
+
const hasCapability = (...names) => names.some((name) => capabilities.has(String(name).toLowerCase()));
|
|
4188
|
+
const declarations = Boolean(value.declarations ?? hasCapability('semanticIndex', 'declarations'));
|
|
4189
|
+
const symbols = Boolean(value.symbols ?? declarations);
|
|
4190
|
+
const references = Boolean(value.references ?? hasCapability('references', 'referenceIndex'));
|
|
4191
|
+
const types = Boolean(value.types ?? hasCapability('types', 'typeResolution', 'typeChecking'));
|
|
4192
|
+
const controlFlow = Boolean(value.controlFlow ?? hasCapability('controlFlow', 'cfg'));
|
|
4193
|
+
return Object.freeze({
|
|
4194
|
+
level: String(value.level ?? inferredSemanticCoverageLevel({ declarations, symbols, references, types, controlFlow })),
|
|
4195
|
+
declarations,
|
|
4196
|
+
symbols,
|
|
4197
|
+
references,
|
|
4198
|
+
types,
|
|
4199
|
+
controlFlow
|
|
4200
|
+
});
|
|
4201
|
+
}
|
|
4202
|
+
|
|
4203
|
+
function inferredAdapterExactness(exactAst, capabilities) {
|
|
4204
|
+
if (exactAst) return 'exact-parser-ast';
|
|
4205
|
+
if (capabilities.has('nativeast')) return 'adapter-reported-native-ast';
|
|
4206
|
+
return 'loss-aware-native-ast';
|
|
4207
|
+
}
|
|
4208
|
+
|
|
4209
|
+
function inferredSemanticCoverageLevel(input) {
|
|
4210
|
+
if (input.references || input.types || input.controlFlow) return 'semantic-index';
|
|
4211
|
+
if (input.declarations || input.symbols) return 'declaration-index';
|
|
4212
|
+
return 'native-ast';
|
|
4213
|
+
}
|
|
4214
|
+
|
|
4215
|
+
function maxSemanticCoverageLevel(left, right) {
|
|
4216
|
+
const ranks = { 'native-ast': 0, 'declaration-index': 1, 'semantic-index': 2 };
|
|
4217
|
+
const leftRank = ranks[left] ?? 0;
|
|
4218
|
+
const rightRank = ranks[right] ?? 0;
|
|
4219
|
+
return rightRank > leftRank ? right : left;
|
|
4220
|
+
}
|
|
4221
|
+
|
|
4222
|
+
function inferredAdapterCoverageNotes(context, coverage) {
|
|
4223
|
+
const notes = [];
|
|
4224
|
+
if (!coverage.exactAst) notes.push('Adapter did not declare exact parser AST/CST coverage; import readiness depends on losses and evidence.');
|
|
4225
|
+
if (!coverage.generatedRanges) notes.push('Adapter does not declare generated-range coverage unless parse output includes generated spans.');
|
|
4226
|
+
if (!coverage.diagnostics) notes.push('Adapter did not declare parser diagnostics support.');
|
|
4227
|
+
if (context.language && context.parser) notes.push(`Coverage summary applies to ${context.language} via ${context.parser}.`);
|
|
4228
|
+
return notes;
|
|
4229
|
+
}
|
|
4230
|
+
|
|
3579
4231
|
function normalizeStringList(value) {
|
|
3580
4232
|
if (value === undefined || value === null) return [];
|
|
3581
4233
|
if (Array.isArray(value)) return value.map((item) => String(item)).filter(Boolean);
|
|
@@ -3669,6 +4321,7 @@ function adapterDiagnosticsEvidence(adapter, diagnostics, input) {
|
|
|
3669
4321
|
parserVersion: input.parserVersion,
|
|
3670
4322
|
sourceHash: input.sourceHash,
|
|
3671
4323
|
capabilities: adapter.capabilities,
|
|
4324
|
+
coverage: adapter.coverage,
|
|
3672
4325
|
supportedExtensions: adapter.supportedExtensions,
|
|
3673
4326
|
diagnostics: diagnostics.map(serializableDiagnostic),
|
|
3674
4327
|
errors,
|
package/package.json
CHANGED