@shapeshift-labs/frontier-lang-compiler 0.2.9 → 0.2.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -50,6 +50,8 @@ console.log(summary.categories);
50
50
  console.log(readiness.readiness);
51
51
  ```
52
52
 
53
+ The loss taxonomy separates broad scanner limits from specific round-trip risks such as conditional compilation, reflection, overload/type-inference gaps, comments/trivia preservation, source-map approximation, parser diagnostics, and target projection loss. These records are evidence labels for merge admission; they are not claims that the lightweight scanner expanded macros, evaluated inactive branches, resolved overloads, or ran a type checker.
54
+
53
55
  Ask the compiler what is actually covered before sending native imports into a merge queue:
54
56
 
55
57
  ```js
@@ -71,6 +73,28 @@ console.log(python.imports.readiness); // scanner imports are intentionally revi
71
73
  console.log(python.parserAdapters); // host-owned exact parsers such as LibCST can be injected
72
74
  ```
73
75
 
76
+ Preserve exact native source text, token/trivia hashes, comments, whitespace, and source directives as evidence. This does not claim full semantic understanding; it keeps round-trip material available while exact parser adapters catch up:
77
+
78
+ ```js
79
+ import {
80
+ createNativeSourcePreservation,
81
+ importNativeSource
82
+ } from '@shapeshift-labs/frontier-lang-compiler';
83
+
84
+ const sourceText = '// kept\nexport function step(frame) { return frame + 1; }\n';
85
+ const preservation = createNativeSourcePreservation({
86
+ language: 'javascript',
87
+ sourcePath: 'src/runtime.js',
88
+ sourceText
89
+ });
90
+ const imported = importNativeSource({ language: 'javascript', sourcePath: 'src/runtime.js', sourceText });
91
+
92
+ console.log(preservation.summary.comments); // comments and whitespace are tracked
93
+ console.log(imported.metadata.sourcePreservation.sourceHash);
94
+ ```
95
+
96
+ When `sourceText` is present, hashes are computed from the actual text. Caller-provided hashes are recorded as declared metadata and cannot make stale text project as exact source. Use `includeTokens`, `includeTrivia`, `includeDirectives`, and `max*` options to keep preservation records compact for large files.
97
+
74
98
  Create a compact semantic sidecar for swarm merge admission. This is the artifact a coordinator can index instead of reading a worker directory by hand:
75
99
 
76
100
  ```js
@@ -96,7 +120,7 @@ console.log(sidecar.ownershipRegions[0].key); // source#src/runtime.ts#class#Run
96
120
  console.log(sidecar.patchHints[0].supportedOperations); // source-region patch operations
97
121
  ```
98
122
 
99
- Project a native import back to source. Exact source is preserved only when the supplied text matches the import hash; otherwise the compiler emits declaration stubs with review-required loss evidence:
123
+ Project a native import back to source. Exact source is preserved when the import carries matching source-preservation evidence or when supplied text matches the import hash; otherwise the compiler emits declaration stubs with review-required loss evidence:
100
124
 
101
125
  ```js
102
126
  import {
@@ -111,7 +135,7 @@ const imported = importNativeSource({
111
135
  sourceText
112
136
  });
113
137
 
114
- const projection = projectNativeImportToSource(imported, { sourceText });
138
+ const projection = projectNativeImportToSource(imported);
115
139
 
116
140
  console.log(projection.mode); // "preserved-source"
117
141
  console.log(projection.readiness.readiness); // "ready"
@@ -146,6 +170,9 @@ const project = await importNativeProject({
146
170
 
147
171
  console.log(imported.universalAst.sourceMaps.length);
148
172
  console.log(project.semanticIndex.symbols.length);
173
+ console.log(imported.adapter.coverage.exactness);
174
+ console.log(imported.adapter.coverage.semanticCoverage.level);
175
+ console.log(project.metadata.sourcePreservationSummary.total);
149
176
  ```
150
177
 
151
178
  The built-in adapter factories are dependency-light wrappers for caller-owned parsers or ASTs:
@@ -155,6 +182,8 @@ The built-in adapter factories are dependency-light wrappers for caller-owned pa
155
182
  - `createTypeScriptCompilerNativeImporterAdapter`
156
183
  - `createTreeSitterNativeImporterAdapter`
157
184
 
185
+ Adapter summaries include a structured `coverage` record so merge queues can distinguish exact parser AST imports from declaration scans. The record declares exactness, parser token/trivia support, diagnostics support, source-range and generated-range support, and semantic coverage. Built-in wrappers normalize native AST/CST nodes and declaration-level semantic indexes; they do not claim resolved references, types, control flow, generated ranges, or token/trivia fidelity unless the host adapter supplies that evidence.
186
+
158
187
  ## Related Packages
159
188
 
160
189
  The published Frontier package family is generated from one shared package catalog so READMEs stay in sync across packages:
package/bench/smoke.mjs CHANGED
@@ -3,6 +3,7 @@ import {
3
3
  compileFrontierSource,
4
4
  createEstreeNativeImporterAdapter,
5
5
  createNativeImportCoverageMatrix,
6
+ createNativeSourcePreservation,
6
7
  createSemanticImportSidecar,
7
8
  importNativeSource,
8
9
  projectNativeImportToSource,
@@ -67,6 +68,15 @@ const matrixStart = performance.now();
67
68
  const coverageMatrix = createNativeImportCoverageMatrix({ imports: nativeImportResults });
68
69
  const matrixDurationMs = performance.now() - matrixStart;
69
70
 
71
+ const preservationStart = performance.now();
72
+ const preservationRecords = nativeImportResults.map((imported) => imported.metadata.sourcePreservation ?? createNativeSourcePreservation({
73
+ language: imported.language,
74
+ sourcePath: imported.sourcePath,
75
+ sourceText: imported.metadata.sourcePreservation?.sourceText ?? ''
76
+ }));
77
+ const preservationDurationMs = performance.now() - preservationStart;
78
+ const preservationTokens = preservationRecords.reduce((sum, record) => sum + record.tokens.length + record.trivia.length, 0);
79
+
70
80
  const sidecarStart = performance.now();
71
81
  const semanticSidecars = nativeImportResults.map((imported) => createSemanticImportSidecar(imported));
72
82
  const sidecarDurationMs = performance.now() - sidecarStart;
@@ -87,6 +97,9 @@ console.log(JSON.stringify({
87
97
  coverageMatrixLanguages: coverageMatrix.summary.languages,
88
98
  coverageMatrixImports: coverageMatrix.summary.imports,
89
99
  coverageMatrixDurationMs: Number(matrixDurationMs.toFixed(2)),
100
+ sourcePreservationRecords: preservationRecords.length,
101
+ sourcePreservationTokens: preservationTokens,
102
+ sourcePreservationDurationMs: Number(preservationDurationMs.toFixed(2)),
90
103
  semanticSidecars: semanticSidecars.length,
91
104
  sidecarOwnershipRegions,
92
105
  sidecarDurationMs: Number(sidecarDurationMs.toFixed(2)),
package/dist/index.d.ts CHANGED
@@ -79,27 +79,42 @@ export type NativeImportTaxonomyKind =
79
79
  | 'opaqueBodies'
80
80
  | 'macroExpansion'
81
81
  | 'preprocessor'
82
+ | 'conditionalCompilation'
82
83
  | 'metaprogramming'
84
+ | 'reflection'
83
85
  | 'generatedCode'
86
+ | 'overloadTypeInference'
84
87
  | 'sourcePreservation'
88
+ | 'commentsTrivia'
85
89
  | 'parserDiagnostics'
86
90
  | 'unsupportedSyntax'
87
91
  | 'partialSemanticIndex'
88
92
  | 'sourceMapApproximation'
93
+ | 'targetProjectionLoss'
89
94
  | string;
90
95
 
91
96
  export type NativeImportKnownLossKind =
92
97
  | 'declarationOnlyCoverage'
93
98
  | 'opaqueNative'
94
99
  | 'macroExpansion'
100
+ | 'macroHygiene'
95
101
  | 'preprocessor'
102
+ | 'conditionalCompilation'
96
103
  | 'metaprogramming'
104
+ | 'reflection'
105
+ | 'dynamicRuntime'
106
+ | 'dynamicDispatch'
97
107
  | 'generatedCode'
108
+ | 'overloadResolution'
109
+ | 'typeInference'
98
110
  | 'sourcePreservation'
111
+ | 'commentsTrivia'
99
112
  | 'parserDiagnostic'
100
113
  | 'unsupportedSyntax'
114
+ | 'unsupportedSemantic'
101
115
  | 'partialSemanticIndex'
102
116
  | 'sourceMapApproximation'
117
+ | 'targetProjectionLoss'
103
118
  | string;
104
119
 
105
120
  export interface NativeImportLossSummaryOptions {
@@ -201,6 +216,82 @@ export interface NativeImportCoverageMatrixOptions {
201
216
  readonly generatedAt?: number;
202
217
  }
203
218
 
219
+ export type NativeSourceTokenKind =
220
+ | 'identifier'
221
+ | 'keyword'
222
+ | 'number'
223
+ | 'string'
224
+ | 'operator'
225
+ | 'punctuation'
226
+ | 'comment'
227
+ | 'whitespace'
228
+ | 'newline'
229
+ | 'directive'
230
+ | 'unknown'
231
+ | string;
232
+
233
+ export interface NativeSourcePreservedToken {
234
+ readonly id: string;
235
+ readonly kind: NativeSourceTokenKind;
236
+ readonly text?: string;
237
+ readonly textHash: string;
238
+ readonly span: SourceSpan;
239
+ readonly metadata?: Record<string, unknown>;
240
+ }
241
+
242
+ export interface NativeSourcePreservedDirective {
243
+ readonly id: string;
244
+ readonly kind: string;
245
+ readonly text?: string;
246
+ readonly textHash: string;
247
+ readonly span: SourceSpan;
248
+ readonly metadata?: Record<string, unknown>;
249
+ }
250
+
251
+ export interface NativeSourcePreservation {
252
+ readonly kind: 'frontier.lang.nativeSourcePreservation';
253
+ readonly version: 1;
254
+ readonly id: string;
255
+ readonly language: FrontierSourceLanguage | string;
256
+ readonly sourcePath?: string;
257
+ readonly sourceHash: string;
258
+ readonly sourceBytes: number;
259
+ readonly lineCount: number;
260
+ readonly newline: 'lf' | 'crlf' | 'mixed' | 'none';
261
+ readonly encoding: string;
262
+ readonly sourceText?: string;
263
+ readonly tokens: readonly NativeSourcePreservedToken[];
264
+ readonly trivia: readonly NativeSourcePreservedToken[];
265
+ readonly directives: readonly NativeSourcePreservedDirective[];
266
+ readonly summary: {
267
+ readonly tokens: number;
268
+ readonly trivia: number;
269
+ readonly directives: number;
270
+ readonly comments: number;
271
+ readonly whitespace: number;
272
+ readonly exactSourceAvailable: boolean;
273
+ readonly truncated: boolean;
274
+ };
275
+ readonly metadata?: Record<string, unknown>;
276
+ }
277
+
278
+ export interface CreateNativeSourcePreservationOptions {
279
+ readonly id?: string;
280
+ readonly language?: FrontierSourceLanguage | string;
281
+ readonly sourcePath?: string;
282
+ readonly sourceHash?: string;
283
+ readonly sourceText: string;
284
+ readonly encoding?: string;
285
+ readonly includeSourceText?: boolean;
286
+ readonly includeTokens?: boolean;
287
+ readonly includeTrivia?: boolean;
288
+ readonly includeDirectives?: boolean;
289
+ readonly maxTokens?: number;
290
+ readonly maxTrivia?: number;
291
+ readonly maxDirectives?: number;
292
+ readonly metadata?: Record<string, unknown>;
293
+ }
294
+
204
295
  export interface SemanticImportOwnershipRegion {
205
296
  readonly id: string;
206
297
  readonly key: string;
@@ -322,6 +413,52 @@ export interface SemanticImportSidecarOptions {
322
413
  readonly metadata?: Record<string, unknown>;
323
414
  }
324
415
 
416
+ export type NativeImporterAdapterExactness =
417
+ | 'exact-parser-ast'
418
+ | 'parser-tree'
419
+ | 'adapter-reported-native-ast'
420
+ | 'loss-aware-native-ast'
421
+ | 'unknown'
422
+ | string;
423
+
424
+ export interface NativeImporterAdapterSemanticCoverage {
425
+ readonly level: 'native-ast' | 'declaration-index' | 'semantic-index' | string;
426
+ readonly declarations: boolean;
427
+ readonly symbols: boolean;
428
+ readonly references: boolean;
429
+ readonly types: boolean;
430
+ readonly controlFlow: boolean;
431
+ }
432
+
433
+ export interface NativeImporterAdapterCoverageObserved {
434
+ readonly diagnostics: number;
435
+ readonly losses: number;
436
+ readonly nativeAstNodes: number;
437
+ readonly semanticSymbols: number;
438
+ readonly sourceMapMappings: number;
439
+ readonly sourceRanges: boolean;
440
+ readonly generatedRanges: boolean;
441
+ }
442
+
443
+ export interface NativeImporterAdapterCoverageSummary {
444
+ readonly exactness: NativeImporterAdapterExactness;
445
+ readonly exactAst: boolean;
446
+ readonly tokens: boolean;
447
+ readonly trivia: boolean;
448
+ readonly diagnostics: boolean;
449
+ readonly sourceRanges: boolean;
450
+ readonly generatedRanges: boolean;
451
+ readonly semanticCoverage: NativeImporterAdapterSemanticCoverage;
452
+ readonly notes: readonly string[];
453
+ readonly observed?: NativeImporterAdapterCoverageObserved;
454
+ }
455
+
456
+ export type NativeImporterAdapterCoverageInput =
457
+ Omit<Partial<NativeImporterAdapterCoverageSummary>, 'semanticCoverage' | 'observed'> & {
458
+ readonly semanticCoverage?: Partial<NativeImporterAdapterSemanticCoverage>;
459
+ readonly observed?: Partial<NativeImporterAdapterCoverageObserved>;
460
+ };
461
+
325
462
  export interface NativeImporterAdapterDiagnostic {
326
463
  readonly id?: string;
327
464
  readonly severity?: 'info' | 'warning' | 'error';
@@ -362,6 +499,7 @@ export interface ImportNativeSourceOptions {
362
499
  readonly losses?: readonly NativeAstLossRecord[];
363
500
  readonly evidence?: readonly EvidenceRecord[];
364
501
  readonly evidenceId?: string;
502
+ readonly sourcePreservation?: NativeSourcePreservation;
365
503
  readonly patch?: SemanticPatchBundle;
366
504
  readonly patchId?: string;
367
505
  readonly author?: string;
@@ -405,6 +543,7 @@ export interface NativeImporterAdapter {
405
543
  readonly parser: string;
406
544
  readonly version?: string;
407
545
  readonly capabilities?: readonly string[];
546
+ readonly coverage?: NativeImporterAdapterCoverageInput;
408
547
  readonly supportedExtensions?: readonly string[];
409
548
  readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
410
549
  readonly parse: (input: NativeImporterAdapterParseInput) => NativeImporterAdapterParseResult | Promise<NativeImporterAdapterParseResult>;
@@ -416,6 +555,7 @@ export interface JavaScriptNativeImporterAdapterOptions {
416
555
  readonly parser?: string;
417
556
  readonly version?: string;
418
557
  readonly capabilities?: readonly string[];
558
+ readonly coverage?: NativeImporterAdapterCoverageInput;
419
559
  readonly supportedExtensions?: readonly string[];
420
560
  readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
421
561
  readonly ast?: unknown;
@@ -432,6 +572,7 @@ export interface TypeScriptCompilerNativeImporterAdapterOptions {
432
572
  readonly parser?: string;
433
573
  readonly version?: string;
434
574
  readonly capabilities?: readonly string[];
575
+ readonly coverage?: NativeImporterAdapterCoverageInput;
435
576
  readonly supportedExtensions?: readonly string[];
436
577
  readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
437
578
  readonly typescript?: unknown;
@@ -451,6 +592,7 @@ export interface TreeSitterNativeImporterAdapterOptions {
451
592
  readonly parserName?: string;
452
593
  readonly version?: string;
453
594
  readonly capabilities?: readonly string[];
595
+ readonly coverage?: NativeImporterAdapterCoverageInput;
454
596
  readonly supportedExtensions?: readonly string[];
455
597
  readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
456
598
  readonly parserInstance?: { readonly parse: (sourceText: string) => unknown };
@@ -466,6 +608,7 @@ export interface NativeImporterAdapterSummary {
466
608
  readonly parser: string;
467
609
  readonly version?: string;
468
610
  readonly capabilities: readonly string[];
611
+ readonly coverage: NativeImporterAdapterCoverageSummary;
469
612
  readonly supportedExtensions: readonly string[];
470
613
  readonly diagnostics: readonly NativeImporterAdapterDiagnostic[];
471
614
  }
@@ -595,6 +738,7 @@ export declare function resolveCapabilityAdapters(document: FrontierLangDocument
595
738
  export declare function summarizeNativeImportLosses(losses?: readonly NativeAstLossRecord[], options?: NativeImportLossSummaryOptions): NativeImportLossSummary;
596
739
  export declare function classifyNativeImportReadiness(losses?: readonly NativeAstLossRecord[], options?: NativeImportLossSummaryOptions): NativeImportReadinessClassification;
597
740
  export declare function createNativeImportCoverageMatrix(options?: NativeImportCoverageMatrixOptions): NativeImportCoverageMatrix;
741
+ export declare function createNativeSourcePreservation(options: CreateNativeSourcePreservationOptions): NativeSourcePreservation;
598
742
  export declare function createSemanticImportSidecar(importResult: NativeSourceImportResult | NativeProjectImportResult, options?: SemanticImportSidecarOptions): SemanticImportSidecar;
599
743
  export declare function createEstreeNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
600
744
  export declare function createBabelNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
package/dist/index.js CHANGED
@@ -73,27 +73,42 @@ export const NativeImportTaxonomyKinds = Object.freeze([
73
73
  'opaqueBodies',
74
74
  'macroExpansion',
75
75
  'preprocessor',
76
+ 'conditionalCompilation',
76
77
  'metaprogramming',
78
+ 'reflection',
77
79
  'generatedCode',
80
+ 'overloadTypeInference',
78
81
  'sourcePreservation',
82
+ 'commentsTrivia',
79
83
  'parserDiagnostics',
80
84
  'unsupportedSyntax',
81
85
  'partialSemanticIndex',
82
- 'sourceMapApproximation'
86
+ 'sourceMapApproximation',
87
+ 'targetProjectionLoss'
83
88
  ]);
84
89
 
85
90
  export const NativeImportLossKinds = Object.freeze([
86
91
  'declarationOnlyCoverage',
87
92
  'opaqueNative',
88
93
  'macroExpansion',
94
+ 'macroHygiene',
89
95
  'preprocessor',
96
+ 'conditionalCompilation',
90
97
  'metaprogramming',
98
+ 'reflection',
99
+ 'dynamicRuntime',
100
+ 'dynamicDispatch',
91
101
  'generatedCode',
102
+ 'overloadResolution',
103
+ 'typeInference',
92
104
  'sourcePreservation',
105
+ 'commentsTrivia',
93
106
  'parserDiagnostic',
94
107
  'unsupportedSyntax',
108
+ 'unsupportedSemantic',
95
109
  'partialSemanticIndex',
96
- 'sourceMapApproximation'
110
+ 'sourceMapApproximation',
111
+ 'targetProjectionLoss'
97
112
  ]);
98
113
 
99
114
  export const NativeImportReadinessBySeverity = Object.freeze({
@@ -356,6 +371,70 @@ export function createNativeImportCoverageMatrix(input = {}) {
356
371
  };
357
372
  }
358
373
 
374
+ export function createNativeSourcePreservation(options) {
375
+ if (!options || typeof options.sourceText !== 'string') {
376
+ throw new Error('createNativeSourcePreservation requires sourceText');
377
+ }
378
+ const language = options.language ?? 'source';
379
+ const sourceText = options.sourceText;
380
+ const computedSourceHash = hashSemanticValue(sourceText);
381
+ const declaredSourceHash = options.sourceHash;
382
+ const sourceHash = computedSourceHash;
383
+ const tokensAndTrivia = scanPreservedSourceTokens(sourceText, {
384
+ language,
385
+ sourcePath: options.sourcePath,
386
+ sourceHash,
387
+ includeTokens: options.includeTokens !== false,
388
+ includeTrivia: options.includeTrivia !== false,
389
+ maxTokens: options.maxTokens,
390
+ maxTrivia: options.maxTrivia
391
+ });
392
+ const directiveScan = options.includeDirectives === false
393
+ ? { directives: [], truncated: false }
394
+ : scanPreservedSourceDirectives(sourceText, {
395
+ language,
396
+ sourcePath: options.sourcePath,
397
+ sourceHash,
398
+ maxDirectives: options.maxDirectives
399
+ });
400
+ const directives = directiveScan.directives;
401
+ const newline = detectNewlineStyle(sourceText);
402
+ return {
403
+ kind: 'frontier.lang.nativeSourcePreservation',
404
+ version: 1,
405
+ id: options.id ?? `native_source_preservation_${idFragment(options.sourcePath ?? language)}_${idFragment(sourceHash)}`,
406
+ language,
407
+ sourcePath: options.sourcePath,
408
+ sourceHash,
409
+ sourceBytes: Buffer.byteLength(sourceText, options.encoding ?? 'utf8'),
410
+ lineCount: sourceText.length ? sourceText.split(/\r\n|\r|\n/).length : 0,
411
+ newline,
412
+ encoding: options.encoding ?? 'utf8',
413
+ ...(options.includeSourceText === false ? {} : { sourceText }),
414
+ tokens: tokensAndTrivia.tokens,
415
+ trivia: tokensAndTrivia.trivia,
416
+ directives,
417
+ summary: {
418
+ tokens: tokensAndTrivia.tokens.length,
419
+ trivia: tokensAndTrivia.trivia.length,
420
+ directives: directives.length,
421
+ comments: tokensAndTrivia.trivia.filter((entry) => entry.kind === 'comment').length,
422
+ whitespace: tokensAndTrivia.trivia.filter((entry) => entry.kind === 'whitespace' || entry.kind === 'newline').length,
423
+ exactSourceAvailable: options.includeSourceText !== false,
424
+ truncated: tokensAndTrivia.truncated || directiveScan.truncated
425
+ },
426
+ metadata: {
427
+ preservation: 'source-text-token-trivia-directive-evidence',
428
+ tokenization: 'frontier-lightweight-lexical-scan',
429
+ ...(declaredSourceHash ? {
430
+ declaredSourceHash,
431
+ sourceHashVerified: declaredSourceHash === computedSourceHash
432
+ } : {}),
433
+ ...options.metadata
434
+ }
435
+ };
436
+ }
437
+
359
438
  export function createSemanticImportSidecar(importResult, options = {}) {
360
439
  const imports = Array.isArray(importResult?.imports) ? importResult.imports : [importResult].filter(Boolean);
361
440
  const importEntries = imports.map((imported, index) => semanticImportSidecarEntry(imported, index, options));
@@ -458,6 +537,20 @@ export function createTypeScriptCompilerNativeImporterAdapter(options = {}) {
458
537
  parser: options.parser ?? 'typescript-compiler-api',
459
538
  version: options.version,
460
539
  capabilities: uniqueStrings(['nativeAst', 'semanticIndex', 'sourceMaps', 'diagnostics', ...(options.capabilities ?? [])]),
540
+ coverage: nativeImporterAdapterCoverage({
541
+ exactness: 'exact-parser-ast',
542
+ exactAst: true,
543
+ tokens: false,
544
+ trivia: false,
545
+ diagnostics: true,
546
+ sourceRanges: true,
547
+ generatedRanges: false,
548
+ semanticCoverage: declarationSemanticCoverage(),
549
+ notes: [
550
+ 'Normalizes a caller-owned TypeScript SourceFile into native AST nodes and declaration-level semantic index records.',
551
+ 'Type resolution, reference resolution, control flow, generated ranges, and parser token/trivia streams require host-supplied adapter evidence.'
552
+ ]
553
+ }, options.coverage),
461
554
  supportedExtensions: options.supportedExtensions ?? ['.ts', '.tsx', '.js', '.jsx'],
462
555
  diagnostics: options.diagnostics,
463
556
  parse(input) {
@@ -488,6 +581,20 @@ export function createTreeSitterNativeImporterAdapter(options = {}) {
488
581
  parser: options.parserName ?? options.parser ?? 'tree-sitter',
489
582
  version: options.version,
490
583
  capabilities: uniqueStrings(['nativeAst', 'semanticIndex', 'sourceMaps', 'diagnostics', ...(options.capabilities ?? [])]),
584
+ coverage: nativeImporterAdapterCoverage({
585
+ exactness: 'parser-tree',
586
+ exactAst: true,
587
+ tokens: false,
588
+ trivia: false,
589
+ diagnostics: true,
590
+ sourceRanges: true,
591
+ generatedRanges: false,
592
+ semanticCoverage: declarationSemanticCoverage(),
593
+ notes: [
594
+ 'Normalizes a caller-owned tree-sitter tree into native AST nodes and declaration-level semantic index records.',
595
+ 'The built-in wrapper walks named syntax nodes; exact token/trivia streams and generated ranges require adapter-specific evidence.'
596
+ ]
597
+ }, options.coverage),
491
598
  supportedExtensions: options.supportedExtensions ?? [],
492
599
  diagnostics: options.diagnostics,
493
600
  parse(input) {
@@ -592,7 +699,14 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
592
699
  parseResult.losses,
593
700
  diagnostics.map((diagnostic, index) => adapterDiagnosticToLoss(diagnostic, index, summary, parseInput))
594
701
  );
595
- const sourceEvidence = adapterDiagnosticsEvidence(summary, diagnostics, {
702
+ const adapterSummary = {
703
+ ...summary,
704
+ coverage: observeNativeImporterAdapterCoverage(summary.coverage, parseResult, {
705
+ diagnostics,
706
+ losses
707
+ })
708
+ };
709
+ const sourceEvidence = adapterDiagnosticsEvidence(adapterSummary, diagnostics, {
596
710
  language,
597
711
  parser,
598
712
  parserVersion,
@@ -614,8 +728,9 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
614
728
  metadata: {
615
729
  adapterId: summary.id,
616
730
  adapterVersion: summary.version,
617
- adapterCapabilities: summary.capabilities,
618
- supportedExtensions: summary.supportedExtensions,
731
+ adapterCapabilities: adapterSummary.capabilities,
732
+ adapterCoverage: adapterSummary.coverage,
733
+ supportedExtensions: adapterSummary.supportedExtensions,
619
734
  diagnostics: diagnostics.map(serializableDiagnostic),
620
735
  ...input.metadata,
621
736
  ...parseResult.metadata
@@ -624,6 +739,7 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
624
739
  adapterId: summary.id,
625
740
  adapterVersion: summary.version,
626
741
  parser,
742
+ adapterCoverage: adapterSummary.coverage,
627
743
  ...input.nativeAstMetadata,
628
744
  ...parseResult.nativeAstMetadata
629
745
  },
@@ -631,6 +747,7 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
631
747
  adapterId: summary.id,
632
748
  adapterVersion: summary.version,
633
749
  parser,
750
+ adapterCoverage: adapterSummary.coverage,
634
751
  ...input.nativeSourceMetadata,
635
752
  ...parseResult.nativeSourceMetadata
636
753
  },
@@ -649,7 +766,7 @@ export async function runNativeImporterAdapter(adapter, input = {}) {
649
766
  };
650
767
  return {
651
768
  ...importNativeSource(importInput),
652
- adapter: summary,
769
+ adapter: adapterSummary,
653
770
  diagnostics
654
771
  };
655
772
  }
@@ -681,6 +798,7 @@ export function projectNativeImportToSource(importResult, options = {}) {
681
798
  sourcePath: context.sourcePath,
682
799
  expectedSourceHash: context.sourceHash,
683
800
  providedSourceHash: candidateSource?.sourceHash,
801
+ sourcePreservationId: candidateSource?.sourcePreservationId,
684
802
  sourceHashVerified: candidateSource?.hashVerified ?? false,
685
803
  declarationCount: declarations.length
686
804
  }
@@ -725,6 +843,7 @@ export function projectNativeImportToSource(importResult, options = {}) {
725
843
  universalAstId: importResult.universalAst?.id,
726
844
  exactSourceAvailable: candidateSource?.exact === true,
727
845
  sourceTextAvailable: typeof candidateSource?.sourceText === 'string',
846
+ sourcePreservationId: candidateSource?.sourcePreservationId,
728
847
  sourceHashVerified: candidateSource?.hashVerified ?? false,
729
848
  nativeImportLossSummary,
730
849
  ...options.metadata
@@ -736,17 +855,30 @@ export function importNativeSource(input) {
736
855
  const language = input.language ?? input.nativeAst?.language;
737
856
  if (!language) throw new Error('importNativeSource requires a language or nativeAst.language');
738
857
  const sourcePath = input.sourcePath ?? input.nativeAst?.sourcePath;
739
- const sourceHash = input.sourceHash ?? input.nativeAst?.sourceHash ?? (input.sourceText ? hashSemanticValue(input.sourceText) : hashSemanticValue(input.nativeAst?.nodes ?? input.nativeAst ?? {}));
858
+ const declaredSourceHash = input.sourceHash ?? input.nativeAst?.sourceHash;
859
+ const sourceHash = typeof input.sourceText === 'string'
860
+ ? hashSemanticValue(input.sourceText)
861
+ : declaredSourceHash ?? hashSemanticValue(input.nativeAst?.nodes ?? input.nativeAst ?? {});
740
862
  const targetPath = input.targetPath ?? input.target?.emitPath;
741
863
  const targetHash = input.targetHash;
742
864
  const importIdPart = idFragment(input.id ?? input.nativeSourceId ?? sourcePath ?? language);
865
+ const sourcePreservation = input.sourcePreservation ?? (typeof input.sourceText === 'string'
866
+ ? createNativeSourcePreservation({
867
+ language,
868
+ sourcePath,
869
+ sourceHash: declaredSourceHash,
870
+ sourceText: input.sourceText,
871
+ metadata: { importIdPart }
872
+ })
873
+ : undefined);
743
874
  const lightweight = !input.nativeAst && !input.nodes && input.sourceText
744
875
  ? createLightweightNativeImport({
745
876
  language,
746
877
  sourceText: input.sourceText,
747
878
  sourcePath,
748
879
  sourceHash,
749
- parser: input.parser
880
+ parser: input.parser,
881
+ sourcePreservation
750
882
  })
751
883
  : undefined;
752
884
  const nativeAst = input.nativeAst ?? createNativeAstRecord({
@@ -769,6 +901,15 @@ export function importNativeSource(input) {
769
901
  losses: input.losses ?? lightweight?.losses,
770
902
  metadata: {
771
903
  ...(input.sourceText ? { sourceBytes: input.sourceText.length } : {}),
904
+ ...(sourcePreservation ? {
905
+ sourcePreservationId: sourcePreservation.id,
906
+ sourcePreservationSummary: sourcePreservation.summary,
907
+ sourcePreservation
908
+ } : {}),
909
+ ...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
910
+ declaredSourceHash,
911
+ sourceHashVerified: false
912
+ } : {}),
772
913
  ...lightweight?.metadata,
773
914
  ...input.nativeAstMetadata
774
915
  }
@@ -793,6 +934,14 @@ export function importNativeSource(input) {
793
934
  metadata: {
794
935
  semanticStatus,
795
936
  mappings: input.mappings ?? [],
937
+ ...(sourcePreservation ? {
938
+ sourcePreservationId: sourcePreservation.id,
939
+ sourcePreservation
940
+ } : {}),
941
+ ...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
942
+ declaredSourceHash,
943
+ sourceHashVerified: false
944
+ } : {}),
796
945
  ...input.nativeSourceMetadata
797
946
  }
798
947
  });
@@ -816,7 +965,16 @@ export function importNativeSource(input) {
816
965
  metadata: {
817
966
  parser: nativeAst.parser,
818
967
  sourcePath,
819
- semanticStatus
968
+ semanticStatus,
969
+ ...(sourcePreservation ? {
970
+ sourcePreservationId: sourcePreservation.id,
971
+ sourcePreservationSummary: sourcePreservation.summary
972
+ } : {})
973
+ ,
974
+ ...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
975
+ declaredSourceHash,
976
+ sourceHashVerified: false
977
+ } : {})
820
978
  }
821
979
  }];
822
980
  const lossSummary = summarizeNativeImportLosses(losses, {
@@ -896,6 +1054,14 @@ export function importNativeSource(input) {
896
1054
  sourcePath,
897
1055
  semanticStatus,
898
1056
  nativeImportLossSummary: lossSummary,
1057
+ ...(sourcePreservation ? {
1058
+ sourcePreservationId: sourcePreservation.id,
1059
+ sourcePreservation
1060
+ } : {}),
1061
+ ...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
1062
+ declaredSourceHash,
1063
+ sourceHashVerified: false
1064
+ } : {}),
899
1065
  ...input.universalAstMetadata
900
1066
  }
901
1067
  });
@@ -915,6 +1081,14 @@ export function importNativeSource(input) {
915
1081
  semanticIndexId: semanticIndex?.id,
916
1082
  universalAstId: universalAst.id,
917
1083
  sourceMapIds: sourceMaps.map((sourceMap) => sourceMap.id),
1084
+ ...(sourcePreservation ? {
1085
+ sourcePreservationId: sourcePreservation.id,
1086
+ sourcePreservationSummary: sourcePreservation.summary
1087
+ } : {}),
1088
+ ...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
1089
+ declaredSourceHash,
1090
+ sourceHashVerified: false
1091
+ } : {}),
918
1092
  nativeImportLossSummary: lossSummary
919
1093
  }
920
1094
  });
@@ -937,6 +1111,14 @@ export function importNativeSource(input) {
937
1111
  sourceMapIds: sourceMaps.map((sourceMap) => sourceMap.id),
938
1112
  semanticStatus,
939
1113
  mappings: resultSourceMapMappings,
1114
+ ...(sourcePreservation ? {
1115
+ sourcePreservationId: sourcePreservation.id,
1116
+ sourcePreservation
1117
+ } : {}),
1118
+ ...(declaredSourceHash && declaredSourceHash !== sourceHash ? {
1119
+ declaredSourceHash,
1120
+ sourceHashVerified: false
1121
+ } : {}),
940
1122
  nativeImportLossSummary: lossSummary,
941
1123
  ...input.metadata
942
1124
  }
@@ -1040,7 +1222,7 @@ function createLightweightNativeImport(input) {
1040
1222
  }
1041
1223
  if (declaration.loss) losses.push(declaration.loss);
1042
1224
  }
1043
- losses.push(...lightweightCoverageLosses(input, declarations));
1225
+ losses.push(...lightweightCoverageLosses(input, declarations, input.sourcePreservation));
1044
1226
 
1045
1227
  const semanticIndex = createSemanticIndexRecord({
1046
1228
  id: `index_${idFragment(input.sourcePath ?? input.language)}`,
@@ -1076,7 +1258,15 @@ function createLightweightNativeImport(input) {
1076
1258
  losses,
1077
1259
  semanticIndex,
1078
1260
  mappings,
1079
- metadata: { parser, scanKind: 'lightweight-declaration-scan', declarationCount: declarations.length }
1261
+ metadata: {
1262
+ parser,
1263
+ scanKind: 'lightweight-declaration-scan',
1264
+ declarationCount: declarations.length,
1265
+ ...(input.sourcePreservation ? {
1266
+ sourcePreservationId: input.sourcePreservation.id,
1267
+ sourcePreservationSummary: input.sourcePreservation.summary
1268
+ } : {})
1269
+ }
1080
1270
  };
1081
1271
  }
1082
1272
 
@@ -1121,20 +1311,32 @@ function nativeImportProjectionContext(importResult, options) {
1121
1311
  }
1122
1312
 
1123
1313
  function nativeProjectionSourceCandidate(context, options) {
1124
- const sourceText = options.sourceText ?? options.preservedSourceText ?? options.exactSourceText;
1314
+ const preservation = sourcePreservationFromProjectionContext(context);
1315
+ const explicitSourceText = options.sourceText ?? options.preservedSourceText ?? options.exactSourceText;
1316
+ const sourceText = explicitSourceText ?? preservation?.sourceText;
1125
1317
  if (typeof sourceText !== 'string') return undefined;
1126
- const sourceHash = options.sourceHash ?? hashSemanticValue(sourceText);
1318
+ const computedSourceHash = hashSemanticValue(sourceText);
1319
+ const declaredSourceHash = options.sourceHash ?? (explicitSourceText === undefined ? preservation?.sourceHash : undefined);
1320
+ const sourceHash = computedSourceHash;
1127
1321
  const hashVerified = Boolean(context.sourceHash);
1128
1322
  const exact = !context.sourceHash || sourceHash === context.sourceHash || options.verifySourceHash === false;
1129
1323
  return {
1130
1324
  sourceText,
1131
1325
  sourceHash,
1326
+ declaredSourceHash,
1132
1327
  hashVerified,
1133
1328
  exact,
1134
- mismatch: hashVerified && sourceHash !== context.sourceHash && options.verifySourceHash !== false
1329
+ mismatch: hashVerified && sourceHash !== context.sourceHash && options.verifySourceHash !== false,
1330
+ sourcePreservationId: preservation?.id
1135
1331
  };
1136
1332
  }
1137
1333
 
1334
+ function sourcePreservationFromProjectionContext(context) {
1335
+ return context.nativeSource?.metadata?.sourcePreservation
1336
+ ?? context.nativeAst?.metadata?.sourcePreservation
1337
+ ?? context.nativeSource?.ast?.metadata?.sourcePreservation;
1338
+ }
1339
+
1138
1340
  function nativeProjectionDeclarations(importResult, context) {
1139
1341
  const semanticIndex = context.semanticIndex;
1140
1342
  const occurrencesBySymbol = new Map();
@@ -1238,14 +1440,15 @@ function nativeProjectionStubLosses(context, candidateSource, declarations, opti
1238
1440
  : 'Exact native source text was not provided; emitted declaration stubs reconstructed from import metadata.';
1239
1441
  const losses = [nativeProjectionLoss(context, {
1240
1442
  id: `loss_${context.idPart}_native_source_stub`,
1241
- kind: 'sourcePreservation',
1443
+ kind: candidateSource?.mismatch ? 'sourcePreservation' : 'targetProjectionLoss',
1242
1444
  severity: 'warning',
1243
1445
  message,
1244
1446
  metadata: {
1245
1447
  reason,
1246
1448
  projectionMode: 'native-source-stubs',
1247
1449
  expectedSourceHash: context.sourceHash,
1248
- providedSourceHash: candidateSource?.sourceHash
1450
+ providedSourceHash: candidateSource?.sourceHash,
1451
+ declaredSourceHash: candidateSource?.declaredSourceHash
1249
1452
  }
1250
1453
  })];
1251
1454
  if (!declarations.length) {
@@ -2249,7 +2452,7 @@ function opaqueBodyLoss(input, lineNumber, nodeId, name) {
2249
2452
  };
2250
2453
  }
2251
2454
 
2252
- function lightweightCoverageLosses(input, declarations) {
2455
+ function lightweightCoverageLosses(input, declarations, sourcePreservation) {
2253
2456
  const id = idFragment(input.sourcePath ?? input.language);
2254
2457
  const span = declarations[0]?.span ?? {
2255
2458
  sourceId: input.sourceHash,
@@ -2291,12 +2494,275 @@ function lightweightCoverageLosses(input, declarations) {
2291
2494
  phase: 'read',
2292
2495
  sourceFormat: input.language,
2293
2496
  kind: 'sourcePreservation',
2294
- message: 'Comments, whitespace, token order, directives, and formatting are not preserved by the lightweight importer.',
2295
- span
2497
+ message: sourcePreservation
2498
+ ? 'Comments, whitespace, token order, directives, and formatting are preserved as opaque native source evidence; exact structural edits still require a parser adapter.'
2499
+ : 'Comments, whitespace, token order, directives, and formatting are not preserved by the lightweight importer.',
2500
+ span,
2501
+ metadata: sourcePreservation ? {
2502
+ sourcePreservationId: sourcePreservation.id,
2503
+ sourcePreservationSummary: sourcePreservation.summary
2504
+ } : undefined
2296
2505
  }
2297
2506
  ];
2298
2507
  }
2299
2508
 
2509
+ function scanPreservedSourceTokens(sourceText, input) {
2510
+ const tokens = [];
2511
+ const trivia = [];
2512
+ const includeTokens = input.includeTokens !== false;
2513
+ const includeTrivia = input.includeTrivia !== false;
2514
+ const maxTokens = Number.isFinite(input.maxTokens) ? Math.max(0, input.maxTokens) : 20000;
2515
+ const maxTrivia = Number.isFinite(input.maxTrivia) ? Math.max(0, input.maxTrivia) : 20000;
2516
+ let offset = 0;
2517
+ let line = 1;
2518
+ let column = 1;
2519
+ let truncated = false;
2520
+ const push = (target, kind, text, start) => {
2521
+ if ((target === tokens && !includeTokens) || (target === trivia && !includeTrivia)) return;
2522
+ const max = target === tokens ? maxTokens : maxTrivia;
2523
+ if (target.length >= max) {
2524
+ truncated = true;
2525
+ return;
2526
+ }
2527
+ target.push(preservedSourceSegment({
2528
+ index: target.length,
2529
+ kind,
2530
+ text,
2531
+ start,
2532
+ end: { offset, line, column },
2533
+ sourceHash: input.sourceHash,
2534
+ sourcePath: input.sourcePath
2535
+ }));
2536
+ };
2537
+ while (offset < sourceText.length) {
2538
+ const start = { offset, line, column };
2539
+ const char = sourceText[offset];
2540
+ const next = sourceText[offset + 1];
2541
+ if (char === '\r' || char === '\n') {
2542
+ const text = char === '\r' && next === '\n' ? '\r\n' : char;
2543
+ offset += text.length;
2544
+ line += 1;
2545
+ column = 1;
2546
+ push(trivia, 'newline', text, start);
2547
+ continue;
2548
+ }
2549
+ if (char === ' ' || char === '\t' || char === '\v' || char === '\f') {
2550
+ let text = '';
2551
+ while (offset < sourceText.length && /[ \t\v\f]/.test(sourceText[offset])) {
2552
+ text += sourceText[offset];
2553
+ offset += 1;
2554
+ column += 1;
2555
+ }
2556
+ push(trivia, 'whitespace', text, start);
2557
+ continue;
2558
+ }
2559
+ if (char === '/' && next === '/') {
2560
+ let text = '';
2561
+ while (offset < sourceText.length && sourceText[offset] !== '\n' && sourceText[offset] !== '\r') {
2562
+ text += sourceText[offset];
2563
+ offset += 1;
2564
+ column += 1;
2565
+ }
2566
+ push(trivia, 'comment', text, start);
2567
+ continue;
2568
+ }
2569
+ if (char === '/' && next === '*') {
2570
+ let text = '';
2571
+ while (offset < sourceText.length) {
2572
+ const current = sourceText[offset];
2573
+ text += current;
2574
+ offset += 1;
2575
+ if (current === '\n') {
2576
+ line += 1;
2577
+ column = 1;
2578
+ } else {
2579
+ column += 1;
2580
+ }
2581
+ if (current === '*' && sourceText[offset] === '/') {
2582
+ text += '/';
2583
+ offset += 1;
2584
+ column += 1;
2585
+ break;
2586
+ }
2587
+ }
2588
+ push(trivia, 'comment', text, start);
2589
+ continue;
2590
+ }
2591
+ if (char === '#' && isHashCommentLanguage(input.language)) {
2592
+ let text = '';
2593
+ while (offset < sourceText.length && sourceText[offset] !== '\n' && sourceText[offset] !== '\r') {
2594
+ text += sourceText[offset];
2595
+ offset += 1;
2596
+ column += 1;
2597
+ }
2598
+ push(trivia, preservedHashLineKind(text), text, start);
2599
+ continue;
2600
+ }
2601
+ if (char === '"' || char === '\'' || char === '`') {
2602
+ const quote = char;
2603
+ let text = char;
2604
+ offset += 1;
2605
+ column += 1;
2606
+ let escaped = false;
2607
+ while (offset < sourceText.length) {
2608
+ const current = sourceText[offset];
2609
+ text += current;
2610
+ offset += 1;
2611
+ if (current === '\n') {
2612
+ line += 1;
2613
+ column = 1;
2614
+ } else {
2615
+ column += 1;
2616
+ }
2617
+ if (escaped) {
2618
+ escaped = false;
2619
+ } else if (current === '\\') {
2620
+ escaped = true;
2621
+ } else if (current === quote) {
2622
+ break;
2623
+ }
2624
+ }
2625
+ push(tokens, 'string', text, start);
2626
+ continue;
2627
+ }
2628
+ if (/[0-9]/.test(char)) {
2629
+ let text = '';
2630
+ while (offset < sourceText.length && /[0-9a-fA-F_xXoObBeE.+-]/.test(sourceText[offset])) {
2631
+ text += sourceText[offset];
2632
+ offset += 1;
2633
+ column += 1;
2634
+ }
2635
+ push(tokens, 'number', text, start);
2636
+ continue;
2637
+ }
2638
+ if (isIdentifierStart(char)) {
2639
+ let text = '';
2640
+ while (offset < sourceText.length && isIdentifierPart(sourceText[offset])) {
2641
+ text += sourceText[offset];
2642
+ offset += 1;
2643
+ column += 1;
2644
+ }
2645
+ push(tokens, preservedKeywordSet.has(text) ? 'keyword' : 'identifier', text, start);
2646
+ continue;
2647
+ }
2648
+ let text = char;
2649
+ if (/[=+\-*/%&|^!<>?:.]/.test(char)) {
2650
+ while (offset + text.length < sourceText.length && /[=+\-*/%&|^!<>?:.]/.test(sourceText[offset + text.length])) text += sourceText[offset + text.length];
2651
+ offset += text.length;
2652
+ column += text.length;
2653
+ push(tokens, 'operator', text, start);
2654
+ } else {
2655
+ offset += 1;
2656
+ column += 1;
2657
+ push(tokens, /[()[\]{};,]/.test(char) ? 'punctuation' : 'unknown', text, start);
2658
+ }
2659
+ }
2660
+ return { tokens, trivia, truncated };
2661
+ }
2662
+
2663
+ function scanPreservedSourceDirectives(sourceText, input) {
2664
+ const directives = [];
2665
+ const maxDirectives = Number.isFinite(input.maxDirectives) ? Math.max(0, input.maxDirectives) : 20000;
2666
+ let truncated = false;
2667
+ let offset = 0;
2668
+ for (const { line, number } of sourceLines(sourceText)) {
2669
+ const trimmed = line.trim();
2670
+ const directiveKind = preservedDirectiveKind(trimmed, input.language);
2671
+ if (directiveKind) {
2672
+ if (directives.length >= maxDirectives) {
2673
+ truncated = true;
2674
+ offset += line.length + 1;
2675
+ continue;
2676
+ }
2677
+ const startColumn = Math.max(1, line.indexOf(trimmed) + 1);
2678
+ directives.push({
2679
+ id: `directive_${idFragment(input.sourcePath ?? input.language)}_${directives.length + 1}`,
2680
+ kind: directiveKind,
2681
+ text: trimmed,
2682
+ textHash: hashSemanticValue(trimmed),
2683
+ span: {
2684
+ sourceId: input.sourceHash,
2685
+ path: input.sourcePath,
2686
+ start: offset + startColumn - 1,
2687
+ end: offset + startColumn - 1 + trimmed.length,
2688
+ startLine: number,
2689
+ startColumn,
2690
+ endLine: number,
2691
+ endColumn: startColumn + trimmed.length
2692
+ },
2693
+ metadata: { language: input.language }
2694
+ });
2695
+ }
2696
+ offset += line.length + 1;
2697
+ }
2698
+ return { directives, truncated };
2699
+ }
2700
+
2701
+ function preservedSourceSegment(input) {
2702
+ const id = `${input.kind}_${input.index + 1}_${idFragment(input.start.offset)}`;
2703
+ return {
2704
+ id,
2705
+ kind: input.kind,
2706
+ text: input.text,
2707
+ textHash: hashSemanticValue(input.text),
2708
+ span: {
2709
+ sourceId: input.sourceHash,
2710
+ path: input.sourcePath,
2711
+ start: input.start.offset,
2712
+ end: input.end.offset,
2713
+ startLine: input.start.line,
2714
+ startColumn: input.start.column,
2715
+ endLine: input.end.line,
2716
+ endColumn: input.end.column
2717
+ }
2718
+ };
2719
+ }
2720
+
2721
+ function preservedDirectiveKind(trimmed, language) {
2722
+ if (!trimmed) return undefined;
2723
+ if (/^#\s*(include|define|if|ifdef|ifndef|elif|else|endif|pragma)\b/.test(trimmed)) return 'preprocessor';
2724
+ if (/^#!\s*/.test(trimmed)) return 'shebang';
2725
+ if (/^['"]use strict['"];?$/.test(trimmed)) return 'runtime-directive';
2726
+ if (/^(import|export|package|module|namespace|use|using|from|require)\b/.test(trimmed)) return 'module-directive';
2727
+ if (normalizeNativeLanguageId(language) === 'python' && /^from\s+\S+\s+import\b/.test(trimmed)) return 'module-directive';
2728
+ return undefined;
2729
+ }
2730
+
2731
+ function preservedHashLineKind(text) {
2732
+ return preservedDirectiveKind(String(text).trim(), 'c') ? 'directive' : 'comment';
2733
+ }
2734
+
2735
+ function isHashCommentLanguage(language) {
2736
+ return ['python', 'ruby', 'shell', 'bash', 'zsh', 'r', 'perl', 'yaml', 'toml'].includes(normalizeNativeLanguageId(language));
2737
+ }
2738
+
2739
+ function detectNewlineStyle(sourceText) {
2740
+ const crlf = (sourceText.match(/\r\n/g) ?? []).length;
2741
+ const normalized = sourceText.replace(/\r\n/g, '');
2742
+ const lf = (normalized.match(/\n/g) ?? []).length;
2743
+ const cr = (normalized.match(/\r/g) ?? []).length;
2744
+ const kinds = [crlf ? 'crlf' : undefined, lf ? 'lf' : undefined, cr ? 'cr' : undefined].filter(Boolean);
2745
+ if (!kinds.length) return 'none';
2746
+ if (kinds.length > 1 || cr) return 'mixed';
2747
+ return kinds[0];
2748
+ }
2749
+
2750
+ const preservedKeywordSet = new Set([
2751
+ 'abstract', 'as', 'async', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue', 'def', 'defer',
2752
+ 'do', 'else', 'enum', 'export', 'extends', 'extern', 'false', 'final', 'fn', 'for', 'from', 'func', 'function',
2753
+ 'if', 'impl', 'import', 'in', 'interface', 'let', 'match', 'mod', 'module', 'mut', 'namespace', 'new', 'nil',
2754
+ 'none', 'null', 'package', 'private', 'protected', 'pub', 'public', 'return', 'self', 'static', 'struct',
2755
+ 'switch', 'this', 'throw', 'trait', 'true', 'try', 'type', 'use', 'using', 'var', 'while', 'yield'
2756
+ ]);
2757
+
2758
+ function isIdentifierStart(char) {
2759
+ return /[A-Za-z_$]/.test(char ?? '');
2760
+ }
2761
+
2762
+ function isIdentifierPart(char) {
2763
+ return /[A-Za-z0-9_$]/.test(char ?? '');
2764
+ }
2765
+
2300
2766
  function sourceLines(sourceText) {
2301
2767
  return String(sourceText ?? '').split(/\r?\n/).map((line, index) => ({ line, number: index + 1 }));
2302
2768
  }
@@ -2636,11 +3102,13 @@ function nativeImportCategoryForLossKind(kind) {
2636
3102
  if (kind === 'preprocessor' || kind === 'conditionalCompilation' || kind === 'macroHygiene') return 'preprocessor';
2637
3103
  if (kind === 'metaprogramming' || kind === 'reflection' || kind === 'dynamicDispatch' || kind === 'dynamicRuntime') return 'metaprogramming';
2638
3104
  if (kind === 'generatedCode') return 'generatedCode';
2639
- if (kind === 'sourcePreservation' || kind === 'nonRoundTrippable') return 'sourcePreservation';
3105
+ if (kind === 'overloadResolution' || kind === 'typeInference') return 'overloadTypeInference';
3106
+ if (kind === 'sourcePreservation' || kind === 'commentsTrivia' || kind === 'nonRoundTrippable') return 'sourcePreservation';
2640
3107
  if (kind === 'parserDiagnostic') return 'parserDiagnostics';
2641
3108
  if (kind === 'unsupportedSyntax' || kind === 'unsupportedSemantic') return 'unsupportedSyntax';
2642
3109
  if (kind === 'partialSemanticIndex') return 'partialSemanticIndex';
2643
3110
  if (kind === 'sourceMapApproximation') return 'sourceMapApproximation';
3111
+ if (kind === 'targetProjectionLoss') return 'targetProjectionLoss';
2644
3112
  return String(kind ?? 'opaqueNative');
2645
3113
  }
2646
3114
 
@@ -3004,6 +3472,20 @@ function createJavaScriptSyntaxImporterAdapter(options) {
3004
3472
  parser: options.parser,
3005
3473
  version: options.version,
3006
3474
  capabilities: uniqueStrings(['nativeAst', 'semanticIndex', 'sourceMaps', 'diagnostics', ...(options.capabilities ?? [])]),
3475
+ coverage: nativeImporterAdapterCoverage({
3476
+ exactness: 'exact-parser-ast',
3477
+ exactAst: true,
3478
+ tokens: false,
3479
+ trivia: false,
3480
+ diagnostics: true,
3481
+ sourceRanges: true,
3482
+ generatedRanges: false,
3483
+ semanticCoverage: declarationSemanticCoverage(),
3484
+ notes: [
3485
+ 'Normalizes a caller-owned ESTree/Babel-compatible AST into native AST nodes and declaration-level semantic index records.',
3486
+ 'The wrapper ignores parser token/trivia/comment arrays unless a host adapter explicitly maps them into preservation evidence.'
3487
+ ]
3488
+ }, options.coverage),
3007
3489
  supportedExtensions: options.supportedExtensions,
3008
3490
  diagnostics: options.diagnostics,
3009
3491
  parse(input) {
@@ -3454,6 +3936,14 @@ function createNativeProjectImportResult(input, imports) {
3454
3936
  mergeCandidates.push(...(result.mergeCandidates ?? []));
3455
3937
  operations.push(...(result.patch?.operations ?? []));
3456
3938
  }
3939
+ const uniqueLosses = uniqueByLossId(losses);
3940
+ const uniqueEvidence = uniqueByEvidenceId(evidence);
3941
+ const nativeImportLossSummary = summarizeNativeImportLosses(uniqueLosses, {
3942
+ evidence: uniqueEvidence,
3943
+ scanKind: 'native-project-import',
3944
+ semanticStatus: uniqueLosses.some((loss) => loss.severity === 'error') ? 'partial' : 'mapped'
3945
+ });
3946
+ const sourcePreservationSummary = summarizeProjectSourcePreservation(imports);
3457
3947
  const document = createDocument({
3458
3948
  id: input.documentId ?? `document_${idPart}`,
3459
3949
  name: input.documentName ?? input.name ?? 'NativeProject',
@@ -3464,6 +3954,8 @@ function createNativeProjectImportResult(input, imports) {
3464
3954
  semanticStatus: losses.some((loss) => loss.severity === 'error') ? 'partial' : 'mapped',
3465
3955
  projectRoot: input.projectRoot,
3466
3956
  sourceCount: imports.length,
3957
+ nativeImportLossSummary,
3958
+ sourcePreservationSummary,
3467
3959
  ...input.documentMetadata
3468
3960
  }
3469
3961
  });
@@ -3473,12 +3965,14 @@ function createNativeProjectImportResult(input, imports) {
3473
3965
  nativeSources,
3474
3966
  semanticIndex,
3475
3967
  sourceMaps,
3476
- losses: uniqueByLossId(losses),
3477
- evidence: uniqueByEvidenceId(evidence),
3968
+ losses: uniqueLosses,
3969
+ evidence: uniqueEvidence,
3478
3970
  metadata: {
3479
3971
  sourceLanguage: input.language ?? 'mixed',
3480
3972
  projectRoot: input.projectRoot,
3481
3973
  sourceCount: imports.length,
3974
+ nativeImportLossSummary,
3975
+ sourcePreservationSummary,
3482
3976
  ...input.universalAstMetadata
3483
3977
  }
3484
3978
  });
@@ -3487,12 +3981,14 @@ function createNativeProjectImportResult(input, imports) {
3487
3981
  author: input.author ?? '@shapeshift-labs/frontier-lang-compiler/importNativeProject',
3488
3982
  risk: losses.some((loss) => loss.severity === 'error') ? 'high' : losses.some((loss) => loss.severity === 'warning') ? 'medium' : 'low',
3489
3983
  operations,
3490
- evidence: uniqueByEvidenceId(evidence),
3984
+ evidence: uniqueEvidence,
3491
3985
  metadata: {
3492
3986
  semanticIndexId: semanticIndex?.id,
3493
3987
  universalAstId: universalAst.id,
3494
3988
  sourceMapIds: sourceMaps.map((sourceMap) => sourceMap.id),
3495
- sourceCount: imports.length
3989
+ sourceCount: imports.length,
3990
+ nativeImportLossSummary,
3991
+ sourcePreservationSummary
3496
3992
  }
3497
3993
  });
3498
3994
  return {
@@ -3508,17 +4004,35 @@ function createNativeProjectImportResult(input, imports) {
3508
4004
  semanticIndex,
3509
4005
  universalAst,
3510
4006
  sourceMaps,
3511
- losses: uniqueByLossId(losses),
3512
- evidence: uniqueByEvidenceId(evidence),
4007
+ losses: uniqueLosses,
4008
+ evidence: uniqueEvidence,
3513
4009
  mergeCandidates,
3514
4010
  metadata: {
3515
4011
  sourceCount: imports.length,
3516
4012
  sourcePaths: imports.map((result) => result.sourcePath).filter(Boolean),
4013
+ nativeImportLossSummary,
4014
+ sourcePreservationSummary,
3517
4015
  ...input.metadata
3518
4016
  }
3519
4017
  };
3520
4018
  }
3521
4019
 
4020
+ function summarizeProjectSourcePreservation(imports) {
4021
+ const records = imports
4022
+ .map((result) => result.metadata?.sourcePreservation ?? result.nativeSource?.metadata?.sourcePreservation ?? result.nativeAst?.metadata?.sourcePreservation)
4023
+ .filter(Boolean);
4024
+ return {
4025
+ total: records.length,
4026
+ exactSourceAvailable: records.filter((record) => record.summary?.exactSourceAvailable).length,
4027
+ sourceBytes: records.reduce((sum, record) => sum + (record.sourceBytes ?? 0), 0),
4028
+ tokens: records.reduce((sum, record) => sum + (record.summary?.tokens ?? record.tokens?.length ?? 0), 0),
4029
+ trivia: records.reduce((sum, record) => sum + (record.summary?.trivia ?? record.trivia?.length ?? 0), 0),
4030
+ directives: records.reduce((sum, record) => sum + (record.summary?.directives ?? record.directives?.length ?? 0), 0),
4031
+ truncated: records.some((record) => record.summary?.truncated === true),
4032
+ ids: records.map((record) => record.id).filter(Boolean)
4033
+ };
4034
+ }
4035
+
3522
4036
  function mergeSemanticIndexes(imports, input, idPart) {
3523
4037
  const indexes = imports.map((result) => result.semanticIndex ?? result.universalAst?.semanticIndex).filter(Boolean);
3524
4038
  if (!indexes.length) return undefined;
@@ -3564,9 +4078,15 @@ function normalizeNativeImporterAdapter(adapter) {
3564
4078
  parser: String(adapter.parser),
3565
4079
  version: adapter.version === undefined ? undefined : String(adapter.version)
3566
4080
  };
4081
+ const capabilities = normalizeStringList(adapter.capabilities);
3567
4082
  return Object.freeze({
3568
4083
  ...summaryInput,
3569
- capabilities: normalizeStringList(adapter.capabilities),
4084
+ capabilities,
4085
+ coverage: normalizeNativeImporterAdapterCoverage(adapter.coverage, {
4086
+ capabilities,
4087
+ language: adapter.language,
4088
+ parser: String(adapter.parser)
4089
+ }),
3570
4090
  supportedExtensions: normalizeStringList(adapter.supportedExtensions).map((extension) => extension.startsWith('.') ? extension.toLowerCase() : `.${extension.toLowerCase()}`),
3571
4091
  diagnostics: normalizeAdapterDiagnostics(adapter.diagnostics, summaryInput, {
3572
4092
  language: adapter.language,
@@ -3576,6 +4096,138 @@ function normalizeNativeImporterAdapter(adapter) {
3576
4096
  });
3577
4097
  }
3578
4098
 
4099
+ function nativeImporterAdapterCoverage(defaults = {}, overrides = {}) {
4100
+ return {
4101
+ ...defaults,
4102
+ ...overrides,
4103
+ semanticCoverage: {
4104
+ ...(defaults.semanticCoverage ?? {}),
4105
+ ...(overrides.semanticCoverage ?? {})
4106
+ },
4107
+ notes: uniqueStrings([...(defaults.notes ?? []), ...(overrides.notes ?? [])])
4108
+ };
4109
+ }
4110
+
4111
+ function normalizeNativeImporterAdapterCoverage(value = {}, context = {}) {
4112
+ const capabilities = new Set(normalizeStringList(context.capabilities).map((capability) => capability.toLowerCase()));
4113
+ const hasCapability = (...names) => names.some((name) => capabilities.has(String(name).toLowerCase()));
4114
+ const exactAst = Boolean(value.exactAst ?? hasCapability('exactAst', 'exactAstImport'));
4115
+ const sourceRanges = Boolean(value.sourceRanges ?? hasCapability('sourceRanges', 'sourceRange', 'ranges', 'sourceMaps'));
4116
+ const generatedRanges = Boolean(value.generatedRanges ?? hasCapability('generatedRanges', 'generatedRange', 'generatedSourceMaps'));
4117
+ const diagnostics = Boolean(value.diagnostics ?? hasCapability('diagnostics', 'parserDiagnostics'));
4118
+ return Object.freeze({
4119
+ exactness: String(value.exactness ?? inferredAdapterExactness(exactAst, capabilities)),
4120
+ exactAst,
4121
+ tokens: Boolean(value.tokens ?? hasCapability('tokens', 'tokenStream')),
4122
+ trivia: Boolean(value.trivia ?? hasCapability('trivia', 'comments', 'formatting')),
4123
+ diagnostics,
4124
+ sourceRanges,
4125
+ generatedRanges,
4126
+ semanticCoverage: normalizeNativeImporterSemanticCoverage(value.semanticCoverage, {
4127
+ capabilities,
4128
+ sourceRanges,
4129
+ generatedRanges
4130
+ }),
4131
+ notes: uniqueStrings(value.notes ?? inferredAdapterCoverageNotes(context, {
4132
+ exactAst,
4133
+ sourceRanges,
4134
+ generatedRanges,
4135
+ diagnostics
4136
+ }))
4137
+ });
4138
+ }
4139
+
4140
+ function observeNativeImporterAdapterCoverage(coverage, parseResult = {}, context = {}) {
4141
+ const nodes = parseResult.nativeAst?.nodes ?? parseResult.nodes ?? {};
4142
+ const nodeList = Object.values(nodes);
4143
+ const sourceMapMappings = parseResult.sourceMaps?.flatMap((sourceMap) => sourceMap.mappings ?? []) ?? parseResult.mappings ?? [];
4144
+ const semanticIndex = parseResult.semanticIndex;
4145
+ const semanticSymbols = semanticIndex?.symbols?.length ?? 0;
4146
+ const observedSourceRanges = nodeList.some((node) => Boolean(node?.span)) || sourceMapMappings.some((mapping) => Boolean(mapping?.sourceSpan));
4147
+ const observedGeneratedRanges = sourceMapMappings.some((mapping) => Boolean(mapping?.generatedSpan));
4148
+ const observedSemanticCoverage = normalizeNativeImporterSemanticCoverage({
4149
+ ...coverage.semanticCoverage,
4150
+ level: semanticSymbols
4151
+ ? maxSemanticCoverageLevel(coverage.semanticCoverage?.level, 'declaration-index')
4152
+ : coverage.semanticCoverage?.level,
4153
+ declarations: coverage.semanticCoverage?.declarations || semanticSymbols > 0,
4154
+ symbols: coverage.semanticCoverage?.symbols || semanticSymbols > 0
4155
+ }, {});
4156
+ return Object.freeze({
4157
+ ...coverage,
4158
+ diagnostics: coverage.diagnostics || (context.diagnostics?.length ?? 0) > 0,
4159
+ sourceRanges: coverage.sourceRanges || observedSourceRanges,
4160
+ generatedRanges: coverage.generatedRanges || observedGeneratedRanges,
4161
+ semanticCoverage: observedSemanticCoverage,
4162
+ observed: {
4163
+ diagnostics: context.diagnostics?.length ?? 0,
4164
+ losses: context.losses?.length ?? 0,
4165
+ nativeAstNodes: nodeList.length,
4166
+ semanticSymbols,
4167
+ sourceMapMappings: sourceMapMappings.length,
4168
+ sourceRanges: observedSourceRanges,
4169
+ generatedRanges: observedGeneratedRanges
4170
+ }
4171
+ });
4172
+ }
4173
+
4174
+ function declarationSemanticCoverage() {
4175
+ return {
4176
+ level: 'declaration-index',
4177
+ declarations: true,
4178
+ symbols: true,
4179
+ references: false,
4180
+ types: false,
4181
+ controlFlow: false
4182
+ };
4183
+ }
4184
+
4185
+ function normalizeNativeImporterSemanticCoverage(value = {}, context = {}) {
4186
+ const capabilities = context.capabilities ?? new Set();
4187
+ const hasCapability = (...names) => names.some((name) => capabilities.has(String(name).toLowerCase()));
4188
+ const declarations = Boolean(value.declarations ?? hasCapability('semanticIndex', 'declarations'));
4189
+ const symbols = Boolean(value.symbols ?? declarations);
4190
+ const references = Boolean(value.references ?? hasCapability('references', 'referenceIndex'));
4191
+ const types = Boolean(value.types ?? hasCapability('types', 'typeResolution', 'typeChecking'));
4192
+ const controlFlow = Boolean(value.controlFlow ?? hasCapability('controlFlow', 'cfg'));
4193
+ return Object.freeze({
4194
+ level: String(value.level ?? inferredSemanticCoverageLevel({ declarations, symbols, references, types, controlFlow })),
4195
+ declarations,
4196
+ symbols,
4197
+ references,
4198
+ types,
4199
+ controlFlow
4200
+ });
4201
+ }
4202
+
4203
+ function inferredAdapterExactness(exactAst, capabilities) {
4204
+ if (exactAst) return 'exact-parser-ast';
4205
+ if (capabilities.has('nativeast')) return 'adapter-reported-native-ast';
4206
+ return 'loss-aware-native-ast';
4207
+ }
4208
+
4209
+ function inferredSemanticCoverageLevel(input) {
4210
+ if (input.references || input.types || input.controlFlow) return 'semantic-index';
4211
+ if (input.declarations || input.symbols) return 'declaration-index';
4212
+ return 'native-ast';
4213
+ }
4214
+
4215
+ function maxSemanticCoverageLevel(left, right) {
4216
+ const ranks = { 'native-ast': 0, 'declaration-index': 1, 'semantic-index': 2 };
4217
+ const leftRank = ranks[left] ?? 0;
4218
+ const rightRank = ranks[right] ?? 0;
4219
+ return rightRank > leftRank ? right : left;
4220
+ }
4221
+
4222
+ function inferredAdapterCoverageNotes(context, coverage) {
4223
+ const notes = [];
4224
+ if (!coverage.exactAst) notes.push('Adapter did not declare exact parser AST/CST coverage; import readiness depends on losses and evidence.');
4225
+ if (!coverage.generatedRanges) notes.push('Adapter does not declare generated-range coverage unless parse output includes generated spans.');
4226
+ if (!coverage.diagnostics) notes.push('Adapter did not declare parser diagnostics support.');
4227
+ if (context.language && context.parser) notes.push(`Coverage summary applies to ${context.language} via ${context.parser}.`);
4228
+ return notes;
4229
+ }
4230
+
3579
4231
  function normalizeStringList(value) {
3580
4232
  if (value === undefined || value === null) return [];
3581
4233
  if (Array.isArray(value)) return value.map((item) => String(item)).filter(Boolean);
@@ -3669,6 +4321,7 @@ function adapterDiagnosticsEvidence(adapter, diagnostics, input) {
3669
4321
  parserVersion: input.parserVersion,
3670
4322
  sourceHash: input.sourceHash,
3671
4323
  capabilities: adapter.capabilities,
4324
+ coverage: adapter.coverage,
3672
4325
  supportedExtensions: adapter.supportedExtensions,
3673
4326
  diagnostics: diagnostics.map(serializableDiagnostic),
3674
4327
  errors,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@shapeshift-labs/frontier-lang-compiler",
3
- "version": "0.2.9",
3
+ "version": "0.2.10",
4
4
  "description": "Compiler facade for Frontier Lang source documents and language projection adapters.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",