@shapeshift-labs/frontier-lang-compiler 0.2.9 → 0.2.11

package/README.md

@@ -50,6 +50,8 @@ console.log(summary.categories);
 console.log(readiness.readiness);
 ```
 
+The loss taxonomy separates broad scanner limits from specific round-trip risks such as conditional compilation, reflection, overload/type-inference gaps, comments/trivia preservation, source-map approximation, parser diagnostics, and target projection loss. These records are evidence labels for merge admission; they are not claims that the lightweight scanner expanded macros, evaluated inactive branches, resolved overloads, or ran a type checker.
+
 Ask the compiler what is actually covered before sending native imports into a merge queue:
 
 ```js
@@ -71,6 +73,28 @@ console.log(python.imports.readiness); // scanner imports are intentionally revi
 console.log(python.parserAdapters); // host-owned exact parsers such as LibCST can be injected
 ```
 
+Preserve exact native source text, token/trivia hashes, comments, whitespace, and source directives as evidence. This does not claim full semantic understanding; it keeps round-trip material available while exact parser adapters catch up:
+
+```js
+import {
+  createNativeSourcePreservation,
+  importNativeSource
+} from '@shapeshift-labs/frontier-lang-compiler';
+
+const sourceText = '// kept\nexport function step(frame) { return frame + 1; }\n';
+const preservation = createNativeSourcePreservation({
+  language: 'javascript',
+  sourcePath: 'src/runtime.js',
+  sourceText
+});
+const imported = importNativeSource({ language: 'javascript', sourcePath: 'src/runtime.js', sourceText });
+
+console.log(preservation.summary.comments); // comments and whitespace are tracked
+console.log(imported.metadata.sourcePreservation.sourceHash);
+```
+
+When `sourceText` is present, hashes are computed from the actual text. Caller-provided hashes are recorded as declared metadata and cannot make stale text project as exact source. Use `includeTokens`, `includeTrivia`, `includeDirectives`, and `max*` options to keep preservation records compact for large files.
+
 Create a compact semantic sidecar for swarm merge admission. This is the artifact a coordinator can index instead of reading a worker directory by hand:
 
 ```js
@@ -96,7 +120,7 @@ console.log(sidecar.ownershipRegions[0].key); // source#src/runtime.ts#class#Run
 console.log(sidecar.patchHints[0].supportedOperations); // source-region patch operations
 ```
 
-Project a native import back to source. Exact source is preserved only when the supplied text matches the import hash; otherwise the compiler emits declaration stubs with review-required loss evidence:
+Project a native import back to source. Exact source is preserved when the import carries matching source-preservation evidence or when supplied text matches the import hash; otherwise the compiler emits declaration stubs with review-required loss evidence:
 
 ```js
 import {
@@ -111,7 +135,7 @@ const imported = importNativeSource({
   sourceText
 });
 
-const projection = projectNativeImportToSource(imported, { sourceText });
+const projection = projectNativeImportToSource(imported);
 
 console.log(projection.mode); // "preserved-source"
 console.log(projection.readiness.readiness); // "ready"
@@ -146,6 +170,9 @@ const project = await importNativeProject({
 
 console.log(imported.universalAst.sourceMaps.length);
 console.log(project.semanticIndex.symbols.length);
+console.log(imported.adapter.coverage.exactness);
+console.log(imported.adapter.coverage.semanticCoverage.level);
+console.log(project.metadata.sourcePreservationSummary.total);
 ```
 
 The built-in adapter factories are dependency-light wrappers for caller-owned parsers or ASTs:
@@ -155,6 +182,8 @@ The built-in adapter factories are dependency-light wrappers for caller-owned pa
 - `createTypeScriptCompilerNativeImporterAdapter`
 - `createTreeSitterNativeImporterAdapter`
 
+Adapter summaries include a structured `coverage` record so merge queues can distinguish exact parser AST imports from declaration scans. The record declares exactness, parser token/trivia support, diagnostics support, source-range and generated-range support, and semantic coverage. Built-in wrappers normalize native AST/CST nodes and declaration-level semantic indexes; they do not claim resolved references, types, control flow, generated ranges, or token/trivia fidelity unless the host adapter supplies that evidence.
+
 ## Related Packages
 
 The published Frontier package family is generated from one shared package catalog so READMEs stay in sync across packages:

package/bench/smoke.mjs

@@ -3,6 +3,7 @@ import {
   compileFrontierSource,
   createEstreeNativeImporterAdapter,
   createNativeImportCoverageMatrix,
+  createNativeSourcePreservation,
   createSemanticImportSidecar,
   importNativeSource,
   projectNativeImportToSource,
@@ -67,6 +68,15 @@ const matrixStart = performance.now();
 const coverageMatrix = createNativeImportCoverageMatrix({ imports: nativeImportResults });
 const matrixDurationMs = performance.now() - matrixStart;
 
+const preservationStart = performance.now();
+const preservationRecords = nativeImportResults.map((imported) => imported.metadata.sourcePreservation ?? createNativeSourcePreservation({
+  language: imported.language,
+  sourcePath: imported.sourcePath,
+  sourceText: imported.metadata.sourcePreservation?.sourceText ?? ''
+}));
+const preservationDurationMs = performance.now() - preservationStart;
+const preservationTokens = preservationRecords.reduce((sum, record) => sum + record.tokens.length + record.trivia.length, 0);
+
 const sidecarStart = performance.now();
 const semanticSidecars = nativeImportResults.map((imported) => createSemanticImportSidecar(imported));
 const sidecarDurationMs = performance.now() - sidecarStart;
@@ -87,6 +97,9 @@ console.log(JSON.stringify({
   coverageMatrixLanguages: coverageMatrix.summary.languages,
   coverageMatrixImports: coverageMatrix.summary.imports,
   coverageMatrixDurationMs: Number(matrixDurationMs.toFixed(2)),
+  sourcePreservationRecords: preservationRecords.length,
+  sourcePreservationTokens: preservationTokens,
+  sourcePreservationDurationMs: Number(preservationDurationMs.toFixed(2)),
   semanticSidecars: semanticSidecars.length,
   sidecarOwnershipRegions,
   sidecarDurationMs: Number(sidecarDurationMs.toFixed(2)),

package/dist/index.d.ts

@@ -79,27 +79,43 @@ export type NativeImportTaxonomyKind =
   | 'opaqueBodies'
   | 'macroExpansion'
   | 'preprocessor'
+  | 'conditionalCompilation'
   | 'metaprogramming'
+  | 'reflection'
   | 'generatedCode'
+  | 'overloadTypeInference'
   | 'sourcePreservation'
+  | 'commentsTrivia'
   | 'parserDiagnostics'
   | 'unsupportedSyntax'
   | 'partialSemanticIndex'
   | 'sourceMapApproximation'
+  | 'targetProjectionLoss'
   | string;
 
 export type NativeImportKnownLossKind =
   | 'declarationOnlyCoverage'
   | 'opaqueNative'
   | 'macroExpansion'
+  | 'macroHygiene'
   | 'preprocessor'
+  | 'conditionalCompilation'
   | 'metaprogramming'
+  | 'reflection'
+  | 'dynamicRuntime'
+  | 'dynamicDispatch'
   | 'generatedCode'
+  | 'overloadResolution'
+  | 'typeInference'
   | 'sourcePreservation'
+  | 'commentsTrivia'
   | 'parserDiagnostic'
   | 'unsupportedSyntax'
+  | 'unsupportedSemantic'
+  | 'unverifiedNativeAst'
   | 'partialSemanticIndex'
   | 'sourceMapApproximation'
+  | 'targetProjectionLoss'
   | string;
 
 export interface NativeImportLossSummaryOptions {
@@ -201,6 +217,82 @@ export interface NativeImportCoverageMatrixOptions {
   readonly generatedAt?: number;
 }
 
+export type NativeSourceTokenKind =
+  | 'identifier'
+  | 'keyword'
+  | 'number'
+  | 'string'
+  | 'operator'
+  | 'punctuation'
+  | 'comment'
+  | 'whitespace'
+  | 'newline'
+  | 'directive'
+  | 'unknown'
+  | string;
+
+export interface NativeSourcePreservedToken {
+  readonly id: string;
+  readonly kind: NativeSourceTokenKind;
+  readonly text?: string;
+  readonly textHash: string;
+  readonly span: SourceSpan;
+  readonly metadata?: Record<string, unknown>;
+}
+
+export interface NativeSourcePreservedDirective {
+  readonly id: string;
+  readonly kind: string;
+  readonly text?: string;
+  readonly textHash: string;
+  readonly span: SourceSpan;
+  readonly metadata?: Record<string, unknown>;
+}
+
+export interface NativeSourcePreservation {
+  readonly kind: 'frontier.lang.nativeSourcePreservation';
+  readonly version: 1;
+  readonly id: string;
+  readonly language: FrontierSourceLanguage | string;
+  readonly sourcePath?: string;
+  readonly sourceHash: string;
+  readonly sourceBytes: number;
+  readonly lineCount: number;
+  readonly newline: 'lf' | 'crlf' | 'mixed' | 'none';
+  readonly encoding: string;
+  readonly sourceText?: string;
+  readonly tokens: readonly NativeSourcePreservedToken[];
+  readonly trivia: readonly NativeSourcePreservedToken[];
+  readonly directives: readonly NativeSourcePreservedDirective[];
+  readonly summary: {
+    readonly tokens: number;
+    readonly trivia: number;
+    readonly directives: number;
+    readonly comments: number;
+    readonly whitespace: number;
+    readonly exactSourceAvailable: boolean;
+    readonly truncated: boolean;
+  };
+  readonly metadata?: Record<string, unknown>;
+}
+
+export interface CreateNativeSourcePreservationOptions {
+  readonly id?: string;
+  readonly language?: FrontierSourceLanguage | string;
+  readonly sourcePath?: string;
+  readonly sourceHash?: string;
+  readonly sourceText: string;
+  readonly encoding?: string;
+  readonly includeSourceText?: boolean;
+  readonly includeTokens?: boolean;
+  readonly includeTrivia?: boolean;
+  readonly includeDirectives?: boolean;
+  readonly maxTokens?: number;
+  readonly maxTrivia?: number;
+  readonly maxDirectives?: number;
+  readonly metadata?: Record<string, unknown>;
+}
+
 export interface SemanticImportOwnershipRegion {
   readonly id: string;
   readonly key: string;
@@ -322,6 +414,52 @@ export interface SemanticImportSidecarOptions {
   readonly metadata?: Record<string, unknown>;
 }
 
+export type NativeImporterAdapterExactness =
+  | 'exact-parser-ast'
+  | 'parser-tree'
+  | 'adapter-reported-native-ast'
+  | 'loss-aware-native-ast'
+  | 'unknown'
+  | string;
+
+export interface NativeImporterAdapterSemanticCoverage {
+  readonly level: 'native-ast' | 'declaration-index' | 'semantic-index' | string;
+  readonly declarations: boolean;
+  readonly symbols: boolean;
+  readonly references: boolean;
+  readonly types: boolean;
+  readonly controlFlow: boolean;
+}
+
+export interface NativeImporterAdapterCoverageObserved {
+  readonly diagnostics: number;
+  readonly losses: number;
+  readonly nativeAstNodes: number;
+  readonly semanticSymbols: number;
+  readonly sourceMapMappings: number;
+  readonly sourceRanges: boolean;
+  readonly generatedRanges: boolean;
+}
+
+export interface NativeImporterAdapterCoverageSummary {
+  readonly exactness: NativeImporterAdapterExactness;
+  readonly exactAst: boolean;
+  readonly tokens: boolean;
+  readonly trivia: boolean;
+  readonly diagnostics: boolean;
+  readonly sourceRanges: boolean;
+  readonly generatedRanges: boolean;
+  readonly semanticCoverage: NativeImporterAdapterSemanticCoverage;
+  readonly notes: readonly string[];
+  readonly observed?: NativeImporterAdapterCoverageObserved;
+}
+
+export type NativeImporterAdapterCoverageInput =
+  Omit<Partial<NativeImporterAdapterCoverageSummary>, 'semanticCoverage' | 'observed'> & {
+    readonly semanticCoverage?: Partial<NativeImporterAdapterSemanticCoverage>;
+    readonly observed?: Partial<NativeImporterAdapterCoverageObserved>;
+  };
+
 export interface NativeImporterAdapterDiagnostic {
   readonly id?: string;
   readonly severity?: 'info' | 'warning' | 'error';
@@ -362,6 +500,7 @@ export interface ImportNativeSourceOptions {
   readonly losses?: readonly NativeAstLossRecord[];
   readonly evidence?: readonly EvidenceRecord[];
   readonly evidenceId?: string;
+  readonly sourcePreservation?: NativeSourcePreservation;
   readonly patch?: SemanticPatchBundle;
   readonly patchId?: string;
   readonly author?: string;
@@ -373,6 +512,7 @@ export interface ImportNativeSourceOptions {
   readonly sourceMaps?: readonly SourceMapRecord[];
   readonly universalAstId?: string;
   readonly universalAstMetadata?: Record<string, unknown>;
+  readonly exactAst?: boolean;
   readonly metadata?: Record<string, unknown>;
 }
 
@@ -405,6 +545,7 @@ export interface NativeImporterAdapter {
   readonly parser: string;
   readonly version?: string;
   readonly capabilities?: readonly string[];
+  readonly coverage?: NativeImporterAdapterCoverageInput;
   readonly supportedExtensions?: readonly string[];
   readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
   readonly parse: (input: NativeImporterAdapterParseInput) => NativeImporterAdapterParseResult | Promise<NativeImporterAdapterParseResult>;
@@ -416,6 +557,7 @@ export interface JavaScriptNativeImporterAdapterOptions {
   readonly parser?: string;
   readonly version?: string;
   readonly capabilities?: readonly string[];
+  readonly coverage?: NativeImporterAdapterCoverageInput;
   readonly supportedExtensions?: readonly string[];
   readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
   readonly ast?: unknown;
@@ -432,6 +574,7 @@ export interface TypeScriptCompilerNativeImporterAdapterOptions {
   readonly parser?: string;
   readonly version?: string;
   readonly capabilities?: readonly string[];
+  readonly coverage?: NativeImporterAdapterCoverageInput;
   readonly supportedExtensions?: readonly string[];
   readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
   readonly typescript?: unknown;
@@ -451,6 +594,7 @@ export interface TreeSitterNativeImporterAdapterOptions {
   readonly parserName?: string;
   readonly version?: string;
   readonly capabilities?: readonly string[];
+  readonly coverage?: NativeImporterAdapterCoverageInput;
   readonly supportedExtensions?: readonly string[];
   readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
   readonly parserInstance?: { readonly parse: (sourceText: string) => unknown };
@@ -466,6 +610,7 @@ export interface NativeImporterAdapterSummary {
   readonly parser: string;
   readonly version?: string;
   readonly capabilities: readonly string[];
+  readonly coverage: NativeImporterAdapterCoverageSummary;
   readonly supportedExtensions: readonly string[];
   readonly diagnostics: readonly NativeImporterAdapterDiagnostic[];
 }
@@ -581,7 +726,62 @@ export interface NativeSourceProjectionResult {
   readonly metadata: Record<string, unknown>;
 }
 
+export type NativeImportRoundtripReadinessStatus =
+  | 'exact'
+  | 'preserved-source'
+  | 'stub-only'
+  | 'blocked'
+  | 'needs-review';
+
+export interface NativeImportRoundtripReadinessOptions extends ProjectNativeImportToSourceOptions {
+  readonly projection?: NativeSourceProjectionResult;
+}
+
+export interface NativeImportRoundtripReadinessClassification {
+  readonly kind: 'frontier.lang.nativeImportRoundtripReadiness';
+  readonly version: 1;
+  readonly status: NativeImportRoundtripReadinessStatus;
+  readonly semanticMergeReadiness: SemanticMergeReadiness;
+  readonly reasons: readonly string[];
+  readonly importReadiness: NativeImportReadinessClassification;
+  readonly projectionReadiness: NativeImportReadinessClassification;
+  readonly projectionMode: NativeSourceProjectionMode;
+  readonly checks: {
+    readonly nativeImport: {
+      readonly imports: number;
+      readonly exactAst: boolean;
+      readonly losses: number;
+      readonly readiness: SemanticMergeReadiness;
+    };
+    readonly universalAst: {
+      readonly present: boolean;
+      readonly valid: boolean;
+      readonly issues: readonly string[];
+      readonly nativeSources: number;
+      readonly semanticSymbols: number;
+      readonly sourceMaps: number;
+      readonly sourceMapMappings: number;
+    };
+    readonly projectedSource: {
+      readonly mode: NativeSourceProjectionMode;
+      readonly outputHash: string;
+      readonly expectedSourceHash?: string;
+      readonly sourceHashVerified: boolean;
+      readonly declarations: number;
+      readonly losses: number;
+      readonly readiness: SemanticMergeReadiness;
+    };
+  };
+  readonly evidence: {
+    readonly importEvidenceIds: readonly string[];
+    readonly projectionEvidenceIds: readonly string[];
+    readonly failedEvidenceIds: readonly string[];
+  };
+  readonly metadata: Record<string, unknown>;
+}
+
 export declare const FrontierCompileTargets: readonly FrontierCompileTarget[];
+export declare const NativeImportRoundtripReadinessStatuses: readonly NativeImportRoundtripReadinessStatus[];
 export declare const NativeImportTaxonomyKinds: readonly NativeImportTaxonomyKind[];
 export declare const NativeImportLossKinds: readonly NativeImportKnownLossKind[];
 export declare const NativeImportReadinessBySeverity: Readonly<Record<NativeImportLossSummary['highestSeverity'], SemanticMergeReadiness>>;
@@ -594,7 +794,9 @@ export declare function renderTargetAst(ast: FrontierTargetAst, target?: Frontie
 export declare function resolveCapabilityAdapters(document: FrontierLangDocument, target?: FrontierCompileOptions['target'], options?: { readonly platform?: string }): readonly CapabilityResolution[];
 export declare function summarizeNativeImportLosses(losses?: readonly NativeAstLossRecord[], options?: NativeImportLossSummaryOptions): NativeImportLossSummary;
 export declare function classifyNativeImportReadiness(losses?: readonly NativeAstLossRecord[], options?: NativeImportLossSummaryOptions): NativeImportReadinessClassification;
+export declare function classifyNativeImportRoundtripReadiness(importResult: NativeSourceImportResult | NativeProjectImportResult, options?: NativeImportRoundtripReadinessOptions): NativeImportRoundtripReadinessClassification;
 export declare function createNativeImportCoverageMatrix(options?: NativeImportCoverageMatrixOptions): NativeImportCoverageMatrix;
+export declare function createNativeSourcePreservation(options: CreateNativeSourcePreservationOptions): NativeSourcePreservation;
 export declare function createSemanticImportSidecar(importResult: NativeSourceImportResult | NativeProjectImportResult, options?: SemanticImportSidecarOptions): SemanticImportSidecar;
 export declare function createEstreeNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
 export declare function createBabelNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;