@shapeshift-labs/frontier-lang-compiler 0.2.24 → 0.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -3
- package/bench/smoke.mjs +32 -0
- package/dist/index.d.ts +201 -1
- package/dist/index.js +799 -8
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -116,6 +116,7 @@ Ask the compiler what is actually covered before sending native imports into a m
|
|
|
116
116
|
```js
|
|
117
117
|
import {
|
|
118
118
|
createNativeImportCoverageMatrix,
|
|
119
|
+
createNativeParserAstFormatMatrix,
|
|
119
120
|
createProjectionTargetLossMatrix,
|
|
120
121
|
importNativeSource
|
|
121
122
|
} from '@shapeshift-labs/frontier-lang-compiler';
|
|
@@ -132,6 +133,9 @@ const python = matrix.languages.find((entry) => entry.language === 'python');
|
|
|
132
133
|
console.log(python.imports.readiness); // scanner imports are intentionally review-required
|
|
133
134
|
console.log(python.parserAdapters); // host-owned exact parsers such as LibCST can be injected
|
|
134
135
|
|
|
136
|
+
const parserMatrix = createNativeParserAstFormatMatrix({ imports: [imported] });
|
|
137
|
+
console.log(parserMatrix.formats.find((entry) => entry.id === 'python-ast')?.adapters.total ?? 0);
|
|
138
|
+
|
|
135
139
|
const projectionMatrix = createProjectionTargetLossMatrix({ imports: [imported] });
|
|
136
140
|
const pythonProjection = projectionMatrix.languages.find((entry) => entry.language === 'python');
|
|
137
141
|
|
|
@@ -229,10 +233,12 @@ const changeSet = diffNativeSources({
|
|
|
229
233
|
|
|
230
234
|
console.log(changeSet.changedSymbols[0]?.changeKind); // "modified"
|
|
231
235
|
console.log(changeSet.changedRegions[0]?.conflictKey); // semantic ownership key
|
|
236
|
+
console.log(changeSet.changedRegions[0]?.metadata.changedRegionProjection.reviewRequired); // true
|
|
237
|
+
console.log(changeSet.metadata.changedRegionProjectionSummary.autoMergeClaims); // 0
|
|
232
238
|
console.log(changeSet.mergeCandidate.readiness); // merge-admission classification
|
|
233
239
|
```
|
|
234
240
|
|
|
235
|
-
Use `diffNativeSourceImports` when the worker or runner already produced `importNativeSource` results. Body-only edits that the lightweight scanner cannot anchor to a symbol are still reported as file-level changed regions instead of being silently treated as safe.
|
|
241
|
+
Use `diffNativeSourceImports` when the worker or runner already produced `importNativeSource` results. Changed regions include a `metadata.changedRegionProjection` envelope with before/after source hashes, source-map links, ownership keys, readiness, and `autoMergeClaim: false` so swarm admission tools can score or port patches without treating semantic metadata as proof. Body-only edits that the lightweight scanner cannot anchor to a symbol are still reported as file-level changed regions instead of being silently treated as safe.
|
|
236
242
|
|
|
237
243
|
Compile native source imports through the same reader/IR/writer facade that swarms use for sidecar evidence. Same-language targets preserve exact source when hashes match; cross-language targets emit declaration stubs until a real adapter provides stronger evidence:
|
|
238
244
|
|
|
@@ -326,6 +332,7 @@ Use injected parser adapters when a real language parser is available but should
|
|
|
326
332
|
```js
|
|
327
333
|
import {
|
|
328
334
|
createBabelNativeImporterAdapter,
|
|
335
|
+
createPythonAstNativeImporterAdapter,
|
|
329
336
|
importNativeProject,
|
|
330
337
|
runNativeImporterAdapter
|
|
331
338
|
} from '@shapeshift-labs/frontier-lang-compiler';
|
|
@@ -333,6 +340,9 @@ import {
|
|
|
333
340
|
const babelAdapter = createBabelNativeImporterAdapter({
|
|
334
341
|
parserModule: await import('@babel/parser')
|
|
335
342
|
});
|
|
343
|
+
const pythonAstAdapter = createPythonAstNativeImporterAdapter({
|
|
344
|
+
parserModule: hostPythonAstParser
|
|
345
|
+
});
|
|
336
346
|
|
|
337
347
|
const imported = await runNativeImporterAdapter(babelAdapter, {
|
|
338
348
|
sourcePath: 'src/todo.ts',
|
|
@@ -341,10 +351,10 @@ const imported = await runNativeImporterAdapter(babelAdapter, {
|
|
|
341
351
|
|
|
342
352
|
const project = await importNativeProject({
|
|
343
353
|
projectRoot: 'src',
|
|
344
|
-
adapters: [babelAdapter],
|
|
354
|
+
adapters: [babelAdapter, pythonAstAdapter],
|
|
345
355
|
sources: [
|
|
346
356
|
{ language: 'typescript', adapter: babelAdapter.id, sourcePath: 'src/todo.ts', sourceText },
|
|
347
|
-
{ language: 'python', sourcePath: 'tools/todo.py', sourceText: pythonSource }
|
|
357
|
+
{ language: 'python', adapter: pythonAstAdapter.id, sourcePath: 'tools/todo.py', sourceText: pythonSource }
|
|
348
358
|
]
|
|
349
359
|
});
|
|
350
360
|
|
|
@@ -363,6 +373,7 @@ The built-in adapter factories are dependency-light wrappers for caller-owned pa
|
|
|
363
373
|
- `createEstreeNativeImporterAdapter`
|
|
364
374
|
- `createBabelNativeImporterAdapter`
|
|
365
375
|
- `createTypeScriptCompilerNativeImporterAdapter`
|
|
376
|
+
- `createPythonAstNativeImporterAdapter`
|
|
366
377
|
- `createTreeSitterNativeImporterAdapter`
|
|
367
378
|
|
|
368
379
|
Adapter summaries include a structured `coverage` record so merge queues can distinguish exact parser AST imports from declaration scans. The record declares exactness, parser token/trivia support, diagnostics support, source-range and generated-range support, and semantic coverage. Built-in wrappers normalize native AST/CST nodes and declaration-level semantic indexes; they do not claim resolved references, types, control flow, generated ranges, or token/trivia fidelity unless the host adapter supplies that evidence.
|
package/bench/smoke.mjs
CHANGED
|
@@ -4,9 +4,12 @@ import {
|
|
|
4
4
|
compileFrontierSource,
|
|
5
5
|
createEstreeNativeImporterAdapter,
|
|
6
6
|
createNativeImportCoverageMatrix,
|
|
7
|
+
createNativeParserAstFormatMatrix,
|
|
7
8
|
createProjectionTargetLossMatrix,
|
|
8
9
|
createNativeSourcePreservation,
|
|
10
|
+
createPythonAstNativeImporterAdapter,
|
|
9
11
|
createSemanticImportSidecar,
|
|
12
|
+
diffNativeSources,
|
|
10
13
|
importExternalSemanticIndex,
|
|
11
14
|
importNativeSource,
|
|
12
15
|
projectNativeImportToSource,
|
|
@@ -72,6 +75,13 @@ const matrixStart = performance.now();
|
|
|
72
75
|
const coverageMatrix = createNativeImportCoverageMatrix({ imports: nativeImportResults });
|
|
73
76
|
const matrixDurationMs = performance.now() - matrixStart;
|
|
74
77
|
|
|
78
|
+
const parserFormatMatrixStart = performance.now();
|
|
79
|
+
const parserFormatMatrix = createNativeParserAstFormatMatrix({
|
|
80
|
+
imports: nativeImportResults,
|
|
81
|
+
adapters: [estreeAdapter, createPythonAstNativeImporterAdapter()]
|
|
82
|
+
});
|
|
83
|
+
const parserFormatMatrixDurationMs = performance.now() - parserFormatMatrixStart;
|
|
84
|
+
|
|
75
85
|
const projectionMatrixStart = performance.now();
|
|
76
86
|
const projectionLossMatrix = createProjectionTargetLossMatrix({ imports: nativeImportResults });
|
|
77
87
|
const projectionMatrixDurationMs = performance.now() - projectionMatrixStart;
|
|
@@ -165,6 +175,20 @@ const regionScanDurationMs = performance.now() - regionScanStart;
|
|
|
165
175
|
const regionScanSymbols = regionScanImports.reduce((sum, entry) => sum + entry.imported.semanticIndex.symbols.length, 0);
|
|
166
176
|
const regionScanOwnershipRegions = regionScanImports.reduce((sum, entry) => sum + entry.sidecar.ownershipRegions.length, 0);
|
|
167
177
|
|
|
178
|
+
const changeProjectionStart = performance.now();
|
|
179
|
+
const changeProjectionSets = [];
|
|
180
|
+
for (let index = 0; index < 80; index += 1) {
|
|
181
|
+
changeProjectionSets.push(diffNativeSources({
|
|
182
|
+
language: 'javascript',
|
|
183
|
+
sourcePath: `src/change-projection-${index}.js`,
|
|
184
|
+
beforeSourceText: `export function changeProjection${index}() { return ${index}; }\n`,
|
|
185
|
+
afterSourceText: `export function changeProjection${index}() { return ${index + 1}; }\nexport const changeProjectionFlag${index} = true;\n`
|
|
186
|
+
}));
|
|
187
|
+
}
|
|
188
|
+
const changeProjectionDurationMs = performance.now() - changeProjectionStart;
|
|
189
|
+
const changedRegionProjections = changeProjectionSets.reduce((sum, changeSet) => sum + changeSet.metadata.changedRegionProjectionSummary.withProjection, 0);
|
|
190
|
+
const changedRegionProjectionSourceMapLinks = changeProjectionSets.reduce((sum, changeSet) => sum + changeSet.metadata.changedRegionProjectionSummary.sourceMapLinks, 0);
|
|
191
|
+
|
|
168
192
|
const externalSemanticStart = performance.now();
|
|
169
193
|
const externalSemanticImports = [];
|
|
170
194
|
for (let index = 0; index < 100; index += 1) {
|
|
@@ -213,6 +237,10 @@ console.log(JSON.stringify({
|
|
|
213
237
|
adapterCoverageTokenGaps: coverageMatrix.summary.adapterCoverage.gaps.tokens ?? 0,
|
|
214
238
|
adapterCoverageReferenceGaps: coverageMatrix.summary.adapterCoverage.gaps.references ?? 0,
|
|
215
239
|
coverageMatrixDurationMs: Number(matrixDurationMs.toFixed(2)),
|
|
240
|
+
parserFormatMatrixFormats: parserFormatMatrix.summary.formats,
|
|
241
|
+
parserFormatMatrixImports: parserFormatMatrix.summary.imports,
|
|
242
|
+
parserFormatMatrixNativeAstNodes: parserFormatMatrix.summary.nativeAstNodes,
|
|
243
|
+
parserFormatMatrixDurationMs: Number(parserFormatMatrixDurationMs.toFixed(2)),
|
|
216
244
|
projectionMatrixLanguages: projectionLossMatrix.summary.languages,
|
|
217
245
|
projectionMatrixMissingAdapters: projectionLossMatrix.summary.missingAdapters,
|
|
218
246
|
projectionMatrixUnsupportedTargetFeatures: projectionLossMatrix.summary.unsupportedTargetFeatures,
|
|
@@ -242,6 +270,10 @@ console.log(JSON.stringify({
|
|
|
242
270
|
regionScanSymbols,
|
|
243
271
|
regionScanOwnershipRegions,
|
|
244
272
|
regionScanDurationMs: Number(regionScanDurationMs.toFixed(2)),
|
|
273
|
+
changeProjectionSets: changeProjectionSets.length,
|
|
274
|
+
changedRegionProjections,
|
|
275
|
+
changedRegionProjectionSourceMapLinks,
|
|
276
|
+
changeProjectionDurationMs: Number(changeProjectionDurationMs.toFixed(2)),
|
|
245
277
|
externalSemanticImports: externalSemanticImports.length,
|
|
246
278
|
externalSemanticSymbols,
|
|
247
279
|
externalSemanticMappings,
|
package/dist/index.d.ts
CHANGED
|
@@ -236,6 +236,88 @@ export interface NativeImportLanguageProfile {
|
|
|
236
236
|
readonly notes: readonly string[];
|
|
237
237
|
}
|
|
238
238
|
|
|
239
|
+
export type NativeParserAstFormatKind =
|
|
240
|
+
| 'abstract-ast'
|
|
241
|
+
| 'concrete-syntax-tree'
|
|
242
|
+
| 'compiler-ast'
|
|
243
|
+
| 'semantic-index'
|
|
244
|
+
| string;
|
|
245
|
+
|
|
246
|
+
export interface NativeParserAstFormatProfile {
|
|
247
|
+
readonly id: string;
|
|
248
|
+
readonly aliases: readonly string[];
|
|
249
|
+
readonly kind: NativeParserAstFormatKind;
|
|
250
|
+
readonly languages: readonly (FrontierSourceLanguage | 'mixed' | string)[];
|
|
251
|
+
readonly parserAdapters: readonly string[];
|
|
252
|
+
readonly exactness: NativeImporterAdapterExactness;
|
|
253
|
+
readonly sourceRangeModel: string;
|
|
254
|
+
readonly preservesTokens: boolean;
|
|
255
|
+
readonly preservesTrivia: boolean;
|
|
256
|
+
readonly supportsIncremental: boolean;
|
|
257
|
+
readonly supportsErrorRecovery: boolean;
|
|
258
|
+
readonly notes: readonly string[];
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export interface NativeParserAstFormatCoverage {
|
|
262
|
+
readonly id: string;
|
|
263
|
+
readonly kind: NativeParserAstFormatKind;
|
|
264
|
+
readonly languages: readonly (FrontierSourceLanguage | 'mixed' | string)[];
|
|
265
|
+
readonly parserAdapters: readonly string[];
|
|
266
|
+
readonly exactness: NativeImporterAdapterExactness;
|
|
267
|
+
readonly sourceRangeModel: string;
|
|
268
|
+
readonly preservesTokens: boolean;
|
|
269
|
+
readonly preservesTrivia: boolean;
|
|
270
|
+
readonly supportsIncremental: boolean;
|
|
271
|
+
readonly supportsErrorRecovery: boolean;
|
|
272
|
+
readonly notes: readonly string[];
|
|
273
|
+
readonly adapters: {
|
|
274
|
+
readonly total: number;
|
|
275
|
+
readonly ids: readonly string[];
|
|
276
|
+
readonly parsers: readonly string[];
|
|
277
|
+
readonly effectiveCapabilities: Readonly<Record<string, number>>;
|
|
278
|
+
};
|
|
279
|
+
readonly imports: {
|
|
280
|
+
readonly total: number;
|
|
281
|
+
readonly sourcePaths: readonly string[];
|
|
282
|
+
readonly readiness: SemanticMergeReadiness;
|
|
283
|
+
readonly nativeAstNodes: number;
|
|
284
|
+
readonly symbols: number;
|
|
285
|
+
readonly sourceMapMappings: number;
|
|
286
|
+
readonly losses: number;
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export interface NativeParserAstFormatMatrix {
|
|
291
|
+
readonly kind: 'frontier.lang.nativeParserAstFormatMatrix';
|
|
292
|
+
readonly version: 1;
|
|
293
|
+
readonly generatedAt: number;
|
|
294
|
+
readonly formats: readonly NativeParserAstFormatCoverage[];
|
|
295
|
+
readonly summary: {
|
|
296
|
+
readonly formats: number;
|
|
297
|
+
readonly adapterSlots: number;
|
|
298
|
+
readonly adapters: number;
|
|
299
|
+
readonly imports: number;
|
|
300
|
+
readonly nativeAstNodes: number;
|
|
301
|
+
readonly symbols: number;
|
|
302
|
+
readonly sourceMapMappings: number;
|
|
303
|
+
readonly losses: number;
|
|
304
|
+
readonly byKind: Readonly<Record<string, number>>;
|
|
305
|
+
readonly byReadiness: Readonly<Record<string, number>>;
|
|
306
|
+
readonly effectiveCapabilities: Readonly<Record<string, number>>;
|
|
307
|
+
};
|
|
308
|
+
readonly metadata: {
|
|
309
|
+
readonly note: string;
|
|
310
|
+
readonly profileIds: readonly string[];
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
export interface NativeParserAstFormatMatrixOptions {
|
|
315
|
+
readonly formats?: readonly NativeParserAstFormatProfile[];
|
|
316
|
+
readonly imports?: readonly NativeSourceImportResult[];
|
|
317
|
+
readonly adapters?: readonly NativeImporterAdapter[];
|
|
318
|
+
readonly generatedAt?: number;
|
|
319
|
+
}
|
|
320
|
+
|
|
239
321
|
export interface NativeImporterAdapterCoverageAggregate {
|
|
240
322
|
readonly total: number;
|
|
241
323
|
readonly declared: Readonly<Record<string, number>>;
|
|
@@ -786,6 +868,91 @@ export interface SemanticImportSidecarOptions {
|
|
|
786
868
|
|
|
787
869
|
export type NativeSourceChangeKind = 'added' | 'removed' | 'modified' | 'unchanged';
|
|
788
870
|
|
|
871
|
+
export interface NativeSourceChangeProjectionEndpoint {
|
|
872
|
+
readonly side: 'before' | 'after';
|
|
873
|
+
readonly importId?: string;
|
|
874
|
+
readonly sidecarId?: string;
|
|
875
|
+
readonly nativeSourceId?: string;
|
|
876
|
+
readonly nativeAstId?: string;
|
|
877
|
+
readonly semanticIndexId?: string;
|
|
878
|
+
readonly universalAstId?: string;
|
|
879
|
+
readonly sourcePath?: string;
|
|
880
|
+
readonly sourceHash?: string;
|
|
881
|
+
readonly sourcePreservationId?: string;
|
|
882
|
+
readonly exactSourceAvailable: boolean;
|
|
883
|
+
readonly ownershipRegionId?: string;
|
|
884
|
+
readonly ownershipKey?: string;
|
|
885
|
+
readonly ownershipRegionKind?: NativeImportRegionTaxonomyKind;
|
|
886
|
+
readonly sourceSpan?: SourceSpan;
|
|
887
|
+
readonly sourceMapIds: readonly string[];
|
|
888
|
+
readonly sourceMapMappingIds: readonly string[];
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
export interface NativeSourceChangeProjectionSourceMapLink {
|
|
892
|
+
readonly id: string;
|
|
893
|
+
readonly side: 'before' | 'after';
|
|
894
|
+
readonly sourceMapId?: string;
|
|
895
|
+
readonly sourceMapMappingId?: string;
|
|
896
|
+
readonly sourcePath?: string;
|
|
897
|
+
readonly sourceHash?: string;
|
|
898
|
+
readonly targetPath?: string;
|
|
899
|
+
readonly targetHash?: string;
|
|
900
|
+
readonly semanticSymbolId?: string;
|
|
901
|
+
readonly semanticOccurrenceId?: string;
|
|
902
|
+
readonly semanticNodeId?: string;
|
|
903
|
+
readonly nativeSourceId?: string;
|
|
904
|
+
readonly nativeAstNodeId?: string;
|
|
905
|
+
readonly precision?: string;
|
|
906
|
+
readonly sourceSpan?: SourceSpan;
|
|
907
|
+
readonly generatedSpan?: SourceMapMappingRecord['generatedSpan'];
|
|
908
|
+
readonly ownershipRegionId?: string;
|
|
909
|
+
readonly ownershipRegionKey?: string;
|
|
910
|
+
readonly ownershipRegionKind?: NativeImportRegionTaxonomyKind;
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
export interface NativeSourceChangeProjectionMetadata {
|
|
914
|
+
readonly schema: 'frontier.lang.changedRegionProjection.v1';
|
|
915
|
+
readonly id: string;
|
|
916
|
+
readonly reviewRequired: true;
|
|
917
|
+
readonly autoMergeClaim: false;
|
|
918
|
+
readonly changeKind: NativeSourceChangeKind;
|
|
919
|
+
readonly language?: FrontierSourceLanguage | string;
|
|
920
|
+
readonly sourcePath?: string;
|
|
921
|
+
readonly conflictKey: string;
|
|
922
|
+
readonly region: {
|
|
923
|
+
readonly id?: string;
|
|
924
|
+
readonly key?: string;
|
|
925
|
+
readonly kind?: NativeImportRegionTaxonomyKind;
|
|
926
|
+
readonly granularity?: string;
|
|
927
|
+
readonly precision?: string;
|
|
928
|
+
readonly sourceSpan?: SourceSpan;
|
|
929
|
+
readonly nativeAstNodeId?: string;
|
|
930
|
+
readonly symbolId?: string;
|
|
931
|
+
readonly symbolName?: string;
|
|
932
|
+
readonly symbolKind?: string;
|
|
933
|
+
};
|
|
934
|
+
readonly before?: NativeSourceChangeProjectionEndpoint;
|
|
935
|
+
readonly after?: NativeSourceChangeProjectionEndpoint;
|
|
936
|
+
readonly sourceMapLinks: readonly NativeSourceChangeProjectionSourceMapLink[];
|
|
937
|
+
readonly admission: {
|
|
938
|
+
readonly readiness: SemanticMergeReadiness;
|
|
939
|
+
readonly action: 'review-addition' | 'review-removal' | 'review-file' | 'review-port' | 'rerun-or-human-port' | string;
|
|
940
|
+
readonly reasons: readonly string[];
|
|
941
|
+
readonly conflictKeys: readonly string[];
|
|
942
|
+
};
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
export interface NativeSourceChangeProjectionSummary {
|
|
946
|
+
readonly schema: 'frontier.lang.changedRegionProjectionSummary.v1';
|
|
947
|
+
readonly total: number;
|
|
948
|
+
readonly withProjection: number;
|
|
949
|
+
readonly reviewRequired: number;
|
|
950
|
+
readonly autoMergeClaims: number;
|
|
951
|
+
readonly sourceMapLinks: number;
|
|
952
|
+
readonly byAction: Readonly<Record<string, number>>;
|
|
953
|
+
readonly byRegionKind: Readonly<Record<string, number>>;
|
|
954
|
+
}
|
|
955
|
+
|
|
789
956
|
export interface NativeSourceChangeSymbol {
|
|
790
957
|
readonly changeKind: NativeSourceChangeKind;
|
|
791
958
|
readonly key: string;
|
|
@@ -813,6 +980,9 @@ export interface NativeSourceChangeSymbol {
|
|
|
813
980
|
export interface NativeSourceChangeRegion extends SemanticImportOwnershipRegion {
|
|
814
981
|
readonly changeKind: NativeSourceChangeKind;
|
|
815
982
|
readonly conflictKey: string;
|
|
983
|
+
readonly metadata?: SemanticImportOwnershipRegion['metadata'] & {
|
|
984
|
+
readonly changedRegionProjection?: NativeSourceChangeProjectionMetadata;
|
|
985
|
+
};
|
|
816
986
|
}
|
|
817
987
|
|
|
818
988
|
export interface NativeSourceChangeSummary {
|
|
@@ -875,7 +1045,9 @@ export interface NativeSourceChangeSet {
|
|
|
875
1045
|
readonly semanticIndex?: SemanticIndexRecord;
|
|
876
1046
|
readonly losses: readonly NativeAstLossRecord[];
|
|
877
1047
|
readonly summary: NativeSourceChangeSummary;
|
|
878
|
-
readonly metadata?: Record<string, unknown
|
|
1048
|
+
readonly metadata?: Record<string, unknown> & {
|
|
1049
|
+
readonly changedRegionProjectionSummary?: NativeSourceChangeProjectionSummary;
|
|
1050
|
+
};
|
|
879
1051
|
}
|
|
880
1052
|
|
|
881
1053
|
export type NativeImporterAdapterExactness =
|
|
@@ -1193,6 +1365,29 @@ export interface TypeScriptCompilerNativeImporterAdapterOptions {
|
|
|
1193
1365
|
readonly includeTokens?: boolean;
|
|
1194
1366
|
}
|
|
1195
1367
|
|
|
1368
|
+
export interface PythonAstNativeImporterAdapterOptions {
|
|
1369
|
+
readonly id?: string;
|
|
1370
|
+
readonly language?: FrontierSourceLanguage;
|
|
1371
|
+
readonly parser?: string;
|
|
1372
|
+
readonly version?: string;
|
|
1373
|
+
readonly capabilities?: readonly string[];
|
|
1374
|
+
readonly coverage?: NativeImporterAdapterCoverageInput;
|
|
1375
|
+
readonly supportedExtensions?: readonly string[];
|
|
1376
|
+
readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
|
|
1377
|
+
readonly ast?: unknown;
|
|
1378
|
+
readonly parse?: (sourceText: string, options: Record<string, unknown>) => unknown;
|
|
1379
|
+
readonly parserModule?: { readonly parse: (sourceText: string, options: Record<string, unknown>) => unknown };
|
|
1380
|
+
readonly pythonAst?: { readonly parse: (sourceText: string, options: Record<string, unknown>) => unknown };
|
|
1381
|
+
readonly parserOptions?: Record<string, unknown>;
|
|
1382
|
+
readonly mode?: 'exec' | 'eval' | 'single' | 'func_type' | string;
|
|
1383
|
+
readonly pythonVersion?: string;
|
|
1384
|
+
readonly featureVersion?: string | number;
|
|
1385
|
+
readonly typeComments?: boolean;
|
|
1386
|
+
readonly optimize?: number;
|
|
1387
|
+
readonly includeAttributes?: boolean;
|
|
1388
|
+
readonly maxNodes?: number;
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1196
1391
|
export interface TreeSitterNativeImporterAdapterOptions {
|
|
1197
1392
|
readonly id?: string;
|
|
1198
1393
|
readonly language?: FrontierSourceLanguage;
|
|
@@ -1535,6 +1730,8 @@ export declare const ProjectionTargetLossClasses: readonly ProjectionTargetLossC
|
|
|
1535
1730
|
export declare const NativeImportReadinessBySeverity: Readonly<Record<NativeImportLossSummary['highestSeverity'], SemanticMergeReadiness>>;
|
|
1536
1731
|
export declare const NativeImportFeatureEvidencePolicies: Readonly<Record<string, NativeImportFeatureEvidencePolicy>>;
|
|
1537
1732
|
export declare const NativeImportLanguageProfiles: readonly NativeImportLanguageProfile[];
|
|
1733
|
+
export declare const NativeParserAstFormatProfiles: readonly NativeParserAstFormatProfile[];
|
|
1734
|
+
export declare const NativeParserAstFormats: readonly string[];
|
|
1538
1735
|
export declare const ExternalSemanticIndexFormats: readonly ExternalSemanticIndexFormat[];
|
|
1539
1736
|
export declare function normalizeCompileTarget(target?: string): FrontierCompileTarget;
|
|
1540
1737
|
export declare function compileFrontierSource(source: string, options?: FrontierCompileOptions): FrontierCompileResult;
|
|
@@ -1551,6 +1748,8 @@ export declare function summarizeNativeImportLosses(losses?: readonly NativeAstL
|
|
|
1551
1748
|
export declare function classifyNativeImportReadiness(losses?: readonly NativeAstLossRecord[], options?: NativeImportLossSummaryOptions): NativeImportReadinessClassification;
|
|
1552
1749
|
export declare function classifyNativeImportRoundtripReadiness(importResult: NativeSourceImportResult | NativeProjectImportResult, options?: NativeImportRoundtripReadinessOptions): NativeImportRoundtripReadinessClassification;
|
|
1553
1750
|
export declare function createNativeImportCoverageMatrix(options?: NativeImportCoverageMatrixOptions): NativeImportCoverageMatrix;
|
|
1751
|
+
export declare function getNativeParserAstFormatProfile(format?: string): NativeParserAstFormatProfile | undefined;
|
|
1752
|
+
export declare function createNativeParserAstFormatMatrix(options?: NativeParserAstFormatMatrixOptions): NativeParserAstFormatMatrix;
|
|
1554
1753
|
export declare function createProjectionTargetLossMatrix(options?: ProjectionTargetLossMatrixOptions): ProjectionTargetLossMatrix;
|
|
1555
1754
|
export declare function createNativeSourcePreservation(options: CreateNativeSourcePreservationOptions): NativeSourcePreservation;
|
|
1556
1755
|
export declare function createSemanticImportSidecar(importResult: NativeSourceImportResult | NativeProjectImportResult, options?: SemanticImportSidecarOptions): SemanticImportSidecar;
|
|
@@ -1558,6 +1757,7 @@ export declare function createNativeImportResultContract(importResult: NativeSou
|
|
|
1558
1757
|
export declare function createEstreeNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1559
1758
|
export declare function createBabelNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1560
1759
|
export declare function createTypeScriptCompilerNativeImporterAdapter(options?: TypeScriptCompilerNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1760
|
+
export declare function createPythonAstNativeImporterAdapter(options?: PythonAstNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1561
1761
|
export declare function createTreeSitterNativeImporterAdapter(options?: TreeSitterNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1562
1762
|
export declare function runNativeImporterAdapter(adapter: NativeImporterAdapter, input: RunNativeImporterAdapterOptions): Promise<NativeImporterAdapterImportResult>;
|
|
1563
1763
|
export declare function runNativeTargetProjectionAdapter(adapter: NativeTargetProjectionAdapter, input: NativeTargetProjectionAdapterInput): NativeTargetProjectionResult;
|
package/dist/index.js
CHANGED
|
@@ -333,6 +333,100 @@ export const NativeImportLanguageProfiles = Object.freeze([
|
|
|
333
333
|
nativeImportLanguageProfile('r', { aliases: ['R'], extensions: ['.r', '.R'], parserAdapters: ['r-parser', 'tree-sitter'], lossKinds: ['declarationOnlyCoverage', 'opaqueNative', 'dynamicRuntime', 'sourceMapApproximation', 'sourcePreservation'] })
|
|
334
334
|
]);
|
|
335
335
|
|
|
336
|
+
export const NativeParserAstFormatProfiles = Object.freeze([
|
|
337
|
+
nativeParserAstFormatProfile('estree', {
|
|
338
|
+
kind: 'abstract-ast',
|
|
339
|
+
languages: ['javascript'],
|
|
340
|
+
parserAdapters: ['estree'],
|
|
341
|
+
exactness: 'exact-parser-ast',
|
|
342
|
+
sourceRangeModel: 'loc-range',
|
|
343
|
+
preservesTokens: false,
|
|
344
|
+
preservesTrivia: false,
|
|
345
|
+
supportsErrorRecovery: false,
|
|
346
|
+
notes: ['Community JavaScript AST shape used by many JS tooling parsers.']
|
|
347
|
+
}),
|
|
348
|
+
nativeParserAstFormatProfile('babel', {
|
|
349
|
+
kind: 'abstract-ast',
|
|
350
|
+
languages: ['javascript', 'typescript'],
|
|
351
|
+
parserAdapters: ['babel'],
|
|
352
|
+
exactness: 'exact-parser-ast',
|
|
353
|
+
sourceRangeModel: 'loc-range',
|
|
354
|
+
preservesTokens: false,
|
|
355
|
+
preservesTrivia: false,
|
|
356
|
+
supportsErrorRecovery: true,
|
|
357
|
+
notes: ['Babel-compatible ESTree-like ASTs can report parser errors when error recovery is enabled.']
|
|
358
|
+
}),
|
|
359
|
+
nativeParserAstFormatProfile('typescript-compiler-api', {
|
|
360
|
+
kind: 'compiler-ast',
|
|
361
|
+
languages: ['typescript', 'javascript'],
|
|
362
|
+
parserAdapters: ['typescript-compiler-api'],
|
|
363
|
+
exactness: 'exact-parser-ast',
|
|
364
|
+
sourceRangeModel: 'pos-end',
|
|
365
|
+
preservesTokens: false,
|
|
366
|
+
preservesTrivia: false,
|
|
367
|
+
supportsErrorRecovery: true,
|
|
368
|
+
notes: ['TypeScript SourceFile trees can be parsed without a full Program; richer type/checker evidence remains host-owned.']
|
|
369
|
+
}),
|
|
370
|
+
nativeParserAstFormatProfile('python-ast', {
|
|
371
|
+
kind: 'abstract-ast',
|
|
372
|
+
languages: ['python'],
|
|
373
|
+
parserAdapters: ['python-ast'],
|
|
374
|
+
exactness: 'exact-parser-ast',
|
|
375
|
+
sourceRangeModel: 'lineno-col-offset',
|
|
376
|
+
preservesTokens: false,
|
|
377
|
+
preservesTrivia: false,
|
|
378
|
+
supportsErrorRecovery: false,
|
|
379
|
+
notes: ['Python stdlib AST exposes versioned abstract grammar and source locations, but not formatting trivia.']
|
|
380
|
+
}),
|
|
381
|
+
nativeParserAstFormatProfile('tree-sitter', {
|
|
382
|
+
kind: 'concrete-syntax-tree',
|
|
383
|
+
languages: ['mixed'],
|
|
384
|
+
parserAdapters: ['tree-sitter'],
|
|
385
|
+
exactness: 'parser-tree',
|
|
386
|
+
sourceRangeModel: 'row-column',
|
|
387
|
+
preservesTokens: false,
|
|
388
|
+
preservesTrivia: false,
|
|
389
|
+
supportsIncremental: true,
|
|
390
|
+
supportsErrorRecovery: true,
|
|
391
|
+
notes: ['Tree-sitter provides cross-language concrete syntax trees; language-specific queries still decide semantic richness.']
|
|
392
|
+
}),
|
|
393
|
+
nativeParserAstFormatProfile('libcst', {
|
|
394
|
+
kind: 'concrete-syntax-tree',
|
|
395
|
+
languages: ['python'],
|
|
396
|
+
parserAdapters: ['libcst'],
|
|
397
|
+
exactness: 'parser-tree',
|
|
398
|
+
sourceRangeModel: 'metadata-position-provider',
|
|
399
|
+
preservesTokens: true,
|
|
400
|
+
preservesTrivia: true,
|
|
401
|
+
supportsErrorRecovery: false,
|
|
402
|
+
notes: ['LibCST-style trees preserve formatting and are best treated as host-owned evidence until normalized explicitly.']
|
|
403
|
+
}),
|
|
404
|
+
nativeParserAstFormatProfile('scip', {
|
|
405
|
+
kind: 'semantic-index',
|
|
406
|
+
languages: ['mixed'],
|
|
407
|
+
parserAdapters: ['scip'],
|
|
408
|
+
exactness: 'loss-aware-native-ast',
|
|
409
|
+
sourceRangeModel: 'range-tuples',
|
|
410
|
+
preservesTokens: false,
|
|
411
|
+
preservesTrivia: false,
|
|
412
|
+
supportsErrorRecovery: false,
|
|
413
|
+
notes: ['SCIP is semantic index evidence rather than a full parser AST; it is useful for symbols/references and source maps.']
|
|
414
|
+
}),
|
|
415
|
+
nativeParserAstFormatProfile('lsif', {
|
|
416
|
+
kind: 'semantic-index',
|
|
417
|
+
languages: ['mixed'],
|
|
418
|
+
parserAdapters: ['lsif'],
|
|
419
|
+
exactness: 'loss-aware-native-ast',
|
|
420
|
+
sourceRangeModel: 'lsp-ranges',
|
|
421
|
+
preservesTokens: false,
|
|
422
|
+
preservesTrivia: false,
|
|
423
|
+
supportsErrorRecovery: false,
|
|
424
|
+
notes: ['LSIF graph dumps are semantic/source-map evidence, not complete native ASTs.']
|
|
425
|
+
})
|
|
426
|
+
]);
|
|
427
|
+
|
|
428
|
+
export const NativeParserAstFormats = Object.freeze(NativeParserAstFormatProfiles.map((profile) => profile.id));
|
|
429
|
+
|
|
336
430
|
export const ExternalSemanticIndexFormats = Object.freeze([
|
|
337
431
|
'frontier-semantic-index',
|
|
338
432
|
'scip',
|
|
@@ -2125,6 +2219,57 @@ export function createNativeImportCoverageMatrix(input = {}) {
|
|
|
2125
2219
|
};
|
|
2126
2220
|
}
|
|
2127
2221
|
|
|
2222
|
+
export function getNativeParserAstFormatProfile(format) {
|
|
2223
|
+
const normalized = normalizeParserAstFormatId(format);
|
|
2224
|
+
return NativeParserAstFormatProfiles.find((profile) => profile.id === normalized || profile.aliases.includes(normalized));
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
export function createNativeParserAstFormatMatrix(input = {}) {
|
|
2228
|
+
const imports = input.imports ?? [];
|
|
2229
|
+
const adapters = input.adapters ?? [];
|
|
2230
|
+
const profiles = mergeNativeParserAstFormatProfiles(input.formats ?? NativeParserAstFormatProfiles, imports, adapters);
|
|
2231
|
+
const formats = profiles.map((profile) => nativeParserAstFormatCoverageForProfile(profile, imports, adapters));
|
|
2232
|
+
const summary = formats.reduce((totals, entry) => {
|
|
2233
|
+
totals.formats += 1;
|
|
2234
|
+
totals.adapterSlots += entry.parserAdapters.length;
|
|
2235
|
+
totals.adapters += entry.adapters.total;
|
|
2236
|
+
totals.imports += entry.imports.total;
|
|
2237
|
+
totals.nativeAstNodes += entry.imports.nativeAstNodes;
|
|
2238
|
+
totals.symbols += entry.imports.symbols;
|
|
2239
|
+
totals.sourceMapMappings += entry.imports.sourceMapMappings;
|
|
2240
|
+
totals.losses += entry.imports.losses;
|
|
2241
|
+
totals.byKind[entry.kind] = (totals.byKind[entry.kind] ?? 0) + 1;
|
|
2242
|
+
totals.byReadiness[entry.imports.readiness] = (totals.byReadiness[entry.imports.readiness] ?? 0) + 1;
|
|
2243
|
+
for (const [capability, count] of Object.entries(entry.adapters.effectiveCapabilities)) {
|
|
2244
|
+
totals.effectiveCapabilities[capability] = (totals.effectiveCapabilities[capability] ?? 0) + count;
|
|
2245
|
+
}
|
|
2246
|
+
return totals;
|
|
2247
|
+
}, {
|
|
2248
|
+
formats: 0,
|
|
2249
|
+
adapterSlots: 0,
|
|
2250
|
+
adapters: 0,
|
|
2251
|
+
imports: 0,
|
|
2252
|
+
nativeAstNodes: 0,
|
|
2253
|
+
symbols: 0,
|
|
2254
|
+
sourceMapMappings: 0,
|
|
2255
|
+
losses: 0,
|
|
2256
|
+
byKind: {},
|
|
2257
|
+
byReadiness: {},
|
|
2258
|
+
effectiveCapabilities: {}
|
|
2259
|
+
});
|
|
2260
|
+
return {
|
|
2261
|
+
kind: 'frontier.lang.nativeParserAstFormatMatrix',
|
|
2262
|
+
version: 1,
|
|
2263
|
+
generatedAt: input.generatedAt ?? Date.now(),
|
|
2264
|
+
formats,
|
|
2265
|
+
summary,
|
|
2266
|
+
metadata: {
|
|
2267
|
+
note: 'Parser AST format coverage describes normalization evidence and host-parser obligations; it is not a lossless portability claim.',
|
|
2268
|
+
profileIds: profiles.map((profile) => profile.id)
|
|
2269
|
+
}
|
|
2270
|
+
};
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2128
2273
|
export function createProjectionTargetLossMatrix(input = {}) {
|
|
2129
2274
|
const imports = input.imports ?? [];
|
|
2130
2275
|
const adapters = input.adapters ?? [];
|
|
@@ -2435,6 +2580,57 @@ export function createTypeScriptCompilerNativeImporterAdapter(options = {}) {
|
|
|
2435
2580
|
};
|
|
2436
2581
|
}
|
|
2437
2582
|
|
|
2583
|
+
export function createPythonAstNativeImporterAdapter(options = {}) {
|
|
2584
|
+
return {
|
|
2585
|
+
id: options.id ?? 'frontier.python-ast-native-importer',
|
|
2586
|
+
language: options.language ?? 'python',
|
|
2587
|
+
parser: options.parser ?? 'python-ast',
|
|
2588
|
+
version: options.version,
|
|
2589
|
+
capabilities: uniqueStrings(['nativeAst', 'semanticIndex', 'sourceMaps', 'diagnostics', ...(options.capabilities ?? [])]),
|
|
2590
|
+
coverage: nativeImporterAdapterCoverage({
|
|
2591
|
+
exactness: 'exact-parser-ast',
|
|
2592
|
+
exactAst: true,
|
|
2593
|
+
tokens: false,
|
|
2594
|
+
trivia: false,
|
|
2595
|
+
diagnostics: true,
|
|
2596
|
+
sourceRanges: true,
|
|
2597
|
+
generatedRanges: false,
|
|
2598
|
+
semanticCoverage: declarationSemanticCoverage(),
|
|
2599
|
+
notes: [
|
|
2600
|
+
'Normalizes caller-owned Python stdlib ast trees into native AST nodes and declaration-level semantic index records.',
|
|
2601
|
+
'Python ast does not preserve comments, whitespace, or concrete formatting; use LibCST/parso-style host evidence for round-trip trivia.'
|
|
2602
|
+
]
|
|
2603
|
+
}, options.coverage),
|
|
2604
|
+
supportedExtensions: options.supportedExtensions ?? ['.py', '.pyi'],
|
|
2605
|
+
diagnostics: options.diagnostics,
|
|
2606
|
+
parse(input) {
|
|
2607
|
+
const parsed = input.options?.ast
|
|
2608
|
+
?? input.options?.nativeAst
|
|
2609
|
+
?? options.ast
|
|
2610
|
+
?? parsePythonAstSource(input, options);
|
|
2611
|
+
const root = pythonAstRoot(parsed);
|
|
2612
|
+
if (!root) {
|
|
2613
|
+
return missingInjectedParserResult(input, {
|
|
2614
|
+
parser: options.parser ?? 'python-ast',
|
|
2615
|
+
adapterId: options.id ?? 'frontier.python-ast-native-importer',
|
|
2616
|
+
message: 'createPythonAstNativeImporterAdapter requires an injected Python AST object, parserModule.parse function, parse function, or adapterOptions.ast.'
|
|
2617
|
+
});
|
|
2618
|
+
}
|
|
2619
|
+
const parseDiagnostics = normalizeParserErrors(parsed?.errors ?? parsed?.diagnostics, input, {
|
|
2620
|
+
parser: options.parser ?? 'python-ast'
|
|
2621
|
+
});
|
|
2622
|
+
return createNativeImportFromPythonAst(root, input, {
|
|
2623
|
+
parser: options.parser ?? 'python-ast',
|
|
2624
|
+
astFormat: 'python-ast',
|
|
2625
|
+
maxNodes: options.maxNodes,
|
|
2626
|
+
diagnostics: parseDiagnostics,
|
|
2627
|
+
pythonVersion: options.pythonVersion ?? input.options?.pythonVersion ?? parsed?.pythonVersion,
|
|
2628
|
+
includeAttributes: options.includeAttributes ?? input.options?.includeAttributes
|
|
2629
|
+
});
|
|
2630
|
+
}
|
|
2631
|
+
};
|
|
2632
|
+
}
|
|
2633
|
+
|
|
2438
2634
|
export function createTreeSitterNativeImporterAdapter(options = {}) {
|
|
2439
2635
|
return {
|
|
2440
2636
|
id: options.id ?? `frontier.tree-sitter-${idFragment(options.language ?? 'source')}-native-importer`,
|
|
@@ -3158,6 +3354,25 @@ export function diffNativeSourceImports(input) {
|
|
|
3158
3354
|
if (sourceChanged && changedSymbols.length === 0 && changedRegions.length === 0) {
|
|
3159
3355
|
changedRegions = [fileLevelNativeChangeRegion({ language, sourcePath, beforeHash, afterHash, before, after })];
|
|
3160
3356
|
}
|
|
3357
|
+
const readiness = maxSemanticMergeReadiness(
|
|
3358
|
+
maxSemanticMergeReadiness(nativeImportReadiness(before), nativeImportReadiness(after)),
|
|
3359
|
+
sourceChanged && changedSymbols.length === 0 ? 'needs-review' : 'ready'
|
|
3360
|
+
);
|
|
3361
|
+
const reasons = nativeSourceChangeReasons({ before, after, beforeHash, afterHash, changedSymbols, changedRegions, readiness });
|
|
3362
|
+
changedRegions = attachNativeChangeRegionProjectionMetadata(changedRegions, {
|
|
3363
|
+
before,
|
|
3364
|
+
after,
|
|
3365
|
+
beforeSidecar,
|
|
3366
|
+
afterSidecar,
|
|
3367
|
+
changedSymbols,
|
|
3368
|
+
language,
|
|
3369
|
+
sourcePath,
|
|
3370
|
+
beforeHash,
|
|
3371
|
+
afterHash,
|
|
3372
|
+
readiness,
|
|
3373
|
+
reasons
|
|
3374
|
+
});
|
|
3375
|
+
const changedRegionProjectionSummary = summarizeNativeChangedRegionProjections(changedRegions);
|
|
3161
3376
|
const evidence = [{
|
|
3162
3377
|
id: input.evidenceId ?? `evidence_${idPart}_native_source_diff`,
|
|
3163
3378
|
kind: 'import',
|
|
@@ -3172,14 +3387,10 @@ export function diffNativeSourceImports(input) {
|
|
|
3172
3387
|
sourceChanged,
|
|
3173
3388
|
addedSymbols: changedSymbols.filter((symbol) => symbol.changeKind === 'added').length,
|
|
3174
3389
|
removedSymbols: changedSymbols.filter((symbol) => symbol.changeKind === 'removed').length,
|
|
3175
|
-
modifiedSymbols: changedSymbols.filter((symbol) => symbol.changeKind === 'modified').length
|
|
3390
|
+
modifiedSymbols: changedSymbols.filter((symbol) => symbol.changeKind === 'modified').length,
|
|
3391
|
+
changedRegionProjectionSummary
|
|
3176
3392
|
}
|
|
3177
3393
|
}];
|
|
3178
|
-
const readiness = maxSemanticMergeReadiness(
|
|
3179
|
-
maxSemanticMergeReadiness(nativeImportReadiness(before), nativeImportReadiness(after)),
|
|
3180
|
-
sourceChanged && changedSymbols.length === 0 ? 'needs-review' : 'ready'
|
|
3181
|
-
);
|
|
3182
|
-
const reasons = nativeSourceChangeReasons({ before, after, beforeHash, afterHash, changedSymbols, changedRegions, readiness });
|
|
3183
3394
|
const conflictKeys = uniqueStrings([
|
|
3184
3395
|
...changedSymbols.map((symbol) => symbol.conflictKey),
|
|
3185
3396
|
...changedRegions.map((region) => region.conflictKey ?? region.key ?? region.id),
|
|
@@ -3230,7 +3441,8 @@ export function diffNativeSourceImports(input) {
|
|
|
3230
3441
|
beforeImportId: before?.id,
|
|
3231
3442
|
afterImportId: after?.id,
|
|
3232
3443
|
sourceChanged,
|
|
3233
|
-
changeSummary: nativeSourceChangeSummary(changedSymbols, changedRegions, sourceChanged)
|
|
3444
|
+
changeSummary: nativeSourceChangeSummary(changedSymbols, changedRegions, sourceChanged),
|
|
3445
|
+
changedRegionProjectionSummary
|
|
3234
3446
|
}
|
|
3235
3447
|
});
|
|
3236
3448
|
return {
|
|
@@ -3259,6 +3471,7 @@ export function diffNativeSourceImports(input) {
|
|
|
3259
3471
|
afterSidecarId: afterSidecar?.id,
|
|
3260
3472
|
beforeImportContract: before?.metadata?.importResultContract,
|
|
3261
3473
|
afterImportContract: after?.metadata?.importResultContract,
|
|
3474
|
+
changedRegionProjectionSummary,
|
|
3262
3475
|
...input.metadata
|
|
3263
3476
|
}
|
|
3264
3477
|
};
|
|
@@ -3404,6 +3617,199 @@ function fileLevelNativeChangeRegion(input) {
|
|
|
3404
3617
|
};
|
|
3405
3618
|
}
|
|
3406
3619
|
|
|
3620
|
+
function attachNativeChangeRegionProjectionMetadata(regions, context) {
|
|
3621
|
+
return (regions ?? []).map((region) => {
|
|
3622
|
+
const projection = nativeChangedRegionProjectionMetadata(region, context);
|
|
3623
|
+
return {
|
|
3624
|
+
...region,
|
|
3625
|
+
metadata: {
|
|
3626
|
+
...(region.metadata ?? {}),
|
|
3627
|
+
changedRegionProjection: projection
|
|
3628
|
+
}
|
|
3629
|
+
};
|
|
3630
|
+
});
|
|
3631
|
+
}
|
|
3632
|
+
|
|
3633
|
+
function nativeChangedRegionProjectionMetadata(region, context) {
|
|
3634
|
+
const beforeRegion = findSemanticImportRegion(context.beforeSidecar, region);
|
|
3635
|
+
const afterRegion = findSemanticImportRegion(context.afterSidecar, region);
|
|
3636
|
+
const regionSymbols = (context.changedSymbols ?? []).filter((symbol) => nativeChangeSymbolTouchesRegion(symbol, region));
|
|
3637
|
+
const sourceMapLinks = uniqueRecordsById([
|
|
3638
|
+
...nativeChangeProjectionSourceMapLinks(context.before, 'before', beforeRegion ?? region, regionSymbols),
|
|
3639
|
+
...nativeChangeProjectionSourceMapLinks(context.after, 'after', afterRegion ?? region, regionSymbols)
|
|
3640
|
+
]).slice(0, 24);
|
|
3641
|
+
const action = nativeChangedRegionProjectionAction(region, context.readiness);
|
|
3642
|
+
const conflictKeys = uniqueStrings([
|
|
3643
|
+
region.conflictKey,
|
|
3644
|
+
region.key ? `region:${region.key}` : undefined,
|
|
3645
|
+
region.id ? `region:${region.id}` : undefined,
|
|
3646
|
+
...regionSymbols.map((symbol) => symbol.conflictKey)
|
|
3647
|
+
].filter(Boolean));
|
|
3648
|
+
return {
|
|
3649
|
+
schema: 'frontier.lang.changedRegionProjection.v1',
|
|
3650
|
+
id: `changed_region_projection_${idFragment(region.conflictKey ?? region.key ?? region.id)}`,
|
|
3651
|
+
reviewRequired: true,
|
|
3652
|
+
autoMergeClaim: false,
|
|
3653
|
+
changeKind: region.changeKind,
|
|
3654
|
+
language: region.language ?? context.language,
|
|
3655
|
+
sourcePath: region.sourcePath ?? context.sourcePath,
|
|
3656
|
+
conflictKey: region.conflictKey,
|
|
3657
|
+
region: {
|
|
3658
|
+
id: region.id,
|
|
3659
|
+
key: region.key,
|
|
3660
|
+
kind: region.regionKind,
|
|
3661
|
+
granularity: region.granularity,
|
|
3662
|
+
precision: region.precision,
|
|
3663
|
+
sourceSpan: region.sourceSpan,
|
|
3664
|
+
nativeAstNodeId: region.nativeAstNodeId,
|
|
3665
|
+
symbolId: region.symbolId,
|
|
3666
|
+
symbolName: region.symbolName,
|
|
3667
|
+
symbolKind: region.symbolKind
|
|
3668
|
+
},
|
|
3669
|
+
before: nativeChangeProjectionEndpoint(context.before, context.beforeSidecar, beforeRegion ?? (region.changeKind === 'added' ? undefined : region), 'before'),
|
|
3670
|
+
after: nativeChangeProjectionEndpoint(context.after, context.afterSidecar, afterRegion ?? (region.changeKind === 'removed' ? undefined : region), 'after'),
|
|
3671
|
+
sourceMapLinks,
|
|
3672
|
+
admission: {
|
|
3673
|
+
readiness: context.readiness,
|
|
3674
|
+
action,
|
|
3675
|
+
reasons: context.reasons ?? [],
|
|
3676
|
+
conflictKeys
|
|
3677
|
+
}
|
|
3678
|
+
};
|
|
3679
|
+
}
|
|
3680
|
+
|
|
3681
|
+
function findSemanticImportRegion(sidecar, region) {
|
|
3682
|
+
return (sidecar?.ownershipRegions ?? []).find((candidate) => (
|
|
3683
|
+
(region.id && candidate.id === region.id) ||
|
|
3684
|
+
(region.key && candidate.key === region.key)
|
|
3685
|
+
));
|
|
3686
|
+
}
|
|
3687
|
+
|
|
3688
|
+
function nativeChangeProjectionEndpoint(imported, sidecar, region, side) {
|
|
3689
|
+
if (!imported && !region) return undefined;
|
|
3690
|
+
const preservation = nativeImportSourcePreservationRecord(imported);
|
|
3691
|
+
const sourceMaps = imported?.sourceMaps ?? imported?.universalAst?.sourceMaps ?? [];
|
|
3692
|
+
const regionMappings = sourceMaps
|
|
3693
|
+
.flatMap((sourceMap) => (sourceMap?.mappings ?? []).map((mapping) => ({ sourceMap, mapping })))
|
|
3694
|
+
.filter(({ mapping }) => nativeChangeMappingTouchesRegion(mapping, region, []));
|
|
3695
|
+
return {
|
|
3696
|
+
side,
|
|
3697
|
+
importId: imported?.id,
|
|
3698
|
+
sidecarId: sidecar?.id,
|
|
3699
|
+
nativeSourceId: imported?.nativeSource?.id,
|
|
3700
|
+
nativeAstId: imported?.nativeAst?.id,
|
|
3701
|
+
semanticIndexId: imported?.semanticIndex?.id,
|
|
3702
|
+
universalAstId: imported?.universalAst?.id,
|
|
3703
|
+
sourcePath: imported?.sourcePath ?? region?.sourcePath,
|
|
3704
|
+
sourceHash: imported?.nativeSource?.sourceHash ?? imported?.nativeAst?.sourceHash ?? region?.sourceHash,
|
|
3705
|
+
sourcePreservationId: preservation?.id,
|
|
3706
|
+
exactSourceAvailable: preservation?.summary?.exactSourceAvailable === true,
|
|
3707
|
+
ownershipRegionId: region?.id,
|
|
3708
|
+
ownershipKey: region?.key,
|
|
3709
|
+
ownershipRegionKind: region?.regionKind,
|
|
3710
|
+
sourceSpan: region?.sourceSpan,
|
|
3711
|
+
sourceMapIds: sourceMaps.map((sourceMap) => sourceMap?.id).filter(Boolean),
|
|
3712
|
+
sourceMapMappingIds: regionMappings.map(({ mapping }) => mapping?.id).filter(Boolean)
|
|
3713
|
+
};
|
|
3714
|
+
}
|
|
3715
|
+
|
|
3716
|
+
function nativeChangeProjectionSourceMapLinks(imported, side, region, symbols) {
|
|
3717
|
+
if (!imported) return [];
|
|
3718
|
+
const sourceMaps = imported.sourceMaps ?? imported.universalAst?.sourceMaps ?? [];
|
|
3719
|
+
const links = [];
|
|
3720
|
+
for (const sourceMap of sourceMaps) {
|
|
3721
|
+
for (const mapping of sourceMap?.mappings ?? []) {
|
|
3722
|
+
if (!nativeChangeMappingTouchesRegion(mapping, region, symbols)) continue;
|
|
3723
|
+
links.push({
|
|
3724
|
+
id: `${side}:${sourceMap.id}:${mapping.id}`,
|
|
3725
|
+
side,
|
|
3726
|
+
sourceMapId: sourceMap.id,
|
|
3727
|
+
sourceMapMappingId: mapping.id,
|
|
3728
|
+
sourcePath: mapping.sourceSpan?.path ?? sourceMap.sourcePath ?? imported.sourcePath,
|
|
3729
|
+
sourceHash: sourceMap.sourceHash ?? imported.nativeSource?.sourceHash ?? imported.nativeAst?.sourceHash,
|
|
3730
|
+
targetPath: mapping.generatedSpan?.targetPath ?? sourceMap.targetPath,
|
|
3731
|
+
targetHash: mapping.generatedSpan?.targetHash ?? sourceMap.targetHash,
|
|
3732
|
+
semanticSymbolId: mapping.semanticSymbolId,
|
|
3733
|
+
semanticOccurrenceId: mapping.semanticOccurrenceId,
|
|
3734
|
+
semanticNodeId: mapping.semanticNodeId,
|
|
3735
|
+
nativeSourceId: mapping.nativeSourceId,
|
|
3736
|
+
nativeAstNodeId: mapping.nativeAstNodeId,
|
|
3737
|
+
precision: mapping.precision,
|
|
3738
|
+
sourceSpan: mapping.sourceSpan,
|
|
3739
|
+
generatedSpan: mapping.generatedSpan,
|
|
3740
|
+
ownershipRegionId: mapping.ownershipRegionId,
|
|
3741
|
+
ownershipRegionKey: mapping.ownershipRegionKey,
|
|
3742
|
+
ownershipRegionKind: mapping.ownershipRegionKind
|
|
3743
|
+
});
|
|
3744
|
+
}
|
|
3745
|
+
}
|
|
3746
|
+
return links;
|
|
3747
|
+
}
|
|
3748
|
+
|
|
3749
|
+
function nativeChangeMappingTouchesRegion(mapping, region, symbols) {
|
|
3750
|
+
if (!mapping || !region) return false;
|
|
3751
|
+
const symbolIds = new Set((symbols ?? []).map((symbol) => symbol.id).filter(Boolean));
|
|
3752
|
+
const occurrenceIds = new Set((symbols ?? []).map((symbol) => symbol.semanticOccurrenceId).filter(Boolean));
|
|
3753
|
+
const mappingIds = new Set((symbols ?? []).map((symbol) => symbol.sourceMapMappingId).filter(Boolean));
|
|
3754
|
+
if (mappingIds.has(mapping.id)) return true;
|
|
3755
|
+
if (region.id && mapping.ownershipRegionId === region.id) return true;
|
|
3756
|
+
if (region.key && mapping.ownershipRegionKey === region.key) return true;
|
|
3757
|
+
if (region.nativeAstNodeId && mapping.nativeAstNodeId === region.nativeAstNodeId) return true;
|
|
3758
|
+
if (symbolIds.has(mapping.semanticSymbolId)) return true;
|
|
3759
|
+
if (occurrenceIds.has(mapping.semanticOccurrenceId)) return true;
|
|
3760
|
+
if (region.granularity === 'file') {
|
|
3761
|
+
return !region.sourcePath || sourceSpanPathMatches(mapping.sourceSpan, region.sourcePath);
|
|
3762
|
+
}
|
|
3763
|
+
return false;
|
|
3764
|
+
}
|
|
3765
|
+
|
|
3766
|
+
function sourceSpanPathMatches(span, sourcePath) {
|
|
3767
|
+
if (!span || !sourcePath) return false;
|
|
3768
|
+
return span.path === sourcePath || span.sourceId === sourcePath;
|
|
3769
|
+
}
|
|
3770
|
+
|
|
3771
|
+
function nativeChangeSymbolTouchesRegion(symbol, region) {
|
|
3772
|
+
return Boolean(symbol && region && (
|
|
3773
|
+
(region.id && symbol.ownershipRegionId === region.id) ||
|
|
3774
|
+
(region.key && (
|
|
3775
|
+
symbol.ownershipKey === region.key ||
|
|
3776
|
+
symbol.beforeOwnershipKey === region.key ||
|
|
3777
|
+
symbol.afterOwnershipKey === region.key
|
|
3778
|
+
))
|
|
3779
|
+
));
|
|
3780
|
+
}
|
|
3781
|
+
|
|
3782
|
+
function nativeChangedRegionProjectionAction(region, readiness) {
|
|
3783
|
+
if (readiness === 'blocked') return 'rerun-or-human-port';
|
|
3784
|
+
if (region.changeKind === 'added') return 'review-addition';
|
|
3785
|
+
if (region.changeKind === 'removed') return 'review-removal';
|
|
3786
|
+
if (region.granularity === 'file') return 'review-file';
|
|
3787
|
+
return 'review-port';
|
|
3788
|
+
}
|
|
3789
|
+
|
|
3790
|
+
function nativeImportSourcePreservationRecord(imported) {
|
|
3791
|
+
return imported?.metadata?.sourcePreservation
|
|
3792
|
+
?? imported?.nativeSource?.metadata?.sourcePreservation
|
|
3793
|
+
?? imported?.nativeAst?.metadata?.sourcePreservation
|
|
3794
|
+
?? imported?.universalAst?.metadata?.sourcePreservation;
|
|
3795
|
+
}
|
|
3796
|
+
|
|
3797
|
+
function summarizeNativeChangedRegionProjections(regions) {
|
|
3798
|
+
const projections = (regions ?? [])
|
|
3799
|
+
.map((region) => region?.metadata?.changedRegionProjection)
|
|
3800
|
+
.filter(Boolean);
|
|
3801
|
+
return {
|
|
3802
|
+
schema: 'frontier.lang.changedRegionProjectionSummary.v1',
|
|
3803
|
+
total: regions?.length ?? 0,
|
|
3804
|
+
withProjection: projections.length,
|
|
3805
|
+
reviewRequired: projections.filter((projection) => projection.reviewRequired === true).length,
|
|
3806
|
+
autoMergeClaims: projections.filter((projection) => projection.autoMergeClaim === true).length,
|
|
3807
|
+
sourceMapLinks: projections.reduce((sum, projection) => sum + (projection.sourceMapLinks?.length ?? 0), 0),
|
|
3808
|
+
byAction: countBy(projections.map((projection) => projection.admission?.action ?? 'unknown')),
|
|
3809
|
+
byRegionKind: countBy(projections.map((projection) => projection.region?.kind ?? 'unknown'))
|
|
3810
|
+
};
|
|
3811
|
+
}
|
|
3812
|
+
|
|
3407
3813
|
function nativeImportReadiness(imported) {
|
|
3408
3814
|
return imported?.metadata?.semanticMergeReadiness
|
|
3409
3815
|
?? imported?.metadata?.nativeImportLossSummary?.semanticMergeReadiness
|
|
@@ -3467,11 +3873,31 @@ function nativeChangeSpans(changedSymbols, changedRegions, input) {
|
|
|
3467
3873
|
symbolId: region.symbolId,
|
|
3468
3874
|
span: region.sourceSpan,
|
|
3469
3875
|
conflictKey: region.conflictKey ?? `region:${region.key ?? region.id}`,
|
|
3470
|
-
metadata: {
|
|
3876
|
+
metadata: {
|
|
3877
|
+
changeKind: region.changeKind,
|
|
3878
|
+
regionKind: region.regionKind,
|
|
3879
|
+
granularity: region.granularity,
|
|
3880
|
+
...(region.metadata?.changedRegionProjection ? {
|
|
3881
|
+
changedRegionProjection: nativeChangedRegionProjectionSpanMetadata(region.metadata.changedRegionProjection)
|
|
3882
|
+
} : {})
|
|
3883
|
+
}
|
|
3471
3884
|
}));
|
|
3472
3885
|
return uniqueRecordsById([...symbolSpans, ...regionSpans]);
|
|
3473
3886
|
}
|
|
3474
3887
|
|
|
3888
|
+
function nativeChangedRegionProjectionSpanMetadata(projection) {
|
|
3889
|
+
return {
|
|
3890
|
+
schema: projection.schema,
|
|
3891
|
+
id: projection.id,
|
|
3892
|
+
reviewRequired: projection.reviewRequired,
|
|
3893
|
+
autoMergeClaim: projection.autoMergeClaim,
|
|
3894
|
+
beforeSourceHash: projection.before?.sourceHash,
|
|
3895
|
+
afterSourceHash: projection.after?.sourceHash,
|
|
3896
|
+
sourceMapLinks: projection.sourceMapLinks?.length ?? 0,
|
|
3897
|
+
admission: projection.admission
|
|
3898
|
+
};
|
|
3899
|
+
}
|
|
3900
|
+
|
|
3475
3901
|
function nativeSourceChangeSummary(changedSymbols, changedRegions, sourceChanged) {
|
|
3476
3902
|
return {
|
|
3477
3903
|
sourceChanged,
|
|
@@ -6537,6 +6963,132 @@ function nativeImportLanguageProfile(language, input = {}) {
|
|
|
6537
6963
|
});
|
|
6538
6964
|
}
|
|
6539
6965
|
|
|
6966
|
+
function nativeParserAstFormatProfile(id, input = {}) {
|
|
6967
|
+
return Object.freeze({
|
|
6968
|
+
id,
|
|
6969
|
+
aliases: Object.freeze(uniqueStrings(input.aliases ?? [])),
|
|
6970
|
+
kind: input.kind ?? 'abstract-ast',
|
|
6971
|
+
languages: Object.freeze(uniqueStrings(input.languages ?? [])),
|
|
6972
|
+
parserAdapters: Object.freeze(uniqueStrings(input.parserAdapters ?? [id])),
|
|
6973
|
+
exactness: input.exactness ?? 'unknown',
|
|
6974
|
+
sourceRangeModel: input.sourceRangeModel ?? 'unknown',
|
|
6975
|
+
preservesTokens: Boolean(input.preservesTokens),
|
|
6976
|
+
preservesTrivia: Boolean(input.preservesTrivia),
|
|
6977
|
+
supportsIncremental: Boolean(input.supportsIncremental),
|
|
6978
|
+
supportsErrorRecovery: Boolean(input.supportsErrorRecovery),
|
|
6979
|
+
notes: Object.freeze(uniqueStrings(input.notes ?? []))
|
|
6980
|
+
});
|
|
6981
|
+
}
|
|
6982
|
+
|
|
6983
|
+
function normalizeParserAstFormatId(format) {
|
|
6984
|
+
return String(format ?? '').trim().toLowerCase().replace(/[_\s]+/g, '-');
|
|
6985
|
+
}
|
|
6986
|
+
|
|
6987
|
+
function mergeNativeParserAstFormatProfiles(profiles, imports, adapters) {
|
|
6988
|
+
const byId = new Map((profiles ?? []).map((profile) => [normalizeParserAstFormatId(profile.id ?? profile), nativeParserAstFormatProfile(normalizeParserAstFormatId(profile.id ?? profile), profile)]));
|
|
6989
|
+
for (const adapter of adapters ?? []) {
|
|
6990
|
+
const summary = safeNativeImporterAdapterSummary(adapter);
|
|
6991
|
+
if (!summary) continue;
|
|
6992
|
+
const formatId = parserAstFormatIdForParser(summary.parser);
|
|
6993
|
+
if (!byId.has(formatId)) {
|
|
6994
|
+
byId.set(formatId, nativeParserAstFormatProfile(formatId, {
|
|
6995
|
+
languages: [summary.language],
|
|
6996
|
+
parserAdapters: [summary.parser],
|
|
6997
|
+
exactness: summary.coverage.exactness,
|
|
6998
|
+
sourceRangeModel: summary.coverage.sourceRanges ? 'adapter-reported' : 'unknown'
|
|
6999
|
+
}));
|
|
7000
|
+
}
|
|
7001
|
+
}
|
|
7002
|
+
for (const imported of imports ?? []) {
|
|
7003
|
+
const formatId = parserAstFormatIdForImport(imported);
|
|
7004
|
+
if (formatId && !byId.has(formatId)) {
|
|
7005
|
+
byId.set(formatId, nativeParserAstFormatProfile(formatId, {
|
|
7006
|
+
languages: [imported.language].filter(Boolean),
|
|
7007
|
+
parserAdapters: [imported.parser ?? imported.nativeAst?.parser ?? formatId],
|
|
7008
|
+
exactness: 'unknown',
|
|
7009
|
+
sourceRangeModel: (imported.sourceMaps ?? []).some((sourceMap) => sourceMap.mappings?.some((mapping) => mapping.sourceSpan)) ? 'adapter-reported' : 'unknown'
|
|
7010
|
+
}));
|
|
7011
|
+
}
|
|
7012
|
+
}
|
|
7013
|
+
return [...byId.values()].sort((left, right) => left.id.localeCompare(right.id));
|
|
7014
|
+
}
|
|
7015
|
+
|
|
7016
|
+
function nativeParserAstFormatCoverageForProfile(profile, imports, adapters) {
|
|
7017
|
+
const formatIds = new Set([profile.id, ...profile.aliases].map(normalizeParserAstFormatId));
|
|
7018
|
+
const adapterParsers = new Set(profile.parserAdapters.map(parserAstFormatIdForParser));
|
|
7019
|
+
const matchingAdapters = (adapters ?? [])
|
|
7020
|
+
.map((adapter) => safeNativeImporterAdapterSummary(adapter))
|
|
7021
|
+
.filter(Boolean)
|
|
7022
|
+
.filter((adapter) => formatIds.has(parserAstFormatIdForParser(adapter.parser)) || adapterParsers.has(parserAstFormatIdForParser(adapter.parser)));
|
|
7023
|
+
const matchingImports = (imports ?? [])
|
|
7024
|
+
.filter((imported) => {
|
|
7025
|
+
const formatId = parserAstFormatIdForImport(imported);
|
|
7026
|
+
return formatId && (formatIds.has(formatId) || adapterParsers.has(formatId));
|
|
7027
|
+
});
|
|
7028
|
+
const effectiveCapabilities = {};
|
|
7029
|
+
for (const adapter of matchingAdapters) {
|
|
7030
|
+
for (const row of adapter.coverage.capabilityEvidence?.capabilities ?? []) {
|
|
7031
|
+
if (row.effective) effectiveCapabilities[row.capability] = (effectiveCapabilities[row.capability] ?? 0) + 1;
|
|
7032
|
+
}
|
|
7033
|
+
}
|
|
7034
|
+
const readiness = matchingImports.reduce(
|
|
7035
|
+
(current, imported) => maxSemanticMergeReadiness(current, nativeImportReadiness(imported)),
|
|
7036
|
+
matchingImports.length ? 'ready' : 'needs-review'
|
|
7037
|
+
);
|
|
7038
|
+
return {
|
|
7039
|
+
id: profile.id,
|
|
7040
|
+
kind: profile.kind,
|
|
7041
|
+
languages: profile.languages,
|
|
7042
|
+
parserAdapters: profile.parserAdapters,
|
|
7043
|
+
exactness: profile.exactness,
|
|
7044
|
+
sourceRangeModel: profile.sourceRangeModel,
|
|
7045
|
+
preservesTokens: profile.preservesTokens,
|
|
7046
|
+
preservesTrivia: profile.preservesTrivia,
|
|
7047
|
+
supportsIncremental: profile.supportsIncremental,
|
|
7048
|
+
supportsErrorRecovery: profile.supportsErrorRecovery,
|
|
7049
|
+
notes: profile.notes,
|
|
7050
|
+
adapters: {
|
|
7051
|
+
total: matchingAdapters.length,
|
|
7052
|
+
ids: matchingAdapters.map((adapter) => adapter.id),
|
|
7053
|
+
parsers: uniqueStrings(matchingAdapters.map((adapter) => adapter.parser)),
|
|
7054
|
+
effectiveCapabilities
|
|
7055
|
+
},
|
|
7056
|
+
imports: {
|
|
7057
|
+
total: matchingImports.length,
|
|
7058
|
+
sourcePaths: matchingImports.map((imported) => imported.sourcePath).filter(Boolean),
|
|
7059
|
+
readiness,
|
|
7060
|
+
nativeAstNodes: matchingImports.reduce((sum, imported) => sum + Object.keys(imported.nativeAst?.nodes ?? {}).length, 0),
|
|
7061
|
+
symbols: matchingImports.reduce((sum, imported) => sum + (imported.semanticIndex?.symbols?.length ?? 0), 0),
|
|
7062
|
+
sourceMapMappings: matchingImports.reduce((sum, imported) => sum + (imported.sourceMaps ?? []).reduce((mapSum, sourceMap) => mapSum + (sourceMap.mappings?.length ?? 0), 0), 0),
|
|
7063
|
+
losses: matchingImports.reduce((sum, imported) => sum + (imported.losses?.length ?? 0), 0)
|
|
7064
|
+
}
|
|
7065
|
+
};
|
|
7066
|
+
}
|
|
7067
|
+
|
|
7068
|
+
function parserAstFormatIdForParser(parser) {
|
|
7069
|
+
const text = normalizeParserAstFormatId(parser);
|
|
7070
|
+
if (text.includes('typescript')) return 'typescript-compiler-api';
|
|
7071
|
+
if (text.includes('python') && text.includes('ast')) return 'python-ast';
|
|
7072
|
+
if (text.includes('tree-sitter') || text.includes('treesitter')) return 'tree-sitter';
|
|
7073
|
+
if (text.includes('babel')) return 'babel';
|
|
7074
|
+
if (text.includes('estree')) return 'estree';
|
|
7075
|
+
if (text.includes('libcst')) return 'libcst';
|
|
7076
|
+
if (text.includes('scip')) return 'scip';
|
|
7077
|
+
if (text.includes('lsif')) return 'lsif';
|
|
7078
|
+
return text || 'unknown';
|
|
7079
|
+
}
|
|
7080
|
+
|
|
7081
|
+
function parserAstFormatIdForImport(imported) {
|
|
7082
|
+
return parserAstFormatIdForParser(
|
|
7083
|
+
imported?.metadata?.adapter?.parser
|
|
7084
|
+
?? imported?.metadata?.astFormat
|
|
7085
|
+
?? imported?.nativeAst?.metadata?.astFormat
|
|
7086
|
+
?? imported?.nativeAst?.parser
|
|
7087
|
+
?? imported?.parser
|
|
7088
|
+
?? imported?.metadata?.parser
|
|
7089
|
+
);
|
|
7090
|
+
}
|
|
7091
|
+
|
|
6540
7092
|
function mergeNativeImportProfiles(languages, imports, adapters, targetAdapters = []) {
|
|
6541
7093
|
const profilesByLanguage = new Map();
|
|
6542
7094
|
for (const profile of languages) {
|
|
@@ -7534,6 +8086,23 @@ function parseTreeSitterSource(input, options) {
|
|
|
7534
8086
|
return undefined;
|
|
7535
8087
|
}
|
|
7536
8088
|
|
|
8089
|
+
function parsePythonAstSource(input, options) {
|
|
8090
|
+
const parse = options.parse ?? options.parserModule?.parse ?? options.pythonAst?.parse;
|
|
8091
|
+
if (typeof parse !== 'function') return undefined;
|
|
8092
|
+
const parserOptions = {
|
|
8093
|
+
sourcePath: input.sourcePath,
|
|
8094
|
+
filename: input.sourcePath,
|
|
8095
|
+
mode: options.mode ?? input.options?.mode ?? 'exec',
|
|
8096
|
+
typeComments: options.typeComments ?? input.options?.typeComments,
|
|
8097
|
+
featureVersion: options.featureVersion ?? input.options?.featureVersion,
|
|
8098
|
+
optimize: options.optimize ?? input.options?.optimize,
|
|
8099
|
+
includeAttributes: options.includeAttributes ?? input.options?.includeAttributes,
|
|
8100
|
+
...(options.parserOptions ?? {}),
|
|
8101
|
+
...(input.options?.parserOptions ?? {})
|
|
8102
|
+
};
|
|
8103
|
+
return parse(input.sourceText, parserOptions);
|
|
8104
|
+
}
|
|
8105
|
+
|
|
7537
8106
|
function createNativeImportFromSyntaxAst(ast, input, options) {
|
|
7538
8107
|
const root = normalizeSyntaxAstRoot(ast, options.astFormat);
|
|
7539
8108
|
if (!root) {
|
|
@@ -7594,6 +8163,36 @@ function createNativeImportFromTypeScriptAst(sourceFile, input, options) {
|
|
|
7594
8163
|
};
|
|
7595
8164
|
}
|
|
7596
8165
|
|
|
8166
|
+
function createNativeImportFromPythonAst(root, input, options) {
|
|
8167
|
+
const context = createAstNormalizationContext(input, options);
|
|
8168
|
+
visitPythonAstNode(root, context, 'root');
|
|
8169
|
+
if (context.truncated) {
|
|
8170
|
+
context.losses.push(truncatedAstLoss(input, context, options));
|
|
8171
|
+
}
|
|
8172
|
+
const semantic = semanticIndexFromNativeDeclarations(context.declarations, input, options);
|
|
8173
|
+
return {
|
|
8174
|
+
rootId: context.rootId,
|
|
8175
|
+
nodes: context.nodes,
|
|
8176
|
+
semanticIndex: semantic.semanticIndex,
|
|
8177
|
+
mappings: semantic.mappings,
|
|
8178
|
+
losses: mergeNativeLosses(context.losses, options.diagnostics?.map((diagnostic, index) => adapterDiagnosticToLoss(diagnostic, index, {
|
|
8179
|
+
id: input.adapterId,
|
|
8180
|
+
version: input.adapterVersion
|
|
8181
|
+
}, input)) ?? []),
|
|
8182
|
+
evidence: semantic.evidence,
|
|
8183
|
+
diagnostics: options.diagnostics,
|
|
8184
|
+
metadata: {
|
|
8185
|
+
astFormat: options.astFormat,
|
|
8186
|
+
parser: options.parser,
|
|
8187
|
+
pythonVersion: options.pythonVersion,
|
|
8188
|
+
includeAttributes: Boolean(options.includeAttributes),
|
|
8189
|
+
normalizedNodeCount: Object.keys(context.nodes).length,
|
|
8190
|
+
declarationCount: context.declarations.length,
|
|
8191
|
+
truncated: context.truncated
|
|
8192
|
+
}
|
|
8193
|
+
};
|
|
8194
|
+
}
|
|
8195
|
+
|
|
7597
8196
|
function createNativeImportFromTreeSitter(root, input, options) {
|
|
7598
8197
|
const context = createAstNormalizationContext(input, options);
|
|
7599
8198
|
visitTreeSitterNode(root, context, 'root');
|
|
@@ -7730,6 +8329,53 @@ function visitTypeScriptAstNode(node, sourceFile, context, propertyPath, ts) {
|
|
|
7730
8329
|
return id;
|
|
7731
8330
|
}
|
|
7732
8331
|
|
|
8332
|
+
function visitPythonAstNode(node, context, propertyPath) {
|
|
8333
|
+
if (!isPythonAstNode(node) || context.truncated) return undefined;
|
|
8334
|
+
if (context.objectIds.has(node)) return context.objectIds.get(node);
|
|
8335
|
+
if (context.counter >= context.maxNodes) {
|
|
8336
|
+
context.truncated = true;
|
|
8337
|
+
return undefined;
|
|
8338
|
+
}
|
|
8339
|
+
const kind = pythonAstKind(node);
|
|
8340
|
+
const span = spanFromPythonAstNode(node, context.input);
|
|
8341
|
+
const id = nativeNodeId(context, kind, { start: { line: span?.startLine, column: span?.startColumn } }, propertyPath);
|
|
8342
|
+
context.objectIds.set(node, id);
|
|
8343
|
+
if (!context.rootId) context.rootId = id;
|
|
8344
|
+
const children = [];
|
|
8345
|
+
for (const [field, value] of pythonAstChildEntries(node)) {
|
|
8346
|
+
if (Array.isArray(value)) {
|
|
8347
|
+
value.forEach((entry, index) => {
|
|
8348
|
+
const childId = visitPythonAstNode(entry, context, `${propertyPath}.${field}[${index}]`);
|
|
8349
|
+
if (childId) children.push(childId);
|
|
8350
|
+
});
|
|
8351
|
+
} else {
|
|
8352
|
+
const childId = visitPythonAstNode(value, context, `${propertyPath}.${field}`);
|
|
8353
|
+
if (childId) children.push(childId);
|
|
8354
|
+
}
|
|
8355
|
+
}
|
|
8356
|
+
const declaration = pythonAstDeclaration(node, kind, id, context.input);
|
|
8357
|
+
const nativeNode = {
|
|
8358
|
+
id,
|
|
8359
|
+
kind,
|
|
8360
|
+
languageKind: `${context.input.language}.${kind}`,
|
|
8361
|
+
span,
|
|
8362
|
+
value: declaration?.name ?? pythonAstNodeValue(node),
|
|
8363
|
+
fields: primitivePythonAstFields(node, kind),
|
|
8364
|
+
children,
|
|
8365
|
+
metadata: {
|
|
8366
|
+
astFormat: context.options.astFormat,
|
|
8367
|
+
propertyPath,
|
|
8368
|
+
lineno: numberOrUndefined(node.lineno ?? node.line),
|
|
8369
|
+
colOffset: numberOrUndefined(node.col_offset ?? node.colOffset),
|
|
8370
|
+
endLineno: numberOrUndefined(node.end_lineno ?? node.endLine),
|
|
8371
|
+
endColOffset: numberOrUndefined(node.end_col_offset ?? node.endColOffset)
|
|
8372
|
+
}
|
|
8373
|
+
};
|
|
8374
|
+
context.nodes[id] = nativeNode;
|
|
8375
|
+
if (declaration) context.declarations.push({ ...declaration, nativeNode });
|
|
8376
|
+
return id;
|
|
8377
|
+
}
|
|
8378
|
+
|
|
7733
8379
|
function visitTreeSitterNode(node, context, propertyPath) {
|
|
7734
8380
|
if (!node || typeof node !== 'object' || context.truncated) return undefined;
|
|
7735
8381
|
if (context.objectIds.has(node)) return context.objectIds.get(node);
|
|
@@ -8225,6 +8871,14 @@ function nativeTargetProjectionDiagnosticToLoss(diagnostic, index, adapter, inpu
|
|
|
8225
8871
|
};
|
|
8226
8872
|
}
|
|
8227
8873
|
|
|
8874
|
+
function safeNativeImporterAdapterSummary(adapter) {
|
|
8875
|
+
try {
|
|
8876
|
+
return normalizeNativeImporterAdapter(adapter);
|
|
8877
|
+
} catch {
|
|
8878
|
+
return undefined;
|
|
8879
|
+
}
|
|
8880
|
+
}
|
|
8881
|
+
|
|
8228
8882
|
function normalizeNativeImporterAdapter(adapter) {
|
|
8229
8883
|
if (!adapter || typeof adapter !== 'object') {
|
|
8230
8884
|
throw new Error('Native importer adapter must be an object');
|
|
@@ -8859,6 +9513,23 @@ function isSyntaxAstNode(value) {
|
|
|
8859
9513
|
return Boolean(value && typeof value === 'object' && typeof (value.type ?? value.kind) === 'string');
|
|
8860
9514
|
}
|
|
8861
9515
|
|
|
9516
|
+
function pythonAstRoot(value) {
|
|
9517
|
+
if (!value || typeof value !== 'object') return undefined;
|
|
9518
|
+
if (isPythonAstNode(value)) return value;
|
|
9519
|
+
if (isPythonAstNode(value.ast)) return value.ast;
|
|
9520
|
+
if (isPythonAstNode(value.root)) return value.root;
|
|
9521
|
+
if (isPythonAstNode(value.module)) return value.module;
|
|
9522
|
+
return undefined;
|
|
9523
|
+
}
|
|
9524
|
+
|
|
9525
|
+
function isPythonAstNode(value) {
|
|
9526
|
+
return Boolean(value && typeof value === 'object' && typeof pythonAstKind(value) === 'string');
|
|
9527
|
+
}
|
|
9528
|
+
|
|
9529
|
+
function pythonAstKind(node) {
|
|
9530
|
+
return node?._type ?? node?.type ?? node?.kind ?? node?.nodeType;
|
|
9531
|
+
}
|
|
9532
|
+
|
|
8862
9533
|
function ignoredSyntaxField(key) {
|
|
8863
9534
|
return key === 'type'
|
|
8864
9535
|
|| key === 'kind'
|
|
@@ -8875,6 +9546,24 @@ function ignoredSyntaxField(key) {
|
|
|
8875
9546
|
|| key === 'parent';
|
|
8876
9547
|
}
|
|
8877
9548
|
|
|
9549
|
+
function ignoredPythonAstField(key) {
|
|
9550
|
+
return key === '_type'
|
|
9551
|
+
|| key === 'type'
|
|
9552
|
+
|| key === 'kind'
|
|
9553
|
+
|| key === 'nodeType'
|
|
9554
|
+
|| key === '_fields'
|
|
9555
|
+
|| key === 'lineno'
|
|
9556
|
+
|| key === 'col_offset'
|
|
9557
|
+
|| key === 'end_lineno'
|
|
9558
|
+
|| key === 'end_col_offset'
|
|
9559
|
+
|| key === 'line'
|
|
9560
|
+
|| key === 'colOffset'
|
|
9561
|
+
|| key === 'endLine'
|
|
9562
|
+
|| key === 'endColOffset'
|
|
9563
|
+
|| key === 'ctx'
|
|
9564
|
+
|| key === 'parent';
|
|
9565
|
+
}
|
|
9566
|
+
|
|
8878
9567
|
function primitiveSyntaxFields(node) {
|
|
8879
9568
|
const fields = {};
|
|
8880
9569
|
for (const key of ['name', 'operator', 'sourceType', 'async', 'generator', 'computed', 'static', 'exportKind', 'importKind', 'optional']) {
|
|
@@ -8888,11 +9577,36 @@ function primitiveSyntaxFields(node) {
|
|
|
8888
9577
|
return fields;
|
|
8889
9578
|
}
|
|
8890
9579
|
|
|
9580
|
+
function primitivePythonAstFields(node, kind) {
|
|
9581
|
+
const fields = { kind };
|
|
9582
|
+
for (const key of ['name', 'id', 'arg', 'module', 'level', 'attr', 'asname', 'type_comment']) {
|
|
9583
|
+
if (typeof node[key] === 'string' || typeof node[key] === 'number' || typeof node[key] === 'boolean' || node[key] === null) {
|
|
9584
|
+
fields[key] = node[key];
|
|
9585
|
+
}
|
|
9586
|
+
}
|
|
9587
|
+
if (Array.isArray(node.names)) {
|
|
9588
|
+
fields.names = node.names
|
|
9589
|
+
.map((entry) => pythonAliasName(entry))
|
|
9590
|
+
.filter(Boolean)
|
|
9591
|
+
.join(',');
|
|
9592
|
+
}
|
|
9593
|
+
const literal = pythonAstLiteralValue(node);
|
|
9594
|
+
if (literal !== undefined) fields.literal = literal;
|
|
9595
|
+
return fields;
|
|
9596
|
+
}
|
|
9597
|
+
|
|
8891
9598
|
function literalSyntaxValue(node) {
|
|
8892
9599
|
if (node.value === null || typeof node.value === 'string' || typeof node.value === 'number' || typeof node.value === 'boolean') return node.value;
|
|
8893
9600
|
return undefined;
|
|
8894
9601
|
}
|
|
8895
9602
|
|
|
9603
|
+
function pythonAstLiteralValue(node) {
|
|
9604
|
+
if (node.value === null || typeof node.value === 'string' || typeof node.value === 'number' || typeof node.value === 'boolean') return node.value;
|
|
9605
|
+
if (typeof node.s === 'string' || typeof node.s === 'number') return node.s;
|
|
9606
|
+
if (typeof node.n === 'number') return node.n;
|
|
9607
|
+
return undefined;
|
|
9608
|
+
}
|
|
9609
|
+
|
|
8896
9610
|
function spanFromLoc(loc, input) {
|
|
8897
9611
|
if (!loc?.start) return undefined;
|
|
8898
9612
|
return {
|
|
@@ -8905,6 +9619,22 @@ function spanFromLoc(loc, input) {
|
|
|
8905
9619
|
};
|
|
8906
9620
|
}
|
|
8907
9621
|
|
|
9622
|
+
function spanFromPythonAstNode(node, input) {
|
|
9623
|
+
const line = node.lineno ?? node.line;
|
|
9624
|
+
if (typeof line !== 'number') return undefined;
|
|
9625
|
+
const col = node.col_offset ?? node.colOffset;
|
|
9626
|
+
const endLine = node.end_lineno ?? node.endLine;
|
|
9627
|
+
const endCol = node.end_col_offset ?? node.endColOffset;
|
|
9628
|
+
return {
|
|
9629
|
+
sourceId: input.sourceHash,
|
|
9630
|
+
path: input.sourcePath,
|
|
9631
|
+
startLine: line,
|
|
9632
|
+
startColumn: typeof col === 'number' ? col + 1 : undefined,
|
|
9633
|
+
endLine: typeof endLine === 'number' ? endLine : undefined,
|
|
9634
|
+
endColumn: typeof endCol === 'number' ? endCol + 1 : undefined
|
|
9635
|
+
};
|
|
9636
|
+
}
|
|
9637
|
+
|
|
8908
9638
|
function syntaxDeclaration(node, nativeNodeId, input) {
|
|
8909
9639
|
const kind = String(node.type ?? node.kind ?? '');
|
|
8910
9640
|
if (kind === 'ImportDeclaration') {
|
|
@@ -8923,6 +9653,24 @@ function syntaxDeclaration(node, nativeNodeId, input) {
|
|
|
8923
9653
|
return undefined;
|
|
8924
9654
|
}
|
|
8925
9655
|
|
|
9656
|
+
function pythonAstDeclaration(node, kind, nativeNodeId, input) {
|
|
9657
|
+
if (kind === 'Import') {
|
|
9658
|
+
const name = (node.names ?? []).map((entry) => pythonAliasName(entry)).find(Boolean);
|
|
9659
|
+
if (name) return declarationRecord(input, nativeNodeId, name, 'module', 'import');
|
|
9660
|
+
}
|
|
9661
|
+
if (kind === 'ImportFrom') {
|
|
9662
|
+
const name = node.module ?? (node.names ?? []).map((entry) => pythonAliasName(entry)).find(Boolean);
|
|
9663
|
+
if (name) return declarationRecord(input, nativeNodeId, name, 'module', 'import');
|
|
9664
|
+
}
|
|
9665
|
+
if (kind === 'FunctionDef' || kind === 'AsyncFunctionDef') return declarationRecord(input, nativeNodeId, node.name, 'function', 'definition');
|
|
9666
|
+
if (kind === 'ClassDef') return declarationRecord(input, nativeNodeId, node.name, 'class', 'definition');
|
|
9667
|
+
if (kind === 'AnnAssign' || kind === 'Assign') {
|
|
9668
|
+
const name = pythonAssignmentName(node);
|
|
9669
|
+
if (name) return declarationRecord(input, nativeNodeId, name, 'variable', 'definition');
|
|
9670
|
+
}
|
|
9671
|
+
return undefined;
|
|
9672
|
+
}
|
|
9673
|
+
|
|
8926
9674
|
function typeScriptDeclaration(node, kind, nativeNodeId, input) {
|
|
8927
9675
|
if (kind === 'ImportDeclaration' || kind === 'ImportEqualsDeclaration') {
|
|
8928
9676
|
const name = stringFromTsExpression(node.moduleSpecifier) ?? stringFromTsExpression(node.externalModuleReference?.expression);
|
|
@@ -8966,6 +9714,49 @@ function namedDeclaration(input, nativeNodeId, nameNode, symbolKind) {
|
|
|
8966
9714
|
return name ? declarationRecord(input, nativeNodeId, name, symbolKind, 'definition') : undefined;
|
|
8967
9715
|
}
|
|
8968
9716
|
|
|
9717
|
+
function pythonAstChildEntries(node) {
|
|
9718
|
+
const fieldNames = Array.isArray(node._fields)
|
|
9719
|
+
? node._fields
|
|
9720
|
+
: Object.keys(node).filter((key) => !ignoredPythonAstField(key));
|
|
9721
|
+
return fieldNames
|
|
9722
|
+
.map((field) => [field, node[field]])
|
|
9723
|
+
.filter(([, value]) => Array.isArray(value)
|
|
9724
|
+
? value.some(isPythonAstNode)
|
|
9725
|
+
: isPythonAstNode(value));
|
|
9726
|
+
}
|
|
9727
|
+
|
|
9728
|
+
function pythonAstNodeValue(node) {
|
|
9729
|
+
return node.name ?? node.id ?? node.arg ?? node.module ?? pythonAstLiteralValue(node);
|
|
9730
|
+
}
|
|
9731
|
+
|
|
9732
|
+
function pythonAliasName(alias) {
|
|
9733
|
+
if (!alias) return undefined;
|
|
9734
|
+
if (typeof alias === 'string') return alias;
|
|
9735
|
+
return alias.name ?? alias.asname ?? alias.id;
|
|
9736
|
+
}
|
|
9737
|
+
|
|
9738
|
+
function pythonAssignmentName(node) {
|
|
9739
|
+
if (node.target) return pythonTargetName(node.target);
|
|
9740
|
+
for (const target of node.targets ?? []) {
|
|
9741
|
+
const name = pythonTargetName(target);
|
|
9742
|
+
if (name) return name;
|
|
9743
|
+
}
|
|
9744
|
+
return undefined;
|
|
9745
|
+
}
|
|
9746
|
+
|
|
9747
|
+
function pythonTargetName(target) {
|
|
9748
|
+
if (!target) return undefined;
|
|
9749
|
+
if (typeof target === 'string') return target;
|
|
9750
|
+
if (typeof target.id === 'string') return target.id;
|
|
9751
|
+
if (typeof target.name === 'string') return target.name;
|
|
9752
|
+
if (typeof target.arg === 'string') return target.arg;
|
|
9753
|
+
if (target.attr && target.value) {
|
|
9754
|
+
const base = pythonTargetName(target.value);
|
|
9755
|
+
return base ? `${base}.${target.attr}` : target.attr;
|
|
9756
|
+
}
|
|
9757
|
+
return undefined;
|
|
9758
|
+
}
|
|
9759
|
+
|
|
8969
9760
|
function declarationRecord(input, nativeNodeId, name, symbolKind, role = 'definition') {
|
|
8970
9761
|
return {
|
|
8971
9762
|
name: String(name),
|
package/package.json
CHANGED