@shapeshift-labs/frontier-lang-compiler 0.2.25 → 0.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/bench/smoke.mjs +13 -0
- package/dist/index.d.ts +110 -0
- package/dist/index.js +561 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -116,6 +116,7 @@ Ask the compiler what is actually covered before sending native imports into a m
|
|
|
116
116
|
```js
|
|
117
117
|
import {
|
|
118
118
|
createNativeImportCoverageMatrix,
|
|
119
|
+
createNativeParserAstFormatMatrix,
|
|
119
120
|
createProjectionTargetLossMatrix,
|
|
120
121
|
importNativeSource
|
|
121
122
|
} from '@shapeshift-labs/frontier-lang-compiler';
|
|
@@ -132,6 +133,9 @@ const python = matrix.languages.find((entry) => entry.language === 'python');
|
|
|
132
133
|
console.log(python.imports.readiness); // scanner imports are intentionally review-required
|
|
133
134
|
console.log(python.parserAdapters); // host-owned exact parsers such as LibCST can be injected
|
|
134
135
|
|
|
136
|
+
const parserMatrix = createNativeParserAstFormatMatrix({ imports: [imported] });
|
|
137
|
+
console.log(parserMatrix.formats.find((entry) => entry.id === 'python-ast')?.adapters.total ?? 0);
|
|
138
|
+
|
|
135
139
|
const projectionMatrix = createProjectionTargetLossMatrix({ imports: [imported] });
|
|
136
140
|
const pythonProjection = projectionMatrix.languages.find((entry) => entry.language === 'python');
|
|
137
141
|
|
|
@@ -328,6 +332,7 @@ Use injected parser adapters when a real language parser is available but should
|
|
|
328
332
|
```js
|
|
329
333
|
import {
|
|
330
334
|
createBabelNativeImporterAdapter,
|
|
335
|
+
createPythonAstNativeImporterAdapter,
|
|
331
336
|
importNativeProject,
|
|
332
337
|
runNativeImporterAdapter
|
|
333
338
|
} from '@shapeshift-labs/frontier-lang-compiler';
|
|
@@ -335,6 +340,9 @@ import {
|
|
|
335
340
|
const babelAdapter = createBabelNativeImporterAdapter({
|
|
336
341
|
parserModule: await import('@babel/parser')
|
|
337
342
|
});
|
|
343
|
+
const pythonAstAdapter = createPythonAstNativeImporterAdapter({
|
|
344
|
+
parserModule: hostPythonAstParser
|
|
345
|
+
});
|
|
338
346
|
|
|
339
347
|
const imported = await runNativeImporterAdapter(babelAdapter, {
|
|
340
348
|
sourcePath: 'src/todo.ts',
|
|
@@ -343,10 +351,10 @@ const imported = await runNativeImporterAdapter(babelAdapter, {
|
|
|
343
351
|
|
|
344
352
|
const project = await importNativeProject({
|
|
345
353
|
projectRoot: 'src',
|
|
346
|
-
adapters: [babelAdapter],
|
|
354
|
+
adapters: [babelAdapter, pythonAstAdapter],
|
|
347
355
|
sources: [
|
|
348
356
|
{ language: 'typescript', adapter: babelAdapter.id, sourcePath: 'src/todo.ts', sourceText },
|
|
349
|
-
{ language: 'python', sourcePath: 'tools/todo.py', sourceText: pythonSource }
|
|
357
|
+
{ language: 'python', adapter: pythonAstAdapter.id, sourcePath: 'tools/todo.py', sourceText: pythonSource }
|
|
350
358
|
]
|
|
351
359
|
});
|
|
352
360
|
|
|
@@ -365,6 +373,7 @@ The built-in adapter factories are dependency-light wrappers for caller-owned pa
|
|
|
365
373
|
- `createEstreeNativeImporterAdapter`
|
|
366
374
|
- `createBabelNativeImporterAdapter`
|
|
367
375
|
- `createTypeScriptCompilerNativeImporterAdapter`
|
|
376
|
+
- `createPythonAstNativeImporterAdapter`
|
|
368
377
|
- `createTreeSitterNativeImporterAdapter`
|
|
369
378
|
|
|
370
379
|
Adapter summaries include a structured `coverage` record so merge queues can distinguish exact parser AST imports from declaration scans. The record declares exactness, parser token/trivia support, diagnostics support, source-range and generated-range support, and semantic coverage. Built-in wrappers normalize native AST/CST nodes and declaration-level semantic indexes; they do not claim resolved references, types, control flow, generated ranges, or token/trivia fidelity unless the host adapter supplies that evidence.
|
package/bench/smoke.mjs
CHANGED
|
@@ -4,8 +4,10 @@ import {
|
|
|
4
4
|
compileFrontierSource,
|
|
5
5
|
createEstreeNativeImporterAdapter,
|
|
6
6
|
createNativeImportCoverageMatrix,
|
|
7
|
+
createNativeParserAstFormatMatrix,
|
|
7
8
|
createProjectionTargetLossMatrix,
|
|
8
9
|
createNativeSourcePreservation,
|
|
10
|
+
createPythonAstNativeImporterAdapter,
|
|
9
11
|
createSemanticImportSidecar,
|
|
10
12
|
diffNativeSources,
|
|
11
13
|
importExternalSemanticIndex,
|
|
@@ -73,6 +75,13 @@ const matrixStart = performance.now();
|
|
|
73
75
|
const coverageMatrix = createNativeImportCoverageMatrix({ imports: nativeImportResults });
|
|
74
76
|
const matrixDurationMs = performance.now() - matrixStart;
|
|
75
77
|
|
|
78
|
+
const parserFormatMatrixStart = performance.now();
|
|
79
|
+
const parserFormatMatrix = createNativeParserAstFormatMatrix({
|
|
80
|
+
imports: nativeImportResults,
|
|
81
|
+
adapters: [estreeAdapter, createPythonAstNativeImporterAdapter()]
|
|
82
|
+
});
|
|
83
|
+
const parserFormatMatrixDurationMs = performance.now() - parserFormatMatrixStart;
|
|
84
|
+
|
|
76
85
|
const projectionMatrixStart = performance.now();
|
|
77
86
|
const projectionLossMatrix = createProjectionTargetLossMatrix({ imports: nativeImportResults });
|
|
78
87
|
const projectionMatrixDurationMs = performance.now() - projectionMatrixStart;
|
|
@@ -228,6 +237,10 @@ console.log(JSON.stringify({
|
|
|
228
237
|
adapterCoverageTokenGaps: coverageMatrix.summary.adapterCoverage.gaps.tokens ?? 0,
|
|
229
238
|
adapterCoverageReferenceGaps: coverageMatrix.summary.adapterCoverage.gaps.references ?? 0,
|
|
230
239
|
coverageMatrixDurationMs: Number(matrixDurationMs.toFixed(2)),
|
|
240
|
+
parserFormatMatrixFormats: parserFormatMatrix.summary.formats,
|
|
241
|
+
parserFormatMatrixImports: parserFormatMatrix.summary.imports,
|
|
242
|
+
parserFormatMatrixNativeAstNodes: parserFormatMatrix.summary.nativeAstNodes,
|
|
243
|
+
parserFormatMatrixDurationMs: Number(parserFormatMatrixDurationMs.toFixed(2)),
|
|
231
244
|
projectionMatrixLanguages: projectionLossMatrix.summary.languages,
|
|
232
245
|
projectionMatrixMissingAdapters: projectionLossMatrix.summary.missingAdapters,
|
|
233
246
|
projectionMatrixUnsupportedTargetFeatures: projectionLossMatrix.summary.unsupportedTargetFeatures,
|
package/dist/index.d.ts
CHANGED
|
@@ -236,6 +236,88 @@ export interface NativeImportLanguageProfile {
|
|
|
236
236
|
readonly notes: readonly string[];
|
|
237
237
|
}
|
|
238
238
|
|
|
239
|
+
export type NativeParserAstFormatKind =
|
|
240
|
+
| 'abstract-ast'
|
|
241
|
+
| 'concrete-syntax-tree'
|
|
242
|
+
| 'compiler-ast'
|
|
243
|
+
| 'semantic-index'
|
|
244
|
+
| string;
|
|
245
|
+
|
|
246
|
+
export interface NativeParserAstFormatProfile {
|
|
247
|
+
readonly id: string;
|
|
248
|
+
readonly aliases: readonly string[];
|
|
249
|
+
readonly kind: NativeParserAstFormatKind;
|
|
250
|
+
readonly languages: readonly (FrontierSourceLanguage | 'mixed' | string)[];
|
|
251
|
+
readonly parserAdapters: readonly string[];
|
|
252
|
+
readonly exactness: NativeImporterAdapterExactness;
|
|
253
|
+
readonly sourceRangeModel: string;
|
|
254
|
+
readonly preservesTokens: boolean;
|
|
255
|
+
readonly preservesTrivia: boolean;
|
|
256
|
+
readonly supportsIncremental: boolean;
|
|
257
|
+
readonly supportsErrorRecovery: boolean;
|
|
258
|
+
readonly notes: readonly string[];
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export interface NativeParserAstFormatCoverage {
|
|
262
|
+
readonly id: string;
|
|
263
|
+
readonly kind: NativeParserAstFormatKind;
|
|
264
|
+
readonly languages: readonly (FrontierSourceLanguage | 'mixed' | string)[];
|
|
265
|
+
readonly parserAdapters: readonly string[];
|
|
266
|
+
readonly exactness: NativeImporterAdapterExactness;
|
|
267
|
+
readonly sourceRangeModel: string;
|
|
268
|
+
readonly preservesTokens: boolean;
|
|
269
|
+
readonly preservesTrivia: boolean;
|
|
270
|
+
readonly supportsIncremental: boolean;
|
|
271
|
+
readonly supportsErrorRecovery: boolean;
|
|
272
|
+
readonly notes: readonly string[];
|
|
273
|
+
readonly adapters: {
|
|
274
|
+
readonly total: number;
|
|
275
|
+
readonly ids: readonly string[];
|
|
276
|
+
readonly parsers: readonly string[];
|
|
277
|
+
readonly effectiveCapabilities: Readonly<Record<string, number>>;
|
|
278
|
+
};
|
|
279
|
+
readonly imports: {
|
|
280
|
+
readonly total: number;
|
|
281
|
+
readonly sourcePaths: readonly string[];
|
|
282
|
+
readonly readiness: SemanticMergeReadiness;
|
|
283
|
+
readonly nativeAstNodes: number;
|
|
284
|
+
readonly symbols: number;
|
|
285
|
+
readonly sourceMapMappings: number;
|
|
286
|
+
readonly losses: number;
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export interface NativeParserAstFormatMatrix {
|
|
291
|
+
readonly kind: 'frontier.lang.nativeParserAstFormatMatrix';
|
|
292
|
+
readonly version: 1;
|
|
293
|
+
readonly generatedAt: number;
|
|
294
|
+
readonly formats: readonly NativeParserAstFormatCoverage[];
|
|
295
|
+
readonly summary: {
|
|
296
|
+
readonly formats: number;
|
|
297
|
+
readonly adapterSlots: number;
|
|
298
|
+
readonly adapters: number;
|
|
299
|
+
readonly imports: number;
|
|
300
|
+
readonly nativeAstNodes: number;
|
|
301
|
+
readonly symbols: number;
|
|
302
|
+
readonly sourceMapMappings: number;
|
|
303
|
+
readonly losses: number;
|
|
304
|
+
readonly byKind: Readonly<Record<string, number>>;
|
|
305
|
+
readonly byReadiness: Readonly<Record<string, number>>;
|
|
306
|
+
readonly effectiveCapabilities: Readonly<Record<string, number>>;
|
|
307
|
+
};
|
|
308
|
+
readonly metadata: {
|
|
309
|
+
readonly note: string;
|
|
310
|
+
readonly profileIds: readonly string[];
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
export interface NativeParserAstFormatMatrixOptions {
|
|
315
|
+
readonly formats?: readonly NativeParserAstFormatProfile[];
|
|
316
|
+
readonly imports?: readonly NativeSourceImportResult[];
|
|
317
|
+
readonly adapters?: readonly NativeImporterAdapter[];
|
|
318
|
+
readonly generatedAt?: number;
|
|
319
|
+
}
|
|
320
|
+
|
|
239
321
|
export interface NativeImporterAdapterCoverageAggregate {
|
|
240
322
|
readonly total: number;
|
|
241
323
|
readonly declared: Readonly<Record<string, number>>;
|
|
@@ -1283,6 +1365,29 @@ export interface TypeScriptCompilerNativeImporterAdapterOptions {
|
|
|
1283
1365
|
readonly includeTokens?: boolean;
|
|
1284
1366
|
}
|
|
1285
1367
|
|
|
1368
|
+
export interface PythonAstNativeImporterAdapterOptions {
|
|
1369
|
+
readonly id?: string;
|
|
1370
|
+
readonly language?: FrontierSourceLanguage;
|
|
1371
|
+
readonly parser?: string;
|
|
1372
|
+
readonly version?: string;
|
|
1373
|
+
readonly capabilities?: readonly string[];
|
|
1374
|
+
readonly coverage?: NativeImporterAdapterCoverageInput;
|
|
1375
|
+
readonly supportedExtensions?: readonly string[];
|
|
1376
|
+
readonly diagnostics?: readonly NativeImporterAdapterDiagnostic[];
|
|
1377
|
+
readonly ast?: unknown;
|
|
1378
|
+
readonly parse?: (sourceText: string, options: Record<string, unknown>) => unknown;
|
|
1379
|
+
readonly parserModule?: { readonly parse: (sourceText: string, options: Record<string, unknown>) => unknown };
|
|
1380
|
+
readonly pythonAst?: { readonly parse: (sourceText: string, options: Record<string, unknown>) => unknown };
|
|
1381
|
+
readonly parserOptions?: Record<string, unknown>;
|
|
1382
|
+
readonly mode?: 'exec' | 'eval' | 'single' | 'func_type' | string;
|
|
1383
|
+
readonly pythonVersion?: string;
|
|
1384
|
+
readonly featureVersion?: string | number;
|
|
1385
|
+
readonly typeComments?: boolean;
|
|
1386
|
+
readonly optimize?: number;
|
|
1387
|
+
readonly includeAttributes?: boolean;
|
|
1388
|
+
readonly maxNodes?: number;
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1286
1391
|
export interface TreeSitterNativeImporterAdapterOptions {
|
|
1287
1392
|
readonly id?: string;
|
|
1288
1393
|
readonly language?: FrontierSourceLanguage;
|
|
@@ -1625,6 +1730,8 @@ export declare const ProjectionTargetLossClasses: readonly ProjectionTargetLossC
|
|
|
1625
1730
|
export declare const NativeImportReadinessBySeverity: Readonly<Record<NativeImportLossSummary['highestSeverity'], SemanticMergeReadiness>>;
|
|
1626
1731
|
export declare const NativeImportFeatureEvidencePolicies: Readonly<Record<string, NativeImportFeatureEvidencePolicy>>;
|
|
1627
1732
|
export declare const NativeImportLanguageProfiles: readonly NativeImportLanguageProfile[];
|
|
1733
|
+
export declare const NativeParserAstFormatProfiles: readonly NativeParserAstFormatProfile[];
|
|
1734
|
+
export declare const NativeParserAstFormats: readonly string[];
|
|
1628
1735
|
export declare const ExternalSemanticIndexFormats: readonly ExternalSemanticIndexFormat[];
|
|
1629
1736
|
export declare function normalizeCompileTarget(target?: string): FrontierCompileTarget;
|
|
1630
1737
|
export declare function compileFrontierSource(source: string, options?: FrontierCompileOptions): FrontierCompileResult;
|
|
@@ -1641,6 +1748,8 @@ export declare function summarizeNativeImportLosses(losses?: readonly NativeAstL
|
|
|
1641
1748
|
export declare function classifyNativeImportReadiness(losses?: readonly NativeAstLossRecord[], options?: NativeImportLossSummaryOptions): NativeImportReadinessClassification;
|
|
1642
1749
|
export declare function classifyNativeImportRoundtripReadiness(importResult: NativeSourceImportResult | NativeProjectImportResult, options?: NativeImportRoundtripReadinessOptions): NativeImportRoundtripReadinessClassification;
|
|
1643
1750
|
export declare function createNativeImportCoverageMatrix(options?: NativeImportCoverageMatrixOptions): NativeImportCoverageMatrix;
|
|
1751
|
+
export declare function getNativeParserAstFormatProfile(format?: string): NativeParserAstFormatProfile | undefined;
|
|
1752
|
+
export declare function createNativeParserAstFormatMatrix(options?: NativeParserAstFormatMatrixOptions): NativeParserAstFormatMatrix;
|
|
1644
1753
|
export declare function createProjectionTargetLossMatrix(options?: ProjectionTargetLossMatrixOptions): ProjectionTargetLossMatrix;
|
|
1645
1754
|
export declare function createNativeSourcePreservation(options: CreateNativeSourcePreservationOptions): NativeSourcePreservation;
|
|
1646
1755
|
export declare function createSemanticImportSidecar(importResult: NativeSourceImportResult | NativeProjectImportResult, options?: SemanticImportSidecarOptions): SemanticImportSidecar;
|
|
@@ -1648,6 +1757,7 @@ export declare function createNativeImportResultContract(importResult: NativeSou
|
|
|
1648
1757
|
export declare function createEstreeNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1649
1758
|
export declare function createBabelNativeImporterAdapter(options?: JavaScriptNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1650
1759
|
export declare function createTypeScriptCompilerNativeImporterAdapter(options?: TypeScriptCompilerNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1760
|
+
export declare function createPythonAstNativeImporterAdapter(options?: PythonAstNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1651
1761
|
export declare function createTreeSitterNativeImporterAdapter(options?: TreeSitterNativeImporterAdapterOptions): NativeImporterAdapter;
|
|
1652
1762
|
export declare function runNativeImporterAdapter(adapter: NativeImporterAdapter, input: RunNativeImporterAdapterOptions): Promise<NativeImporterAdapterImportResult>;
|
|
1653
1763
|
export declare function runNativeTargetProjectionAdapter(adapter: NativeTargetProjectionAdapter, input: NativeTargetProjectionAdapterInput): NativeTargetProjectionResult;
|
package/dist/index.js
CHANGED
|
@@ -333,6 +333,100 @@ export const NativeImportLanguageProfiles = Object.freeze([
|
|
|
333
333
|
nativeImportLanguageProfile('r', { aliases: ['R'], extensions: ['.r', '.R'], parserAdapters: ['r-parser', 'tree-sitter'], lossKinds: ['declarationOnlyCoverage', 'opaqueNative', 'dynamicRuntime', 'sourceMapApproximation', 'sourcePreservation'] })
|
|
334
334
|
]);
|
|
335
335
|
|
|
336
|
+
export const NativeParserAstFormatProfiles = Object.freeze([
|
|
337
|
+
nativeParserAstFormatProfile('estree', {
|
|
338
|
+
kind: 'abstract-ast',
|
|
339
|
+
languages: ['javascript'],
|
|
340
|
+
parserAdapters: ['estree'],
|
|
341
|
+
exactness: 'exact-parser-ast',
|
|
342
|
+
sourceRangeModel: 'loc-range',
|
|
343
|
+
preservesTokens: false,
|
|
344
|
+
preservesTrivia: false,
|
|
345
|
+
supportsErrorRecovery: false,
|
|
346
|
+
notes: ['Community JavaScript AST shape used by many JS tooling parsers.']
|
|
347
|
+
}),
|
|
348
|
+
nativeParserAstFormatProfile('babel', {
|
|
349
|
+
kind: 'abstract-ast',
|
|
350
|
+
languages: ['javascript', 'typescript'],
|
|
351
|
+
parserAdapters: ['babel'],
|
|
352
|
+
exactness: 'exact-parser-ast',
|
|
353
|
+
sourceRangeModel: 'loc-range',
|
|
354
|
+
preservesTokens: false,
|
|
355
|
+
preservesTrivia: false,
|
|
356
|
+
supportsErrorRecovery: true,
|
|
357
|
+
notes: ['Babel-compatible ESTree-like ASTs can report parser errors when error recovery is enabled.']
|
|
358
|
+
}),
|
|
359
|
+
nativeParserAstFormatProfile('typescript-compiler-api', {
|
|
360
|
+
kind: 'compiler-ast',
|
|
361
|
+
languages: ['typescript', 'javascript'],
|
|
362
|
+
parserAdapters: ['typescript-compiler-api'],
|
|
363
|
+
exactness: 'exact-parser-ast',
|
|
364
|
+
sourceRangeModel: 'pos-end',
|
|
365
|
+
preservesTokens: false,
|
|
366
|
+
preservesTrivia: false,
|
|
367
|
+
supportsErrorRecovery: true,
|
|
368
|
+
notes: ['TypeScript SourceFile trees can be parsed without a full Program; richer type/checker evidence remains host-owned.']
|
|
369
|
+
}),
|
|
370
|
+
nativeParserAstFormatProfile('python-ast', {
|
|
371
|
+
kind: 'abstract-ast',
|
|
372
|
+
languages: ['python'],
|
|
373
|
+
parserAdapters: ['python-ast'],
|
|
374
|
+
exactness: 'exact-parser-ast',
|
|
375
|
+
sourceRangeModel: 'lineno-col-offset',
|
|
376
|
+
preservesTokens: false,
|
|
377
|
+
preservesTrivia: false,
|
|
378
|
+
supportsErrorRecovery: false,
|
|
379
|
+
notes: ['Python stdlib AST exposes versioned abstract grammar and source locations, but not formatting trivia.']
|
|
380
|
+
}),
|
|
381
|
+
nativeParserAstFormatProfile('tree-sitter', {
|
|
382
|
+
kind: 'concrete-syntax-tree',
|
|
383
|
+
languages: ['mixed'],
|
|
384
|
+
parserAdapters: ['tree-sitter'],
|
|
385
|
+
exactness: 'parser-tree',
|
|
386
|
+
sourceRangeModel: 'row-column',
|
|
387
|
+
preservesTokens: false,
|
|
388
|
+
preservesTrivia: false,
|
|
389
|
+
supportsIncremental: true,
|
|
390
|
+
supportsErrorRecovery: true,
|
|
391
|
+
notes: ['Tree-sitter provides cross-language concrete syntax trees; language-specific queries still decide semantic richness.']
|
|
392
|
+
}),
|
|
393
|
+
nativeParserAstFormatProfile('libcst', {
|
|
394
|
+
kind: 'concrete-syntax-tree',
|
|
395
|
+
languages: ['python'],
|
|
396
|
+
parserAdapters: ['libcst'],
|
|
397
|
+
exactness: 'parser-tree',
|
|
398
|
+
sourceRangeModel: 'metadata-position-provider',
|
|
399
|
+
preservesTokens: true,
|
|
400
|
+
preservesTrivia: true,
|
|
401
|
+
supportsErrorRecovery: false,
|
|
402
|
+
notes: ['LibCST-style trees preserve formatting and are best treated as host-owned evidence until normalized explicitly.']
|
|
403
|
+
}),
|
|
404
|
+
nativeParserAstFormatProfile('scip', {
|
|
405
|
+
kind: 'semantic-index',
|
|
406
|
+
languages: ['mixed'],
|
|
407
|
+
parserAdapters: ['scip'],
|
|
408
|
+
exactness: 'loss-aware-native-ast',
|
|
409
|
+
sourceRangeModel: 'range-tuples',
|
|
410
|
+
preservesTokens: false,
|
|
411
|
+
preservesTrivia: false,
|
|
412
|
+
supportsErrorRecovery: false,
|
|
413
|
+
notes: ['SCIP is semantic index evidence rather than a full parser AST; it is useful for symbols/references and source maps.']
|
|
414
|
+
}),
|
|
415
|
+
nativeParserAstFormatProfile('lsif', {
|
|
416
|
+
kind: 'semantic-index',
|
|
417
|
+
languages: ['mixed'],
|
|
418
|
+
parserAdapters: ['lsif'],
|
|
419
|
+
exactness: 'loss-aware-native-ast',
|
|
420
|
+
sourceRangeModel: 'lsp-ranges',
|
|
421
|
+
preservesTokens: false,
|
|
422
|
+
preservesTrivia: false,
|
|
423
|
+
supportsErrorRecovery: false,
|
|
424
|
+
notes: ['LSIF graph dumps are semantic/source-map evidence, not complete native ASTs.']
|
|
425
|
+
})
|
|
426
|
+
]);
|
|
427
|
+
|
|
428
|
+
export const NativeParserAstFormats = Object.freeze(NativeParserAstFormatProfiles.map((profile) => profile.id));
|
|
429
|
+
|
|
336
430
|
export const ExternalSemanticIndexFormats = Object.freeze([
|
|
337
431
|
'frontier-semantic-index',
|
|
338
432
|
'scip',
|
|
@@ -2125,6 +2219,57 @@ export function createNativeImportCoverageMatrix(input = {}) {
|
|
|
2125
2219
|
};
|
|
2126
2220
|
}
|
|
2127
2221
|
|
|
2222
|
+
export function getNativeParserAstFormatProfile(format) {
|
|
2223
|
+
const normalized = normalizeParserAstFormatId(format);
|
|
2224
|
+
return NativeParserAstFormatProfiles.find((profile) => profile.id === normalized || profile.aliases.includes(normalized));
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
export function createNativeParserAstFormatMatrix(input = {}) {
|
|
2228
|
+
const imports = input.imports ?? [];
|
|
2229
|
+
const adapters = input.adapters ?? [];
|
|
2230
|
+
const profiles = mergeNativeParserAstFormatProfiles(input.formats ?? NativeParserAstFormatProfiles, imports, adapters);
|
|
2231
|
+
const formats = profiles.map((profile) => nativeParserAstFormatCoverageForProfile(profile, imports, adapters));
|
|
2232
|
+
const summary = formats.reduce((totals, entry) => {
|
|
2233
|
+
totals.formats += 1;
|
|
2234
|
+
totals.adapterSlots += entry.parserAdapters.length;
|
|
2235
|
+
totals.adapters += entry.adapters.total;
|
|
2236
|
+
totals.imports += entry.imports.total;
|
|
2237
|
+
totals.nativeAstNodes += entry.imports.nativeAstNodes;
|
|
2238
|
+
totals.symbols += entry.imports.symbols;
|
|
2239
|
+
totals.sourceMapMappings += entry.imports.sourceMapMappings;
|
|
2240
|
+
totals.losses += entry.imports.losses;
|
|
2241
|
+
totals.byKind[entry.kind] = (totals.byKind[entry.kind] ?? 0) + 1;
|
|
2242
|
+
totals.byReadiness[entry.imports.readiness] = (totals.byReadiness[entry.imports.readiness] ?? 0) + 1;
|
|
2243
|
+
for (const [capability, count] of Object.entries(entry.adapters.effectiveCapabilities)) {
|
|
2244
|
+
totals.effectiveCapabilities[capability] = (totals.effectiveCapabilities[capability] ?? 0) + count;
|
|
2245
|
+
}
|
|
2246
|
+
return totals;
|
|
2247
|
+
}, {
|
|
2248
|
+
formats: 0,
|
|
2249
|
+
adapterSlots: 0,
|
|
2250
|
+
adapters: 0,
|
|
2251
|
+
imports: 0,
|
|
2252
|
+
nativeAstNodes: 0,
|
|
2253
|
+
symbols: 0,
|
|
2254
|
+
sourceMapMappings: 0,
|
|
2255
|
+
losses: 0,
|
|
2256
|
+
byKind: {},
|
|
2257
|
+
byReadiness: {},
|
|
2258
|
+
effectiveCapabilities: {}
|
|
2259
|
+
});
|
|
2260
|
+
return {
|
|
2261
|
+
kind: 'frontier.lang.nativeParserAstFormatMatrix',
|
|
2262
|
+
version: 1,
|
|
2263
|
+
generatedAt: input.generatedAt ?? Date.now(),
|
|
2264
|
+
formats,
|
|
2265
|
+
summary,
|
|
2266
|
+
metadata: {
|
|
2267
|
+
note: 'Parser AST format coverage describes normalization evidence and host-parser obligations; it is not a lossless portability claim.',
|
|
2268
|
+
profileIds: profiles.map((profile) => profile.id)
|
|
2269
|
+
}
|
|
2270
|
+
};
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2128
2273
|
export function createProjectionTargetLossMatrix(input = {}) {
|
|
2129
2274
|
const imports = input.imports ?? [];
|
|
2130
2275
|
const adapters = input.adapters ?? [];
|
|
@@ -2435,6 +2580,57 @@ export function createTypeScriptCompilerNativeImporterAdapter(options = {}) {
|
|
|
2435
2580
|
};
|
|
2436
2581
|
}
|
|
2437
2582
|
|
|
2583
|
+
export function createPythonAstNativeImporterAdapter(options = {}) {
|
|
2584
|
+
return {
|
|
2585
|
+
id: options.id ?? 'frontier.python-ast-native-importer',
|
|
2586
|
+
language: options.language ?? 'python',
|
|
2587
|
+
parser: options.parser ?? 'python-ast',
|
|
2588
|
+
version: options.version,
|
|
2589
|
+
capabilities: uniqueStrings(['nativeAst', 'semanticIndex', 'sourceMaps', 'diagnostics', ...(options.capabilities ?? [])]),
|
|
2590
|
+
coverage: nativeImporterAdapterCoverage({
|
|
2591
|
+
exactness: 'exact-parser-ast',
|
|
2592
|
+
exactAst: true,
|
|
2593
|
+
tokens: false,
|
|
2594
|
+
trivia: false,
|
|
2595
|
+
diagnostics: true,
|
|
2596
|
+
sourceRanges: true,
|
|
2597
|
+
generatedRanges: false,
|
|
2598
|
+
semanticCoverage: declarationSemanticCoverage(),
|
|
2599
|
+
notes: [
|
|
2600
|
+
'Normalizes caller-owned Python stdlib ast trees into native AST nodes and declaration-level semantic index records.',
|
|
2601
|
+
'Python ast does not preserve comments, whitespace, or concrete formatting; use LibCST/parso-style host evidence for round-trip trivia.'
|
|
2602
|
+
]
|
|
2603
|
+
}, options.coverage),
|
|
2604
|
+
supportedExtensions: options.supportedExtensions ?? ['.py', '.pyi'],
|
|
2605
|
+
diagnostics: options.diagnostics,
|
|
2606
|
+
parse(input) {
|
|
2607
|
+
const parsed = input.options?.ast
|
|
2608
|
+
?? input.options?.nativeAst
|
|
2609
|
+
?? options.ast
|
|
2610
|
+
?? parsePythonAstSource(input, options);
|
|
2611
|
+
const root = pythonAstRoot(parsed);
|
|
2612
|
+
if (!root) {
|
|
2613
|
+
return missingInjectedParserResult(input, {
|
|
2614
|
+
parser: options.parser ?? 'python-ast',
|
|
2615
|
+
adapterId: options.id ?? 'frontier.python-ast-native-importer',
|
|
2616
|
+
message: 'createPythonAstNativeImporterAdapter requires an injected Python AST object, parserModule.parse function, parse function, or adapterOptions.ast.'
|
|
2617
|
+
});
|
|
2618
|
+
}
|
|
2619
|
+
const parseDiagnostics = normalizeParserErrors(parsed?.errors ?? parsed?.diagnostics, input, {
|
|
2620
|
+
parser: options.parser ?? 'python-ast'
|
|
2621
|
+
});
|
|
2622
|
+
return createNativeImportFromPythonAst(root, input, {
|
|
2623
|
+
parser: options.parser ?? 'python-ast',
|
|
2624
|
+
astFormat: 'python-ast',
|
|
2625
|
+
maxNodes: options.maxNodes,
|
|
2626
|
+
diagnostics: parseDiagnostics,
|
|
2627
|
+
pythonVersion: options.pythonVersion ?? input.options?.pythonVersion ?? parsed?.pythonVersion,
|
|
2628
|
+
includeAttributes: options.includeAttributes ?? input.options?.includeAttributes
|
|
2629
|
+
});
|
|
2630
|
+
}
|
|
2631
|
+
};
|
|
2632
|
+
}
|
|
2633
|
+
|
|
2438
2634
|
export function createTreeSitterNativeImporterAdapter(options = {}) {
|
|
2439
2635
|
return {
|
|
2440
2636
|
id: options.id ?? `frontier.tree-sitter-${idFragment(options.language ?? 'source')}-native-importer`,
|
|
@@ -6767,6 +6963,132 @@ function nativeImportLanguageProfile(language, input = {}) {
|
|
|
6767
6963
|
});
|
|
6768
6964
|
}
|
|
6769
6965
|
|
|
6966
|
+
function nativeParserAstFormatProfile(id, input = {}) {
|
|
6967
|
+
return Object.freeze({
|
|
6968
|
+
id,
|
|
6969
|
+
aliases: Object.freeze(uniqueStrings(input.aliases ?? [])),
|
|
6970
|
+
kind: input.kind ?? 'abstract-ast',
|
|
6971
|
+
languages: Object.freeze(uniqueStrings(input.languages ?? [])),
|
|
6972
|
+
parserAdapters: Object.freeze(uniqueStrings(input.parserAdapters ?? [id])),
|
|
6973
|
+
exactness: input.exactness ?? 'unknown',
|
|
6974
|
+
sourceRangeModel: input.sourceRangeModel ?? 'unknown',
|
|
6975
|
+
preservesTokens: Boolean(input.preservesTokens),
|
|
6976
|
+
preservesTrivia: Boolean(input.preservesTrivia),
|
|
6977
|
+
supportsIncremental: Boolean(input.supportsIncremental),
|
|
6978
|
+
supportsErrorRecovery: Boolean(input.supportsErrorRecovery),
|
|
6979
|
+
notes: Object.freeze(uniqueStrings(input.notes ?? []))
|
|
6980
|
+
});
|
|
6981
|
+
}
|
|
6982
|
+
|
|
6983
|
+
function normalizeParserAstFormatId(format) {
|
|
6984
|
+
return String(format ?? '').trim().toLowerCase().replace(/[_\s]+/g, '-');
|
|
6985
|
+
}
|
|
6986
|
+
|
|
6987
|
+
function mergeNativeParserAstFormatProfiles(profiles, imports, adapters) {
|
|
6988
|
+
const byId = new Map((profiles ?? []).map((profile) => [normalizeParserAstFormatId(profile.id ?? profile), nativeParserAstFormatProfile(normalizeParserAstFormatId(profile.id ?? profile), profile)]));
|
|
6989
|
+
for (const adapter of adapters ?? []) {
|
|
6990
|
+
const summary = safeNativeImporterAdapterSummary(adapter);
|
|
6991
|
+
if (!summary) continue;
|
|
6992
|
+
const formatId = parserAstFormatIdForParser(summary.parser);
|
|
6993
|
+
if (!byId.has(formatId)) {
|
|
6994
|
+
byId.set(formatId, nativeParserAstFormatProfile(formatId, {
|
|
6995
|
+
languages: [summary.language],
|
|
6996
|
+
parserAdapters: [summary.parser],
|
|
6997
|
+
exactness: summary.coverage.exactness,
|
|
6998
|
+
sourceRangeModel: summary.coverage.sourceRanges ? 'adapter-reported' : 'unknown'
|
|
6999
|
+
}));
|
|
7000
|
+
}
|
|
7001
|
+
}
|
|
7002
|
+
for (const imported of imports ?? []) {
|
|
7003
|
+
const formatId = parserAstFormatIdForImport(imported);
|
|
7004
|
+
if (formatId && !byId.has(formatId)) {
|
|
7005
|
+
byId.set(formatId, nativeParserAstFormatProfile(formatId, {
|
|
7006
|
+
languages: [imported.language].filter(Boolean),
|
|
7007
|
+
parserAdapters: [imported.parser ?? imported.nativeAst?.parser ?? formatId],
|
|
7008
|
+
exactness: 'unknown',
|
|
7009
|
+
sourceRangeModel: (imported.sourceMaps ?? []).some((sourceMap) => sourceMap.mappings?.some((mapping) => mapping.sourceSpan)) ? 'adapter-reported' : 'unknown'
|
|
7010
|
+
}));
|
|
7011
|
+
}
|
|
7012
|
+
}
|
|
7013
|
+
return [...byId.values()].sort((left, right) => left.id.localeCompare(right.id));
|
|
7014
|
+
}
|
|
7015
|
+
|
|
7016
|
+
function nativeParserAstFormatCoverageForProfile(profile, imports, adapters) {
|
|
7017
|
+
const formatIds = new Set([profile.id, ...profile.aliases].map(normalizeParserAstFormatId));
|
|
7018
|
+
const adapterParsers = new Set(profile.parserAdapters.map(parserAstFormatIdForParser));
|
|
7019
|
+
const matchingAdapters = (adapters ?? [])
|
|
7020
|
+
.map((adapter) => safeNativeImporterAdapterSummary(adapter))
|
|
7021
|
+
.filter(Boolean)
|
|
7022
|
+
.filter((adapter) => formatIds.has(parserAstFormatIdForParser(adapter.parser)) || adapterParsers.has(parserAstFormatIdForParser(adapter.parser)));
|
|
7023
|
+
const matchingImports = (imports ?? [])
|
|
7024
|
+
.filter((imported) => {
|
|
7025
|
+
const formatId = parserAstFormatIdForImport(imported);
|
|
7026
|
+
return formatId && (formatIds.has(formatId) || adapterParsers.has(formatId));
|
|
7027
|
+
});
|
|
7028
|
+
const effectiveCapabilities = {};
|
|
7029
|
+
for (const adapter of matchingAdapters) {
|
|
7030
|
+
for (const row of adapter.coverage.capabilityEvidence?.capabilities ?? []) {
|
|
7031
|
+
if (row.effective) effectiveCapabilities[row.capability] = (effectiveCapabilities[row.capability] ?? 0) + 1;
|
|
7032
|
+
}
|
|
7033
|
+
}
|
|
7034
|
+
const readiness = matchingImports.reduce(
|
|
7035
|
+
(current, imported) => maxSemanticMergeReadiness(current, nativeImportReadiness(imported)),
|
|
7036
|
+
matchingImports.length ? 'ready' : 'needs-review'
|
|
7037
|
+
);
|
|
7038
|
+
return {
|
|
7039
|
+
id: profile.id,
|
|
7040
|
+
kind: profile.kind,
|
|
7041
|
+
languages: profile.languages,
|
|
7042
|
+
parserAdapters: profile.parserAdapters,
|
|
7043
|
+
exactness: profile.exactness,
|
|
7044
|
+
sourceRangeModel: profile.sourceRangeModel,
|
|
7045
|
+
preservesTokens: profile.preservesTokens,
|
|
7046
|
+
preservesTrivia: profile.preservesTrivia,
|
|
7047
|
+
supportsIncremental: profile.supportsIncremental,
|
|
7048
|
+
supportsErrorRecovery: profile.supportsErrorRecovery,
|
|
7049
|
+
notes: profile.notes,
|
|
7050
|
+
adapters: {
|
|
7051
|
+
total: matchingAdapters.length,
|
|
7052
|
+
ids: matchingAdapters.map((adapter) => adapter.id),
|
|
7053
|
+
parsers: uniqueStrings(matchingAdapters.map((adapter) => adapter.parser)),
|
|
7054
|
+
effectiveCapabilities
|
|
7055
|
+
},
|
|
7056
|
+
imports: {
|
|
7057
|
+
total: matchingImports.length,
|
|
7058
|
+
sourcePaths: matchingImports.map((imported) => imported.sourcePath).filter(Boolean),
|
|
7059
|
+
readiness,
|
|
7060
|
+
nativeAstNodes: matchingImports.reduce((sum, imported) => sum + Object.keys(imported.nativeAst?.nodes ?? {}).length, 0),
|
|
7061
|
+
symbols: matchingImports.reduce((sum, imported) => sum + (imported.semanticIndex?.symbols?.length ?? 0), 0),
|
|
7062
|
+
sourceMapMappings: matchingImports.reduce((sum, imported) => sum + (imported.sourceMaps ?? []).reduce((mapSum, sourceMap) => mapSum + (sourceMap.mappings?.length ?? 0), 0), 0),
|
|
7063
|
+
losses: matchingImports.reduce((sum, imported) => sum + (imported.losses?.length ?? 0), 0)
|
|
7064
|
+
}
|
|
7065
|
+
};
|
|
7066
|
+
}
|
|
7067
|
+
|
|
7068
|
+
function parserAstFormatIdForParser(parser) {
|
|
7069
|
+
const text = normalizeParserAstFormatId(parser);
|
|
7070
|
+
if (text.includes('typescript')) return 'typescript-compiler-api';
|
|
7071
|
+
if (text.includes('python') && text.includes('ast')) return 'python-ast';
|
|
7072
|
+
if (text.includes('tree-sitter') || text.includes('treesitter')) return 'tree-sitter';
|
|
7073
|
+
if (text.includes('babel')) return 'babel';
|
|
7074
|
+
if (text.includes('estree')) return 'estree';
|
|
7075
|
+
if (text.includes('libcst')) return 'libcst';
|
|
7076
|
+
if (text.includes('scip')) return 'scip';
|
|
7077
|
+
if (text.includes('lsif')) return 'lsif';
|
|
7078
|
+
return text || 'unknown';
|
|
7079
|
+
}
|
|
7080
|
+
|
|
7081
|
+
function parserAstFormatIdForImport(imported) {
|
|
7082
|
+
return parserAstFormatIdForParser(
|
|
7083
|
+
imported?.metadata?.adapter?.parser
|
|
7084
|
+
?? imported?.metadata?.astFormat
|
|
7085
|
+
?? imported?.nativeAst?.metadata?.astFormat
|
|
7086
|
+
?? imported?.nativeAst?.parser
|
|
7087
|
+
?? imported?.parser
|
|
7088
|
+
?? imported?.metadata?.parser
|
|
7089
|
+
);
|
|
7090
|
+
}
|
|
7091
|
+
|
|
6770
7092
|
function mergeNativeImportProfiles(languages, imports, adapters, targetAdapters = []) {
|
|
6771
7093
|
const profilesByLanguage = new Map();
|
|
6772
7094
|
for (const profile of languages) {
|
|
@@ -7764,6 +8086,23 @@ function parseTreeSitterSource(input, options) {
|
|
|
7764
8086
|
return undefined;
|
|
7765
8087
|
}
|
|
7766
8088
|
|
|
8089
|
+
function parsePythonAstSource(input, options) {
|
|
8090
|
+
const parse = options.parse ?? options.parserModule?.parse ?? options.pythonAst?.parse;
|
|
8091
|
+
if (typeof parse !== 'function') return undefined;
|
|
8092
|
+
const parserOptions = {
|
|
8093
|
+
sourcePath: input.sourcePath,
|
|
8094
|
+
filename: input.sourcePath,
|
|
8095
|
+
mode: options.mode ?? input.options?.mode ?? 'exec',
|
|
8096
|
+
typeComments: options.typeComments ?? input.options?.typeComments,
|
|
8097
|
+
featureVersion: options.featureVersion ?? input.options?.featureVersion,
|
|
8098
|
+
optimize: options.optimize ?? input.options?.optimize,
|
|
8099
|
+
includeAttributes: options.includeAttributes ?? input.options?.includeAttributes,
|
|
8100
|
+
...(options.parserOptions ?? {}),
|
|
8101
|
+
...(input.options?.parserOptions ?? {})
|
|
8102
|
+
};
|
|
8103
|
+
return parse(input.sourceText, parserOptions);
|
|
8104
|
+
}
|
|
8105
|
+
|
|
7767
8106
|
function createNativeImportFromSyntaxAst(ast, input, options) {
|
|
7768
8107
|
const root = normalizeSyntaxAstRoot(ast, options.astFormat);
|
|
7769
8108
|
if (!root) {
|
|
@@ -7824,6 +8163,36 @@ function createNativeImportFromTypeScriptAst(sourceFile, input, options) {
|
|
|
7824
8163
|
};
|
|
7825
8164
|
}
|
|
7826
8165
|
|
|
8166
|
+
function createNativeImportFromPythonAst(root, input, options) {
|
|
8167
|
+
const context = createAstNormalizationContext(input, options);
|
|
8168
|
+
visitPythonAstNode(root, context, 'root');
|
|
8169
|
+
if (context.truncated) {
|
|
8170
|
+
context.losses.push(truncatedAstLoss(input, context, options));
|
|
8171
|
+
}
|
|
8172
|
+
const semantic = semanticIndexFromNativeDeclarations(context.declarations, input, options);
|
|
8173
|
+
return {
|
|
8174
|
+
rootId: context.rootId,
|
|
8175
|
+
nodes: context.nodes,
|
|
8176
|
+
semanticIndex: semantic.semanticIndex,
|
|
8177
|
+
mappings: semantic.mappings,
|
|
8178
|
+
losses: mergeNativeLosses(context.losses, options.diagnostics?.map((diagnostic, index) => adapterDiagnosticToLoss(diagnostic, index, {
|
|
8179
|
+
id: input.adapterId,
|
|
8180
|
+
version: input.adapterVersion
|
|
8181
|
+
}, input)) ?? []),
|
|
8182
|
+
evidence: semantic.evidence,
|
|
8183
|
+
diagnostics: options.diagnostics,
|
|
8184
|
+
metadata: {
|
|
8185
|
+
astFormat: options.astFormat,
|
|
8186
|
+
parser: options.parser,
|
|
8187
|
+
pythonVersion: options.pythonVersion,
|
|
8188
|
+
includeAttributes: Boolean(options.includeAttributes),
|
|
8189
|
+
normalizedNodeCount: Object.keys(context.nodes).length,
|
|
8190
|
+
declarationCount: context.declarations.length,
|
|
8191
|
+
truncated: context.truncated
|
|
8192
|
+
}
|
|
8193
|
+
};
|
|
8194
|
+
}
|
|
8195
|
+
|
|
7827
8196
|
function createNativeImportFromTreeSitter(root, input, options) {
|
|
7828
8197
|
const context = createAstNormalizationContext(input, options);
|
|
7829
8198
|
visitTreeSitterNode(root, context, 'root');
|
|
@@ -7960,6 +8329,53 @@ function visitTypeScriptAstNode(node, sourceFile, context, propertyPath, ts) {
|
|
|
7960
8329
|
return id;
|
|
7961
8330
|
}
|
|
7962
8331
|
|
|
8332
|
+
function visitPythonAstNode(node, context, propertyPath) {
|
|
8333
|
+
if (!isPythonAstNode(node) || context.truncated) return undefined;
|
|
8334
|
+
if (context.objectIds.has(node)) return context.objectIds.get(node);
|
|
8335
|
+
if (context.counter >= context.maxNodes) {
|
|
8336
|
+
context.truncated = true;
|
|
8337
|
+
return undefined;
|
|
8338
|
+
}
|
|
8339
|
+
const kind = pythonAstKind(node);
|
|
8340
|
+
const span = spanFromPythonAstNode(node, context.input);
|
|
8341
|
+
const id = nativeNodeId(context, kind, { start: { line: span?.startLine, column: span?.startColumn } }, propertyPath);
|
|
8342
|
+
context.objectIds.set(node, id);
|
|
8343
|
+
if (!context.rootId) context.rootId = id;
|
|
8344
|
+
const children = [];
|
|
8345
|
+
for (const [field, value] of pythonAstChildEntries(node)) {
|
|
8346
|
+
if (Array.isArray(value)) {
|
|
8347
|
+
value.forEach((entry, index) => {
|
|
8348
|
+
const childId = visitPythonAstNode(entry, context, `${propertyPath}.${field}[${index}]`);
|
|
8349
|
+
if (childId) children.push(childId);
|
|
8350
|
+
});
|
|
8351
|
+
} else {
|
|
8352
|
+
const childId = visitPythonAstNode(value, context, `${propertyPath}.${field}`);
|
|
8353
|
+
if (childId) children.push(childId);
|
|
8354
|
+
}
|
|
8355
|
+
}
|
|
8356
|
+
const declaration = pythonAstDeclaration(node, kind, id, context.input);
|
|
8357
|
+
const nativeNode = {
|
|
8358
|
+
id,
|
|
8359
|
+
kind,
|
|
8360
|
+
languageKind: `${context.input.language}.${kind}`,
|
|
8361
|
+
span,
|
|
8362
|
+
value: declaration?.name ?? pythonAstNodeValue(node),
|
|
8363
|
+
fields: primitivePythonAstFields(node, kind),
|
|
8364
|
+
children,
|
|
8365
|
+
metadata: {
|
|
8366
|
+
astFormat: context.options.astFormat,
|
|
8367
|
+
propertyPath,
|
|
8368
|
+
lineno: numberOrUndefined(node.lineno ?? node.line),
|
|
8369
|
+
colOffset: numberOrUndefined(node.col_offset ?? node.colOffset),
|
|
8370
|
+
endLineno: numberOrUndefined(node.end_lineno ?? node.endLine),
|
|
8371
|
+
endColOffset: numberOrUndefined(node.end_col_offset ?? node.endColOffset)
|
|
8372
|
+
}
|
|
8373
|
+
};
|
|
8374
|
+
context.nodes[id] = nativeNode;
|
|
8375
|
+
if (declaration) context.declarations.push({ ...declaration, nativeNode });
|
|
8376
|
+
return id;
|
|
8377
|
+
}
|
|
8378
|
+
|
|
7963
8379
|
function visitTreeSitterNode(node, context, propertyPath) {
|
|
7964
8380
|
if (!node || typeof node !== 'object' || context.truncated) return undefined;
|
|
7965
8381
|
if (context.objectIds.has(node)) return context.objectIds.get(node);
|
|
@@ -8455,6 +8871,14 @@ function nativeTargetProjectionDiagnosticToLoss(diagnostic, index, adapter, inpu
|
|
|
8455
8871
|
};
|
|
8456
8872
|
}
|
|
8457
8873
|
|
|
8874
|
+
function safeNativeImporterAdapterSummary(adapter) {
|
|
8875
|
+
try {
|
|
8876
|
+
return normalizeNativeImporterAdapter(adapter);
|
|
8877
|
+
} catch {
|
|
8878
|
+
return undefined;
|
|
8879
|
+
}
|
|
8880
|
+
}
|
|
8881
|
+
|
|
8458
8882
|
function normalizeNativeImporterAdapter(adapter) {
|
|
8459
8883
|
if (!adapter || typeof adapter !== 'object') {
|
|
8460
8884
|
throw new Error('Native importer adapter must be an object');
|
|
@@ -9089,6 +9513,23 @@ function isSyntaxAstNode(value) {
|
|
|
9089
9513
|
return Boolean(value && typeof value === 'object' && typeof (value.type ?? value.kind) === 'string');
|
|
9090
9514
|
}
|
|
9091
9515
|
|
|
9516
|
+
function pythonAstRoot(value) {
|
|
9517
|
+
if (!value || typeof value !== 'object') return undefined;
|
|
9518
|
+
if (isPythonAstNode(value)) return value;
|
|
9519
|
+
if (isPythonAstNode(value.ast)) return value.ast;
|
|
9520
|
+
if (isPythonAstNode(value.root)) return value.root;
|
|
9521
|
+
if (isPythonAstNode(value.module)) return value.module;
|
|
9522
|
+
return undefined;
|
|
9523
|
+
}
|
|
9524
|
+
|
|
9525
|
+
function isPythonAstNode(value) {
|
|
9526
|
+
return Boolean(value && typeof value === 'object' && typeof pythonAstKind(value) === 'string');
|
|
9527
|
+
}
|
|
9528
|
+
|
|
9529
|
+
function pythonAstKind(node) {
|
|
9530
|
+
return node?._type ?? node?.type ?? node?.kind ?? node?.nodeType;
|
|
9531
|
+
}
|
|
9532
|
+
|
|
9092
9533
|
function ignoredSyntaxField(key) {
|
|
9093
9534
|
return key === 'type'
|
|
9094
9535
|
|| key === 'kind'
|
|
@@ -9105,6 +9546,24 @@ function ignoredSyntaxField(key) {
|
|
|
9105
9546
|
|| key === 'parent';
|
|
9106
9547
|
}
|
|
9107
9548
|
|
|
9549
|
+
function ignoredPythonAstField(key) {
|
|
9550
|
+
return key === '_type'
|
|
9551
|
+
|| key === 'type'
|
|
9552
|
+
|| key === 'kind'
|
|
9553
|
+
|| key === 'nodeType'
|
|
9554
|
+
|| key === '_fields'
|
|
9555
|
+
|| key === 'lineno'
|
|
9556
|
+
|| key === 'col_offset'
|
|
9557
|
+
|| key === 'end_lineno'
|
|
9558
|
+
|| key === 'end_col_offset'
|
|
9559
|
+
|| key === 'line'
|
|
9560
|
+
|| key === 'colOffset'
|
|
9561
|
+
|| key === 'endLine'
|
|
9562
|
+
|| key === 'endColOffset'
|
|
9563
|
+
|| key === 'ctx'
|
|
9564
|
+
|| key === 'parent';
|
|
9565
|
+
}
|
|
9566
|
+
|
|
9108
9567
|
function primitiveSyntaxFields(node) {
|
|
9109
9568
|
const fields = {};
|
|
9110
9569
|
for (const key of ['name', 'operator', 'sourceType', 'async', 'generator', 'computed', 'static', 'exportKind', 'importKind', 'optional']) {
|
|
@@ -9118,11 +9577,36 @@ function primitiveSyntaxFields(node) {
|
|
|
9118
9577
|
return fields;
|
|
9119
9578
|
}
|
|
9120
9579
|
|
|
9580
|
+
function primitivePythonAstFields(node, kind) {
|
|
9581
|
+
const fields = { kind };
|
|
9582
|
+
for (const key of ['name', 'id', 'arg', 'module', 'level', 'attr', 'asname', 'type_comment']) {
|
|
9583
|
+
if (typeof node[key] === 'string' || typeof node[key] === 'number' || typeof node[key] === 'boolean' || node[key] === null) {
|
|
9584
|
+
fields[key] = node[key];
|
|
9585
|
+
}
|
|
9586
|
+
}
|
|
9587
|
+
if (Array.isArray(node.names)) {
|
|
9588
|
+
fields.names = node.names
|
|
9589
|
+
.map((entry) => pythonAliasName(entry))
|
|
9590
|
+
.filter(Boolean)
|
|
9591
|
+
.join(',');
|
|
9592
|
+
}
|
|
9593
|
+
const literal = pythonAstLiteralValue(node);
|
|
9594
|
+
if (literal !== undefined) fields.literal = literal;
|
|
9595
|
+
return fields;
|
|
9596
|
+
}
|
|
9597
|
+
|
|
9121
9598
|
function literalSyntaxValue(node) {
|
|
9122
9599
|
if (node.value === null || typeof node.value === 'string' || typeof node.value === 'number' || typeof node.value === 'boolean') return node.value;
|
|
9123
9600
|
return undefined;
|
|
9124
9601
|
}
|
|
9125
9602
|
|
|
9603
|
+
function pythonAstLiteralValue(node) {
|
|
9604
|
+
if (node.value === null || typeof node.value === 'string' || typeof node.value === 'number' || typeof node.value === 'boolean') return node.value;
|
|
9605
|
+
if (typeof node.s === 'string' || typeof node.s === 'number') return node.s;
|
|
9606
|
+
if (typeof node.n === 'number') return node.n;
|
|
9607
|
+
return undefined;
|
|
9608
|
+
}
|
|
9609
|
+
|
|
9126
9610
|
function spanFromLoc(loc, input) {
|
|
9127
9611
|
if (!loc?.start) return undefined;
|
|
9128
9612
|
return {
|
|
@@ -9135,6 +9619,22 @@ function spanFromLoc(loc, input) {
|
|
|
9135
9619
|
};
|
|
9136
9620
|
}
|
|
9137
9621
|
|
|
9622
|
+
function spanFromPythonAstNode(node, input) {
|
|
9623
|
+
const line = node.lineno ?? node.line;
|
|
9624
|
+
if (typeof line !== 'number') return undefined;
|
|
9625
|
+
const col = node.col_offset ?? node.colOffset;
|
|
9626
|
+
const endLine = node.end_lineno ?? node.endLine;
|
|
9627
|
+
const endCol = node.end_col_offset ?? node.endColOffset;
|
|
9628
|
+
return {
|
|
9629
|
+
sourceId: input.sourceHash,
|
|
9630
|
+
path: input.sourcePath,
|
|
9631
|
+
startLine: line,
|
|
9632
|
+
startColumn: typeof col === 'number' ? col + 1 : undefined,
|
|
9633
|
+
endLine: typeof endLine === 'number' ? endLine : undefined,
|
|
9634
|
+
endColumn: typeof endCol === 'number' ? endCol + 1 : undefined
|
|
9635
|
+
};
|
|
9636
|
+
}
|
|
9637
|
+
|
|
9138
9638
|
function syntaxDeclaration(node, nativeNodeId, input) {
|
|
9139
9639
|
const kind = String(node.type ?? node.kind ?? '');
|
|
9140
9640
|
if (kind === 'ImportDeclaration') {
|
|
@@ -9153,6 +9653,24 @@ function syntaxDeclaration(node, nativeNodeId, input) {
|
|
|
9153
9653
|
return undefined;
|
|
9154
9654
|
}
|
|
9155
9655
|
|
|
9656
|
+
function pythonAstDeclaration(node, kind, nativeNodeId, input) {
|
|
9657
|
+
if (kind === 'Import') {
|
|
9658
|
+
const name = (node.names ?? []).map((entry) => pythonAliasName(entry)).find(Boolean);
|
|
9659
|
+
if (name) return declarationRecord(input, nativeNodeId, name, 'module', 'import');
|
|
9660
|
+
}
|
|
9661
|
+
if (kind === 'ImportFrom') {
|
|
9662
|
+
const name = node.module ?? (node.names ?? []).map((entry) => pythonAliasName(entry)).find(Boolean);
|
|
9663
|
+
if (name) return declarationRecord(input, nativeNodeId, name, 'module', 'import');
|
|
9664
|
+
}
|
|
9665
|
+
if (kind === 'FunctionDef' || kind === 'AsyncFunctionDef') return declarationRecord(input, nativeNodeId, node.name, 'function', 'definition');
|
|
9666
|
+
if (kind === 'ClassDef') return declarationRecord(input, nativeNodeId, node.name, 'class', 'definition');
|
|
9667
|
+
if (kind === 'AnnAssign' || kind === 'Assign') {
|
|
9668
|
+
const name = pythonAssignmentName(node);
|
|
9669
|
+
if (name) return declarationRecord(input, nativeNodeId, name, 'variable', 'definition');
|
|
9670
|
+
}
|
|
9671
|
+
return undefined;
|
|
9672
|
+
}
|
|
9673
|
+
|
|
9156
9674
|
function typeScriptDeclaration(node, kind, nativeNodeId, input) {
|
|
9157
9675
|
if (kind === 'ImportDeclaration' || kind === 'ImportEqualsDeclaration') {
|
|
9158
9676
|
const name = stringFromTsExpression(node.moduleSpecifier) ?? stringFromTsExpression(node.externalModuleReference?.expression);
|
|
@@ -9196,6 +9714,49 @@ function namedDeclaration(input, nativeNodeId, nameNode, symbolKind) {
|
|
|
9196
9714
|
return name ? declarationRecord(input, nativeNodeId, name, symbolKind, 'definition') : undefined;
|
|
9197
9715
|
}
|
|
9198
9716
|
|
|
9717
|
+
function pythonAstChildEntries(node) {
|
|
9718
|
+
const fieldNames = Array.isArray(node._fields)
|
|
9719
|
+
? node._fields
|
|
9720
|
+
: Object.keys(node).filter((key) => !ignoredPythonAstField(key));
|
|
9721
|
+
return fieldNames
|
|
9722
|
+
.map((field) => [field, node[field]])
|
|
9723
|
+
.filter(([, value]) => Array.isArray(value)
|
|
9724
|
+
? value.some(isPythonAstNode)
|
|
9725
|
+
: isPythonAstNode(value));
|
|
9726
|
+
}
|
|
9727
|
+
|
|
9728
|
+
function pythonAstNodeValue(node) {
|
|
9729
|
+
return node.name ?? node.id ?? node.arg ?? node.module ?? pythonAstLiteralValue(node);
|
|
9730
|
+
}
|
|
9731
|
+
|
|
9732
|
+
function pythonAliasName(alias) {
|
|
9733
|
+
if (!alias) return undefined;
|
|
9734
|
+
if (typeof alias === 'string') return alias;
|
|
9735
|
+
return alias.name ?? alias.asname ?? alias.id;
|
|
9736
|
+
}
|
|
9737
|
+
|
|
9738
|
+
function pythonAssignmentName(node) {
|
|
9739
|
+
if (node.target) return pythonTargetName(node.target);
|
|
9740
|
+
for (const target of node.targets ?? []) {
|
|
9741
|
+
const name = pythonTargetName(target);
|
|
9742
|
+
if (name) return name;
|
|
9743
|
+
}
|
|
9744
|
+
return undefined;
|
|
9745
|
+
}
|
|
9746
|
+
|
|
9747
|
+
function pythonTargetName(target) {
|
|
9748
|
+
if (!target) return undefined;
|
|
9749
|
+
if (typeof target === 'string') return target;
|
|
9750
|
+
if (typeof target.id === 'string') return target.id;
|
|
9751
|
+
if (typeof target.name === 'string') return target.name;
|
|
9752
|
+
if (typeof target.arg === 'string') return target.arg;
|
|
9753
|
+
if (target.attr && target.value) {
|
|
9754
|
+
const base = pythonTargetName(target.value);
|
|
9755
|
+
return base ? `${base}.${target.attr}` : target.attr;
|
|
9756
|
+
}
|
|
9757
|
+
return undefined;
|
|
9758
|
+
}
|
|
9759
|
+
|
|
9199
9760
|
function declarationRecord(input, nativeNodeId, name, symbolKind, role = 'definition') {
|
|
9200
9761
|
return {
|
|
9201
9762
|
name: String(name),
|
package/package.json
CHANGED