@shapeshift-labs/frontier-lang-compiler 0.2.103 → 0.2.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/declarations/bidirectional-target-change-source-edit.d.ts +30 -0
- package/dist/declarations/bidirectional-target-change.d.ts +10 -0
- package/dist/declarations/js-ts-safe-member-merge.d.ts +58 -0
- package/dist/declarations/js-ts-safe-merge.d.ts +120 -0
- package/dist/declarations/js-ts-semantic-conflict-sidecars.d.ts +235 -0
- package/dist/declarations/js-ts-semantic-merge-contracts.d.ts +287 -0
- package/dist/declarations/js-ts-semantic-merge.d.ts +4 -0
- package/dist/declarations/native-import-losses.d.ts +3 -0
- package/dist/declarations/semantic-edit-replay-diagnostics.d.ts +12 -0
- package/dist/declarations/semantic-edit-script.d.ts +7 -4
- package/dist/declarations/semantic-patch-bundle-index.d.ts +45 -0
- package/dist/declarations/semantic-patch-bundle.d.ts +6 -4
- package/dist/declarations/semantic-sidecar-example.d.ts +18 -0
- package/dist/declarations/semantic-transform-identity.d.ts +3 -0
- package/dist/declarations/source-preservation.d.ts +72 -0
- package/dist/declarations/universal-capability.d.ts +4 -0
- package/dist/declarations/universal-conversion-artifacts.d.ts +61 -1
- package/dist/declarations/universal-conversion-compact-counts.d.ts +51 -0
- package/dist/declarations/universal-conversion-plan.d.ts +6 -1
- package/dist/declarations/universal-representation-coverage.d.ts +90 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +3 -0
- package/dist/internal/index-impl/bidirectionalExactSourceBackprojection.js +199 -0
- package/dist/internal/index-impl/bidirectionalSameLanguageSourceProjection.js +112 -0
- package/dist/internal/index-impl/bidirectionalSourceEditProjection.js +319 -0
- package/dist/internal/index-impl/bidirectionalSourceEditProjectionArtifacts.js +67 -0
- package/dist/internal/index-impl/bidirectionalTargetChangeRecordInternals.js +17 -5
- package/dist/internal/index-impl/bidirectionalTargetRoundtripEvidence.js +58 -20
- package/dist/internal/index-impl/createBidirectionalTargetChangeRecord.js +60 -7
- package/dist/internal/index-impl/createLightweightNativeImport.js +1 -0
- package/dist/internal/index-impl/createNativeSourcePreservation.js +28 -2
- package/dist/internal/index-impl/diffNativeSymbols.js +3 -3
- package/dist/internal/index-impl/nativeChangeProjectionSourceMapLinks.js +2 -0
- package/dist/internal/index-impl/projectSemanticEditScriptToSource.js +43 -8
- package/dist/internal/index-impl/replaySemanticEditLineEndings.js +34 -0
- package/dist/internal/index-impl/replaySemanticEditProjection.js +39 -19
- package/dist/internal/index-impl/semanticEditBundleAdmission.js +7 -3
- package/dist/internal/index-impl/semanticEditBundleIndex.js +47 -1
- package/dist/internal/index-impl/semanticEditExplicitSourceReplacement.js +40 -0
- package/dist/internal/index-impl/semanticEditOperationCoverage.js +33 -3
- package/dist/internal/index-impl/semanticEditProjectionRecord.js +29 -0
- package/dist/internal/index-impl/semanticEditReplayDiagnostics.js +39 -0
- package/dist/internal/index-impl/semanticEditReplaySourceReplacement.js +85 -0
- package/dist/internal/index-impl/semanticEditScripts.js +4 -0
- package/dist/internal/index-impl/semanticEditSourceRanges.js +27 -0
- package/dist/internal/index-impl/semanticIndexFromNativeDeclarations.js +1 -0
- package/dist/internal/index-impl/semanticPatchBundleAdmission.js +41 -7
- package/dist/internal/index-impl/semanticPatchBundleRecords.js +16 -0
- package/dist/internal/index-impl/semanticPatchBundleSourceRecords.js +2 -0
- package/dist/internal/index-impl/semanticSidecarQuality.js +111 -0
- package/dist/internal/index-impl/semanticSourceEditDedupe.js +69 -9
- package/dist/internal/index-impl/semanticTransformIdentityRecords.js +85 -9
- package/dist/js-ts-safe-member-merge-result.js +158 -0
- package/dist/js-ts-safe-member-merge.js +202 -0
- package/dist/js-ts-safe-merge-analyze.js +279 -0
- package/dist/js-ts-safe-merge-constants.js +50 -0
- package/dist/js-ts-safe-merge-context.js +118 -0
- package/dist/js-ts-safe-merge-ledger-validation.js +92 -0
- package/dist/js-ts-safe-merge-ledger.js +85 -0
- package/dist/js-ts-safe-merge-parse-declarations.js +210 -0
- package/dist/js-ts-safe-merge-parse-statements.js +155 -0
- package/dist/js-ts-safe-merge-plan.js +190 -0
- package/dist/js-ts-safe-merge.js +175 -0
- package/dist/js-ts-semantic-conflict-sidecar-constants.js +77 -0
- package/dist/js-ts-semantic-conflict-sidecar-detectors.js +195 -0
- package/dist/js-ts-semantic-conflict-sidecar-normalize.js +203 -0
- package/dist/js-ts-semantic-conflict-sidecar-utils.js +190 -0
- package/dist/js-ts-semantic-conflict-sidecars.js +81 -0
- package/dist/js-ts-semantic-merge-contract-helpers.js +128 -0
- package/dist/js-ts-semantic-merge-contracts.js +217 -0
- package/dist/js-ts-semantic-merge-member-containers.js +100 -0
- package/dist/js-ts-semantic-merge-member-keys.js +142 -0
- package/dist/js-ts-semantic-merge-member-segments.js +185 -0
- package/dist/js-ts-semantic-merge-member-source.js +64 -0
- package/dist/js-ts-semantic-merge-member-utils.js +18 -0
- package/dist/js-ts-semantic-merge-parse.js +15 -0
- package/dist/js-ts-semantic-merge.js +21 -0
- package/dist/lightweight-dependency-effects.js +51 -0
- package/dist/lightweight-dependency-language.js +12 -1
- package/dist/lightweight-dependency-relations.js +14 -27
- package/dist/native-region-scanner-core.js +33 -1
- package/dist/native-region-scanner-csharp.js +151 -0
- package/dist/native-region-scanner-dart.js +91 -0
- package/dist/native-region-scanner-dynamic.js +21 -151
- package/dist/native-region-scanner-functional.js +40 -13
- package/dist/native-region-scanner-java.js +97 -0
- package/dist/native-region-scanner-js-class.js +100 -0
- package/dist/native-region-scanner-js-helpers.js +28 -86
- package/dist/native-region-scanner-js-imports.js +121 -1
- package/dist/native-region-scanner-js-nested.js +96 -8
- package/dist/native-region-scanner-js-structure.js +27 -0
- package/dist/native-region-scanner-js-types.js +99 -0
- package/dist/native-region-scanner-js.js +70 -118
- package/dist/native-region-scanner-kotlin.js +94 -0
- package/dist/native-region-scanner-main.js +15 -181
- package/dist/native-region-scanner-php.js +80 -0
- package/dist/native-region-scanner-python.js +62 -0
- package/dist/native-region-scanner-ruby.js +72 -0
- package/dist/native-region-scanner-scala.js +91 -0
- package/dist/native-region-scanner-spans.js +74 -0
- package/dist/native-region-scanner-swift.js +155 -0
- package/dist/native-region-scanner.js +14 -10
- package/dist/native-source-ledger-helpers.js +195 -0
- package/dist/native-source-ledger.js +306 -0
- package/dist/native-source-preservation-scanner.js +4 -0
- package/dist/semantic-import-callsite-regions.js +136 -0
- package/dist/semantic-import-effect-regions.js +283 -0
- package/dist/semantic-import-regions.js +11 -2
- package/dist/semantic-import-sidecar-entry.js +16 -2
- package/dist/semantic-import-sidecar-types.d.ts +2 -0
- package/dist/semantic-sidecar-example.js +68 -0
- package/dist/universal-capability-matrix.js +23 -0
- package/dist/universal-conversion-artifact-query.js +79 -2
- package/dist/universal-conversion-artifact-semantic-edit.js +103 -0
- package/dist/universal-conversion-artifact-summary.js +33 -1
- package/dist/universal-conversion-artifacts.js +13 -48
- package/dist/universal-conversion-plan-scoring.js +21 -1
- package/dist/universal-conversion-plan-summary.js +30 -0
- package/dist/universal-conversion-plan.js +25 -9
- package/dist/universal-conversion-route-metadata.js +96 -0
- package/dist/universal-conversion-route-operations.js +7 -0
- package/dist/universal-representation-coverage.js +193 -0
- package/package.json +1 -1
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
const regexStartKeywords = new Set(['case', 'delete', 'else', 'in', 'instanceof', 'new', 'return', 'throw', 'typeof', 'void', 'yield']);
|
|
2
|
+
|
|
3
|
+
export function isSourceMapComment(text) {
|
|
4
|
+
return /^(?:\/\/|\/\*)[#@]\s*sourceMappingURL=/.test(String(text).trim());
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export function readToLineEnd(text, start) {
|
|
8
|
+
let index = start;
|
|
9
|
+
while (index < text.length && text[index] !== '\n' && text[index] !== '\r') index += 1;
|
|
10
|
+
return index;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function readBlockCommentEnd(text, start) {
|
|
14
|
+
const end = text.indexOf('*/', start + 2);
|
|
15
|
+
return end === -1 ? text.length : end + 2;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function readQuotedEnd(text, start, quote) {
|
|
19
|
+
let escaped = false;
|
|
20
|
+
for (let index = start + 1; index < text.length; index += 1) {
|
|
21
|
+
const char = text[index];
|
|
22
|
+
if (escaped) {
|
|
23
|
+
escaped = false;
|
|
24
|
+
} else if (char === '\\') {
|
|
25
|
+
escaped = true;
|
|
26
|
+
} else if (char === quote) {
|
|
27
|
+
return index + 1;
|
|
28
|
+
} else if (char === '\n' || char === '\r') {
|
|
29
|
+
return index;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return text.length;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function readTemplateEnd(text, start) {
|
|
36
|
+
let escaped = false;
|
|
37
|
+
for (let index = start + 1; index < text.length; index += 1) {
|
|
38
|
+
const char = text[index];
|
|
39
|
+
if (escaped) escaped = false;
|
|
40
|
+
else if (char === '\\') escaped = true;
|
|
41
|
+
else if (char === '`') return index + 1;
|
|
42
|
+
}
|
|
43
|
+
return text.length;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function readRegexEnd(text, start) {
|
|
47
|
+
let escaped = false;
|
|
48
|
+
let inClass = false;
|
|
49
|
+
for (let index = start + 1; index < text.length; index += 1) {
|
|
50
|
+
const char = text[index];
|
|
51
|
+
if (char === '\n' || char === '\r') return undefined;
|
|
52
|
+
if (escaped) escaped = false;
|
|
53
|
+
else if (char === '\\') escaped = true;
|
|
54
|
+
else if (char === '[') inClass = true;
|
|
55
|
+
else if (char === ']') inClass = false;
|
|
56
|
+
else if (char === '/' && !inClass) return readWhile(text, index + 1, (value) => /[A-Za-z]/.test(value));
|
|
57
|
+
}
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function readJsxRegionEnd(text, start) {
|
|
62
|
+
let index = start;
|
|
63
|
+
let depth = 0;
|
|
64
|
+
while (index < text.length) {
|
|
65
|
+
if (text[index] === '{') {
|
|
66
|
+
index = readBalancedBraceEnd(text, index);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (text[index] !== '<') {
|
|
70
|
+
index += 1;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const tag = readJsxTag(text, index);
|
|
74
|
+
if (!tag) return index > start ? index : start + 1;
|
|
75
|
+
if (tag.close) depth -= 1;
|
|
76
|
+
else if (!tag.selfClosing) depth += 1;
|
|
77
|
+
index = tag.end;
|
|
78
|
+
if (depth <= 0) return index;
|
|
79
|
+
}
|
|
80
|
+
return text.length;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function readStatementEnd(text, start) {
|
|
84
|
+
const end = readToLineEnd(text, start);
|
|
85
|
+
const semicolon = text.indexOf(';', start);
|
|
86
|
+
return semicolon !== -1 && semicolon < end ? semicolon + 1 : end;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function readWhile(text, start, predicate) {
|
|
90
|
+
let index = start;
|
|
91
|
+
while (index < text.length && predicate(text[index])) index += 1;
|
|
92
|
+
return index;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function endPosition(start, text) {
|
|
96
|
+
let line = start.line;
|
|
97
|
+
let column = start.column;
|
|
98
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
99
|
+
if (text[index] === '\r') {
|
|
100
|
+
if (text[index + 1] === '\n') index += 1;
|
|
101
|
+
line += 1;
|
|
102
|
+
column = 1;
|
|
103
|
+
} else if (text[index] === '\n') {
|
|
104
|
+
line += 1;
|
|
105
|
+
column = 1;
|
|
106
|
+
} else {
|
|
107
|
+
column += 1;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return { offset: start.offset + text.length, line, column };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function looksLikeJsxStart(text, offset, previous) {
|
|
114
|
+
if (text[offset] !== '<' || !(isIdentifierStart(text[offset + 1]) || text[offset + 1] === '>')) return false;
|
|
115
|
+
if (!previous) return true;
|
|
116
|
+
if (previous.kind === 'operator' || ['(', '[', '{', ',', ':', '?', '=>'].includes(previous.text)) return true;
|
|
117
|
+
return previous.kind === 'keyword' && ['return', 'case', 'throw'].includes(previous.text);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export function mayStartRegex(previous) {
|
|
121
|
+
if (!previous) return true;
|
|
122
|
+
if (previous.kind === 'operator') return true;
|
|
123
|
+
if (previous.kind === 'keyword') return regexStartKeywords.has(previous.text);
|
|
124
|
+
return ['(', '[', '{', ',', ';', ':', '?'].includes(previous.text);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function braceMetadata(char, stack, tokenId) {
|
|
128
|
+
const kind = char === '{' || char === '}' ? 'curly' : char === '(' || char === ')' ? 'paren' : 'bracket';
|
|
129
|
+
if (char === '{' || char === '(' || char === '[') {
|
|
130
|
+
stack.push({ char, tokenId, kind });
|
|
131
|
+
return { braceKind: kind, side: 'open', depth: stack.length };
|
|
132
|
+
}
|
|
133
|
+
const open = char === '}' ? '{' : char === ')' ? '(' : '[';
|
|
134
|
+
const matchIndex = stack.map((entry) => entry.char).lastIndexOf(open);
|
|
135
|
+
const matched = matchIndex >= 0 ? stack.splice(matchIndex, 1)[0] : undefined;
|
|
136
|
+
return { braceKind: kind, side: 'close', depth: stack.length + 1, openTokenId: matched?.tokenId };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function isBrace(char) {
|
|
140
|
+
return char === '{' || char === '}' || char === '(' || char === ')' || char === '[' || char === ']';
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function isHorizontalWhitespace(char) {
|
|
144
|
+
return char === ' ' || char === '\t' || char === '\v' || char === '\f';
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export function isIdentifierStart(char) {
|
|
148
|
+
return /[A-Za-z_$]/.test(char ?? '');
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export function isIdentifierPart(char) {
|
|
152
|
+
return /[A-Za-z0-9_$]/.test(char ?? '');
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function readJsxTag(text, start) {
|
|
156
|
+
if (text[start + 1] === '!' && text.startsWith('<!--', start)) {
|
|
157
|
+
const end = text.indexOf('-->', start + 4);
|
|
158
|
+
return { end: end === -1 ? text.length : end + 3, selfClosing: true, close: false };
|
|
159
|
+
}
|
|
160
|
+
let index = start + 1;
|
|
161
|
+
const close = text[index] === '/';
|
|
162
|
+
if (close) index += 1;
|
|
163
|
+
if (text[index] === '>') return { end: index + 1, selfClosing: false, close };
|
|
164
|
+
if (!isIdentifierStart(text[index])) return undefined;
|
|
165
|
+
let quote = undefined;
|
|
166
|
+
for (; index < text.length; index += 1) {
|
|
167
|
+
const char = text[index];
|
|
168
|
+
if (quote) {
|
|
169
|
+
if (char === '\\') index += 1;
|
|
170
|
+
else if (char === quote) quote = undefined;
|
|
171
|
+
} else if (char === '\'' || char === '"') {
|
|
172
|
+
quote = char;
|
|
173
|
+
} else if (char === '>') {
|
|
174
|
+
return { end: index + 1, selfClosing: text[index - 1] === '/', close };
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return { end: text.length, selfClosing: true, close };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function readBalancedBraceEnd(text, start) {
|
|
181
|
+
let depth = 1;
|
|
182
|
+
for (let index = start + 1; index < text.length; index += 1) {
|
|
183
|
+
const char = text[index];
|
|
184
|
+
if (char === '\'' || char === '"') index = readQuotedEnd(text, index, char) - 1;
|
|
185
|
+
else if (char === '`') index = readTemplateEnd(text, index) - 1;
|
|
186
|
+
else if (char === '/' && text[index + 1] === '/') index = readToLineEnd(text, index) - 1;
|
|
187
|
+
else if (char === '/' && text[index + 1] === '*') index = readBlockCommentEnd(text, index) - 1;
|
|
188
|
+
else if (char === '{') depth += 1;
|
|
189
|
+
else if (char === '}') {
|
|
190
|
+
depth -= 1;
|
|
191
|
+
if (depth === 0) return index + 1;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return text.length;
|
|
195
|
+
}
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
import { hashSemanticValue } from '@shapeshift-labs/frontier-lang-kernel';
|
|
2
|
+
import { countBy, idFragment, normalizeNativeLanguageId } from './native-import-utils.js';
|
|
3
|
+
import { braceMetadata, endPosition, isBrace, isHorizontalWhitespace, isIdentifierPart, isIdentifierStart, isSourceMapComment, looksLikeJsxStart, mayStartRegex, readBlockCommentEnd, readJsxRegionEnd, readQuotedEnd, readRegexEnd, readStatementEnd, readTemplateEnd, readToLineEnd, readWhile } from './native-source-ledger-helpers.js';
|
|
4
|
+
|
|
5
|
+
const jsTsKeywords = new Set([
|
|
6
|
+
'abstract', 'as', 'async', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
|
|
7
|
+
'declare', 'default', 'delete', 'do', 'else', 'enum', 'export', 'extends', 'false', 'finally',
|
|
8
|
+
'for', 'from', 'function', 'get', 'if', 'implements', 'import', 'in', 'infer', 'instanceof',
|
|
9
|
+
'interface', 'keyof', 'let', 'module', 'namespace', 'never', 'new', 'null', 'of', 'package',
|
|
10
|
+
'private', 'protected', 'public', 'readonly', 'require', 'return', 'satisfies', 'set',
|
|
11
|
+
'static', 'super', 'switch', 'symbol', 'this', 'throw', 'true', 'try', 'type', 'typeof',
|
|
12
|
+
'undefined', 'unique', 'unknown', 'var', 'void', 'while', 'with', 'yield'
|
|
13
|
+
]);
|
|
14
|
+
|
|
15
|
+
export function isJavaScriptTypeScriptSource(language, sourcePath) {
|
|
16
|
+
const normalized = normalizeNativeLanguageId(language);
|
|
17
|
+
if (normalized === 'javascript' || normalized === 'typescript') return true;
|
|
18
|
+
return /\.(?:[cm]?js|jsx|[cm]?ts|tsx)$/i.test(String(sourcePath ?? ''));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function scanJavaScriptTypeScriptSourceLedger(sourceText, input = {}) {
|
|
22
|
+
const maxSpans = Number.isFinite(input.maxLedgerSpans) ? Math.max(0, input.maxLedgerSpans) : 40000;
|
|
23
|
+
const sourceHash = input.sourceHash ?? hashSemanticValue(sourceText);
|
|
24
|
+
const ledger = {
|
|
25
|
+
kind: 'frontier.lang.jsTsSourceLedger',
|
|
26
|
+
version: 1,
|
|
27
|
+
language: input.language ?? 'javascript',
|
|
28
|
+
sourcePath: input.sourcePath,
|
|
29
|
+
sourceHash,
|
|
30
|
+
spans: [],
|
|
31
|
+
tokens: [],
|
|
32
|
+
trivia: [],
|
|
33
|
+
comments: [],
|
|
34
|
+
shebangs: [],
|
|
35
|
+
directives: [],
|
|
36
|
+
importExportSpans: [],
|
|
37
|
+
braces: [],
|
|
38
|
+
protectedRegions: []
|
|
39
|
+
};
|
|
40
|
+
let offset = 0;
|
|
41
|
+
let line = 1;
|
|
42
|
+
let column = 1;
|
|
43
|
+
let truncated = false;
|
|
44
|
+
let previous = undefined;
|
|
45
|
+
const braceStack = [];
|
|
46
|
+
|
|
47
|
+
const push = (collection, kind, role, text, start, end, metadata = {}) => {
|
|
48
|
+
if (ledger.spans.length >= maxSpans) {
|
|
49
|
+
truncated = true;
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
const entry = sourceLedgerSpan({
|
|
53
|
+
index: ledger.spans.length,
|
|
54
|
+
kind,
|
|
55
|
+
role,
|
|
56
|
+
text,
|
|
57
|
+
start,
|
|
58
|
+
end,
|
|
59
|
+
sourceHash,
|
|
60
|
+
sourcePath: input.sourcePath,
|
|
61
|
+
metadata
|
|
62
|
+
});
|
|
63
|
+
ledger.spans.push(entry);
|
|
64
|
+
collection.push(entry);
|
|
65
|
+
return entry;
|
|
66
|
+
};
|
|
67
|
+
const consume = (endOffset) => {
|
|
68
|
+
const start = { offset, line, column };
|
|
69
|
+
const text = sourceText.slice(offset, endOffset);
|
|
70
|
+
advanceText(text);
|
|
71
|
+
return { start, end: { offset, line, column }, text };
|
|
72
|
+
};
|
|
73
|
+
const pushToken = (kind, text, start, end, metadata) => {
|
|
74
|
+
const token = push(ledger.tokens, kind, 'token', text, start, end, metadata);
|
|
75
|
+
previous = token ?? { kind, text, metadata };
|
|
76
|
+
return token;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
while (offset < sourceText.length) {
|
|
80
|
+
const char = sourceText[offset];
|
|
81
|
+
const next = sourceText[offset + 1];
|
|
82
|
+
if (offset === 0 && char === '#' && next === '!') {
|
|
83
|
+
const part = consume(readToLineEnd(sourceText, offset));
|
|
84
|
+
push(ledger.trivia, 'shebang', 'trivia', part.text, part.start, part.end);
|
|
85
|
+
push(ledger.comments, 'shebang', 'comment', part.text, part.start, part.end);
|
|
86
|
+
ledger.shebangs.push(sourceLedgerSpan({
|
|
87
|
+
index: ledger.spans.length,
|
|
88
|
+
kind: 'shebang',
|
|
89
|
+
role: 'directive',
|
|
90
|
+
text: part.text,
|
|
91
|
+
start: part.start,
|
|
92
|
+
end: part.end,
|
|
93
|
+
sourceHash,
|
|
94
|
+
sourcePath: input.sourcePath
|
|
95
|
+
}));
|
|
96
|
+
ledger.spans.push(ledger.shebangs[ledger.shebangs.length - 1]);
|
|
97
|
+
previous = undefined;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (char === '\r' || char === '\n') {
|
|
101
|
+
const part = consume(char === '\r' && next === '\n' ? offset + 2 : offset + 1);
|
|
102
|
+
push(ledger.trivia, 'newline', 'trivia', part.text, part.start, part.end);
|
|
103
|
+
previous = undefined;
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (isHorizontalWhitespace(char)) {
|
|
107
|
+
const part = consume(readWhile(sourceText, offset, isHorizontalWhitespace));
|
|
108
|
+
push(ledger.trivia, 'whitespace', 'trivia', part.text, part.start, part.end);
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
if (char === '/' && next === '/') {
|
|
112
|
+
const part = consume(readToLineEnd(sourceText, offset));
|
|
113
|
+
const kind = isSourceMapComment(part.text) ? 'source-map-comment' : 'comment';
|
|
114
|
+
const metadata = kind === 'source-map-comment' ? { sourceMap: true } : {};
|
|
115
|
+
push(ledger.trivia, kind, 'trivia', part.text, part.start, part.end, metadata);
|
|
116
|
+
push(ledger.comments, kind, 'comment', part.text, part.start, part.end, metadata);
|
|
117
|
+
previous = undefined;
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (char === '/' && next === '*') {
|
|
121
|
+
const part = consume(readBlockCommentEnd(sourceText, offset));
|
|
122
|
+
const kind = isSourceMapComment(part.text) ? 'source-map-comment' : 'comment';
|
|
123
|
+
const metadata = kind === 'source-map-comment' ? { sourceMap: true } : {};
|
|
124
|
+
push(ledger.trivia, kind, 'trivia', part.text, part.start, part.end, metadata);
|
|
125
|
+
push(ledger.comments, kind, 'comment', part.text, part.start, part.end, metadata);
|
|
126
|
+
previous = undefined;
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
if (char === '\'' || char === '"') {
|
|
130
|
+
const part = consume(readQuotedEnd(sourceText, offset, char));
|
|
131
|
+
pushToken('string', part.text, part.start, part.end, { quote: char });
|
|
132
|
+
push(ledger.protectedRegions, 'string', 'protected', part.text, part.start, part.end, { quote: char });
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
if (char === '`') {
|
|
136
|
+
const part = consume(readTemplateEnd(sourceText, offset));
|
|
137
|
+
pushToken('template', part.text, part.start, part.end);
|
|
138
|
+
push(ledger.protectedRegions, 'template', 'protected', part.text, part.start, part.end);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (looksLikeJsxStart(sourceText, offset, previous)) {
|
|
142
|
+
const jsxEnd = readJsxRegionEnd(sourceText, offset);
|
|
143
|
+
if (jsxEnd > offset + 1) {
|
|
144
|
+
const part = consume(jsxEnd);
|
|
145
|
+
pushToken('jsx', part.text, part.start, part.end);
|
|
146
|
+
push(ledger.protectedRegions, 'jsx', 'protected', part.text, part.start, part.end);
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (char === '/' && mayStartRegex(previous)) {
|
|
151
|
+
const regexEnd = readRegexEnd(sourceText, offset);
|
|
152
|
+
if (regexEnd) {
|
|
153
|
+
const part = consume(regexEnd);
|
|
154
|
+
pushToken('regex-like', part.text, part.start, part.end);
|
|
155
|
+
push(ledger.protectedRegions, 'regex-like', 'protected', part.text, part.start, part.end);
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (/[0-9]/.test(char)) {
|
|
160
|
+
const part = consume(readWhile(sourceText, offset, (value) => /[0-9a-fA-F_xXoObBeE.+-]/.test(value)));
|
|
161
|
+
pushToken('number', part.text, part.start, part.end);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
if (isIdentifierStart(char)) {
|
|
165
|
+
const part = consume(readWhile(sourceText, offset, isIdentifierPart));
|
|
166
|
+
const kind = jsTsKeywords.has(part.text) ? 'keyword' : 'identifier';
|
|
167
|
+
pushToken(kind, part.text, part.start, part.end);
|
|
168
|
+
if (part.text === 'import' || part.text === 'export') {
|
|
169
|
+
push(ledger.importExportSpans, part.text, 'module-keyword', part.text, part.start, part.end, {
|
|
170
|
+
statementEnd: readStatementEnd(sourceText, offset)
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
if (isBrace(char)) {
|
|
176
|
+
const part = consume(offset + 1);
|
|
177
|
+
const token = pushToken('punctuation', part.text, part.start, part.end);
|
|
178
|
+
const brace = braceMetadata(char, braceStack, token?.id);
|
|
179
|
+
push(ledger.braces, 'brace', 'brace', part.text, part.start, part.end, brace);
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
if (/[=+\-*/%&|^!<>?:.]/.test(char)) {
|
|
183
|
+
const part = consume(readWhile(sourceText, offset, (value) => /[=+\-*/%&|^!<>?:.]/.test(value)));
|
|
184
|
+
pushToken('operator', part.text, part.start, part.end);
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const part = consume(offset + 1);
|
|
188
|
+
pushToken(/[;,]/.test(char) ? 'punctuation' : 'unknown', part.text, part.start, part.end);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
for (const directive of sourceLedgerDirectives(sourceText, input, sourceHash)) {
|
|
192
|
+
if (ledger.spans.length >= maxSpans) {
|
|
193
|
+
truncated = true;
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
ledger.spans.push(directive);
|
|
197
|
+
ledger.directives.push(directive);
|
|
198
|
+
}
|
|
199
|
+
ledger.summary = {
|
|
200
|
+
spans: ledger.spans.length,
|
|
201
|
+
tokens: ledger.tokens.length,
|
|
202
|
+
trivia: ledger.trivia.length,
|
|
203
|
+
comments: ledger.comments.length,
|
|
204
|
+
shebangs: ledger.shebangs.length,
|
|
205
|
+
directives: ledger.directives.length,
|
|
206
|
+
sourceMapComments: ledger.comments.filter((entry) => entry.kind === 'source-map-comment').length,
|
|
207
|
+
importExportSpans: ledger.importExportSpans.length,
|
|
208
|
+
braces: ledger.braces.length,
|
|
209
|
+
protectedRegions: ledger.protectedRegions.length,
|
|
210
|
+
stringRegions: ledger.protectedRegions.filter((entry) => entry.kind === 'string').length,
|
|
211
|
+
templateRegions: ledger.protectedRegions.filter((entry) => entry.kind === 'template').length,
|
|
212
|
+
regexLikeRegions: ledger.protectedRegions.filter((entry) => entry.kind === 'regex-like').length,
|
|
213
|
+
jsxRegions: ledger.protectedRegions.filter((entry) => entry.kind === 'jsx').length,
|
|
214
|
+
triviaByKind: countBy(ledger.trivia.map((entry) => entry.kind)),
|
|
215
|
+
tokenByKind: countBy(ledger.tokens.map((entry) => entry.kind)),
|
|
216
|
+
directiveByKind: countBy(ledger.directives.map((entry) => entry.kind)),
|
|
217
|
+
truncated
|
|
218
|
+
};
|
|
219
|
+
return ledger;
|
|
220
|
+
|
|
221
|
+
function advanceText(text) {
|
|
222
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
223
|
+
if (text[index] === '\r') {
|
|
224
|
+
if (text[index + 1] === '\n') {
|
|
225
|
+
offset += 2;
|
|
226
|
+
index += 1;
|
|
227
|
+
} else {
|
|
228
|
+
offset += 1;
|
|
229
|
+
}
|
|
230
|
+
line += 1;
|
|
231
|
+
column = 1;
|
|
232
|
+
} else if (text[index] === '\n') {
|
|
233
|
+
offset += 1;
|
|
234
|
+
line += 1;
|
|
235
|
+
column = 1;
|
|
236
|
+
} else {
|
|
237
|
+
offset += 1;
|
|
238
|
+
column += 1;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function sourceLedgerSpan(input) {
|
|
245
|
+
const entry = {
|
|
246
|
+
id: `${input.role}_${input.kind}_${input.index + 1}_${idFragment(input.start.offset)}`,
|
|
247
|
+
kind: input.kind,
|
|
248
|
+
role: input.role,
|
|
249
|
+
text: input.text,
|
|
250
|
+
textHash: hashSemanticValue(input.text),
|
|
251
|
+
span: {
|
|
252
|
+
sourceId: input.sourceHash,
|
|
253
|
+
path: input.sourcePath,
|
|
254
|
+
start: input.start.offset,
|
|
255
|
+
end: input.end.offset,
|
|
256
|
+
startLine: input.start.line,
|
|
257
|
+
startColumn: input.start.column,
|
|
258
|
+
endLine: input.end.line,
|
|
259
|
+
endColumn: input.end.column
|
|
260
|
+
}
|
|
261
|
+
};
|
|
262
|
+
if (input.metadata && Object.keys(input.metadata).length) entry.metadata = input.metadata;
|
|
263
|
+
return entry;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function sourceLedgerDirectives(sourceText, input, sourceHash) {
|
|
267
|
+
const directives = [];
|
|
268
|
+
let offset = 0;
|
|
269
|
+
let line = 1;
|
|
270
|
+
while (offset <= sourceText.length) {
|
|
271
|
+
const lineEnd = readToLineEnd(sourceText, offset);
|
|
272
|
+
const lineText = sourceText.slice(offset, lineEnd);
|
|
273
|
+
const trimmed = lineText.trim();
|
|
274
|
+
const kind = directiveKind(trimmed);
|
|
275
|
+
if (kind) {
|
|
276
|
+
const startColumn = lineText.indexOf(trimmed) + 1;
|
|
277
|
+
const start = { offset: offset + startColumn - 1, line, column: startColumn };
|
|
278
|
+
directives.push(sourceLedgerSpan({
|
|
279
|
+
index: directives.length,
|
|
280
|
+
kind,
|
|
281
|
+
role: 'directive',
|
|
282
|
+
text: trimmed,
|
|
283
|
+
start,
|
|
284
|
+
end: endPosition(start, trimmed),
|
|
285
|
+
sourceHash,
|
|
286
|
+
sourcePath: input.sourcePath,
|
|
287
|
+
metadata: { language: input.language ?? 'javascript' }
|
|
288
|
+
}));
|
|
289
|
+
}
|
|
290
|
+
if (lineEnd >= sourceText.length) break;
|
|
291
|
+
const newlineLength = sourceText[lineEnd] === '\r' && sourceText[lineEnd + 1] === '\n' ? 2 : 1;
|
|
292
|
+
offset = lineEnd + newlineLength;
|
|
293
|
+
line += 1;
|
|
294
|
+
}
|
|
295
|
+
return directives;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function directiveKind(trimmed) {
|
|
299
|
+
if (!trimmed) return undefined;
|
|
300
|
+
if (/^#!\s*/.test(trimmed)) return 'shebang';
|
|
301
|
+
if (isSourceMapComment(trimmed)) return 'source-map-comment';
|
|
302
|
+
if (/^\/\/\/\s*<reference\b/.test(trimmed)) return 'typescript-reference';
|
|
303
|
+
if (/^['"]use\s+(?:strict|client|server)['"];?$/.test(trimmed)) return 'runtime-directive';
|
|
304
|
+
if (/^(?:import|export)\b/.test(trimmed)) return 'module-directive';
|
|
305
|
+
return undefined;
|
|
306
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { caseSensitiveIdFragment, idFragment, uniqueRecordsById } from './native-import-utils.js';
|
|
2
|
+
|
|
3
|
+
export function semanticCallsiteRecordsForImport(imported, semanticIndex, options = {}) {
|
|
4
|
+
const relations = (semanticIndex?.relations ?? []).filter((relation) => relation?.predicate === 'calls');
|
|
5
|
+
if (!relations.length) return { symbols: [], ownershipRegions: [] };
|
|
6
|
+
const symbolsById = new Map((semanticIndex?.symbols ?? []).map((symbol) => [symbol.id, symbol]));
|
|
7
|
+
const occurrences = semanticIndex?.occurrences ?? [];
|
|
8
|
+
const occurrencesById = new Map(occurrences.map((occurrence) => [occurrence.id, occurrence]));
|
|
9
|
+
const sourceText = nativeImportSourceText(imported);
|
|
10
|
+
const records = relations.map((relation, index) => {
|
|
11
|
+
const occurrence = occurrenceForRelation(relation, occurrencesById, occurrences);
|
|
12
|
+
const rawSpan = relation.metadata?.sourceSpan ?? occurrence?.span;
|
|
13
|
+
const spanInfo = callsiteSpan(sourceText, rawSpan);
|
|
14
|
+
const sourceName = relation.metadata?.sourceName ?? symbolsById.get(relation.sourceId)?.name ?? relation.sourceId;
|
|
15
|
+
const targetName = relation.metadata?.targetName ?? symbolsById.get(relation.targetId)?.name ?? relation.targetId;
|
|
16
|
+
const sourcePath = spanInfo.span?.path ?? imported?.sourcePath ?? imported?.nativeSource?.sourcePath;
|
|
17
|
+
const language = imported?.language ?? imported?.nativeSource?.language ?? imported?.nativeAst?.language;
|
|
18
|
+
const symbolName = `${sourceName}->${targetName}`;
|
|
19
|
+
const key = callsiteKey(options.regionPrefix, sourcePath, symbolName, spanInfo.span, index);
|
|
20
|
+
const region = {
|
|
21
|
+
id: `region_${caseSensitiveIdFragment(key)}`,
|
|
22
|
+
key,
|
|
23
|
+
regionKind: 'call',
|
|
24
|
+
granularity: 'callsite',
|
|
25
|
+
language,
|
|
26
|
+
sourcePath,
|
|
27
|
+
sourceHash: imported?.nativeSource?.sourceHash ?? imported?.nativeAst?.sourceHash ?? imported?.sourceHash,
|
|
28
|
+
symbolId: `symbol:${language ?? 'source'}:call:${idFragment(relation.id ?? key)}`,
|
|
29
|
+
symbolName,
|
|
30
|
+
symbolKind: 'call',
|
|
31
|
+
nativeAstNodeId: occurrence?.nativeAstNodeId,
|
|
32
|
+
sourceSpan: spanInfo.span,
|
|
33
|
+
precision: spanInfo.expanded ? 'callsite' : spanInfo.span ? 'line' : 'unknown',
|
|
34
|
+
mergePolicy: 'callsite-overlap-review-required',
|
|
35
|
+
metadata: {
|
|
36
|
+
semanticRegionTaxonomy: true,
|
|
37
|
+
relationId: relation.id,
|
|
38
|
+
occurrenceId: occurrence?.id ?? relation.metadata?.occurrenceId,
|
|
39
|
+
sourceSymbolId: relation.sourceId,
|
|
40
|
+
targetSymbolId: relation.targetId,
|
|
41
|
+
sourceName,
|
|
42
|
+
targetName
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
return {
|
|
46
|
+
region,
|
|
47
|
+
symbol: {
|
|
48
|
+
id: region.symbolId,
|
|
49
|
+
name: symbolName,
|
|
50
|
+
kind: 'call',
|
|
51
|
+
language,
|
|
52
|
+
nativeAstNodeId: region.nativeAstNodeId,
|
|
53
|
+
semanticOccurrenceId: region.metadata.occurrenceId,
|
|
54
|
+
sourceSpan: region.sourceSpan,
|
|
55
|
+
ownershipRegionId: region.id,
|
|
56
|
+
ownershipKey: region.key,
|
|
57
|
+
ownershipRegionKind: 'call',
|
|
58
|
+
readiness: 'needs-review'
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
});
|
|
62
|
+
return {
|
|
63
|
+
symbols: uniqueRecordsById(records.map((record) => record.symbol)),
|
|
64
|
+
ownershipRegions: uniqueRecordsById(records.map((record) => record.region))
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function occurrenceForRelation(relation, occurrencesById, occurrences) {
|
|
69
|
+
const explicit = occurrencesById.get(relation.metadata?.occurrenceId);
|
|
70
|
+
if (explicit) return explicit;
|
|
71
|
+
return occurrences.find((occurrence) => occurrence.symbolId === relation.targetId
|
|
72
|
+
&& (!relation.metadata?.nativeAstNodeId || occurrence.nativeAstNodeId === relation.metadata.nativeAstNodeId));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function callsiteKey(prefix = 'source', sourcePath, symbolName, span, index) {
|
|
76
|
+
const location = span ? `${span.startLine ?? 0}:${span.startColumn ?? 0}` : `unknown:${index + 1}`;
|
|
77
|
+
return [prefix, sourcePath ?? 'memory', 'call', `${symbolName}@${location}`]
|
|
78
|
+
.map((part) => String(part).replace(/\s+/g, ' ').trim())
|
|
79
|
+
.join('#');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function callsiteSpan(sourceText, span) {
|
|
83
|
+
if (typeof sourceText !== 'string' || !span || typeof span.startLine !== 'number') {
|
|
84
|
+
return { span, expanded: false };
|
|
85
|
+
}
|
|
86
|
+
const line = sourceText.split(/\r\n|\n|\r/)[span.startLine - 1];
|
|
87
|
+
if (line === undefined) return { span, expanded: false };
|
|
88
|
+
const nameEnd = Math.max(0, Number(span.endColumn ?? span.startColumn ?? 1) - 1);
|
|
89
|
+
const open = nextNonSpaceIndex(line, nameEnd);
|
|
90
|
+
if (line[open] !== '(') return { span, expanded: false };
|
|
91
|
+
const close = matchingParenIndex(line, open);
|
|
92
|
+
if (close === undefined) return { span, expanded: false };
|
|
93
|
+
return {
|
|
94
|
+
span: {
|
|
95
|
+
...span,
|
|
96
|
+
endColumn: close + 2
|
|
97
|
+
},
|
|
98
|
+
expanded: true
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function nextNonSpaceIndex(line, index) {
|
|
103
|
+
for (let cursor = index; cursor < line.length; cursor += 1) {
|
|
104
|
+
if (!/\s/.test(line[cursor])) return cursor;
|
|
105
|
+
}
|
|
106
|
+
return -1;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function matchingParenIndex(line, open) {
|
|
110
|
+
let depth = 0;
|
|
111
|
+
let quote;
|
|
112
|
+
let escaped = false;
|
|
113
|
+
for (let index = open; index < line.length; index += 1) {
|
|
114
|
+
const char = line[index];
|
|
115
|
+
if (quote) {
|
|
116
|
+
if (escaped) escaped = false;
|
|
117
|
+
else if (char === '\\') escaped = true;
|
|
118
|
+
else if (char === quote) quote = undefined;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
if (char === '\'' || char === '"' || char === '`') {
|
|
122
|
+
quote = char;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
if (char === '(') depth += 1;
|
|
126
|
+
else if (char === ')' && --depth === 0) return index;
|
|
127
|
+
}
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function nativeImportSourceText(imported) {
|
|
132
|
+
return imported?.metadata?.sourcePreservation?.sourceText
|
|
133
|
+
?? imported?.nativeSource?.metadata?.sourcePreservation?.sourceText
|
|
134
|
+
?? imported?.nativeAst?.metadata?.sourcePreservation?.sourceText
|
|
135
|
+
?? imported?.universalAst?.metadata?.sourcePreservation?.sourceText;
|
|
136
|
+
}
|