@shapeshift-labs/frontier-lang-compiler 0.2.102 → 0.2.104

package/dist/native-source-ledger.js

@@ -0,0 +1,306 @@
+import { hashSemanticValue } from '@shapeshift-labs/frontier-lang-kernel';
+import { countBy, idFragment, normalizeNativeLanguageId } from './native-import-utils.js';
+import { braceMetadata, endPosition, isBrace, isHorizontalWhitespace, isIdentifierPart, isIdentifierStart, isSourceMapComment, looksLikeJsxStart, mayStartRegex, readBlockCommentEnd, readJsxRegionEnd, readQuotedEnd, readRegexEnd, readStatementEnd, readTemplateEnd, readToLineEnd, readWhile } from './native-source-ledger-helpers.js';
+
+const jsTsKeywords = new Set([
+  'abstract', 'as', 'async', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
+  'declare', 'default', 'delete', 'do', 'else', 'enum', 'export', 'extends', 'false', 'finally',
+  'for', 'from', 'function', 'get', 'if', 'implements', 'import', 'in', 'infer', 'instanceof',
+  'interface', 'keyof', 'let', 'module', 'namespace', 'never', 'new', 'null', 'of', 'package',
+  'private', 'protected', 'public', 'readonly', 'require', 'return', 'satisfies', 'set',
+  'static', 'super', 'switch', 'symbol', 'this', 'throw', 'true', 'try', 'type', 'typeof',
+  'undefined', 'unique', 'unknown', 'var', 'void', 'while', 'with', 'yield'
+]);
+
+export function isJavaScriptTypeScriptSource(language, sourcePath) {
+  const normalized = normalizeNativeLanguageId(language);
+  if (normalized === 'javascript' || normalized === 'typescript') return true;
+  return /\.(?:[cm]?js|jsx|[cm]?ts|tsx)$/i.test(String(sourcePath ?? ''));
+}
+
+export function scanJavaScriptTypeScriptSourceLedger(sourceText, input = {}) {
+  const maxSpans = Number.isFinite(input.maxLedgerSpans) ? Math.max(0, input.maxLedgerSpans) : 40000;
+  const sourceHash = input.sourceHash ?? hashSemanticValue(sourceText);
+  const ledger = {
+    kind: 'frontier.lang.jsTsSourceLedger',
+    version: 1,
+    language: input.language ?? 'javascript',
+    sourcePath: input.sourcePath,
+    sourceHash,
+    spans: [],
+    tokens: [],
+    trivia: [],
+    comments: [],
+    shebangs: [],
+    directives: [],
+    importExportSpans: [],
+    braces: [],
+    protectedRegions: []
+  };
+  let offset = 0;
+  let line = 1;
+  let column = 1;
+  let truncated = false;
+  let previous = undefined;
+  const braceStack = [];
+
+  const push = (collection, kind, role, text, start, end, metadata = {}) => {
+    if (ledger.spans.length >= maxSpans) {
+      truncated = true;
+      return undefined;
+    }
+    const entry = sourceLedgerSpan({
+      index: ledger.spans.length,
+      kind,
+      role,
+      text,
+      start,
+      end,
+      sourceHash,
+      sourcePath: input.sourcePath,
+      metadata
+    });
+    ledger.spans.push(entry);
+    collection.push(entry);
+    return entry;
+  };
+  const consume = (endOffset) => {
+    const start = { offset, line, column };
+    const text = sourceText.slice(offset, endOffset);
+    advanceText(text);
+    return { start, end: { offset, line, column }, text };
+  };
+  const pushToken = (kind, text, start, end, metadata) => {
+    const token = push(ledger.tokens, kind, 'token', text, start, end, metadata);
+    previous = token ?? { kind, text, metadata };
+    return token;
+  };
+
+  while (offset < sourceText.length) {
+    const char = sourceText[offset];
+    const next = sourceText[offset + 1];
+    if (offset === 0 && char === '#' && next === '!') {
+      const part = consume(readToLineEnd(sourceText, offset));
+      push(ledger.trivia, 'shebang', 'trivia', part.text, part.start, part.end);
+      push(ledger.comments, 'shebang', 'comment', part.text, part.start, part.end);
+      ledger.shebangs.push(sourceLedgerSpan({
+        index: ledger.spans.length,
+        kind: 'shebang',
+        role: 'directive',
+        text: part.text,
+        start: part.start,
+        end: part.end,
+        sourceHash,
+        sourcePath: input.sourcePath
+      }));
+      ledger.spans.push(ledger.shebangs[ledger.shebangs.length - 1]);
+      previous = undefined;
+      continue;
+    }
+    if (char === '\r' || char === '\n') {
+      const part = consume(char === '\r' && next === '\n' ? offset + 2 : offset + 1);
+      push(ledger.trivia, 'newline', 'trivia', part.text, part.start, part.end);
+      previous = undefined;
+      continue;
+    }
+    if (isHorizontalWhitespace(char)) {
+      const part = consume(readWhile(sourceText, offset, isHorizontalWhitespace));
+      push(ledger.trivia, 'whitespace', 'trivia', part.text, part.start, part.end);
+      continue;
+    }
+    if (char === '/' && next === '/') {
+      const part = consume(readToLineEnd(sourceText, offset));
+      const kind = isSourceMapComment(part.text) ? 'source-map-comment' : 'comment';
+      const metadata = kind === 'source-map-comment' ? { sourceMap: true } : {};
+      push(ledger.trivia, kind, 'trivia', part.text, part.start, part.end, metadata);
+      push(ledger.comments, kind, 'comment', part.text, part.start, part.end, metadata);
+      previous = undefined;
+      continue;
+    }
+    if (char === '/' && next === '*') {
+      const part = consume(readBlockCommentEnd(sourceText, offset));
+      const kind = isSourceMapComment(part.text) ? 'source-map-comment' : 'comment';
+      const metadata = kind === 'source-map-comment' ? { sourceMap: true } : {};
+      push(ledger.trivia, kind, 'trivia', part.text, part.start, part.end, metadata);
+      push(ledger.comments, kind, 'comment', part.text, part.start, part.end, metadata);
+      previous = undefined;
+      continue;
+    }
+    if (char === '\'' || char === '"') {
+      const part = consume(readQuotedEnd(sourceText, offset, char));
+      pushToken('string', part.text, part.start, part.end, { quote: char });
+      push(ledger.protectedRegions, 'string', 'protected', part.text, part.start, part.end, { quote: char });
+      continue;
+    }
+    if (char === '`') {
+      const part = consume(readTemplateEnd(sourceText, offset));
+      pushToken('template', part.text, part.start, part.end);
+      push(ledger.protectedRegions, 'template', 'protected', part.text, part.start, part.end);
+      continue;
+    }
+    if (looksLikeJsxStart(sourceText, offset, previous)) {
+      const jsxEnd = readJsxRegionEnd(sourceText, offset);
+      if (jsxEnd > offset + 1) {
+        const part = consume(jsxEnd);
+        pushToken('jsx', part.text, part.start, part.end);
+        push(ledger.protectedRegions, 'jsx', 'protected', part.text, part.start, part.end);
+        continue;
+      }
+    }
+    if (char === '/' && mayStartRegex(previous)) {
+      const regexEnd = readRegexEnd(sourceText, offset);
+      if (regexEnd) {
+        const part = consume(regexEnd);
+        pushToken('regex-like', part.text, part.start, part.end);
+        push(ledger.protectedRegions, 'regex-like', 'protected', part.text, part.start, part.end);
+        continue;
+      }
+    }
+    if (/[0-9]/.test(char)) {
+      const part = consume(readWhile(sourceText, offset, (value) => /[0-9a-fA-F_xXoObBeE.+-]/.test(value)));
+      pushToken('number', part.text, part.start, part.end);
+      continue;
+    }
+    if (isIdentifierStart(char)) {
+      const part = consume(readWhile(sourceText, offset, isIdentifierPart));
+      const kind = jsTsKeywords.has(part.text) ? 'keyword' : 'identifier';
+      pushToken(kind, part.text, part.start, part.end);
+      if (part.text === 'import' || part.text === 'export') {
+        push(ledger.importExportSpans, part.text, 'module-keyword', part.text, part.start, part.end, {
+          statementEnd: readStatementEnd(sourceText, offset)
+        });
+      }
+      continue;
+    }
+    if (isBrace(char)) {
+      const part = consume(offset + 1);
+      const token = pushToken('punctuation', part.text, part.start, part.end);
+      const brace = braceMetadata(char, braceStack, token?.id);
+      push(ledger.braces, 'brace', 'brace', part.text, part.start, part.end, brace);
+      continue;
+    }
+    if (/[=+\-*/%&|^!<>?:.]/.test(char)) {
+      const part = consume(readWhile(sourceText, offset, (value) => /[=+\-*/%&|^!<>?:.]/.test(value)));
+      pushToken('operator', part.text, part.start, part.end);
+      continue;
+    }
+    const part = consume(offset + 1);
+    pushToken(/[;,]/.test(char) ? 'punctuation' : 'unknown', part.text, part.start, part.end);
+  }
+
+  for (const directive of sourceLedgerDirectives(sourceText, input, sourceHash)) {
+    if (ledger.spans.length >= maxSpans) {
+      truncated = true;
+      break;
+    }
+    ledger.spans.push(directive);
+    ledger.directives.push(directive);
+  }
+  ledger.summary = {
+    spans: ledger.spans.length,
+    tokens: ledger.tokens.length,
+    trivia: ledger.trivia.length,
+    comments: ledger.comments.length,
+    shebangs: ledger.shebangs.length,
+    directives: ledger.directives.length,
+    sourceMapComments: ledger.comments.filter((entry) => entry.kind === 'source-map-comment').length,
+    importExportSpans: ledger.importExportSpans.length,
+    braces: ledger.braces.length,
+    protectedRegions: ledger.protectedRegions.length,
+    stringRegions: ledger.protectedRegions.filter((entry) => entry.kind === 'string').length,
+    templateRegions: ledger.protectedRegions.filter((entry) => entry.kind === 'template').length,
+    regexLikeRegions: ledger.protectedRegions.filter((entry) => entry.kind === 'regex-like').length,
+    jsxRegions: ledger.protectedRegions.filter((entry) => entry.kind === 'jsx').length,
+    triviaByKind: countBy(ledger.trivia.map((entry) => entry.kind)),
+    tokenByKind: countBy(ledger.tokens.map((entry) => entry.kind)),
+    directiveByKind: countBy(ledger.directives.map((entry) => entry.kind)),
+    truncated
+  };
+  return ledger;
+
+  function advanceText(text) {
+    for (let index = 0; index < text.length; index += 1) {
+      if (text[index] === '\r') {
+        if (text[index + 1] === '\n') {
+          offset += 2;
+          index += 1;
+        } else {
+          offset += 1;
+        }
+        line += 1;
+        column = 1;
+      } else if (text[index] === '\n') {
+        offset += 1;
+        line += 1;
+        column = 1;
+      } else {
+        offset += 1;
+        column += 1;
+      }
+    }
+  }
+}
+
+function sourceLedgerSpan(input) {
+  const entry = {
+    id: `${input.role}_${input.kind}_${input.index + 1}_${idFragment(input.start.offset)}`,
+    kind: input.kind,
+    role: input.role,
+    text: input.text,
+    textHash: hashSemanticValue(input.text),
+    span: {
+      sourceId: input.sourceHash,
+      path: input.sourcePath,
+      start: input.start.offset,
+      end: input.end.offset,
+      startLine: input.start.line,
+      startColumn: input.start.column,
+      endLine: input.end.line,
+      endColumn: input.end.column
+    }
+  };
+  if (input.metadata && Object.keys(input.metadata).length) entry.metadata = input.metadata;
+  return entry;
+}
+
+function sourceLedgerDirectives(sourceText, input, sourceHash) {
+  const directives = [];
+  let offset = 0;
+  let line = 1;
+  while (offset <= sourceText.length) {
+    const lineEnd = readToLineEnd(sourceText, offset);
+    const lineText = sourceText.slice(offset, lineEnd);
+    const trimmed = lineText.trim();
+    const kind = directiveKind(trimmed);
+    if (kind) {
+      const startColumn = lineText.indexOf(trimmed) + 1;
+      const start = { offset: offset + startColumn - 1, line, column: startColumn };
+      directives.push(sourceLedgerSpan({
+        index: directives.length,
+        kind,
+        role: 'directive',
+        text: trimmed,
+        start,
+        end: endPosition(start, trimmed),
+        sourceHash,
+        sourcePath: input.sourcePath,
+        metadata: { language: input.language ?? 'javascript' }
+      }));
+    }
+    if (lineEnd >= sourceText.length) break;
+    const newlineLength = sourceText[lineEnd] === '\r' && sourceText[lineEnd + 1] === '\n' ? 2 : 1;
+    offset = lineEnd + newlineLength;
+    line += 1;
+  }
+  return directives;
+}
+
+function directiveKind(trimmed) {
+  if (!trimmed) return undefined;
+  if (/^#!\s*/.test(trimmed)) return 'shebang';
+  if (isSourceMapComment(trimmed)) return 'source-map-comment';
+  if (/^\/\/\/\s*<reference\b/.test(trimmed)) return 'typescript-reference';
+  if (/^['"]use\s+(?:strict|client|server)['"];?$/.test(trimmed)) return 'runtime-directive';
+  if (/^(?:import|export)\b/.test(trimmed)) return 'module-directive';
+  return undefined;
+}

package/dist/native-source-preservation-scanner.js

@@ -264,3 +264,7 @@ export {
   scanPreservedSourceDirectives,
   scanPreservedSourceTokens
 };
+export {
+  isJavaScriptTypeScriptSource,
+  scanJavaScriptTypeScriptSourceLedger
+} from './native-source-ledger.js';

package/dist/semantic-import-callsite-regions.js

@@ -0,0 +1,136 @@
+import { caseSensitiveIdFragment, idFragment, uniqueRecordsById } from './native-import-utils.js';
+
+export function semanticCallsiteRecordsForImport(imported, semanticIndex, options = {}) {
+  const relations = (semanticIndex?.relations ?? []).filter((relation) => relation?.predicate === 'calls');
+  if (!relations.length) return { symbols: [], ownershipRegions: [] };
+  const symbolsById = new Map((semanticIndex?.symbols ?? []).map((symbol) => [symbol.id, symbol]));
+  const occurrences = semanticIndex?.occurrences ?? [];
+  const occurrencesById = new Map(occurrences.map((occurrence) => [occurrence.id, occurrence]));
+  const sourceText = nativeImportSourceText(imported);
+  const records = relations.map((relation, index) => {
+    const occurrence = occurrenceForRelation(relation, occurrencesById, occurrences);
+    const rawSpan = relation.metadata?.sourceSpan ?? occurrence?.span;
+    const spanInfo = callsiteSpan(sourceText, rawSpan);
+    const sourceName = relation.metadata?.sourceName ?? symbolsById.get(relation.sourceId)?.name ?? relation.sourceId;
+    const targetName = relation.metadata?.targetName ?? symbolsById.get(relation.targetId)?.name ?? relation.targetId;
+    const sourcePath = spanInfo.span?.path ?? imported?.sourcePath ?? imported?.nativeSource?.sourcePath;
+    const language = imported?.language ?? imported?.nativeSource?.language ?? imported?.nativeAst?.language;
+    const symbolName = `${sourceName}->${targetName}`;
+    const key = callsiteKey(options.regionPrefix, sourcePath, symbolName, spanInfo.span, index);
+    const region = {
+      id: `region_${caseSensitiveIdFragment(key)}`,
+      key,
+      regionKind: 'call',
+      granularity: 'callsite',
+      language,
+      sourcePath,
+      sourceHash: imported?.nativeSource?.sourceHash ?? imported?.nativeAst?.sourceHash ?? imported?.sourceHash,
+      symbolId: `symbol:${language ?? 'source'}:call:${idFragment(relation.id ?? key)}`,
+      symbolName,
+      symbolKind: 'call',
+      nativeAstNodeId: occurrence?.nativeAstNodeId,
+      sourceSpan: spanInfo.span,
+      precision: spanInfo.expanded ? 'callsite' : spanInfo.span ? 'line' : 'unknown',
+      mergePolicy: 'callsite-overlap-review-required',
+      metadata: {
+        semanticRegionTaxonomy: true,
+        relationId: relation.id,
+        occurrenceId: occurrence?.id ?? relation.metadata?.occurrenceId,
+        sourceSymbolId: relation.sourceId,
+        targetSymbolId: relation.targetId,
+        sourceName,
+        targetName
+      }
+    };
+    return {
+      region,
+      symbol: {
+        id: region.symbolId,
+        name: symbolName,
+        kind: 'call',
+        language,
+        nativeAstNodeId: region.nativeAstNodeId,
+        semanticOccurrenceId: region.metadata.occurrenceId,
+        sourceSpan: region.sourceSpan,
+        ownershipRegionId: region.id,
+        ownershipKey: region.key,
+        ownershipRegionKind: 'call',
+        readiness: 'needs-review'
+      }
+    };
+  });
+  return {
+    symbols: uniqueRecordsById(records.map((record) => record.symbol)),
+    ownershipRegions: uniqueRecordsById(records.map((record) => record.region))
+  };
+}
+
+function occurrenceForRelation(relation, occurrencesById, occurrences) {
+  const explicit = occurrencesById.get(relation.metadata?.occurrenceId);
+  if (explicit) return explicit;
+  return occurrences.find((occurrence) => occurrence.symbolId === relation.targetId
+    && (!relation.metadata?.nativeAstNodeId || occurrence.nativeAstNodeId === relation.metadata.nativeAstNodeId));
+}
+
+function callsiteKey(prefix = 'source', sourcePath, symbolName, span, index) {
+  const location = span ? `${span.startLine ?? 0}:${span.startColumn ?? 0}` : `unknown:${index + 1}`;
+  return [prefix, sourcePath ?? 'memory', 'call', `${symbolName}@${location}`]
+    .map((part) => String(part).replace(/\s+/g, ' ').trim())
+    .join('#');
+}
+
+function callsiteSpan(sourceText, span) {
+  if (typeof sourceText !== 'string' || !span || typeof span.startLine !== 'number') {
+    return { span, expanded: false };
+  }
+  const line = sourceText.split(/\r\n|\n|\r/)[span.startLine - 1];
+  if (line === undefined) return { span, expanded: false };
+  const nameEnd = Math.max(0, Number(span.endColumn ?? span.startColumn ?? 1) - 1);
+  const open = nextNonSpaceIndex(line, nameEnd);
+  if (line[open] !== '(') return { span, expanded: false };
+  const close = matchingParenIndex(line, open);
+  if (close === undefined) return { span, expanded: false };
+  return {
+    span: {
+      ...span,
+      endColumn: close + 2
+    },
+    expanded: true
+  };
+}
+
+function nextNonSpaceIndex(line, index) {
+  for (let cursor = index; cursor < line.length; cursor += 1) {
+    if (!/\s/.test(line[cursor])) return cursor;
+  }
+  return -1;
+}
+
+function matchingParenIndex(line, open) {
+  let depth = 0;
+  let quote;
+  let escaped = false;
+  for (let index = open; index < line.length; index += 1) {
+    const char = line[index];
+    if (quote) {
+      if (escaped) escaped = false;
+      else if (char === '\\') escaped = true;
+      else if (char === quote) quote = undefined;
+      continue;
+    }
+    if (char === '\'' || char === '"' || char === '`') {
+      quote = char;
+      continue;
+    }
+    if (char === '(') depth += 1;
+    else if (char === ')' && --depth === 0) return index;
+  }
+  return undefined;
+}
+
+function nativeImportSourceText(imported) {
+  return imported?.metadata?.sourcePreservation?.sourceText
+    ?? imported?.nativeSource?.metadata?.sourcePreservation?.sourceText
+    ?? imported?.nativeAst?.metadata?.sourcePreservation?.sourceText
+    ?? imported?.universalAst?.metadata?.sourcePreservation?.sourceText;
+}