@shapeshift-labs/frontier-lang-parser 0.3.15 → 0.3.16

package/README.md

@@ -239,6 +239,26 @@ conversion TodoJavascriptToRust @id("conversion_todo_js_rust") {
 
 `sourceRuntime` and `targetRuntime` become runtime maps. `runtimeRequirement` rows become proof obligations for host/runtime capabilities. `dialect` and `extern` rows preserve dialect-specific constructs, projection readiness, loss/evidence ids, and binding metadata without requiring the authored Frontier file to drop down to raw JSON.
 
+## Authored native source evidence
+
+`nativeSource` blocks can also carry source-bound merge evidence. This keeps parser/source-map/merge-candidate facts in `.frontier` text instead of requiring a raw JSON sidecar for the first authored program slice.
+
+```frontier
+nativeSource TodoTs @id("native_todo_ts") {
+  language typescript
+  parser typescript
+  sourcePath src/todo.ts
+  sourceHash sha256:example
+  frontierNodes ent_todo, action_add
+  evidence todoTitleProbe @id("artifact_todo_title_probe") kind test status passed path reports/todo-title.json
+  sourceMap todoProjection @id("sourcemap_todo_ts") target typescript targetPath src/generated/todo.ts evidence artifact_todo_title_probe
+  mapping todoTitle @id("map_todo_title") sourceMap sourcemap_todo_ts semanticNode field_title sourceSpan src/todo.ts:1:1-1:12 generatedSpan src/generated/todo.ts:1:1-1:20 precision exact evidence artifact_todo_title_probe
+  mergeCandidate todoTitle @id("candidate_todo_title") symbol symbol:Todo.title semanticNode field_title conflictKey symbol:Todo.title readiness ready sourceMap sourcemap_todo_ts sourceMapMapping map_todo_title
+}
+```
+
+The parser projects these rows into `metadata.universalAst.sourceMaps`, `metadata.universalAst.mergeCandidates`, and `metadata.universalAst.evidence`. `nativeSource` nodes keep id links through `sourceMapIds`, `mergeCandidateIds`, and `evidenceIds`. Runtime/browser equivalence still requires separate proof artifacts.
+
 ## Benchmarks
 
 Run the package-local benchmark with:

package/dist/index.js

@@ -1,9 +1,10 @@
-import { actionNode, capabilityNode, createDocument, effectNode, entityNode, externNode, latticeNode, migrationNode, nativeSourceNode, stateNode, targetNode, typeNode } from '@shapeshift-labs/frontier-lang-kernel';
+import { actionNode, capabilityNode, createDocument, effectNode, entityNode, externNode, latticeNode, migrationNode, stateNode, targetNode, typeNode } from '@shapeshift-labs/frontier-lang-kernel';
 import { parseConversionBlock } from './conversion.js';
 import { createParsedMetadata } from './metadata.js';
 import { parseSemanticOperationsBlock } from './operations.js';
 import { parseParadigmBlock } from './paradigm.js';
 import { parseProofBlock } from './proof.js';
+import { parseNativeSourceBlock } from './source-evidence.js';
 import { parseViewBlock } from './view.js';
 
 export function parseFrontierSource(source, options = {}) {
@@ -12,6 +13,7 @@ export function parseFrontierSource(source, options = {}) {
   const paradigmBlocks = [];
   const operationBlocks = [];
   const conversionBlocks = [];
+  const nativeSourceBlocks = [];
   const documentId = options.id ?? readId(source) ?? 'mod_frontier';
   const documentName = options.name ?? readName(source) ?? 'FrontierModule';
   for (const block of readBlocks(source)) {
@@ -25,14 +27,18 @@ export function parseFrontierSource(source, options = {}) {
     if (block.kind === 'type') nodes.push(parseType(block));
     if (block.kind === 'extern') nodes.push(parseExtern(block));
     if (block.kind === 'lattice') nodes.push(parseLattice(block));
-    if (block.kind === 'nativeSource') nodes.push(parseNativeSource(block));
+    if (block.kind === 'nativeSource') {
+      const parsed = parseNativeSourceBlock(block);
+      nodes.push(parsed.node);
+      nativeSourceBlocks.push(parsed);
+    }
     if (block.kind === 'target') nodes.push(parseTarget(block));
     if (block.kind === 'proof') proofBlocks.push(parseProofBlock(block));
     if (block.kind === 'paradigm' || block.kind === 'paradigmSemantics') paradigmBlocks.push(parseParadigmBlock(block));
     if (block.kind === 'operations' || block.kind === 'semanticOperations') operationBlocks.push(parseSemanticOperationsBlock(block));
     if (block.kind === 'conversion' || block.kind === 'universalConversionPlan') conversionBlocks.push(parseConversionBlock(block));
   }
-  const metadata = createParsedMetadata({ proofBlocks, paradigmBlocks, operationBlocks, conversionBlocks });
+  const metadata = createParsedMetadata({ proofBlocks, paradigmBlocks, operationBlocks, conversionBlocks, nativeSourceBlocks });
   return createDocument({ id: documentId, name: documentName, nodes, ...(metadata ? { metadata } : {}) });
 }
 
@@ -173,32 +179,6 @@ function parseLattice(block) {
     } : undefined
   });
 }
-function parseNativeSource(block) {
-  const name = nameFrom(block.header);
-  const losses = [];
-  const lossRe = /^\s*loss\s+([A-Za-z][\w-]*)\s+["']([^"']+)["'](?:\s+severity\s+([A-Za-z][\w-]*))?/gm;
-  let match;
-  while ((match = lossRe.exec(block.body))) {
-    losses.push({
-      id: `loss_${name}_${losses.length}`,
-      kind: match[1],
-      message: match[2],
-      severity: match[3] ?? 'warning'
-    });
-  }
-  return nativeSourceNode({
-    id: idFrom(block.header, `native_${name}`),
-    name,
-    language: readWord('language', block.body) ?? name,
-    parser: readWord('parser', block.body),
-    parserVersion: readWord('parserVersion', block.body),
-    sourcePath: readWord('sourcePath', block.body) ?? readWord('path', block.body),
-    sourceHash: readWord('sourceHash', block.body),
-    symbol: readWord('symbol', block.body),
-    frontierNodeIds: readList('frontierNodes', block.body),
-    losses: losses.length ? losses : undefined
-  });
-}
 function parseTarget(block) {
   const name = nameFrom(block.header);
   return targetNode({

package/dist/metadata.js

@@ -21,12 +21,15 @@ const PARADIGM_GROUPS = [
   'loweringRecords'
 ];
 
-export function createParsedMetadata({ proofBlocks = [], paradigmBlocks = [], operationBlocks = [], conversionBlocks = [] } = {}) {
+export function createParsedMetadata({ proofBlocks = [], paradigmBlocks = [], operationBlocks = [], conversionBlocks = [], nativeSourceBlocks = [] } = {}) {
   const metadata = {};
   if (proofBlocks.length) metadata.proof = mergeProofBlocks(proofBlocks);
   if (paradigmBlocks.length) metadata.paradigmSemantics = mergeParadigmBlocks(paradigmBlocks);
   if (operationBlocks.length) metadata.semanticOperations = mergeOperationBlocks(operationBlocks);
   if (conversionBlocks.length) metadata.universalConversionPlan = mergeConversionBlocks(conversionBlocks);
+  if (nativeSourceBlocks.some((block) => block.sourceMaps.length || block.mergeCandidates.length || block.evidence.length)) {
+    metadata.universalAst = mergeNativeSourceBlocks(nativeSourceBlocks);
+  }
   return Object.keys(metadata).length ? metadata : undefined;
 }
 
@@ -73,3 +76,15 @@ function mergeConversionBlocks(blocks) {
   }
   return plan;
 }
+
+function mergeNativeSourceBlocks(blocks) {
+  return {
+    id: blocks.length === 1 ? `universalAst:${blocks[0].node.id}` : 'universalAst:source',
+    nativeSourceIds: blocks.map((block) => block.node.id),
+    sourceMaps: blocks.flatMap((block) => block.sourceMaps ?? []),
+    mergeCandidates: blocks.flatMap((block) => block.mergeCandidates ?? []),
+    evidence: blocks.flatMap((block) => block.evidence ?? []),
+    losses: blocks.flatMap((block) => block.losses ?? []),
+    metadata: { authoredNativeSourceIds: blocks.map((block) => block.node.id) }
+  };
+}

package/dist/source-evidence.js

@@ -0,0 +1,230 @@
+import { createSemanticMergeCandidateRecord, createSourceMapRecord, nativeSourceNode } from '@shapeshift-labs/frontier-lang-kernel';
+
+export function parseNativeSourceBlock(block) {
+  const name = nameFrom(block.header);
+  const losses = parseLosses(name, block.body);
+  const nativeSourceId = idFrom(block.header, `native_${name}`);
+  const evidence = parseEvidenceRecords(block.body);
+  const sourceMaps = parseSourceMaps(block.body, {
+    nativeSourceId,
+    sourcePath: readWord('sourcePath', block.body) ?? readWord('path', block.body),
+    sourceHash: readWord('sourceHash', block.body),
+    evidence
+  });
+  const mergeCandidates = parseMergeCandidates(block.body, {
+    nativeSourceId,
+    language: readWord('language', block.body) ?? name,
+    sourcePath: readWord('sourcePath', block.body) ?? readWord('path', block.body),
+    sourceMaps,
+    evidence
+  });
+  const node = nativeSourceNode({
+    id: nativeSourceId,
+    name,
+    language: readWord('language', block.body) ?? name,
+    parser: readWord('parser', block.body),
+    parserVersion: readWord('parserVersion', block.body),
+    sourcePath: readWord('sourcePath', block.body) ?? readWord('path', block.body),
+    sourceHash: readWord('sourceHash', block.body),
+    symbol: readWord('symbol', block.body),
+    frontierNodeIds: readList('frontierNodes', block.body),
+    sourceMapIds: sourceMaps.map((sourceMap) => sourceMap.id),
+    mergeCandidateIds: mergeCandidates.map((candidate) => candidate.id),
+    evidenceIds: evidence.map((record) => record.id),
+    losses: losses.length ? losses : undefined
+  });
+  return {
+    node,
+    sourceMaps,
+    mergeCandidates,
+    evidence,
+    losses
+  };
+}
+
+function parseLosses(name, body) {
+  const losses = [];
+  const lossRe = /^\s*loss\s+([A-Za-z][\w-]*)\s+["']([^"']+)["'](?:\s+severity\s+([A-Za-z][\w-]*))?/gm;
+  let match;
+  while ((match = lossRe.exec(body))) {
+    losses.push({
+      id: `loss_${name}_${losses.length}`,
+      kind: match[1],
+      message: match[2],
+      severity: match[3] ?? 'warning'
+    });
+  }
+  return losses;
+}
+
+function parseEvidenceRecords(body) {
+  const records = [];
+  for (const { name, rest } of matchingRows(body, /^(?:evidence|proofEvidence)\s+([A-Za-z_$][\w$-]*)(.*)$/)) {
+    records.push(cleanRecord({
+      id: idFrom(rest, `evidence_${name}`),
+      kind: readInlineWord('kind', rest) ?? 'note',
+      status: readInlineWord('status', rest) ?? 'unknown',
+      path: readInlineWord('path', rest),
+      summary: readInlineQuoted('summary', rest),
+      metadata: { name }
+    }));
+  }
+  return records;
+}
+
+function parseSourceMaps(body, context) {
+  const rows = matchingRows(body, /^(?:sourceMap|sourcemap)\s+([A-Za-z_$][\w$-]*)(.*)$/);
+  const evidenceById = new Map(context.evidence.map((record) => [record.id, record]));
+  const mappingRows = matchingRows(body, /^(?:mapping|sourceMapMapping)\s+([A-Za-z_$][\w$-]*)(.*)$/);
+  const defaultSourceMapId = rows.length === 1 ? idFrom(rows[0].rest, `sourcemap_${rows[0].name}`) : undefined;
+  return rows.map(({ name, rest }) => {
+    const id = idFrom(rest, `sourcemap_${name}`);
+    const evidenceIds = readInlineList(rest, 'evidence', 'evidenceIds');
+    const mappings = mappingRows
+      .filter((row) => (readInlineWord('sourceMap', row.rest) ?? readInlineWord('sourceMapId', row.rest) ?? defaultSourceMapId) === id)
+      .map((row) => parseSourceMapMapping(row.name, row.rest, {
+        sourceMapId: id,
+        nativeSourceId: context.nativeSourceId,
+        sourcePath: readInlineWord('sourcePath', rest) ?? readInlineWord('path', rest) ?? context.sourcePath
+      }));
+    return createSourceMapRecord(cleanRecord({
+      id,
+      sourcePath: readInlineWord('sourcePath', rest) ?? readInlineWord('path', rest) ?? context.sourcePath,
+      sourceHash: readInlineWord('sourceHash', rest) ?? context.sourceHash,
+      target: readInlineWord('target', rest),
+      targetPath: readInlineWord('targetPath', rest),
+      targetHash: readInlineWord('targetHash', rest),
+      semanticIndexId: readInlineWord('semanticIndex', rest) ?? readInlineWord('semanticIndexId', rest),
+      universalAstId: readInlineWord('universalAst', rest) ?? readInlineWord('universalAstId', rest),
+      nativeAstId: readInlineWord('nativeAst', rest) ?? readInlineWord('nativeAstId', rest),
+      nativeSourceId: readInlineWord('nativeSource', rest) ?? readInlineWord('nativeSourceId', rest) ?? context.nativeSourceId,
+      mappings,
+      evidence: evidenceIds?.map((evidenceId) => evidenceById.get(evidenceId) ?? { id: evidenceId, kind: 'note', status: 'unknown' })
+    }));
+  });
+}
+
+function parseSourceMapMapping(name, text, context) {
+  return cleanRecord({
+    id: idFrom(text, `mapping_${name}`),
+    semanticNodeId: readInlineWord('semanticNode', text) ?? readInlineWord('semanticNodeId', text),
+    nativeSourceId: readInlineWord('nativeSource', text) ?? readInlineWord('nativeSourceId', text) ?? context.nativeSourceId,
+    nativeAstNodeId: readInlineWord('nativeAstNode', text) ?? readInlineWord('nativeAstNodeId', text),
+    semanticSymbolId: readInlineWord('semanticSymbol', text) ?? readInlineWord('symbol', text) ?? readInlineWord('semanticSymbolId', text),
+    semanticOccurrenceId: readInlineWord('semanticOccurrence', text) ?? readInlineWord('semanticOccurrenceId', text),
+    mergeCandidateId: readInlineWord('mergeCandidate', text) ?? readInlineWord('mergeCandidateId', text),
+    sourceSpan: parseSpan(readInlineWord('sourceSpan', text), context.sourcePath),
+    generatedSpan: parseSpan(readInlineWord('generatedSpan', text), readInlineWord('targetPath', text)),
+    target: readInlineWord('target', text),
+    generatedName: readInlineWord('generatedName', text),
+    evidenceIds: readInlineList(text, 'evidence', 'evidenceIds'),
+    lossIds: readInlineList(text, 'loss', 'lossIds'),
+    precision: readInlineWord('precision', text) ?? 'unknown',
+    preservation: readInlineWord('preservation', text),
+    metadata: cleanRecord({
+      sourceMapId: context.sourceMapId,
+      ownershipRegionId: readInlineWord('ownershipRegion', text) ?? readInlineWord('ownershipRegionId', text),
+      ownershipRegionKey: readInlineWord('ownershipKey', text) ?? readInlineWord('ownershipRegionKey', text)
+    })
+  });
+}
+
+function parseMergeCandidates(body, context) {
+  const sourceMapIds = context.sourceMaps.map((sourceMap) => sourceMap.id);
+  return matchingRows(body, /^(?:mergeCandidate|candidate)\s+([A-Za-z_$][\w$-]*)(.*)$/).map(({ name, rest }) => {
+    const conflictKeys = readInlineList(rest, 'conflictKey', 'conflictKeys', 'ownershipKey', 'ownershipKeys');
+    const symbols = readInlineList(rest, 'semanticSymbol', 'semanticSymbols', 'symbol', 'symbols', 'semanticSymbolIds');
+    const nodes = readInlineList(rest, 'semanticNode', 'semanticNodes', 'semanticNodeIds');
+    const evidenceIds = readInlineList(rest, 'evidence', 'evidenceIds');
+    const sourceMapMappingIds = readInlineList(rest, 'sourceMapMapping', 'sourceMapMappings', 'sourceMapMappingIds');
+    return createSemanticMergeCandidateRecord(cleanRecord({
+      id: idFrom(rest, `merge_candidate_${name}`),
+      language: readInlineWord('language', rest) ?? context.language,
+      sourcePath: readInlineWord('sourcePath', rest) ?? readInlineWord('path', rest) ?? context.sourcePath,
+      baseHash: readInlineWord('baseHash', rest),
+      targetHash: readInlineWord('targetHash', rest),
+      touchedSymbols: symbols?.map((symbol) => ({ id: symbol, conflictKey: conflictKeys?.[0] ?? symbol })),
+      touchedSemanticNodes: nodes?.map((node) => ({ id: node, conflictKey: conflictKeys?.[0] ?? node })),
+      nativeSpans: parseCandidateNativeSpans(rest, context),
+      conflictKeys,
+      readiness: readInlineWord('readiness', rest),
+      reasons: uniqueList([readInlineQuoted('reason', rest), ...(readInlineList(rest, 'reasonCode', 'reasonCodes', 'reasons') ?? [])]),
+      evidence: evidenceIds?.map((evidenceId) => context.evidence.find((record) => record.id === evidenceId) ?? { id: evidenceId, kind: 'note', status: 'unknown' }),
+      metadata: cleanRecord({
+        name,
+        nativeSourceId: readInlineWord('nativeSource', rest) ?? readInlineWord('nativeSourceId', rest) ?? context.nativeSourceId,
+        sourceMapIds: readInlineList(rest, 'sourceMap', 'sourceMaps', 'sourceMapIds') ?? sourceMapIds,
+        sourceMapMappingIds,
+        ownershipKeys: readInlineList(rest, 'ownershipKey', 'ownershipKeys')
+      })
+    }));
+  });
+}
+
+function parseCandidateNativeSpans(text, context) {
+  const span = parseSpan(readInlineWord('sourceSpan', text), context.sourcePath);
+  const nativeAstNodeId = readInlineWord('nativeAstNode', text) ?? readInlineWord('nativeAstNodeId', text);
+  if (!span && !nativeAstNodeId) return undefined;
+  const conflictKey = readInlineList(text, 'conflictKey', 'conflictKeys')?.[0]
+    ?? readInlineWord('semanticSymbol', text)
+    ?? readInlineWord('symbol', text)
+    ?? nativeAstNodeId
+    ?? 'native-span';
+  return [cleanRecord({
+    id: readInlineWord('nativeSpan', text) ?? `native_span_${hashableId(conflictKey)}`,
+    path: span?.path ?? context.sourcePath,
+    language: readInlineWord('language', text) ?? context.language,
+    nativeAstNodeId,
+    semanticNodeId: readInlineWord('semanticNode', text) ?? readInlineWord('semanticNodeId', text),
+    symbolId: readInlineWord('semanticSymbol', text) ?? readInlineWord('symbol', text),
+    span,
+    conflictKey
+  })];
+}
+
+function parseSpan(value, fallbackPath) {
+  if (!value) return undefined;
+  const match = /^(.+?):(\d+):(\d+)-(\d+):(\d+)$/.exec(value);
+  if (!match) return { path: value };
+  return {
+    path: match[1] || fallbackPath,
+    startLine: Number(match[2]),
+    startColumn: Number(match[3]),
+    endLine: Number(match[4]),
+    endColumn: Number(match[5])
+  };
+}
+
+function matchingRows(body, pattern) {
+  const rows = [];
+  for (const rawLine of body.split('\n')) {
+    const line = rawLine.trim();
+    if (!line || line.startsWith('#')) continue;
+    const match = pattern.exec(line);
+    if (match) rows.push({ name: match[1], rest: match[2] ?? '' });
+  }
+  return rows;
+}
+
+function idFrom(text, fallback) { return /@id\(\s*["']([^"']+)["']\s*\)/.exec(text)?.[1] ?? fallback; }
+function nameFrom(header) { return /^([A-Za-z_$][\w$]*)/.exec(header)?.[1] ?? 'Unnamed'; }
+function readWord(label, body) { return new RegExp('^\\s*' + label + '\\s+([^\\s]+)', 'm').exec(body)?.[1]?.trim(); }
+function readList(label, body) { const line = new RegExp('^\\s*' + label + '\\s+([^\\n]+)', 'm').exec(body)?.[1]; return line ? line.split(',').map((item) => item.trim()).filter(Boolean) : undefined; }
+function readInlineWord(label, text) { return new RegExp('(?:^|\\s)' + label + '\\s+([^\\s,]+)').exec(text)?.[1]?.trim(); }
+function readInlineQuoted(label, text) { return new RegExp("(?:^|\\s)" + label + "\\s+[\"']([^\"']+)[\"']").exec(text)?.[1]?.trim(); }
+function readInlineList(text, ...labels) {
+  for (const label of labels) {
+    const value = new RegExp('(?:^|\\s)' + label + '\\s+([^\\s]+)').exec(text)?.[1]?.trim();
+    if (value) return value.split(/[|,]/).map((item) => item.trim()).filter(Boolean);
+  }
+  return undefined;
+}
+function cleanRecord(record) {
+  return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined && (!Array.isArray(value) || value.length > 0) && (!value || typeof value !== 'object' || Object.keys(value).length > 0)));
+}
+function uniqueList(values) {
+  return [...new Set(values.filter(Boolean))];
+}
+function hashableId(value) {
+  return String(value ?? 'unknown').replace(/[^A-Za-z0-9_$-]+/g, '_');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shapeshift-labs/frontier-lang-parser",
-  "version": "0.3.15",
+  "version": "0.3.16",
   "description": "Parser for the first Frontier Lang .frontier syntax slice.",
   "type": "module",
   "main": "./dist/index.js",