@pieceful/ravel-core 0.1.1 → 0.2.0

package/src/index.js

@@ -1,6 +1,16 @@
 const componentPattern = /^[a-z][a-z0-9-]*$/;
+const chunkPathPattern = /^[a-z][a-z0-9-]*(?:\/(?:[a-z][a-z0-9-]*)?)*$/;
+const relativeChunkPathPattern = /^(?:\.\.?)(?:\/(?:\.\.?|[a-z][a-z0-9-]*))*$/;
 
 export { directiveKinds, compose, append, newline, pipe, pass, createDirective, aliasDirective } from "./directives.js";
+export { parseRavelDirectiveBlock } from "./directive-syntax.js";
+export {
+  cloneRavelValue,
+  executeLiveProgram,
+  planLiveExecutions,
+  ravelValueIssue,
+  serializeRavelValue
+} from "./live.js";
 
 const clone = (value) => JSON.parse(JSON.stringify(value));
 
@@ -198,6 +208,7 @@ const diagnostic = (code, message, source, related = []) => ({
 });
 
 const validComponent = (value) => value === null || (typeof value === "string" && componentPattern.test(value));
+const validChunkPath = (value) => value === null || (typeof value === "string" && chunkPathPattern.test(value));
 
 export const formatChunkId = (identity) => {
   const prefix = identity.document === null ? "" : identity.document + "::";
@@ -223,7 +234,8 @@ export const parseChunkId = (input, { reference = false } = {}) => {
 
   let type = null;
   const typeIndex = remainder.lastIndexOf(".");
-  if (typeIndex !== -1) {
+  const pathIndex = remainder.lastIndexOf("/");
+  if (typeIndex > pathIndex) {
     type = remainder.slice(typeIndex + 1);
     remainder = remainder.slice(0, typeIndex);
     if (!componentPattern.test(type)) return null;
@@ -236,11 +248,16 @@ export const parseChunkId = (input, { reference = false } = {}) => {
     chunk = remainder.slice(0, minorIndex);
     minor = remainder.slice(minorIndex + 1);
     if (!componentPattern.test(minor)) return null;
+    if (chunk.endsWith("/") && /^\.{1,2}(?:\/|$)/.test(chunk)) chunk = chunk.slice(0, -1);
   }
 
   if (chunk === "") chunk = null;
-  if (!validComponent(chunk)) return null;
+  const relativePath = reference && typeof chunk === "string" && relativeChunkPathPattern.test(chunk)
+    ? chunk
+    : null;
+  if (relativePath === null && !validChunkPath(chunk)) return null;
   if (!validComponent(document) || !validComponent(minor) || !validComponent(type)) return null;
+  if (relativePath !== null && explicitDocument) return null;
 
   if (!reference && document === null && chunk === null) return null;
   if (!reference && document === null && chunk !== null && input.includes("::")) return null;
@@ -253,7 +270,14 @@ export const parseChunkId = (input, { reference = false } = {}) => {
   }
   if (reference && !explicitDocument && chunk === null && minor === null && type === null) return null;
 
-  return { document, chunk, minor, type, explicitDocument };
+  return {
+    document,
+    chunk,
+    minor,
+    type,
+    explicitDocument,
+    ...(relativePath === null ? {} : { relativePath })
+  };
 };
 
 const validateIdentity = (identity) => {
@@ -267,7 +291,7 @@ const validateIdentity = (identity) => {
     type: identity.type,
     explicitDocument: identity.document !== null
   };
-  if (!validComponent(result.document) || !validComponent(result.chunk) ||
+  if (!validComponent(result.document) || !validChunkPath(result.chunk) ||
       !validComponent(result.minor) || !validComponent(result.type)) {
     return null;
   }
@@ -483,6 +507,27 @@ const parseExpression = (expression, source, expressionOffset, diagnostics, { al
   };
 };
 
+/**
+ * Parse a definition-time transform pipeline without executing it.
+ * Adapters use this entry point so every markup dialect shares one grammar.
+ */
+export const parseDefinitionPipeline = (text, source) => {
+  const diagnostics = [];
+  if (typeof text !== "string" || !text.trim()) return { pipeline: [], diagnostics };
+  const expression = text.trim();
+  const pipeline = [];
+  for (const part of splitTopLevel(expression, "|")) {
+    const step = parsePipeStep(part, expression, source, 0, diagnostics);
+    if (!step) continue;
+    if (step.type !== "transform") {
+      diagnostics.push(diagnostic("RV120", "Definition pipelines accept transform calls only.", step.source ?? source));
+      continue;
+    }
+    pipeline.push({ name: step.name, arguments: step.arguments });
+  }
+  return { pipeline, diagnostics };
+};
+
 // An embedded reference inherits the indentation of its containing source line
 // on continuation lines. The first line is already positioned by the literal
 // text before the reference, so it deliberately remains untouched.
@@ -513,18 +558,103 @@ const applyContinuationIndentMapped = (value, indentation, source, chunk) => {
   return concatMapped(...parts);
 };
 
-export const parseChunk = (body, source) => {
+const firstUnescapedPipe = (value) => {
+  let escaped = false;
+  for (let index = 0; index < value.length; index += 1) {
+    if (escaped) escaped = false;
+    else if (value[index] === "\\") escaped = true;
+    else if (value[index] === "|") return index;
+  }
+  return -1;
+};
+
+const nowebExpression = (value, options) => {
+  const pipe = options.nowebPlus ? firstUnescapedPipe(value) : -1;
+  const authoredName = (pipe === -1 ? value : value.slice(0, pipe)).trim().replace(/\\\|/g, "|");
+  const canonicalName = options.referenceAliases?.[authoredName] ?? authoredName;
+  return pipe === -1
+    ? canonicalName
+    : canonicalName + " | " + value.slice(pipe + 1).trim();
+};
+
+export const parseChunk = (body, source, options = {}) => {
   const nodes = [];
   const diagnostics = [];
   let literalStart = 0;
   let index = 0;
 
   while (index < body.length) {
+    if (options.nowebReferences && body.startsWith("<<", index)) {
+      const end = body.indexOf(">>", index + 2);
+      if (end === -1) {
+        diagnostics.push(diagnostic("RV110", "Unterminated noweb chunk reference.", span(source, body, index, body.length)));
+        break;
+      }
+      if (literalStart < index) {
+        nodes.push({ type: "literal", value: body.slice(literalStart, index), source: span(source, body, literalStart, index) });
+      }
+      const authored = body.slice(index + 2, end);
+      const expression = nowebExpression(authored, options);
+      const parsed = parseExpression(expression, source, index + 2, diagnostics);
+      if (parsed) {
+        nodes.push({
+          ...parsed,
+          authoredReference: authored,
+          source: span(source, body, index + 2, end),
+          continuationIndent: continuationIndentAt(body, index)
+        });
+      }
+      index = end + 2;
+      literalStart = index;
+      continue;
+    }
+    const counted = options.underscoreReferences !== false && body[index] === "\\"
+      ? /^\\([1-9][0-9]*)_(["'`])/.exec(body.slice(index))
+      : null;
+    if (counted) {
+      const quote = counted[2];
+      const expressionStart = index + counted[0].length;
+      let end = expressionStart;
+      let escaped = false;
+      while (end < body.length) {
+        if (escaped) escaped = false;
+        else if (body[end] === "\\") escaped = true;
+        else if (body[end] === quote) break;
+        end += 1;
+      }
+      if (end >= body.length) {
+        diagnostics.push(diagnostic("RV110", "Unterminated counted chunk reference.", span(source, body, index, body.length)));
+        break;
+      }
+      if (literalStart < index) {
+        nodes.push({ type: "literal", value: body.slice(literalStart, index), source: span(source, body, literalStart, index) });
+      }
+      const expression = body.slice(expressionStart, end);
+      const parsed = parseExpression(expression, source, expressionStart, diagnostics, { allowDelay: false });
+      if (parsed) {
+        nodes.push({
+          type: "delay",
+          value: {
+            kind: "ravel-command-argument",
+            command: { type: "chunk", expression, value: parsed, source: span(source, body, expressionStart, end) }
+          },
+          expression,
+          phase: Number(counted[1]),
+          safeSymbol: undefined,
+          source: span(source, body, index, end + 1),
+          continuationIndent: continuationIndentAt(body, index)
+        });
+      }
+      index = end + 1;
+      literalStart = index;
+      continue;
+    }
     if (body[index] === "\\" && body[index + 1] === "_" && ["\"", "'", "`"].includes(body[index + 2])) {
       index += 3;
       continue;
     }
-    if (body[index] !== "_" || !["\"", "'", "`"].includes(body[index + 1])) {
+    if (options.underscoreReferences === false ||
+        body[index] !== "_" || !["\"", "'", "`"].includes(body[index + 1])) {
       index += 1;
       continue;
     }
@@ -565,8 +695,16 @@ export const parseChunk = (body, source) => {
 };
 
 const parseChunkFragments = (raw) => {
-  const parsed = parseChunk(raw.body, raw.source);
-  if (!Array.isArray(raw.fragments) || raw.fragments.length < 2 ||
+  const referenceSyntax = raw.metadata?.data?.ravel?.referenceSyntax;
+  const parsed = parseChunk(raw.body, raw.source, referenceSyntax && typeof referenceSyntax === "object"
+    ? {
+        nowebReferences: referenceSyntax.noweb === true,
+        nowebPlus: referenceSyntax.dialect === "noweb-plus",
+        underscoreReferences: referenceSyntax.underscore !== false,
+        referenceAliases: referenceSyntax.aliases
+      }
+    : {});
+  if (!Array.isArray(raw.fragments) || raw.fragments.length === 0 ||
       raw.fragments.some((fragment) => typeof fragment?.body !== "string" || !fragment.source) ||
       raw.fragments.map((fragment) => fragment.body).join("") !== raw.body) {
     return parsed;
@@ -583,16 +721,41 @@ const parseChunkFragments = (raw) => {
     start: source.range.start.offset - syntheticStart,
     end: source.range.end.offset - syntheticStart
   });
+  const fragmentSlice = (fragment, start, end) =>
+    fragment.precision === "coarse"
+      ? fragment.source
+      : span(
+          fragment.source,
+          fragment.body,
+          start - fragment.start,
+          end - fragment.start
+        );
   const remapLocation = (source) => {
     const range = relativeRange(source);
-    const fragment = fragments.find((entry) => range.start >= entry.start && range.end <= entry.end);
-    if (!fragment) return source;
-    return span(
-      fragment.source,
-      fragment.body,
-      range.start - fragment.start,
-      range.end - fragment.start
+    const overlapping = fragments.filter((entry) =>
+      range.start < entry.end && range.end > entry.start
+    );
+    if (!overlapping.length) return source;
+    const first = overlapping[0];
+    const last = overlapping.at(-1);
+    const firstSlice = fragmentSlice(
+      first,
+      Math.max(range.start, first.start),
+      Math.min(range.end, first.end)
+    );
+    const lastSlice = fragmentSlice(
+      last,
+      Math.max(range.start, last.start),
+      Math.min(range.end, last.end)
     );
+    if (firstSlice.uri !== lastSlice.uri) return firstSlice;
+    return {
+      uri: firstSlice.uri,
+      range: {
+        start: firstSlice.range.start,
+        end: lastSlice.range.end
+      }
+    };
   };
   const remapSources = (value) => {
     if (!value || typeof value !== "object") return value;
@@ -617,7 +780,8 @@ const parseChunkFragments = (raw) => {
       nodes.push({
         type: "literal",
         value: raw.body.slice(start, end),
-        source: span(fragment.source, fragment.body, start - fragment.start, end - fragment.start)
+        source: fragmentSlice(fragment, start, end),
+        precision: fragment.precision ?? "exact"
       });
     }
   }
@@ -709,6 +873,30 @@ const applyTransform = (value, step, diagnostics, transforms, context = {}) => {
   if (step.name === "quote-reference") {
     return value.replace(/_(["'\x60])/g, "\\_$1");
   }
+  if (step.name === "jsontext") {
+    if (step.arguments.length > 1 ||
+        (step.arguments.length === 1 && typeof step.arguments[0] !== "string")) {
+      diagnostics.push(diagnostic("RV120", "jsontext accepts no arguments or one string key.", step.source));
+      return value;
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(value);
+    } catch {
+      diagnostics.push(diagnostic("RV120", "jsontext requires a live-result JSON value.", step.source));
+      return value;
+    }
+    if (step.arguments.length === 0) return JSON.stringify(parsed);
+
+    const key = step.arguments[0];
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed) ||
+        !Object.hasOwn(parsed, key)) {
+      diagnostics.push(diagnostic("RV120", "jsontext could not find object key " + JSON.stringify(key) + ".", step.source));
+      return value;
+    }
+    const selected = parsed[key];
+    return typeof selected === "string" ? selected : JSON.stringify(selected);
+  }
 
   const transform = customTransform(transforms, step.name);
   if (typeof transform === "function") {
@@ -797,9 +985,22 @@ const definitionFromComposeEmission = (owner, capture, emit) => {
 };
 
 const resolveTarget = (target, owner, definitions) => {
-  const withOwnerChunk = target.chunk === null && !target.explicitDocument
-    ? owner.identity.chunk
-    : target.chunk;
+  let withOwnerChunk;
+  if (target.relativePath) {
+    const segments = (owner.identity.chunk ?? "").split("/").filter((segment, index, entries) =>
+      segment || (index > 0 && index < entries.length - 1)
+    );
+    for (const segment of target.relativePath.split("/")) {
+      if (segment === ".") continue;
+      if (segment === "..") segments.pop();
+      else segments.push(segment);
+    }
+    withOwnerChunk = segments.join("/");
+  } else {
+    withOwnerChunk = target.chunk === null && !target.explicitDocument
+      ? owner.identity.chunk
+      : target.chunk;
+  }
   const parts = {
     chunk: withOwnerChunk,
     minor: target.minor,
@@ -829,6 +1030,11 @@ export const transformGraph = (pretransform, options = {}) => {
   const delayTokenCorpus = JSON.stringify(pretransform);
   const compareText = (left, right) => left < right ? -1 : left > right ? 1 : 0;
   const orderedEntries = (value) => Object.entries(value).sort(([left], [right]) => compareText(left, right));
+  const liveExecutions = options.liveResults?.executions ?? options.liveResults;
+  const liveExecution = (id) => liveExecutions instanceof Map
+    ? liveExecutions.get(id)
+    : liveExecutions?.[id];
+  const isExecutable = (definition) => definition?.metadata?.data?.ravel?.run === true;
 
   const normalizeDefinitionPipeline = (steps, source) => {
     if (steps === undefined) return [];
@@ -1067,10 +1273,47 @@ export const transformGraph = (pretransform, options = {}) => {
       to: resolved,
       source: clone(node.source)
     };
-    let value = mappedValue(dependency.value, dependency.segments.map((segment) => ({
-      ...clone(segment),
-      via: [...(segment.via ?? []), reference]
-    })));
+    const resolvedDefinition = definitions.get(resolved);
+    const ownerIsExecutable = isExecutable(owner.definition);
+    const shouldUseLiveResult = isExecutable(resolvedDefinition) && !ownerIsExecutable;
+    if (shouldUseLiveResult && options.deferLiveResults && !liveExecutions) {
+      return mappedValue();
+    }
+    let value;
+    if (shouldUseLiveResult && liveExecutions) {
+      const execution = liveExecution(resolved);
+      if (execution?.status !== "succeeded") {
+        diagnostics.push(diagnostic("RV140", "Live result is unavailable for reference " + node.reference + ".", node.source));
+        return mappedValue();
+      }
+      const first = node.pipeline[0];
+      const hasJsonTextBoundary = first?.type === "transform" && first.name === "jsontext";
+      const serializeWholeValue = hasJsonTextBoundary && first.arguments.length === 0;
+      if (typeof execution.value === "string" && !serializeWholeValue) {
+        value = coarseMapped(execution.value, resolvedDefinition.source, resolved, "live-result", [reference]);
+      } else if (hasJsonTextBoundary) {
+        value = coarseMapped(
+          execution.serialized ?? JSON.stringify(execution.value),
+          resolvedDefinition.source,
+          resolved,
+          "live-result",
+          [reference]
+        );
+      } else {
+        diagnostics.push(diagnostic(
+          "RV140",
+          "Non-string live result " + resolved +
+            " requires jsontext() or jsontext(\"key\") before ordinary Ravel processing.",
+          node.source
+        ));
+        return mappedValue();
+      }
+    } else {
+      value = mappedValue(dependency.value, dependency.segments.map((segment) => ({
+        ...clone(segment),
+        via: [...(segment.via ?? []), reference]
+      })));
+    }
     return evaluatePipeline(value, node.pipeline, owner);
   };
 
@@ -1136,7 +1379,13 @@ export const transformGraph = (pretransform, options = {}) => {
 
   const evaluateNode = (node, owner) => {
     if (node.type === "literal") {
-      return sourceSegment(node.value, node.source, owner.definition.id, "literal");
+      return sourceSegment(
+        node.value,
+        node.source,
+        owner.definition.id,
+        "literal",
+        node.precision ?? "exact"
+      );
     }
     if (node.type === "delay") return evaluateDelay(node, owner);
     const value = evaluateExpression(node, owner);
@@ -1295,6 +1544,7 @@ export const transformGraph = (pretransform, options = {}) => {
       value: value.text,
       segments: value.segments,
       metadata: definition.metadata,
+      source: definition.source,
       dependencies: [...owner.dependencies].sort(),
       // References retain authored source order, which is deterministic and is
       // more useful for explaining a chunk than lexical target order.
@@ -1327,7 +1577,26 @@ export const transformGraph = (pretransform, options = {}) => {
       diagnostics.push(diagnostic("RV130", "out requires a fully qualified existing source chunk ID.", directive.source));
       continue;
     }
-    const chunk = evaluate(id, directive.source);
+    let chunk = evaluate(id, directive.source);
+    const definition = definitions.get(id);
+    if (isExecutable(definition) && liveExecutions) {
+      const execution = liveExecution(id);
+      if (execution?.status !== "succeeded") {
+        diagnostics.push(diagnostic("RV140", "Live result is unavailable for output " + name + ".", directive.source));
+        continue;
+      }
+      if (typeof execution.value !== "string") {
+        diagnostics.push(diagnostic(
+          "RV140",
+          "Output " + name +
+            " requires a live string; use jsontext() or jsontext(\"key\") in a derived chunk.",
+          directive.source
+        ));
+        continue;
+      }
+      const mapped = coarseMapped(execution.value, definition.source, id, "live-result");
+      chunk = { ...chunk, value: mapped.text, segments: mapped.segments };
+    }
     if (deliverables[name]) {
       diagnostics.push(diagnostic("RV101", "Duplicate out deliverable: " + name, directive.source));
       continue;