@pieceful/ravel-core 0.2.0 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pieceful/ravel-core",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "type": "module",
   "description": "Ravel chunk syntax, resolution, graph evaluation, and diagnostics.",
   "license": "MIT",

package/src/directives.d.ts

@@ -5,5 +5,5 @@ export function append(reference: string, source: SourceLocation): unknown;
 export function newline(count: number, source: SourceLocation): unknown;
 export function pipe(steps: unknown[], source: SourceLocation): unknown;
 export function pass(steps: unknown[], source: SourceLocation): unknown;
-export function createDirective(name: string, value: unknown, source: SourceLocation): unknown;
-export function aliasDirective(name: string, reference: string, source: SourceLocation): unknown;
+export function createDirective(name: string, value: unknown[] | string, source: SourceLocation, document?: string): { kind: "create"; document?: string; name: string; compose?: unknown[]; body?: string; source: SourceLocation };
+export function aliasDirective(name: string, reference: string, source: SourceLocation, document?: string): { kind: "alias"; document?: string; name: string; reference: string; source: SourceLocation };

package/src/directives.js

@@ -12,5 +12,18 @@ export const pass = (steps, source) => ({ kind: "pass", steps, source });
  * document's canonical chunk identity. Alias is intentionally the only
  * directive that may introduce an alternate graph name for an existing node.
  */
-export const createDirective = (name, value, source) => ({ kind: "create", name, value, source });
-export const aliasDirective = (name, reference, source) => ({ kind: "alias", name, reference, source });
+export const createDirective = (name, value, source, document) => ({
+  kind: "create",
+  ...(document === undefined ? {} : { document }),
+  name,
+  // An array is a compose step list; a string is a chunk body.
+  ...(Array.isArray(value) ? { compose: value } : { body: String(value ?? "") }),
+  source
+});
+export const aliasDirective = (name, reference, source, document) => ({
+  kind: "alias",
+  ...(document === undefined ? {} : { document }),
+  name,
+  reference,
+  source
+});

package/src/index.d.ts

@@ -104,6 +104,7 @@ export interface ChunkReferenceSyntaxOptions {
   referenceAliases?: Record<string, string>;
 }
 export function parseChunk(body: string, source: SourceLocation, options?: ChunkReferenceSyntaxOptions): { nodes: unknown[]; diagnostics: Diagnostic[] };
+export function nowebReferenceAt(body: string, index: number): { authored: string; end: number } | null;
 export function parseDefinitionPipeline(text: string, source: SourceLocation): { pipeline: TransformCall[]; diagnostics: Diagnostic[] };
 export function combineMaps(maps: RavelMap[]): PretransformGraph;
 export function transformGraph(graph: PretransformGraph, options?: {
@@ -137,8 +138,8 @@ export function append(reference: string, source: SourceLocation): unknown;
 export function newline(count: number, source: SourceLocation): unknown;
 export function pipe(steps: unknown[], source: SourceLocation): unknown;
 export function pass(steps: unknown[], source: SourceLocation): unknown;
-export function createDirective(name: string, value: unknown, source: SourceLocation): unknown;
-export function aliasDirective(name: string, reference: string, source: SourceLocation): unknown;
+export function createDirective(name: string, value: unknown[] | string, source: SourceLocation, document?: string): { kind: "create"; document?: string; name: string; compose?: unknown[]; body?: string; source: SourceLocation };
+export function aliasDirective(name: string, reference: string, source: SourceLocation, document?: string): { kind: "alias"; document?: string; name: string; reference: string; source: SourceLocation };
 export function parseRavelDirectiveBlock(text: string, options: {
   document: string;
   sourceAt(start: number, end: number): SourceLocation;

package/src/index.js

@@ -177,18 +177,32 @@ const replaceMappedOnce = (value, search, replacement) => {
   );
 };
 
+// Parsing asks for many positions in the same chunk body, so keep the newline
+// offsets of the most recent text and binary-search them instead of rescanning.
+const lineIndexCache = new Map();
+const lineStartsOf = (text) => {
+  let starts = lineIndexCache.get(text);
+  if (starts) return starts;
+  starts = [];
+  for (let i = text.indexOf("\n"); i !== -1; i = text.indexOf("\n", i + 1)) starts.push(i + 1);
+  // A handful of entries covers a chunk body and the expressions inside it.
+  if (lineIndexCache.size >= 16) lineIndexCache.delete(lineIndexCache.keys().next().value);
+  lineIndexCache.set(text, starts);
+  return starts;
+};
+
 const advance = (start, text, index) => {
-  let line = start.line;
-  let column = start.column;
-  for (let i = 0; i < index; i += 1) {
-    if (text[i] === "\n") {
-      line += 1;
-      column = 0;
-    } else {
-      column += 1;
-    }
+  const starts = lineStartsOf(text);
+  let low = 0;
+  let high = starts.length;
+  while (low < high) {
+    const middle = (low + high) >> 1;
+    if (starts[middle] <= index) low = middle + 1;
+    else high = middle;
   }
-  return { line, column, offset: start.offset + index };
+  return low === 0
+    ? { line: start.line, column: start.column + index, offset: start.offset + index }
+    : { line: start.line + low, column: index - starts[low - 1], offset: start.offset + index };
 };
 
 const span = (source, text, start, end) => ({
@@ -382,9 +396,10 @@ const parseEmitSuffix = (value) => {
   return { minor, type, inheritMinor: false };
 };
 
-const parsePipeStep = (part, expression, source, expressionOffset, diagnostics) => {
+// `text` is the string that `expressionOffset` and `part` offsets index into.
+const parsePipeStep = (part, text, source, expressionOffset, diagnostics) => {
   const match = /^([a-z][a-z0-9-]*)\s*(?:\((.*)\))?$/s.exec(part.value);
-  const location = span(source, expression, expressionOffset + part.start, expressionOffset + part.end);
+  const location = span(source, text, expressionOffset + part.start, expressionOffset + part.end);
   if (!match) {
     diagnostics.push(diagnostic("RV110", "Malformed pipeline step: " + part.value, location));
     return null;
@@ -430,7 +445,7 @@ const parsePipeStep = (part, expression, source, expressionOffset, diagnostics)
       diagnostics.push(diagnostic("RV121", "ch requires one quoted chunk expression.", location));
       return null;
     }
-    const parsed = parseExpression(argumentsValue[0], source, expressionOffset + part.start, diagnostics, { allowDelay: false });
+    const parsed = parseExpression(argumentsValue[0], source, expressionOffset + part.start, diagnostics, { allowDelay: false, text });
     if (!parsed) return null;
     return { type: "chunk", expression: argumentsValue[0], value: parsed, source: location };
   }
@@ -438,7 +453,7 @@ const parsePipeStep = (part, expression, source, expressionOffset, diagnostics)
   const argumentsParsed = rawArguments.map((argument) => {
     const value = parseValue(argument.value);
     if (typeof value !== "undefined") return value;
-    const nested = parsePipeStep({ value: argument.value, start: part.start + argument.start, end: part.start + argument.end }, expression, source, expressionOffset, diagnostics);
+    const nested = parsePipeStep({ value: argument.value, start: part.start + argument.start, end: part.start + argument.end }, text, source, expressionOffset, diagnostics);
     return nested?.type === "text" || nested?.type === "chunk"
       ? { kind: "ravel-command-argument", command: nested }
       : undefined;
@@ -450,12 +465,14 @@ const parsePipeStep = (part, expression, source, expressionOffset, diagnostics)
   return { type: name === "delay" ? "delay" : "transform", name, arguments: argumentsParsed, source: location };
 };
 
-const parseExpression = (expression, source, expressionOffset, diagnostics, { allowDelay = true } = {}) => {
+// `text` is the chunk body that `expressionOffset` indexes into; positions
+// must be computed against it, not against the expression alone.
+const parseExpression = (expression, source, expressionOffset, diagnostics, { allowDelay = true, text = expression } = {}) => {
   const parts = splitTopLevel(expression, "|");
   const reference = parts.shift();
   const target = parseChunkId(reference?.value, { reference: true });
   const pipeline = parts
-    .map((part) => parsePipeStep(part, expression, source, expressionOffset, diagnostics))
+    .map((part) => parsePipeStep(part, text, source, expressionOffset, diagnostics))
     .filter(Boolean);
   if (!reference || !target) {
     const delay = pipeline.length === 1 && pipeline[0].type === "delay" ? pipeline[0] : null;
@@ -474,7 +491,7 @@ const parseExpression = (expression, source, expressionOffset, diagnostics, { al
         expression: typeof value === "string" ? value : command.expression ?? command.value,
         phase,
         safeSymbol,
-        source: span(source, expression, expressionOffset, expressionOffset + expression.length)
+        source: span(source, text, expressionOffset, expressionOffset + expression.length)
       };
     }
     if (reference?.value === "" && pipeline.length && pipeline[0].type !== "delay" &&
@@ -483,13 +500,13 @@ const parseExpression = (expression, source, expressionOffset, diagnostics, { al
         diagnostics.push(diagnostic("RV121", "delay may only appear as the sole command in _\"|delay(...)\".", pipeline.find((step) => step.type === "delay").source));
         return null;
       }
-      return { type: "pipeline", pipeline, source: span(source, expression, expressionOffset, expressionOffset + expression.length) };
+      return { type: "pipeline", pipeline, source: span(source, text, expressionOffset, expressionOffset + expression.length) };
     }
     if (pipeline.some((step) => step.type === "delay")) {
       diagnostics.push(diagnostic("RV121", "delay may only appear as the sole command in _\"|delay(...)\".", pipeline.find((step) => step.type === "delay").source));
       return null;
     }
-    diagnostics.push(diagnostic("RV110", "Malformed chunk reference: " + (reference?.value ?? ""), span(source, expression, expressionOffset, expressionOffset + expression.length)));
+    diagnostics.push(diagnostic("RV110", "Malformed chunk reference: " + (reference?.value ?? ""), span(source, text, expressionOffset, expressionOffset + expression.length)));
     return null;
   }
 
@@ -503,7 +520,7 @@ const parseExpression = (expression, source, expressionOffset, diagnostics, { al
     reference: reference.value,
     target,
     pipeline,
-    source: span(source, expression, expressionOffset, expressionOffset + expression.length)
+    source: span(source, text, expressionOffset, expressionOffset + expression.length)
   };
 };
 
@@ -532,7 +549,10 @@ export const parseDefinitionPipeline = (text, source) => {
 // on continuation lines. The first line is already positioned by the literal
 // text before the reference, so it deliberately remains untouched.
 const continuationIndentAt = (body, index) => {
-  const lineStart = Math.max(body.lastIndexOf("\n", index - 1), body.lastIndexOf("\r", index - 1)) + 1;
+  // Walk back only to the start of this line; lastIndexOf for an absent "\r"
+  // would rescan the whole body for every reference.
+  let lineStart = index;
+  while (lineStart > 0 && body[lineStart - 1] !== "\n" && body[lineStart - 1] !== "\r") lineStart -= 1;
   return /^[\t ]*/.exec(body.slice(lineStart, index))[0];
 };
 
@@ -577,6 +597,26 @@ const nowebExpression = (value, options) => {
     : canonicalName + " | " + value.slice(pipe + 1).trim();
 };
 
+/**
+ * Recognize a noweb-style `<<name>>` reference starting at `index`.
+ *
+ * A reference must close on the same line and its name must be non-empty
+ * without leading or trailing whitespace, so shift operators such as
+ * `std::cout << x` or `a << 2; b >> 1` remain ordinary code. Returns the
+ * authored text and the offset of the closing `>>`, or null.
+ */
+const countedReferencePattern = /\\([1-9][0-9]*)_(["'`])/y;
+
+export const nowebReferenceAt = (body, index) => {
+  if (!body.startsWith("<<", index)) return null;
+  const lineEnd = body.indexOf("\n", index + 2);
+  const end = body.indexOf(">>", index + 2);
+  if (end === -1 || (lineEnd !== -1 && end > lineEnd)) return null;
+  const authored = body.slice(index + 2, end);
+  if (!authored || authored !== authored.trim() || authored.startsWith("<")) return null;
+  return { authored, end };
+};
+
 export const parseChunk = (body, source, options = {}) => {
   const nodes = [];
   const diagnostics = [];
@@ -584,18 +624,23 @@ export const parseChunk = (body, source, options = {}) => {
   let index = 0;
 
   while (index < body.length) {
-    if (options.nowebReferences && body.startsWith("<<", index)) {
-      const end = body.indexOf(">>", index + 2);
-      if (end === -1) {
-        diagnostics.push(diagnostic("RV110", "Unterminated noweb chunk reference.", span(source, body, index, body.length)));
-        break;
+    if (options.nowebReferences && body.startsWith("@<<", index)) {
+      // `@<<` is noweb's escape for a literal `<<`; drop the `@`.
+      if (literalStart < index) {
+        nodes.push({ type: "literal", value: body.slice(literalStart, index), source: span(source, body, literalStart, index) });
       }
+      literalStart = index + 1;
+      index += 3;
+      continue;
+    }
+    const noweb = options.nowebReferences ? nowebReferenceAt(body, index) : null;
+    if (noweb) {
+      const { end, authored } = noweb;
       if (literalStart < index) {
         nodes.push({ type: "literal", value: body.slice(literalStart, index), source: span(source, body, literalStart, index) });
       }
-      const authored = body.slice(index + 2, end);
       const expression = nowebExpression(authored, options);
-      const parsed = parseExpression(expression, source, index + 2, diagnostics);
+      const parsed = parseExpression(expression, source, index + 2, diagnostics, { text: body });
       if (parsed) {
         nodes.push({
           ...parsed,
@@ -608,9 +653,11 @@ export const parseChunk = (body, source, options = {}) => {
       literalStart = index;
       continue;
     }
-    const counted = options.underscoreReferences !== false && body[index] === "\\"
-      ? /^\\([1-9][0-9]*)_(["'`])/.exec(body.slice(index))
-      : null;
+    let counted = null;
+    if (options.underscoreReferences !== false && body[index] === "\\") {
+      countedReferencePattern.lastIndex = index;
+      counted = countedReferencePattern.exec(body);
+    }
     if (counted) {
       const quote = counted[2];
       const expressionStart = index + counted[0].length;
@@ -630,7 +677,7 @@ export const parseChunk = (body, source, options = {}) => {
         nodes.push({ type: "literal", value: body.slice(literalStart, index), source: span(source, body, literalStart, index) });
       }
       const expression = body.slice(expressionStart, end);
-      const parsed = parseExpression(expression, source, expressionStart, diagnostics, { allowDelay: false });
+      const parsed = parseExpression(expression, source, expressionStart, diagnostics, { allowDelay: false, text: body });
       if (parsed) {
         nodes.push({
           type: "delay",
@@ -682,7 +729,7 @@ export const parseChunk = (body, source, options = {}) => {
       nodes.push({ type: "literal", value: body.slice(literalStart, index), source: span(source, body, literalStart, index) });
     }
     const expression = body.slice(index + 2, end);
-    const parsed = parseExpression(expression, source, index + 2, diagnostics);
+    const parsed = parseExpression(expression, source, index + 2, diagnostics, { text: body });
     if (parsed) nodes.push({ ...parsed, continuationIndent: continuationIndentAt(body, index) });
     index = end + 1;
     literalStart = index;
@@ -901,9 +948,12 @@ const applyTransform = (value, step, diagnostics, transforms, context = {}) => {
   const transform = customTransform(transforms, step.name);
   if (typeof transform === "function") {
     try {
-      const result = transform(value, { ...context, arguments: clone(step.arguments), transform: step.name });
+      // `parts: true` invites a transform to describe which output came from
+      // its input or arguments; a plain string result is always accepted.
+      const result = transform(value, { ...context, arguments: clone(step.arguments), transform: step.name, parts: true });
       if (typeof result === "string") return result;
-      diagnostics.push(diagnostic("RV121", "Transform " + step.name + " must return a string.", step.source));
+      if (Array.isArray(result?.parts)) return result;
+      diagnostics.push(diagnostic("RV121", "Transform " + step.name + " must return a string or { parts }.", step.source));
     } catch (error) {
       diagnostics.push(diagnostic("RV121", "Transform " + step.name + " failed: " + (error?.message ?? String(error)), step.source));
     }
@@ -1235,14 +1285,52 @@ export const transformGraph = (pretransform, options = {}) => {
     return (hash >>> 0).toString(36).toUpperCase();
   };
 
+  // Assemble a `{ parts }` transform result: input and argument ranges keep
+  // their provenance; literal parts are attributed to the transform step.
+  const mappedTransformParts = (input, parts, step, definition, argumentValues = []) => {
+    const derivation = { kind: "transform", name: step.name, source: clone(step.source) };
+    const argumentMapped = (index) => argumentValues[index] ??
+      (typeof step.arguments[index] === "string" ? coarseMapped(step.arguments[index], step.source, definition.id, "transform", [derivation]) : null);
+    const validRange = (text, start, end) => Number.isInteger(start) && Number.isInteger(end) && start >= 0 && start <= end && end <= text.length;
+    let value = mappedValue();
+    for (const part of parts) {
+      let piece = null;
+      if (typeof part === "string") {
+        piece = coarseMapped(part, step.source, definition.id, "transform", [derivation]);
+      } else if (Array.isArray(part?.input) && validRange(input.text, part.input[0], part.input[1])) {
+        piece = withMappedDerivation(sliceMapped(input, part.input[0], part.input[1]), derivation);
+      } else if (Number.isInteger(part?.argument) && typeof part.text === "string") {
+        const source = argumentMapped(part.argument);
+        piece = source
+          ? coarseDerivedMapped(part.text, step.source, definition.id, "transform", [derivation], source)
+          : coarseMapped(part.text, step.source, definition.id, "transform", [derivation]);
+      } else if (Number.isInteger(part?.argument)) {
+        const source = argumentMapped(part.argument);
+        if (source && validRange(source.text, part.start, part.end)) {
+          piece = withMappedDerivation(sliceMapped(source, part.start, part.end), derivation);
+        }
+      }
+      if (!piece) {
+        diagnostics.push(diagnostic("RV121", "Transform " + step.name + " returned an invalid part.", step.source));
+        return null;
+      }
+      value = concatMapped(value, piece);
+    }
+    return value;
+  };
+
   const evaluateTransformMapped = (input, step, definition, context = {}) => {
+    const { argumentValues, ...transformContext } = context;
     const output = applyTransform(
       input.text,
       step,
       diagnostics,
       options.transforms,
-      { chunk: definition, ...context }
+      { chunk: definition, ...transformContext }
     );
+    if (typeof output !== "string") {
+      return mappedTransformParts(input, output.parts, step, definition, argumentValues) ?? input;
+    }
     return mapBuiltinTransform(input, output, step, definition.id) ?? coarseDerivedMapped(
       output,
       step.source,
@@ -1358,8 +1446,16 @@ export const transformGraph = (pretransform, options = {}) => {
       } else if (step.type === "chunk") {
         value = evaluateExpression(step.value, owner);
       } else if (step.type === "transform") {
-        const argumentsValue = step.arguments.map((argument) => evaluateArgument(argument, owner));
-        value = evaluateTransformMapped(value, { ...step, arguments: argumentsValue }, owner.definition);
+        // Evaluate each ch()/text() argument once, keeping its mapped value so
+        // a parts-aware transform can carry its provenance through.
+        const argumentValues = step.arguments.map((argument) => {
+          const command = argument?.kind === "ravel-command-argument" ? argument.command : null;
+          if (command?.type === "chunk") return evaluateExpression(command.value, owner);
+          if (command?.type === "text") return coarseMapped(command.value, command.source ?? step.source, owner.definition.id, "text");
+          return null;
+        });
+        const argumentsValue = step.arguments.map((argument, index) => argumentValues[index]?.text ?? evaluateArgument(argument, owner));
+        value = evaluateTransformMapped(value, { ...step, arguments: argumentsValue }, owner.definition, { argumentValues });
       }
     }
     return value;