@tinta-lang/syntax 0.0.0-stage → 0.1.0-dev.0

package/vendor/micromark-core-commonmark/lib/code-indented.js

@@ -0,0 +1,159 @@
+/**
+ * @import {
+ *   Construct,
+ *   State,
+ *   TokenizeContext,
+ *   Tokenizer
+ * } from 'micromark-util-types'
+ */
+
+import { factorySpaceMinMax } from 'micromark-factory-space';
+import { markdownLineEnding, markdownSpace } from 'micromark-util-character';
+/** @type {Construct} */
+export const codeIndented = {
+  name: 'codeIndented',
+  tokenize: tokenizeCodeIndented
+};
+
+/** @type {Construct} */
+const furtherStart = {
+  partial: true,
+  tokenize: tokenizeFurtherStart
+};
+
+/**
+ * @this {TokenizeContext}
+ *   Context.
+ * @type {Tokenizer}
+ */
+function tokenizeCodeIndented(effects, ok, nok) {
+  return start;
+
+  /**
+   * Start of code (indented).
+   *
+   * > **Parsing note**: it is not needed to check if this first line is a
+   * > filled line (that it has a non-whitespace character), because blank lines
+   * > are parsed already, so we never run into that.
+   *
+   * ```markdown
+   * > |     aaa
+   *     ^
+   * ```
+   *
+   * @type {State}
+   */
+  function start(code) {
+    // To do: manually check if interrupting like `markdown-rs`.
+
+    effects.enter("codeIndented");
+    return factorySpaceMinMax(effects, atBreak, nok, "linePrefix", 4, 4)(code);
+  }
+
+  /**
+   * At a break.
+   *
+   * ```markdown
+   * > |     aaa
+   *         ^  ^
+   * ```
+   *
+   * @type {State}
+   */
+  function atBreak(code) {
+    if (code === null) {
+      return after(code);
+    }
+    if (markdownLineEnding(code)) {
+      return effects.attempt(furtherStart, atBreak, after)(code);
+    }
+    effects.enter("codeFlowValue");
+    return inside(code);
+  }
+
+  /**
+   * In code content.
+   *
+   * ```markdown
+   * > |     aaa
+   *         ^^^^
+   * ```
+   *
+   * @type {State}
+   */
+  function inside(code) {
+    if (code === null || markdownLineEnding(code)) {
+      effects.exit("codeFlowValue");
+      return atBreak(code);
+    }
+    effects.consume(code);
+    return inside;
+  }
+
+  /** @type {State} */
+  function after(code) {
+    effects.exit("codeIndented");
+    // To do: allow interrupting like `markdown-rs`.
+    // Feel free to interrupt.
+    // tokenizer.interrupt = false
+    return ok(code);
+  }
+}
+
+/**
+ * @this {TokenizeContext}
+ *   Context.
+ * @type {Tokenizer}
+ */
+function tokenizeFurtherStart(effects, ok, nok) {
+  const self = this;
+  return furtherStart;
+
+  /**
+   * At eol, trying to parse another indent.
+   *
+   * ```markdown
+   * > |     aaa
+   *            ^
+   *   |     bbb
+   * ```
+   *
+   * @type {State}
+   */
+  function furtherStart(code) {
+    // To do: improve `lazy` / `pierce` handling.
+    // If this is a lazy line, it can’t be code.
+    if (self.parser.lazy[self.now().line]) {
+      return nok(code);
+    }
+    if (markdownLineEnding(code)) {
+      effects.enter("lineEnding");
+      effects.consume(code);
+      effects.exit("lineEnding");
+      return furtherStart;
+    }
+
+    // To do: the code here in `micromark-js` is a bit different from
+    // `markdown-rs` because there it can attempt spaces.
+    // We can’t yet.
+    return factorySpaceMinMax(effects, ok, onNotEnoughPrefix, "linePrefix", 4, 4)(code);
+  }
+
+  /**
+   * After not enough of a prefix.
+   *
+   * A following line ending is another (potentially blank) line to try,
+   * anything else means this isn’t a continuation.
+   *
+   * ```markdown
+   * > |     aaa
+   *
+   *   |     bbb
+   * ```
+   *
+   * @type {State}
+   */
+  function onNotEnoughPrefix(code) {
+    return markdownLineEnding(code) ? furtherStart(code) : nok(code);
+  }
+}

package/vendor/micromark-core-commonmark/lib/code-text.js

@@ -0,0 +1,225 @@
+/**
+ * @import {
+ *   Construct,
+ *   Previous,
+ *   Resolver,
+ *   State,
+ *   TokenizeContext,
+ *   Tokenizer,
+ *   Token
+ * } from 'micromark-util-types'
+ */
+
+import { markdownLineEnding } from 'micromark-util-character';
+/** @type {Construct} */
+export const codeText = {
+  name: 'codeText',
+  previous,
+  resolve: resolveCodeText,
+  tokenize: tokenizeCodeText
+};
+
+// To do: next major: don’t resolve, like `markdown-rs`.
+/** @type {Resolver} */
+function resolveCodeText(events) {
+  let tailExitIndex = events.length - 4;
+  let headEnterIndex = 3;
+  /** @type {number} */
+  let index;
+  /** @type {number | undefined} */
+  let enter;
+
+  // If we start and end with an EOL or a space.
+  if ((events[headEnterIndex][1].type === "lineEnding" || events[headEnterIndex][1].type === 'space') && (events[tailExitIndex][1].type === "lineEnding" || events[tailExitIndex][1].type === 'space')) {
+    index = headEnterIndex;
+
+    // And we have data.
+    while (++index < tailExitIndex) {
+      if (events[index][1].type === "codeTextData") {
+        // Then we have padding.
+        events[headEnterIndex][1].type = "codeTextPadding";
+        events[tailExitIndex][1].type = "codeTextPadding";
+        headEnterIndex += 2;
+        tailExitIndex -= 2;
+        break;
+      }
+    }
+  }
+
+  // Merge adjacent spaces and data.
+  index = headEnterIndex - 1;
+  tailExitIndex++;
+  while (++index <= tailExitIndex) {
+    if (enter === undefined) {
+      if (index !== tailExitIndex && events[index][1].type !== "lineEnding") {
+        enter = index;
+      }
+    } else if (index === tailExitIndex || events[index][1].type === "lineEnding") {
+      events[enter][1].type = "codeTextData";
+      if (index !== enter + 2) {
+        events[enter][1].end = events[index - 1][1].end;
+        events.splice(enter + 2, index - enter - 2);
+        tailExitIndex -= index - enter - 2;
+        index = enter + 2;
+      }
+      enter = undefined;
+    }
+  }
+  return events;
+}
+
+/**
+ * @this {TokenizeContext}
+ *   Context.
+ * @type {Previous}
+ */
+function previous(code) {
+  // If there is a previous code, there will always be a tail.
+  return code !== 96 || this.events[this.events.length - 1][1].type === "characterEscape";
+}
+
+/**
+ * @this {TokenizeContext}
+ *   Context.
+ * @type {Tokenizer}
+ */
+function tokenizeCodeText(effects, ok, nok) {
+  const self = this;
+  let sizeOpen = 0;
+  /** @type {number} */
+  let size;
+  /** @type {Token} */
+  let token;
+  return start;
+
+  /**
+   * Start of code (text).
+   *
+   * ```markdown
+   * > | `a`
+   *     ^
+   * > | \`a`
+   *      ^
+   * ```
+   *
+   * @type {State}
+   */
+  function start(code) {
+    effects.enter("codeText");
+    effects.enter("codeTextSequence");
+    return sequenceOpen(code);
+  }
+
+  /**
+   * In opening sequence.
+   *
+   * ```markdown
+   * > | `a`
+   *     ^
+   * ```
+   *
+   * @type {State}
+   */
+  function sequenceOpen(code) {
+    if (code === 96) {
+      effects.consume(code);
+      sizeOpen++;
+      return sequenceOpen;
+    }
+    effects.exit("codeTextSequence");
+    return between(code);
+  }
+
+  /**
+   * Between something and something else.
+   *
+   * ```markdown
+   * > | `a`
+   *      ^^
+   * ```
+   *
+   * @type {State}
+   */
+  function between(code) {
+    // EOF.
+    if (code === null) {
+      return nok(code);
+    }
+
+    // To do: next major: don’t do spaces in resolve, but when compiling,
+    // like `markdown-rs`.
+    // Tabs don’t work, and virtual spaces don’t make sense.
+    if (code === 32) {
+      effects.enter('space');
+      effects.consume(code);
+      effects.exit('space');
+      return between;
+    }
+
+    // Closing fence? Could also be data.
+    if (code === 96) {
+      token = effects.enter("codeTextSequence");
+      size = 0;
+      return sequenceClose(code);
+    }
+    if (markdownLineEnding(code)) {
+      effects.enter("lineEnding");
+      effects.consume(code);
+      effects.exit("lineEnding");
+      return between;
+    }
+
+    // Data.
+    effects.enter("codeTextData");
+    return data(code);
+  }
+
+  /**
+   * In data.
+   *
+   * ```markdown
+   * > | `a`
+   *      ^
+   * ```
+   *
+   * @type {State}
+   */
+  function data(code) {
+    if (code === null || code === 32 || code === 96 || markdownLineEnding(code)) {
+      effects.exit("codeTextData");
+      return between(code);
+    }
+    effects.consume(code);
+    return data;
+  }
+
+  /**
+   * In closing sequence.
+   *
+   * ```markdown
+   * > | `a`
+   *       ^
+   * ```
+   *
+   * @type {State}
+   */
+  function sequenceClose(code) {
+    // More.
+    if (code === 96) {
+      effects.consume(code);
+      size++;
+      return sequenceClose;
+    }
+
+    // Done!
+    if (size === sizeOpen) {
+      effects.exit("codeTextSequence");
+      effects.exit("codeText");
+      return ok(code);
+    }
+
+    // More or less accents: mark as data.
+    token.type = "codeTextData";
+    return data(code);
+  }
+}

package/vendor/micromark-core-commonmark/lib/content.js

@@ -0,0 +1,163 @@
+/**
+ * @import {
+ *   Construct,
+ *   Resolver,
+ *   State,
+ *   TokenizeContext,
+ *   Tokenizer,
+ *   Token
+ * } from 'micromark-util-types'
+ */
+
+import { factorySpace } from 'micromark-factory-space';
+import { markdownLineEnding } from 'micromark-util-character';
+import { subtokenize } from 'micromark-util-subtokenize';
+/**
+ * No name because it must not be turned off.
+ * @type {Construct}
+ */
+export const content = {
+  resolve: resolveContent,
+  tokenize: tokenizeContent
+};
+
+/** @type {Construct} */
+const continuationConstruct = {
+  partial: true,
+  tokenize: tokenizeContinuation
+};
+
+/**
+ * Content is transparent: it’s parsed right now. That way, definitions are also
+ * parsed right now: before text in paragraphs (specifically, media) are parsed.
+ *
+ * @type {Resolver}
+ */
+function resolveContent(events) {
+  subtokenize(events);
+  return events;
+}
+
+/**
+ * @this {TokenizeContext}
+ *   Context.
+ * @type {Tokenizer}
+ */
+function tokenizeContent(effects, ok) {
+  /** @type {Token | undefined} */
+  let previous;
+  return chunkStart;
+
+  /**
+   * Before a content chunk.
+   *
+   * ```markdown
+   * > | abc
+   *     ^
+   * ```
+   *
+   * @type {State}
+   */
+  function chunkStart(code) {
+    effects.enter("content");
+    previous = effects.enter("chunkContent", {
+      contentType: "content"
+    });
+    return chunkInside(code);
+  }
+
+  /**
+   * In a content chunk.
+   *
+   * ```markdown
+   * > | abc
+   *     ^^^
+   * ```
+   *
+   * @type {State}
+   */
+  function chunkInside(code) {
+    if (code === null) {
+      return contentEnd(code);
+    }
+
+    // To do: in `markdown-rs`, each line is parsed on its own, and everything
+    // is stitched together resolving.
+    if (markdownLineEnding(code)) {
+      return effects.check(continuationConstruct, contentContinue, contentEnd)(code);
+    }
+
+    // Data.
+    effects.consume(code);
+    return chunkInside;
+  }
+
+  /**
+   *
+   *
+   * @type {State}
+   */
+  function contentEnd(code) {
+    effects.exit("chunkContent");
+    effects.exit("content");
+    return ok(code);
+  }
+
+  /**
+   *
+   *
+   * @type {State}
+   */
+  function contentContinue(code) {
+    effects.consume(code);
+    effects.exit("chunkContent");
+    previous.next = effects.enter("chunkContent", {
+      contentType: "content",
+      previous
+    });
+    previous = previous.next;
+    return chunkInside;
+  }
+}
+
+/**
+ * @this {TokenizeContext}
+ *   Context.
+ * @type {Tokenizer}
+ */
+function tokenizeContinuation(effects, ok, nok) {
+  const self = this;
+  return startLookahead;
+
+  /**
+   *
+   *
+   * @type {State}
+   */
+  function startLookahead(code) {
+    effects.exit("chunkContent");
+    effects.enter("lineEnding");
+    effects.consume(code);
+    effects.exit("lineEnding");
+    return factorySpace(effects, prefixed, "linePrefix");
+  }
+
+  /**
+   *
+   *
+   * @type {State}
+   */
+  function prefixed(code) {
+    if (code === null || markdownLineEnding(code)) {
+      return nok(code);
+    }
+
+    // Always populated by defaults.
+
+    const tail = self.events[self.events.length - 1];
+    if (!self.parser.constructs.disable.null.includes('codeIndented') && tail && tail[1].type === "linePrefix" && tail[2].sliceSerialize(tail[1], true).length >= 4) {
+      return ok(code);
+    }
+    return effects.interrupt(self.parser.constructs.flow, nok, ok)(code);
+  }
+}