@funcstache/stache-stream 0.2.2 → 0.2.3

package/src/lib/template/Template.ts

@@ -0,0 +1,614 @@
+import { ReadableStream } from "node:stream/web";
+import { EventEmitter } from "node:events";
+import { escape } from "lodash";
+import { Log } from "@funcstache/logger";
+import type {
+  Context,
+  ContextProvider,
+  ContextTypes,
+  PartialTagContext,
+  SectionTagCallback,
+  Tag as TagType,
+  VariableTagContext,
+  VariableTagContextLambda,
+  WriteToOutput,
+} from "../types";
+import { Parse } from "../parse/Parse";
+import { Queue, type TokenizeAllEvent } from "../queue/Queue";
+import {
+  Tokenize,
+  type TokenizeTagEvent,
+  type TokenizeTextEvent,
+} from "../tokenize/Tokenize";
+
+let instance = 0;
+
+export class Template
+  extends EventEmitter<{ inactive: [undefined] }>
+  implements ContextProvider
+{
+  #logger = new Log({
+    category: "TML",
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    level: (process.env as any).LOG_LEVEL || "warn",
+  });
+
+  #_parse: Parse | undefined;
+  #_queue: Queue | undefined;
+  #_tokenize: Tokenize | undefined;
+  /** The context for this template. AKA "hash" in the mustache documentation. */
+  #contextProvider: ContextProvider | undefined;
+  #consumeNextNewline = false;
+  #instance: number;
+  #lineHasContent = false;
+  #pendingWhitespace = "";
+  #readable: ReadableStream<string>;
+  #writeToOutput: WriteToOutput;
+
+  constructor({
+    contextProvider,
+    readable,
+    writeToOutput,
+    ...options
+  }: ConstructorParameters<
+    typeof EventEmitter<{ inactive: [undefined] }>
+  >[0] & {
+    contextProvider?: ContextProvider;
+    readable: ReadableStream<string>;
+    writeToOutput: WriteToOutput;
+  }) {
+    super(options);
+
+    instance++;
+    this.#instance = instance;
+
+    if (contextProvider) {
+      this.#contextProvider = contextProvider;
+    }
+
+    this.#readable = readable;
+    this.#writeToOutput = (...args: Parameters<WriteToOutput>) => {
+      this.#logger.debug(() => [
+        `~~~ ctor.writeToOutput[${this.#instance}]: args=`,
+        args,
+      ]);
+      return writeToOutput(...args);
+    };
+  }
+
+  async getContextValue<CTX extends ContextTypes>(
+    tag: TagType
+  ): Promise<CTX | undefined> {
+    // this.#logger.debug(() => [
+    //   `~~~ getContextValue[${this.#instance}]: tag='${
+    //     tag.value
+    //   }', this.context='\n%o\n'`,
+    //   this.context,
+    // ]);
+
+    let found = false;
+    let value: CTX | undefined;
+    if (isContext(this.context)) {
+      ({ found, value } = await this.#findContextForTag<CTX>(
+        tag,
+        this.context
+      ));
+    }
+
+    // this.#logger.debug(
+    //   () =>
+    //     `~~~ getContextValue[${
+    //       this.#instance
+    //     }]: found='${found}', value='${value}'`
+    // );
+
+    if (!found) {
+      if (this.#contextProvider?.getContextValue) {
+        value = await this.#contextProvider.getContextValue(tag);
+      }
+    }
+
+    // this.#logger.debug(() => [
+    //   `~~~ getContextValue[${this.#instance}]: tag='${
+    //     tag.value
+    //   }', value='${value}'`,
+    //   this.context,
+    // ]);
+
+    return value;
+  }
+
+  read(): Template {
+    this.#parse.read(this.#readable);
+    return this;
+  }
+
+  get context(): ContextTypes {
+    return this.#contextProvider?.context || {};
+  }
+
+  get #parse(): Parse {
+    if (!this.#_parse) {
+      this.#_parse = new Parse({
+        onChar: this.#tokenize.push.bind(this.#tokenize),
+      });
+    }
+
+    return this.#_parse;
+  }
+
+  get #queue(): Queue {
+    if (!this.#_queue) {
+      this.#_queue = new Queue()
+        .on("error", (err) => {
+          throw err;
+        })
+        .on("comment", this.#handleComment.bind(this))
+        .on("implicit", this.#handleImplicit.bind(this))
+        .on("inactive", this.#handleInactive.bind(this))
+        .on("inverted", this.#handleInverted.bind(this))
+        .on("partial", this.#handlePartial.bind(this))
+        .on("section", this.#handleSection.bind(this))
+        .on("text", this.#handleText.bind(this))
+        .on("variable", this.#handleVariable.bind(this));
+    }
+
+    return this.#_queue;
+  }
+
+  get #tokenize(): Tokenize {
+    if (!this.#_tokenize) {
+      this.#_tokenize = new Tokenize().on(
+        "token",
+        this.#queue.push.bind(this.#queue)
+      );
+    }
+
+    return this.#_tokenize;
+  }
+
+  async #applyStandaloneState(
+    bodyEvents: TokenizeAllEvent[]
+  ): Promise<boolean> {
+    const firstBodyEvent = bodyEvents[0];
+    const isStandalone =
+      !this.#lineHasContent &&
+      firstBodyEvent !== undefined &&
+      firstBodyEvent.type === "text" &&
+      /^(\r\n|\n)/.test(firstBodyEvent.data.toString());
+
+    if (isStandalone) {
+      this.#pendingWhitespace = "";
+      this.#consumeNextNewline = true;
+    } else {
+      await this.#flushPendingWhitespace();
+    }
+
+    return isStandalone;
+  }
+
+  /**
+   * Gets the context for a tag. This is a complex operation so it's been broken into several
+   * functions. If context is not found in this template's context an attempt will be made to find
+   * the context in this Template's parent - if it exists.
+   * @param tag Find context for this tag.
+   * @param context The context provided to the template.
+   */
+  async #findContextForTag<CTX extends ContextTypes>(
+    tag: TagType,
+    context: Context
+  ): Promise<{ found: boolean; value: CTX | undefined }> {
+    // this.#logger.debug(() => [
+    //   `~~~ #findContextForTag[${this.#instance}]: tag='${
+    //     tag.value
+    //   }', this.context='\n%o\n'`,
+    //   this.context,
+    // ]);
+
+    let ctx = context;
+    let found = false;
+    let keys = tag.key.split(".");
+    let value: Context | ContextTypes | undefined = undefined;
+    while (keys.length) {
+      try {
+        ({ found = true, keys, value } = this.#getFromContext(keys, ctx));
+
+        // this.#logger.debug(
+        //   () =>
+        //     `~~~ #findContextForTag[${
+        //       this.#instance
+        //     }]: found='${found}', keys='${keys}', value=${value}`
+        // );
+
+        if (!found) {
+          break;
+        }
+
+        if (keys.length < 1) {
+          // No keys left, return the value.
+          found = true;
+          break;
+        }
+
+        if (typeof value === "function") {
+          // We got a function for the key and there are more keys, invoke the lambda function and
+          // expect the result to be an object we can use for the remainder of the keys.
+          ctx = (await (value as VariableTagContextLambda)()) as Context;
+          continue;
+        }
+
+        ctx = value as Context;
+      } catch (err) {
+        this.#logger.error(() => [
+          `#findContextForTag[${this.#instance}]: error;\n   tag='${
+            tag.value
+          }',\n   keys='%0',\n   ctx=%1,\n   err=%2`,
+          keys,
+          ctx,
+          err,
+        ]);
+        break;
+      }
+    }
+
+    // this.#logger.debug(
+    //   () =>
+    //     `~~~ #findContextForTag[${
+    //       this.#instance
+    //     }]: found='${found}', value=${value}`
+    // );
+
+    return { found, value: value as CTX };
+  }
+
+  #getFromContext(keys: string[], context: Context | undefined) {
+    if (isContext(context)) {
+      const key = keys[0];
+      if (!key) {
+        throw Error(`No keys were provided.`);
+      }
+
+      if (!(key in context)) {
+        return { found: false, keys };
+      }
+
+      const value = context[key];
+
+      if (isPrimitive(value)) {
+        if (1 < keys.length) {
+          throw Error(
+            `key '${key}', returned '${typeof value}' from context however, more keys remain so expected an object.`
+          );
+        }
+
+        return { value, keys: [] };
+      }
+
+      return { value, keys: keys.slice(1) };
+    }
+
+    throw new Error(
+      `Expect an object for context, received a '${typeof context}'.`
+    );
+  }
+
+  async #handleComment(): Promise<void> {
+    if (!this.#lineHasContent) {
+      this.#pendingWhitespace = "";
+      this.#consumeNextNewline = true;
+    } else {
+      await this.#flushPendingWhitespace();
+    }
+  }
+
+  async #handleInverted(events: TokenizeAllEvent[]): Promise<void> {
+    const bodyEvents = events.slice(1, -1);
+    const isStandalone = await this.#applyStandaloneState(bodyEvents);
+
+    const startEvent = events[0];
+    if (!startEvent || startEvent.type !== "tag") {
+      return;
+    }
+
+    const value = await this.getContextValue(startEvent.data);
+
+    const isFalsy =
+      value === undefined ||
+      value === false ||
+      value === "" ||
+      (Array.isArray(value) && value.length === 0);
+
+    if (!isFalsy) {
+      return;
+    }
+
+    let templateStr = bodyEvents.map((e) => e.data.toString()).join("");
+    if (isStandalone) {
+      templateStr = stripLeadingNewline(templateStr);
+    }
+
+    await this.#renderSectionContent(templateStr, this.context);
+  }
+
+  async #handleImplicit({ data: tag }: TokenizeTagEvent): Promise<void> {
+    await this.#flushPendingWhitespace();
+    const ctx = this.context;
+
+    if (isPrimitive(ctx)) {
+      this.#writeToOutput(tag.raw ? "" + ctx : escape("" + ctx));
+    }
+  }
+
+  async #handleInactive(): Promise<void> {
+    await this.#flushPendingWhitespace();
+    this.emit("inactive", undefined);
+  }
+
+  async #handlePartial(event: TokenizeTagEvent): Promise<void> {
+    const partialContext = await this.getContextValue<PartialTagContext>(
+      event.data
+    );
+    if (typeof partialContext !== "function") {
+      await this.#flushPendingWhitespace();
+      return;
+    }
+
+    const isStandalone = !this.#lineHasContent;
+    const indent = isStandalone ? this.#pendingWhitespace : "";
+
+    if (isStandalone) {
+      this.#pendingWhitespace = "";
+      this.#consumeNextNewline = true;
+    } else {
+      await this.#flushPendingWhitespace();
+    }
+
+    const { input, context } = await partialContext(event.data);
+    const readable = await input();
+
+    await this.#renderPartialContent(
+      readable as ReadableStream<string>,
+      context,
+      indent
+    );
+  }
+
+  async #handleSection(events: TokenizeAllEvent[]): Promise<void> {
+    const bodyEvents = events.slice(1, -1);
+    const isStandalone = await this.#applyStandaloneState(bodyEvents);
+
+    const startEvent = events[0];
+    if (!startEvent || startEvent.type !== "tag") {
+      return;
+    }
+
+    const sectionContext =
+      startEvent.data.key === "."
+        ? this.context
+        : await this.getContextValue(startEvent.data);
+
+    if (
+      sectionContext === undefined ||
+      sectionContext === false ||
+      sectionContext === "" ||
+      (Array.isArray(sectionContext) && sectionContext.length === 0)
+    ) {
+      return;
+    }
+
+    let templateStr = bodyEvents.map((e) => e.data.toString()).join("");
+    if (isStandalone) {
+      templateStr = stripLeadingNewline(templateStr);
+    }
+
+    if (isSectionLambda(sectionContext)) {
+      // TODO: make `SectionTagCallback` async
+      await this.#writeToOutput(sectionContext(templateStr));
+      return;
+    }
+
+    const items: ContextTypes[] = Array.isArray(sectionContext)
+      ? sectionContext
+      : [sectionContext];
+
+    for (const item of items) {
+      await this.#renderSectionContent(templateStr, item);
+    }
+  }
+
+  async #renderSectionContent(
+    templateStr: string,
+    context: ContextTypes
+  ): Promise<void> {
+    return new Promise<void>((resolve) => {
+      const childContextProvider: ContextProvider = {
+        context,
+        getContextValue: <CTX extends ContextTypes>(
+          tag: TagType
+        ): Promise<CTX | undefined> => this.getContextValue<CTX>(tag),
+      };
+
+      new Template({
+        contextProvider: childContextProvider,
+        readable: stringToReadableStream(templateStr),
+        writeToOutput: this.#writeToOutput,
+      })
+        .on("inactive", () => resolve())
+        .read();
+    });
+  }
+
+  async #flushPendingWhitespace(): Promise<void> {
+    if (this.#pendingWhitespace) {
+      await this.#writeToOutput(this.#pendingWhitespace);
+      this.#pendingWhitespace = "";
+    }
+  }
+
+  async #handleText(event: TokenizeTextEvent): Promise<void> {
+    let text = event.data;
+    // After a standalone partial the trailing newline belongs to the partial tag, not the output.
+    if (this.#consumeNextNewline) {
+      if (text.startsWith("\r\n")) {
+        text = text.slice(2);
+      } else if (text.startsWith("\n")) {
+        text = text.slice(1);
+      }
+      this.#consumeNextNewline = false;
+      if (!text) {
+        return;
+      }
+    }
+
+    const lastNewlineEnd = lastNewlineIndex(text);
+    if (lastNewlineEnd !== -1) {
+      // Flush everything up to and including the last newline, then reset line state.
+      await this.#writeToOutput(
+        this.#pendingWhitespace + text.slice(0, lastNewlineEnd)
+      );
+      this.#pendingWhitespace = "";
+      this.#lineHasContent = false;
+      text = text.slice(lastNewlineEnd);
+      if (!text) {
+        return;
+      }
+    }
+
+    // text contains no newlines — process as current-line fragment.
+    if (isHorizontalWhitespace(text)) {
+      this.#pendingWhitespace += text;
+    } else {
+      await this.#writeToOutput(this.#pendingWhitespace + text);
+      this.#pendingWhitespace = "";
+      this.#lineHasContent = true;
+    }
+  }
+
+  async #renderPartialContent(
+    readable: ReadableStream<string>,
+    context: ContextTypes | undefined,
+    indent: string
+  ): Promise<void> {
+    return new Promise<void>((resolve) => {
+      const childContextProvider: ContextProvider = {
+        context: context ?? this.context,
+        getContextValue: <CTX extends ContextTypes>(
+          tag: TagType
+        ): Promise<CTX | undefined> => this.getContextValue<CTX>(tag),
+      };
+
+      const indentingWriter: WriteToOutput = indent
+        ? makeIndentingWriter(this.#writeToOutput, indent)
+        : this.#writeToOutput;
+
+      new Template({
+        contextProvider: childContextProvider,
+        readable,
+        writeToOutput: indentingWriter,
+      })
+        .on("inactive", () => resolve())
+        .read();
+    });
+  }
+
+  async #handleVariable({ data: tag }: TokenizeTagEvent): Promise<void> {
+    await this.#flushPendingWhitespace();
+    // this.#logger.debug(
+    //   () => `~~~ #handleVariable[${this.#instance}]: tag=${tag.toString()}`
+    // );
+
+    let tagVariableContext = await this.getContextValue<VariableTagContext>(
+      tag
+    );
+
+    // this.#logger.debug(() => [
+    //   `~~~ #handleVariable[${this.#instance}]: tagVariableContext=`,
+    //   tagVariableContext,
+    // ]);
+
+    // If the variable context value is a function then invoke it to get the value.
+    if (typeof tagVariableContext === "function") {
+      tagVariableContext = await (
+        tagVariableContext as VariableTagContextLambda
+      )();
+    }
+
+    if (isPrimitive(tagVariableContext)) {
+      this.#writeToOutput(
+        tag.raw ? "" + tagVariableContext : escape("" + tagVariableContext)
+      );
+    } else {
+      this.#logger.log(
+        () =>
+          `#handleVariable[${this.#instance}]: variable tag '${
+            tag.valueOption
+          }' returns context value '${tagVariableContext}'`
+      );
+    }
+
+    return;
+  }
+}
+
+function isContext(obj: unknown): obj is Context {
+  return !!(
+    obj &&
+    typeof obj !== "boolean" &&
+    typeof obj !== "number" &&
+    typeof obj !== "string" &&
+    typeof obj !== "function" &&
+    !Array.isArray(obj)
+  );
+}
+
+function isHorizontalWhitespace(text: string): boolean {
+  return /^[ \t]*$/.test(text);
+}
+
+function isPrimitive(obj: unknown): obj is boolean | number | string {
+  return (
+    typeof obj === "boolean" ||
+    typeof obj === "number" ||
+    typeof obj === "string"
+  );
+}
+
+function isSectionLambda(value: ContextTypes): value is SectionTagCallback {
+  return !!value && typeof value === "function";
+}
+
+/** Returns the index of the first character after the last `\n` (or `\r\n`), or -1 if none. */
+function lastNewlineIndex(text: string): number {
+  const index = text.lastIndexOf("\n");
+  if (index === -1) {
+    return -1;
+  }
+  return index + 1;
+}
+
+function makeIndentingWriter(
+  writeToOutput: WriteToOutput,
+  indent: string
+): WriteToOutput {
+  let atLineStart = true;
+  return async (text: string) => {
+    const indented =
+      (atLineStart ? indent : "") + text.replace(/\n(?!\n|$)/g, "\n" + indent);
+    atLineStart = text.endsWith("\n");
+    return writeToOutput(indented);
+  };
+}
+
+function stripLeadingNewline(str: string): string {
+  const index = str.startsWith("\r\n") ? 2 : str.startsWith("\n") ? 1 : 0;
+  return !index ? str : str.slice(index);
+}
+
+function stringToReadableStream(str: string): ReadableStream<string> {
+  return new ReadableStream<string>({
+    start(controller) {
+      controller.enqueue(str);
+      controller.close();
+    },
+  });
+}

package/src/lib/test/streams.ts

@@ -0,0 +1,36 @@
+import { ReadableStream } from "node:stream/web";
+
+export function createReadableStream(
+  source: string,
+  charCount = -1,
+  pause = 0
+) {
+  const encoder = new TextEncoder();
+  let sourceIndex = 0;
+  let pullCount = 0;
+
+  return new ReadableStream<AllowSharedBufferSource>({
+    pull: async (controller) => {
+      pullCount++;
+      if (source.length <= sourceIndex) {
+        controller.close();
+        return;
+      }
+
+      for (let i = 0; i < (0 < charCount ? charCount : source.length); i++) {
+        if (sourceIndex < source.length) {
+          controller.enqueue(encoder.encode(source.charAt(sourceIndex)));
+        } else {
+          controller.close();
+          return;
+        }
+
+        sourceIndex++;
+
+        if (0 < pause) {
+          await new Promise<void>((resolve) => setTimeout(resolve, pause));
+        }
+      }
+    },
+  });
+}