@funcstache/stache-stream 0.2.2 → 0.2.3

package/src/lib/tokenize/Tokenize.ts

@@ -0,0 +1,374 @@
+import { EventEmitter } from "node:events";
+import { Log } from "@funcstache/logger";
+import { Tag } from "../tag/Tag";
+
+export class Tokenize extends EventEmitter<TokenizeEventMap> {
+  readonly #TOKEN_ENTER_CHAR = "{";
+  readonly #TOKEN_EXIT_CHAR = "}";
+
+  #logger = new Log({
+    category: "TKZ",
+    level: (process.env as any).LOG_LEVEL || "warn",
+  });
+
+  #stateCurrent: StateNames = "outsideTag";
+  #statePrevious: StateNames | undefined;
+  #leftBuffer = "";
+  #leftTokenBuffer = "";
+  #middleBuffer = "";
+  #rightTokenBuffer = "";
+
+  #emitTagToken() {
+    if (this.#leftTokenBuffer.length === this.#rightTokenBuffer.length) {
+      const data = Tag.parse(
+        `${this.#leftTokenBuffer}${this.#middleBuffer}${this.#rightTokenBuffer}`
+      );
+
+      data &&
+        this.#emitToken({
+          data,
+          uid: performance.now(),
+          type: "tag",
+        });
+    }
+
+    this.#leftBuffer = "";
+    this.#leftTokenBuffer = "";
+    this.#middleBuffer = "";
+    this.#rightTokenBuffer = "";
+  }
+
+  #emitToken(event: TokenizeTagEvent | TokenizeTextEvent | null) {
+    this.#logger.debug(() => [`!!! #emitEvent: event=`, JSON.stringify(event)]);
+    this.emit("token", event);
+  }
+
+  async push(char: string | null): Promise<void> {
+    this.#logger.debug(() => "-".repeat(80));
+
+    char && this.#updateStatus(char);
+    await this.#processChar(char);
+  }
+
+  async #processChar(char: string | null): Promise<void> {
+    this.#logger.debug(
+      () =>
+        `=== #processChar: current=${this.#stateCurrent}, previous=${
+          this.#statePrevious
+        }`
+    );
+
+    if (char === null) {
+      if (this.#leftBuffer) {
+        this.#emitToken({
+          data: this.#leftBuffer,
+          type: "text",
+          uid: performance.now(),
+        });
+      }
+      this.#emitTagToken();
+      this.#emitToken(null);
+      return;
+    }
+
+    // The only state changes that should happen here are resets that need to be done after `char`
+    // is processed.
+    switch (this.#stateCurrent) {
+      case "tagEnteringMaybe": {
+        // This state change happens in the case of nested tags, for example: `{{#a}}{{m}}{{/a}}`
+        // It's why we can't set `#leftTokenBuffer` in the `#updateStatus` method.
+        if (this.#statePrevious === "tagExitingMaybe") {
+          this.#emitTagToken();
+        }
+
+        this.#leftTokenBuffer += char;
+
+        break;
+      }
+
+      // case "tagExitingMaybe": {
+      //   break;
+      // }
+
+      case "inTagValue": {
+        if (this.#leftBuffer) {
+          this.#emitToken({
+            data: this.#leftBuffer,
+            type: "text",
+            uid: performance.now(),
+          });
+          this.#leftBuffer = "";
+        }
+        break;
+      }
+
+      case "outsideTag": {
+        if (this.#statePrevious === "tagEnteringMaybe") {
+          // Single `{` that was not the start of a mustache tag — treat as plain text.
+          // #updateStatus already appended the current char to leftBuffer, so insert
+          // leftTokenBuffer before that last char to preserve the correct order.
+          this.#leftBuffer =
+            this.#leftBuffer.slice(0, -1) +
+            this.#leftTokenBuffer +
+            this.#leftBuffer.slice(-1);
+          this.#leftTokenBuffer = "";
+        } else if (this.#leftTokenBuffer && this.#rightTokenBuffer) {
+          const data = Tag.parse(
+            `${this.#leftTokenBuffer}${this.#middleBuffer}${
+              this.#rightTokenBuffer
+            }`
+          );
+
+          data &&
+            this.#emitToken({
+              data,
+              type: "tag",
+              uid: performance.now(),
+            });
+
+          this.#leftTokenBuffer = "";
+          this.#middleBuffer = "";
+          this.#rightTokenBuffer = "";
+        }
+        break;
+      }
+    }
+
+    this.#logger.debug(
+      () =>
+        `--- #processChar: '${char}' - ${this.#leftBuffer},${
+          this.#leftTokenBuffer ? ` _${this.#leftTokenBuffer}_` : " "
+        }, ${this.#middleBuffer}, ${
+          this.#rightTokenBuffer ? `_${this.#rightTokenBuffer}_` : ""
+        }`
+    );
+  }
+
+  #updateStateFromModel(
+    current: StateNames,
+    options: UpdateStateModelOptions,
+    instantStateCallback?: (state: StateNames) => void
+  ): StateNames {
+    const currentStateData = stateModel[current];
+    let nextState: StateNames = current;
+    for (const stateName of Object.keys(currentStateData) as StateNames[]) {
+      const dataArr = currentStateData[stateName];
+      if (!dataArr) {
+        continue;
+      }
+
+      if (
+        dataArr.some((data) => {
+          const allFalse =
+            "isFalse" in data
+              ? data.isFalse.every(
+                  (optionName) => options[optionName] === false
+                )
+              : true;
+
+          if (!allFalse) {
+            return false;
+          }
+
+          const allTrue =
+            "isTrue" in data
+              ? data.isTrue.every((optionName) => options[optionName] === true)
+              : true;
+
+          if (allTrue) {
+            nextState = stateName;
+
+            // If the next state is "instant" invoke the callback then invoke
+            // `#updateStateFromModel` function again recursively.
+            if ("instant" in data && data.instant) {
+              instantStateCallback && instantStateCallback(nextState);
+              nextState = this.#updateStateFromModel(nextState, options);
+            }
+
+            return true;
+          }
+
+          return false;
+        })
+      ) {
+        break;
+      }
+    }
+
+    return nextState;
+  }
+
+  #updateStatus(char: string): void {
+    this.#logger.debug(
+      () =>
+        `--- #updateStatus: '${char}' - ${this.#leftBuffer},${
+          this.#leftTokenBuffer ? ` _${this.#leftTokenBuffer}_` : " "
+        }, ${this.#middleBuffer}, ${
+          this.#rightTokenBuffer ? `_${this.#rightTokenBuffer}_` : ""
+        }`
+    );
+
+    this.#statePrevious = this.#stateCurrent;
+
+    // This code sets the state for how `char` should be processed.
+    const isEnterToken = char === this.#TOKEN_ENTER_CHAR;
+    const isExitToken = char === this.#TOKEN_EXIT_CHAR;
+
+    const options: UpdateStateModelOptions = {
+      currentCharEnterToken: isEnterToken,
+      currentCharExitToken: isExitToken,
+      currentCharNotToken: !isEnterToken && !isExitToken,
+      enterTokenCountEqualsOne: 1 === this.#leftTokenBuffer.length,
+      enterTokenCountEqualsTwo: 2 === this.#leftTokenBuffer.length,
+      enterTokenCountEqualsThree: 3 === this.#leftTokenBuffer.length,
+      exitTokenCountEqualsEnterTokenCount:
+        this.#leftTokenBuffer.length === this.#rightTokenBuffer.length,
+    };
+
+    this.#stateCurrent = this.#updateStateFromModel(
+      this.#stateCurrent,
+      options,
+      (state) => {
+        this.#logger.debug(
+          () =>
+            `>>> #updateStatus: instant;\n  #statePrevious='${
+              this.#statePrevious
+            }',\n  state='${state}'`
+        );
+      }
+    );
+
+    // Manage most of the state changes here. a small number of them have to be done in
+    // `#processChar` because certain state values have to be reset after the char is processed.
+    switch (this.#stateCurrent) {
+      case "tagEnteringMaybe": {
+        //Changing to "tagEnteringMaybe" could happen because of nested tags, for example:
+        // `{{#a}}{{m}}{{/a}}`. It's why we can't set `#leftTokenBuffer` here, it has to be done as
+        // part of #processChar.
+
+        // this.#leftTokenBuffer += char;
+        break;
+      }
+
+      case "tagExitingMaybe": {
+        this.#rightTokenBuffer += char;
+        break;
+      }
+
+      case "outsideTag": {
+        this.#leftBuffer += char;
+        break;
+      }
+
+      case "inTagValue": {
+        this.#middleBuffer += char;
+        break;
+      }
+    }
+  }
+}
+
+interface TokenizeEvent {
+  uid: number;
+}
+
+export interface TokenizeTagEvent extends TokenizeEvent {
+  data: Tag;
+  type: "tag";
+}
+
+export interface TokenizeTextEvent extends TokenizeEvent {
+  data: string;
+  type: "text";
+}
+
+export type TokenizeEventMap = {
+  token: [TokenizeTagEvent | TokenizeTextEvent | null];
+};
+
+export type StateNames =
+  | "inTagValue"
+  | "outsideTag"
+  | "tagExitingMaybe"
+  | "tagEnteringMaybe";
+
+const stateModel: StateModel = {
+  inTagValue: {
+    tagExitingMaybe: [
+      {
+        isTrue: ["currentCharExitToken"],
+      },
+    ],
+  },
+
+  outsideTag: {
+    tagEnteringMaybe: [
+      {
+        isTrue: ["currentCharEnterToken"],
+      },
+    ],
+  },
+
+  tagExitingMaybe: {
+    tagEnteringMaybe: [
+      {
+        isTrue: [
+          "currentCharEnterToken",
+          "exitTokenCountEqualsEnterTokenCount",
+        ],
+      },
+    ],
+    outsideTag: [
+      {
+        isTrue: ["currentCharNotToken", "exitTokenCountEqualsEnterTokenCount"],
+      },
+    ],
+  },
+
+  tagEnteringMaybe: {
+    outsideTag: [
+      {
+        isFalse: ["currentCharEnterToken"],
+        isTrue: ["enterTokenCountEqualsOne"],
+      },
+    ],
+    inTagValue: [
+      {
+        isTrue: ["currentCharNotToken", "enterTokenCountEqualsTwo"],
+      },
+      {
+        isTrue: ["enterTokenCountEqualsThree"],
+      },
+    ],
+  },
+} as const;
+
+type StateModel = {
+  [currentState in StateNames]: {
+    [moveToStateWhen in StateNames]?: (
+      | {
+          instant?: boolean;
+          isFalse: (keyof UpdateStateModelOptions)[];
+          isTrue: (keyof UpdateStateModelOptions)[];
+        }
+      | {
+          instant?: boolean;
+          isFalse: (keyof UpdateStateModelOptions)[];
+        }
+      | {
+          instant?: boolean;
+          isTrue: (keyof UpdateStateModelOptions)[];
+        }
+    )[];
+  };
+};
+
+interface UpdateStateModelOptions {
+  currentCharEnterToken: boolean;
+  currentCharExitToken: boolean;
+  currentCharNotToken: boolean;
+  enterTokenCountEqualsOne: boolean;
+  enterTokenCountEqualsTwo: boolean;
+  enterTokenCountEqualsThree: boolean;
+  exitTokenCountEqualsEnterTokenCount: boolean;
+}

package/src/lib/{types.d.ts → types.ts}

@@ -1,14 +1,24 @@
 import { ReadableStream } from "node:stream/web";
+
 export type ContentTagTagTypes = (typeof ContentTagTypes)[number];
-export declare const ContentTagTypes: readonly ["block", "end", "inverted", "parent", "section"];
+
+export const ContentTagTypes = [
+  "block", // `{{$tag}}{{/tag}}`
+  "end", // `{{/tag}}`
+  "inverted", // `{{^tag}}{{/tag}}`
+  "parent", // `{{<tag}}{{/tag}}`
+  "section", // `{{#tag}}{{/tag}}`
+] as const;
+
 /**
  * A JavaScript object that supplies replacements for tags in a Mustache template. The Mustache
  * specification uses the term "hash." In stache-stream "hash" and "context" are synonyms.
  * @category Context
  */
 export interface Context {
-    readonly [key: string]: ContextTypes | undefined;
+  readonly [key: string]: ContextTypes | undefined;
 }
+
 /**
  * When a Mustache template is rendered data needs to be provided to replace tags in the template; a
  * ContextProvider delivers that information. The ContextProvider includes context for a template as
@@ -20,19 +30,30 @@ export interface Context {
  * look for a value in parent context.
  */
 export interface ContextProvider {
-    readonly context: ContextTypes;
-    readonly getContextValue?: <CTX extends ContextTypes>(tag: Tag) => Promise<CTX | undefined>;
+  readonly context: ContextTypes;
+  readonly getContextValue?: <CTX extends ContextTypes>(
+    tag: Tag
+  ) => Promise<CTX | undefined>;
 }
+
 /**
  * Properties in a { @link Context } object implement these types. The specific types to implement
  * are determined by the tag type in a Mustache template.
  * @category Context
  */
-export type ContextTypes = Context | PartialTagContext | SectionTagContext | VariableTagContext;
+export type ContextTypes =
+  | Context
+  // | JsonType
+  | PartialTagContext
+  | SectionTagContext
+  | VariableTagContext;
+// | (Context | JsonType | SectionTagContextList)[];
+
 /**
  * A primitive JavaScript type that can be serialized in a JSON object.
  */
 export type JsonType = boolean | number | string;
+
 /**
  * Implement one of these interfaces to provide replacements for
  * {@link https://mustache.github.io/mustache.5.html#Variables Partial tags}. See
@@ -41,6 +62,7 @@ export type JsonType = boolean | number | string;
  * @category Partial
  */
 export type PartialTagContext = PartialTagContextLambda;
+
 /**
  * Implement this function to asynchronously return a `ReadableStream` that contains a Mustache
  * template and its associated context that replaces a
@@ -62,20 +84,22 @@ export type PartialTagContext = PartialTagContextLambda;
  * @category Partial
  */
 export interface PartialTagContextLambda {
-    (tag: Tag): Promise<PartialTagContextLambdaResult>;
+  (tag: Tag): Promise<PartialTagContextLambdaResult>;
 }
+
 /**
  * The result of invoking a {@link PartialTagContextLambda}.
  * @category Context
  * @category Partial
  */
 export interface PartialTagContextLambdaResult {
-    /** The optional context associated with this template. */
-    context?: ContextTypes;
-    /** When invoked `input` resolves to a stream containing the Mustache template that replaces a
-     * partial tag. */
-    input: () => Promise<ReadableStream>;
+  /** The optional context associated with this template. */
+  context?: ContextTypes;
+  /** When invoked `input` resolves to a stream containing the Mustache template that replaces a
+   * partial tag. */
+  input: () => Promise<ReadableStream>;
 }
+
 /**
  * Implement one of these interfaces to provide replacements for
  * {@link https://mustache.github.io/mustache.5.html#Sections Section tags}. See
@@ -84,6 +108,7 @@ export interface PartialTagContextLambdaResult {
  * @category Section
  */
 export type SectionTagContext = SectionTagCallback | SectionTagContextRecord[];
+
 /**
  * Provides zero or more items that will replace a
  * {@link https://mustache.github.io/mustache.5.html#Sections Section tag}. See
@@ -92,8 +117,9 @@ export type SectionTagContext = SectionTagCallback | SectionTagContextRecord[];
  * @category Section
  */
 export interface SectionTagContextRecord {
-    [key: string]: JsonType | SectionTagContextRecord;
+  [key: string]: JsonType | SectionTagContextRecord;
 }
+
 /**
  * When provided this callback is invoked when the
  * {@link https://mustache.github.io/mustache.5.html#Sections Section tag} is encountered. The
@@ -117,8 +143,9 @@ export interface SectionTagContextRecord {
  * @category Section
  */
 export interface SectionTagCallback {
-    (tagContent: string): string;
+  (tagContent: string): string;
 }
+
 /**
  * Represents a tag in a Mustache template providing information about a tag and its
  * characteristics.
@@ -165,12 +192,14 @@ export interface SectionTagCallback {
  * "{{ #items }}" the `value` is "#items".
  */
 export interface Tag {
-    readonly content: string | null;
-    readonly key: string;
-    readonly raw: boolean;
-    readonly type: ContentTagTagTypes | ValueTagTagTypes;
-    readonly value: string;
+  readonly content: string | null;
+  readonly key: string;
+  readonly raw: boolean;
+  readonly type: ContentTagTagTypes | ValueTagTagTypes;
+  readonly value: string;
+  // process: () => Promise<void>;
 }
+
 /**
  * Defines how to access a template file.
  *   - string: The name of a template file in the same directory as this module file. The name must
@@ -179,9 +208,19 @@ export interface Tag {
  *     in the "string" type above. The second element is a relative path to the directory where the
  *     template file is located; the path is relative to the location of this module file.
  */
-export type TemplateName = string | [templateName: string, templateFilename: string];
+export type TemplateName =
+  | string
+  | [templateName: string, templateFilename: string];
+
 export type ValueTagTagTypes = (typeof ValueTagTypes)[number];
-export declare const ValueTagTypes: readonly ["comment", "implicit", "partial", "variable"];
+
+export const ValueTagTypes = [
+  "comment", // {{!text}}
+  "implicit", // `{{.}}`
+  "partial", // `{{>tag}}`
+  "variable",
+] as const;
+
 /**
  * Interfaces that can supply replacements for
  * {@link https://mustache.github.io/mustache.5.html#Variables variable tags}. See
@@ -189,7 +228,11 @@ export declare const ValueTagTypes: readonly ["comment", "implicit", "partial",
  * @category Context
  * @category Variable
  */
-export type VariableTagContext = VariableTagContextPrimitive | VariableTagContextLambda | VariableTagContextRecord;
+export type VariableTagContext =
+  | VariableTagContextPrimitive
+  | VariableTagContextLambda
+  | VariableTagContextRecord;
+
 /**
  * A function that resolves to the value that replaces a
  * {@link https://mustache.github.io/mustache.5.html#Variables variable tag}. See
@@ -198,15 +241,19 @@ export type VariableTagContext = VariableTagContextPrimitive | VariableTagContex
  * @category Variable
  */
 export interface VariableTagContextLambda {
-    (): Promise<VariableTagContextPrimitive | VariableTagContextRecord>;
+  (): Promise<VariableTagContextPrimitive | VariableTagContextRecord>;
 }
+
 /**
  * A {@link https://mustache.github.io/mustache.5.html#Variables variable tag} can be replaced by
  * these JavaScript primitives. See {@link VariableTagContext VariableTagContext}.
  * @category Context
  * @category Variable
  */
-export type VariableTagContextPrimitive = JsonType | Array<boolean | number | string>;
+export type VariableTagContextPrimitive =
+  | JsonType
+  | Array<boolean | number | string>;
+
 /**
  * A {@link https://mustache.github.io/mustache.5.html#Variables variable tag} with a "dotted name"
  * can be replaced by a JavaScript object. See {@link VariableTagContext VariableTagContext}.
@@ -214,11 +261,12 @@ export type VariableTagContextPrimitive = JsonType | Array<boolean | number | st
  * @category Variable
  */
 export interface VariableTagContextRecord {
-    [key: string]: unknown;
+  [key: string]: unknown;
 }
+
 /**
  * Invoke this function to write to the output `WritableStream`.
  */
 export interface WriteToOutput {
-    (text: string): Promise<void>;
+  (text: string): Promise<void>;
 }

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "noImplicitOverride": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

package/tsconfig.lib.json

@@ -0,0 +1,16 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": [
+    "vite.config.ts",
+    "vitest.config.ts",
+    "vitest.setup.ts",
+    "src/**/*.spec.ts",
+    "src/**/*.test.ts"
+  ]
+}

package/tsconfig.spec.json

@@ -0,0 +1,21 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "types": [
+      "vitest/globals",
+      "vitest/importMeta",
+      "vite/client",
+      "node",
+      "vitest"
+    ]
+  },
+  "include": [
+    "vite.config.ts",
+    "vitest.config.ts",
+    "vitest.setup.ts",
+    "src/**/*.d.ts",
+    "src/**/*.test.ts",
+    "src/**/*.spec.ts"
+  ]
+}

package/typedoc.mjs

@@ -0,0 +1,15 @@
+// what?
+const config = {
+  entryPoints: ["src/index.ts"],
+  includeVersion: true,
+  githubPages: true,
+  gitRevision: "main",
+  // navigation: {
+  //   includeCategories: true,
+  // },
+  out: "docs",
+  readme: ["README.md"],
+  tsconfig: "tsconfig.lib.json",
+};
+
+export default config;

package/vite.config.ts

@@ -0,0 +1,27 @@
+import { defineConfig } from "vite";
+import { nxViteTsPaths } from "@nx/vite/plugins/nx-tsconfig-paths.plugin";
+
+export default defineConfig({
+  root: __dirname,
+  cacheDir: "../../node_modules/.vite/libs/stache-stream",
+
+  plugins: [nxViteTsPaths()],
+
+  // Uncomment this if you are using workers.
+  // worker: {
+  //  plugins: [ nxViteTsPaths() ],
+  // },
+
+  test: {
+    watch: false,
+    globals: true,
+    environment: "node",
+    include: ["src/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}"],
+    reporters: ["default"],
+    setupFiles: ["./vitest.setup.ts"],
+    coverage: {
+      reportsDirectory: "../../coverage/libs/stache-stream",
+      provider: "v8",
+    },
+  },
+});

package/vitest.setup.ts

@@ -0,0 +1,6 @@
+import { populateGlobal } from "vitest/environments";
+import "./src/index";
+
+beforeAll(() => {
+  populateGlobal({}, globalThis);
+});