@funcstache/stache-stream 0.2.2 → 0.2.3

package/src/lib/tokenize/README.md

@@ -0,0 +1,97 @@
+# Tokenize
+
+`Tokenize` receives a character stream from `Parse` and groups characters into mustache tokens. It emits a `token` event for each plain-text run and each mustache tag it recognizes, then emits `null` to signal end-of-stream.
+
+## How it works
+
+### Character input
+
+Characters are pushed one at a time via `push(char)`. Each call first updates the state machine (`#updateStatus`) then processes the character against the new state (`#processChar`). `null` signals end-of-stream and flushes any buffered text or tag before emitting a final `null` token.
+
+### State machine
+
+Each character moves the tokenizer through one of four states:
+
+| State              | Meaning                                    |
+| ------------------ | ------------------------------------------ |
+| `outsideTag`       | Reading plain text between tags            |
+| `tagEnteringMaybe` | Seen one or more `{`; may be opening a tag |
+| `inTagValue`       | Inside a tag, reading its key              |
+| `tagExitingMaybe`  | Seen one or more `}`; may be closing a tag |
+
+Transitions are driven by a declarative `stateModel` table. Each entry lists the conditions (flags derived from the current character and buffer lengths) that must be true or false for the transition to fire. `#updateStateFromModel` evaluates the table and returns the next state. An optional `instant` flag allows passing through an intermediate state in the same call.
+
+### Buffers
+
+Four buffers track the in-progress token:
+
+| Buffer              | Contents                                    |
+| ------------------- | ------------------------------------------- |
+| `#leftBuffer`       | Plain text accumulated outside any tag      |
+| `#leftTokenBuffer`  | Opening brace characters (`{`, `{{`, `{{{`) |
+| `#middleBuffer`     | Tag key (the content between the braces)    |
+| `#rightTokenBuffer` | Closing brace characters                    |
+
+A tag is complete when `#leftTokenBuffer.length === #rightTokenBuffer.length`. At that point the concatenated string is passed to `Tag.parse()` and a `TokenizeTagEvent` is emitted.
+
+### Adjacent tags
+
+When two tags appear back-to-back (e.g. `{{a}}{{b}}`), the closing `}` of the first tag and the opening `{` of the second arrive in sequence. The state machine transitions from `tagExitingMaybe` to `tagEnteringMaybe` when a `{` is seen and the exit brace count matches the enter brace count. `#processChar` detects this transition and calls `#emitTagToken()` to flush the completed first tag before the second tag starts accumulating.
+
+### Finalization
+
+When `push(null)` is called, any text remaining in `#leftBuffer` is emitted as a `text` token, `#emitTagToken()` is called to flush any in-progress tag, then a `null` token is emitted to signal end-of-stream.
+
+## Usage
+
+```ts
+import { Tokenize } from "./Tokenize";
+import { Parse } from "../parse/Parse";
+
+const tokenize = new Tokenize().on("token", async (token) => {
+  if (token === null) return; // end of stream
+  if (token.type === "text") {
+    /* plain text */
+  }
+  if (token.type === "tag") {
+    /* mustache tag */
+  }
+});
+
+const parse = new Parse({ onChar: tokenize.push.bind(tokenize) });
+parse.read(myReadableStream);
+```
+
+## API
+
+### `tokenize.on(eventName, listener)`
+
+Inherited from `EventEmitter`. Registers a listener for the `token` event.
+
+### `tokenize.push(char: string | null): Promise<void>`
+
+Pushes a single character into the tokenizer. Pass `null` to signal end-of-stream and flush remaining buffers.
+
+### Events
+
+| Event   | Payload             | Description                                  |
+| ------- | ------------------- | -------------------------------------------- |
+| `token` | `TokenizeTextEvent` | A run of plain text outside any tag.         |
+| `token` | `TokenizeTagEvent`  | A complete mustache tag.                     |
+| `token` | `null`              | End of stream; all tokens have been emitted. |
+
+#### `TokenizeTextEvent`
+
+| Property | Type     | Description                                 |
+| -------- | -------- | ------------------------------------------- |
+| `type`   | `"text"` | Discriminant.                               |
+| `data`   | `string` | The plain-text content.                     |
+| `uid`    | `number` | `performance.now()` timestamp at emit time. |
+
+#### `TokenizeTagEvent`
+
+| Property | Type     | Description                                 |
+| -------- | -------- | ------------------------------------------- |
+| `type`   | `"tag"`  | Discriminant.                               |
+| `data`   | `Tag`    | Parsed tag object (see `Tag`).              |
+| `uid`    | `number` | `performance.now()` timestamp at emit time. |

package/src/lib/tokenize/Tokenize.spec.ts

@@ -0,0 +1,364 @@
+import { TextDecoderStream } from "node:stream/web";
+import { createReadableStream } from "../test/streams";
+import { Parse, type ParseOptions } from "../parse/Parse";
+import { Tokenize, TokenizeTagEvent, TokenizeTextEvent } from "./Tokenize";
+
+(process.env as any).LOG_LEVEL = "debug";
+
+describe("Tokenize", () => {
+  it("tokenizes a mustache template: single variable tag", async () => {
+    const queue: (TokenizeTagEvent | TokenizeTextEvent | null)[] = [];
+
+    const t = new Tokenize().on("token", (evt) => queue.push(evt));
+
+    const options: ParseOptions = {
+      onChar: t.push.bind(t),
+    };
+
+    const mockHandleInactive = vitest.fn();
+    await new Promise<void>((resolve) => {
+      new Parse(options)
+        .on("inactive", mockHandleInactive.mockImplementation(resolve))
+        .read(
+          createReadableStream("a{{m}}z").pipeThrough(new TextDecoderStream())
+        );
+    });
+
+    expect(queue).toHaveLength(4);
+
+    expect(queue[0]).toStrictEqual(
+      expect.objectContaining({
+        data: "a",
+        type: "text",
+        uid: expect.any(Number),
+      })
+    );
+
+    const { data, type } = queue[1] as TokenizeTagEvent;
+    expect(type).toBe("tag");
+    expect(data.type).toBe("variable");
+    expect(data.value).toBe("m");
+
+    expect(queue[2]).toStrictEqual(
+      expect.objectContaining({
+        data: "z",
+        type: "text",
+        uid: expect.any(Number),
+      })
+    );
+
+    expect(queue[3]).toBeNull();
+
+    expect(mockHandleInactive).toHaveBeenCalledTimes(1);
+  });
+
+  it("tokenizes a mustache template: single partial tag", async () => {
+    const queue: (TokenizeTagEvent | TokenizeTextEvent | null)[] = [];
+
+    const t = new Tokenize();
+    t.on("token", (evt) => queue.push(evt));
+
+    const options: ParseOptions = {
+      onChar: t.push.bind(t),
+    };
+
+    const mockHandleInactive = vitest.fn();
+    await new Promise<void>((resolve) => {
+      new Parse(options)
+        .on("inactive", mockHandleInactive.mockImplementation(resolve))
+        .read(
+          createReadableStream("a{{>m}}z").pipeThrough(new TextDecoderStream())
+        );
+    });
+
+    expect(queue.length).toBe(4);
+
+    let { data, type } = queue[0] as TokenizeTextEvent;
+    expect(type).toBe("text");
+    expect(data).toBe("a");
+
+    let { data: tagData, type: tagType } = queue[1] as TokenizeTagEvent;
+    expect(tagType).toBe("tag");
+    expect(tagData.type).toBe("partial");
+    expect(tagData.value).toBe(">m");
+
+    ({ data, type } = queue[2] as TokenizeTextEvent);
+    expect(type).toBe("text");
+    expect(data).toBe("z");
+
+    expect(queue[3]).toBeNull();
+
+    expect(mockHandleInactive).toHaveBeenCalledTimes(1);
+  });
+
+  it("tokenizes a mustache template: single section tag", async () => {
+    const queue: (TokenizeTagEvent | TokenizeTextEvent | null)[] = [];
+
+    const t = new Tokenize();
+    t.on("token", (evt) => queue.push(evt));
+
+    const options: ParseOptions = {
+      onChar: t.push.bind(t),
+    };
+
+    const mockHandleInactive = vitest.fn();
+    await new Promise<void>((resolve) => {
+      new Parse(options)
+        .on("inactive", mockHandleInactive.mockImplementation(resolve))
+        .read(
+          createReadableStream("ab{{#g}}m{{/g}}yz").pipeThrough(
+            new TextDecoderStream()
+          )
+        );
+    });
+
+    expect(queue).toHaveLength(6);
+
+    let { data: textData, type: textType } = queue[0] as TokenizeTagEvent;
+    expect(textType).toBe("text");
+    expect(textData).toBe("ab");
+
+    let { data, type } = queue[1] as TokenizeTagEvent;
+    expect(type).toBe("tag");
+    expect(data.type).toBe("section");
+    expect(data.value).toBe("#g");
+
+    ({ data: textData, type: textType } = queue[2] as TokenizeTagEvent);
+    expect(textType).toBe("text");
+    expect(textData).toBe("m");
+
+    ({ data, type } = queue[3] as TokenizeTagEvent);
+    expect(type).toBe("tag");
+    expect(data.type).toBe("end");
+    expect(data.value).toBe("/g");
+
+    ({ data: textData, type: textType } = queue[4] as TokenizeTagEvent);
+    expect(textType).toBe("text");
+    expect(textData).toBe("yz");
+
+    expect(queue[5]).toBeNull();
+
+    expect(mockHandleInactive).toHaveBeenCalledTimes(1);
+  });
+
+  it("tokenizes a mustache template: single variable context tag", async () => {
+    const queue: (TokenizeTagEvent | TokenizeTextEvent | null)[] = [];
+
+    const t = new Tokenize();
+    t.on("token", (evt) => queue.push(evt));
+
+    const options: ParseOptions = {
+      onChar: t.push.bind(t),
+    };
+
+    const mockHandleInactive = vitest.fn();
+    await new Promise<void>((resolve) => {
+      new Parse(options)
+        .on("inactive", mockHandleInactive.mockImplementation(resolve))
+        .read(
+          createReadableStream("a{{.}}z").pipeThrough(new TextDecoderStream())
+        );
+    });
+
+    console.log(`\n${"-".repeat(80)}\nqueue=`, JSON.stringify(queue));
+
+    expect(queue).toHaveLength(4);
+
+    let { type } = queue[0] as TokenizeTextEvent;
+    expect(type).toBe("text");
+
+    let { data: tagData } = queue[1] as TokenizeTagEvent;
+    expect(tagData.type).toBe("implicit");
+    expect(tagData.value).toBe(".");
+
+    ({ type } = queue[2] as TokenizeTextEvent);
+    expect(type).toBe("text");
+
+    expect(queue[3]).toBeNull();
+
+    expect(mockHandleInactive).toHaveBeenCalledTimes(1);
+  });
+
+  it("tokenizes a mustache template: nested tags", async () => {
+    const queue: (TokenizeTagEvent | TokenizeTextEvent | null)[] = [];
+
+    const t = new Tokenize();
+    t.on("token", (evt) => queue.push(evt));
+
+    const options: ParseOptions = {
+      onChar: vitest.fn(t.push.bind(t)),
+    };
+
+    const mockHandleInactive = vitest.fn();
+    await new Promise<void>((resolve) => {
+      new Parse(options)
+        .on("inactive", mockHandleInactive.mockImplementation(resolve))
+        .read(
+          createReadableStream("a{{#e}}{{>m}}{{/e}}z").pipeThrough(
+            new TextDecoderStream()
+          )
+        );
+    });
+
+    expect(queue.length).toBe(6);
+
+    let { data: textData, type: textType } = queue[0] as TokenizeTextEvent;
+    expect(textType).toBe("text");
+    expect(textData).toBe("a");
+
+    let { data } = queue[1] as TokenizeTagEvent;
+    expect(data.type).toBe("section");
+    expect(data.value).toBe("#e");
+
+    ({ data } = queue[2] as TokenizeTagEvent);
+    expect(data.type).toBe("partial");
+    expect(data.value).toBe(">m");
+
+    ({ data } = queue[3] as TokenizeTagEvent);
+    expect(data.type).toBe("end");
+    expect(data.value).toBe("/e");
+
+    ({ data: textData, type: textType } = queue[4] as TokenizeTextEvent);
+    expect(textType).toBe("text");
+    expect(textData).toBe("z");
+
+    expect(queue[5]).toBeNull();
+
+    expect(mockHandleInactive).toHaveBeenCalledTimes(1);
+  });
+
+  it("tokenizes an empty section", async () => {
+    const queue: (TokenizeTagEvent | TokenizeTextEvent | null)[] = [];
+
+    const t = new Tokenize();
+    t.on("token", (evt) => queue.push(evt));
+
+    const options: ParseOptions = {
+      onChar: vitest.fn(t.push.bind(t)),
+    };
+
+    const mockHandleInactive = vitest.fn();
+    await new Promise<void>((resolve) => {
+      new Parse(options)
+        .on("inactive", mockHandleInactive.mockImplementation(resolve))
+        .read(
+          createReadableStream("a{{#e}}{{/e}}z").pipeThrough(
+            new TextDecoderStream()
+          )
+        );
+    });
+
+    expect(queue.length).toBe(5);
+
+    let { data: textData, type: textType } = queue[0] as TokenizeTextEvent;
+    expect(textType).toBe("text");
+    expect(textData).toBe("a");
+
+    let { data } = queue[1] as TokenizeTagEvent;
+    expect(data.type).toBe("section");
+    expect(data.value).toBe("#e");
+
+    ({ data } = queue[2] as TokenizeTagEvent);
+    expect(data.type).toBe("end");
+    expect(data.value).toBe("/e");
+
+    ({ data: textData, type: textType } = queue[3] as TokenizeTextEvent);
+    expect(textType).toBe("text");
+    expect(textData).toBe("z");
+
+    expect(queue[4]).toBeNull();
+
+    expect(mockHandleInactive).toHaveBeenCalledTimes(1);
+  });
+
+  it("tokenizes text containing single curly braces as plain text", async () => {
+    const queue: (TokenizeTagEvent | TokenizeTextEvent | null)[] = [];
+
+    const t = new Tokenize().on("token", (evt) => queue.push(evt));
+
+    const options: ParseOptions = {
+      onChar: t.push.bind(t),
+    };
+
+    const mockHandleInactive = vitest.fn();
+    await new Promise<void>((resolve) => {
+      new Parse(options)
+        .on("inactive", mockHandleInactive.mockImplementation(resolve))
+        .read(
+          createReadableStream("a {\n  color: red;\n} b").pipeThrough(
+            new TextDecoderStream()
+          )
+        );
+    });
+
+    expect(queue).toHaveLength(2);
+
+    expect(queue[0]).toStrictEqual(
+      expect.objectContaining({
+        data: "a {\n  color: red;\n} b",
+        type: "text",
+        uid: expect.any(Number),
+      })
+    );
+
+    expect(queue[1]).toBeNull();
+
+    expect(mockHandleInactive).toHaveBeenCalledTimes(1);
+  });
+
+  it("tokenizes nested sections", async () => {
+    const queue: (TokenizeTagEvent | TokenizeTextEvent | null)[] = [];
+
+    const t = new Tokenize();
+    t.on("token", (evt) => queue.push(evt));
+
+    const options: ParseOptions = {
+      onChar: vitest.fn(t.push.bind(t)),
+    };
+
+    const mockHandleInactive = vitest.fn();
+    await new Promise<void>((resolve) => {
+      new Parse(options)
+        .on("inactive", mockHandleInactive.mockImplementation(resolve))
+        .read(
+          createReadableStream(
+            "aa{{#ee}}{{#.}}{{.}}{{/.}}{{/ee}}zz"
+          ).pipeThrough(new TextDecoderStream())
+        );
+    });
+
+    expect(queue.length).toBe(8);
+
+    let { data: textData, type: textType } = queue[0] as TokenizeTextEvent;
+    expect(textType).toBe("text");
+    expect(textData).toBe("aa");
+
+    let { data } = queue[1] as TokenizeTagEvent;
+    expect(data.type).toBe("section");
+    expect(data.value).toBe("#ee");
+
+    ({ data } = queue[2] as TokenizeTagEvent);
+    expect(data.type).toBe("section");
+    expect(data.value).toBe("#.");
+
+    ({ data } = queue[3] as TokenizeTagEvent);
+    expect(data.type).toBe("implicit");
+    expect(data.value).toBe(".");
+
+    ({ data } = queue[4] as TokenizeTagEvent);
+    expect(data.type).toBe("end");
+    expect(data.value).toBe("/.");
+
+    ({ data } = queue[5] as TokenizeTagEvent);
+    expect(data.type).toBe("end");
+    expect(data.value).toBe("/ee");
+
+    ({ data: textData, type: textType } = queue[6] as TokenizeTextEvent);
+    expect(textType).toBe("text");
+    expect(textData).toBe("zz");
+
+    expect(queue[7]).toBeNull();
+
+    expect(mockHandleInactive).toHaveBeenCalledTimes(1);
+  });
+});