@funcstache/stache-stream 0.2.2 → 0.2.3

package/src/lib/queue/Queue.spec.ts

@@ -0,0 +1,275 @@
+import { TextDecoderStream } from "node:stream/web";
+import { createReadableStream } from "../test/streams";
+import { Parse } from "../parse/Parse";
+import {
+  Tokenize,
+  type TokenizeTagEvent,
+  type TokenizeTextEvent,
+} from "../tokenize/Tokenize";
+import { Queue, type TokenizeAllEvent } from "./Queue";
+
+// (process.env as any).LOG_LEVEL = "debug";
+
+describe("Queue", () => {
+  it("queues variable", async () => {
+    let resolve: () => void;
+    const mockHandler = vitest.fn();
+    const q = new Queue()
+      .on("inactive", () => Promise.resolve(resolve()))
+      .on("text", mockHandler)
+      .on("variable", mockHandler);
+
+    const t = new Tokenize().on("token", q.push.bind(q));
+
+    new Parse({ onChar: t.push.bind(t) }).read(
+      createReadableStream("ab{{m}}yz").pipeThrough(new TextDecoderStream())
+    );
+
+    await new Promise<void>((res) => {
+      resolve = res;
+    });
+
+    expect(mockHandler).toHaveBeenCalledTimes(3);
+
+    let [textEvt] = mockHandler.mock.calls[0] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("ab");
+    expect(textEvt.type).toBe("text");
+
+    const [evt] = mockHandler.mock.calls[1] as [TokenizeTagEvent];
+    expect(evt.data.value).toBe("m");
+    expect(evt.data.type).toBe("variable");
+
+    [textEvt] = mockHandler.mock.calls[2] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("yz");
+    expect(textEvt.type).toBe("text");
+  });
+
+  it("queues partial", async () => {
+    let resolve: () => void;
+    const mockHandler = vitest.fn();
+    const q = new Queue()
+      .on("inactive", () => Promise.resolve(resolve()))
+      .on("text", mockHandler)
+      .on("partial", mockHandler);
+
+    const t = new Tokenize().on("token", q.push.bind(q));
+
+    new Parse({ onChar: t.push.bind(t) }).read(
+      createReadableStream("ab{{>m}}yz").pipeThrough(new TextDecoderStream())
+    );
+
+    await new Promise<void>((res) => {
+      resolve = res;
+    });
+
+    expect(mockHandler).toHaveBeenCalledTimes(3);
+
+    let [textEvt] = mockHandler.mock.calls[0] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("ab");
+    expect(textEvt.type).toBe("text");
+
+    const [evt] = mockHandler.mock.calls[1] as [TokenizeTagEvent];
+    expect(evt.data.value).toBe(">m");
+    expect(evt.data.type).toBe("partial");
+
+    [textEvt] = mockHandler.mock.calls[2] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("yz");
+    expect(textEvt.type).toBe("text");
+  });
+
+  it("queues two simple tags", async () => {
+    let resolve: () => void;
+    const mockHandler = vitest.fn();
+    const q = new Queue()
+      .on("inactive", () => Promise.resolve(resolve()))
+      .on("text", mockHandler)
+      .on("partial", mockHandler)
+      .on("variable", mockHandler);
+
+    const t = new Tokenize().on("token", q.push.bind(q));
+
+    new Parse({ onChar: t.push.bind(t) }).read(
+      createReadableStream("ab{{m}}{{>n}}yz").pipeThrough(
+        new TextDecoderStream()
+      )
+    );
+
+    await new Promise<void>((res) => {
+      resolve = res;
+    });
+
+    expect(mockHandler).toHaveBeenCalledTimes(4);
+
+    let [textEvt] = mockHandler.mock.calls[0] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("ab");
+    expect(textEvt.type).toBe("text");
+
+    let [evt] = mockHandler.mock.calls[1] as [TokenizeTagEvent];
+    expect(evt.data.value).toBe("m");
+    expect(evt.data.type).toBe("variable");
+
+    [evt] = mockHandler.mock.calls[2] as [TokenizeTagEvent];
+    expect(evt.data.value).toBe(">n");
+    expect(evt.data.type).toBe("partial");
+
+    [textEvt] = mockHandler.mock.calls[3] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("yz");
+    expect(textEvt.type).toBe("text");
+  });
+
+  it("queues section tag", async () => {
+    let resolve: () => void;
+    const mockHandler = vitest.fn();
+    const q = new Queue()
+      .on("inactive", () => Promise.resolve(resolve()))
+      .on("text", mockHandler)
+      .on("partial", mockHandler)
+      .on("section", mockHandler)
+      .on("variable", mockHandler);
+
+    const t = new Tokenize().on("token", q.push.bind(q));
+
+    new Parse({ onChar: t.push.bind(t) }).read(
+      createReadableStream("ab{{#m}}{{n}}{{/m}}yz").pipeThrough(
+        new TextDecoderStream()
+      )
+    );
+
+    await new Promise<void>((res) => {
+      resolve = res;
+    });
+
+    expect(mockHandler).toHaveBeenCalledTimes(3);
+
+    let [textEvt] = mockHandler.mock.calls[0] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("ab");
+    expect(textEvt.type).toBe("text");
+
+    const [acc] = mockHandler.mock.calls[1] as [TokenizeTagEvent[]];
+    expect(acc).toHaveLength(3);
+
+    const [one, two, three] = acc;
+    expect(one.data.value).toBe("#m");
+    expect(one.data.type).toBe("section");
+    expect(two.data.value).toBe("n");
+    expect(two.data.type).toBe("variable");
+    expect(three.data.value).toBe("/m");
+    expect(three.data.type).toBe("end");
+
+    [textEvt] = mockHandler.mock.calls[2] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("yz");
+    expect(textEvt.type).toBe("text");
+  });
+
+  it("accumulates text inside a section", async () => {
+    let resolve: () => void;
+    const mockHandler = vitest.fn();
+    const q = new Queue()
+      .on("inactive", () => Promise.resolve(resolve()))
+      .on("section", mockHandler)
+      .on("text", mockHandler);
+
+    const t = new Tokenize().on("token", q.push.bind(q));
+
+    new Parse({ onChar: t.push.bind(t) }).read(
+      createReadableStream("{{#m}}text{{/m}}").pipeThrough(
+        new TextDecoderStream()
+      )
+    );
+
+    await new Promise<void>((res) => {
+      resolve = res;
+    });
+
+    expect(mockHandler).toHaveBeenCalledTimes(1);
+
+    const [acc] = mockHandler.mock.calls[0] as [TokenizeAllEvent[]];
+    expect(acc).toHaveLength(3);
+
+    const [one, two, three] = acc;
+    expect(one.data.toString()).toBe("{{#m}}");
+    expect(two.data).toBe("text");
+    expect(two.type).toBe("text");
+    expect(three.data.toString()).toBe("{{/m}}");
+  });
+
+  it("accumulates mixed text and tags inside a section", async () => {
+    let resolve: () => void;
+    const mockHandler = vitest.fn();
+    const q = new Queue()
+      .on("inactive", () => Promise.resolve(resolve()))
+      .on("section", mockHandler)
+      .on("text", mockHandler);
+
+    const t = new Tokenize().on("token", q.push.bind(q));
+
+    new Parse({ onChar: t.push.bind(t) }).read(
+      createReadableStream("{{#m}} L {{n}} R {{/m}}").pipeThrough(
+        new TextDecoderStream()
+      )
+    );
+
+    await new Promise<void>((res) => {
+      resolve = res;
+    });
+
+    expect(mockHandler).toHaveBeenCalledTimes(1);
+
+    const [acc] = mockHandler.mock.calls[0] as [TokenizeAllEvent[]];
+    expect(acc).toHaveLength(5);
+
+    const [one, two, three, four, five] = acc;
+    expect(one.data.toString()).toBe("{{#m}}");
+    expect(two.data).toBe(" L ");
+    expect(two.type).toBe("text");
+    expect(three.data.toString()).toBe("{{n}}");
+    expect(four.data).toBe(" R ");
+    expect(four.type).toBe("text");
+    expect(five.data.toString()).toBe("{{/m}}");
+  });
+
+  it("queues implicit tag", async () => {
+    let resolve: () => void;
+    const mockHandler = vitest.fn();
+    const q = new Queue()
+      .on("implicit", mockHandler)
+      .on("inactive", () => Promise.resolve(resolve()))
+      .on("text", mockHandler)
+      .on("partial", mockHandler)
+      .on("section", mockHandler)
+      .on("variable", mockHandler);
+
+    const t = new Tokenize().on("token", q.push.bind(q));
+
+    new Parse({ onChar: t.push.bind(t) }).read(
+      createReadableStream("ab{{#m}}{{.}}{{/m}}yz").pipeThrough(
+        new TextDecoderStream()
+      )
+    );
+
+    await new Promise<void>((res) => {
+      resolve = res;
+    });
+
+    expect(mockHandler).toHaveBeenCalledTimes(3);
+
+    let [textEvt] = mockHandler.mock.calls[0] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("ab");
+    expect(textEvt.type).toBe("text");
+
+    const [acc] = mockHandler.mock.calls[1] as [TokenizeTagEvent[]];
+    expect(acc).toHaveLength(3);
+
+    const [one, two, three] = acc;
+    expect(one.data.value).toBe("#m");
+    expect(one.data.type).toBe("section");
+    expect(two.data.value).toBe(".");
+    expect(two.data.type).toBe("implicit");
+    expect(three.data.value).toBe("/m");
+    expect(three.data.type).toBe("end");
+
+    [textEvt] = mockHandler.mock.calls[2] as [TokenizeTextEvent];
+    expect(textEvt.data).toBe("yz");
+    expect(textEvt.type).toBe("text");
+  });
+});

package/src/lib/queue/Queue.ts

@@ -0,0 +1,253 @@
+import { Log } from "@funcstache/logger";
+import { TokenizeTagEvent, TokenizeTextEvent } from "../tokenize/Tokenize";
+
+export class Queue implements AsyncEventEmitter<QueueEventMap> {
+  #logger = new Log({
+    category: "QUE",
+    level: (process.env as any).LOG_LEVEL || "warn",
+  });
+
+  #active = false;
+  #listeners: Listeners<QueueEventMap>[] = [];
+  #accumulator: Accumulator | undefined;
+  #tokenQueue: TokenizeEvent[] = [];
+
+  get tokenQueue() {
+    return Array.from(this.#tokenQueue);
+  }
+
+  on<E extends keyof QueueEventMap>(
+    eventName: Listeners<QueueEventMap, E>["eventName"],
+    listener: Listeners<QueueEventMap, E>["listener"]
+  ) {
+    if (
+      !this.#listeners.some(
+        ({ eventName: evn, listener: lst }) =>
+          eventName === evn && listener === lst
+      )
+    ) {
+      const l = listener as any;
+      this.#listeners.push({ eventName, listener: l });
+    }
+
+    return this;
+  }
+
+  async push(token: TokenizeEvent): Promise<void> {
+    // this.#logger.debug(() => [`••• push: token=`, token]);
+
+    this.#tokenQueue.push(token);
+    this.#processToken();
+  }
+
+  async #emit<K extends keyof QueueEventMap>(
+    eventName: K,
+    event: QueueEventMap[K]
+  ): Promise<void> {
+    for (const item of this.#listeners) {
+      if (item.eventName !== eventName) {
+        continue;
+      }
+
+      this.#logger.debug(() => [`••• #emit: event=`, event]);
+      await item.listener(event);
+    }
+  }
+
+  async #processToken(): Promise<void> {
+    if (this.#active) {
+      return;
+    }
+
+    this.#active = true;
+
+    const token = this.#tokenQueue.shift();
+    if (token === undefined) {
+      return;
+    }
+
+    // this.#logger.debug(() => [
+    //   `••• #processToken: token='${JSON.stringify(token)}'`,
+    // ]);
+
+    if (token === null) {
+      await this.#emit("inactive", undefined);
+      return;
+    }
+
+    if (token.type === "text") {
+      if (this.#accumulator) {
+        this.#accumulator.push(token);
+      } else {
+        await this.#emit("text", token);
+      }
+    } else if (token.type === "tag") {
+      // this.#logger.debug(() => [
+      //   `    #processToken: token type='${token.data.type}'`,
+      // ]);
+
+      switch (token.data.type) {
+        case "comment":
+        case "implicit":
+        case "partial":
+        case "variable": {
+          if (this.#accumulator) {
+            this.#accumulator.push(token);
+          } else {
+            await this.#emit(token.data.type, token);
+          }
+
+          break;
+        }
+
+        case "inverted":
+        case "section": {
+          if (!this.#accumulator) {
+            const acc: Accumulator = [token] as any;
+            acc.level = 0;
+            this.#accumulator = acc;
+          } else {
+            this.#accumulator.level++;
+            this.#accumulator.push(token);
+          }
+
+          break;
+        }
+
+        case "end": {
+          if (!this.#accumulator) {
+            this.#emit(
+              "error",
+              new QueueError<TokenizeAllEvent>(
+                `Received end tag '${token.data.value}' without a start tag.`,
+                token
+              )
+            );
+            break;
+          }
+
+          // Received an end tag, but we are not at the lowest level. Add this tag to the
+          // accumulator and reduce the level by one then exit.
+          if (0 < this.#accumulator.level) {
+            this.#accumulator.level--;
+            this.#accumulator.push(token);
+            break;
+          }
+
+          // We are at accumulator level 0 so if we've done everything right thing we should emit a
+          // "section" event with all the accumulated tags...
+          const start = this.#accumulator[0];
+          if (!isTagEvent(start)) {
+            this.#emit(
+              "error",
+              new QueueError<TokenizeAllEvent>(
+                `The start tag '${JSON.stringify(
+                  start
+                )}' is not a TokenizeTagEvent.`,
+                start
+              )
+            );
+            break;
+          }
+
+          if (start.data.key !== token.data.key) {
+            this.#emit(
+              "error",
+              new QueueError<TokenizeAllEvent[]>(
+                `The start tag '${JSON.stringify(
+                  start
+                )}' does not match the end tag '${JSON.stringify(token)}'`,
+                [start, token]
+              )
+            );
+            break;
+          }
+
+          if (
+            start.data.type === "inverted" ||
+            start.data.type === "section"
+          ) {
+            await this.#emit(start.data.type, [
+              ...Array.from(this.#accumulator),
+              token,
+            ]);
+
+            this.#accumulator = undefined;
+          } else {
+            this.#emit(
+              "error",
+              new QueueError<TokenizeAllEvent>(
+                `The start tag '${JSON.stringify(start)}' is not a section tag`,
+                start
+              )
+            );
+          }
+
+          break;
+        }
+      }
+    }
+
+    // this.#logger.debug(() => [
+    //   `    #processToken: this.#accumulator.level=${
+    //     this.#accumulator?.level
+    //   }, this.#accumulator='${JSON.stringify(this.#accumulator)}'`,
+    //   ,
+    // ]);
+
+    this.#active = false;
+
+    if (this.#tokenQueue.length) {
+      this.#processToken();
+    }
+  }
+}
+
+export class QueueError<
+  D extends TokenizeAllEvent | TokenizeAllEvent[] = TokenizeAllEvent
+> extends Error {
+  readonly data: D;
+
+  constructor(message: string, token: D) {
+    super(message);
+
+    this.data = token;
+  }
+}
+
+function isTagEvent(obj: any): obj is TokenizeTagEvent {
+  return obj && "type" in obj && obj.type === "tag";
+}
+
+interface AsyncEventEmitter<T extends object, K extends keyof T = keyof T> {
+  on: (
+    eventName: K,
+    listener: (event: T[K]) => Promise<void>
+  ) => AsyncEventEmitter<T>;
+}
+
+type Accumulator = TokenizeAllEvent[] & {
+  level: number;
+};
+
+interface Listeners<T extends object, K extends keyof T = keyof T> {
+  eventName: K;
+  listener: (event: T[K]) => Promise<void>;
+}
+
+export type TokenizeAllEvent = TokenizeTagEvent | TokenizeTextEvent;
+
+type TokenizeEvent = TokenizeAllEvent | null;
+
+type QueueEventMap = {
+  comment: TokenizeTagEvent;
+  // end: TokenizeTagEvent;
+  error: QueueError<TokenizeAllEvent | TokenizeAllEvent[]>;
+  implicit: TokenizeTagEvent;
+  inactive: undefined;
+  inverted: TokenizeAllEvent[];
+  partial: TokenizeTagEvent;
+  section: TokenizeAllEvent[];
+  text: TokenizeTextEvent;
+  variable: TokenizeTagEvent;
+};

package/src/lib/queue/README.md

@@ -0,0 +1,110 @@
+# Queue
+
+`Queue` receives a stream of tokenized mustache events from `Tokenize` and re-emits them as higher-level, typed events. Its primary job is to accumulate the tokens that belong to a section or inverted section and emit them together as a single event once the closing tag is seen.
+
+## How it works
+
+### Token input
+
+Tokens are pushed one at a time via `push(token)`. `null` signals end-of-stream. Tokens are added to an internal array (`#tokenQueue`) and processed serially — `#processToken` sets `#active = true` while running and recurses at the end if more tokens arrived in the meantime, preventing concurrent processing.
+
+### Simple tokens (no accumulator active)
+
+When no section is open, simple tokens are emitted immediately:
+
+| Token type | Event emitted | Payload             |
+| ---------- | ------------- | ------------------- |
+| `text`     | `text`        | `TokenizeTextEvent` |
+| `variable` | `variable`    | `TokenizeTagEvent`  |
+| `implicit` | `implicit`    | `TokenizeTagEvent`  |
+| `partial`  | `partial`     | `TokenizeTagEvent`  |
+| `null`     | `inactive`    | `undefined`         |
+
+### Section / inverted-section accumulation
+
+When a `section` or `inverted` opening tag is received, `Queue` creates an `Accumulator` — an array of `TokenizeAllEvent` with an extra `level` counter — and adds the opening tag to it. All subsequent tokens (text, variables, nested sections, etc.) are pushed into the accumulator instead of being emitted.
+
+Nested sections increment `level`; their matching `end` tags decrement it. When an `end` tag is received at `level === 0`, the accumulator (opening tag through closing tag inclusive) is emitted as a single `section` or `inverted` event and the accumulator is cleared.
+
+### Error cases
+
+`Queue` emits an `error` event (with a `QueueError` payload) for:
+
+- An `end` tag received when no accumulator is active (unmatched close).
+- The first element of the accumulator is not a tag event.
+- The `end` tag key does not match the opening tag key.
+- The opening tag stored in the accumulator is not a `section` or `inverted` type.
+
+### Event registration
+
+`on(eventName, listener)` stores async listeners. Duplicate registrations (same event name and same listener reference) are silently ignored. All listener calls are `await`ed in registration order.
+
+## Usage
+
+```ts
+import { Queue } from "./Queue";
+import { Tokenize } from "../tokenize/Tokenize";
+import { Parse } from "../parse/Parse";
+
+const queue = new Queue()
+  .on("text", async (event) => {
+    /* plain text node */
+  })
+  .on("variable", async (event) => {
+    /* {{variable}} */
+  })
+  .on("partial", async (event) => {
+    /* {{>partial}} */
+  })
+  .on("implicit", async (event) => {
+    /* {{.}} */
+  })
+  .on("section", async (events) => {
+    /* {{#section}}...{{/section}} */
+  })
+  .on("inverted", async (events) => {
+    /* {{^inverted}}...{{/inverted}} */
+  })
+  .on("error", async (err) => {
+    /* QueueError */
+  })
+  .on("inactive", async () => {
+    /* end of stream */
+  });
+
+const tokenize = new Tokenize().on("token", queue.push.bind(queue));
+const parse = new Parse({ onChar: tokenize.push.bind(tokenize) });
+
+parse.read(myReadableStream);
+```
+
+## API
+
+### `queue.on(eventName, listener)`
+
+Registers an async listener for a queue event. Returns `this` for chaining. Safe to call multiple times with the same arguments — duplicates are ignored.
+
+### `queue.push(token: TokenizeAllEvent | null): Promise<void>`
+
+Pushes a token into the queue for processing. Pass `null` to signal end-of-stream.
+
+### `queue.tokenQueue`
+
+Read-only snapshot of the pending token queue (copy of the internal array).
+
+### Events
+
+| Event      | Payload              | Description                                              |
+| ---------- | -------------------- | -------------------------------------------------------- |
+| `text`     | `TokenizeTextEvent`  | A plain-text node outside any section.                   |
+| `variable` | `TokenizeTagEvent`   | A `{{variable}}` tag outside any section.                |
+| `implicit` | `TokenizeTagEvent`   | A `{{.}}` tag outside any section.                       |
+| `partial`  | `TokenizeTagEvent`   | A `{{>partial}}` tag outside any section.                |
+| `section`  | `TokenizeAllEvent[]` | All tokens from `{{#key}}` through `{{/key}}` inclusive. |
+| `inverted` | `TokenizeAllEvent[]` | All tokens from `{{^key}}` through `{{/key}}` inclusive. |
+| `error`    | `QueueError`         | A structural error (unmatched tags, key mismatch).       |
+| `inactive` | `undefined`          | End of stream; all tokens have been processed.           |
+
+### `QueueError`
+
+Extends `Error`. Has a `data` property containing the offending `TokenizeAllEvent` or `TokenizeAllEvent[]`.

package/src/lib/stache-stream/README.md

@@ -0,0 +1,45 @@
+# StacheTransformStream
+
+`StacheTransformStream` is a `TransformStream<string, string>` that renders a mustache template from a stream of string chunks. Incoming chunks are fed into the `Template` pipeline (`Parse → Tokenize → Queue → Template`) and rendered output chunks are enqueued to the readable side as they are produced.
+
+## How it works
+
+### Internal pipeline
+
+The constructor creates an internal `ReadableStream` whose controller is captured as `push` and `close` functions. On `start`, a `Template` is instantiated with that stream and a `writeToOutput` that enqueues rendered text to the outer `TransformStream` controller.
+
+Each chunk received by `transform` is pushed into the internal stream. When the writable side closes, `flush` closes the internal stream and waits for the `Template` to emit `"inactive"` before resolving, ensuring all rendered output is flushed before the readable side ends.
+
+### Error propagation
+
+If `controller.enqueue` throws (e.g. the readable side has been cancelled), the error is captured and re-thrown so that the `inactive` promise rejects. `flush` catches that rejection and forwards it to `controller.error`, surfacing it to the pipeline consumer.
+
+## Usage
+
+Pipe any `ReadableStream<string>` through `StacheTransformStream` to get a rendered `ReadableStream<string>`:
+
+```ts
+import { StacheTransformStream } from "./StacheStream";
+
+const stache = new StacheTransformStream({
+  contextProvider: { context: { name: "world" } },
+});
+
+const rendered = templateStream.pipeThrough(stache);
+
+for await (const chunk of rendered) {
+  process.stdout.write(chunk);
+}
+```
+
+## API
+
+### `new StacheTransformStream(options?)`
+
+| Option            | Type                         | Description                                                   |
+| ----------------- | ---------------------------- | ------------------------------------------------------------- |
+| `contextProvider` | `ContextProvider` (optional) | Supplies the data context and parent lookup for the template. |
+
+### Events
+
+`StacheTransformStream` does not emit events directly. Use standard `ReadableStream` consumption (`for await`, `pipeTo`, etc.) to receive rendered output and detect completion.