@techspokes/typescript-wsdl-client 0.16.1 → 0.17.0

package/docs/testing.md

@@ -246,6 +246,20 @@ const client = createMockClient({
 
 Mock responses use the pre-unwrap SOAP wrapper shape. The generated `unwrapArrayWrappers()` function handles conversion at runtime.
 
+For operations opted in via `--stream-config`, the mock returns `records: AsyncIterable<RecordType>` (via a small `asyncIterableOf` helper) and the generated happy-path test asserts on the NDJSON content-type and parseable record lines. Override a stream op with a multi-record iterable to exercise downstream backpressure:
+
+```typescript
+const client = createMockClient({
+  UnitDescriptiveInfoStream: async () => ({
+    records: (async function* () {
+      yield { Id: "1", Name: "Villa A" };
+      yield { Id: "2", Name: "Villa B" };
+    })(),
+    headers: {},
+  }),
+});
+```
+
 ## For Consumer Projects
 
 If you're using wsdl-tsc as a dependency and want to test your integration:

package/docs/troubleshooting.md

@@ -17,6 +17,12 @@ See [README](../README.md) for quick start and [CLI Reference](cli-reference.md)
 | TypeScript compilation errors | Check --import-extensions matches your tsconfig moduleResolution |
 | Gateway validation failures | Ensure OpenAPI has valid $ref paths and all schemas in components.schemas |
 | Catalog file not found | Catalog defaults to output directory; use --catalog-file to specify |
+| Stream config references unknown operation | Operation name must match the WSDL exactly; check spelling and casing |
+| Stream record type not found | `recordType` must exist in the main catalog or a companion `shapeCatalog` must supply it; confirm the companion WSDL compiles cleanly in isolation |
+| Structural collision between main and companion catalog | Two types share a name but differ structurally; rename in the companion source or point `recordType` at a distinct subtree |
+| NDJSON response ends abruptly | Mid-stream upstream error per the terminal-error policy; check gateway logs for the classified error |
+| Stream recordPath does not match | SAX matching is positional and case-sensitive; verify duplicate local-name segments are spelled exactly |
+| Stream client throws "stream request failed" | The upstream SOAP endpoint rejected the hand-built envelope; check `requestRaw` on the response and verify SOAP action and namespaces match the WSDL binding |
 
 ## SOAP Wire Logging
 
@@ -68,3 +74,15 @@ Or inspect catalog from client generation:
 ```bash
 cat ./src/services/hotel/catalog.json | jq '.types'
 ```
+
+## Streaming Debug
+
+Inspect stream metadata on the compiled catalog to confirm the config was parsed and applied:
+
+```bash
+cat ./src/services/hotel/catalog.json | jq '.operations[] | select(.stream) | {name, stream}'
+```
+
+Each entry shows the normalized `OperationStreamMetadata` (mode, format, mediaType, recordPath, recordTypeName, and any `shapeCatalogName`). If an expected operation is missing, the config either did not match the WSDL operation name or was not passed to the generation command.
+
+For record-path and chunk-boundary issues, the reference integration test pattern lives in `test/integration/stream-end-to-end.test.ts` and the SAX record matcher is exercised by `test/unit/stream-xml.test.ts`. Running them against a local fixture isolates whether the issue is in the config, the parser, or the upstream SOAP server.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@techspokes/typescript-wsdl-client",
-  "version": "0.16.1",
+  "version": "0.17.0",
   "description": "Turn legacy WSDL/SOAP services into typed TypeScript clients, OpenAPI 3.1 specs, and production-ready Fastify REST gateways. Built for enterprise SOAP modernization.",
   "keywords": [
     "wsdl",
@@ -53,7 +53,9 @@
     "docs",
     "README.md",
     "LICENSE",
-    "llms.txt"
+    "llms.txt",
+    "src/runtime/*.ts",
+    "src/runtime/*.tpl.txt"
   ],
   "scripts": {
     "dev": "tsx src/cli.ts",
@@ -93,6 +95,7 @@
     "@apidevtools/swagger-parser": "^12.1.0",
     "fast-xml-parser": "^5.7.1",
     "js-yaml": "^4.1.1",
+    "saxes": "^6.0.0",
     "soap": "^1.9.1",
     "yargs": "^18.0.0"
   },

package/src/runtime/clientStreamMethods.tpl.txt

@@ -0,0 +1,183 @@
+
+  /**
+   * Streaming transport for operations flagged with a stream configuration.
+   *
+   * node-soap buffers the full response before invoking the operation callback
+   * (verified empirically in ADR-002 / phase-0 research), so this method
+   * bypasses node-soap and POSTs a hand-built SOAP envelope directly, then
+   * pipes the response body through the SAX-driven record parser.
+   */
+  protected async callStream<RequestType, RecordType, HeadersType>(
+    args: RequestType,
+    operation: string,
+    requestType: string | undefined,
+    recordTypeName: string,
+    inputElementLocal: string,
+    inputElementNs: string,
+    soapAction: string,
+    recordPath: string[]
+  ): Promise<StreamOperationResponse<RecordType, HeadersType>> {
+    const client = await this.soapClient();
+    const endpoint = this.resolveStreamEndpoint(client);
+    const envelope = this.buildSoapEnvelope(args, operation, requestType, inputElementLocal, inputElementNs);
+    const res = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        "content-type": "text/xml; charset=utf-8",
+        "soapaction": '"' + soapAction + '"',
+      },
+      body: envelope,
+    });
+    if (!res.ok) {
+      throw new Error(
+        operation + " stream request failed: " + res.status + " " + res.statusText
+      );
+    }
+    if (!res.body) {
+      throw new Error(operation + " stream request returned an empty response body");
+    }
+    const headers: Record<string, string> = {};
+    res.headers.forEach((value, key) => { headers[key] = value; });
+    const spec: RecordParseSpec = {
+      recordPath,
+      recordTypeName,
+      attributesKey: this.attributesKeyOut,
+      childType: this.dataTypes?.ChildrenTypes,
+    };
+    const records = parseRecords<RecordType>(
+      this.webStreamToAsyncIterable(res.body),
+      spec
+    );
+    return {
+      records,
+      headers: headers as unknown as HeadersType,
+      requestRaw: envelope,
+    };
+  }
+
+  /**
+   * Walk the node-soap WSDL descriptor to locate the first service port's
+   * SOAP address. Falls back to the \`source\` URL when the descriptor is
+   * unavailable (e.g., \`source\` was given as a direct endpoint URL).
+   */
+  protected resolveStreamEndpoint(client: soap.Client): string {
+    const services = (client as any)?.wsdl?.definitions?.services ?? {};
+    for (const svc of Object.values(services)) {
+      const ports = (svc as any)?.ports ?? {};
+      for (const port of Object.values(ports)) {
+        const location = (port as any)?.location;
+        if (typeof location === "string" && location) return location;
+      }
+    }
+    return this.source;
+  }
+
+  /**
+   * Build a SOAP 1.1 envelope around the operation's input element. Uses the
+   * existing attributes / children metadata so attribute bags render as XML
+   * attributes and child properties render as nested elements.
+   */
+  protected buildSoapEnvelope(
+    args: unknown,
+    operation: string,
+    requestType: string | undefined,
+    inputElementLocal: string,
+    inputElementNs: string
+  ): string {
+    const body = this.toXmlElement(args, requestType, inputElementLocal, inputElementNs);
+    return '<?xml version="1.0" encoding="utf-8"?>' +
+      '<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">' +
+      '<soap:Body>' + body + '</soap:Body>' +
+      '</soap:Envelope>';
+  }
+
+  /**
+   * Serialize a value as a single XML element. The element namespace is only
+   * emitted when \`namespace\` is provided (i.e., the envelope's top-level body
+   * element). Nested elements inherit the namespace via XML scoping.
+   */
+  protected toXmlElement(
+    value: unknown,
+    typeName: string | undefined,
+    elementName: string,
+    namespace?: string
+  ): string {
+    const nsAttr = namespace ? ' xmlns="' + this.escapeXml(namespace) + '"' : "";
+    if (value === null || value === undefined) {
+      return '<' + elementName + nsAttr +
+        ' xsi:nil="true" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"/>';
+    }
+    if (typeof value !== "object") {
+      return '<' + elementName + nsAttr + '>' + this.escapeXml(String(value)) + '</' + elementName + '>';
+    }
+    if (Array.isArray(value)) {
+      return value.map((v) => this.toXmlElement(v, typeName, elementName, namespace)).join("");
+    }
+    const obj = value as Record<string, unknown>;
+    const attributesList = (typeName && this.dataTypes?.Attributes?.[typeName]) || [];
+    const childrenTypes = (typeName && this.dataTypes?.ChildrenTypes?.[typeName]) || {};
+    const attrPairs: Array<[string, string]> = [];
+    const bagIn = (obj as any)[this.attributesKeyIn] ?? (obj as any)["attributes"];
+    if (bagIn && typeof bagIn === "object") {
+      for (const [k, v] of Object.entries(bagIn)) attrPairs.push([k, this.normalizeAttr(v)]);
+    }
+    const childParts: string[] = [];
+    let textContent: string | undefined;
+    if ("$value" in obj) textContent = String(obj.$value ?? "");
+    for (const [k, v] of Object.entries(obj)) {
+      if (k === "$value" || k === this.attributesKeyIn || k === "attributes") continue;
+      if ((attributesList as readonly string[]).includes(k)) {
+        attrPairs.push([k, this.normalizeAttr(v)]);
+        continue;
+      }
+      const childTypeRaw = (childrenTypes as Record<string, string>)[k];
+      const childType = childTypeRaw?.endsWith("[]") ? childTypeRaw.slice(0, -2) : childTypeRaw;
+      if (Array.isArray(v)) {
+        for (const item of v) childParts.push(this.toXmlElement(item, childType, k));
+      } else {
+        childParts.push(this.toXmlElement(v, childType, k));
+      }
+    }
+    const attrStr = attrPairs.map(([k, val]) => ' ' + k + '="' + this.escapeXml(val) + '"').join("");
+    if (childParts.length === 0 && textContent === undefined) {
+      return '<' + elementName + nsAttr + attrStr + '/>';
+    }
+    const inner = childParts.join("") + (textContent !== undefined ? this.escapeXml(textContent) : "");
+    return '<' + elementName + nsAttr + attrStr + '>' + inner + '</' + elementName + '>';
+  }
+
+  protected normalizeAttr(v: unknown): string {
+    if (v === null || v === undefined) return "";
+    if (typeof v === "boolean") return v ? "true" : "false";
+    if (typeof v === "number") return Number.isFinite(v) ? String(v) : "";
+    return String(v);
+  }
+
+  protected escapeXml(s: string): string {
+    return s
+      .replace(/&/g, "&amp;")
+      .replace(/</g, "&lt;")
+      .replace(/>/g, "&gt;")
+      .replace(/"/g, "&quot;")
+      .replace(/'/g, "&apos;");
+  }
+
+  /**
+   * Iterate an uploaded web ReadableStream as Uint8Array chunks. Node 20+
+   * ReadableStream implements Symbol.asyncIterator natively; this helper
+   * normalizes the types so TypeScript can drive it through \`for await\`.
+   */
+  protected async *webStreamToAsyncIterable(
+    body: ReadableStream<Uint8Array>
+  ): AsyncIterable<Uint8Array> {
+    const reader = body.getReader();
+    try {
+      while (true) {
+        const {value, done} = await reader.read();
+        if (done) return;
+        if (value) yield value;
+      }
+    } finally {
+      try { reader.releaseLock(); } catch { /* noop */ }
+    }
+  }

package/src/runtime/ndjson.ts

@@ -0,0 +1,32 @@
+/**
+ * NDJSON adapter for record iterables.
+ *
+ * Phase 3 of ADR-002. Given an `AsyncIterable<T>` (typically the output of
+ * `parseRecords`), produce a Node `Readable` that emits one JSON-encoded line
+ * per record and respects downstream backpressure.
+ *
+ * Terminal-error policy (Q3 resolved): the stream aborts on source errors.
+ * Before-first-byte errors surface as the stream's `error` event before any
+ * bytes are pushed, so Fastify can translate them into a normal JSON error
+ * envelope. Errors after the first byte propagate as `error` events too, but
+ * callers should treat them as a truncated response.
+ */
+import {Readable} from "node:stream";
+
+/**
+ * Wrap an async iterable of records in a Node `Readable` stream that emits
+ * NDJSON (one JSON document per line, LF-terminated). Downstream backpressure
+ * is honored via `Readable.from`'s default behavior: the iterator's `next()`
+ * is not called until the internal buffer has room.
+ *
+ * Source errors are forwarded to the returned stream's `error` event.
+ */
+export function toNdjson<T>(records: AsyncIterable<T>): Readable {
+  return Readable.from(encode(records), {objectMode: false, encoding: "utf-8"});
+}
+
+async function* encode<T>(records: AsyncIterable<T>): AsyncIterable<string> {
+  for await (const record of records) {
+    yield JSON.stringify(record) + "\n";
+  }
+}

package/src/runtime/operationsStreamHelper.tpl.txt

@@ -0,0 +1,13 @@
+/**
+ * Response shape for streaming __CLIENT_NAME__ operations. `records` is a
+ * single-pass async iterable of parsed record objects; iteration pulls bytes
+ * from the upstream SOAP response on demand. `headers` is the HTTP response
+ * header map captured before the first record is parsed. `requestRaw`, when
+ * populated, contains the serialized SOAP envelope that was sent upstream.
+ */
+export type StreamOperationResponse<RecordType, HeadersType = Record<string, unknown>> = {
+  records: AsyncIterable<RecordType>;
+  headers: HeadersType;
+  requestRaw?: string;
+};
+

package/src/runtime/streamXml.ts

@@ -0,0 +1,293 @@
+/**
+ * Streaming SOAP-payload to record iterator.
+ *
+ * Phase 3 of ADR-002. Driven by saxes (chunk-boundary safe — proven in
+ * test/research/sax-record-path.test.ts) and the compiled-catalog metadata
+ * that the buffered client already relies on. The parser accepts an async
+ * iterable of bytes/strings (typically an upstream SOAP HTTP response) and
+ * yields fully-materialized record objects as the corresponding end tags
+ * close. Consumers never see partial records.
+ *
+ * Open questions resolved here:
+ *   Q3 (terminal error policy): stream aborts. Errors that happen before the
+ *        first record bubble out as a rejected promise from the first
+ *        iterator.next(). Errors after a record was emitted throw from a
+ *        later iterator.next() — callers are expected to treat that as a
+ *        truncated stream.
+ *   Q4 (saxes placement): runtime dependency of the wsdl-tsc package; the
+ *        generated client imports the emitted copy of this module and
+ *        inherits saxes via its own dependency tree.
+ */
+import {SaxesParser, type SaxesTagPlain} from "saxes";
+
+/**
+ * Catalog-driven parse specification. `recordPath` is an ordered XML element
+ * path from the SOAP body payload down to the repeated record element. The
+ * path is matched as a suffix of the open-tag stack, so callers may either
+ * pre-strip the SOAP envelope or feed the entire response body.
+ *
+ * Duplicate local names in `recordPath` are supported and expected (Escapia's
+ * EVRN content service nests two elements named `EVRN_UnitDescriptiveInfoRS`).
+ */
+export interface RecordParseSpec {
+  recordPath: string[];
+  /** TypeScript type name of the record, used to look up propMeta. */
+  recordTypeName: string;
+  /** Attribute bag key to stash XML attributes under. Defaults to "$attributes". */
+  attributesKey?: string;
+  /**
+   * Compiled-catalog child-type map: `childType[typeName][propName] = tsType`.
+   * Used to descend into nested complex types and to detect array-valued
+   * props (trailing `[]`). Optional — absent means occurrence-based array
+   * detection only.
+   */
+  childType?: Record<string, Record<string, string>>;
+  /**
+   * Compiled-catalog prop-meta map: carries min/max/nillable/declaredType.
+   * When present, `max > 1` or `max === "unbounded"` drives array emission
+   * even for props that happen to occur just once in a given record.
+   */
+  propMeta?: Record<string, Record<string, PropMeta>>;
+}
+
+export interface PropMeta {
+  min?: number;
+  max?: number | "unbounded";
+  nillable?: boolean;
+  declaredType?: string;
+}
+
+/**
+ * Consume an async iterable of XML bytes/strings and yield parsed records.
+ *
+ * The returned iterator is single-pass; iteration must complete (or be
+ * interrupted by the consumer via `break`/`return`) before the upstream
+ * source is released. Errors from the source or from the SAX parser abort
+ * the iteration via a rejected `next()`.
+ */
+export async function* parseRecords<T = unknown>(
+  source: AsyncIterable<string | Uint8Array>,
+  spec: RecordParseSpec,
+): AsyncIterable<T> {
+  const parser = new SaxesParser({xmlns: false, position: false});
+  const recordPath = spec.recordPath;
+  if (recordPath.length === 0) {
+    throw new Error("parseRecords: recordPath must not be empty");
+  }
+  const attrsKey = spec.attributesKey ?? "$attributes";
+
+  // Global tag stack, maintained across the entire document.
+  const stack: string[] = [];
+  // Records materialized during the current chunk write, awaiting yield.
+  const pending: T[] = [];
+  // When a parser.on('error') fires we buffer the error for re-throw on the
+  // next yield cycle; saxes emits 'error' and continues unless we stop it.
+  let parseError: Error | null = null;
+
+  // While we're inside the record element (stack tail matches recordPath),
+  // we maintain a stack of "open" element nodes capturing their XML shape.
+  interface OpenNode {
+    obj: Record<string, unknown>;
+    /** Type name for `childType`/`propMeta` lookup on direct children. */
+    typeName: string | null;
+    /** Name this node was opened as, in the parent object. null for the record root. */
+    propName: string | null;
+    /** Accumulated text content (including CDATA). */
+    textBuf: string[];
+    /** True once we've seen at least one child element — distinguishes leaves from containers. */
+    hadChildren: boolean;
+    /** xsi:nil="true" marker → materialize as null regardless of content. */
+    isNil: boolean;
+  }
+  const openNodes: OpenNode[] = [];
+
+  parser.on("opentag", (tag: SaxesTagPlain) => {
+    stack.push(tag.name);
+    if (openNodes.length === 0) {
+      // Not yet inside a record. Check whether entering the path tail.
+      if (tailMatches(stack, recordPath)) {
+        openNodes.push(newNode(tag, spec.recordTypeName, null, attrsKey));
+      }
+      return;
+    }
+    // Inside a record: descend.
+    const parent = openNodes[openNodes.length - 1];
+    parent.hadChildren = true;
+    const parentType = parent.typeName;
+    const childTypeRaw = parentType ? spec.childType?.[parentType]?.[tag.name] : undefined;
+    const childTypeName = childTypeRaw ? stripArraySuffix(childTypeRaw) : null;
+    openNodes.push(newNode(tag, childTypeName, tag.name, attrsKey));
+  });
+
+  const appendText = (t: string) => {
+    if (openNodes.length > 0) openNodes[openNodes.length - 1].textBuf.push(t);
+  };
+  parser.on("text", appendText);
+  parser.on("cdata", appendText);
+
+  parser.on("closetag", (_tag: SaxesTagPlain) => {
+    if (openNodes.length > 0) {
+      const closing = openNodes.pop()!;
+      const value = materialize(closing, attrsKey);
+      if (openNodes.length === 0) {
+        // We just closed the record root. Stack tail must match once more.
+        if (tailMatches(stack, recordPath)) {
+          pending.push(value as T);
+        }
+      } else {
+        assignChild(openNodes[openNodes.length - 1], closing.propName!, value, spec);
+      }
+    }
+    stack.pop();
+  });
+
+  parser.on("error", (err: Error) => {
+    parseError = err;
+  });
+
+  try {
+    for await (const chunk of source) {
+      if (parseError) throw parseError;
+      const text = typeof chunk === "string" ? chunk : decodeUtf8(chunk);
+      parser.write(text);
+      if (parseError) throw parseError;
+      while (pending.length > 0) {
+        yield pending.shift()!;
+      }
+    }
+    parser.close();
+    if (parseError) throw parseError;
+    while (pending.length > 0) {
+      yield pending.shift()!;
+    }
+  } finally {
+    // Best-effort cleanup: detach handlers so the parser can be GC'd even
+    // when the consumer aborts iteration early.
+    parser.off("opentag");
+    parser.off("closetag");
+    parser.off("text");
+    parser.off("cdata");
+    parser.off("error");
+  }
+}
+
+function newNode(
+  tag: SaxesTagPlain,
+  typeName: string | null,
+  propName: string | null,
+  attrsKey: string,
+): {
+  obj: Record<string, unknown>;
+  typeName: string | null;
+  propName: string | null;
+  textBuf: string[];
+  hadChildren: boolean;
+  isNil: boolean;
+} {
+  const attrs = tag.attributes;
+  const obj: Record<string, unknown> = {};
+  let isNil = false;
+  const attrKeys = Object.keys(attrs);
+  if (attrKeys.length > 0) {
+    // Detect xsi:nil and drop it from the attribute bag — it's a wire-level
+    // concern that should not pollute the user-visible record.
+    const cleaned: Record<string, string> = {};
+    for (const k of attrKeys) {
+      if ((k === "xsi:nil" || k === "nil") && attrs[k] === "true") {
+        isNil = true;
+        continue;
+      }
+      cleaned[k] = attrs[k];
+    }
+    if (Object.keys(cleaned).length > 0) {
+      obj[attrsKey] = cleaned;
+    }
+  }
+  return {
+    obj,
+    typeName,
+    propName,
+    textBuf: [],
+    hadChildren: false,
+    isNil,
+  };
+}
+
+function materialize(
+  node: {obj: Record<string, unknown>; textBuf: string[]; hadChildren: boolean; isNil: boolean},
+  attrsKey: string,
+): unknown {
+  if (node.isNil) return null;
+  if (!node.hadChildren) {
+    // Leaf with no child elements: it's simple text. Preserve attributes via
+    // a `$value` pairing when present, mirroring how the buffered mapper
+    // surfaces simpleContent-with-attributes types.
+    const text = node.textBuf.join("");
+    if (attrsKey in node.obj) {
+      return {...node.obj, $value: text};
+    }
+    return text;
+  }
+  return node.obj;
+}
+
+function assignChild(
+  parent: {obj: Record<string, unknown>; typeName: string | null},
+  propName: string,
+  value: unknown,
+  spec: RecordParseSpec,
+): void {
+  const parentType = parent.typeName;
+  const propMetaEntry = parentType ? spec.propMeta?.[parentType]?.[propName] : undefined;
+  const childTypeHint = parentType ? spec.childType?.[parentType]?.[propName] : undefined;
+
+  // Array if: (a) propMeta says max > 1 or "unbounded", or (b) childType hint
+  // ends in `[]`, or (c) the slot is already occupied (implicit repetition).
+  const metaSaysArray =
+    propMetaEntry?.max === "unbounded" ||
+    (typeof propMetaEntry?.max === "number" && propMetaEntry.max > 1);
+  const hintSaysArray = !!childTypeHint && childTypeHint.endsWith("[]");
+  const existing = parent.obj[propName];
+  const slotTaken = existing !== undefined;
+
+  if (metaSaysArray || hintSaysArray) {
+    if (Array.isArray(existing)) {
+      existing.push(value);
+    } else if (slotTaken) {
+      parent.obj[propName] = [existing, value];
+    } else {
+      parent.obj[propName] = [value];
+    }
+    return;
+  }
+  if (slotTaken) {
+    // Schema said scalar but the wire repeated it. Promote to array rather
+    // than drop data.
+    if (Array.isArray(existing)) {
+      existing.push(value);
+    } else {
+      parent.obj[propName] = [existing, value];
+    }
+    return;
+  }
+  parent.obj[propName] = value;
+}
+
+function stripArraySuffix(tsType: string): string {
+  return tsType.endsWith("[]") ? tsType.slice(0, -2) : tsType;
+}
+
+function tailMatches(stack: string[], path: string[]): boolean {
+  if (stack.length < path.length) return false;
+  const offset = stack.length - path.length;
+  for (let i = 0; i < path.length; i++) {
+    if (stack[offset + i] !== path[i]) return false;
+  }
+  return true;
+}
+
+const UTF8_DECODER = new TextDecoder("utf-8");
+
+function decodeUtf8(buf: Uint8Array): string {
+  return UTF8_DECODER.decode(buf);
+}