@llm4ts/shell 2.11.0 → 2.13.0

package/kits/soap-ace/flows/lib/soap/Scenarios.js

@@ -0,0 +1,132 @@
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import { elementByName, operationByName } from "./Catalog.js";
+import { readIfPresent } from "./Discover.js";
+import { skeletonYaml, validateInstance } from "./Instance.js";
+import { renderRequestFile, samplePaths, toSampleName } from "./Samples.js";
+// Scenario sets: for one operation the reasoning seat proposes a handful of
+// request bodies that together exercise it — the happy path, an empty
+// result, a pagination boundary, a business fault, a schema edge. The model
+// only drafts; each body is validated against the XSD and written as an
+// ordinary request file (with its problems listed at the top), so the user
+// reviews and edits before anything is called. Existing files are kept.
+export class ProposedScenario extends Schema.Class("ProposedScenario")({
+    name: Schema.String,
+    purpose: Schema.String,
+    body: Schema.Json
+}) {
+}
+export class ScenarioSet extends Schema.Class("ScenarioSet")({
+    scenarios: Schema.Array(ProposedScenario)
+}) {
+}
+const scenarioJsonSchema = {
+    type: "object",
+    required: ["scenarios"],
+    properties: {
+        scenarios: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["name", "purpose", "body"],
+                properties: {
+                    name: { type: "string", description: "kebab-case, e.g. happy-path" },
+                    purpose: { type: "string", description: "what the scenario exercises, one sentence" },
+                    body: { type: "object", description: "the request body, shaped like the skeleton" }
+                }
+            }
+        }
+    }
+};
+/** JSON from a model → request values: every leaf a string, as XML text is. */
+export const toYamlValue = (value) => {
+    if (value === null)
+        return null;
+    if (typeof value === "string")
+        return value;
+    if (typeof value === "number" || typeof value === "boolean")
+        return String(value);
+    if (Array.isArray(value))
+        return value.map(toYamlValue);
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, toYamlValue(item)]));
+};
+export const scenarioPrompt = (options) => {
+    const operation = operationByName(options.catalog, options.operation);
+    const input = operation === undefined ? undefined : elementByName(options.catalog, operation.input);
+    if (operation === undefined || input === undefined)
+        return undefined;
+    const output = operation.output === undefined ? undefined : elementByName(options.catalog, operation.output);
+    const seeds = [...options.seeds].map(([field, value]) => `- ${field}: ${value}`);
+    return [
+        `You are preparing sample requests to explore the SOAP operation ${operation.name} of a bank's backend,`,
+        "before an equivalent REST API is designed. Propose 3 to 6 scenarios that together show how it really",
+        "behaves. Cover, where the operation allows it: the happy path; an empty or not-found result; a",
+        "pagination boundary; one business error (an esito KO code or a SOAP fault); one schema edge (maximum",
+        "lengths, optional fields left out, optional fields all present).",
+        "",
+        `Operation: ${operation.name} (${options.operationClass})`,
+        ...(operation.documentation === undefined ? [] : [`Documentation: ${operation.documentation}`]),
+        `Faults: ${operation.faults.map((fault) => fault.name).join(", ") || "none declared"}`,
+        "",
+        "The request body, as YAML with type notes (required fields filled, optional ones commented):",
+        "```yaml",
+        ...skeletonYaml(options.catalog, input, 0),
+        "```",
+        ...(output === undefined
+            ? []
+            : [
+                "",
+                "The response body shape:",
+                "```yaml",
+                ...skeletonYaml(options.catalog, output, 0, { maxDepth: 3 }),
+                "```"
+            ]),
+        "",
+        "Rules:",
+        "- Use only synthetic data. Never invent a real person's name, tax code, or account.",
+        "- Reuse these identifiers seen in earlier samples wherever an identifier is needed; they are masked",
+        "  test values the service is likely to know:",
+        ...(seeds.length === 0
+            ? ["  (none yet: use the examples in the skeleton)"]
+            : seeds.map((seed) => `  ${seed}`)),
+        "- Every value is a string. Respect the type notes (patterns, lengths, enumerations, digits).",
+        "- Omit optional fields you do not need; never add fields that are not in the skeleton.",
+        "- Names are kebab-case and unique; the purpose says what the scenario should reveal.",
+        "",
+        'Reply with JSON: {"scenarios": [{"name": "...", "purpose": "...", "body": {...}}]}.'
+    ].join("\n");
+};
+export const proposeScenarios = (options) => Effect.gen(function* () {
+    const prompt = scenarioPrompt(options);
+    const operation = operationByName(options.catalog, options.operation);
+    const input = operation === undefined ? undefined : elementByName(options.catalog, operation.input);
+    if (prompt === undefined || input === undefined)
+        return [];
+    const proposal = yield* options.reasoning.executeStructured(prompt, ScenarioSet, scenarioJsonSchema);
+    const written = [];
+    const names = new Set();
+    for (const scenario of proposal.scenarios) {
+        let name = toSampleName(scenario.name);
+        for (let suffix = 2; names.has(name); suffix++)
+            name = `${toSampleName(scenario.name).slice(0, 58)}-${suffix}`;
+        names.add(name);
+        const paths = samplePaths(options.service, options.operation, name);
+        const exists = (yield* readIfPresent(options.workspace, paths.yaml)) !== undefined ||
+            (yield* readIfPresent(options.workspace, paths.xml)) !== undefined;
+        const body = toYamlValue(scenario.body);
+        const issues = validateInstance(options.catalog, input, body);
+        if (exists) {
+            written.push({ name, path: paths.yaml, issues, kept: true });
+            continue;
+        }
+        yield* options.workspace.write(paths.yaml, renderRequestFile({
+            catalog: options.catalog,
+            operation: options.operation,
+            purpose: scenario.purpose,
+            body,
+            issues
+        }));
+        written.push({ name, path: paths.yaml, issues, kept: false });
+    }
+    return written;
+});

package/kits/soap-ace/flows/lib/soap/SoapUiImport.js

@@ -0,0 +1,200 @@
+import * as Effect from "effect/Effect";
+import { elementByName, operationByName } from "./Catalog.js";
+import { readIfPresent } from "./Discover.js";
+import { readEnvelope, stripSecurity } from "./Envelope.js";
+import { instanceFromXml } from "./Instance.js";
+import { kindsFromCatalog, maskDocument, maskText, MaskingReport, mergeReports } from "./Masking.js";
+import { Exchange, issueRecords, RecordedResponse, renderRequestFile, samplePaths, toSampleName, writeExchange } from "./Samples.js";
+import { attribute, elements, parseXml, renderXml, textOf } from "./Xml.js";
+// SoapUI / ReadyAPI projects are where bank teams keep their working
+// requests. Import turns each saved request (interface calls and test
+// steps) into a request file and each mock response into a recorded
+// exchange. Everything passes through the masker first: a project copied
+// from a shared drive often holds real customer data. Stored credentials
+// (`con:credentials`, WS-S passwords) are counted and never read.
+export const SoapUiNamespace = "http://eviware.com/soapui/config";
+const con = (element, local) => elements(element).filter((child) => child.name.namespace === SoapUiNamespace && child.name.local === local);
+const conText = (element, local) => {
+    const found = con(element, local)[0];
+    return found === undefined ? undefined : textOf(found);
+};
+const hasCredentials = (element) => {
+    const credentials = con(element, "credentials")[0];
+    if (credentials === undefined)
+        return false;
+    return elements(credentials).some((child) => textOf(child).trim() !== "" && /password|username|token/i.test(child.name.local));
+};
+/** Every saved request and mock response in a project, with its operation. */
+export const findMessages = (project) => {
+    const messages = [];
+    let credentials = 0;
+    for (const iface of con(project, "interface")) {
+        for (const operation of con(iface, "operation")) {
+            const name = attribute(operation, "name") ?? "";
+            for (const call of con(operation, "call")) {
+                if (hasCredentials(call))
+                    credentials++;
+                const envelope = conText(call, "request");
+                if (envelope !== undefined && envelope.trim() !== "") {
+                    messages.push({
+                        operation: name,
+                        label: attribute(call, "name") ?? "call",
+                        envelope,
+                        kind: "request"
+                    });
+                }
+            }
+        }
+    }
+    const walkSuites = (element, path) => {
+        for (const child of elements(element)) {
+            if (child.name.namespace !== SoapUiNamespace)
+                continue;
+            if (child.name.local === "testSuite" || child.name.local === "testCase") {
+                walkSuites(child, [...path, attribute(child, "name") ?? child.name.local]);
+            }
+            else if (child.name.local === "testStep" && attribute(child, "type") === "request") {
+                const config = con(child, "config")[0];
+                const operation = config === undefined ? undefined : conText(config, "operation");
+                const request = config === undefined ? undefined : con(config, "request")[0];
+                if (request !== undefined && hasCredentials(request))
+                    credentials++;
+                const envelope = request === undefined ? undefined : conText(request, "request");
+                if (operation !== undefined && envelope !== undefined && envelope.trim() !== "") {
+                    messages.push({
+                        operation: operation.trim(),
+                        label: [...path, attribute(child, "name") ?? "step"].join(" "),
+                        envelope,
+                        kind: "request"
+                    });
+                }
+            }
+        }
+    };
+    walkSuites(project, []);
+    for (const mock of con(project, "mockService")) {
+        for (const operation of con(mock, "mockOperation")) {
+            const name = attribute(operation, "operation") ?? attribute(operation, "name") ?? "";
+            for (const response of con(operation, "response")) {
+                const envelope = conText(response, "responseContent");
+                if (envelope !== undefined && envelope.trim() !== "") {
+                    messages.push({
+                        operation: name,
+                        label: `mock ${attribute(response, "name") ?? "response"}`,
+                        envelope,
+                        kind: "response"
+                    });
+                }
+            }
+        }
+    }
+    return { messages, credentials };
+};
+export const importSoapUiProject = (options) => Effect.gen(function* () {
+    const { catalog, workspace, service } = options;
+    const project = yield* parseXml(options.project, { source: "SoapUI project" });
+    const { messages, credentials } = findMessages(project);
+    const maskOptions = {
+        key: options.key,
+        typeKinds: kindsFromCatalog(catalog),
+        ...(options.overrides === undefined ? {} : { overrides: options.overrides })
+    };
+    const requests = [];
+    const responses = [];
+    const skipped = [];
+    const reports = [];
+    const used = new Set();
+    for (const found of messages) {
+        // SoapUI labels are free text people type ("Conti di Mario Rossi …"):
+        // they reach file names and headers, so they are masked like values.
+        const message = { ...found, label: maskText(options.key, found.label).text };
+        const operation = operationByName(catalog, message.operation);
+        if (operation === undefined) {
+            skipped.push(`${message.label}: operation ${message.operation} is not in the catalog`);
+            continue;
+        }
+        // SoapUI keeps property expansions (${#Project#iban}) inside the XML
+        // text; they parse fine and are reported by validation.
+        const parsed = yield* parseXml(message.envelope, { source: message.label }).pipe(Effect.option);
+        if (parsed._tag === "None") {
+            skipped.push(`${message.operation} ${message.label}: not well-formed XML`);
+            continue;
+        }
+        const masked = maskDocument(stripSecurity(parsed.value), maskOptions);
+        reports.push(masked.report);
+        const envelope = yield* readEnvelope(masked.document).pipe(Effect.option);
+        const payload = envelope._tag === "Some" ? envelope.value.payload : undefined;
+        let base = toSampleName(`soapui-${message.label}`);
+        for (let suffix = 2; used.has(`${message.operation}/${base}`); suffix++) {
+            base = `${toSampleName(`soapui-${message.label}`).slice(0, 58)}-${suffix}`;
+        }
+        used.add(`${message.operation}/${base}`);
+        const paths = samplePaths(service, message.operation, base);
+        if (message.kind === "request") {
+            const input = elementByName(catalog, operation.input);
+            if (input === undefined || payload === undefined) {
+                skipped.push(`${message.operation} ${message.label}: no request body`);
+                continue;
+            }
+            if ((yield* readIfPresent(workspace, paths.yaml)) !== undefined ||
+                (yield* readIfPresent(workspace, paths.xml)) !== undefined) {
+                skipped.push(`${message.operation}/${base}: a request file already exists; kept it`);
+                continue;
+            }
+            const read = instanceFromXml(catalog, input, payload);
+            const placeholders = read.issues.length > 0 && JSON.stringify(read.value).includes('"?"');
+            const issues = placeholders
+                ? [
+                    { path: "(root)", detail: "SoapUI placeholders (?) are still in this request" },
+                    ...read.issues
+                ]
+                : read.issues;
+            yield* workspace.write(paths.yaml, renderRequestFile({
+                catalog,
+                operation: message.operation,
+                purpose: `imported from SoapUI: ${message.label}`,
+                body: read.value,
+                issues
+            }));
+            requests.push({ operation: message.operation, name: base, path: paths.yaml, issues });
+        }
+        else {
+            const output = operation.output === undefined ? undefined : elementByName(catalog, operation.output);
+            const responseIssues = envelope._tag === "Some" &&
+                envelope.value.fault === undefined &&
+                output !== undefined &&
+                payload !== undefined
+                ? instanceFromXml(catalog, output, payload).issues
+                : [];
+            const exchange = new Exchange({
+                version: 1,
+                operation: message.operation,
+                name: base,
+                purpose: `SoapUI ${message.label}`,
+                provenance: "soapui",
+                recordedAt: (options.now ?? (() => new Date()))().toISOString(),
+                response: new RecordedResponse({
+                    status: 200,
+                    headers: {},
+                    envelope: renderXml(masked.document),
+                    elapsedMs: 0
+                }),
+                ...(envelope._tag === "Some" && envelope.value.fault !== undefined
+                    ? { fault: envelope.value.fault }
+                    : {}),
+                requestIssues: [],
+                responseIssues: issueRecords(responseIssues),
+                masking: masked.report
+            });
+            const path = yield* writeExchange(workspace, service, exchange);
+            responses.push({ operation: message.operation, name: base, path });
+        }
+    }
+    return {
+        requests,
+        responses,
+        skipped,
+        credentialsIgnored: credentials,
+        masking: reports.length === 0 ? new MaskingReport({ entries: [], kept: [] }) : mergeReports(reports)
+    };
+});

package/kits/soap-ace/flows/lib/soap/Stub.js

@@ -0,0 +1,34 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import * as Effect from "effect/Effect";
+import { Soap11Namespace } from "./Envelope.js";
+// A backend stand-in: answers each call with `<directory>/<operation>.xml`,
+// the operation found from the SOAPAction (SOAP 1.1 header or the SOAP 1.2
+// content-type `action`). For rehearsals without a reachable service
+// (LLM4TS_SOAP_STUB) and for the ACE tests' recorded backend.
+export const soapActionOf = (request) => {
+    const header = request.headers["soapaction"];
+    if (header !== undefined)
+        return header.replace(/^"|"$/g, "");
+    return /action="([^"]*)"/.exec(request.headers["content-type"] ?? "")?.[1] ?? "";
+};
+const stubFault = (reason) => ({
+    status: 500,
+    headers: { "content-type": "text/xml; charset=utf-8" },
+    body: new TextEncoder().encode(`<s:Envelope xmlns:s="${Soap11Namespace}"><s:Body><s:Fault><faultcode>s:Server</faultcode><faultstring>${reason}</faultstring></s:Fault></s:Body></s:Envelope>`),
+    elapsedMs: 1
+});
+export const makeDirectoryStubTransport = (directory, catalog) => ({
+    send: (request) => {
+        const action = soapActionOf(request);
+        const operation = catalog.operations.find((candidate) => candidate.soapAction === action);
+        if (operation === undefined)
+            return Effect.succeed(stubFault(`stub: no operation for action ${action}`));
+        return Effect.tryPromise(() => readFile(join(directory, `${operation.name}.xml`))).pipe(Effect.map((bytes) => ({
+            status: 200,
+            headers: { "content-type": "text/xml; charset=utf-8" },
+            body: new Uint8Array(bytes),
+            elapsedMs: 1
+        })), Effect.orElseSucceed(() => stubFault(`stub: no ${operation.name}.xml`)));
+    }
+});

package/kits/soap-ace/flows/lib/soap/Transport.js

@@ -0,0 +1,167 @@
+import * as http from "node:http";
+import * as https from "node:https";
+import * as Context from "effect/Context";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Redacted from "effect/Redacted";
+import * as Ref from "effect/Ref";
+import * as Schema from "effect/Schema";
+import { DocumentLoadError, makeFileDocumentLoader } from "./Wsdl.js";
+export class TransportError extends Schema.TaggedError()("TransportError", {
+    url: Schema.String,
+    reason: Schema.Literals(["scheme", "network", "tls", "timeout", "aborted"]),
+    detail: Schema.String
+}) {
+    get message() {
+        return `${this.url}: ${this.reason}: ${this.detail}`;
+    }
+}
+export class SoapTransport extends Context.Service()("@llm4ts/kits/soap-ace/SoapTransport") {
+}
+/** Strip query string and credentials from a URL before it lands in an error or a file. */
+export const displayUrl = (raw) => {
+    try {
+        const url = new URL(raw);
+        return `${url.protocol}//${url.host}${url.pathname}`;
+    }
+    catch {
+        return "(invalid url)";
+    }
+};
+const errorCode = (cause) => typeof cause === "object" && cause !== null && "code" in cause && typeof cause.code === "string"
+    ? cause.code
+    : "request failed";
+const tlsCodes = /CERT|SSL|TLS|SELF_SIGNED|UNABLE_TO|DEPTH_ZERO|ERR_OSSL/;
+const flatHeaders = (headers) => Object.fromEntries(Object.entries(headers).flatMap(([name, value]) => value === undefined ? [] : [[name, Array.isArray(value) ? value.join(", ") : value]]));
+const bufferOf = (secret) => secret === undefined ? undefined : Buffer.from(Redacted.value(secret));
+export const makeNodeSoapTransport = () => ({
+    send: (request) => {
+        const shown = displayUrl(request.url);
+        let url;
+        try {
+            url = new URL(request.url);
+        }
+        catch {
+            return Effect.fail(new TransportError({ url: shown, reason: "scheme", detail: "invalid URL" }));
+        }
+        if (url.protocol !== "https:" && url.protocol !== "http:") {
+            return Effect.fail(new TransportError({
+                url: shown,
+                reason: "scheme",
+                detail: `${url.protocol} is not http(s)`
+            }));
+        }
+        if (url.protocol === "http:" && request.tls !== undefined) {
+            return Effect.fail(new TransportError({
+                url: shown,
+                reason: "scheme",
+                detail: "client TLS needs an https URL"
+            }));
+        }
+        const headers = { ...request.headers };
+        for (const [name, value] of request.secretHeaders)
+            headers[name] = Redacted.value(value);
+        const body = request.body === undefined ? undefined : Buffer.from(Redacted.value(request.body), "utf8");
+        if (body !== undefined)
+            headers["content-length"] = String(body.length);
+        return Effect.tryPromise({
+            try: (signal) => new Promise((resolve, reject) => {
+                const started = performance.now();
+                const tls = request.tls;
+                const options = {
+                    method: request.method,
+                    headers,
+                    signal,
+                    ...(tls === undefined
+                        ? {}
+                        : {
+                            cert: bufferOf(tls.cert),
+                            key: bufferOf(tls.key),
+                            pfx: bufferOf(tls.pfx),
+                            ca: bufferOf(tls.ca),
+                            passphrase: tls.passphrase === undefined ? undefined : Redacted.value(tls.passphrase)
+                        })
+                };
+                const handler = (response) => {
+                    const chunks = [];
+                    response.on("data", (chunk) => chunks.push(chunk));
+                    response.on("error", reject);
+                    response.on("end", () => resolve({
+                        status: response.statusCode ?? 0,
+                        headers: flatHeaders(response.headers),
+                        body: new Uint8Array(Buffer.concat(chunks)),
+                        elapsedMs: Math.round(performance.now() - started)
+                    }));
+                };
+                const outgoing = url.protocol === "https:"
+                    ? https.request(url, options, handler)
+                    : http.request(url, options, handler);
+                outgoing.on("error", reject);
+                if (body !== undefined)
+                    outgoing.write(body);
+                outgoing.end();
+            }),
+            catch: (cause) => {
+                const code = errorCode(cause);
+                return new TransportError({
+                    url: shown,
+                    reason: code === "ABORT_ERR" ? "aborted" : tlsCodes.test(code) ? "tls" : "network",
+                    detail: code
+                });
+            }
+        }).pipe(Effect.timeoutOrElse({
+            duration: request.timeout,
+            orElse: () => Effect.fail(new TransportError({
+                url: shown,
+                reason: "timeout",
+                detail: `no response within ${Duration.format(request.timeout)}`
+            }))
+        }));
+    }
+});
+/**
+ * A scripted transport for tests: each request goes to `respond`, and every
+ * request is recorded (secrets still redacted) for assertions.
+ */
+export const makeFakeSoapTransport = (respond) => Effect.map(Ref.make([]), (log) => ({
+    transport: {
+        send: (request) => Effect.andThen(Ref.update(log, (all) => [...all, request]), respond(request))
+    },
+    requests: Ref.get(log)
+}));
+export const xmlResponse = (xml, status = 200) => ({
+    status,
+    headers: { "content-type": "text/xml; charset=utf-8" },
+    body: new TextEncoder().encode(xml),
+    elapsedMs: 5
+});
+const isUrl = (location) => /^https?:\/\//i.test(location);
+/**
+ * The document loader for discovery: local paths from disk, http(s) URLs
+ * through the transport with the profile's fetch-side auth and TLS.
+ */
+export const makeTransportDocumentLoader = (transport, side, timeout = Duration.seconds(30)) => {
+    const files = makeFileDocumentLoader();
+    return {
+        load: (location) => !isUrl(location)
+            ? files.load(location)
+            : transport
+                .send({
+                method: "GET",
+                url: location,
+                headers: { accept: "text/xml, application/xml, */*" },
+                secretHeaders: side.headers,
+                ...(side.tls === undefined ? {} : { tls: side.tls }),
+                timeout
+            })
+                .pipe(Effect.mapError((error) => new DocumentLoadError({
+                location: displayUrl(location),
+                detail: `${error.reason}: ${error.detail}`
+            })), Effect.flatMap((response) => response.status >= 200 && response.status < 300
+                ? Effect.succeed(response.body)
+                : Effect.fail(new DocumentLoadError({
+                    location: displayUrl(location),
+                    detail: `HTTP ${response.status}`
+                }))))
+    };
+};