@llm4ts/shell 2.26.0 → 2.28.0

package/kits/soap-ace/flows/lib/soap/Transport.js

@@ -1,15 +1,18 @@
 import * as http from "node:http";
 import * as https from "node:https";
+import * as net from "node:net";
+import * as tls from "node:tls";
 import * as Context from "effect/Context";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
 import * as Redacted from "effect/Redacted";
 import * as Ref from "effect/Ref";
 import * as Schema from "effect/Schema";
+import { bypassesProxy } from "./Auth.js";
 import { DocumentLoadError, makeFileDocumentLoader } from "./Wsdl.js";
 export class TransportError extends Schema.TaggedError()("TransportError", {
     url: Schema.String,
-    reason: Schema.Literals(["scheme", "network", "tls", "timeout", "aborted"]),
+    reason: Schema.Literals(["scheme", "network", "tls", "timeout", "aborted", "proxy"]),
     detail: Schema.String
 }) {
     get message() {
@@ -32,32 +35,166 @@ const errorCode = (cause) => typeof cause === "object" && cause !== null && "cod
     ? cause.code
     : "request failed";
 const tlsCodes = /CERT|SSL|TLS|SELF_SIGNED|UNABLE_TO|DEPTH_ZERO|ERR_OSSL/;
+const unknownIssuer = /SELF_SIGNED|UNABLE_TO_VERIFY_LEAF_SIGNATURE|UNABLE_TO_GET_ISSUER|DEPTH_ZERO|CERT_UNTRUSTED|UNABLE_TO_GET_LOCAL_ISSUER/;
+export const trustHint = 'the chain is not trusted; capture it with soap-sample "trust <side>"';
+const describeTlsFailure = (code) => unknownIssuer.test(code) ? `${code} (${trustHint})` : code;
+class ProxyRefused extends Error {
+    status;
+    constructor(status) {
+        super(`proxy refused the tunnel: HTTP ${status}`);
+        this.status = status;
+    }
+}
 const flatHeaders = (headers) => Object.fromEntries(Object.entries(headers).flatMap(([name, value]) => value === undefined ? [] : [[name, Array.isArray(value) ? value.join(", ") : value]]));
 const bufferOf = (secret) => secret === undefined ? undefined : Buffer.from(Redacted.value(secret));
+/** The CA list a connection verifies against: system roots plus the profile's, or the profile's alone. */
+export const caListFor = (resolved) => {
+    if (resolved?.ca === undefined) {
+        return undefined;
+    }
+    const own = new TextDecoder().decode(Redacted.value(resolved.ca));
+    return resolved.trust === "ca-only" ? [own] : [...tls.rootCertificates, own];
+};
+const tlsOptionsOf = (resolved, host) => {
+    const ca = caListFor(resolved);
+    return {
+        ...(resolved === undefined
+            ? {}
+            : {
+                cert: bufferOf(resolved.cert),
+                key: bufferOf(resolved.key),
+                pfx: bufferOf(resolved.pfx),
+                passphrase: resolved.passphrase === undefined ? undefined : Redacted.value(resolved.passphrase)
+            }),
+        // A pinned chain may hold only what the server sent (a leaf signed by a
+        // CA it never presents): trust it as captured, not only self-signed roots.
+        ...(ca === undefined ? {} : { ca: [...ca], allowPartialTrustChain: true }),
+        servername: resolved?.servername ?? (net.isIP(host) === 0 ? host : undefined)
+    };
+};
+const portOf = (url) => url.port !== "" ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
+/** Whether this request goes through the proxy: configured and the host not excluded. */
+export const proxyFor = (url, proxy) => proxy === undefined || bypassesProxy(url.hostname, proxy.noProxy) ? undefined : proxy;
+/** An HTTP CONNECT tunnel to `target` through `proxy`; the raw socket, not yet TLS. */
+const connectTunnel = (proxy, target, signal) => new Promise((resolve, reject) => {
+    const proxyUrl = new URL(proxy.url);
+    const headers = { host: `${target.hostname}:${portOf(target)}` };
+    if (proxy.authorization !== undefined) {
+        headers["proxy-authorization"] = Redacted.value(proxy.authorization);
+    }
+    const request = http.request({
+        host: proxyUrl.hostname,
+        port: portOf(proxyUrl),
+        method: "CONNECT",
+        path: `${target.hostname}:${portOf(target)}`,
+        headers,
+        signal
+    });
+    request.on("connect", (response, socket) => {
+        if (response.statusCode === 200) {
+            resolve(socket);
+        }
+        else {
+            socket.destroy();
+            reject(new ProxyRefused(response.statusCode ?? 0));
+        }
+    });
+    request.on("error", reject);
+    request.end();
+});
+/** The socket a request runs on: direct, or through the tunnel; TLS-wrapped for https. */
+const openSocket = async (url, resolved, proxy, signal) => {
+    const via = proxyFor(url, proxy);
+    if (via === undefined) {
+        return undefined;
+    }
+    const tunnel = await connectTunnel(via, url, signal);
+    if (url.protocol !== "https:") {
+        return tunnel;
+    }
+    return new Promise((resolve, reject) => {
+        const secure = tls.connect({ ...tlsOptionsOf(resolved, url.hostname), socket: tunnel }, () => resolve(secure));
+        secure.on("error", reject);
+    });
+};
+const transportFailure = (shown, cause) => {
+    if (cause instanceof ProxyRefused) {
+        return new TransportError({ url: shown, reason: "proxy", detail: `HTTP ${cause.status}` });
+    }
+    const code = errorCode(cause);
+    return new TransportError({
+        url: shown,
+        reason: code === "ABORT_ERR" ? "aborted" : tlsCodes.test(code) ? "tls" : "network",
+        detail: tlsCodes.test(code) ? describeTlsFailure(code) : code
+    });
+};
+const withTimeout = (shown, timeout) => (effect) => effect.pipe(Effect.timeoutOrElse({
+    duration: timeout,
+    orElse: () => Effect.fail(new TransportError({
+        url: shown,
+        reason: "timeout",
+        detail: `no response within ${Duration.format(timeout)}`
+    }))
+}));
+const parseTarget = (raw, needsTls) => {
+    const shown = displayUrl(raw);
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        return new TransportError({ url: shown, reason: "scheme", detail: "invalid URL" });
+    }
+    if (url.protocol !== "https:" && url.protocol !== "http:") {
+        return new TransportError({
+            url: shown,
+            reason: "scheme",
+            detail: `${url.protocol} is not http(s)`
+        });
+    }
+    if (url.protocol === "http:" && needsTls) {
+        return new TransportError({
+            url: shown,
+            reason: "scheme",
+            detail: "client TLS needs an https URL"
+        });
+    }
+    return url;
+};
+const pemOf = (der) => `-----BEGIN CERTIFICATE-----\n${der
+    .toString("base64")
+    .replace(/(.{64})/g, "$1\n")
+    .trimEnd()}\n-----END CERTIFICATE-----\n`;
+const nameOf = (subject) => Object.entries(subject ?? {})
+    .map(([key, value]) => `${key}=${Array.isArray(value) ? value.join("+") : String(value)}`)
+    .join(", ");
+/** Walk `issuerCertificate` links, leaf first, stopping at the self-signed root. */
+const chainOf = (leaf) => {
+    const out = [];
+    const seen = new Set();
+    let current = leaf;
+    while (current !== undefined && current.raw !== undefined && !seen.has(current.fingerprint256)) {
+        seen.add(current.fingerprint256);
+        out.push({
+            subject: nameOf(current.subject),
+            issuer: nameOf(current.issuer),
+            validFrom: current.valid_from,
+            validTo: current.valid_to,
+            fingerprint256: current.fingerprint256,
+            pem: pemOf(current.raw)
+        });
+        current = current.issuerCertificate === current ? undefined : current.issuerCertificate;
+    }
+    return out;
+};
 export const makeNodeSoapTransport = () => ({
     send: (request) => {
         const shown = displayUrl(request.url);
-        let url;
-        try {
-            url = new URL(request.url);
-        }
-        catch {
-            return Effect.fail(new TransportError({ url: shown, reason: "scheme", detail: "invalid URL" }));
-        }
-        if (url.protocol !== "https:" && url.protocol !== "http:") {
-            return Effect.fail(new TransportError({
-                url: shown,
-                reason: "scheme",
-                detail: `${url.protocol} is not http(s)`
-            }));
-        }
-        if (url.protocol === "http:" && request.tls !== undefined) {
-            return Effect.fail(new TransportError({
-                url: shown,
-                reason: "scheme",
-                detail: "client TLS needs an https URL"
-            }));
+        const target = parseTarget(request.url, request.tls !== undefined);
+        if (target instanceof TransportError) {
+            return Effect.fail(target);
         }
+        const url = target;
         const headers = { ...request.headers };
         for (const [name, value] of request.secretHeaders)
             headers[name] = Redacted.value(value);
@@ -65,67 +202,99 @@ export const makeNodeSoapTransport = () => ({
         if (body !== undefined)
             headers["content-length"] = String(body.length);
         return Effect.tryPromise({
-            try: (signal) => new Promise((resolve, reject) => {
+            try: (signal) => {
                 const started = performance.now();
-                const tls = request.tls;
-                const options = {
-                    method: request.method,
-                    headers,
-                    signal,
-                    ...(tls === undefined
-                        ? {}
-                        : {
-                            cert: bufferOf(tls.cert),
-                            key: bufferOf(tls.key),
-                            pfx: bufferOf(tls.pfx),
-                            ca: bufferOf(tls.ca),
-                            passphrase: tls.passphrase === undefined ? undefined : Redacted.value(tls.passphrase)
-                        })
-                };
-                const handler = (response) => {
-                    const chunks = [];
-                    response.on("data", (chunk) => chunks.push(chunk));
-                    response.on("error", reject);
-                    response.on("end", () => resolve({
-                        status: response.statusCode ?? 0,
-                        headers: flatHeaders(response.headers),
-                        body: new Uint8Array(Buffer.concat(chunks)),
-                        elapsedMs: Math.round(performance.now() - started)
-                    }));
-                };
-                const outgoing = url.protocol === "https:"
-                    ? https.request(url, options, handler)
-                    : http.request(url, options, handler);
-                outgoing.on("error", reject);
-                if (body !== undefined)
-                    outgoing.write(body);
-                outgoing.end();
-            }),
-            catch: (cause) => {
-                const code = errorCode(cause);
-                return new TransportError({
-                    url: shown,
-                    reason: code === "ABORT_ERR" ? "aborted" : tlsCodes.test(code) ? "tls" : "network",
-                    detail: code
-                });
-            }
-        }).pipe(Effect.timeoutOrElse({
-            duration: request.timeout,
-            orElse: () => Effect.fail(new TransportError({
+                return openSocket(url, request.tls, request.proxy, signal).then((socket) => new Promise((resolve, reject) => {
+                    const options = {
+                        method: request.method,
+                        headers,
+                        signal,
+                        ...(url.protocol === "https:" ? tlsOptionsOf(request.tls, url.hostname) : {}),
+                        ...(socket === undefined ? {} : { createConnection: () => socket })
+                    };
+                    const handler = (response) => {
+                        const chunks = [];
+                        response.on("data", (chunk) => chunks.push(chunk));
+                        response.on("error", reject);
+                        response.on("end", () => resolve({
+                            status: response.statusCode ?? 0,
+                            headers: flatHeaders(response.headers),
+                            body: new Uint8Array(Buffer.concat(chunks)),
+                            elapsedMs: Math.round(performance.now() - started)
+                        }));
+                    };
+                    const outgoing = url.protocol === "https:"
+                        ? https.request(url, options, handler)
+                        : http.request(url, options, handler);
+                    outgoing.on("error", reject);
+                    if (body !== undefined)
+                        outgoing.write(body);
+                    outgoing.end();
+                }));
+            },
+            catch: (cause) => transportFailure(shown, cause)
+        }).pipe(withTimeout(shown, request.timeout));
+    },
+    peerChain: (raw, options) => {
+        const shown = displayUrl(raw);
+        const target = parseTarget(raw, false);
+        if (target instanceof TransportError) {
+            return Effect.fail(target);
+        }
+        const url = target;
+        if (url.protocol !== "https:") {
+            return Effect.fail(new TransportError({
                 url: shown,
-                reason: "timeout",
-                detail: `no response within ${Duration.format(request.timeout)}`
-            }))
-        }));
+                reason: "scheme",
+                detail: "only an https endpoint has a chain"
+            }));
+        }
+        return Effect.tryPromise({
+            try: (signal) => {
+                const via = proxyFor(url, options.proxy);
+                const tunnelled = via === undefined ? Promise.resolve(undefined) : connectTunnel(via, url, signal);
+                return tunnelled.then((tunnel) => new Promise((resolve, reject) => {
+                    const socket = tls.connect({
+                        ...(tunnel === undefined
+                            ? { host: url.hostname, port: portOf(url) }
+                            : { socket: tunnel }),
+                        servername: options.servername ?? (net.isIP(url.hostname) === 0 ? url.hostname : undefined),
+                        rejectUnauthorized: false
+                    }, () => {
+                        const leaf = socket.getPeerCertificate(true);
+                        const chain = {
+                            host: url.hostname,
+                            certificates: chainOf(leaf),
+                            trustedBySystem: socket.authorized,
+                            authorizationError: socket.authorizationError === undefined || socket.authorizationError === null
+                                ? undefined
+                                : String(socket.authorizationError)
+                        };
+                        socket.end();
+                        resolve(chain);
+                    });
+                    socket.on("error", reject);
+                    signal.addEventListener("abort", () => socket.destroy());
+                }));
+            },
+            catch: (cause) => transportFailure(shown, cause)
+        }).pipe(withTimeout(shown, options.timeout));
     }
 });
 /**
  * A scripted transport for tests: each request goes to `respond`, and every
- * request is recorded (secrets still redacted) for assertions.
+ * request is recorded (secrets still redacted) for assertions. `chain`
+ * answers `peerChain`; by default an endpoint presents nothing.
  */
-export const makeFakeSoapTransport = (respond) => Effect.map(Ref.make([]), (log) => ({
+export const makeFakeSoapTransport = (respond, chain = (url) => Effect.succeed({
+    host: new URL(url).hostname,
+    certificates: [],
+    trustedBySystem: true,
+    authorizationError: undefined
+})) => Effect.map(Ref.make([]), (log) => ({
     transport: {
-        send: (request) => Effect.andThen(Ref.update(log, (all) => [...all, request]), respond(request))
+        send: (request) => Effect.andThen(Ref.update(log, (all) => [...all, request]), respond(request)),
+        peerChain: (url) => chain(url)
     },
     requests: Ref.get(log)
 }));
@@ -138,7 +307,7 @@ export const xmlResponse = (xml, status = 200) => ({
 const isUrl = (location) => /^https?:\/\//i.test(location);
 /**
  * The document loader for discovery: local paths from disk, http(s) URLs
- * through the transport with the profile's fetch-side auth and TLS.
+ * through the transport with the profile's fetch-side auth, TLS and proxy.
  */
 export const makeTransportDocumentLoader = (transport, side, timeout = Duration.seconds(30)) => {
     const files = makeFileDocumentLoader();
@@ -152,6 +321,7 @@ export const makeTransportDocumentLoader = (transport, side, timeout = Duration.
                 headers: { accept: "text/xml, application/xml, */*" },
                 secretHeaders: side.headers,
                 ...(side.tls === undefined ? {} : { tls: side.tls }),
+                ...(side.proxy === undefined ? {} : { proxy: side.proxy }),
                 timeout
             })
                 .pipe(Effect.mapError((error) => new DocumentLoadError({

package/kits/soap-ace/flows/lib/soap/Xml.js

@@ -297,7 +297,8 @@ class Parser {
                 attributes,
                 children: [],
                 scope,
-                line
+                line,
+                span: { start, end: this.position }
             };
         }
         if (this.text[this.position] !== ">")
@@ -340,7 +341,8 @@ class Parser {
                     attributes,
                     children,
                     scope,
-                    line
+                    line,
+                    span: { start, end: this.position }
                 };
             }
             if (this.text.startsWith("<![CDATA[", this.position)) {

package/kits/soap-ace/flows/soap-discover.js

@@ -47,7 +47,7 @@ const program = Effect.gen(function* () {
         const workspace = yield* makeNodeWorkspace(input.workDir);
         const profile = service === undefined || service === ""
             ? undefined
-            : yield* loadAuthProfile(workspace, service);
+            : yield* loadAuthProfile(workspace, service, process.env.LLM4TS_SOAP_ENV?.trim() || undefined);
         if (isUrl(location) && profile === undefined) {
             yield* say("fetching without credentials (set LLM4TS_SOAP_SERVICE and write its auth.json for an authenticated WSDL)");
         }

package/kits/soap-ace/flows/soap-explore.js

@@ -37,8 +37,9 @@ import { FlowAborted, Info, makeNodeWorkspace, mock, resolveFlowInput, runFlowMa
 import { coderFor, judgmentConnectorFromEnvironment } from "@llm4ts/runner/Connectors";
 import { nodeSecretSource, resolveSide } from "./lib/soap/Auth.js";
 import { heuristicClass, judgeClass } from "./lib/soap/Classification.js";
-import { discoverService, loadAuthProfile, loadCatalog, loadOperationsFile } from "./lib/soap/Discover.js";
+import { discoverService, loadAuthProfile, profilePath, loadCatalog, loadOperationsFile } from "./lib/soap/Discover.js";
 import { designStep, explorePaths, exploreService, renderExploreReport, writeEvidence } from "./lib/soap/Explore.js";
+import { makeTokenSource } from "./lib/soap/Sts.js";
 import { makeDirectoryStubTransport } from "./lib/soap/Stub.js";
 import { makeNodeSoapTransport, makeTransportDocumentLoader } from "./lib/soap/Transport.js";
 import { DocumentLoader } from "./lib/soap/Wsdl.js";
@@ -94,7 +95,9 @@ const program = Effect.gen(function* () {
         let catalog;
         let judged;
         if (location !== undefined) {
-            const fetchProfile = requested === undefined ? undefined : yield* loadAuthProfile(workspace, requested);
+            const fetchProfile = requested === undefined
+                ? undefined
+                : yield* loadAuthProfile(workspace, requested, environment.LLM4TS_SOAP_ENV?.trim() || undefined);
             const fetchSide = yield* resolveSide(nodeSecretSource(environment), fetchProfile?.fetch);
             const discovered = yield* discoverService({
                 workspace,
@@ -124,7 +127,10 @@ const program = Effect.gen(function* () {
             catalog = yield* loadCatalog(workspace, service);
             yield* say(`continuing ${service}: ${catalog.operations.length} operations`);
         }
-        const profile = yield* loadAuthProfile(workspace, service);
+        const env = environment.LLM4TS_SOAP_ENV?.trim() || undefined;
+        const profile = yield* loadAuthProfile(workspace, service, env);
+        if (profile !== undefined)
+            yield* say(`profile: ${profilePath(service, env)}`);
         const operations = yield* loadOperationsFile(workspace, service);
         // Provisional probing asks the judgment only where the listed class
         // and the heuristic already say read.
@@ -149,6 +155,17 @@ const program = Effect.gen(function* () {
             judged,
             secrets: nodeSecretSource(environment),
             transport,
+            ...(profile?.sts === undefined
+                ? {}
+                : {
+                    token: yield* makeTokenSource({
+                        workspace,
+                        service,
+                        config: profile.sts,
+                        secrets: nodeSecretSource(environment),
+                        transport
+                    })
+                }),
             refresh: flags.refresh,
             progress: (line) => say(`  ${line}`)
         });