@llm4ts/shell 2.12.0 → 2.13.1

package/kits/soap-ace/flows/lib/soap/Auth.js

@@ -0,0 +1,179 @@
+import { readFile } from "node:fs/promises";
+import * as Effect from "effect/Effect";
+import * as Redacted from "effect/Redacted";
+import * as Schema from "effect/Schema";
+// The auth profile, `.llm4ts/soap/<service>/auth.json` (gitignored). It holds
+// references only — `env:NAME` or `file:path` — never a secret value; a
+// literal where a secret belongs is a decode error. References are resolved
+// at call time into `Redacted` values that live inside the transport; no
+// secret reaches arguments, logs, traces, persisted files, or error
+// messages (errors name the reference, never its value).
+export const Environment = Schema.Literals(["dev", "test", "uat"]);
+/** `env:NAME` or `file:path`. */
+export const SecretRef = Schema.String.check(Schema.isPattern(/^(env:[A-Za-z_][A-Za-z0-9_]*|file:.+)$/, {
+    message: "must be a reference, env:NAME or file:path — never the secret itself"
+}));
+/** A username may be written literally or referenced. */
+export const Principal = Schema.String;
+export class BasicAuth extends Schema.Class("BasicAuth")({
+    scheme: Schema.Literal("basic"),
+    user: Principal,
+    password: SecretRef
+}) {
+}
+export class BearerAuth extends Schema.Class("BearerAuth")({
+    scheme: Schema.Literal("bearer"),
+    token: SecretRef
+}) {
+}
+export class NoAuth extends Schema.Class("NoAuth")({
+    scheme: Schema.Literal("none")
+}) {
+}
+export const HttpAuth = Schema.Union([NoAuth, BasicAuth, BearerAuth]);
+/** Client TLS: PEM cert+key, or a PKCS#12 bundle; an optional CA bundle. */
+export class TlsConfig extends Schema.Class("TlsConfig")({
+    cert: Schema.optionalKey(SecretRef),
+    key: Schema.optionalKey(SecretRef),
+    pfx: Schema.optionalKey(SecretRef),
+    passphrase: Schema.optionalKey(SecretRef),
+    ca: Schema.optionalKey(SecretRef)
+}) {
+}
+export class HttpSide extends Schema.Class("HttpSide")({
+    auth: Schema.optionalKey(HttpAuth),
+    tls: Schema.optionalKey(TlsConfig)
+}) {
+}
+export class UsernameToken extends Schema.Class("UsernameToken")({
+    user: Principal,
+    password: SecretRef,
+    passwordType: Schema.Literals(["text", "digest"])
+}) {
+}
+export const MutatingPolicy = Schema.Literals(["confirm", "flag", "deny"]);
+export class MutatingPolicies extends Schema.Class("MutatingPolicies")({
+    dev: Schema.optionalKey(MutatingPolicy),
+    test: Schema.optionalKey(MutatingPolicy),
+    uat: Schema.optionalKey(MutatingPolicy)
+}) {
+}
+export class AuthProfile extends Schema.Class("AuthProfile")({
+    environment: Environment,
+    /** Overrides the WSDL's address for calls (the WSDL often names another environment). */
+    endpoint: Schema.optionalKey(Schema.String.check(Schema.isPattern(/^https?:\/\/[^/?#@]+(?:[/?#][^@]*)?$/i, {
+        message: "an http(s) URL without credentials; secrets go in references"
+    }))),
+    /** Fetching the WSDL and its schemas. */
+    fetch: Schema.optionalKey(HttpSide),
+    /** Calling operations. */
+    call: Schema.optionalKey(HttpSide),
+    wsSecurity: Schema.optionalKey(UsernameToken),
+    mutating: Schema.optionalKey(MutatingPolicies),
+    timeoutSeconds: Schema.optionalKey(Schema.Int)
+}) {
+}
+export const defaultMutatingPolicy = (environment) => environment === "dev" ? "confirm" : "flag";
+export const mutatingPolicy = (profile) => profile.mutating?.[profile.environment] ?? defaultMutatingPolicy(profile.environment);
+export class AuthProfileError extends Schema.TaggedError()("AuthProfileError", {
+    /** The reference or field involved; never a secret value. */
+    subject: Schema.String,
+    detail: Schema.String
+}) {
+    get message() {
+        return `auth profile: ${this.subject}: ${this.detail}`;
+    }
+}
+export const decodeAuthProfile = (text) => Schema.decodeUnknownEffect(Schema.fromJsonString(AuthProfile))(text).pipe(Effect.mapError((error) => 
+// Schema issues quote the offending input; a mistyped secret could be
+// in it, so only the paths of the issue are kept.
+new AuthProfileError({
+    subject: "auth.json",
+    detail: `invalid profile (${issuePaths(error.message)})`
+})));
+/** Keep only the `at ["a"]["b"]` locations of a schema message, dropping any quoted value. */
+const issuePaths = (message) => {
+    const paths = [...message.matchAll(/at ((?:\[[^\]]*\])+)/g)].map((match) => [...(match[1] ?? "").matchAll(/\["?([^"\]]*)"?\]/g)].map((part) => part[1] ?? "").join("."));
+    return paths.length === 0
+        ? "see the soap-ace README for the profile shape"
+        : `at ${[...new Set(paths)].join(", ")}; secrets must be env:NAME or file:path references`;
+};
+export const resolveSecret = (source, reference) => {
+    if (reference.startsWith("env:")) {
+        const name = reference.slice(4);
+        const value = source.environment[name];
+        return value === undefined || value === ""
+            ? Effect.fail(new AuthProfileError({ subject: reference, detail: "variable is not set" }))
+            : Effect.succeed(Redacted.make(new TextEncoder().encode(value)));
+    }
+    if (reference.startsWith("file:")) {
+        return Effect.map(source.readFile(reference.slice(5)), (bytes) => Redacted.make(bytes));
+    }
+    return Effect.fail(new AuthProfileError({ subject: "reference", detail: "not env: or file:" }));
+};
+export const resolveText = (source, reference) => Effect.map(resolveSecret(source, reference), (secret) => Redacted.make(new TextDecoder().decode(Redacted.value(secret)).replace(/\r?\n$/, "")));
+/** A principal: resolved when it is a reference, else the literal. */
+export const resolvePrincipal = (source, value) => value.startsWith("env:") || value.startsWith("file:")
+    ? Effect.map(resolveText(source, value), Redacted.value)
+    : Effect.succeed(value);
+const optionalSecret = (source, name, reference) => reference === undefined
+    ? Effect.succeed({})
+    : Effect.map(resolveSecret(source, reference), (value) => ({ [name]: value }));
+export const resolveTls = (source, tls) => tls === undefined
+    ? Effect.succeed(undefined)
+    : Effect.gen(function* () {
+        if (tls.pfx !== undefined && (tls.cert !== undefined || tls.key !== undefined)) {
+            return yield* new AuthProfileError({
+                subject: "tls",
+                detail: "use either pfx or cert+key, not both"
+            });
+        }
+        if ((tls.cert === undefined) !== (tls.key === undefined)) {
+            return yield* new AuthProfileError({
+                subject: "tls",
+                detail: "cert and key go together"
+            });
+        }
+        const cert = yield* optionalSecret(source, "cert", tls.cert);
+        const key = yield* optionalSecret(source, "key", tls.key);
+        const pfx = yield* optionalSecret(source, "pfx", tls.pfx);
+        const ca = yield* optionalSecret(source, "ca", tls.ca);
+        const passphrase = tls.passphrase === undefined
+            ? {}
+            : { passphrase: yield* resolveText(source, tls.passphrase) };
+        return { ...cert, ...key, ...pfx, ...ca, ...passphrase };
+    });
+const basicHeader = (user, password) => Redacted.make(`Basic ${Buffer.from(`${user}:${Redacted.value(password)}`, "utf8").toString("base64")}`);
+export const resolveSide = (source, side) => Effect.gen(function* () {
+    const auth = side?.auth;
+    const tls = yield* resolveTls(source, side?.tls);
+    if (auth === undefined || auth.scheme === "none")
+        return { headers: [], tls };
+    if (auth.scheme === "basic") {
+        const user = yield* resolvePrincipal(source, auth.user);
+        const password = yield* resolveText(source, auth.password);
+        return { headers: [["authorization", basicHeader(user, password)]], tls };
+    }
+    const token = yield* resolveText(source, auth.token);
+    return {
+        headers: [["authorization", Redacted.make(`Bearer ${Redacted.value(token)}`)]],
+        tls
+    };
+});
+export const resolveUsernameToken = (source, token) => token === undefined
+    ? Effect.succeed(undefined)
+    : Effect.gen(function* () {
+        return {
+            user: yield* resolvePrincipal(source, token.user),
+            password: yield* resolveText(source, token.password),
+            passwordType: token.passwordType
+        };
+    });
+/** Secrets from the process environment and local files. */
+export const nodeSecretSource = (environment = process.env) => ({
+    environment,
+    readFile: (path) => Effect.tryPromise({
+        try: async () => new Uint8Array(await readFile(path)),
+        catch: () => new AuthProfileError({ subject: `file:${path}`, detail: "not readable" })
+    })
+});

package/kits/soap-ace/flows/lib/soap/Call.js

@@ -0,0 +1,186 @@
+import { randomBytes } from "node:crypto";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Redacted from "effect/Redacted";
+import * as Schema from "effect/Schema";
+import { resolveSide, resolveUsernameToken } from "./Auth.js";
+import { checkCall } from "./CallPolicy.js";
+import { elementByName, operationByName } from "./Catalog.js";
+import { effectiveClass } from "./Classification.js";
+import { buildEnvelope, readEnvelope, safeHeaders, soapHeaders, stripSecurity, usernameTokenHeader } from "./Envelope.js";
+import { instanceFromXml, instanceToXml, validateInstance } from "./Instance.js";
+import { kindsFromCatalog, maskDocument, MaskingReport, mergeReports } from "./Masking.js";
+import { Exchange, issueRecords, loadMaskingKey, loadMaskingOverrides, readRequestFile, RecordedMessage, RecordedResponse, samplePaths, writeExchange } from "./Samples.js";
+import { displayUrl } from "./Transport.js";
+import { parseXml, parseXmlBytes, renderXml } from "./Xml.js";
+// One call, end to end: read and validate the request file, check the call
+// policy, render the envelope (WS-Security header when configured), send it
+// with the profile's call-side auth and TLS, read the response against the
+// output element's schema, mask both messages, and persist the exchange.
+// A request that does not validate is never sent. A response that breaks
+// its schema is recorded with the issues; that is a finding for the design.
+export class RequestInvalid extends Schema.TaggedError()("RequestInvalid", {
+    operation: Schema.String,
+    name: Schema.String,
+    issues: Schema.Array(Schema.Struct({ path: Schema.String, detail: Schema.String }))
+}) {
+    get message() {
+        return `${this.operation}/${this.name} not sent; the request does not validate:\n${this.issues
+            .map((issue) => `  ${issue.path}: ${issue.detail}`)
+            .join("\n")}`;
+    }
+}
+export class CallSetupError extends Schema.TaggedError()("CallSetupError", {
+    operation: Schema.String,
+    detail: Schema.String
+}) {
+    get message() {
+        return `${this.operation}: ${this.detail}`;
+    }
+}
+/** The endpoint for an operation: the profile's override, else the WSDL port of its binding. */
+export const endpointFor = (catalog, operation, profile) => profile.endpoint ??
+    catalog.endpoints.find((endpoint) => endpoint.binding === operation.binding)?.address ??
+    catalog.endpoints.find((endpoint) => endpoint.soapVersion === operation.soapVersion)?.address;
+const maskEnvelope = (root, key, options) => {
+    const masked = maskDocument(stripSecurity(root), { key, ...options });
+    return { document: masked.document, xml: renderXml(masked.document), report: masked.report };
+};
+export const callOperation = (options) => Effect.gen(function* () {
+    const { catalog, workspace, service, name } = options;
+    const operation = operationByName(catalog, options.operation);
+    if (operation === undefined) {
+        return yield* new CallSetupError({
+            operation: options.operation,
+            detail: "not an operation of this service"
+        });
+    }
+    const input = elementByName(catalog, operation.input);
+    if (input === undefined) {
+        return yield* new CallSetupError({
+            operation: operation.name,
+            detail: "request element missing from the catalog"
+        });
+    }
+    const profile = options.profile;
+    if (profile === undefined) {
+        return yield* new CallSetupError({
+            operation: operation.name,
+            detail: `no auth profile; write ${`.llm4ts/soap/${service}/auth.json`} with at least {"environment": "dev"}`
+        });
+    }
+    const request = yield* readRequestFile(workspace, catalog, service, operation.name, name);
+    const issues = [
+        ...request.readIssues,
+        ...validateInstance(catalog, input, request.body)
+    ];
+    if (issues.length > 0) {
+        return yield* new RequestInvalid({ operation: operation.name, name, issues });
+    }
+    const permission = yield* checkCall({
+        operation: operation.name,
+        operationClass: effectiveClass(options.operations, operation.name),
+        profile,
+        allowMutating: options.allowMutating,
+        confirm: options.confirm
+    });
+    const address = endpointFor(catalog, operation, profile);
+    if (address === undefined) {
+        return yield* new CallSetupError({
+            operation: operation.name,
+            detail: 'no endpoint: the WSDL declares none for this binding; set "endpoint" in auth.json'
+        });
+    }
+    const side = yield* resolveSide(options.secrets, profile.call);
+    const token = yield* resolveUsernameToken(options.secrets, profile.wsSecurity);
+    const now = options.now ?? (() => new Date());
+    const random = options.random ?? ((size) => new Uint8Array(randomBytes(size)));
+    const header = token === undefined
+        ? undefined
+        : usernameTokenHeader(token, { nonce: random(16), created: now().toISOString() });
+    const bodyXml = instanceToXml(catalog, input, request.body);
+    const envelope = buildEnvelope(operation.soapVersion, bodyXml, header);
+    const headers = soapHeaders(operation.soapVersion, operation.soapAction);
+    const response = yield* options.transport.send({
+        method: "POST",
+        url: address,
+        headers,
+        secretHeaders: side.headers,
+        body: envelope,
+        ...(side.tls === undefined ? {} : { tls: side.tls }),
+        timeout: Duration.seconds(profile.timeoutSeconds ?? 60)
+    });
+    const key = yield* loadMaskingKey(workspace, service, random);
+    const overrides = yield* loadMaskingOverrides(workspace, service);
+    const maskOptions = {
+        typeKinds: kindsFromCatalog(catalog),
+        ...(overrides === undefined ? {} : { overrides })
+    };
+    // The request is re-parsed from what was sent, so the recorded envelope
+    // is exactly the message on the wire minus its security header.
+    const sent = yield* parseXml(Redacted.value(envelope));
+    const maskedRequest = maskEnvelope(sent, key, maskOptions);
+    const responseIssues = [];
+    const parsedResponse = yield* parseXmlBytes(response.body).pipe(Effect.option);
+    let maskedResponse = {
+        xml: "",
+        report: new MaskingReport({ entries: [], kept: [] })
+    };
+    // Everything recorded about the response (fault text, schema findings,
+    // which quote values) is read from the masked, security-stripped tree:
+    // nothing unmasked reaches the exchange.
+    let fault;
+    if (parsedResponse._tag === "None") {
+        responseIssues.push({
+            path: "(response)",
+            detail: `HTTP ${response.status} with a body that is not XML (${response.body.length} bytes)`
+        });
+    }
+    else {
+        const masked = maskEnvelope(parsedResponse.value, key, maskOptions);
+        maskedResponse = masked;
+        const read = yield* readEnvelope(masked.document).pipe(Effect.option);
+        if (read._tag === "None") {
+            responseIssues.push({ path: "(response)", detail: "XML that is not a SOAP envelope" });
+        }
+        else if (read.value.fault !== undefined) {
+            fault = read.value.fault;
+        }
+        else if (operation.output !== undefined) {
+            const output = elementByName(catalog, operation.output);
+            const payload = read.value.payload;
+            if (output !== undefined && payload !== undefined) {
+                responseIssues.push(...instanceFromXml(catalog, output, payload).issues);
+            }
+            else if (payload === undefined) {
+                responseIssues.push({ path: "(response)", detail: "empty Body" });
+            }
+        }
+        if (options.keepRaw === true) {
+            yield* workspace.write(samplePaths(service, operation.name, name).raw, renderXml(stripSecurity(parsedResponse.value)));
+        }
+    }
+    const exchange = new Exchange({
+        version: 1,
+        operation: operation.name,
+        name,
+        purpose: request.purpose,
+        provenance: "call",
+        recordedAt: now().toISOString(),
+        environment: profile.environment,
+        endpoint: displayUrl(address),
+        request: new RecordedMessage({ headers: safeHeaders(headers), envelope: maskedRequest.xml }),
+        response: new RecordedResponse({
+            status: response.status,
+            headers: safeHeaders(response.headers),
+            envelope: maskedResponse.xml,
+            elapsedMs: response.elapsedMs
+        }),
+        ...(fault === undefined ? {} : { fault }),
+        requestIssues: [],
+        responseIssues: issueRecords(responseIssues),
+        masking: mergeReports([maskedRequest.report, maskedResponse.report])
+    });
+    const path = yield* writeExchange(workspace, service, exchange);
+    return { exchange, path, permission };
+});

package/kits/soap-ace/flows/lib/soap/CallPolicy.js

@@ -0,0 +1,50 @@
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import { mutatingPolicy } from "./Auth.js";
+// Whether one operation may be called now. Non-production environments
+// still share data and reach other teams' downstream test systems, so the
+// guard stays on everywhere:
+//
+//   unclassified → refused (confirm operations.md first)
+//   read         → allowed
+//   mutating     → the environment's policy: `confirm` (default in dev)
+//                  asks interactively, `flag` (default in test and uat)
+//                  needs `--allow-mutating <operation>` naming exactly this
+//                  operation, `deny` refuses. Profiles may override per
+//                  environment.
+export class CallRefused extends Schema.TaggedError()("CallRefused", {
+    operation: Schema.String,
+    detail: Schema.String
+}) {
+    get message() {
+        return `${this.operation} not called: ${this.detail}`;
+    }
+}
+const refuse = (operation, detail) => Effect.fail(new CallRefused({ operation, detail }));
+export const checkCall = (request) => {
+    const { operation, operationClass, profile } = request;
+    if (request.allowMutating !== undefined && /[*?,\s]/.test(request.allowMutating)) {
+        return refuse(operation, "--allow-mutating names exactly one operation; no wildcards or lists");
+    }
+    if (operationClass === "unclassified") {
+        return refuse(operation, "the operation is unclassified; classify it in operations.md and set Status: confirmed");
+    }
+    if (operationClass === "read")
+        return Effect.succeed({ operation, operationClass });
+    const policy = mutatingPolicy(profile);
+    const flagged = request.allowMutating === operation;
+    switch (policy) {
+        case "deny":
+            return refuse(operation, `mutating calls are denied in ${profile.environment}`);
+        case "flag":
+            return flagged
+                ? Effect.succeed({ operation, operationClass, allowedBy: "flag" })
+                : refuse(operation, `mutating calls in ${profile.environment} need --allow-mutating ${operation}`);
+        case "confirm":
+            if (flagged)
+                return Effect.succeed({ operation, operationClass, allowedBy: "flag" });
+            return Effect.flatMap(request.confirm(`${operation} changes state in ${profile.environment}. Call it now? [y/N]`), (confirmed) => confirmed
+                ? Effect.succeed({ operation, operationClass, allowedBy: "confirmation" })
+                : refuse(operation, "not confirmed"));
+    }
+};