@llm4ts/shell 2.11.0 → 2.13.0

package/kits/soap-ace/flows/lib/soap/OpenApi.js

@@ -0,0 +1,288 @@
+import { ErrorMapping } from "./Design.js";
+const ref = (name) => ({ $ref: `#/components/schemas/${name}` });
+const withDefined = (entries) => Object.fromEntries(entries.filter((entry) => entry[1] !== undefined));
+/**
+ * One property. OpenAPI 3.1 is JSON Schema: `type: [t, "null"]`, and keys
+ * beside `$ref` are honoured. OpenAPI 3.0 (what IBM ACE 12 imports) needs
+ * `nullable: true`, and a `$ref` with siblings wrapped in `allOf`.
+ */
+const propertySchema = (property, dialect) => {
+    const common = withDefined([
+        ["description", property.description],
+        ["x-soap-source", property.source],
+        ["x-derivation", property.derivation],
+        ["x-soap-enum-map", property.enumMap === undefined ? undefined : { ...property.enumMap }]
+    ]);
+    const nullable = property.nullable === true;
+    if (property.type === "object" && property.ref !== undefined) {
+        if (dialect === "3.0") {
+            return Object.keys(common).length === 0 && !nullable
+                ? ref(property.ref)
+                : { allOf: [ref(property.ref)], ...(nullable ? { nullable: true } : {}), ...common };
+        }
+        return nullable
+            ? { oneOf: [ref(property.ref), { type: "null" }], ...common }
+            : { ...ref(property.ref), ...common };
+    }
+    const typeOf = (type) => dialect === "3.0"
+        ? { type, ...(nullable ? { nullable: true } : {}) }
+        : { type: nullable ? [type, "null"] : type };
+    if (property.type === "array") {
+        return {
+            ...typeOf("array"),
+            items: property.ref !== undefined ? ref(property.ref) : { type: property.items ?? "string" },
+            ...common
+        };
+    }
+    return {
+        ...typeOf(property.type),
+        ...withDefined([
+            ["format", property.format],
+            ["enum", property.enum === undefined ? undefined : [...property.enum]]
+        ]),
+        ...common
+    };
+};
+const modelSchema = (model, dialect) => ({
+    type: "object",
+    ...withDefined([["description", model.description]]),
+    ...(model.properties.some((property) => property.required)
+        ? {
+            required: model.properties
+                .filter((property) => property.required)
+                .map((property) => property.name)
+        }
+        : {}),
+    properties: Object.fromEntries(model.properties.map((property) => [property.name, propertySchema(property, dialect)])),
+    additionalProperties: false,
+    ...withDefined([["x-soap-source-type", model.sourceType]])
+});
+const problemSchema = {
+    type: "object",
+    description: "RFC 9457 problem details; `code` is the stable, machine-readable error code.",
+    required: ["type", "title", "status", "code"],
+    properties: {
+        type: { type: "string", format: "uri-reference" },
+        title: { type: "string" },
+        status: { type: "integer", format: "int32" },
+        detail: { type: "string" },
+        instance: { type: "string", format: "uri-reference" },
+        code: { type: "string" }
+    }
+};
+const pageSchema = {
+    type: "object",
+    required: ["page", "size"],
+    properties: {
+        page: { type: "integer", format: "int32", minimum: 1 },
+        size: { type: "integer", format: "int32", minimum: 1 },
+        totalItems: { type: "integer", format: "int64" },
+        hasNext: { type: "boolean" }
+    }
+};
+const listName = (response) => `${response.model ?? "Item"}${response.paged === true ? "Page" : "List"}`;
+const parameterObject = (parameter) => ({
+    name: parameter.name,
+    in: parameter.in,
+    required: parameter.required,
+    ...withDefined([["description", parameter.description]]),
+    schema: {
+        type: parameter.type,
+        ...withDefined([
+            ["format", parameter.format],
+            ["enum", parameter.enum === undefined ? undefined : [...parameter.enum]]
+        ])
+    },
+    ...withDefined([
+        ["x-soap-source", parameter.source],
+        ["x-derivation", parameter.derivation],
+        ["x-soap-enum-map", parameter.enumMap === undefined ? undefined : { ...parameter.enumMap }]
+    ])
+});
+const successObject = (response, created) => {
+    const schema = response.model === undefined
+        ? undefined
+        : response.list === true
+            ? ref(listName(response))
+            : ref(response.model);
+    return {
+        description: response.description,
+        ...(created && response.status === 201
+            ? {
+                headers: {
+                    Location: {
+                        description: "URL of the created resource",
+                        schema: { type: "string", format: "uri-reference" }
+                    }
+                }
+            }
+            : {}),
+        ...(schema === undefined ? {} : { content: { "application/json": { schema } } }),
+        ...withDefined([["x-soap-source", response.source]])
+    };
+};
+const errorResponses = (errors) => {
+    const byStatus = new Map();
+    for (const mapping of errors)
+        byStatus.set(mapping.status, [...(byStatus.get(mapping.status) ?? []), mapping]);
+    if (!byStatus.has(502)) {
+        byStatus.set(502, [
+            new ErrorMapping({
+                status: 502,
+                code: "backend-error",
+                title: "The backend service returned an unmapped error",
+                from: ["transport"]
+            })
+        ]);
+    }
+    return Object.fromEntries([...byStatus]
+        .sort((left, right) => left[0] - right[0])
+        .map(([status, mappings]) => [
+        String(status),
+        {
+            description: mappings.map((mapping) => mapping.title).join("; "),
+            content: { "application/problem+json": { schema: ref("Problem") } },
+            "x-problem-codes": mappings.map((mapping) => ({
+                code: mapping.code,
+                title: mapping.title,
+                from: [...mapping.from]
+            }))
+        }
+    ]));
+};
+const operationObject = (endpoint) => {
+    const created = endpoint.method === "POST";
+    const tag = endpoint.path.split("/")[1] ?? "default";
+    return {
+        operationId: endpoint.operationId,
+        summary: endpoint.summary,
+        ...withDefined([["description", endpoint.description]]),
+        tags: [tag],
+        ...(endpoint.parameters.length === 0
+            ? {}
+            : { parameters: endpoint.parameters.map(parameterObject) }),
+        ...(endpoint.requestBody === undefined
+            ? {}
+            : {
+                requestBody: {
+                    required: endpoint.requestBody.required,
+                    content: { "application/json": { schema: ref(endpoint.requestBody.model) } }
+                }
+            }),
+        responses: {
+            ...Object.fromEntries(endpoint.responses.map((response) => [
+                String(response.status),
+                successObject(response, created)
+            ])),
+            ...errorResponses(endpoint.errors)
+        },
+        "x-soap-operations": [...endpoint.sources],
+        ...(endpoint.evidence.length === 0 ? {} : { "x-evidence": [...endpoint.evidence] })
+    };
+};
+export const projectOpenApi = (design, options = {}) => {
+    const dialect = options.dialect ?? "3.1";
+    const paths = {};
+    for (const endpoint of design.endpoints) {
+        paths[endpoint.path] = {
+            ...(paths[endpoint.path] ?? {}),
+            [endpoint.method.toLowerCase()]: operationObject(endpoint)
+        };
+    }
+    const schemas = {};
+    for (const model of design.models)
+        schemas[model.name] = modelSchema(model, dialect);
+    for (const endpoint of design.endpoints) {
+        for (const response of endpoint.responses) {
+            if (response.list !== true || response.model === undefined)
+                continue;
+            schemas[listName(response)] = {
+                type: "object",
+                required: response.paged === true ? ["items", "page"] : ["items"],
+                properties: {
+                    items: { type: "array", items: ref(response.model) },
+                    ...(response.paged === true ? { page: ref("PageInfo") } : {})
+                }
+            };
+            if (response.paged === true)
+                schemas["PageInfo"] = pageSchema;
+        }
+    }
+    schemas["Problem"] = problemSchema;
+    return {
+        openapi: dialect === "3.0" ? "3.0.3" : "3.1.0",
+        info: {
+            title: design.title,
+            version: options.version ?? "1.0.0",
+            description: "Projected by soap-design from the approved REST design; do not edit. x-soap-* extensions trace every element to the SOAP service."
+        },
+        servers: [{ url: design.basePath }],
+        paths,
+        components: { schemas }
+    };
+};
+// ---------------------------------------------------------------------------
+// Typed YAML
+// A string that a YAML 1.1 or 1.2 reader would turn into something else
+// (a number, boolean, null, date) or that is not plain-safe is quoted.
+const ambiguous = new RegExp([
+    "^(true|false|yes|no|on|off|y|n|null|~|)$", // booleans and nulls, YAML 1.1 included
+    "^[-+]?(\\.\\d+|\\d[\\d_]*(\\.[\\d_]*)?)([eE][-+]?\\d+)?$", // numbers, 1_000 included
+    "^[-+]?\\.(inf|nan)$", // .inf, .nan
+    "^[-+]?\\d[\\d_]*(:[0-5]?\\d)+(\\.\\d*)?$", // YAML 1.1 sexagesimal (12:30)
+    "^0[xob]",
+    "^\\d{4}-\\d{2}-\\d{2}", // dates and timestamps
+    "^[-?:,[\\]{}#&*!|>'\"%@`\\s]",
+    "\\s$",
+    ":$",
+    ": ",
+    "\\s#",
+    "[\\n\\r\\t]"
+].join("|"), "i");
+const scalar = (value) => typeof value === "string"
+    ? ambiguous.test(value)
+        ? JSON.stringify(value)
+        : value
+    : value === null
+        ? "null"
+        : String(value);
+const key = (name) => /^[A-Za-z_$/][A-Za-z0-9_.$/{}-]*$/.test(name) && !ambiguous.test(name)
+    ? name
+    : JSON.stringify(name);
+const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+/** YAML for JSON data, with types preserved for any conforming reader. */
+export const renderTypedYaml = (value, indent = 0) => {
+    const pad = " ".repeat(indent);
+    if (Array.isArray(value)) {
+        if (value.length === 0)
+            return `${pad}[]\n`;
+        return value
+            .map((item) => {
+            if (!isObject(item) && !Array.isArray(item))
+                return `${pad}- ${scalar(item)}\n`;
+            if ((Array.isArray(item) && item.length === 0) ||
+                (isObject(item) && Object.keys(item).length === 0)) {
+                return `${pad}- ${Array.isArray(item) ? "[]" : "{}"}\n`;
+            }
+            return `${pad}- ${renderTypedYaml(item, indent + 2).slice(indent + 2)}`;
+        })
+            .join("");
+    }
+    if (isObject(value)) {
+        const entries = Object.entries(value);
+        if (entries.length === 0)
+            return `${pad}{}\n`;
+        return entries
+            .map(([name, item]) => {
+            if (!isObject(item) && !Array.isArray(item))
+                return `${pad}${key(name)}: ${scalar(item)}\n`;
+            if ((Array.isArray(item) && item.length === 0) ||
+                (isObject(item) && Object.keys(item).length === 0)) {
+                return `${pad}${key(name)}: ${Array.isArray(item) ? "[]" : "{}"}\n`;
+            }
+            return `${pad}${key(name)}:\n${renderTypedYaml(item, indent + 2)}`;
+        })
+            .join("");
+    }
+    return `${pad}${scalar(value)}\n`;
+};

package/kits/soap-ace/flows/lib/soap/Samples.js

@@ -0,0 +1,287 @@
+import { randomBytes } from "node:crypto";
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import { elementByName, operationByName } from "./Catalog.js";
+import { readIfPresent, serviceDirectory, servicePaths } from "./Discover.js";
+import { readEnvelope, SoapFault } from "./Envelope.js";
+import { instanceFromXml, skeletonYaml } from "./Instance.js";
+import { MaskingOverrides, MaskingReport } from "./Masking.js";
+import { parseXml } from "./Xml.js";
+import { isYamlMap, parseYaml, renderYaml } from "./Yaml.js";
+// Request files and recorded exchanges under `.llm4ts/soap/<service>/`:
+//
+//   samples/<operation>/<name>.request.yaml   the user's request (primary form)
+//   samples/<operation>/<name>.request.xml    or the raw body element / envelope
+//   samples/<operation>/<name>.exchange.json  one masked request/response pair
+//   raw/<operation>/<name>.response.xml       only with --keep-raw (gitignored)
+//   .masking-key                               the service's pseudonym key (gitignored)
+//
+// Exchanges are always masked and auth-stripped before they are written.
+export const Provenance = Schema.Literals(["call", "soapui"]);
+export class IssueRecord extends Schema.Class("IssueRecord")({
+    path: Schema.String,
+    detail: Schema.String
+}) {
+}
+export class RecordedMessage extends Schema.Class("RecordedMessage")({
+    headers: Schema.Record(Schema.String, Schema.String),
+    /** The masked envelope, security headers removed. */
+    envelope: Schema.String
+}) {
+}
+export class RecordedResponse extends Schema.Class("RecordedResponse")({
+    status: Schema.Int,
+    headers: Schema.Record(Schema.String, Schema.String),
+    envelope: Schema.String,
+    elapsedMs: Schema.Int
+}) {
+}
+export class Exchange extends Schema.Class("Exchange")({
+    version: Schema.Literal(1),
+    operation: Schema.String,
+    name: Schema.String,
+    purpose: Schema.String,
+    provenance: Provenance,
+    recordedAt: Schema.String,
+    environment: Schema.optionalKey(Schema.String),
+    /** Endpoint without query string or credentials. */
+    endpoint: Schema.optionalKey(Schema.String),
+    request: Schema.optionalKey(RecordedMessage),
+    response: Schema.optionalKey(RecordedResponse),
+    fault: Schema.optionalKey(SoapFault),
+    requestIssues: Schema.Array(IssueRecord),
+    /** Where the response breaks its schema; findings, not errors. */
+    responseIssues: Schema.Array(IssueRecord),
+    masking: MaskingReport
+}) {
+}
+export class SampleError extends Schema.TaggedError()("SampleError", {
+    path: Schema.String,
+    detail: Schema.String
+}) {
+    get message() {
+        return `${this.path}: ${this.detail}`;
+    }
+}
+export const isSampleName = (name) => /^[a-z0-9][a-z0-9-]{0,62}$/.test(name);
+export const toSampleName = (raw) => raw
+    .normalize("NFKD")
+    .replace(/[^\w\s-]/g, "")
+    .trim()
+    .toLowerCase()
+    .replace(/[\s_]+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 63) || "sample";
+export const samplePaths = (service, operation, name) => {
+    const directory = `${serviceDirectory(service)}/samples/${operation}`;
+    return {
+        directory,
+        yaml: `${directory}/${name}.request.yaml`,
+        xml: `${directory}/${name}.request.xml`,
+        exchange: `${directory}/${name}.exchange.json`,
+        raw: `${serviceDirectory(service)}/raw/${operation}/${name}.response.xml`
+    };
+};
+// ---------------------------------------------------------------------------
+// Request files
+export const renderRequestFile = (options) => {
+    const operation = operationByName(options.catalog, options.operation);
+    const input = operation === undefined ? undefined : elementByName(options.catalog, operation.input);
+    const header = [
+        `# ${options.operation}: ${options.purpose}`,
+        '# Edit the values, then: llm4ts run soap-sample "call <operation>/<name>"',
+        "# Optional fields are commented out; uncomment to send them. `~` sends xsi:nil.",
+        ...(options.issues === undefined || options.issues.length === 0
+            ? []
+            : [
+                "#",
+                "# To fix before calling:",
+                ...options.issues.map((issue) => `#   ${issue.path}: ${issue.detail}`)
+            ])
+    ];
+    // A body that is not a non-empty map or list is written inline (`body: {}`),
+    // never as an indented block the reader would not accept.
+    const inline = options.body !== undefined &&
+        !((isYamlMap(options.body) && Object.keys(options.body).length > 0) ||
+            (Array.isArray(options.body) && options.body.length > 0));
+    const body = options.body !== undefined
+        ? inline
+            ? []
+            : renderYaml(options.body, 2).trimEnd().split("\n")
+        : input === undefined
+            ? []
+            : skeletonYaml(options.catalog, input, 2, options.seeds === undefined ? {} : { seeds: options.seeds });
+    const bodyKey = options.body !== undefined && inline ? `body: ${renderYaml(options.body).trim()}` : "body:";
+    return [
+        ...header,
+        `operation: ${options.operation}`,
+        `purpose: ${JSON.stringify(options.purpose)}`,
+        bodyKey,
+        ...body,
+        ""
+    ].join("\n");
+};
+export const readRequestFile = (workspace, catalog, service, operation, name) => Effect.gen(function* () {
+    const paths = samplePaths(service, operation, name);
+    const yaml = yield* readIfPresent(workspace, paths.yaml);
+    const xml = yield* readIfPresent(workspace, paths.xml);
+    if (yaml !== undefined && xml !== undefined) {
+        return yield* new SampleError({
+            path: paths.directory,
+            detail: `${name} has both a .yaml and an .xml request; keep one`
+        });
+    }
+    if (yaml !== undefined) {
+        const document = yield* parseYaml(yaml);
+        if (!isYamlMap(document) || document["body"] === undefined) {
+            return yield* new SampleError({
+                path: paths.yaml,
+                detail: "expected operation:, purpose:, and body:"
+            });
+        }
+        if (document["operation"] !== undefined && document["operation"] !== operation) {
+            return yield* new SampleError({
+                path: paths.yaml,
+                detail: `operation: says ${String(document["operation"])}, the file sits under ${operation}`
+            });
+        }
+        const purpose = typeof document["purpose"] === "string" ? document["purpose"] : name;
+        return { purpose, body: document["body"] ?? null, form: "yaml", readIssues: [] };
+    }
+    if (xml !== undefined) {
+        const declared = operationByName(catalog, operation);
+        const input = declared === undefined ? undefined : elementByName(catalog, declared.input);
+        if (input === undefined) {
+            return yield* new SampleError({
+                path: paths.xml,
+                detail: `${operation} is not in the catalog`
+            });
+        }
+        const root = yield* parseXml(xml, { source: paths.xml });
+        const envelope = yield* readEnvelope(root).pipe(Effect.option);
+        const bodyElement = envelope._tag === "Some" ? envelope.value.payload : root;
+        if (bodyElement === undefined) {
+            return yield* new SampleError({ path: paths.xml, detail: "the envelope Body is empty" });
+        }
+        const read = instanceFromXml(catalog, input, bodyElement);
+        const purpose = /<!--\s*purpose:\s*([\s\S]*?)-->/.exec(xml)?.[1]?.trim() ?? name;
+        return { purpose, body: read.value, form: "xml", readIssues: read.issues };
+    }
+    return yield* new SampleError({
+        path: paths.directory,
+        detail: `no request ${name}; create one with: soap-sample "init ${operation} ${name}"`
+    });
+});
+/** Request names under an operation, from either form. */
+export const listRequests = (workspace, service, operation) => Effect.map(workspace.discover(`${serviceDirectory(service)}/samples/${operation}/*.request.*`), (files) => [
+    ...new Set(files.map((file) => (file.split("/").pop() ?? "").replace(/\.request\.(yaml|xml)$/, "")))
+].sort());
+// ---------------------------------------------------------------------------
+// Exchanges
+export const encodeExchange = (exchange) => `${JSON.stringify(Schema.encodeSync(Exchange)(exchange), null, 2)}\n`;
+export const writeExchange = (workspace, service, exchange) => {
+    const path = samplePaths(service, exchange.operation, exchange.name).exchange;
+    return Effect.as(workspace.write(path, encodeExchange(exchange)), path);
+};
+export const readExchanges = (workspace, service, operation) => Effect.gen(function* () {
+    const pattern = `${serviceDirectory(service)}/samples/${operation ?? "*"}/*.exchange.json`;
+    const files = [...(yield* workspace.discover(pattern))].sort();
+    return yield* Effect.forEach(files, (file) => Effect.flatMap(workspace.read(file), (text) => Schema.decodeUnknownEffect(Schema.fromJsonString(Exchange))(text).pipe(Effect.mapError((error) => new SampleError({ path: file, detail: `not an exchange: ${error.message}` })))));
+});
+// ---------------------------------------------------------------------------
+// Masking key and overrides
+const keyPath = (service) => `${serviceDirectory(service)}/.masking-key`;
+/**
+ * The service's pseudonym key: 32 random bytes, created on first use and
+ * kept (gitignored) so pseudonyms stay stable across runs.
+ */
+export const loadMaskingKey = (workspace, service, random = (size) => new Uint8Array(randomBytes(size))) => Effect.gen(function* () {
+    const existing = yield* readIfPresent(workspace, keyPath(service));
+    if (existing !== undefined) {
+        const hex = existing.trim();
+        if (!/^[0-9a-f]{64}$/.test(hex)) {
+            return yield* new SampleError({ path: keyPath(service), detail: "not a 64-digit hex key" });
+        }
+        return Uint8Array.from(Buffer.from(hex, "hex"));
+    }
+    const key = random(32);
+    yield* workspace.write(keyPath(service), `${Buffer.from(key).toString("hex")}\n`);
+    yield* workspace.write(servicePaths(service).gitignore, ["auth.json", "raw/", ".masking-key", ""].join("\n"));
+    return key;
+});
+export const loadMaskingOverrides = (workspace, service) => {
+    const path = servicePaths(service).masking;
+    return Effect.flatMap(readIfPresent(workspace, path), (text) => text === undefined
+        ? Effect.succeed(undefined)
+        : Schema.decodeUnknownEffect(Schema.fromJsonString(MaskingOverrides))(text).pipe(Effect.mapError(() => new SampleError({
+            path,
+            detail: 'expected { "fields": { "<element>": "mask" | "keep" } }'
+        }))));
+};
+/** First value seen per field name across request bodies; SoapUI `${...}` expansions skipped. */
+export const collectSeeds = (bodies) => {
+    const seeds = new Map();
+    const collect = (value) => {
+        if (value === null || typeof value === "string")
+            return;
+        if (Array.isArray(value)) {
+            for (const item of value)
+                collect(item);
+            return;
+        }
+        if (!isYamlMap(value))
+            return;
+        for (const [key, item] of Object.entries(value)) {
+            if (typeof item === "string") {
+                if (!seeds.has(key) && !item.includes("${") && item !== "?")
+                    seeds.set(key, item);
+            }
+            else
+                collect(item);
+        }
+    };
+    for (const body of bodies)
+        collect(body);
+    return seeds;
+};
+/** Request bodies of recorded exchanges (already masked). */
+export const bodiesFromExchanges = (exchanges, catalog) => Effect.gen(function* () {
+    const bodies = [];
+    for (const exchange of exchanges) {
+        if (exchange.request === undefined)
+            continue;
+        const operation = operationByName(catalog, exchange.operation);
+        const input = operation === undefined ? undefined : elementByName(catalog, operation.input);
+        if (input === undefined)
+            continue;
+        const parsed = yield* parseXml(exchange.request.envelope).pipe(Effect.option);
+        if (parsed._tag === "None")
+            continue;
+        const envelope = yield* readEnvelope(parsed.value).pipe(Effect.option);
+        const payload = envelope._tag === "Some" ? envelope.value.payload : undefined;
+        if (payload !== undefined)
+            bodies.push(instanceFromXml(catalog, input, payload).value);
+    }
+    return bodies;
+});
+/** Values seen in earlier requests, by field name, to seed new skeletons. */
+export const seedsFromExchanges = (exchanges, catalog) => Effect.map(bodiesFromExchanges(exchanges, catalog), collectSeeds);
+/**
+ * Seeds from everything on disk for the service: recorded exchanges first
+ * (values the service answered to), then request files (imported ones are
+ * masked). Unreadable request files are skipped.
+ */
+export const seedsForService = (workspace, catalog, service) => Effect.gen(function* () {
+    const fromExchanges = yield* bodiesFromExchanges(yield* readExchanges(workspace, service), catalog);
+    const fromFiles = [];
+    for (const operation of catalog.operations) {
+        for (const name of yield* listRequests(workspace, service, operation.name)) {
+            const request = yield* readRequestFile(workspace, catalog, service, operation.name, name).pipe(Effect.option);
+            if (request._tag === "Some")
+                fromFiles.push(request.value.body);
+        }
+    }
+    return collectSeeds([...fromExchanges, ...fromFiles]);
+});
+export const issueRecords = (issues) => issues.map((issue) => new IssueRecord(issue));