@obov/mext 0.5.0 → 0.6.0

package/dist/inspect/scenario.js

@@ -0,0 +1,555 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+import { CallError } from "../client/typed.js";
+import { AGENT_TIMEOUT_MS, checkAudit, connectAgent, instructAgent } from "./agent.js";
+import { isRecord, partial } from "./compare.js";
+import { orderedBody, outcomeOf } from "./inspector.js";
+import { findProfile, INSPECT_DIR, ProfileUnavailableError } from "./profiles.js";
+/**
+ * 시나리오 (specs/inspector.md §4): 호출 순서 + 기대 결과를 적은 JSON 파일. 사람 · CLI · 웹 UI · 에이전트가 같은 파일을 쓴다.
+ * 코드 실행 없음 (값 비교만)
+ */
+/** <프로젝트>/inspect/scenarios */
+export const SCENARIOS_DIR = "scenarios";
+/** agent 단계 결과의 as */
+export const AGENT_AS = "agent";
+/** 시나리오 파일 형식 오류. issues = "<위치>: <문제>" */
+export class ScenarioFileError extends Error {
+    file;
+    issues;
+    constructor(file, issues) {
+        super(`${file}: invalid scenario file\n${issues.map((i) => `  ${i}`).join("\n")}`);
+        this.name = "ScenarioFileError";
+        this.file = file;
+        this.issues = issues;
+    }
+}
+/** 경로 · 폴더 → 시나리오 파일 목록 (폴더 = 그 안의 *.json, 이름 순). 경로 없음 = <cwd>/inspect/scenarios */
+export function scenarioFiles(cwd, paths) {
+    const targets = paths.length > 0 ? paths.map((p) => resolve(cwd, p)) : [join(resolve(cwd), INSPECT_DIR, SCENARIOS_DIR)];
+    const files = [];
+    for (const target of targets) {
+        if (!existsSync(target))
+            throw new ScenarioFileError(target, ["file not found"]);
+        if (!statSync(target).isDirectory()) {
+            files.push(target);
+            continue;
+        }
+        const found = readdirSync(target)
+            .filter((name) => name.endsWith(".json"))
+            .sort()
+            .map((name) => join(target, name));
+        if (found.length === 0)
+            throw new ScenarioFileError(target, ["no *.json scenario files in this folder"]);
+        files.push(...found);
+    }
+    return files;
+}
+/** 파일 읽기 + 검증. profiles 를 주면 없는 프로필 이름도 오류 */
+export function loadScenario(file, profiles) {
+    let value;
+    try {
+        value = JSON.parse(readFileSync(file, "utf8"));
+    }
+    catch (error) {
+        throw new ScenarioFileError(file, [`not valid JSON (${error instanceof Error ? error.message : String(error)})`]);
+    }
+    return parseScenario(value, file, profiles);
+}
+const SCENARIO_KEYS = ["name", "description", "as", "steps"];
+const CALL_KEYS = ["name", "as", "call", "input", "confirmed", "answers", "decline", "expect", "save"];
+const COMMANDS_KEYS = ["name", "as", "commands"];
+const AGENT_KEYS = ["name", "agent", "from", "expect", "timeoutMs"];
+/** 시나리오 내용 검증 (문제를 모두 모아 한 번에) */
+export function parseScenario(value, file, profiles) {
+    if (!isRecord(value))
+        throw new ScenarioFileError(file, ['must be an object ({ "name": …, "steps": [ … ] })']);
+    const issues = [];
+    const unknownKeys = (at, object, allowed) => {
+        for (const key of Object.keys(object))
+            if (!allowed.includes(key))
+                issues.push(`${at}${key}: unknown key`);
+    };
+    const profileName = (at, as) => {
+        if (as === undefined)
+            return undefined;
+        if (typeof as !== "string" || as === "") {
+            issues.push(`${at}: must be a profile name`);
+            return undefined;
+        }
+        if (profiles && !findProfile(profiles, as)) {
+            issues.push(`${at}: unknown profile "${as}" (profiles: ${profiles.profiles.map((p) => p.name).join(", ")})`);
+        }
+        return as;
+    };
+    /** 감사 이벤트의 일부 목록 */
+    const eventPatterns = (at, list) => {
+        if (list === undefined)
+            return undefined;
+        if (!Array.isArray(list) || !list.every(isRecord)) {
+            issues.push(`${at}: must be an array of objects (parts of audit events)`);
+            return undefined;
+        }
+        return list;
+    };
+    unknownKeys("", value, SCENARIO_KEYS);
+    const scenario = { file, name: basename(file, ".json"), steps: [] };
+    if (value.name !== undefined) {
+        if (typeof value.name !== "string" || value.name === "")
+            issues.push("name: must be a non-empty string");
+        else
+            scenario.name = value.name;
+    }
+    if (value.description !== undefined) {
+        if (typeof value.description !== "string")
+            issues.push("description: must be a string");
+        else
+            scenario.description = value.description;
+    }
+    const as = profileName("as", value.as);
+    if (as !== undefined)
+        scenario.as = as;
+    if (!Array.isArray(value.steps) || value.steps.length === 0) {
+        issues.push("steps: must be a non-empty array");
+    }
+    else {
+        value.steps.forEach((raw, i) => {
+            const at = `steps[${i + 1}]`;
+            if (!isRecord(raw))
+                return void issues.push(`${at}: must be an object`);
+            const hasCall = raw.call !== undefined;
+            const hasCommands = raw.commands !== undefined;
+            const hasAgent = raw.agent !== undefined;
+            if (hasCall && hasCommands)
+                return void issues.push(`${at}: use either "call" or "commands", not both`);
+            if (hasAgent && (hasCall || hasCommands))
+                return void issues.push(`${at}: "agent" cannot be combined with "call" or "commands"`);
+            if (!hasCall && !hasCommands && !hasAgent)
+                return void issues.push(`${at}: needs "call" (a command ID), "commands", or "agent" (an instruction)`);
+            const before = issues.length;
+            if (hasAgent) {
+                unknownKeys(`${at}.`, raw, AGENT_KEYS);
+                const step = { agent: "" };
+                if (raw.name !== undefined) {
+                    if (typeof raw.name !== "string")
+                        issues.push(`${at}.name: must be a string`);
+                    else
+                        step.name = raw.name;
+                }
+                if (typeof raw.agent !== "string" || raw.agent.trim() === "")
+                    issues.push(`${at}.agent: must be the instruction text`);
+                else
+                    step.agent = raw.agent;
+                if (raw.from !== undefined) {
+                    if (!isRecord(raw.from))
+                        issues.push(`${at}.from: must be an object (part of an audit event)`);
+                    else
+                        step.from = raw.from;
+                }
+                if (raw.timeoutMs !== undefined) {
+                    if (typeof raw.timeoutMs !== "number" || !Number.isFinite(raw.timeoutMs) || raw.timeoutMs <= 0)
+                        issues.push(`${at}.timeoutMs: must be a positive number`);
+                    else
+                        step.timeoutMs = raw.timeoutMs;
+                }
+                if (raw.expect !== undefined) {
+                    const e = raw.expect;
+                    if (!isRecord(e))
+                        issues.push(`${at}.expect: must be { "audit"?: [ … ], "none"?: [ … ] }`);
+                    else {
+                        unknownKeys(`${at}.expect.`, e, ["audit", "none"]);
+                        const expect = {};
+                        for (const key of ["audit", "none"]) {
+                            const list = eventPatterns(`${at}.expect.${key}`, e[key]);
+                            if (list)
+                                expect[key] = list;
+                        }
+                        step.expect = expect;
+                    }
+                }
+                if (issues.length === before)
+                    scenario.steps.push(step);
+                return;
+            }
+            unknownKeys(`${at}.`, raw, hasCall ? CALL_KEYS : COMMANDS_KEYS);
+            if (raw.name !== undefined && typeof raw.name !== "string")
+                issues.push(`${at}.name: must be a string`);
+            const stepAs = profileName(`${at}.as`, raw.as);
+            const common = { ...(typeof raw.name === "string" ? { name: raw.name } : {}), ...(stepAs !== undefined ? { as: stepAs } : {}) };
+            if (hasCommands) {
+                const c = raw.commands;
+                const list = (key) => {
+                    const v = isRecord(c) ? c[key] : undefined;
+                    if (v === undefined)
+                        return undefined;
+                    if (!Array.isArray(v) || !v.every((x) => typeof x === "string")) {
+                        issues.push(`${at}.commands.${key}: must be an array of command IDs`);
+                        return undefined;
+                    }
+                    return v;
+                };
+                if (!isRecord(c))
+                    issues.push(`${at}.commands: must be { "visible"?: [ids], "hidden"?: [ids] }`);
+                else {
+                    unknownKeys(`${at}.commands.`, c, ["visible", "hidden"]);
+                    if (c.visible === undefined && c.hidden === undefined)
+                        issues.push(`${at}.commands: needs "visible" or "hidden"`);
+                }
+                const visible = list("visible");
+                const hidden = list("hidden");
+                if (issues.length === before) {
+                    scenario.steps.push({ ...common, commands: { ...(visible ? { visible } : {}), ...(hidden ? { hidden } : {}) } });
+                }
+                return;
+            }
+            const step = { ...common, call: "" };
+            if (typeof raw.call !== "string" || raw.call === "")
+                issues.push(`${at}.call: must be a command ID`);
+            else
+                step.call = raw.call;
+            if (raw.input !== undefined) {
+                if (!isRecord(raw.input))
+                    issues.push(`${at}.input: must be an object`);
+                else
+                    step.input = raw.input;
+            }
+            for (const key of ["confirmed", "decline"]) {
+                if (raw[key] === undefined)
+                    continue;
+                if (typeof raw[key] !== "boolean")
+                    issues.push(`${at}.${key}: must be true or false`);
+                else if (raw[key])
+                    step[key] = true;
+            }
+            if (raw.answers !== undefined) {
+                const a = raw.answers;
+                if (!isRecord(a))
+                    issues.push(`${at}.answers: must be { "confirm"?: boolean, "form"?: object, "url"?: boolean }`);
+                else {
+                    unknownKeys(`${at}.answers.`, a, ["confirm", "form", "url"]);
+                    const answers = {};
+                    if (a.confirm !== undefined) {
+                        if (typeof a.confirm !== "boolean")
+                            issues.push(`${at}.answers.confirm: must be true or false`);
+                        else
+                            answers.confirm = a.confirm;
+                    }
+                    if (a.url !== undefined) {
+                        if (typeof a.url !== "boolean")
+                            issues.push(`${at}.answers.url: must be true or false`);
+                        else
+                            answers.url = a.url;
+                    }
+                    if (a.form !== undefined) {
+                        if (a.form !== false && !isRecord(a.form))
+                            issues.push(`${at}.answers.form: must be an object (the values to send) or false`);
+                        else
+                            answers.form = a.form;
+                    }
+                    step.answers = answers;
+                }
+            }
+            if (raw.expect !== undefined) {
+                const e = raw.expect;
+                if (!isRecord(e))
+                    issues.push(`${at}.expect: must be { "ok"?: boolean, "value"?: …, "error"?: code | { … } }`);
+                else {
+                    unknownKeys(`${at}.expect.`, e, ["ok", "value", "error", "audit"]);
+                    const expect = {};
+                    const audit = eventPatterns(`${at}.expect.audit`, e.audit);
+                    if (audit)
+                        expect.audit = audit;
+                    if (e.ok !== undefined) {
+                        if (typeof e.ok !== "boolean")
+                            issues.push(`${at}.expect.ok: must be true or false`);
+                        else
+                            expect.ok = e.ok;
+                    }
+                    if (e.value !== undefined)
+                        expect.value = e.value;
+                    if (e.error !== undefined) {
+                        if (typeof e.error !== "string" && !isRecord(e.error))
+                            issues.push(`${at}.expect.error: must be an error code or an object`);
+                        else
+                            expect.error = e.error;
+                    }
+                    if (e.error !== undefined && (e.ok === true || e.value !== undefined)) {
+                        issues.push(`${at}.expect: "error" cannot be combined with "ok": true or "value"`);
+                    }
+                    if (e.ok === false && e.value !== undefined)
+                        issues.push(`${at}.expect: "value" cannot be combined with "ok": false`);
+                    step.expect = expect;
+                }
+            }
+            if (raw.save !== undefined) {
+                if (!isRecord(raw.save) || !Object.values(raw.save).every((v) => typeof v === "string")) {
+                    issues.push(`${at}.save: must be { "<variable>": "<path in the result>" }`);
+                }
+                else
+                    step.save = raw.save;
+            }
+            if (issues.length === before)
+                scenario.steps.push(step);
+        });
+    }
+    if (issues.length > 0)
+        throw new ScenarioFileError(file, issues);
+    return scenario;
+}
+/** 단계 순서대로 실행. 실패한 단계가 있어도 끝까지 (bail 제외) */
+export async function runScenario(inspector, scenario, options = {}) {
+    const started = Date.now();
+    const vars = new Map();
+    const steps = [];
+    for (const [i, step] of scenario.steps.entries()) {
+        if (options.signal?.aborted)
+            break;
+        const as = "agent" in step ? AGENT_AS : inspector.profile(step.as ?? scenario.as).name;
+        const name = step.name ?? defaultName(step, as);
+        options.onStepStart?.({ step: i + 1, name, as });
+        const result = "agent" in step
+            ? await runAgent(inspector, step, vars, options)
+            : "call" in step
+                ? await runCall(inspector, step, as, vars, options.signal)
+                : await runCommands(inspector, step, as);
+        const full = { step: i + 1, name, as, ...result, pass: result.failures.length === 0 };
+        steps.push(full);
+        options.onStep?.(full);
+        if (!full.pass && options.bail === true)
+            break;
+    }
+    const result = {
+        name: scenario.name,
+        pass: steps.length === scenario.steps.length && steps.every((s) => s.pass),
+        durationMs: Date.now() - started,
+        steps,
+    };
+    if (scenario.file !== undefined)
+        result.file = scenario.file;
+    return result;
+}
+async function runCall(inspector, step, as, vars, signal) {
+    const failures = [];
+    const missing = [];
+    const input = substitute(step.input ?? {}, vars, missing);
+    if (missing.length > 0) {
+        for (const name of missing)
+            failures.push({ path: "input", expected: `{{${name}}}`, actual: undefined });
+        return { durationMs: 0, outcome: undefined, failures };
+    }
+    const record = await inspector.call(step.call, input, {
+        profile: as,
+        ...(step.confirmed ? { confirmed: true } : {}),
+        ...(step.answers ? { answers: step.answers } : {}),
+        ...(step.decline ? { decline: true } : {}),
+        ...(signal ? { signal } : {}),
+    });
+    const outcome = outcomeOf(record);
+    compare(step.expect, outcome, failures);
+    if (step.expect?.audit) {
+        if (!inspector.auditConnected)
+            failures.push({ path: "audit", expected: "an audit source", actual: AUDIT_UNAVAILABLE });
+        else
+            partial(step.expect.audit, await auditOf(inspector, record, step.expect.audit.length, signal), "audit", failures);
+    }
+    for (const [name, path] of Object.entries(step.save ?? {})) {
+        const value = outcome.ok ? valueAt(outcome.value, path) : undefined;
+        if (value === undefined)
+            failures.push({ path: `save.${name}`, expected: path, actual: undefined });
+        else
+            vars.set(name, value);
+    }
+    return { durationMs: record.durationMs, outcome, failures, record: record.id };
+}
+const AUDIT_UNAVAILABLE = "AUDIT_UNAVAILABLE: audit events are not available for this target (run against the mext dev server of this project, or pass an audit source to createInspector)";
+const AGENT_UNAVAILABLE = "AGENT_UNAVAILABLE: no agent client (use mext inspect run --agent <module>, or runScenario(…, { agent }))";
+/** 호출의 감사 이벤트가 응답보다 늦게 올 때 기다리는 시간 */
+const CALL_AUDIT_WAIT_MS = 2_000;
+/** 이 호출의 감사 이벤트 (trace ID 가 같은 것). 기대한 수만큼 올 때까지 잠깐 기다린다 */
+async function auditOf(inspector, record, count, signal) {
+    const read = () => inspector.auditEvents().filter((event) => event.traceId === record.traceId);
+    const deadline = Date.now() + CALL_AUDIT_WAIT_MS;
+    while (read().length < count && Date.now() < deadline && signal?.aborted !== true)
+        await new Promise((resolve) => setTimeout(resolve, 20));
+    return read();
+}
+async function runAgent(inspector, step, vars, options) {
+    const started = Date.now();
+    const failures = [];
+    const missing = [];
+    const text = substitute(step.agent, vars, missing);
+    for (const name of missing)
+        failures.push({ path: "agent", expected: `{{${name}}}`, actual: undefined });
+    const agent = options.agent;
+    if (!agent)
+        failures.push({ path: "agent", expected: "an agent client", actual: AGENT_UNAVAILABLE });
+    if (!inspector.auditConnected)
+        failures.push({ path: "audit", expected: "an audit source", actual: AUDIT_UNAVAILABLE });
+    if (failures.length > 0 || !agent)
+        return { durationMs: 0, outcome: undefined, failures };
+    try {
+        await connectAgent(agent, options.signal);
+    }
+    catch (error) {
+        failures.push({ path: "agent", expected: "connected", actual: error instanceof Error ? error.message : String(error) });
+        return { durationMs: Date.now() - started, outcome: undefined, failures };
+    }
+    const observation = await instructAgent(inspector, agent, {
+        text,
+        timeoutMs: step.timeoutMs ?? AGENT_TIMEOUT_MS,
+        ...(step.from ? { from: step.from } : {}),
+        ...(options.agentSettleMs !== undefined ? { settleMs: options.agentSettleMs } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+    });
+    if (!observation.ok)
+        failures.push({ path: "agent", expected: "idle", actual: observation.problem });
+    checkAudit(step.expect, observation.events, failures);
+    return { durationMs: Date.now() - started, outcome: observation, failures };
+}
+async function runCommands(inspector, step, as) {
+    const started = Date.now();
+    const failures = [];
+    let ids;
+    try {
+        ids = (await inspector.commands(as)).commands.map((c) => c.id);
+    }
+    catch (error) {
+        const body = listErrorBody(error);
+        failures.push({ path: "commands", expected: "a command list", actual: body });
+        return { durationMs: Date.now() - started, outcome: { ok: false, error: body }, failures };
+    }
+    const notVisible = (step.commands.visible ?? []).filter((id) => !ids.includes(id));
+    const notHidden = (step.commands.hidden ?? []).filter((id) => ids.includes(id));
+    if (notVisible.length > 0)
+        failures.push({ path: "commands.visible", expected: notVisible, actual: "hidden" });
+    if (notHidden.length > 0)
+        failures.push({ path: "commands.hidden", expected: notHidden, actual: "visible" });
+    return { durationMs: Date.now() - started, outcome: { ok: true, commands: ids }, failures };
+}
+/** 목록 조회 실패 → 오류 본문 */
+export function listErrorBody(error) {
+    if (error instanceof CallError)
+        return orderedBody(error);
+    if (error instanceof ProfileUnavailableError) {
+        return { code: error.code, message: error.message, profile: error.profile, missing: [...error.missing] };
+    }
+    return { code: "CLIENT_ERROR", message: error instanceof Error ? error.message : String(error) };
+}
+function defaultName(step, as) {
+    if ("agent" in step)
+        return `agent: ${step.agent.length > 60 ? `${step.agent.slice(0, 57)}…` : step.agent}`;
+    return "call" in step ? `${step.call} as ${as}` : `commands as ${as}`;
+}
+/** 기대 결과 비교 (expect 없음 = 통과) */
+export function compare(expect, outcome, failures) {
+    if (!expect)
+        return;
+    const wantsError = expect.error !== undefined || expect.ok === false;
+    const wantsOk = expect.ok === true || expect.value !== undefined;
+    if (wantsOk && !outcome.ok) {
+        failures.push({ path: "ok", expected: true, actual: false });
+        return;
+    }
+    if (wantsError && outcome.ok) {
+        failures.push({ path: "ok", expected: false, actual: true });
+        return;
+    }
+    if (outcome.ok) {
+        if (expect.value !== undefined)
+            partial(expect.value, outcome.value, "value", failures);
+        return;
+    }
+    if (typeof expect.error === "string") {
+        if (outcome.error.code !== expect.error)
+            failures.push({ path: "error.code", expected: expect.error, actual: outcome.error.code });
+    }
+    else if (expect.error !== undefined)
+        partial(expect.error, outcome.error, "error", failures);
+}
+const WHOLE = /^\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}$/;
+const PART = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
+/** {{변수}}: 문자열 전체 = 값 그대로 (타입 유지) · 일부 = 문자열로. 없는 변수 = missing 에 이름 */
+function substitute(value, vars, missing) {
+    if (typeof value === "string") {
+        const whole = WHOLE.exec(value);
+        const lookup = (name) => {
+            if (!vars.has(name) && !missing.includes(name))
+                missing.push(name);
+            return vars.get(name);
+        };
+        if (whole)
+            return lookup(whole[1]);
+        return value.replace(PART, (_, name) => {
+            const v = lookup(name);
+            return typeof v === "string" ? v : JSON.stringify(v ?? null);
+        });
+    }
+    if (Array.isArray(value))
+        return value.map((v) => substitute(v, vars, missing));
+    if (isRecord(value))
+        return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, substitute(v, vars, missing)]));
+    return value;
+}
+/** 점 구분 경로 (items.0.id) */
+function valueAt(value, path) {
+    let current = value;
+    for (const key of path === "" ? [] : path.split(".")) {
+        if (Array.isArray(current))
+            current = current[Number(key)];
+        else if (isRecord(current))
+            current = current[key];
+        else
+            return undefined;
+    }
+    return current;
+}
+/** 호출 기록 → 시나리오 단계 (관찰한 결과를 기대 결과로). 오류 = code · reason 만 */
+export function stepFromRecord(record, options = {}) {
+    const step = { as: record.profile, call: record.command };
+    if (Object.keys(record.input).length > 0)
+        step.input = record.input;
+    if (record.confirmed)
+        step.confirmed = true;
+    const answers = options.answers ?? (options.decline ? undefined : answersOf(record));
+    if (answers)
+        step.answers = answers;
+    if (options.decline)
+        step.decline = true;
+    if (record.ok)
+        step.expect = record.value === undefined ? { ok: true } : { ok: true, value: record.value };
+    else {
+        const error = record.error;
+        step.expect = { error: typeof error?.reason === "string" ? { code: error.code, reason: error.reason } : (error?.code ?? "UNKNOWN") };
+    }
+    return step;
+}
+/** 기록의 입력 요청에 보낸 답 → 시나리오 answers (종류별 마지막 답). 답한 요청이 없으면 undefined */
+function answersOf(record) {
+    const answers = {};
+    for (const { kind, answer } of record.inputs) {
+        if (answer === undefined)
+            continue;
+        const accepted = answer === true || (typeof answer === "object" && answer.action === "accept");
+        if (kind === "form")
+            answers.form = accepted ? ((typeof answer === "object" && "content" in answer ? answer.content : undefined) ?? {}) : false;
+        else
+            answers[kind] = accepted;
+    }
+    return Object.keys(answers).length > 0 ? answers : undefined;
+}
+/** 시나리오 파일 끝에 단계를 덧붙인다 (파일이 없으면 만든다). 반환 = 단계 수 */
+export function appendStep(file, step) {
+    let content;
+    if (existsSync(file)) {
+        loadScenario(file); // 형식 확인
+        content = JSON.parse(readFileSync(file, "utf8"));
+        content.steps = [...content.steps, step];
+    }
+    else {
+        mkdirSync(dirname(file), { recursive: true });
+        content = { name: basename(file, ".json"), steps: [step] };
+    }
+    writeFileSync(file, `${JSON.stringify(content, null, 2)}\n`);
+    return content.steps.length;
+}
+//# sourceMappingURL=scenario.js.map