@testmuai/evidence-cli 0.1.1

package/dist/validate/l1.js

@@ -0,0 +1,117 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runL1Checks = runL1Checks;
+const contract_1 = require("../contract");
+const diagnostics_1 = require("../diagnostics");
+const compile_1 = require("../schemas/compile");
+const yaml_1 = require("../yaml");
+const checks_1 = require("./checks");
+// The L1 artifact layer (decision 0040). Status-gated like L0: artifact PRESENCE
+// (logs/, steps/, coverage/) is required only when finalized; shape checks on
+// whatever artifacts ARE present run at any status.
+async function runL1Checks(input) {
+    const diags = [];
+    const finalized = input.status === "finalized";
+    const { logsMeta, video } = (0, compile_1.loadL1Schemas)(input.version);
+    if (finalized && !(await input.container.isDir("coverage"))) {
+        diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_COVERAGE_MISSING, "coverage/", "L1 requires a global coverage/ directory"));
+    }
+    for (const { testId, result } of input.tests) {
+        const base = `tests/${testId}`;
+        await checkLogs(input.container, diags, base, finalized, logsMeta);
+        await checkSteps(input.container, diags, base, finalized, result);
+        await checkVideo(input.container, diags, base, video);
+    }
+    return diags;
+}
+async function checkLogs(c, diags, base, finalized, logsMeta) {
+    const dir = `${base}/logs`;
+    if (!(await c.isDir(dir))) {
+        if (finalized)
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_LOGS_MISSING, `${dir}/`, "L1 requires a logs/ directory"));
+        return;
+    }
+    const metaRel = `${dir}/meta.yaml`;
+    const raw = await c.readText(metaRel);
+    if (raw == null) {
+        if (finalized)
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_LOGS_META_MISSING, metaRel, "logs/ is present but logs/meta.yaml is missing"));
+        return;
+    }
+    let meta;
+    try {
+        meta = (0, yaml_1.parseYaml)(raw);
+    }
+    catch (e) {
+        diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_LOGS_META_INVALID, metaRel, `logs/meta.yaml is not valid YAML: ${e?.message ?? e}`));
+        return;
+    }
+    if (!logsMeta(meta)) {
+        for (const e of logsMeta.errors ?? []) {
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_LOGS_META_INVALID, `${metaRel}${e.instancePath || ""}`, `${e.instancePath || "/"} ${e.message ?? "schema violation"}`));
+        }
+        return; // can't trust file declarations if the shape is wrong
+    }
+    for (const entry of meta.logs) {
+        const file = entry?.file;
+        if (typeof file !== "string")
+            continue;
+        if (!(0, checks_1.isContained)(file)) {
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_LOG_PATH_ESCAPE, metaRel, `declared log file "${file}" escapes logs/`));
+            continue;
+        }
+        if (!(await c.exists(`${dir}/${file}`))) {
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_LOG_FILE_MISSING, `${dir}/${file}`, `declared log file "${file}" does not exist`));
+        }
+    }
+}
+async function checkSteps(c, diags, base, finalized, result) {
+    const dir = `${base}/steps`;
+    if (!(await c.isDir(dir))) {
+        if (finalized)
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_STEPS_MISSING, `${dir}/`, "L1 requires a steps/ directory"));
+        return;
+    }
+    const steps = Array.isArray(result?.steps) ? result.steps : [];
+    const stepKeys = new Set(steps.map((s) => `${s?.ordinal}-${s?.id}`));
+    for (const entry of await c.listDir(dir)) {
+        if (!entry.isDir)
+            continue;
+        const m = entry.name.match(/^(\d+)-(.+)$/);
+        if (!m) {
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_STEPS_MALFORMED, `${dir}/${entry.name}`, `step folder "${entry.name}" is not <ordinal>-<id>`));
+            continue;
+        }
+        if (!stepKeys.has(`${Number(m[1])}-${m[2]}`)) {
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_STEPS_UNMATCHED, `${dir}/${entry.name}`, `step folder "${entry.name}" matches no result.yaml step`));
+            continue;
+        }
+        const files = await c.listDir(`${dir}/${entry.name}`);
+        const hasShot = files.some((f) => !f.isDir && /^screenshot\.[^.]+$/i.test(f.name));
+        if (!hasShot) {
+            // Advisory only (decision 0040): not every framework captures a frame per
+            // step; a missing screenshot never fails the pack.
+            diags.push((0, diagnostics_1.warn)(contract_1.Codes.L1_STEPS_SCREENSHOT_MISSING, `${dir}/${entry.name}`, `step folder "${entry.name}" has no screenshot.<ext>`));
+        }
+    }
+}
+async function checkVideo(c, diags, base, video) {
+    const rel = `${base}/video.yaml`;
+    const raw = await c.readText(rel);
+    if (raw == null)
+        return; // optional; an inline video.<ext> is opaque
+    let v;
+    try {
+        v = (0, yaml_1.parseYaml)(raw);
+    }
+    catch (e) {
+        diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_VIDEO_INVALID, rel, `video.yaml is not valid YAML: ${e?.message ?? e}`));
+        return;
+    }
+    if (!video(v)) {
+        for (const e of video.errors ?? []) {
+            diags.push((0, diagnostics_1.err)(contract_1.Codes.L1_VIDEO_INVALID, `${rel}${e.instancePath || ""}`, `${e.instancePath || "/"} ${e.message ?? "schema violation"}`));
+        }
+    }
+}
+//# sourceMappingURL=l1.js.map

package/dist/validate/l1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"l1.js","sourceRoot":"","sources":["../../src/validate/l1.ts"],"names":[],"mappings":";;AAmBA,kCAgBC;AAnCD,0CAAoC;AAEpC,gDAA2C;AAE3C,gDAAmD;AACnD,kCAAoC;AACpC,qCAAuC;AAUvC,iFAAiF;AACjF,8EAA8E;AAC9E,oDAAoD;AAC7C,KAAK,UAAU,WAAW,CAAC,KAAc;IAC9C,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC;IAC/C,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAA,uBAAa,EAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAEzD,IAAI,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,mBAAmB,EAAE,WAAW,EAAE,0CAA0C,CAAC,CAAC,CAAC;IACtG,CAAC;IAED,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,SAAS,MAAM,EAAE,CAAC;QAC/B,MAAM,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QACnE,MAAM,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,CAAgB,EAChB,KAAmB,EACnB,IAAY,EACZ,SAAkB,EAClB,QAA0B;IAE1B,MAAM,GAAG,GAAG,GAAG,IAAI,OAAO,CAAC;IAC3B,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC1B,IAAI,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,eAAe,EAAE,GAAG,GAAG,GAAG,EAAE,+BAA+B,CAAC,CAAC,CAAC;QAClG,OAAO;IACT,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,GAAG,YAAY,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QAChB,IAAI,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,oBAAoB,EAAE,OAAO,EAAE,gDAAgD,CAAC,CAAC,CAAC;QACtH,OAAO;IACT,CAAC;IACD,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAA,gBAAS,EAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,oBAAoB,EAAE,OAAO,EAAE,qCAAqC,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7G,OAAO;IACT,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpB,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,oBAAoB,EAAE,GAAG,OAAO,GAAG,CAAC,CAAC,YAAY,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,YAAY,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,IAAI,kBAAkB,EAAE,CAAC,CAAC,CAAC;QAClJ,CAAC;QACD,OAAO,CAAC,sDAAsD;IAChE,CAAC;IACD,KAAK,MAAM,KAAK,IAAK,IAAY,CAAC,IAAa,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC;QACzB,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,SAAS;QACvC,IAAI,CAAC,IAAA,oBAAW,EAAC,IAAI,CAAC,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,kBAAkB,EAAE,OAAO,EAAE,sBAAsB,IAAI,iBAAiB,CAAC,CAAC,CAAC;YAChG,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,mBAAmB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,sBAAsB,IAAI,kBAAkB,CAAC,CAAC,CAAC;QAC7G,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,CAAgB,EAChB,KAAmB,EACnB,IAAY,EACZ,SAAkB,EAClB,MAAW;IAEX,MAAM,GAAG,GAAG,GAAG,IAAI,QAAQ,CAAC;IAC5B,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC1B,IAAI,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,gBAAgB,EAAE,GAAG,GAAG,GAAG,EAAE,gCAAgC,CAAC,CAAC,CAAC;QACpG,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1E,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,KAAK;YAAE,SAAS;QAC3B,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC3C,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,kBAAkB,EAAE,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,KAAK,CAAC,IAAI,yBAAyB,CAAC,CAAC,CAAC;YACvH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,kBAAkB,EAAE,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,KAAK,CAAC,IAAI,+BAA+B,CAAC,CAAC,CAAC;YAC7H,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,0EAA0E;YAC1E,mDAAmD;YACnD,KAAK,CAAC,IAAI,CAAC,IAAA,kBAAI,EAAC,gBAAK,CAAC,2BAA2B,EAAE,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,KAAK,CAAC,IAAI,2BAA2B,CAAC,CAAC,CAAC;QACrI,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,CAAgB,EAChB,KAAmB,EACnB,IAAY,EACZ,KAAuB;IAEvB,MAAM,GAAG,GAAG,GAAG,IAAI,aAAa,CAAC;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,CAAC,4CAA4C;IACrE,IAAI,CAAM,CAAC;IACX,IAAI,CAAC;QACH,CAAC,GAAG,IAAA,gBAAS,EAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,gBAAgB,EAAE,GAAG,EAAE,iCAAiC,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACjG,OAAO;IACT,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,gBAAgB,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC,YAAY,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,YAAY,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,IAAI,kBAAkB,EAAE,CAAC,CAAC,CAAC;QAC1I,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/yaml.d.ts

@@ -0,0 +1,4 @@
+import type { Document } from "yaml";
+export declare function parseYaml(raw: string): unknown;
+export declare function parseDoc(raw: string): Document.Parsed;
+export declare function stringifyDoc(doc: Document.Parsed): string;

package/dist/yaml.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseYaml = parseYaml;
+exports.parseDoc = parseDoc;
+exports.stringifyDoc = stringifyDoc;
+const yaml_1 = require("yaml");
+function parseYaml(raw) {
+    return (0, yaml_1.parse)(raw);
+}
+function parseDoc(raw) {
+    return (0, yaml_1.parseDocument)(raw);
+}
+function stringifyDoc(doc) {
+    return doc.toString();
+}
+//# sourceMappingURL=yaml.js.map

package/dist/yaml.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"yaml.js","sourceRoot":"","sources":["../src/yaml.ts"],"names":[],"mappings":";;AAGA,8BAEC;AAED,4BAEC;AAED,oCAEC;AAbD,+BAA4C;AAG5C,SAAgB,SAAS,CAAC,GAAW;IACnC,OAAO,IAAA,YAAK,EAAC,GAAG,CAAC,CAAC;AACpB,CAAC;AAED,SAAgB,QAAQ,CAAC,GAAW;IAClC,OAAO,IAAA,oBAAa,EAAC,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,SAAgB,YAAY,CAAC,GAAoB;IAC/C,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC"}

package/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "@testmuai/evidence-cli",
+  "version": "0.1.1",
+  "description": "An open, framework-agnostic format for what a test run produced — the .evidence pack, and the library + CLI that validate and seal it.",
+  "license": "Apache-2.0",
+  "author": "TestMu AI (formerly LambdaTest)",
+  "homepage": "https://github.com/LambdaTest/evidence-cli#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/LambdaTest/evidence-cli.git"
+  },
+  "bugs": {
+    "url": "https://github.com/LambdaTest/evidence-cli/issues"
+  },
+  "keywords": [
+    "evidence",
+    "test-results",
+    "test-report",
+    "conformance",
+    "json-schema",
+    "validation",
+    "ci",
+    "test-automation",
+    "framework-agnostic"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "bin": {
+    "evidence": "dist/cli.js"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && node scripts/copy-schemas.mjs",
+    "prepack": "npm run build",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "docs": "npm --prefix design/web run dev",
+    "docs:build": "npm --prefix design/web run build",
+    "docs:install": "npm --prefix design/web install"
+  },
+  "dependencies": {
+    "adm-zip": "^0.5.14",
+    "ajv": "^8.17.1",
+    "ajv-formats": "^3.0.1",
+    "commander": "^12.1.0",
+    "picocolors": "^1.0.1",
+    "yaml": "^2.5.0"
+  },
+  "devDependencies": {
+    "@types/adm-zip": "^0.5.5",
+    "@types/node": "^20.14.0",
+    "typescript": "^5.5.0",
+    "vitest": "^2.0.0"
+  }
+}