@examplary/cli 1.9.0 → 1.10.0

package/README.md

@@ -24,3 +24,13 @@ npx @examplary/cli upload
 You'll be required to pass a `--key` argument with your API key, which you can
 obtain from your Examplary account settings. You can also set the `EXAMPLARY_API_KEY`
 environment variable to avoid passing the key every time.
+
+## Validating
+
+To run all of the pre-upload checks without uploading anything, use:
+
+```bash
+npx @examplary/cli validate
+```
+
+This verifies required fields, JSONata expressions (including `responseProcessing.condition`), response processing test cases, that components bundle, and the icon file. The same checks run automatically as part of `upload`.

package/dist/commands/upload.js

@@ -1,12 +1,8 @@
 #! /usr/bin/env node
-import { readFile } from "fs/promises";
-import { resolve } from "path";
 import { cwd, exit } from "process";
 import { setApiHost, setApiKey, uploadFile } from "../lib/api.js";
-import { buildComponent } from "../lib/bundle.js";
-import { deepCompileJsonata } from "../lib/jsonata.js";
 import { readMetaFile } from "../lib/read-meta.js";
-import { buildStyles } from "../lib/styles.js";
+import { isValid, printValidationResult, validateDefinition, } from "../lib/validate.js";
 export const uploadCommand = async (argv) => {
     setApiHost(argv.host);
     setApiKey(argv.key);
@@ -20,98 +16,35 @@ export const uploadCommand = async (argv) => {
         exit(1);
     }
     console.log(`Uploading question type: ${definition.id}...`);
-    if (!definition.id) {
-        console.error(`🚫 Missing required field: id`);
+    // Run all checks (incl. response processing test cases) before uploading
+    const result = await validateDefinition(definition, dir);
+    printValidationResult(result);
+    if (!isValid(result)) {
+        console.error(`\n🚫 Validation failed, aborting upload`);
         exit(1);
     }
-    if (!definition.name) {
-        console.error(`🚫 Missing required field: name`);
-        exit(1);
-    }
-    // Validate JSONata expressions in the definition
-    if (definition.export?.qti?.interaction?.options) {
-        try {
-            await deepCompileJsonata(definition.export.qti.interaction.options);
-        }
-        catch (e) {
-            console.error(`🚫 Invalid JSONata expression in export.qti.interaction.options: ${e.message}`);
-            exit(1);
-        }
-    }
-    // Bundle and upload components
+    // Upload the bundled components produced during validation
     const components = {};
-    const validComponentTypes = [
-        "settings-area",
-        "assessment",
-        "print",
-        "results",
-    ];
-    for (const [componentType, componentPath] of Object.entries(definition.components || {})) {
-        // Validation
-        if (!validComponentTypes.includes(componentType)) {
-            console.error(`🚫 Invalid component type specified: ${componentType}`);
-            exit(1);
-        }
-        const realPath = resolve(dir, componentPath);
-        if (!realPath) {
-            console.error(`🚫 Missing required file: ${componentPath}`);
-            exit(1);
-        }
-        // Bundle component
-        let componentOutput = { js: "" };
-        try {
-            componentOutput = await buildComponent(realPath);
-            const sizeInKb = componentOutput.js.length / 1000;
-            const sizeHint = `(${sizeInKb.toFixed(2)} kB)`;
-            console.log(`• Bundled component ${componentType} from ${componentPath} ${sizeHint}`);
-            if (sizeInKb > 100) {
-                console.warn(`  ⚠️ Component ${componentType} is larger than 100 kB! Consider optimizing it.`);
-                if (process.env.GITHUB_ACTIONS) {
-                    console.log(`::warning file=${realPath},title=Large component size::Component ${componentType} is larger than 100 kB ${sizeHint}! Consider optimizing it.`);
-                }
-            }
-        }
-        catch (error) {
-            console.error(`🚫 Error bundling component ${componentType} from ${componentPath}: ${error.message}`);
-            exit(1);
-        }
+    for (const [componentType, output] of Object.entries(result.bundles)) {
         // Upload source map first if it exists
         let sourceMapStatement = "";
-        if (componentOutput.sourcemap) {
-            const url = await uploadFile(`component-${componentType}.js.map`, componentOutput.sourcemap, "application/json");
+        if (output.sourcemap) {
+            const url = await uploadFile(`component-${componentType}.js.map`, output.sourcemap, "application/json");
             sourceMapStatement = `\n//# sourceMappingURL=${url}`;
         }
-        // Upload component
-        components[componentType] = await uploadFile(`component-${componentType}.js`, componentOutput.js + sourceMapStatement, "application/javascript");
+        components[componentType] = await uploadFile(`component-${componentType}.js`, output.js + sourceMapStatement, "application/javascript");
         console.log(`• Uploaded component ${componentType}`);
     }
     definition.components = components;
     // Upload styles if present
-    if (Object.keys(definition.components).length) {
-        const styles = await buildStyles();
-        if (styles) {
-            definition.stylesheet = await uploadFile("styles.css", styles, "text/css");
-            const sizeInKb = styles.length / 1000;
-            const sizeHint = `(${sizeInKb.toFixed(2)} kB)`;
-            console.log(`• Uploaded styles ${sizeHint}`);
-        }
+    if (result.styles) {
+        definition.stylesheet = await uploadFile("styles.css", result.styles, "text/css");
+        const sizeInKb = (result.styles.length / 1000).toFixed(2);
+        console.log(`• Uploaded styles (${sizeInKb} kB)`);
     }
     // Upload icon if specified
-    if (definition.icon) {
-        const iconPath = resolve(dir, definition.icon);
-        let iconContents;
-        try {
-            iconContents = await readFile(iconPath, "utf-8");
-        }
-        catch (error) {
-            console.error(`🚫 Error reading icon file: ${error.message}`);
-            exit(1);
-        }
-        if (!iconContents) {
-            console.error(`🚫 Icon file is empty: ${iconPath}`);
-            exit(1);
-        }
-        definition.icon = await uploadFile("icon.svg", iconContents, "image/svg+xml");
+    if (definition.icon && result.iconContents) {
+        definition.icon = await uploadFile("icon.svg", result.iconContents, "image/svg+xml");
         console.log(`• Uploaded icon`);
     }
     // Upload the main question type definition

package/dist/commands/validate.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export declare const validateCommand: () => Promise<void>;

package/dist/commands/validate.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+import { cwd, exit } from "process";
+import { readMetaFile } from "../lib/read-meta.js";
+import { isValid, printValidationResult, validateDefinition, } from "../lib/validate.js";
+export const validateCommand = async () => {
+    const dir = cwd();
+    let definition = {};
+    try {
+        ({ definition } = await readMetaFile());
+    }
+    catch (e) {
+        console.error(`🚫 ${e.message}`);
+        exit(1);
+    }
+    console.log(`Validating question type: ${definition.id}...`);
+    const result = await validateDefinition(definition, dir);
+    printValidationResult(result);
+    if (!isValid(result)) {
+        console.error(`\n🚫 Validation failed`);
+        exit(1);
+    }
+    console.log(`\n✅ Validation passed`);
+};

package/dist/index.js

@@ -3,6 +3,7 @@ import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import { previewCommand } from "./commands/preview.js";
 import { uploadCommand } from "./commands/upload.js";
+import { validateCommand } from "./commands/validate.js";
 yargs()
     .usage("examplary <cmd> [args]")
     .command("upload", "Bundle and upload a question type definition", (yargs) => {
@@ -18,6 +19,7 @@ yargs()
         default: "https://api.examplary.ai",
     });
 }, (args) => uploadCommand({ host: args.host, key: args.key }))
+    .command("validate", "Validate a question type definition without uploading", (yargs) => yargs, () => validateCommand())
     .command("preview", "Run local development server", (yargs) => {
     return yargs
         .option("key", {

package/dist/lib/read-meta.d.ts

@@ -1,16 +1,12 @@
+import type { QuestionType } from "@examplary/schemas";
+export type QuestionTypeDefinition = Partial<QuestionType> & {
+    $schema?: string;
+};
 /**
  * Reads and parses the question-type definition file.
  * Prefers question-type.json over question-type.yml.
  */
 export declare const readMetaFile: () => Promise<{
-    definition: {
-        $schema?: string;
-        id?: string;
-        name?: string;
-        components?: Record<string, string>;
-        export?: any;
-        stylesheet?: string;
-        icon?: string;
-    };
+    definition: QuestionTypeDefinition;
     path: string;
 }>;

package/dist/lib/response-processing.d.ts

@@ -0,0 +1,14 @@
+import type { QuestionTypeResponseProcessingType } from "@examplary/schemas";
+export type TestCaseResult = {
+    index: number;
+    expected: boolean;
+    actual?: boolean;
+    passed: boolean;
+    error?: string;
+};
+/**
+ * Runs every `responseProcessing.testCases` through the `condition` and checks
+ * that the evaluated outcome matches the `expectedOutcome`. Returns one result
+ * per test case; an empty array is returned when there are no test cases.
+ */
+export declare const runResponseProcessingTestCases: (responseProcessing: QuestionTypeResponseProcessingType) => Promise<TestCaseResult[]>;

package/dist/lib/response-processing.js

@@ -0,0 +1,45 @@
+import jsonata from "jsonata";
+/**
+ * Evaluates the response processing `condition` against a single test case's
+ * `answer` and `question`, mirroring how the API grades responses
+ * (see api/src/lib/grading/process-response.ts).
+ */
+const evaluateCondition = async (condition, data) => {
+    if (!condition.startsWith("=")) {
+        throw new Error("Invalid response processing condition: must start with '='");
+    }
+    const outcome = await jsonata(condition.substring(1)).evaluate(data);
+    return Boolean(outcome);
+};
+/**
+ * Runs every `responseProcessing.testCases` through the `condition` and checks
+ * that the evaluated outcome matches the `expectedOutcome`. Returns one result
+ * per test case; an empty array is returned when there are no test cases.
+ */
+export const runResponseProcessingTestCases = async (responseProcessing) => {
+    const { condition, testCases } = responseProcessing;
+    if (!testCases?.length)
+        return [];
+    const results = [];
+    for (let index = 0; index < testCases.length; index++) {
+        const { answer, question, expectedOutcome } = testCases[index];
+        try {
+            const actual = await evaluateCondition(condition, { answer, question });
+            results.push({
+                index,
+                expected: expectedOutcome,
+                actual,
+                passed: actual === expectedOutcome,
+            });
+        }
+        catch (error) {
+            results.push({
+                index,
+                expected: expectedOutcome,
+                passed: false,
+                error: error.message,
+            });
+        }
+    }
+    return results;
+};

package/dist/lib/validate.d.ts

@@ -0,0 +1,32 @@
+import type { QuestionTypeDefinition } from "./read-meta.js";
+import { TestCaseResult } from "./response-processing.js";
+export type BuiltComponent = {
+    js: string;
+    sourcemap?: string;
+};
+export type ValidationWarning = {
+    message: string;
+    file?: string;
+};
+export type ValidationResult = {
+    errors: string[];
+    warnings: ValidationWarning[];
+    testCaseResults: TestCaseResult[];
+    /** Bundled component output, keyed by component type, ready to upload. */
+    bundles: Record<string, BuiltComponent>;
+    /** Generated stylesheet, present only when components are bundled. */
+    styles?: string;
+    /** Raw icon file contents, present only when an icon is configured. */
+    iconContents?: string;
+};
+/**
+ * Runs every check the `upload` command performs, short of actually uploading:
+ * required fields, JSONata syntax, component bundling, icon presence, and the
+ * response processing test cases. Bundled artifacts are returned so callers can
+ * reuse them for an upload without bundling twice.
+ */
+export declare const validateDefinition: (definition: QuestionTypeDefinition, dir: string) => Promise<ValidationResult>;
+/** Whether the definition is safe to upload. */
+export declare const isValid: (result: ValidationResult) => boolean;
+/** Prints warnings, errors, and a test-case summary to the console. */
+export declare const printValidationResult: (result: ValidationResult) => void;

package/dist/lib/validate.js

@@ -0,0 +1,126 @@
+import { readFile } from "fs/promises";
+import { resolve } from "path";
+import { buildComponent } from "./bundle.js";
+import { deepCompileJsonata } from "./jsonata.js";
+import { runResponseProcessingTestCases, } from "./response-processing.js";
+import { buildStyles } from "./styles.js";
+const VALID_COMPONENT_TYPES = [
+    "settings-area",
+    "assessment",
+    "print",
+    "results",
+];
+/**
+ * Runs every check the `upload` command performs, short of actually uploading:
+ * required fields, JSONata syntax, component bundling, icon presence, and the
+ * response processing test cases. Bundled artifacts are returned so callers can
+ * reuse them for an upload without bundling twice.
+ */
+export const validateDefinition = async (definition, dir) => {
+    const errors = [];
+    const warnings = [];
+    const bundles = {};
+    let styles;
+    let iconContents;
+    if (!definition.id)
+        errors.push("Missing required field: id");
+    if (!definition.name)
+        errors.push("Missing required field: name");
+    // Validate JSONata expressions in the export options
+    if (definition.export?.qti3?.interaction?.options) {
+        try {
+            await deepCompileJsonata(definition.export.qti3.interaction.options);
+        }
+        catch (e) {
+            errors.push(`Invalid JSONata expression in export.qti3.interaction.options: ${e.message}`);
+        }
+    }
+    // Validate the response processing condition and run its test cases
+    let testCaseResults = [];
+    if (definition.responseProcessing) {
+        const condition = definition.responseProcessing.condition;
+        if (typeof condition !== "string" || !condition.startsWith("=")) {
+            errors.push("Invalid response processing condition: must start with '='");
+        }
+        else {
+            try {
+                await deepCompileJsonata(condition);
+                testCaseResults = await runResponseProcessingTestCases(definition.responseProcessing);
+            }
+            catch (e) {
+                errors.push(`Invalid JSONata expression in responseProcessing.condition: ${e.message}`);
+            }
+        }
+    }
+    // Bundle components to confirm they build and to capture their output
+    for (const [componentType, componentPath] of Object.entries(definition.components || {})) {
+        if (!VALID_COMPONENT_TYPES.includes(componentType)) {
+            errors.push(`Invalid component type specified: ${componentType}`);
+            continue;
+        }
+        const realPath = resolve(dir, componentPath);
+        try {
+            const output = await buildComponent(realPath);
+            bundles[componentType] = output;
+            const sizeInKb = output.js.length / 1000;
+            if (sizeInKb > 100) {
+                warnings.push({
+                    message: `Component ${componentType} is larger than 100 kB (${sizeInKb.toFixed(2)} kB)! Consider optimizing it.`,
+                    file: realPath,
+                });
+            }
+        }
+        catch (error) {
+            errors.push(`Error bundling component ${componentType} from ${componentPath}: ${error.message}`);
+        }
+    }
+    // Build styles if any components are present
+    if (Object.keys(bundles).length) {
+        styles = (await buildStyles()) ?? undefined;
+    }
+    // Validate the icon file
+    if (definition.icon) {
+        const iconPath = resolve(dir, definition.icon);
+        try {
+            iconContents = await readFile(iconPath, "utf-8");
+        }
+        catch (error) {
+            errors.push(`Error reading icon file: ${error.message}`);
+        }
+        if (iconContents !== undefined && !iconContents) {
+            errors.push(`Icon file is empty: ${iconPath}`);
+        }
+    }
+    return { errors, warnings, testCaseResults, bundles, styles, iconContents };
+};
+/** Whether the definition is safe to upload. */
+export const isValid = (result) => result.errors.length === 0 && result.testCaseResults.every((r) => r.passed);
+/** Prints warnings, errors, and a test-case summary to the console. */
+export const printValidationResult = (result) => {
+    for (const [componentType, output] of Object.entries(result.bundles)) {
+        const sizeInKb = (output.js.length / 1000).toFixed(2);
+        console.log(`• Bundled component ${componentType} (${sizeInKb} kB)`);
+    }
+    for (const warning of result.warnings) {
+        console.warn(`  ⚠️ ${warning.message}`);
+        if (warning.file && process.env.GITHUB_ACTIONS) {
+            console.log(`::warning file=${warning.file},title=Validation warning::${warning.message}`);
+        }
+    }
+    if (result.testCaseResults.length) {
+        const passed = result.testCaseResults.filter((r) => r.passed).length;
+        const total = result.testCaseResults.length;
+        console.log(`• Response processing test cases: ${passed}/${total} passed`);
+        for (const r of result.testCaseResults) {
+            if (r.passed)
+                continue;
+            const detail = r.error
+                ? `errored: ${r.error}`
+                : `expected outcome ${r.expected} but got ${r.actual}`;
+            console.error(`  🚫 Test case #${r.index + 1} failed (${detail})`);
+        }
+    }
+    for (const error of result.errors) {
+        console.error(`🚫 ${error}`);
+    }
+};