@syndicalt/snow-cli 2.0.0 → 2.0.1

package/README.md

@@ -28,6 +28,10 @@ A portable CLI for ServiceNow. Query tables, inspect schemas, edit and search sc
 
 ---
 
+## Full Documentation
+
+The full documentation is available on the [documentation page](https://syndicalt.github.io/snow_cli/). This README provides a quick overview and examples to get started.
+
 ## Installation
 
 **From npm (when published):**
@@ -89,6 +93,9 @@ snow diff all --against test --scripts --scope x_myco_myapp
 # 11. Run the full factory pipeline: plan → build → test → promote
 snow factory "Build a hardware asset request app with approval workflow" --envs test,prod
 
+# 11b. Run tests and auto-fix failures with the LLM optimization loop
+snow factory "Create a Script Include named SLACalculator with risk level logic" --run-tests --optimize
+
 # 12. Start an interactive session to build iteratively
 snow ai chat
 
@@ -845,6 +852,12 @@ snow factory "Build an incident escalation app" --dry-run
 # Generate and immediately run ATF tests
 snow factory "Build a change approval workflow" --run-tests
 
+# Generate ATF tests, run them, and auto-fix failures with the LLM (up to 3 retries)
+snow factory "Create a priority calculator script include" --run-tests --optimize
+
+# Same, but allow up to 5 fix iterations
+snow factory "Create a priority calculator script include" --run-tests --optimize --max-retries 5
+
 # Skip ATF test generation (faster)
 snow factory "Add a business rule to auto-assign P1 incidents" --skip-tests
 
@@ -863,6 +876,8 @@ snow factory "" --list
 | `--scope <prefix>` | Override the application scope prefix for all artifacts (e.g. `x_myco_myapp`) |
 | `--skip-tests` | Skip ATF test generation |
 | `--run-tests` | Execute the generated ATF test suite immediately after pushing |
+| `--optimize` | After running tests, auto-fix failures via LLM feedback loop (implies `--run-tests`) |
+| `--max-retries <n>` | Max optimization iterations when using `--optimize` (default: `3`) |
 | `--dry-run` | Show the generated plan only — no builds, no deployments |
 | `--resume <id>` | Resume a previous run from its checkpoint (see `--list`) |
 | `--list` | Show recent factory runs and their component completion status |
@@ -886,7 +901,13 @@ snow factory "" --list
   [4/N] Generating ATF tests (unless --skip-tests)
         LLM generates a test suite with server-side script assertions
         Tests pushed to the instance as sys_atf_test + sys_atf_step records
-        Optionally executed if --run-tests is set
+        If --run-tests or --optimize: suite is executed via the CICD API
+        Pass/fail counts and per-test results are displayed
+
+        If --optimize and tests failed:
+          → LLM receives the failing test names + error messages + artifact source
+          → Generates corrected artifacts, re-pushes them, re-runs the suite
+          → Repeats until all pass or --max-retries is exhausted
 
   [5/N] Promoting to <env>  (once per additional env in --envs)
         Checks for pre-existing artifacts on the target
@@ -906,6 +927,40 @@ snow factory "" --resume abc12345     # resume from last successful phase
 
 The resume prompt argument is ignored when `--resume` is provided — the original prompt and plan are loaded from the checkpoint.
 
+#### Auto-optimization loop
+
+When `--optimize` is set, the factory runs an automated fix cycle after the initial ATF test run. For each iteration:
+
+1. Failing test names and error messages are collected from the `sys_atf_result` table
+2. The current source code of every script-bearing artifact is extracted from the local manifest
+3. A structured fix prompt is sent to the LLM — no questions, just a corrected build
+4. Fixed artifacts are merged into the build (replaced by `type + name`), re-pushed to the instance
+5. The existing ATF suite is re-run; if all tests pass, the loop exits early
+
+```
+  [Optimize 1/3]  2 test(s) failing — asking LLM to fix...
+  ────────────────────────────────────────────────
+  Patching:
+    ~ script_include     SLACalculator
+  Re-pushing fixed artifacts to instance...
+  Re-running ATF tests...
+  3/3 tests passed
+    ✓ Test low risk level
+    ✓ Test medium risk level
+    ✓ Test high risk level
+
+  ✔ All tests passing after optimization!
+```
+
+If failures persist after all retries, a warning is shown and the best-effort build is kept on disk.
+
+**Requirements for ATF test execution:**
+- The `sn_cicd` plugin must be active on the instance
+- The authenticated user must have the `sn_cicd` role
+- ATF must be enabled (`System ATF` → `Properties` → `Enable ATF`)
+
+The optimization loop uses the same LLM provider as the build. Each iteration costs one LLM call, so `--max-retries 3` (default) means at most 3 additional calls beyond the initial build.
+
 #### Example output
 
 ```
@@ -968,6 +1023,12 @@ The resume prompt argument is ignored when `--resume` is provided — the origin
     • Test manager notification trigger  (2 steps)
   ✓ Test suite created
     https://dev12345.service-now.com/nav_to.do?uri=sys_atf_test_suite.do?sys_id=...
+  Running ATF tests...
+  4/4 tests passed
+    ✓ Test employee record creation and field defaults
+    ✓ Test OnboardingUtils task generation
+    ✓ Test business rule fires on insert
+    ✓ Test manager notification trigger
 ```
 
 #### Permission-denied errors

package/dist/index.js

@@ -1075,18 +1075,83 @@ async function pushATFSuite(client, suite, stepConfigSysId) {
   return { suiteSysId, suiteUrl, testCount: suite.tests.length, stepCount };
 }
 async function runATFSuite(client, suiteSysId) {
-  const res = await client.post("/api/now/atf/test_suite/run", { id: suiteSysId });
-  const r = res?.result?.test_suite_result ?? {};
+  const startRes = await client.post("/api/sn_cicd/testsuite/run", void 0, {
+    params: { test_suite_sys_id: suiteSysId }
+  });
+  const resData = startRes;
+  const inner = resData["result"] ?? resData;
+  const links = inner["links"];
+  const progressId = links?.["progress"]?.["id"] ?? links?.["results"]?.["id"];
+  if (!progressId) {
+    throw new Error(
+      "CICD API did not return a progress link.\nRaw response: " + JSON.stringify(startRes, null, 2) + "\nEnsure the sn_cicd plugin is active and your user has the sn_cicd role."
+    );
+  }
+  const resultsId = links?.["results"]?.["id"] ?? progressId;
+  const TERMINAL = /* @__PURE__ */ new Set(["2", "3", "4"]);
+  const POLL_MS = 5e3;
+  const MAX_POLLS = 120;
+  let done = false;
+  let lastProgressRes = {};
+  for (let i = 0; i < MAX_POLLS; i++) {
+    await new Promise((resolve2) => setTimeout(resolve2, POLL_MS));
+    const progress = await client.get(
+      `/api/sn_cicd/progress/${progressId}`
+    );
+    lastProgressRes = progress["result"] ?? progress;
+    if (TERMINAL.has(String(lastProgressRes["status"] ?? ""))) {
+      done = true;
+      break;
+    }
+  }
+  if (!done) {
+    throw new Error("ATF test suite timed out after 10 minutes");
+  }
+  const progressLinks = lastProgressRes["links"];
+  const finalResultsId = progressLinks?.["results"]?.["id"] ?? resultsId;
+  const resultsRes = await client.get(
+    `/api/sn_cicd/testsuite/results/${finalResultsId}`
+  );
+  const resultData = resultsRes["result"] ?? resultsRes;
+  const successCount = parseInt(String(resultData["rolledup_test_success_count"] ?? 0), 10);
+  const failureCount = parseInt(String(resultData["rolledup_test_failure_count"] ?? 0), 10) + parseInt(String(resultData["rolledup_test_error_count"] ?? 0), 10);
+  const skipCount = parseInt(String(resultData["rolledup_test_skip_count"] ?? 0), 10);
+  const total = successCount + failureCount + skipCount;
+  const suiteResultLinks = resultData["links"];
+  const suiteResultSysId = suiteResultLinks?.["results"]?.["id"] ?? finalResultsId;
+  let testResults = [];
+  try {
+    const rows = await client.queryTable("sys_atf_result", {
+      sysparmQuery: `test_suite_result=${suiteResultSysId}`,
+      sysparmFields: "test,status,output",
+      sysparmLimit: 200,
+      sysparmDisplayValue: true
+    });
+    testResults = rows.map((r) => {
+      const testField = r["test"];
+      const name = testField && typeof testField === "object" ? String(testField["display_value"] ?? "(unknown)") : String(testField ?? "(unknown)");
+      const statusStr = String(r["status"] ?? "");
+      return {
+        name,
+        status: statusStr === "success" || statusStr === "pass" ? "success" : "failure",
+        message: r["output"] ? String(r["output"]) : void 0
+      };
+    });
+  } catch {
+    testResults = Array.from({ length: successCount }, (_, i) => ({
+      name: `Test ${i + 1}`,
+      status: "success"
+    }));
+    for (let i = 0; i < failureCount; i++) {
+      testResults.push({ name: `Test ${successCount + i + 1}`, status: "failure" });
+    }
+  }
   return {
-    status: res?.result?.status ?? "unknown",
-    passed: parseInt(r.tests_passed ?? "0", 10),
-    failed: parseInt(r.tests_failed ?? "0", 10),
-    total: parseInt(r.total_tests ?? "0", 10),
-    testResults: (r.test_results ?? []).map((t) => ({
-      name: t.test_name ?? "(unknown)",
-      status: t.status ?? "unknown",
-      message: t.message
-    }))
+    status: String(resultData["test_suite_status"] ?? resultData["status"] ?? "unknown"),
+    passed: successCount,
+    failed: failureCount,
+    total,
+    testResults
   };
 }
 var ATF_SYSTEM_PROMPT;
@@ -5627,8 +5692,148 @@ Scope prefix for all custom names: ${plan.scope.prefix}`;
   writeFileSync8(join7(compDir, `${base}.manifest.json`), JSON.stringify(build, null, 2), "utf-8");
   return build;
 }
+async function runOptimizationLoop(opts) {
+  const { provider, client, atfSuiteSysId, maxRetries, runDir } = opts;
+  let currentBuild = opts.build;
+  let runResult = opts.initialRunResult;
+  const { SN_SYSTEM_PROMPT: SN_SYSTEM_PROMPT2 } = await Promise.resolve().then(() => (init_sn_context(), sn_context_exports));
+  for (let iteration = 1; iteration <= maxRetries; iteration++) {
+    const failures = runResult.testResults.filter((t) => t.status !== "success");
+    if (failures.length === 0) break;
+    console.log();
+    printDivider();
+    console.log(`  ${chalk13.bold.magenta(`[Optimize ${iteration}/${maxRetries}]`)} ${chalk13.red(`${failures.length} test(s) failing`)} \u2014 asking LLM to fix...`);
+    printDivider();
+    const failureLines = failures.map((t) => `  - "${t.name}": ${t.message ?? "no details"}`).join("\n");
+    const artifactLines = currentBuild.artifacts.filter((a) => a.fields["script"] ?? a.fields["condition"]).map((a) => {
+      const name = String(a.fields["name"] ?? "");
+      const script = String(a.fields["script"] ?? "");
+      const condition = String(a.fields["condition"] ?? "");
+      let out = `[${a.type}: ${name}]`;
+      if (script) out += `
+script:
+${script}`;
+      if (condition) out += `
+condition:
+${condition}`;
+      return out;
+    }).join("\n\n---\n\n");
+    const fixPrompt = `OPTIMIZATION LOOP \u2014 Iteration ${iteration} of ${maxRetries}
+
+The following ATF tests failed after deploying the generated artifacts:
+
+FAILING TESTS:
+${failureLines}
+
+CURRENT ARTIFACT CODE:
+${artifactLines}
+
+Your task: analyze the failures, identify root causes, and return corrected artifacts.
+Rules:
+- Return ONLY a JSON object wrapped in a \`\`\`json code fence
+- Use the same schema: { "name": "...", "description": "...", "artifacts": [...] }
+- Only include artifacts that need changes
+- All scripts must be ES5 (var only, no arrow functions, no const/let)
+- Do NOT ask clarifying questions \u2014 provide the fix immediately`;
+    const messages = [
+      { role: "system", content: SN_SYSTEM_PROMPT2 },
+      { role: "user", content: fixPrompt }
+    ];
+    const llmSpinner = ora12(`  Calling LLM for fix...`).start();
+    let fixedArtifacts;
+    try {
+      const raw = await provider.complete(messages);
+      llmSpinner.stop();
+      const json = extractJSON(raw);
+      const parsed = JSON.parse(json);
+      const rawArtifacts = Array.isArray(parsed["artifacts"]) ? parsed["artifacts"] : [];
+      fixedArtifacts = rawArtifacts.map((a) => {
+        const art = a;
+        const type = String(art["type"] ?? "");
+        let fields;
+        if (art["fields"] && typeof art["fields"] === "object" && !Array.isArray(art["fields"])) {
+          fields = art["fields"];
+        } else {
+          const { type: _t, ...rest } = art;
+          fields = rest;
+        }
+        return { type, fields };
+      });
+    } catch (err) {
+      llmSpinner.fail(chalk13.yellow(`  LLM call failed on iteration ${iteration} \u2014 stopping optimization`));
+      console.error(chalk13.dim(`  ${err instanceof Error ? err.message : String(err)}`));
+      break;
+    }
+    const mergedArtifacts = currentBuild.artifacts.map((orig) => {
+      const fix = fixedArtifacts.find(
+        (f) => f.type === orig.type && String(f.fields["name"]) === String(orig.fields["name"])
+      );
+      return fix ?? orig;
+    });
+    const addedArtifacts = fixedArtifacts.filter(
+      (f) => !currentBuild.artifacts.some(
+        (orig) => orig.type === f.type && String(orig.fields["name"]) === String(f.fields["name"])
+      )
+    );
+    const updatedBuild = {
+      ...currentBuild,
+      artifacts: [...mergedArtifacts, ...addedArtifacts]
+    };
+    const changed = fixedArtifacts.filter(
+      (f) => currentBuild.artifacts.some(
+        (orig) => orig.type === f.type && String(orig.fields["name"]) === String(f.fields["name"])
+      )
+    );
+    if (changed.length > 0) {
+      console.log(`  ${chalk13.dim("Patching:")}`);
+      for (const a of changed) {
+        console.log(`    ${chalk13.yellow("~")} ${chalk13.cyan(a.type.padEnd(16))} ${String(a.fields["name"] ?? "")}`);
+      }
+    }
+    const pushSpinner = ora12(`  Re-pushing fixed artifacts to instance...`).start();
+    try {
+      await pushArtifacts(client, updatedBuild);
+      pushSpinner.stop();
+    } catch (err) {
+      pushSpinner.fail(chalk13.yellow("Re-push failed \u2014 stopping optimization"));
+      console.error(chalk13.dim(`  ${err instanceof Error ? err.message : String(err)}`));
+      break;
+    }
+    const base = slugify2(updatedBuild.name);
+    writeFileSync8(join7(runDir, `${base}.xml`), generateUpdateSetXML(updatedBuild), "utf-8");
+    writeFileSync8(join7(runDir, `${base}.manifest.json`), JSON.stringify(updatedBuild, null, 2), "utf-8");
+    const testSpinner = ora12(`  Re-running ATF tests...`).start();
+    try {
+      runResult = await runATFSuite(client, atfSuiteSysId);
+      testSpinner.stop();
+    } catch (err) {
+      testSpinner.fail(chalk13.yellow("ATF re-run failed \u2014 stopping optimization"));
+      console.error(chalk13.dim(`  ${err instanceof Error ? err.message : String(err)}`));
+      break;
+    }
+    const passColor = runResult.failed === 0 ? chalk13.green : chalk13.yellow;
+    console.log(`  ${passColor(`${runResult.passed}/${runResult.total} tests passed`)}`);
+    for (const t of runResult.testResults) {
+      const icon = t.status === "success" ? chalk13.green("\u2713") : chalk13.red("\u2717");
+      const msg = t.message ? chalk13.dim(`  (${t.message})`) : "";
+      console.log(`    ${icon} ${t.name}${msg}`);
+    }
+    currentBuild = updatedBuild;
+    if (runResult.failed === 0) {
+      console.log();
+      console.log(chalk13.green("  \u2714 All tests passing after optimization!"));
+      break;
+    }
+  }
+  if (runResult.failed > 0) {
+    console.log();
+    console.log(chalk13.yellow(`  \u26A0 ${runResult.failed} test(s) still failing after ${maxRetries} optimization iteration(s).`));
+    console.log(chalk13.dim("  Review the ATF suite in ServiceNow for details."));
+  }
+  return currentBuild;
+}
 function factoryCommand() {
-  const cmd = new Command13("factory").description("AI-orchestrated multi-component ServiceNow app pipeline: plan \u2192 build \u2192 test \u2192 promote").argument("<prompt>", "Natural language description of the application to build").option("--envs <aliases>", "Comma-separated instance aliases to deploy to in order (default: active instance only)", "").option("--scope <prefix>", "Override the application scope prefix (e.g. x_myco_myapp)").option("--skip-tests", "Skip ATF test generation").option("--run-tests", "Execute ATF tests on the source instance after generating them").option("--dry-run", "Show the plan only \u2014 do not build or deploy").option("--resume <id>", "Resume a previous factory run from its checkpoint").option("--list", "List recent factory runs and their status").action(async (prompt, opts) => {
+  const cmd = new Command13("factory").description("AI-orchestrated multi-component ServiceNow app pipeline: plan \u2192 build \u2192 test \u2192 promote").argument("<prompt>", "Natural language description of the application to build").option("--envs <aliases>", "Comma-separated instance aliases to deploy to in order (default: active instance only)", "").option("--scope <prefix>", "Override the application scope prefix (e.g. x_myco_myapp)").option("--skip-tests", "Skip ATF test generation").option("--run-tests", "Execute ATF tests on the source instance after generating them").option("--optimize", "Auto-fix failing ATF tests via LLM feedback loop (implies --run-tests)").option("--max-retries <n>", "Max optimization iterations (used with --optimize)", "3").option("--dry-run", "Show the plan only \u2014 do not build or deploy").option("--resume <id>", "Resume a previous factory run from its checkpoint").option("--list", "List recent factory runs and their status").action(async (prompt, opts) => {
     if (opts.list) {
       const runs = listCheckpoints();
       if (runs.length === 0) {
@@ -5819,7 +6024,7 @@ function factoryCommand() {
         console.log(`     ${chalk13.green("+")} ${chalk13.cyan(a.type.padEnd(16))} ${name}`);
       }
     }
-    const combinedBuild = {
+    let combinedBuild = {
       name: plan.name,
       description: plan.description,
       scope: plan.scope,
@@ -5906,7 +6111,7 @@ function factoryCommand() {
             console.error(chalk13.dim(`  ${err instanceof Error ? err.message : String(err)}`));
             atfResult = null;
           }
-          if (opts.runTests && atfResult) {
+          if ((opts.runTests || opts.optimize) && atfResult) {
             const runSpinner = ora12("  Running ATF tests...").start();
             try {
               const runResult = await runATFSuite(sourceClient, atfResult.suiteSysId);
@@ -5918,6 +6123,17 @@ function factoryCommand() {
                 const msg = t.message ? chalk13.dim(`  (${t.message})`) : "";
                 console.log(`    ${icon} ${t.name}${msg}`);
               }
+              if (opts.optimize && runResult.failed > 0) {
+                combinedBuild = await runOptimizationLoop({
+                  provider,
+                  client: sourceClient,
+                  build: combinedBuild,
+                  atfSuiteSysId: atfResult.suiteSysId,
+                  initialRunResult: runResult,
+                  maxRetries: parseInt(opts.maxRetries ?? "3", 10),
+                  runDir
+                });
+              }
             } catch (err) {
               runSpinner.fail(chalk13.yellow("ATF run failed (non-fatal)"));
               console.error(chalk13.dim(`  ${err instanceof Error ? err.message : String(err)}`));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syndicalt/snow-cli",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "CLI tool for ServiceNow automated build and instance management",
   "type": "module",
   "bin": {