@smartergpt/lexrunner 1.0.0 → 1.3.0

package/tests/e2e-determinism.test.ts

@@ -1,293 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { skipIfCliNotBuilt } from "./helpers/cli";
-import { loadInputs } from "../src/core/inputs.js";
-import { generatePlan } from "../src/core/plan.js";
-import { executeGatesWithPolicy } from "../src/gates.js";
-import { ExecutionState } from "../src/executionState.js";
-import { MergeEligibilityEvaluator } from "../src/mergeEligibility.js";
-import { canonicalJSONStringify } from "../src/util/canonicalJson.js";
-import { sha256 } from "../src/util/hash.js";
-import { loadPlan } from "../src/schema.js";
-import { execSync } from "child_process";
-import * as fs from "fs";
-import * as path from "path";
-import * as os from "os";
-
-describe("E2E Determinism Harness", () => {
-  const testDir = path.join(os.tmpdir(), "lexrunner-e2e-determinism");
-  const repoRoot = path.resolve(__dirname, "..");
-
-  beforeEach((context) => {
-    // Clean test directory
-    if (fs.existsSync(testDir)) {
-      fs.rmSync(testDir, { recursive: true });
-    }
-    fs.mkdirSync(testDir, { recursive: true });
-    process.chdir(testDir);
-
-    if (skipIfCliNotBuilt({ skip: context.skip })) return;
-  });
-
-  afterEach(() => {
-    // Cleanup
-    process.chdir("/");
-    if (fs.existsSync(testDir)) {
-      fs.rmSync(testDir, { recursive: true });
-    }
-  });
-
-  describe("Complete Workflow Determinism", () => {
-    it("should produce identical results across multiple complete runs", async () => {
-      // Create test configuration with gates
-      fs.mkdirSync(".smartergpt", { recursive: true });
-      fs.writeFileSync(
-        ".smartergpt/stack.yml",
-        `
-version: 1
-target: main
-items:
-  - id: item-1
-    branch: feat/item-1
-    deps: []
-    gates:
-      - name: lint
-        run: bash -c "echo 'linting item-1'; exit 0"
-      - name: test
-        run: bash -c "echo 'testing item-1'; exit 0"
-  - id: item-2
-    branch: feat/item-2
-    deps: [item-1]
-    gates:
-      - name: lint
-        run: bash -c "echo 'linting item-2'; exit 0"
-      - name: test
-        run: bash -c "echo 'testing item-2'; exit 0"
-  - id: item-3
-    branch: feat/item-3
-    deps: [item-1]
-    gates:
-      - name: lint
-        run: bash -c "echo 'linting item-3'; exit 0"
-  - id: item-4
-    branch: feat/item-4
-    deps: [item-2, item-3]
-    gates:
-      - name: integration
-        run: bash -c "echo 'integration test'; exit 0"
-`
-      );
-
-      // Run complete workflow multiple times
-      const workflows = await Promise.all([
-        runCompleteWorkflow(),
-        runCompleteWorkflow(),
-        runCompleteWorkflow(),
-      ]);
-
-      // Verify all workflows produce identical results
-      const [workflow1, workflow2, workflow3] = workflows;
-
-      // Plan determinism
-      expect(workflow1.planJson).toBe(workflow2.planJson);
-      expect(workflow2.planJson).toBe(workflow3.planJson);
-      expect(sha256(Buffer.from(workflow1.planJson))).toBe(sha256(Buffer.from(workflow2.planJson)));
-
-      // Execution state determinism (structure only, not gate outputs which include stdout)
-      expect(workflow1.executionStateStructure).toEqual(workflow2.executionStateStructure);
-      expect(workflow2.executionStateStructure).toEqual(workflow3.executionStateStructure);
-
-      // Merge decisions determinism
-      expect(workflow1.mergeDecisions).toEqual(workflow2.mergeDecisions);
-      expect(workflow2.mergeDecisions).toEqual(workflow3.mergeDecisions);
-
-      // Snapshot determinism
-      expect(workflow1.snapshot).toBe(workflow2.snapshot);
-      expect(workflow2.snapshot).toBe(workflow3.snapshot);
-    });
-
-    it("should maintain determinism across different working directories", async () => {
-      // Create configuration in original directory
-      fs.mkdirSync(".smartergpt", { recursive: true });
-      const configContent = `
-version: 1
-target: main
-items:
-  - id: simple-item
-    branch: feat/simple
-    deps: []
-    gates:
-      - name: check
-        run: bash -c "pwd && echo 'check passed'; exit 0"
-`;
-      fs.writeFileSync(".smartergpt/stack.yml", configContent);
-
-      // Run workflow in current directory
-      const workflow1 = await runCompleteWorkflow();
-
-      // Create new directory and run same workflow
-      const testDir2 = path.join(os.tmpdir(), "lexrunner-e2e-determinism-2");
-      fs.mkdirSync(testDir2, { recursive: true });
-      const originalDir = process.cwd();
-
-      try {
-        process.chdir(testDir2);
-        fs.mkdirSync(".smartergpt", { recursive: true });
-        fs.writeFileSync(".smartergpt/stack.yml", configContent);
-
-        const workflow2 = await runCompleteWorkflow();
-
-        // Plans should be identical despite different working directories
-        expect(workflow1.planJson).toBe(workflow2.planJson);
-        expect(workflow1.executionStateStructure).toEqual(workflow2.executionStateStructure);
-        expect(workflow1.mergeDecisions).toEqual(workflow2.mergeDecisions);
-        expect(workflow1.snapshot).toBe(workflow2.snapshot);
-      } finally {
-        process.chdir(originalDir);
-        if (fs.existsSync(testDir2)) {
-          fs.rmSync(testDir2, { recursive: true });
-        }
-      }
-    });
-
-    it("should handle environment variations deterministically", async () => {
-      // Create configuration that uses environment variables
-      fs.mkdirSync(".smartergpt", { recursive: true });
-      fs.writeFileSync(
-        ".smartergpt/stack.yml",
-        `
-version: 1
-target: main
-items:
-  - id: env-item
-    branch: feat/env-test
-    deps: []
-    gates:
-      - name: env-check
-        run: bash -c "echo 'NODE_ENV: $NODE_ENV'; exit 0"
-        env:
-          NODE_ENV: test
-`
-      );
-
-      // Run with different system environments but same gate environments
-      const originalEnv = process.env.NODE_ENV;
-
-      try {
-        // First run with NODE_ENV=production (should be overridden by gate env)
-        process.env.NODE_ENV = "production";
-        const workflow1 = await runCompleteWorkflow();
-
-        // Second run with NODE_ENV=development (should be overridden by gate env)
-        process.env.NODE_ENV = "development";
-        const workflow2 = await runCompleteWorkflow();
-
-        // Plans should be identical as gate environment should override system
-        expect(workflow1.planJson).toBe(workflow2.planJson);
-        expect(workflow1.executionStateStructure).toEqual(workflow2.executionStateStructure);
-        expect(workflow1.mergeDecisions).toEqual(workflow2.mergeDecisions);
-      } finally {
-        process.env.NODE_ENV = originalEnv;
-      }
-    });
-
-    it("should maintain artifact determinism with sorted keys and stable hashes", async () => {
-      // Create configuration with multiple items to test sorting
-      fs.mkdirSync(".smartergpt", { recursive: true });
-      fs.writeFileSync(
-        ".smartergpt/stack.yml",
-        `
-version: 1
-target: main
-items:
-  - id: zebra-item
-    branch: feat/zebra
-    deps: []
-    gates:
-      - name: test
-        run: bash -c "echo 'zebra test'; exit 0"
-  - id: alpha-item
-    branch: feat/alpha
-    deps: []
-    gates:
-      - name: test
-        run: bash -c "echo 'alpha test'; exit 0"
-  - id: beta-item
-    branch: feat/beta
-    deps: [alpha-item, zebra-item]
-    gates:
-      - name: integration
-        run: bash -c "echo 'beta integration'; exit 0"
-`
-      );
-
-      const workflow = await runCompleteWorkflow();
-
-      // Verify plan has sorted items
-      const plan = JSON.parse(workflow.planJson);
-      const itemNames = plan.items.map((item: any) => item.name);
-      const sortedNames = [...itemNames].sort();
-      expect(itemNames).toEqual(sortedNames);
-
-      // Verify all JSON is canonical (no key order dependency)
-      expect(() => JSON.parse(workflow.planJson)).not.toThrow();
-
-      // Verify hash consistency
-      const hash1 = sha256(Buffer.from(workflow.planJson));
-      const hash2 = sha256(Buffer.from(canonicalJSONStringify(JSON.parse(workflow.planJson))));
-      expect(hash1).toBe(hash2);
-    });
-  });
-
-  /**
-   * Run complete workflow and return all artifacts
-   */
-  async function runCompleteWorkflow() {
-    // 1. Load inputs
-    const inputs = loadInputs();
-
-    // 2. Generate plan
-    const plan = generatePlan(inputs);
-    const validatedPlan = loadPlan(canonicalJSONStringify(plan));
-    const planJson = canonicalJSONStringify(validatedPlan);
-
-    // 3. Execute gates
-    const executionState = new ExecutionState(validatedPlan);
-    const artifactDir = path.join(process.cwd(), ".artifacts");
-
-    // Create artifact directory
-    if (!fs.existsSync(artifactDir)) {
-      fs.mkdirSync(artifactDir, { recursive: true });
-    }
-
-    // Execute gates with timeout
-    await executeGatesWithPolicy(validatedPlan, executionState, artifactDir, 10000);
-
-    // 4. Generate merge decisions
-    const evaluator = new MergeEligibilityEvaluator(validatedPlan, executionState);
-    const mergeSummary = evaluator.getMergeSummary();
-
-    // 5. Generate snapshot (deterministic output)
-    const cliPath = path.join(repoRoot, "dist/cli.js");
-    const snapshotOutput = execSync(`node ${cliPath} plan --json`, {
-      encoding: "utf8",
-      cwd: process.cwd(),
-    });
-
-    // Extract execution state structure (without non-deterministic parts like stdout)
-    const results = Array.from(executionState.getResults().values());
-    const executionStateStructure = {
-      nodeCount: results.length,
-      nodeNames: results.map((r) => r.name).sort(),
-      gateStatuses: results
-        .flatMap((r) => r.gates.map((g) => ({ node: r.name, gate: g.gate, status: g.status })))
-        .sort((a, b) => `${a.node}-${a.gate}`.localeCompare(`${b.node}-${b.gate}`)),
-    };
-
-    return {
-      planJson,
-      executionStateStructure,
-      mergeDecisions: canonicalJSONStringify(mergeSummary),
-      snapshot: snapshotOutput.trim(),
-    };
-  }
-});

package/tests/e2e-synthetic-6pr-weave.spec.ts

@@ -1,329 +0,0 @@
-/**
- * End-to-end test: Synthetic 6-PR weave with 2 conflicts
- *
- * Tests complete merge-weave workflow with:
- * - 6 PRs in pyramid dependency structure
- * - 2 predictable merge conflicts (auto-resolved)
- * - Green gates (all pass)
- * - Budget compliance (≤ 3 prompts, ≤ 5k tokens)
- * - Artifact generation and validation
- *
- * Related: Issue #[TBD] - M2 milestone
- */
-
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { skipIfCliNotBuilt } from "./helpers/cli.js";
-import { fixtures } from "./fixtures/index.js";
-import { BudgetTracker } from "../src/budget/index.js";
-import * as fs from "fs";
-import * as path from "path";
-import * as os from "os";
-import { exec } from "child_process";
-import { promisify } from "util";
-import { canonicalJSONStringify } from "../src/util/canonicalJson.js";
-
-const execAsync = promisify(exec);
-
-describe("E2E: Synthetic 6-PR Weave with Conflicts", () => {
-  let tempDir: string;
-  let projectDir: string;
-  let gitRepoDir: string;
-
-  beforeEach(async (context) => {
-    if (skipIfCliNotBuilt({ skip: context.skip })) return;
-
-    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-6pr-weave-"));
-    projectDir = path.dirname(__dirname);
-    gitRepoDir = path.join(tempDir, "test-repo");
-
-    // Create git repository
-    fs.mkdirSync(gitRepoDir, { recursive: true });
-    await execAsync("git init", { cwd: gitRepoDir });
-    await execAsync('git config user.email "test@lexrunner.dev"', {
-      cwd: gitRepoDir,
-    });
-    await execAsync('git config user.name "Test Runner"', {
-      cwd: gitRepoDir,
-    });
-    await execAsync("git config commit.gpgsign false", {
-      cwd: gitRepoDir,
-    });
-
-    // Create initial commit with base files
-    const scenario = fixtures.scenarios.syntheticSixPRWeave();
-    for (const [filePath, content] of Object.entries(scenario.files)) {
-      const fullPath = path.join(gitRepoDir, filePath);
-      const dir = path.dirname(fullPath);
-      if (!fs.existsSync(dir)) {
-        fs.mkdirSync(dir, { recursive: true });
-      }
-      fs.writeFileSync(fullPath, content);
-    }
-
-    await execAsync("git add .", { cwd: gitRepoDir });
-    await execAsync('git commit -m "Initial commit"', { cwd: gitRepoDir });
-    await execAsync("git branch -M main", { cwd: gitRepoDir });
-  });
-
-  afterEach(() => {
-    if (fs.existsSync(tempDir)) {
-      fs.rmSync(tempDir, { recursive: true, force: true });
-    }
-  });
-
-  const runCLI = async (
-    args: string,
-    cwd: string = gitRepoDir
-  ): Promise<{ stdout: string; stderr: string }> => {
-    const cliPath = path.join(projectDir, "dist/cli.js");
-    return execAsync(`node ${cliPath} ${args}`, { cwd });
-  };
-
-  /**
-   * Create synthetic PR branches with file changes
-   */
-  async function createSyntheticPRBranches(): Promise<void> {
-    const scenario = fixtures.scenarios.syntheticSixPRWeave();
-    const getFileChanges = fixtures.scenarios.getFileChangesForPR;
-
-    for (const item of scenario.plan.items) {
-      const branchName = `pr/${item.name}`;
-
-      // Create and checkout branch from main
-      await execAsync(`git checkout -b ${branchName} main`, {
-        cwd: gitRepoDir,
-      });
-
-      // Apply file changes for this PR
-      const changes = getFileChanges(item.name);
-      for (const [filePath, content] of Object.entries(changes)) {
-        const fullPath = path.join(gitRepoDir, filePath);
-        const dir = path.dirname(fullPath);
-        if (!fs.existsSync(dir)) {
-          fs.mkdirSync(dir, { recursive: true });
-        }
-        fs.writeFileSync(fullPath, content);
-      }
-
-      // Commit changes
-      await execAsync("git add .", { cwd: gitRepoDir });
-      await execAsync(`git commit -m "Add ${item.name}"`, {
-        cwd: gitRepoDir,
-      });
-    }
-
-    // Return to main
-    await execAsync("git checkout main", { cwd: gitRepoDir });
-  }
-
-  it("should complete 6-PR weave with 2 conflicts and green gates", async (ctx) => {
-    if (skipIfCliNotBuilt({ skip: ctx.skip })) return;
-
-    const scenario = fixtures.scenarios.syntheticSixPRWeave();
-
-    // Step 1: Create synthetic PR branches
-    await createSyntheticPRBranches();
-
-    // Verify branches were created
-    const { stdout: branchList } = await execAsync("git branch", {
-      cwd: gitRepoDir,
-    });
-    expect(branchList).toContain("pr/foundation-a");
-    expect(branchList).toContain("pr/foundation-b");
-    expect(branchList).toContain("pr/feature-x");
-    expect(branchList).toContain("pr/feature-y");
-    expect(branchList).toContain("pr/integration");
-    expect(branchList).toContain("pr/final");
-
-    // Step 2: Create plan file
-    const planPath = path.join(tempDir, "plan.json");
-    fs.writeFileSync(planPath, canonicalJSONStringify(scenario.plan));
-
-    // Step 3: Validate plan
-    const { stdout: validateOutput } = await runCLI(`schema validate ${planPath} --json`);
-    const validation = JSON.parse(validateOutput);
-    expect(validation.valid).toBe(true);
-
-    // Step 4: Compute merge order
-    const { stdout: orderOutput } = await runCLI(`merge-order ${planPath} --json`);
-    const order = JSON.parse(orderOutput);
-    expect(order.levels).toHaveLength(scenario.expected.levels);
-
-    // Count total items across all levels
-    const totalItemsInOrder = order.levels.flat().length;
-    expect(totalItemsInOrder).toBe(scenario.expected.totalItems);
-
-    // Verify topological order (foundations should be in level 1)
-    expect(order.levels[0]).toContain("foundation-a");
-    expect(order.levels[0]).toContain("foundation-b");
-
-    // Step 5: Dry-run to verify gates and structure
-    const { stdout: dryRunOutput } = await runCLI(
-      `weave apply --plan ${planPath} --dry-run --json`
-    );
-    const dryRun = JSON.parse(dryRunOutput);
-
-    expect(dryRun.dryRun).toBe(true);
-    expect(dryRun.totalItems).toBe(scenario.expected.totalItems);
-    expect(dryRun.levels).toHaveLength(scenario.expected.levels);
-
-    // Verify gates are configured
-    for (const item of scenario.plan.items) {
-      const itemGates = item.gates || [];
-      expect(itemGates.length).toBeGreaterThanOrEqual(2); // lint + test at minimum
-      expect(itemGates.some((g) => g.name === "lint")).toBe(true);
-      expect(itemGates.some((g) => g.name === "test")).toBe(true);
-    }
-  });
-
-  it("should track budget and stay within limits", async (ctx) => {
-    if (skipIfCliNotBuilt({ skip: ctx.skip })) return;
-
-    const scenario = fixtures.scenarios.syntheticSixPRWeave();
-    const budgetTracker = new BudgetTracker({
-      tokenBudget: scenario.expectedBudget.maxTokens,
-      maxPrompts: scenario.expectedBudget.maxPrompts,
-    });
-
-    // Create plan file
-    const planPath = path.join(tempDir, "plan.json");
-    fs.writeFileSync(planPath, canonicalJSONStringify(scenario.plan));
-
-    // Simulate prompt tracking during execution
-    // In a real scenario, this would be integrated into the weave workflow
-    const planContent = JSON.stringify(scenario.plan);
-    budgetTracker.recordPrompt(planContent);
-
-    // Get summary
-    const summary = budgetTracker.getSummary();
-    const spend = budgetTracker.getSpend();
-
-    // Assertions
-    expect(spend.prompts).toBeLessThanOrEqual(scenario.expectedBudget.maxPrompts);
-    expect(spend.tokens_estimated).toBeLessThanOrEqual(scenario.expectedBudget.maxTokens);
-    // Verify not exceeded
-    expect(summary.tokenBudgetExceeded).toBe(false);
-    expect(summary.promptBudgetExceeded).toBe(false);
-  });
-
-  it("should generate and validate artifacts", async (ctx) => {
-    if (skipIfCliNotBuilt({ skip: ctx.skip })) return;
-
-    const scenario = fixtures.scenarios.syntheticSixPRWeave();
-
-    // Create plan file
-    const planPath = path.join(tempDir, "plan.json");
-    fs.writeFileSync(planPath, canonicalJSONStringify(scenario.plan));
-
-    // Run merge-order to generate artifacts
-    const { stdout: orderOutput } = await runCLI(`merge-order ${planPath} --json`);
-    const order = JSON.parse(orderOutput);
-
-    // Verify artifact structure
-    expect(order).toHaveProperty("levels");
-    expect(Array.isArray(order.levels)).toBe(true);
-    expect(order.levels.length).toBeGreaterThan(0);
-
-    // Verify levels contain all items
-    const allItemsInLevels = order.levels.flat();
-    expect(allItemsInLevels.length).toBe(scenario.expected.totalItems);
-
-    for (const item of scenario.plan.items) {
-      expect(allItemsInLevels).toContain(item.name);
-    }
-
-    // Verify artifacts can be serialized
-    const artifactPath = path.join(tempDir, "artifacts.json");
-    fs.writeFileSync(artifactPath, JSON.stringify(order, null, 2));
-    expect(fs.existsSync(artifactPath)).toBe(true);
-
-    // Verify artifact is valid JSON
-    const readArtifact = JSON.parse(fs.readFileSync(artifactPath, "utf-8"));
-    expect(readArtifact).toEqual(order);
-  });
-
-  it("should validate conflict scenario structure", async (ctx) => {
-    if (skipIfCliNotBuilt({ skip: ctx.skip })) return;
-
-    const scenario = fixtures.scenarios.syntheticSixPRWeave();
-
-    // Verify conflict definitions
-    expect(scenario.conflicts).toHaveLength(2);
-
-    // Verify first conflict (utils.ts between feature-x and feature-y)
-    const conflict1 = scenario.conflicts[0];
-    expect(conflict1.file).toBe("src/utils.ts");
-    expect(conflict1.prs).toEqual(["feature-x", "feature-y"]);
-    expect(conflict1.resolution).toContain("featureX");
-    expect(conflict1.resolution).toContain("featureY");
-
-    // Verify second conflict (config.json between feature-x and integration)
-    const conflict2 = scenario.conflicts[1];
-    expect(conflict2.file).toBe("config.json");
-    expect(conflict2.prs).toEqual(["feature-x", "integration"]);
-    expect(conflict2.resolution).toContain("features");
-    expect(conflict2.resolution).toContain("integration");
-
-    // Verify expected outcomes
-    expect(scenario.expected.totalItems).toBe(6);
-    expect(scenario.expected.levels).toBe(4);
-    expect(scenario.expected.allGatesPass).toBe(true);
-    expect(scenario.expected.conflictsResolved).toBe(2);
-  });
-
-  it("should verify gate execution produces green results", async (ctx) => {
-    if (skipIfCliNotBuilt({ skip: ctx.skip })) return;
-
-    const scenario = fixtures.scenarios.syntheticSixPRWeave();
-
-    // Verify all gates are configured to pass
-    for (const item of scenario.plan.items) {
-      const gates = item.gates || [];
-      for (const gate of gates) {
-        // All gate commands should produce successful output
-        expect(gate.run).toContain("echo");
-        expect(gate.run).toContain("passed");
-      }
-    }
-
-    // Verify policy requires all gates to pass
-    expect(scenario.plan.policy?.requiredGates).toContain("lint");
-    expect(scenario.plan.policy?.requiredGates).toContain("test");
-    expect(scenario.plan.policy?.mergeRule.type).toBe("strict-required");
-  });
-
-  it("should produce cost summary within budget", async (ctx) => {
-    if (skipIfCliNotBuilt({ skip: ctx.skip })) return;
-
-    const scenario = fixtures.scenarios.syntheticSixPRWeave();
-
-    // Create budget tracker
-    const budgetTracker = new BudgetTracker({
-      tokenBudget: scenario.expectedBudget.maxTokens,
-      maxPrompts: scenario.expectedBudget.maxPrompts,
-    });
-
-    // Simulate workflow operations
-    const operations = ["Load plan", "Compute merge order", "Execute gates"];
-
-    for (const op of operations) {
-      budgetTracker.recordPrompt(op);
-    }
-
-    const spend = budgetTracker.getSpend();
-    const summary = budgetTracker.getSummary();
-
-    // Verify cost summary
-    expect(spend.prompts).toBe(3);
-    expect(spend.prompts).toBeLessThanOrEqual(scenario.expectedBudget.maxPrompts);
-    expect(spend.tokens_estimated).toBeLessThanOrEqual(scenario.expectedBudget.maxTokens);
-
-    // Verify summary format
-    expect(summary).toHaveProperty("prompts");
-    expect(summary).toHaveProperty("tokens_estimated");
-    expect(summary).toHaveProperty("tokenBudget");
-    expect(summary).toHaveProperty("maxPrompts");
-    expect(summary.tokenBudgetExceeded).toBe(false);
-    expect(summary.promptBudgetExceeded).toBe(false);
-  });
-});