@awak-app/simy-cli 0.5.15 → 0.5.16

package/src/orchestrator/evidence.js

@@ -1,11 +1,14 @@
 import { collectMinimumCompletion } from "./minimum-completion.js";
 import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { lstat, readFile, realpath } from "node:fs/promises";
+import path from "node:path";
 import { promisify } from "node:util";
 
 import { cleanString, stringArray } from "./shared.js";
 
 const execFileAsync = promisify(execFile);
-const SUCCESS_CONCLUSIONS = new Set(["SUCCESS", "NEUTRAL", "SKIPPED"]);
+const SUCCESS_CONCLUSIONS = new Set(["SUCCESS", "NEUTRAL"]);
 const FAILURE_CONCLUSIONS = new Set([
   "ACTION_REQUIRED",
   "CANCELLED",
@@ -20,14 +23,25 @@ export async function collectPrEvidence({
   repositoryPath,
   runCommand = execFileAsync,
 }) {
+  const baseRefresh = await command(
+    runCommand,
+    "git",
+    ["fetch", "--no-tags", "origin", charter.base_branch],
+    repositoryPath,
+  );
   const local = await collectLocalGitEvidence({
     baseBranch: charter.base_branch,
     repositoryPath,
     runCommand,
   });
+  local.base_ref_refreshed = baseRefresh.ok;
+  local.base_ref_error = baseRefresh.ok ? null : baseRefresh.error;
   const github = await collectGitHubEvidence({ charter, attempt, repositoryPath, runCommand });
   const minimumCompletion = await collectMinimumCompletion({ charter, attempt, repositoryPath, local });
+  const prPreparationRefs = await verifyPrPreparationRefs({ attempt, repositoryPath, local, github });
   const latest = await collectLocalGitEvidence({ baseBranch: charter.base_branch, repositoryPath, runCommand });
+  latest.base_ref_refreshed = baseRefresh.ok;
+  latest.base_ref_error = baseRefresh.ok ? null : baseRefresh.error;
   if (!latest.available || latest.head_sha !== local.head_sha || !latest.working_tree_clean) {
     minimumCompletion.status = "evidence_incomplete";
     minimumCompletion.errors = ["Checkout changed while collecting minimum evidence; verify again."];
@@ -37,27 +51,69 @@ export async function collectPrEvidence({
     collected_at: new Date().toISOString(),
     local: latest,
     github,
+    pr_preparation_refs: prPreparationRefs,
+  };
+}
+
+async function verifyPrPreparationRefs({ attempt, repositoryPath, local, github }) {
+  const refs = [...new Set((attempt.pr_preparation || []).flatMap((row) => row.evidence_refs || []))];
+  return Promise.all(refs.map(async (ref) => {
+    const observed = observedEvidenceRef(ref, local, github);
+    if (observed !== null) return { ref, verified: observed, kind: "observed" };
+    try {
+      const root = await realpath(repositoryPath);
+      const target = await realpath(path.resolve(root, ref));
+      const relative = path.relative(root, target);
+      const details = await lstat(target);
+      const verified = !relative.startsWith("..") && !path.isAbsolute(relative) &&
+        details.isFile() && details.size <= 50 * 1024 * 1024;
+      const sha256 = verified
+        ? createHash("sha256").update(await readFile(target)).digest("hex")
+        : null;
+      return { ref, verified, kind: "local_file", sha256 };
+    } catch {
+      return { ref, verified: false, kind: "unverified" };
+    }
+  }));
+}
+
+function observedEvidenceRef(ref, local, github) {
+  const sources = {
+    "observed:local.head_sha": local.head_sha,
+    "observed:local.base_sha": local.base_sha,
+    "observed:local.base_is_ancestor": local.base_is_ancestor === true,
+    "observed:github.head_sha": github.head_sha,
+    "observed:github.base_sha": github.base_sha,
+    "observed:github.checks": Array.isArray(github.checks) && github.checks.length > 0,
   };
+  return Object.hasOwn(sources, ref) ? Boolean(sources[ref]) : null;
 }
 
 async function collectLocalGitEvidence({ baseBranch, repositoryPath, runCommand }) {
-  const [head, branch, headline, status, changed] = await Promise.all([
+  const baseRef = `origin/${baseBranch}`;
+  const [head, branch, headline, status, changed, base, mergeBase, baseAncestor] = await Promise.all([
     command(runCommand, "git", ["rev-parse", "HEAD"], repositoryPath),
     command(runCommand, "git", ["branch", "--show-current"], repositoryPath),
     command(runCommand, "git", ["log", "-1", "--pretty=%s"], repositoryPath),
     command(runCommand, "git", ["status", "--porcelain"], repositoryPath),
     changedFiles(runCommand, baseBranch, repositoryPath),
+    command(runCommand, "git", ["rev-parse", baseRef], repositoryPath),
+    command(runCommand, "git", ["merge-base", baseRef, "HEAD"], repositoryPath),
+    command(runCommand, "git", ["merge-base", "--is-ancestor", baseRef, "HEAD"], repositoryPath),
   ]);
   const statusLines = lines(status.stdout);
   return {
-    available: head.ok && branch.ok && headline.ok && status.ok && changed.ok,
+    available: head.ok && branch.ok && headline.ok && status.ok && changed.ok && base.ok && mergeBase.ok,
     head_sha: cleanString(head.stdout) || null,
     branch_name: cleanString(branch.stdout) || null,
     commit_headline: cleanString(headline.stdout) || null,
     working_tree_clean: status.ok && statusLines.length === 0,
     status_lines: statusLines,
     changed_files: lines(changed.stdout),
-    errors: [head, branch, headline, status, changed]
+    base_sha: cleanString(base.stdout) || null,
+    merge_base_sha: cleanString(mergeBase.stdout) || null,
+    base_is_ancestor: baseAncestor.ok,
+    errors: [head, branch, headline, status, changed, base, mergeBase]
       .filter((item) => !item.ok)
       .map((item) => item.error),
   };
@@ -93,6 +149,7 @@ async function collectGitHubEvidence({ charter, attempt, repositoryPath, runComm
     "headRepository",
     "headRepositoryOwner",
     "baseRefName",
+    "baseRefOid",
     "reviews",
     "statusCheckRollup",
   ].join(",");
@@ -137,10 +194,11 @@ async function collectGitHubEvidence({ charter, attempt, repositoryPath, runComm
         ? `${cleanString(payload.headRepositoryOwner.login)}/${cleanString(payload.headRepository.name)}`
         : null,
     base_branch: cleanString(payload.baseRefName) || null,
+    base_sha: cleanString(payload.baseRefOid) || null,
     checks,
     checks_present: checks.length > 0,
     checks_passing:
-      checks.length > 0 && checks.every((item) => item.state === "success"),
+      checks.length > 0 && checks.every((item) => ["success", "skipped"].includes(item.state)),
     required_checks_present: requiredChecks.every((name) => checkNames.has(name)),
     error: null,
   };
@@ -154,6 +212,7 @@ function normalizeChecks(value) {
     const conclusion = cleanString(item?.conclusion || item?.state).toUpperCase();
     let state = "pending";
     if (SUCCESS_CONCLUSIONS.has(conclusion)) state = "success";
+    if (conclusion === "SKIPPED") state = "skipped";
     if (FAILURE_CONCLUSIONS.has(conclusion)) state = "failure";
     if (status === "COMPLETED" && !conclusion) state = "failure";
     return { name, status: status || null, conclusion: conclusion || null, state };

package/src/orchestrator/independent-audit.js

@@ -5,6 +5,7 @@ import {
   normalizeCriterionEvidence,
   normalizeVisualReview,
 } from "./completion-contract.js";
+import { independentAuditEvidenceIdentity } from "./pr-preparation.js";
 
 export function buildIndependentAuditInstruction(charter, attempt) {
   const localEvidence = attempt.observed_evidence?.local || {};
@@ -47,9 +48,14 @@ export function buildIndependentAuditInstruction(charter, attempt) {
         `Executor criterion evidence: ${JSON.stringify(attempt.criterion_evidence || [])}`,
         `Minimum evidence matrix: ${attempt.observed_evidence?.minimum_completion?.evidence_path || "missing"}`,
         `Minimum evidence digest: ${attempt.observed_evidence?.minimum_completion?.matrix_sha256 || "missing"}`,
+        `Complete audit evidence identity: ${independentAuditEvidenceIdentity(attempt)}`,
         `Executor visual review: ${JSON.stringify(attempt.visual_review || {})}`,
+        `Executor PR preparation evidence: ${JSON.stringify(attempt.pr_preparation || [])}`,
+        `Executor skipped-check disclosures: ${JSON.stringify(attempt.skipped_checks || [])}`,
         `Observed local branch: ${localEvidence.branch_name || "unknown"}`,
         `Observed local HEAD: ${localEvidence.head_sha || "unknown"}`,
+        `Observed fetched base SHA: ${localEvidence.base_sha || "unknown"}`,
+        `Observed base ancestry: ${localEvidence.base_is_ancestor === true ? "integrated" : "not integrated or unknown"}`,
         `Observed changed files:\n${bullets(localEvidence.changed_files || [])}`,
         `Observed PR: ${githubEvidence.url || attempt.pr_url || "unknown"}`,
         `Observed passing GitHub checks:\n${bullets(passingChecks)}`,
@@ -63,6 +69,10 @@ export function buildIndependentAuditInstruction(charter, attempt) {
         "Inspect the final diff and related code directly from the repository.",
         "Look for unmet acceptance criteria, regressions, security issues, data and concurrency risks, missing tests, and unrelated changes.",
         "Treat executor claims as untrusted until supported by repository evidence.",
+        "Inspect every structured PR preparation row. Confirm required rows are bound to the current head and raw evidence; confirm not-applicable rows have a genuine boundary reason and authoritative source. Check exact companion/manifest revisions, scenario outcomes, cleanup residual count, telemetry redaction, locale/browser evidence, and residual-work disclosure where applicable.",
+        "For conditional safeguards, inspect the actual changed boundary: checklist rendering must resist H1-H6, leading-space, and trailing-hash lookalikes while keeping one official controller table; release work must derive its target from authoritative topology and define exact-revision observation; live verification must declare its write policy and prove residual test data zero; loop-state/rendering changes must preserve artifacts and recompute derived views across normal/checkpoint and supported paid resumes.",
+        "Confirm the remote base was freshly read, is an ancestor of the final head, and matches the PR base SHA. Treat DIRTY/BEHIND, stale companion SHAs, changed manifests, or conflict resolution without rerun evidence as blockers.",
+        "Inspect skipped GitHub checks independently. A documented skip may be legitimate, but it must not be represented as an executed full/live/E2E result.",
         "Compare the original request and frozen experience to the complete stable criterion-ID list and implementation mapping. Report forgotten features, dead controls, unsaved results and broken integration. Open the raw minimum evidence, verify its oracle and actual runtime observations; matching expected/actual JSON and hashes alone do not prove truth. Check reload/session return and persisted values/ownership when saving is required.",
         "For every stable criterion ID, return a criterion_results entry only after directly checking its cited proof and observed outcome.",
         "For production-relevant criteria, verify that each outcome and guardrail measure names its signal, independent source, observation window, threshold, decision, owner, and criterion ID.",

package/src/orchestrator/index.js

@@ -4,3 +4,10 @@ export { buildIndependentAuditInstruction } from "./independent-audit.js";
 export { recheckPrReadiness, runCodingLoop } from "./loop.js";
 export { mergeTokenUsage, parseStructuredMarker, parseStructuredResult } from "./result.js";
 export { buildRetryFingerprint, evaluateRetryCircuit } from "./retry.js";
+export {
+  PR_PREPARATION_CHECKS,
+  defaultPrPreparationContract,
+  evaluatePrPreparation,
+  independentAuditEvidenceIdentity,
+  skippedChecksDisclosure,
+} from "./pr-preparation.js";

package/src/orchestrator/instruction.js

@@ -2,6 +2,7 @@ import { minimumCharter } from "../codex-plugins/simy/skills/agentic-delivery-lo
 import { bullets, looksLikeUiTask, section } from "./shared.js";
 import { problemSolvingResultContract } from "./problem-solving.js";
 import { acceptanceCriterionRows } from "./completion-contract.js";
+import { PR_PREPARATION_CHECKS } from "./pr-preparation.js";
 
 export function buildCodingInstruction(
   charter,
@@ -72,6 +73,24 @@ export function buildCodingInstruction(
       `Context: run_id=${charter.id}, stage=locally_verified, environment=local, fixture_id=${charter.id}; input_sha256 is contextDigest(spec.source); source_revisions maps ${charter.repository} to current full HEAD.`,
     ].join("\n")),
     section("expected_tests", bullets(charter.expected_tests)),
+    section("pr_preparation_contract", [
+      "Before reporting the PR ready, complete every preparation row below against the final committed HEAD. Do not reuse evidence after the head, fetched base, companion revision, manifest, policy, or relevant environment changes.",
+      ...PR_PREPARATION_CHECKS.map((row) => `- ${row.id}: ${row.label}`),
+      "Return one pr_preparation entry per ID. A required row needs passed=true, a concrete observation, evidence_refs that resolve to a checked repository file or an observed:* controller fact, and the current full commit SHA in bound_revisions.",
+      "Use applicability=not_applicable only when the changed boundary genuinely excludes the check; include reason, authoritative_source, verified evidence_refs, and the current full commit SHA. Missing tools, failed checks, time, or lack of evidence are pending/failed, not not-applicable.",
+      "exact_revision_integration requires companion_revisions entries with repository/full 40-character sha/evidence_ref and/or manifest_revisions entries with manifest/SHA-256/evidence_ref, or a verified N/A boundary when neither exists. e2e_outcome_and_cleanup requires cleanup_residual_count=0.",
+      "security_and_privacy requires security_privacy_checks. ui_and_i18n requires browser_artifacts, locales_checked, console_error_count=0, and network_failure_count=0, or a verified N/A boundary for non-UI work.",
+      "The final four rows are conditional safeguards. Mark them not_applicable only with current-head-bound verified boundary evidence, never merely because verification is inconvenient.",
+      "official_checklist_ownership applies when checklist/chat rendering changes: verify heading levels 1-6, leading spaces, trailing hashes, exactly one controller-owned official checklist, and an explicit unverified label for model-generated lookalikes.",
+      "deployment_target_mapping applies when release or live verification is in scope: report deployment_target, a verified deployment_target_source evidence ref, and revision_observation_method. Derive the environment from authoritative topology instead of assuming app-dev or production.",
+      "live_data_safety applies to shared/live verification: report live_write_policy as no_persistence, scoped_fixture, or authorized_production_write and cleanup_residual_count=0. Do not infer production-write authority from verification scope.",
+      "resume_compatibility applies when Delivery Loop state, derived views, or rendering changes: exercise normal and checkpoint resume; report paid_resume_support as supported or not_supported with a verified paid_resume_support_source evidence ref, exercise paid resume when supported, and prove derived_views_recomputed and artifact_content_preserved.",
+      "For current_base_and_head, fetch the remote base immediately before final verification, integrate it, and rerun invalidated checks after conflict resolution.",
+      "For current_ci_and_skips, verify the published current HEAD. Return every skipped GitHub check in skipped_checks with name, reason, and impact; a skipped full/live job must not be described as executed E2E.",
+      "For exact_revision_integration, pin every companion repository/manifest SHA and exercise the combined path. For e2e_outcome_and_cleanup, inspect expected scenario states and prove residual fixture count is zero; a green job with failed cleanup is not passing evidence.",
+      "For security_and_privacy, verify changed trust boundaries and that arbitrary messages, email addresses, URLs, identifiers, credentials, or user data are not retained in logs, breadcrumbs, or telemetry. For UI changes, exercise the actual locale/browser, layout overflow, console/network, and screenshot evidence.",
+      "Disclose remaining work, intentional skips, uncertainty, and post-merge migration/deploy/monitoring steps without calling them complete.",
+    ].join("\n")),
     section("must_not", bullets(charter.must_not)),
     section(
       "active_request_reconciliation",
@@ -161,11 +180,12 @@ export function buildCodingInstruction(
       "result_contract",
       [
         "Finish with exactly one line beginning SIMY_RESULT_JSON: followed by one JSON object.",
-        "Required keys: outcome_kind, summary, branch_name, commit_sha, commit_message_headline, pr_url, pr_number, pr_title, pr_base_branch, tests_run, tests_passed, ui_evidence_path, unrelated_changes_detected, secret_scan_passed, changed_files, residual_risks, problem_solving, uncertainty_verification, criterion_evidence, visual_review, delivery_evidence_path.",
+        "Required keys: outcome_kind, summary, branch_name, commit_sha, commit_message_headline, pr_url, pr_number, pr_title, pr_base_branch, tests_run, tests_passed, ui_evidence_path, unrelated_changes_detected, secret_scan_passed, changed_files, residual_risks, problem_solving, uncertainty_verification, criterion_evidence, visual_review, delivery_evidence_path, pr_preparation, skipped_checks.",
         problemSolvingResultContract(),
         "uncertainty_verification must contain top_uncertainty, first_verification, evidence_checked, observed_result, resolved, and fallback. Only set resolved=true when concrete observed evidence resolves it.",
         "criterion_evidence must contain one entry for every stable criterion ID with criterion_id, passed, verification_method, evidence_refs, and observed_result. Only set passed=true when the evidence directly proves that criterion.",
         "For UI or browser-extension work, visual_review must contain inspected=true, the local screenshot or video paths in artifacts, and concrete acceptance_observations from viewing the actual content. Capturing a file without viewing it does not count.",
+        "pr_preparation must contain the eight stable preparation IDs and four conditional safeguard IDs. skipped_checks must contain name, reason, and impact for every GitHub check that intentionally skipped.",
         "Use null for unavailable scalar values and [] for unavailable arrays. Do not claim evidence that was not observed.",
       ].join("\n"),
     ),

package/src/orchestrator/loop.js

@@ -9,6 +9,7 @@ import { canStartProvider, refreshBudgetState } from "./budget.js";
 import { buildAttempt, mergeTokenUsage } from "./result.js";
 import { evaluateRetryCircuit } from "./retry.js";
 import { appendEvent, publish } from "./shared.js";
+import { independentAuditEvidenceIdentity } from "./pr-preparation.js";
 
 async function runCodingLoopOwned({
   snapshot,
@@ -212,6 +213,7 @@ async function runCodingLoopOwned({
     if (shouldStop()) return stopCodingLoop(snapshot, onUpdate);
     const auditInstruction = buildIndependentAuditInstruction(snapshot.charter, attempt);
     const reviewedMatrix = attempt.observed_evidence?.minimum_completion?.matrix_sha256;
+    const reviewedEvidenceIdentity = independentAuditEvidenceIdentity(attempt);
     const auditExecution = await safeExecution(() =>
       executeIndependentAudit({
         attemptNumber,
@@ -231,6 +233,7 @@ async function runCodingLoopOwned({
     attempt.budget = refreshBudgetState(snapshot);
 
     attempt.independent_audit.minimum_matrix_sha256 = reviewedMatrix;
+    attempt.independent_audit.evidence_identity_sha256 = reviewedEvidenceIdentity;
 
     // Re-collect after the read-only auditor to catch any mutated HEAD or working tree.
     attempt.observed_evidence = await safeEvidence(collectEvidence, attempt);