@sun-asterisk/sungen 3.0.0-beta.71 → 3.0.0-beta.72

package/src/harness/script-check.ts

@@ -28,10 +28,62 @@ export interface ScriptCheckResult {
   extraInSpec: string[];        // test() titles with no scenario
   drift: 'in-sync' | 'drift' | 'no-spec';
   driftHunks: string[];         // sample differing lines (committed vs regenerated)
+  // C. Anti-bypass / faithfulness — a present test that proves nothing.
+  assertionlessTests: string[];                    // non-manual test() with 0 expect()
+  hollowSteps: { test: string; step: string }[];   // a step-comment that emitted no code
+  bypass: boolean;
   status: 'OK' | 'FAIL';
   findings: string[];
 }
 
+/** Split a spec into test() blocks (title + body) via brace counting. */
+export function extractTestBlocks(specSrc: string): { title: string; body: string[] }[] {
+  const lines = specSrc.split('\n');
+  const blocks: { title: string; body: string[] }[] = [];
+  for (let i = 0; i < lines.length; i++) {
+    const m = lines[i].match(/\btest(?:\.(?:only|skip|fixme))?\(\s*(['"`])([^'"`]+)\1/);
+    if (!m) continue;
+    let depth = 0, started = false;
+    const body: string[] = [];
+    for (let j = i; j < lines.length; j++) {
+      for (const ch of lines[j]) { if (ch === '{') { depth++; started = true; } else if (ch === '}') depth--; }
+      body.push(lines[j]);
+      if (started && depth <= 0) break;
+    }
+    blocks.push({ title: m[2].trim(), body });
+  }
+  return blocks;
+}
+
+const CODE_LINE = /\b(await|expect|testData|page\.)\b|=\s*[^=]/;
+
+/**
+ * Faithfulness / anti-bypass: a non-@manual test that performs actions but proves
+ * nothing (0 assertions), or a step-comment that emitted no executable code (hollow
+ * — the testcase step is not run). Structural 1:1 can't see these.
+ */
+export function analyzeFaithfulness(specSrc: string, automatedTitles: Set<string>) {
+  const assertionlessTests: string[] = [];
+  const hollowSteps: { test: string; step: string }[] = [];
+  for (const blk of extractTestBlocks(specSrc)) {
+    if (!automatedTitles.has(blk.title)) continue; // only non-@manual scenarios
+    const body = blk.body;
+    if (!body.some((l) => /expect\(/.test(l))) assertionlessTests.push(blk.title);
+    // hollow step: a `// step` whose region (until the NEXT step-comment / block end)
+    // contains no executable code. The region — not just the next line — is checked,
+    // so block-style steps (`// Assert all … { … expect … }`) are correctly counted.
+    const stepIdx: number[] = [];
+    for (let k = 1; k < body.length; k++) if (body[k].trim().startsWith('//')) stepIdx.push(k);
+    for (let s = 0; s < stepIdx.length; s++) {
+      const start = stepIdx[s] + 1;
+      const end = s + 1 < stepIdx.length ? stepIdx[s + 1] : body.length;
+      const hasCode = body.slice(start, end).some((l) => CODE_LINE.test(l));
+      if (!hasCode) hollowSteps.push({ test: blk.title, step: body[stepIdx[s]].trim().slice(0, 70) });
+    }
+  }
+  return { assertionlessTests, hollowSteps };
+}
+
 function extractTestTitles(specSrc: string): string[] {
   // Count real test cases only: test(...), test.only/.skip/.fixme(...).
   // Exclude test.describe / test.beforeAll / hooks (not test cases).
@@ -51,8 +103,12 @@ function normalize(src: string): string {
     .trim();
 }
 
-function findSpec(dir: string, screen: string): string | null {
-  // generated spec: <dir>/<screen>/<feature>.spec.ts  (or nested)
+function findSpec(dir: string, name: string, flowMode: boolean): string | null {
+  // Screens compile to  <dir>/<name>/<feature>.spec.ts
+  // Flows   compile to  <dir>/flows/<name>/<feature>.spec.ts
+  // Scope the search to THIS target's own subdir — otherwise the first spec of
+  // ANY other screen/flow is returned, which (for an uncompiled flow) falsely
+  // reports the wrong screen's tests as drift.
   const hits: string[] = [];
   const walk = (d: string) => {
     if (!fs.existsSync(d)) return;
@@ -62,7 +118,9 @@ function findSpec(dir: string, screen: string): string | null {
       else if (e.name.endsWith('.spec.ts')) hits.push(p);
     }
   };
-  walk(dir);
+  const scoped = flowMode ? path.join(dir, 'flows', name) : path.join(dir, name);
+  if (!fs.existsSync(scoped)) return null; // no spec for this target (e.g. not compiled yet)
+  walk(scoped);
   return hits[0] ?? null;
 }
 
@@ -72,7 +130,7 @@ export async function runScriptCheck(screenDir: string, screenName: string, flow
   const automated = scenarios.filter((s) => !s.manual);
   const manual = scenarios.filter((s) => s.manual);
 
-  const committedSpec = findSpec(path.join(process.cwd(), 'specs', 'generated'), screenName);
+  const committedSpec = findSpec(path.join(process.cwd(), 'specs', 'generated'), screenName, flowMode);
 
   const findings: string[] = [];
   let specTitles: string[] = [];
@@ -106,7 +164,7 @@ export async function runScriptCheck(screenDir: string, screenName: string, flow
       const qaSourceDir = path.join(process.cwd(), 'qa', flowMode ? 'flows' : 'screens');
       const gen = new CodeGenerator({ framework: 'playwright', screenName, runtimeData: true, flowMode });
       await gen.generateAllTests(qaSourceDir, tmp, [featurePath]);
-      const fresh = findSpec(tmp, screenName);
+      const fresh = findSpec(tmp, screenName, flowMode);
       if (fresh) {
         const a = normalize(specSrc);
         const b = normalize(fs.readFileSync(fresh, 'utf-8'));
@@ -130,7 +188,19 @@ export async function runScriptCheck(screenDir: string, screenName: string, flow
     }
   }
 
-  const ok = !!committedSpec && countMatch && missingInSpec.length === 0 && extraInSpec.length === 0 && drift === 'in-sync';
+  // C. Anti-bypass / faithfulness
+  const { assertionlessTests, hollowSteps } = committedSpec
+    ? analyzeFaithfulness(specSrc, scenTitleSet)
+    : { assertionlessTests: [], hollowSteps: [] };
+  for (const t of assertionlessTests) {
+    findings.push(`BYPASS: test "${t}" has 0 assertions (action-only — proves nothing). The testcase is not really automated.`);
+  }
+  for (const h of hollowSteps) {
+    findings.push(`BYPASS: in "${h.test}", step "${h.step}" emitted no code (hollow — the testcase step is not executed).`);
+  }
+  const bypass = assertionlessTests.length > 0 || hollowSteps.length > 0;
+
+  const ok = !!committedSpec && countMatch && missingInSpec.length === 0 && extraInSpec.length === 0 && drift === 'in-sync' && !bypass;
 
   return {
     screen: screenName,
@@ -143,6 +213,9 @@ export async function runScriptCheck(screenDir: string, screenName: string, flow
     extraInSpec,
     drift,
     driftHunks,
+    assertionlessTests,
+    hollowSteps,
+    bypass,
     status: ok ? 'OK' : 'FAIL',
     findings,
   };

package/src/orchestrator/project-initializer.ts

@@ -42,6 +42,9 @@ export class ProjectInitializer {
     // Create qa/context.md for QA lead to fill project-wide context
     this.createContext();
 
+    // Create qa/capabilities.yaml — the platform driver profile (default: web/Playwright)
+    this.createCapabilities();
+
     // Ensure package.json and install Playwright
     await this.setupDependencies();
 
@@ -386,6 +389,26 @@ export class ProjectInitializer {
     this.createdItems.push('qa/context.md');
   }
 
+  /**
+   * Create qa/capabilities.yaml — the platform driver profile (Phase 2a).
+   * Defaults to `web` (Playwright). Mobile/API projects switch via `sungen capability add`.
+   */
+  private createCapabilities(): void {
+    const p = path.join(this.cwd, 'qa', 'capabilities.yaml');
+    if (fs.existsSync(p)) {
+      this.skippedItems.push('qa/capabilities.yaml');
+      return;
+    }
+    const content =
+      '# Sungen capability profile — which runtime/drivers this project uses.\n' +
+      '# platform: the runtime adapter — web → Playwright, mobile → Appium.\n' +
+      '# enabled: drivers turned on (add more via `sungen capability add <driver>`).\n\n' +
+      'platform: web\n' +
+      'enabled:\n  - web\n';
+    fs.writeFileSync(p, content, 'utf-8');
+    this.createdItems.push('qa/capabilities.yaml');
+  }
+
   /**
    * Create specs/base.ts for shared browser context
    */

package/src/orchestrator/templates/specs-test-data.ts

@@ -41,13 +41,18 @@ export class TestDataLoader {
   }
 
   get(key: string): string {
-    const parts = key.split('.');
-    let current: any = this.data;
-    for (const part of parts) {
-      if (current == null || typeof current !== 'object') {
-        throw new Error(`Test data key not found: ${key} (failed at '${part}')`);
+    // Captured/runtime vars (set() below) are stored under their literal — possibly
+    // dotted — key (e.g. "cart.product_name"), so check the flat key first.
+    let current: any = this.data[key];
+    if (current === undefined || current === null) {
+      // Fall back to nested navigation for YAML-structured keys (e.g. "cart.qty_two").
+      current = this.data;
+      for (const part of key.split('.')) {
+        if (current == null || typeof current !== 'object') {
+          throw new Error(`Test data key not found: ${key} (failed at '${part}')`);
+        }
+        current = current[part];
       }
-      current = current[part];
     }
     if (current === undefined || current === null) {
       throw new Error(`Test data key not found: ${key}`);