canary-test-cli 6.5.0 → 6.6.0

package/dist/engine/core/framework-registry.js

@@ -12,7 +12,10 @@ import { readFileSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { def } from '../util/coalesce.js';
-import { scaffoldableFrameworks } from './scaffolder.js';
+// The leaf data module, not `scaffolder.js`: the scaffolder imports this
+// registry to degrade loudly on an unknown framework, so taking the set from
+// there would close a cycle (#543).
+import { scaffoldableFrameworks } from './scaffold-templates.js';
 /** Default registry path: `<module dir>/../data/frameworks/registry.json`. */
 export function defaultRegistryPath() {
     const here = dirname(fileURLToPath(import.meta.url));

package/dist/engine/core/scaffold-templates.js

@@ -0,0 +1,123 @@
+/**
+ * Scaffold templates - the config-file bodies canary writes for each supported
+ * framework, and the derived set of frameworks it can scaffold.
+ *
+ * Extracted from `scaffolder.ts` to break a circular dependency (#543):
+ * `framework-registry` needs {@link scaffoldableFrameworks} to answer the
+ * scaffold capability question, while `scaffolder` needs the registry to
+ * degrade loudly on an unknown framework. With the data here, both depend on a
+ * leaf module and neither depends on the other. `scaffolder.ts` re-exports
+ * both names, so existing importers (the migrator, the parity suite) are
+ * unaffected.
+ *
+ * The template strings remain a **byte-exact contract** with the Python
+ * originals - see the note in `scaffolder.ts`. They were moved by line slice,
+ * not retyped, and `core-parity.test.ts` compares TEMPLATES against the golden
+ * fixture.
+ */
+// Exported so the migrator port can compute would-create / already-present sets
+// in its dry-run path (Python: `from agent.core.scaffolder import TEMPLATES`).
+export const TEMPLATES = {
+    playwright: {
+        files: {
+            'playwright.config.ts': `import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests/e2e',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: 'html',
+  use: {
+    trace: 'on-first-retry',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});
+`,
+        },
+        dirs: ['tests/e2e'],
+    },
+    vitest: {
+        files: {
+            'vitest.config.ts': `import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    environment: 'node',
+    include: ['tests/unit/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
+  },
+});
+`,
+        },
+        dirs: ['tests/unit'],
+    },
+    pytest: {
+        files: {
+            'pytest.ini': `[pytest]
+testpaths = tests
+python_files = test_*.py *_test.py
+python_classes = Test*
+python_functions = test_*
+`,
+        },
+        dirs: ['tests'],
+    },
+    k6: {
+        files: {
+            'k6.config.js': `export const options = {
+  vus: 10,
+  duration: '30s',
+};
+`,
+        },
+        dirs: ['tests/performance'],
+    },
+    wdio: {
+        files: {
+            'wdio.conf.ts': `import type { Options } from "@wdio/types";
+
+// Appium + WebdriverIO config. Fill in the capabilities stub below for the
+// device/platform under test (Android shown; add an iOS entry as needed).
+export const config: Options.Testrunner = {
+  runner: "local",
+  specs: ["./tests/**/*.spec.ts"],
+  maxInstances: 1,
+  // Appium capabilities stub \u{2014} replace deviceName / app / versions to match
+  // your emulator or real device.
+  capabilities: [
+    {
+      platformName: "Android",
+      "appium:automationName": "UiAutomator2",
+      "appium:deviceName": "Android Emulator",
+      "appium:app": "./app/build/outputs/apk/debug/app-debug.apk",
+    },
+  ],
+  framework: "mocha",
+  mochaOpts: {
+    ui: "bdd",
+    timeout: 60000,
+  },
+  reporters: ["spec"],
+  // Requires the Appium service: \`npm i -D @wdio/appium-service appium\`.
+  services: ["appium"],
+};
+`,
+        },
+        dirs: ['tests'],
+    },
+};
+/**
+ * Frameworks canary can scaffold - the single source of truth for the
+ * `scaffold` capability, derived from the templates that actually exist
+ * (Python: `scaffoldable_frameworks`).
+ */
+export function scaffoldableFrameworks() {
+    return new Set(Object.keys(TEMPLATES));
+}
+//# sourceMappingURL=scaffold-templates.js.map

package/dist/engine/core/scaffolder.js

@@ -23,6 +23,10 @@
 import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { FrameworkRegistry } from './framework-registry.js';
+import { TEMPLATES } from './scaffold-templates.js';
+// Re-exported so existing importers keep one entry point for the scaffold
+// surface (Python: `from agent.core.scaffolder import TEMPLATES`).
+export { TEMPLATES, scaffoldableFrameworks } from './scaffold-templates.js';
 // ---------------------------------------------------------------------------
 // Python-compatibility helper (copied locally per-module, matching reporter.ts)
 // ---------------------------------------------------------------------------
@@ -41,103 +45,6 @@ function pyTruthy(value) {
         return Object.keys(value).length > 0;
     return Boolean(value);
 }
-// Exported so the migrator port can compute would-create / already-present sets
-// in its dry-run path (Python: `from agent.core.scaffolder import TEMPLATES`).
-export const TEMPLATES = {
-    playwright: {
-        files: {
-            'playwright.config.ts': `import { defineConfig, devices } from '@playwright/test';
-
-export default defineConfig({
-  testDir: './tests/e2e',
-  fullyParallel: true,
-  forbidOnly: !!process.env.CI,
-  retries: process.env.CI ? 2 : 0,
-  workers: process.env.CI ? 1 : undefined,
-  reporter: 'html',
-  use: {
-    trace: 'on-first-retry',
-  },
-  projects: [
-    {
-      name: 'chromium',
-      use: { ...devices['Desktop Chrome'] },
-    },
-  ],
-});
-`,
-        },
-        dirs: ['tests/e2e'],
-    },
-    vitest: {
-        files: {
-            'vitest.config.ts': `import { defineConfig } from 'vitest/config';
-
-export default defineConfig({
-  test: {
-    environment: 'node',
-    include: ['tests/unit/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
-  },
-});
-`,
-        },
-        dirs: ['tests/unit'],
-    },
-    pytest: {
-        files: {
-            'pytest.ini': `[pytest]
-testpaths = tests
-python_files = test_*.py *_test.py
-python_classes = Test*
-python_functions = test_*
-`,
-        },
-        dirs: ['tests'],
-    },
-    k6: {
-        files: {
-            'k6.config.js': `export const options = {
-  vus: 10,
-  duration: '30s',
-};
-`,
-        },
-        dirs: ['tests/performance'],
-    },
-    wdio: {
-        files: {
-            'wdio.conf.ts': `import type { Options } from "@wdio/types";
-
-// Appium + WebdriverIO config. Fill in the capabilities stub below for the
-// device/platform under test (Android shown; add an iOS entry as needed).
-export const config: Options.Testrunner = {
-  runner: "local",
-  specs: ["./tests/**/*.spec.ts"],
-  maxInstances: 1,
-  // Appium capabilities stub \u{2014} replace deviceName / app / versions to match
-  // your emulator or real device.
-  capabilities: [
-    {
-      platformName: "Android",
-      "appium:automationName": "UiAutomator2",
-      "appium:deviceName": "Android Emulator",
-      "appium:app": "./app/build/outputs/apk/debug/app-debug.apk",
-    },
-  ],
-  framework: "mocha",
-  mochaOpts: {
-    ui: "bdd",
-    timeout: 60000,
-  },
-  reporters: ["spec"],
-  // Requires the Appium service: \`npm i -D @wdio/appium-service appium\`.
-  services: ["appium"],
-};
-`,
-        },
-        dirs: ['tests'],
-    },
-};
 /**
  * Handles initialization and scaffolding of test suites.
  *
@@ -225,12 +132,4 @@ export class Scaffolder {
         };
     }
 }
-/**
- * Frameworks canary can scaffold - the single source of truth for the
- * `scaffold` capability, derived from the templates that actually exist
- * (Python: `scaffoldable_frameworks`).
- */
-export function scaffoldableFrameworks() {
-    return new Set(Object.keys(TEMPLATES));
-}
 //# sourceMappingURL=scaffolder.js.map

package/dist/engine/core/static-linter.js

@@ -26,10 +26,36 @@ const BARE_PLAYWRIGHT_CALL = /(?<!await\s)(?<!return\s)(?<!\w)(?:page|frame|loca
 const TEST_FN_PY = /^(\s*)def (test_\w+)\s*\(/gm;
 const TEST_FN_JS = /(?:^|\s)(?:it|test)\s*\(\s*['"]([^'"]*)['"]/gm;
 const ASSERT_PY = /\bassert\b|\bpytest\.raises\b/;
-const ASSERT_JS = /\bexpect\s*\(|\bto(?:Be|Equal|Contain|Have|Match|Throw|Raise)\b/;
+// Assertion styles a JS/TS test may use. `expect()` (jest/vitest/playwright)
+// was the only one recognized until canary was pointed at its own suites and
+// reported 216 assertion-free tests of which 13 were real -- the other 200 were
+// `node:test` + `node:assert`, a whole framework the linter could not see.
+// Kept as a union of shapes rather than an import-aware parse: a static linter
+// that needs to resolve imports to judge one line is the wrong trade.
+// The `expectX()/assertX()` alternative covers a test that delegates its
+// assertion to a named helper (`expectAuthoringAllowed(res)`) -- 9 of canary's
+// own 16 residual findings. A regex linter cannot follow the call, so the NAME
+// carries the signal; the `[A-Z]` keeps it to the convention rather than
+// excusing any call that merely starts with those letters.
+const ASSERT_JS = /\bexpect\s*\(|\bto(?:Be|Equal|Contain|Have|Match|Throw|Raise)\b|\bassert\s*\.\s*\w+\s*\(|\bassert\s*\(|\bshould\s*\.|\.should\b|\b(?:expect|assert)[A-Z]\w*\s*\(/;
 // Strippers
 const STRING_LITERAL = /(['"])(?:\\.|(?!\1).)*?\1/g;
-// Magic numbers
+// Magic numbers -- scoped to TIMING values only.
+//
+// "Extract the magic number to a named constant" is a production-code
+// principle, and it inverts in a test: the literal IS the specification.
+// `expect(notes.length).toBe(2048)` states the contract that
+// `expect(notes.length).toBe(MAX_NOTES)` hides behind a name the reader now has
+// to go look up. Since this linter only ever reads TEST files, the unscoped
+// rule was misapplied across its entire domain -- measured at 0-for-157
+// actionable when canary was first pointed at its own suites.
+//
+// A timing value is the one case that survives: a bare `5000` in a
+// setTimeout/retry/interval position is a duration whose units and intent are
+// genuinely unclear, and naming it genuinely helps. (Hardcoded sleeps are
+// separately flagged at CRITICAL by FLAKE-001/002; this is the softer signal
+// for the non-sleep timing values those rules do not cover.)
+const TIMING_CONTEXT = /\b(?:setTimeout|setInterval|waitForTimeout|sleep|delay|timeout|interval|retryDelay|retries|backoff|pollInterval|debounce|throttle)\b/i;
 const NUMERIC_LITERAL = /(?<![\w.])-?\d+(?:\.\d+)?(?![\w.])/g;
 const ALLOWED_NUMBERS = new Set(['0', '1', '2', '-1', '10', '100']);
 const HTTP_STATUS = new Set([
@@ -99,11 +125,33 @@ const FLAKINESS_RULES = [
         suggestion: 'Mock Date.now()/datetime.now() or use a fixed reference date.',
     },
 ];
+/**
+ * Blank single-line string literals so a rule matching CODE cannot fire on test
+ * DATA.
+ *
+ * `scanMagicNumbers` has always done this; the flakiness and missing-await
+ * rules never did, so `const src = 'const t = Date.now();'` -- a fixture string
+ * feeding a linter test -- was reported as a real timestamp dependency. Any
+ * suite that carries the patterns it tests as string data hits this, and
+ * canary's own linter tests are the worst case.
+ *
+ * The same defect shipped in `canary-blackhawk`'s pragma parser (#499) and was
+ * guarded in `canary-savant` (#495/#498): data must never act as code, nor as
+ * directive.
+ *
+ * NOT applied to the selector rules: LINT-001/002/003 match `'.btn'` / `'#id'`
+ * inside quotes by construction, because a selector IS a string. Stripping
+ * would delete those rules outright.
+ */
+function blankStrings(line) {
+    return line.replace(STRING_LITERAL, '""');
+}
 function scanFlakiness(lines, file) {
     const out = [];
-    lines.forEach((line, idx) => {
-        if (isComment(line))
+    lines.forEach((raw, idx) => {
+        if (isComment(raw))
             return;
+        const line = blankStrings(raw);
         for (const r of FLAKINESS_RULES) {
             if (r.re.test(line) && (!r.guard || r.guard(line))) {
                 out.push(mk(file, idx + 1, r.rule, r.severity, r.message, r.suggestion));
@@ -137,25 +185,70 @@ function scanSelectors(lines, file) {
 }
 function scanMissingAwait(lines, file) {
     const out = [];
-    lines.forEach((line, idx) => {
-        if (isComment(line))
+    lines.forEach((raw, idx) => {
+        if (isComment(raw))
             return;
+        // A `page.click(...)` inside a string is fixture data, not a missing await.
+        const line = blankStrings(raw);
         if (BARE_PLAYWRIGHT_CALL.test(line) && !line.includes('await')) {
             out.push(mk(file, idx + 1, 'LINT-004', 'critical', 'Playwright action called without await.', 'Add `await` before the call to ensure it completes before the next step.'));
         }
     });
     return out;
 }
+/** Multi-line string delimiters: JS template literal, Python triple quotes. */
+const MULTILINE_DELIMS = ['`', '"""', "'''"];
+/**
+ * Blank the INTERIOR of multi-line strings, preserving line count and numbering.
+ *
+ * Every per-line rule (magic numbers, selectors, flakiness) sees one line at a
+ * time, so a line inside a multi-line template literal or a Python
+ * triple-quoted block reads as bare code. Canary's own diff fixtures are
+ * template literals, so `100644` -- a git file mode sitting in test DATA -- was
+ * reported as a magic number 30 times. Any consumer with a multi-line SQL,
+ * JSON, HTML, or diff fixture has the same defect.
+ *
+ * Deliberately conservative about an UNBALANCED delimiter (a stray backtick in
+ * a comment, say): blanking to end-of-file would silently disable these rules
+ * from that point down -- the abstention shape, one layer inside the linter. An
+ * unclosed run is therefore discarded rather than applied.
+ */
+function blankMultilineStrings(lines) {
+    const out = [...lines];
+    for (const delim of MULTILINE_DELIMS) {
+        let openAt = null;
+        for (let i = 0; i < out.length; i += 1) {
+            const hits = out[i].split(delim).length - 1;
+            // An even count opens and closes on the same line, which the single-line
+            // stripper already handles; only an odd count toggles the state.
+            if (hits === 0 || hits % 2 === 0)
+                continue;
+            if (openAt === null) {
+                openAt = i;
+            }
+            else {
+                for (let j = openAt + 1; j < i; j += 1)
+                    out[j] = '';
+                openAt = null;
+            }
+        }
+    }
+    return out;
+}
 function scanMagicNumbers(lines, file) {
     const out = [];
     lines.forEach((raw, idx) => {
         if (isComment(raw))
             return;
         const scrubbed = raw.replace(STRING_LITERAL, '""');
+        // Only timing positions: everywhere else in a test file the literal is the
+        // specification, not a smell. See TIMING_CONTEXT above.
+        if (!TIMING_CONTEXT.test(scrubbed))
+            return;
         for (const m of scrubbed.matchAll(NUMERIC_LITERAL)) {
             if (isAllowedNumber(m[0]))
                 continue;
-            out.push(mk(file, idx + 1, 'LINT-005', 'info', `Magic number ${m[0]}.`, 'Extract to a named constant or derive from test data.'));
+            out.push(mk(file, idx + 1, 'LINT-005', 'info', `Magic timing value ${m[0]}.`, 'Name the duration (e.g. RETRY_DELAY_MS) so its units and intent are readable.'));
             break; // one finding per line
         }
     });
@@ -185,9 +278,23 @@ function scanAssertionFreePy(code, file) {
 }
 function scanAssertionFreeJs(code, file) {
     const out = [];
-    for (const m of code.matchAll(TEST_FN_JS)) {
+    // The test declarations, in source order, so each body can be bounded by the
+    // NEXT one -- the JS analogue of what the pytest scanner already does with
+    // "next `def` at the same indent".
+    //
+    // This replaces a fixed 2000-character lookahead that was wrong in BOTH
+    // directions: a long test whose first assertion fell past the window was
+    // flagged (false positive), and a short empty test could borrow the next
+    // test's assertion from inside the window (false negative). Neither failure
+    // is visible without a real codebase to run it against, which is why
+    // dogfooding found them and the unit tests did not.
+    const decls = [...code.matchAll(TEST_FN_JS)];
+    for (let i = 0; i < decls.length; i += 1) {
+        const m = decls[i];
         const start = m.index;
-        const rest = code.slice(start + m[0].length, start + m[0].length + 2000);
+        const bodyStart = start + m[0].length;
+        const bodyEnd = decls[i + 1]?.index ?? code.length;
+        const rest = code.slice(bodyStart, bodyEnd);
         if (!ASSERT_JS.test(rest)) {
             out.push(mk(file, lineOf(code, start), 'LINT-006', 'warning', `Test "${m[1]}" contains no assertions.`, 'Add an expect() call; a test that never asserts always passes.'));
         }
@@ -209,7 +316,12 @@ export class StaticLinter {
     /** Full quality audit — all rules. */
     lint(path, framework) {
         const code = readFileSync(path, 'utf-8');
-        const lines = code.split('\n');
+        // No rule may read the interior of a multi-line string as code. Blanking
+        // preserves line count, so `scanned` re-joins to the same line numbers the
+        // per-line scanners report -- both halves must use it, or the assertion
+        // scanners go on mining `it(...)` declarations out of diff fixtures.
+        const lines = blankMultilineStrings(code.split('\n'));
+        const scanned = lines.join('\n');
         const fw = framework || detectFramework(path);
         const findings = [
             ...scanFlakiness(lines, path),
@@ -217,8 +329,8 @@ export class StaticLinter {
             ...scanMissingAwait(lines, path),
             ...scanMagicNumbers(lines, path),
             ...(fw === 'pytest'
-                ? scanAssertionFreePy(code, path)
-                : scanAssertionFreeJs(code, path)),
+                ? scanAssertionFreePy(scanned, path)
+                : scanAssertionFreeJs(scanned, path)),
         ];
         findings.sort((a, b) => a.line - b.line || cmp(a.rule, b.rule));
         return findings;
@@ -226,7 +338,7 @@ export class StaticLinter {
     /** Flakiness-only subset. */
     flakeCheck(path) {
         const code = readFileSync(path, 'utf-8');
-        const findings = scanFlakiness(code.split('\n'), path);
+        const findings = scanFlakiness(blankMultilineStrings(code.split('\n')), path);
         findings.sort((a, b) => a.line - b.line);
         return findings;
     }

package/dist/engine/history/async-store.js

@@ -0,0 +1,20 @@
+/**
+ * The async history-store contract.
+ *
+ * Extracted from `store.ts` to break a circular dependency (#543): the store
+ * factory has to import every concrete backend it can return, while each
+ * backend has to import the interface it implements. Both now point at this
+ * leaf and neither points at the other. `store.ts` re-exports the type, so
+ * existing importers are unaffected.
+ *
+ * The dependency was type-only, and type-only imports are erased before
+ * anything runs — so this was never a runtime hazard. It is fixed anyway
+ * because `harness check-deps` counts it, and a cycle the gate reports is a
+ * cycle whatever the emitted JavaScript does.
+ *
+ * The contract itself is async (unlike the synchronous Python `HistoryStore`
+ * ABC) because `@supabase/supabase-js` is Promise-based; see `store.ts` for the
+ * full boundary note.
+ */
+export {};
+//# sourceMappingURL=async-store.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "canary-test-cli",
-  "version": "6.5.0",
+  "version": "6.6.0",
   "description": "Canary — AI-powered test automation agent",
   "license": "MIT",
   "repository": {