flakeiq 1.0.11

package/lib/seed.js

@@ -0,0 +1,30 @@
+const { spawn } = require('child_process');
+const path = require('path');
+const { getConfig } = require('./config');
+
+function seed(dbPath, options = {}) {
+  const config = getConfig();
+  const python = config.venvPython;
+  const seedScript = path.join(config.pythonDir, 'seed.py');
+
+  const args = [seedScript];
+  if (dbPath) args.push(dbPath);
+
+  const child = spawn(python, args, {
+    stdio: 'inherit',
+    env: { ...process.env, FLAKE_DB: dbPath || config.dbPath },
+  });
+
+  child.on('error', (err) => {
+    if (err.code === 'ENOENT') {
+      console.error('Python not found. Run "npx flakeiq setup" or install Python 3.11+.');
+    } else {
+      console.error('Seed error:', err.message);
+    }
+    process.exit(1);
+  });
+
+  child.on('exit', (code) => { process.exit(code || 0); });
+}
+
+module.exports = { seed };

package/lib/serve.js

@@ -0,0 +1,59 @@
+const { spawn } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const { getConfig } = require('./config');
+const { findAvailablePort } = require('./utils');
+
+async function serve(options = {}) {
+  const config = getConfig();
+  const python = config.venvPython;
+  const dashboardScript = path.join(config.pythonDir, 'dashboard.py');
+  const seedScript = path.join(config.pythonDir, 'seed.py');
+
+  const port = options.port || config.port;
+  const host = options.host || config.host;
+  const actualPort = await findAvailablePort(port, host);
+  const useSeed = options.seed || false;
+  const dbPath = options.db || config.dbPath;
+
+  if (useSeed && !fs.existsSync(dbPath)) {
+    console.log('Seed database not found. Generating...');
+    await new Promise((resolve, reject) => {
+      const seedChild = spawn(python, [seedScript, dbPath], {
+        stdio: 'inherit',
+        env: { ...process.env, FLAKE_DB: dbPath },
+      });
+      seedChild.on('exit', (code) => code === 0 ? resolve() : reject(new Error('Seed failed')));
+      seedChild.on('error', reject);
+    });
+  }
+
+  const args = [dashboardScript, '--port', String(actualPort), '--host', host, '--db', dbPath];
+  if (options.open !== false) args.push('--open');
+
+  console.log(`Starting FlakeIQ dashboard on http://${host}:${actualPort}`);
+  if (useSeed) console.log('Using seed database...');
+
+  const child = spawn(python, args, {
+    stdio: 'inherit',
+    env: { ...process.env, FLAKE_DB: dbPath },
+  });
+
+  child.on('error', (err) => {
+    if (err.code === 'ENOENT') {
+      console.error('Python not found. Run "npx flakeiq setup" or install Python 3.11+.');
+    } else {
+      console.error('Dashboard error:', err.message);
+    }
+    process.exit(1);
+  });
+
+  child.on('exit', (code) => {
+    process.exit(code || 0);
+  });
+
+  process.on('SIGINT', () => { child.kill('SIGINT'); });
+  process.on('SIGTERM', () => { child.kill('SIGTERM'); });
+}
+
+module.exports = { serve };

package/lib/setup.js

@@ -0,0 +1,79 @@
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const { getVenvDir, getVenvPython, PYTHON_DIR } = require('./config');
+const { getPythonVersion } = require('./utils');
+
+const REQUIREMENTS = path.join(PYTHON_DIR, 'requirements.txt');
+
+function findSystemPython() {
+  const candidates = process.platform === 'win32'
+    ? ['python', 'python3', 'py -3']
+    : ['python3', 'python'];
+
+  for (const cmd of candidates) {
+    const ver = getPythonVersion(cmd);
+    if (ver && (ver.major > 3 || (ver.major === 3 && ver.minor >= 11))) {
+      return cmd;
+    }
+  }
+  return null;
+}
+
+function createVenv(pythonCmd) {
+  const venvDir = getVenvDir();
+  if (fs.existsSync(getVenvPython())) return true;
+
+  console.log('  Creating Python virtual environment...');
+  try {
+    execSync(`"${pythonCmd}" -m venv "${venvDir}"`, { stdio: 'pipe', timeout: 60000 });
+    return true;
+  } catch (e) {
+    console.error('  Failed to create venv:', e.message);
+    return false;
+  }
+}
+
+function installDeps() {
+  const pip = process.platform === 'win32'
+    ? path.join(getVenvDir(), 'Scripts', 'pip.exe')
+    : path.join(getVenvDir(), 'bin', 'pip3');
+
+  if (!fs.existsSync(pip)) {
+    console.error('  pip not found in venv');
+    return false;
+  }
+
+  console.log('  Installing Python dependencies...');
+  try {
+    execSync(`"${pip}" install -r "${REQUIREMENTS}" --quiet`, { stdio: 'pipe', timeout: 120000 });
+    return true;
+  } catch (e) {
+    console.error('  Failed to install deps:', e.message);
+    return false;
+  }
+}
+
+function setup() {
+  const python = findSystemPython();
+  if (!python) {
+    console.log('');
+    console.log('  WARNING: Python 3.11+ not found on your system.');
+    console.log('  FlakeIQ dashboard requires Python. Install it from:');
+    console.log('    https://www.python.org/downloads/');
+    console.log('');
+    console.log('  The flake-reporter will still work without Python.');
+    console.log('');
+    return false;
+  }
+
+  console.log(`  Found Python: ${python}`);
+
+  if (!createVenv(python)) return false;
+  if (!installDeps()) return false;
+
+  console.log('  Python environment ready.');
+  return true;
+}
+
+module.exports = { setup, findSystemPython };

package/lib/utils.js

@@ -0,0 +1,40 @@
+const net = require('net');
+const { execSync } = require('child_process');
+
+function isPortAvailable(port, host = '127.0.0.1') {
+  return new Promise((resolve) => {
+    const server = net.createServer();
+    server.once('error', () => resolve(false));
+    server.once('listening', () => { server.close(() => resolve(true)); });
+    server.listen(port, host);
+  });
+}
+
+function findAvailablePort(startPort, host = '127.0.0.1') {
+  return new Promise(async (resolve) => {
+    for (let port = startPort; port < startPort + 100; port++) {
+      if (await isPortAvailable(port, host)) { resolve(port); return; }
+    }
+    resolve(startPort);
+  });
+}
+
+function openBrowser(url) {
+  const cmd = process.platform === 'win32'
+    ? `start "" "${url}"`
+    : process.platform === 'darwin'
+      ? `open "${url}"`
+      : `xdg-open "${url}"`;
+  try { execSync(cmd, { stdio: 'ignore' }); } catch {}
+}
+
+function getPythonVersion(pythonPath) {
+  try {
+    const out = execSync(`"${pythonPath}" --version`, { encoding: 'utf8', timeout: 5000 }).trim();
+    const match = out.match(/Python (\d+)\.(\d+)/);
+    if (match) return { major: parseInt(match[1]), minor: parseInt(match[2]), raw: out };
+  } catch {}
+  return null;
+}
+
+module.exports = { isPortAvailable, findAvailablePort, openBrowser, getPythonVersion };

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "flakeiq",
+  "version": "1.0.11",
+  "description": "Test flake tracking + LLM failure classification for E2E tests — works with Playwright, Detox, Appium, Selenium, and any framework",
+  "main": "reporter/flake-reporter.js",
+  "bin": {
+    "flakeiq": "./bin/flakeiq.js"
+  },
+  "files": [
+    "bin/",
+    "lib/",
+    "python/",
+    "reporter/",
+    "scripts/"
+  ],
+  "scripts": {
+    "postinstall": "node scripts/postinstall.js",
+    "test": "node --test tests/",
+    "start": "node bin/flakeiq.js serve"
+  },
+  "keywords": [
+    "playwright",
+    "detox",
+    "appium",
+    "selenium",
+    "flaky-tests",
+    "test-results",
+    "e2e",
+    "test-tracker",
+    "llm",
+    "classification",
+    "test-flake"
+  ],
+  "author": "",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18"
+  },
+  "peerDependencies": {
+    "@playwright/test": ">=1.40"
+  },
+  "peerDependenciesMeta": {
+    "@playwright/test": {
+      "optional": true
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sarang-code2/FlakeIQ-Package.git"
+  },
+  "homepage": "https://github.com/sarang-code2/FlakeIQ-Package#readme",
+  "bugs": {
+    "url": "https://github.com/sarang-code2/FlakeIQ-Package/issues"
+  }
+}

package/python/classify.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+import json
+import sqlite3
+import sys
+import os
+import argparse
+from datetime import datetime
+from urllib.request import Request, urlopen
+from urllib.error import URLError
+
+DB_PATH = os.environ.get("FLAKE_DB", "flake.db")
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434/api/generate")
+MODEL = os.environ.get("OLLAMA_MODEL", "llama3.2")
+
+CLASSIFICATION_PROMPT = """You are a test failure classifier. Given a mobile E2E test failure, classify it into exactly one category.
+
+Categories:
+- REAL_BUG: the app behavior is wrong (element missing, wrong UI state, crash, assertion failure)
+- TIMEOUT_FLAKE: action timed out but the element typically appears eventually (race condition, slow render)
+- DEVICE_FLAKE: device/emulator issue (connection lost, ANR dialog, memory pressure, "could not verify foreground")
+- LOCATOR_FLAKE: element found but not visible/interactable, scrollIntoViewIfNeeded failed, stale element
+- UNKNOWN: can't determine from the given information
+
+Reply with ONLY the category name and a one-sentence reason separated by "|".
+Example: "TIMEOUT_FLAKE|actionTimeout exceeded on fill, element appeared on retry"
+
+---
+Test: {test_name}
+Platform: {platform}
+Error: {error_message}
+Last actions: {last_actions}
+"""
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS test_runs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    test_file TEXT,
+    test_name TEXT,
+    platform TEXT,
+    device_id TEXT,
+    duration_ms INTEGER,
+    result TEXT,
+    error_message TEXT,
+    last_actions TEXT,
+    classification TEXT,
+    classification_reason TEXT,
+    screen_name TEXT,
+    action_type TEXT,
+    session_id TEXT,
+    run_at TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_run_at ON test_runs(run_at);
+CREATE INDEX IF NOT EXISTS idx_screen ON test_runs(screen_name);
+CREATE INDEX IF NOT EXISTS idx_result ON test_runs(result);
+"""
+
+
+def init_db():
+    db = sqlite3.connect(DB_PATH)
+    db.executescript(SCHEMA)
+    db.commit()
+    return db
+
+
+def classify_failure(record: dict) -> tuple[str | None, str | None]:
+    prompt = CLASSIFICATION_PROMPT.format(
+        test_name=record["test_name"],
+        platform=record.get("platform", "unknown"),
+        error_message=record.get("error_message", "")[:2000],
+        last_actions=json.dumps(record.get("last_actions", [])),
+    )
+    payload = json.dumps({"model": MODEL, "prompt": prompt, "stream": False}).encode()
+    req = Request(OLLAMA_URL, data=payload, headers={"Content-Type": "application/json"})
+    try:
+        resp = urlopen(req, timeout=30)
+        body = json.loads(resp.read())
+        text = body.get("response", "").strip()
+        if "|" in text:
+            cat, reason = text.split("|", 1)
+            cat = cat.strip().upper()
+            if cat in ("REAL_BUG", "TIMEOUT_FLAKE", "DEVICE_FLAKE", "LOCATOR_FLAKE", "UNKNOWN"):
+                return cat, reason.strip()
+        return "UNKNOWN", f"Could not parse: {text[:200]}"
+    except URLError as e:
+        print(f"  [warn] Ollama unavailable ({e.reason}), skipping classification")
+        return None, None
+    except Exception as e:
+        print(f"  [warn] Classification failed: {e}")
+        return None, None
+
+
+def guess_action_type(last_actions: list[str]) -> str:
+    for action in reversed(last_actions):
+        al = action.lower()
+        if "fill" in al: return "fill"
+        if "tap" in al or "click" in al: return "tap"
+        if "swipe" in al or "scroll" in al: return "swipe"
+        if "scrollintoview" in al: return "scrollIntoView"
+        if "press" in al: return "press"
+    return "other"
+
+
+def process_jsonl(jsonl_path: str, db: sqlite3.Connection):
+    count = 0
+    classified = 0
+    skipped = 0
+    cursor = db.cursor()
+
+    with open(jsonl_path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            record = json.loads(line)
+            count += 1
+
+            # detect duplicate by test_name + run_at
+            cursor.execute(
+                "SELECT id FROM test_runs WHERE test_name=? AND run_at=?",
+                (record["test_name"], record["run_at"]),
+            )
+            if cursor.fetchone():
+                skipped += 1
+                continue
+
+            action_type = guess_action_type(record.get("last_actions", []))
+
+            if record["result"] != "passed":
+                cat, reason = classify_failure(record)
+                if cat:
+                    classified += 1
+                    record["classification"] = cat
+                    record["classification_reason"] = reason
+            else:
+                record["classification"] = None
+                record["classification_reason"] = None
+
+            cursor.execute(
+                """INSERT INTO test_runs
+                   (test_file, test_name, platform, device_id, duration_ms, result,
+                    error_message, last_actions, classification, classification_reason,
+                    screen_name, action_type, session_id, run_at)
+                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                (
+                    record.get("test_file"),
+                    record["test_name"],
+                    record.get("platform"),
+                    record.get("device_id"),
+                    record.get("duration_ms"),
+                    record["result"],
+                    record.get("error_message", ""),
+                    json.dumps(record.get("last_actions", [])),
+                    record.get("classification"),
+                    record.get("classification_reason"),
+                    record.get("screen_name"),
+                    action_type,
+                    record.get("session_id"),
+                    record["run_at"],
+                ),
+            )
+
+    db.commit()
+    print(f"Processed: {count} records ({classified} classified, {skipped} skipped duplicates)")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="FlakeIQ: classify test failures and store in SQLite")
+    parser.add_argument("jsonl", nargs="?", default="flake-results.jsonl",
+                        help="Path to JSONL file from flake-reporter.ts")
+    parser.add_argument("--db", default=DB_PATH, help="SQLite database path")
+    parser.add_argument("--ollama-url", default=OLLAMA_URL, help="Ollama API URL")
+    parser.add_argument("--model", default=MODEL, help="Ollama model name")
+    args = parser.parse_args()
+
+    os.environ["FLAKE_DB"] = args.db
+    os.environ["OLLAMA_URL"] = args.ollama_url
+    os.environ["OLLAMA_MODEL"] = args.model
+
+    if not os.path.exists(args.jsonl):
+        print(f"JSONL file not found: {args.jsonl}")
+        sys.exit(1)
+
+    db = sqlite3.connect(args.db)
+    db.executescript(SCHEMA)
+    db.commit()
+    process_jsonl(args.jsonl, db)
+    db.close()
+
+
+if __name__ == "__main__":
+    main()