feed-the-machine 1.3.1 → 1.4.0

package/ftm-map/scripts/test_parser.py

@@ -1,106 +1,174 @@
-"""Tests for ftm-map parser module."""
+"""Tests for parser.py -- Aider-style def/ref tag extraction."""
 import os
 import sys
-import unittest
+import pytest
 
 sys.path.insert(0, os.path.dirname(__file__))
+from parser import get_tags, detect_language, Tag, EXTENSION_MAP
 
-FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "tests", "fixtures", "sample_project")
-
-class TestParser(unittest.TestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        """Check if tree-sitter-language-pack is available."""
-        try:
-            from parser import parse_file, extract_relationships, detect_language
-            cls.parse_file = staticmethod(parse_file)
-            cls.extract_relationships = staticmethod(extract_relationships)
-            cls.detect_language = staticmethod(detect_language)
-            cls.skip_reason = None
-        except ImportError as e:
-            cls.skip_reason = f"tree-sitter-language-pack not installed: {e}"
-
-    def setUp(self):
-        if self.skip_reason:
-            self.skipTest(self.skip_reason)
-
-    def test_detect_language(self):
-        self.assertEqual(self.detect_language("foo.py"), "python")
-        self.assertEqual(self.detect_language("bar.ts"), "typescript")
-        self.assertEqual(self.detect_language("baz.js"), "javascript")
-        self.assertIsNone(self.detect_language("README.md"))
-
-    def test_parse_python_file(self):
-        """Should extract Python functions."""
-        path = os.path.join(FIXTURES_DIR, "auth.py")
-        if not os.path.exists(path):
-            self.skipTest("Fixture not found")
-        symbols = self.parse_file(path)
-        names = {s.name for s in symbols}
-        self.assertIn("handleAuth", names)
-        self.assertIn("validateToken", names)
-        self.assertIn("getUser", names)
-
-    def test_parse_typescript_file(self):
-        """Should extract TypeScript functions and classes."""
-        path = os.path.join(FIXTURES_DIR, "api.ts")
-        if not os.path.exists(path):
-            self.skipTest("Fixture not found")
-        symbols = self.parse_file(path)
-        names = {s.name for s in symbols}
-        self.assertIn("processRequest", names)
-        self.assertIn("ApiController", names)
-
-    def test_parse_javascript_file(self):
-        """Should extract JavaScript functions."""
-        path = os.path.join(FIXTURES_DIR, "utils.js")
-        if not os.path.exists(path):
-            self.skipTest("Fixture not found")
-        symbols = self.parse_file(path)
-        names = {s.name for s in symbols}
-        self.assertIn("formatDate", names)
-        self.assertIn("parseConfig", names)
-
-    def test_symbol_has_content_hash(self):
-        """Every symbol should have a content hash."""
-        path = os.path.join(FIXTURES_DIR, "auth.py")
-        if not os.path.exists(path):
-            self.skipTest("Fixture not found")
-        symbols = self.parse_file(path)
-        for sym in symbols:
-            self.assertTrue(len(sym.content_hash) > 0, f"{sym.name} missing content_hash")
-
-    def test_symbol_has_line_numbers(self):
-        """Every symbol should have start and end line."""
-        path = os.path.join(FIXTURES_DIR, "auth.py")
-        if not os.path.exists(path):
-            self.skipTest("Fixture not found")
-        symbols = self.parse_file(path)
-        for sym in symbols:
-            self.assertGreater(sym.start_line, 0)
-            self.assertGreaterEqual(sym.end_line, sym.start_line)
-
-    def test_extract_relationships(self):
-        """Should extract call relationships."""
-        path = os.path.join(FIXTURES_DIR, "auth.py")
-        if not os.path.exists(path):
-            self.skipTest("Fixture not found")
-        rels = self.extract_relationships(path)
-        # handleAuth calls validateToken and getUser
-        call_targets = {r.target_name for r in rels if r.kind == "calls"}
-        self.assertIn("validateToken", call_targets)
-        self.assertIn("getUser", call_targets)
-
-    def test_unsupported_file_returns_empty(self):
-        """Unsupported file types should return empty list."""
-        symbols = self.parse_file("/tmp/fake.xyz")
-        self.assertEqual(symbols, [])
-
-    def test_nonexistent_file_returns_empty(self):
-        """Non-existent files should return empty list, not error."""
-        symbols = self.parse_file("/tmp/nonexistent_file_12345.py")
-        self.assertEqual(symbols, [])
-
-if __name__ == "__main__":
-    unittest.main()
+FIXTURES = os.path.join(os.path.dirname(__file__), "tests", "fixtures", "sample_project")
+
+
+class TestDetectLanguage:
+    def test_python(self):
+        assert detect_language("foo.py") == "python"
+
+    def test_typescript(self):
+        assert detect_language("foo.ts") == "typescript"
+
+    def test_javascript(self):
+        assert detect_language("foo.js") == "javascript"
+
+    def test_tsx(self):
+        assert detect_language("foo.tsx") == "tsx"
+
+    def test_unknown(self):
+        assert detect_language("foo.txt") is None
+
+    def test_all_extensions_mapped(self):
+        for ext, lang in EXTENSION_MAP.items():
+            assert lang is not None, f"Extension {ext} mapped to None"
+
+
+class TestPythonTags:
+    def test_extracts_defs_and_refs(self):
+        fpath = os.path.join(FIXTURES, "auth.py")
+        tags = get_tags(fpath, "auth.py")
+        defs = [t for t in tags if t.kind == "def"]
+        refs = [t for t in tags if t.kind == "ref"]
+        assert len(defs) >= 3, f"Expected >=3 defs, got {len(defs)}: {[t.name for t in defs]}"
+        assert len(refs) >= 2, f"Expected >=2 refs, got {len(refs)}: {[t.name for t in refs]}"
+
+    def test_def_names_correct(self):
+        fpath = os.path.join(FIXTURES, "auth.py")
+        tags = get_tags(fpath, "auth.py")
+        def_names = {t.name for t in tags if t.kind == "def"}
+        assert "handleAuth" in def_names
+        assert "validateToken" in def_names
+        assert "getUser" in def_names
+
+    def test_ref_names_correct(self):
+        fpath = os.path.join(FIXTURES, "auth.py")
+        tags = get_tags(fpath, "auth.py")
+        ref_names = {t.name for t in tags if t.kind == "ref"}
+        assert "validateToken" in ref_names
+        assert "getUser" in ref_names
+
+    def test_no_garbled_names(self):
+        fpath = os.path.join(FIXTURES, "auth.py")
+        tags = get_tags(fpath, "auth.py")
+        for tag in tags:
+            assert tag.name.isidentifier(), f"Garbled name: {tag.name}"
+
+    def test_tag_has_correct_fields(self):
+        fpath = os.path.join(FIXTURES, "auth.py")
+        tags = get_tags(fpath, "auth.py")
+        assert len(tags) > 0
+        tag = tags[0]
+        assert hasattr(tag, "rel_fname")
+        assert hasattr(tag, "fname")
+        assert hasattr(tag, "line")
+        assert hasattr(tag, "name")
+        assert hasattr(tag, "kind")
+        assert tag.kind in ("def", "ref")
+
+    def test_rel_fname_propagated(self):
+        fpath = os.path.join(FIXTURES, "auth.py")
+        tags = get_tags(fpath, "auth.py")
+        for tag in tags:
+            assert tag.rel_fname == "auth.py"
+
+    def test_no_duplicates(self):
+        fpath = os.path.join(FIXTURES, "auth.py")
+        tags = get_tags(fpath, "auth.py")
+        seen = set()
+        for tag in tags:
+            key = (tag.name, tag.line, tag.kind)
+            assert key not in seen, f"Duplicate tag: {tag}"
+            seen.add(key)
+
+
+class TestTypeScriptTags:
+    def test_extracts_defs_and_refs(self):
+        fpath = os.path.join(FIXTURES, "api.ts")
+        tags = get_tags(fpath, "api.ts")
+        defs = [t for t in tags if t.kind == "def"]
+        refs = [t for t in tags if t.kind == "ref"]
+        assert len(defs) >= 3, f"Expected >=3 defs, got {len(defs)}: {[t.name for t in defs]}"
+        assert len(refs) >= 2, f"Expected >=2 refs, got {len(refs)}: {[t.name for t in refs]}"
+
+    def test_captures_class(self):
+        fpath = os.path.join(FIXTURES, "api.ts")
+        tags = get_tags(fpath, "api.ts")
+        def_names = {t.name for t in tags if t.kind == "def"}
+        assert "ApiController" in def_names
+
+    def test_captures_function_defs(self):
+        fpath = os.path.join(FIXTURES, "api.ts")
+        tags = get_tags(fpath, "api.ts")
+        def_names = {t.name for t in tags if t.kind == "def"}
+        assert "processRequest" in def_names
+        assert "formatResponse" in def_names
+
+    def test_captures_call_refs(self):
+        fpath = os.path.join(FIXTURES, "api.ts")
+        tags = get_tags(fpath, "api.ts")
+        ref_names = {t.name for t in tags if t.kind == "ref"}
+        assert "handleAuth" in ref_names or "formatResponse" in ref_names
+
+    def test_captures_cross_file_ref(self):
+        """api.ts imports and calls handleAuth from auth.py -- should appear as ref."""
+        fpath = os.path.join(FIXTURES, "api.ts")
+        tags = get_tags(fpath, "api.ts")
+        ref_names = {t.name for t in tags if t.kind == "ref"}
+        assert "handleAuth" in ref_names
+
+
+class TestJavaScriptTags:
+    def test_extracts_defs_and_refs(self):
+        fpath = os.path.join(FIXTURES, "utils.js")
+        tags = get_tags(fpath, "utils.js")
+        defs = [t for t in tags if t.kind == "def"]
+        refs = [t for t in tags if t.kind == "ref"]
+        assert len(defs) >= 3, f"Expected >=3 defs, got {len(defs)}: {[t.name for t in defs]}"
+        assert len(refs) >= 2, f"Expected >=2 refs, got {len(refs)}: {[t.name for t in refs]}"
+
+    def test_arrow_function_defs(self):
+        fpath = os.path.join(FIXTURES, "utils.js")
+        tags = get_tags(fpath, "utils.js")
+        def_names = {t.name for t in tags if t.kind == "def"}
+        assert "processData" in def_names
+
+    def test_regular_function_defs(self):
+        fpath = os.path.join(FIXTURES, "utils.js")
+        tags = get_tags(fpath, "utils.js")
+        def_names = {t.name for t in tags if t.kind == "def"}
+        assert "formatDate" in def_names
+        assert "parseConfig" in def_names
+
+    def test_cross_function_refs(self):
+        fpath = os.path.join(FIXTURES, "utils.js")
+        tags = get_tags(fpath, "utils.js")
+        ref_names = {t.name for t in tags if t.kind == "ref"}
+        assert "parseConfig" in ref_names
+        assert "formatDate" in ref_names
+
+
+class TestNonexistentFile:
+    def test_returns_empty(self):
+        tags = get_tags("/nonexistent/file.py", "file.py")
+        assert tags == []
+
+    def test_unknown_extension(self):
+        tags = get_tags("file.xyz", "file.xyz")
+        assert tags == []
+
+
+class TestTagNamedTuple:
+    def test_tag_is_namedtuple(self):
+        tag = Tag(rel_fname="a.py", fname="/a.py", line=1, name="foo", kind="def")
+        assert tag.rel_fname == "a.py"
+        assert tag.fname == "/a.py"
+        assert tag.line == 1
+        assert tag.name == "foo"
+        assert tag.kind == "def"

package/ftm-map/scripts/test_query.py

@@ -1,66 +1,183 @@
-"""Tests for ftm-map query module."""
+"""Tests for query.py -- all query modes with new schema."""
 import os
 import sys
 import tempfile
-import unittest
+import pytest
 
 sys.path.insert(0, os.path.dirname(__file__))
-from db import get_connection, add_symbol, add_edge
-from query import blast_radius, dependency_chain, search, symbol_info
-
-class TestQuery(unittest.TestCase):
-    def setUp(self):
-        self.tmpdir = tempfile.mkdtemp()
-        self.conn = get_connection(self.tmpdir)
-        # Build test graph: A -> B -> C, D imports A
-        self.a = add_symbol(self.conn, "handleAuth", "function", "auth.py", 1, 5, "def handleAuth(req)", "Auth handler")
-        self.b = add_symbol(self.conn, "validateToken", "function", "auth.py", 7, 10, "def validateToken(token)")
-        self.c = add_symbol(self.conn, "getUser", "function", "users.py", 1, 5, "def getUser(uid)")
-        self.d = add_symbol(self.conn, "processRequest", "function", "api.ts", 1, 8, "function processRequest(req)")
-        add_edge(self.conn, self.a, self.b, "calls")
-        add_edge(self.conn, self.a, self.c, "calls")
-        add_edge(self.conn, self.d, self.a, "calls")
-        self.conn.commit()
-
-    def tearDown(self):
-        self.conn.close()
-
-    def test_blast_radius(self):
-        """Blast radius of getUser should include handleAuth and processRequest."""
-        result = blast_radius(self.conn, "getUser")
-        names = {r["name"] for r in result["results"]}
-        self.assertIn("handleAuth", names)
-        self.assertIn("processRequest", names)
-        self.assertEqual(result["affected_count"], 2)
-
-    def test_dependency_chain(self):
-        """handleAuth depends on validateToken and getUser."""
-        result = dependency_chain(self.conn, "handleAuth")
-        names = {r["name"] for r in result["results"]}
-        self.assertIn("validateToken", names)
-        self.assertIn("getUser", names)
-
-    def test_search(self):
-        """Search for 'auth' should return handleAuth."""
-        result = search(self.conn, "auth")
-        self.assertGreater(result["result_count"], 0)
-        names = {r["name"] for r in result["results"]}
-        self.assertIn("handleAuth", names)
-
-    def test_symbol_info(self):
-        """Should return full details with callers and callees."""
-        result = symbol_info(self.conn, "handleAuth")
-        self.assertEqual(result["name"], "handleAuth")
-        callee_names = {c["name"] for c in result["callees"]}
-        self.assertIn("validateToken", callee_names)
-        self.assertIn("getUser", callee_names)
-        caller_names = {c["name"] for c in result["callers"]}
-        self.assertIn("processRequest", caller_names)
-
-    def test_missing_symbol(self):
-        """Query for non-existent symbol should return error."""
-        result = blast_radius(self.conn, "nonexistent")
-        self.assertIn("error", result)
-
-if __name__ == "__main__":
-    unittest.main()
+from db import get_connection, add_file, add_symbol, add_reference, rebuild_file_edges, rebuild_symbol_edges
+from query import blast_radius, dependency_chain, search, symbol_info, context, stats
+
+
+@pytest.fixture
+def indexed_conn():
+    """Connection with a fully indexed mini-project."""
+    with tempfile.TemporaryDirectory() as tmp:
+        conn = get_connection(tmp)
+
+        f1 = add_file(conn, "src/auth.py", "python", 1.0, line_count=50)
+        f2 = add_file(conn, "src/api.py", "python", 1.0, line_count=100)
+        f3 = add_file(conn, "src/utils.py", "python", 1.0, line_count=30)
+
+        add_symbol(conn, f1, "authenticate", "function", 1, 20, signature="def authenticate(req)")
+        add_symbol(conn, f1, "verify_token", "function", 25, 40)
+        add_symbol(conn, f2, "handle_request", "function", 1, 50)
+        add_symbol(conn, f3, "format_date", "function", 1, 10)
+        add_symbol(conn, f3, "parse_config", "function", 15, 25)
+
+        add_reference(conn, f2, "authenticate", 10)
+        add_reference(conn, f2, "format_date", 20)
+        add_reference(conn, f1, "format_date", 30)
+
+        rebuild_file_edges(conn)
+        rebuild_symbol_edges(conn)
+        conn.commit()
+
+        yield conn
+        conn.close()
+
+
+class TestBlastRadius:
+    def test_finds_affected(self, indexed_conn):
+        result = blast_radius(indexed_conn, "authenticate")
+        assert result["affected_count"] >= 1
+
+    def test_returns_symbol_name(self, indexed_conn):
+        result = blast_radius(indexed_conn, "authenticate")
+        assert result["symbol"] == "authenticate"
+
+    def test_results_have_file_path(self, indexed_conn):
+        result = blast_radius(indexed_conn, "authenticate")
+        if result["results"]:
+            assert "file_path" in result["results"][0]
+
+    def test_results_have_depth(self, indexed_conn):
+        result = blast_radius(indexed_conn, "authenticate")
+        if result["results"]:
+            assert "depth" in result["results"][0]
+
+    def test_not_found(self, indexed_conn):
+        result = blast_radius(indexed_conn, "nonexistent")
+        assert "error" in result
+
+
+class TestDependencyChain:
+    def test_finds_deps(self, indexed_conn):
+        result = dependency_chain(indexed_conn, "handle_request")
+        assert result["dependency_count"] >= 1
+
+    def test_returns_symbol_name(self, indexed_conn):
+        result = dependency_chain(indexed_conn, "handle_request")
+        assert result["symbol"] == "handle_request"
+
+    def test_not_found(self, indexed_conn):
+        result = dependency_chain(indexed_conn, "nonexistent")
+        assert "error" in result
+
+    def test_leaf_has_no_deps(self, indexed_conn):
+        result = dependency_chain(indexed_conn, "verify_token")
+        # verify_token has no references to other symbols in its scope
+        assert result["dependency_count"] == 0 or "error" not in result
+
+
+class TestSearch:
+    def test_finds_by_name(self, indexed_conn):
+        result = search(indexed_conn, "authenticate")
+        assert result["result_count"] >= 1
+
+    def test_returns_query(self, indexed_conn):
+        result = search(indexed_conn, "authenticate")
+        assert result["query"] == "authenticate"
+
+    def test_results_have_file_path(self, indexed_conn):
+        result = search(indexed_conn, "authenticate")
+        if result["results"]:
+            assert "file_path" in result["results"][0]
+
+    def test_results_have_rank(self, indexed_conn):
+        result = search(indexed_conn, "authenticate")
+        if result["results"]:
+            assert "rank" in result["results"][0]
+
+    def test_empty_results(self, indexed_conn):
+        result = search(indexed_conn, "zzzznonexistent")
+        assert result["result_count"] == 0
+
+
+class TestSymbolInfo:
+    def test_returns_name(self, indexed_conn):
+        result = symbol_info(indexed_conn, "authenticate")
+        assert result["name"] == "authenticate"
+
+    def test_returns_kind(self, indexed_conn):
+        result = symbol_info(indexed_conn, "authenticate")
+        assert "kind" in result
+
+    def test_returns_reference_count(self, indexed_conn):
+        result = symbol_info(indexed_conn, "authenticate")
+        assert "reference_count" in result
+        assert result["reference_count"] >= 1  # api.py references it
+
+    def test_returns_callers_callees(self, indexed_conn):
+        result = symbol_info(indexed_conn, "authenticate")
+        assert "callers" in result
+        assert "callees" in result
+
+    def test_returns_blast_radius_count(self, indexed_conn):
+        result = symbol_info(indexed_conn, "authenticate")
+        assert "blast_radius_count" in result
+
+    def test_returns_file(self, indexed_conn):
+        result = symbol_info(indexed_conn, "authenticate")
+        assert result["file"] == "src/auth.py"
+
+    def test_not_found(self, indexed_conn):
+        result = symbol_info(indexed_conn, "nonexistent")
+        assert "error" in result
+
+
+class TestContext:
+    def test_returns_files(self, indexed_conn):
+        result = context(indexed_conn, seed_symbols=["authenticate"])
+        assert "files" in result
+        assert len(result["files"]) > 0
+
+    def test_files_have_path(self, indexed_conn):
+        result = context(indexed_conn, seed_symbols=["authenticate"])
+        if result["files"]:
+            assert "path" in result["files"][0]
+
+    def test_files_have_score(self, indexed_conn):
+        result = context(indexed_conn, seed_symbols=["authenticate"])
+        if result["files"]:
+            assert "score" in result["files"][0]
+
+    def test_respects_budget(self, indexed_conn):
+        result = context(indexed_conn, token_budget=50)
+        if result.get("total_tokens"):
+            assert result["total_tokens"] <= 50 * 1.15
+
+    def test_no_budget_returns_all(self, indexed_conn):
+        result = context(indexed_conn, token_budget=None)
+        assert "files" in result
+
+    def test_seed_files(self, indexed_conn):
+        result = context(indexed_conn, seed_files=["src/auth.py"])
+        assert "files" in result
+        assert len(result["files"]) > 0
+
+
+class TestStats:
+    def test_returns_counts(self, indexed_conn):
+        result = stats(indexed_conn)
+        assert result["file_count"] == 3
+        assert result["symbol_count"] == 5
+
+    def test_returns_edge_counts(self, indexed_conn):
+        result = stats(indexed_conn)
+        assert "edge_count" in result
+        assert "file_edge_count" in result
+
+    def test_returns_reference_count(self, indexed_conn):
+        result = stats(indexed_conn)
+        assert result["reference_count"] == 3