@design-ai/cli 4.56.0 → 4.58.0

package/tools/audit/registry-smoke.py

@@ -39,6 +39,8 @@ from smoke_assertions import (
     EXPECTED_HELP_ALIASES,
     EXPECTED_NUMERIC_VALUE_SMOKES,
     EXPECTED_PACK_MAX_BYTES,
+    EXPECTED_RANKED_SEARCH_LIMIT,
+    EXPECTED_RANKED_SEARCH_NOT_BUILT_NOTICE,
     EXPECTED_REPOSITORY_URL,
     EXPECTED_ROUTE_BRIEF,
     EXPECTED_ROUTE_ID,
@@ -61,6 +63,9 @@ from smoke_assertions import (
     assert_force_overwrite_replaced,
     assert_functional_alias_smokes,
     assert_help_topic_output,
+    assert_index_build_json,
+    assert_index_status_json,
+    assert_index_verify_json,
     assert_install_doctor_lifecycle_output,
     assert_list_catalog_output,
     assert_list_catalog_json,
@@ -72,6 +77,8 @@ from smoke_assertions import (
     assert_pack_json_component_spec,
     assert_pack_markdown_body_component_spec,
     assert_pack_markdown_component_spec,
+    assert_ranked_search_determinism,
+    assert_ranked_search_json,
     assert_site_repair_apply_report_payload,
     assert_site_repair_guidance_report_contract,
     assert_site_repair_preview_report_payload,
@@ -132,6 +139,10 @@ from smoke_assertions import (
     parse_help_topics,
     passing_doctor_report_json,
     passing_check_artifact_content,
+    passing_index_build_json,
+    passing_index_status_json,
+    passing_index_verify_json,
+    passing_ranked_search_json,
     passing_site_json,
     passing_site_mcp_check_json,
     passing_site_mcp_check_probes_human,
@@ -2205,6 +2216,102 @@ def assert_learning_redact_smoke(
     )
 
 
+def assert_learning_recall_json(
+    raw: str,
+    *,
+    context: str,
+    cmd: list[str],
+) -> None:
+    assert_no_ansi(raw, cmd)
+    try:
+        payload = json.loads(raw)
+    except json.JSONDecodeError as error:
+        raise SystemExit(f"{context}: failed to parse learn recall JSON") from error
+
+    require_registry_smoke(
+        isinstance(payload, dict) and payload.get("query") == "korean mobile",
+        context=context,
+        cmd=cmd,
+        message="learn recall JSON must echo the query",
+    )
+    corpus = payload.get("corpus")
+    require_registry_smoke(
+        isinstance(corpus, dict)
+        and isinstance(corpus.get("candidateCount"), int)
+        and corpus.get("candidateCount") > 0
+        and isinstance(corpus.get("selectedCount"), int)
+        and isinstance(corpus.get("selected"), list),
+        context=context,
+        cmd=cmd,
+        message="learn recall corpus block shape changed",
+    )
+    learning = payload.get("learning")
+    require_registry_smoke(
+        isinstance(learning, dict)
+        and learning.get("mode") == "brief-relevance"
+        and learning.get("candidateCount") == 3
+        and isinstance(learning.get("selected"), list),
+        context=context,
+        cmd=cmd,
+        message="learn recall learning block shape changed",
+    )
+    selected = learning.get("selected")
+    require_registry_smoke(
+        len(selected) >= 1
+        and isinstance(selected[0], dict)
+        and selected[0].get("id") == "registry-korean"
+        and selected[0].get("category") == "korean"
+        and isinstance(selected[0].get("matchedTokens"), list),
+        context=context,
+        cmd=cmd,
+        message="learn recall should rank the Korean learning entry for a Korean query",
+    )
+
+
+def assert_learning_recall_smoke(
+    command_factory,
+    profile_path: Path,
+    *,
+    env: dict[str, str],
+    cwd: Path | None = None,
+    context: str,
+) -> None:
+    write_learning_stats_fixture(profile_path)
+
+    json_cmd = command_factory(
+        "learn",
+        "--recall",
+        "korean mobile",
+        "--file",
+        str(profile_path),
+        "--json",
+    )
+    json_result = run_plain(json_cmd, cwd=cwd, env=env)
+    assert_learning_recall_json(json_result.stdout, context=f"{context} JSON", cmd=json_cmd)
+
+    # Read-only: the profile file is unchanged after recall.
+    before = write_learning_stats_fixture_snapshot(profile_path)
+    human_cmd = command_factory("learn", "--recall", "korean mobile", "--file", str(profile_path))
+    human_result = run_plain(human_cmd, cwd=cwd, env=env)
+    require_registry_smoke(
+        "Recall (read-only)" in human_result.stdout
+        and "Privacy: recall is read-only" in human_result.stdout,
+        context=f"{context} human",
+        cmd=human_cmd,
+        message="learn recall human output missing read-only markers",
+    )
+    require_registry_smoke(
+        profile_path.read_text(encoding="utf-8") == before,
+        context=f"{context} read-only",
+        cmd=human_cmd,
+        message="learn recall must not mutate the learning profile",
+    )
+
+
+def write_learning_stats_fixture_snapshot(profile_path: Path) -> str:
+    return profile_path.read_text(encoding="utf-8")
+
+
 def assert_learning_stats_json(
     raw: str,
     *,
@@ -4412,6 +4519,56 @@ def assert_pack_stdin_smoke(
     )
 
 
+def assert_recall_context(payload: dict[str, object], *, context: str, cmd: list[str]) -> None:
+    recall = payload.get("recall")
+    require_registry_smoke(
+        isinstance(recall, dict),
+        context=context,
+        cmd=cmd,
+        message="recall should be present when --with-recall is used",
+    )
+    require_registry_smoke(
+        recall.get("mode") == "lexical",
+        context=context,
+        cmd=cmd,
+        message="recall should use the deterministic lexical scorer",
+    )
+    selected_count = recall.get("selectedCount")
+    require_registry_smoke(
+        isinstance(selected_count, int) and selected_count >= 1,
+        context=context,
+        cmd=cmd,
+        message="recall should select at least one corpus file for the Button brief",
+    )
+    selected = recall.get("selected")
+    require_registry_smoke(
+        isinstance(selected, list) and len(selected) == selected_count and len(selected) >= 1,
+        context=context,
+        cmd=cmd,
+        message="recall selected list should match the reported selected count",
+    )
+    top = selected[0]
+    require_registry_smoke(
+        isinstance(top, dict) and isinstance(top.get("id"), str) and top.get("id"),
+        context=context,
+        cmd=cmd,
+        message="recall selection should cite the corpus relPath as its id",
+    )
+    require_registry_smoke(
+        type(top.get("score")) in (int, float) and top.get("score") > 0,
+        context=context,
+        cmd=cmd,
+        message="recall selection should include a positive relevance score",
+    )
+    markdown = recall.get("markdown")
+    require_registry_smoke(
+        isinstance(markdown, str) and "## Recalled design knowledge" in markdown and top.get("id") in markdown,
+        context=context,
+        cmd=cmd,
+        message="recall markdown should carry the Recalled design knowledge section and cite the top file",
+    )
+
+
 def assert_learning_relevance_context(payload: dict[str, object], *, context: str, cmd: list[str]) -> None:
     learning_context = payload.get("learningContext")
     require_registry_smoke(
@@ -4628,7 +4785,7 @@ def assert_learning_query_human(raw: str, *, context: str, cmd: list[str]) -> No
         "Limit: 2",
         "Explain: selection score, matched tokens, and reason",
         "[accessibility] Prioritize keyboard accessibility details for Button component API specs",
-        "matched keyboard, accessibility",
+        "matched accessibility, keyboard",
         "reason brief-match",
     ):
         require_registry_smoke(
@@ -4989,6 +5146,49 @@ def assert_learning_relevance_smoke(
     require_registry_smoke(isinstance(plan, dict), context=f"{context} pack", cmd=pack_cmd, message="pack plan missing")
     assert_learning_relevance_context(plan, context=f"{context} pack plan", cmd=pack_cmd)
 
+    recall_prompt_cmd = command_factory(
+        "prompt",
+        EXPECTED_ROUTE_BRIEF,
+        "--route",
+        EXPECTED_ROUTE_ID,
+        "--with-recall",
+        "--recall-limit",
+        "3",
+        "--json",
+    )
+    recall_prompt_result = run_plain(recall_prompt_cmd, cwd=cwd, env=relevance_env)
+    try:
+        recall_prompt_payload = json.loads(recall_prompt_result.stdout)
+    except json.JSONDecodeError as error:
+        raise SystemExit(f"{context}: failed to parse prompt recall JSON") from error
+    assert_recall_context(recall_prompt_payload, context=f"{context} prompt recall", cmd=recall_prompt_cmd)
+
+    recall_pack_cmd = command_factory(
+        "pack",
+        EXPECTED_ROUTE_BRIEF,
+        "--route",
+        EXPECTED_ROUTE_ID,
+        "--with-recall",
+        "--recall-limit",
+        "3",
+        "--max-bytes",
+        str(EXPECTED_PACK_MAX_BYTES),
+        "--json",
+    )
+    recall_pack_result = run_plain(recall_pack_cmd, cwd=cwd, env=relevance_env)
+    try:
+        recall_pack_payload = json.loads(recall_pack_result.stdout)
+    except json.JSONDecodeError as error:
+        raise SystemExit(f"{context}: failed to parse pack recall JSON") from error
+    recall_plan = recall_pack_payload.get("plan")
+    require_registry_smoke(
+        isinstance(recall_plan, dict),
+        context=f"{context} pack recall",
+        cmd=recall_pack_cmd,
+        message="pack recall plan missing",
+    )
+    assert_recall_context(recall_plan, context=f"{context} pack recall plan", cmd=recall_pack_cmd)
+
     eval_template_path = profile_path.with_name(f"{profile_path.stem}-eval-template.json")
     eval_template_path.write_text("stale eval template output\n", encoding="utf-8")
     eval_template_cmd = command_factory(
@@ -5055,6 +5255,166 @@ def assert_learning_relevance_smoke(
     )
 
 
+def assert_index_roundtrip_smoke(
+    command_factory,
+    index_dir: Path,
+    profile_path: Path,
+    *,
+    env: dict[str, str],
+    cwd: Path | None = None,
+    context: str,
+) -> None:
+    """`index --build` -> `index --status` -> `index --verify` round-trip against a
+    temp index dir and learning profile, isolated from the ambient environment."""
+    index_dir.mkdir(parents=True, exist_ok=True)
+    write_learning_relevance_fixture(profile_path)
+    index_env = env.copy()
+    index_env["DESIGN_AI_INDEX_DIR"] = str(index_dir)
+    index_env["DESIGN_AI_LEARNING_FILE"] = str(profile_path)
+
+    build_cmd = command_factory("index", "--build", "--json")
+    build_result = run_plain(build_cmd, cwd=cwd, env=index_env)
+    assert_index_build_json(
+        build_result.stdout,
+        index_dir=str(index_dir),
+        context=f"{context} build",
+        cmd=build_cmd,
+    )
+
+    status_cmd = command_factory("index", "--status", "--json")
+    status_result = run_plain(status_cmd, cwd=cwd, env=index_env)
+    assert_index_status_json(
+        status_result.stdout,
+        index_dir=str(index_dir),
+        context=f"{context} status",
+        cmd=status_cmd,
+    )
+
+    verify_cmd = command_factory("index", "--verify", "--json")
+    verify_result = run_plain(verify_cmd, cwd=cwd, env=index_env)
+    assert_index_verify_json(
+        verify_result.stdout,
+        index_dir=str(index_dir),
+        context=f"{context} verify",
+        cmd=verify_cmd,
+    )
+
+
+def assert_ranked_search_determinism_smoke(
+    command_factory,
+    index_dir: Path,
+    *,
+    env: dict[str, str],
+    cwd: Path | None = None,
+    context: str,
+) -> None:
+    """Two identical `search --ranked --json` runs must produce byte-identical output."""
+    index_dir.mkdir(parents=True, exist_ok=True)
+    ranked_env = env.copy()
+    ranked_env["DESIGN_AI_INDEX_DIR"] = str(index_dir)
+
+    ranked_cmd = command_factory(
+        "search",
+        EXPECTED_CORPUS_SEARCH_QUERY,
+        "--dir",
+        "knowledge",
+        "--limit",
+        str(EXPECTED_RANKED_SEARCH_LIMIT),
+        "--ranked",
+        "--json",
+    )
+    first_result = run_plain(ranked_cmd, cwd=cwd, env=ranked_env)
+    second_result = run_plain(ranked_cmd, cwd=cwd, env=ranked_env)
+    assert_ranked_search_json(
+        first_result.stdout,
+        context=f"{context} payload",
+        cmd=ranked_cmd,
+        expected_notice=EXPECTED_RANKED_SEARCH_NOT_BUILT_NOTICE,
+    )
+    assert_ranked_search_determinism(
+        first_result.stdout,
+        second_result.stdout,
+        context=context,
+        cmd=ranked_cmd,
+    )
+
+
+def assert_embeddings_off_by_default_smoke(
+    command_factory,
+    index_dir: Path,
+    profile_path: Path,
+    *,
+    env: dict[str, str],
+    cwd: Path | None = None,
+    context: str,
+) -> None:
+    """Phase B is opt-in: a plain `index --build` never touches embeddings, and
+    `index --status` reports `embeddings: null` (the feature is simply unused)."""
+    index_dir.mkdir(parents=True, exist_ok=True)
+    write_learning_relevance_fixture(profile_path)
+    index_env = env.copy()
+    index_env["DESIGN_AI_INDEX_DIR"] = str(index_dir)
+    index_env["DESIGN_AI_LEARNING_FILE"] = str(profile_path)
+    # Point at a config file that is guaranteed not to exist, rather than relying on
+    # the ambient environment lacking DESIGN_AI_CONFIG_FILE — a real ~/.design-ai/
+    # config.json on the machine running this smoke must not leak into the test.
+    index_env["DESIGN_AI_CONFIG_FILE"] = str(index_dir.parent / "no-config-here" / "config.json")
+
+    build_cmd = command_factory("index", "--build", "--json")
+    build_result = run_plain(build_cmd, cwd=cwd, env=index_env)
+    build_payload = json.loads(build_result.stdout)
+    if "embeddings" in build_payload:
+        raise SystemExit(f"index build JSON after {context} unexpectedly ran provider execution with plain --build")
+
+    status_cmd = command_factory("index", "--status", "--json")
+    status_result = run_plain(status_cmd, cwd=cwd, env=index_env)
+    assert_index_status_json(
+        status_result.stdout,
+        index_dir=str(index_dir),
+        context=f"{context} status",
+        cmd=status_cmd,
+        expect_embeddings=False,
+    )
+
+
+def assert_search_embeddings_no_provider_fallback_smoke(
+    command_factory,
+    *,
+    env: dict[str, str],
+    cwd: Path | None = None,
+    context: str,
+    config_file: Path,
+) -> None:
+    """`search --ranked --embeddings` with no provider configured (flag or config)
+    degrades to the lexical backend with a notice and exit code 0."""
+    fallback_env = env.copy()
+    # Same isolation rationale as assert_embeddings_off_by_default_smoke: point at a
+    # config path guaranteed not to exist rather than trusting the ambient environment.
+    fallback_env["DESIGN_AI_CONFIG_FILE"] = str(config_file)
+
+    search_cmd = command_factory(
+        "search",
+        EXPECTED_CORPUS_SEARCH_QUERY,
+        "--dir",
+        "knowledge",
+        "--limit",
+        str(EXPECTED_RANKED_SEARCH_LIMIT),
+        "--ranked",
+        "--embeddings",
+        "--json",
+    )
+    result = run_plain(search_cmd, cwd=cwd, env=fallback_env)
+    assert_ranked_search_json(
+        result.stdout,
+        context=f"{context} payload",
+        cmd=search_cmd,
+        expected_backend="lexical",
+    )
+    payload = json.loads(result.stdout)
+    if "embedding provider" not in payload.get("notice", "") and "no embedding provider configured" not in payload.get("notice", ""):
+        raise SystemExit(f"search ranked JSON after {context} fallback notice does not mention the missing provider")
+
+
 def wait_for_registry_package(
     package_spec: str,
     *,
@@ -6086,6 +6446,13 @@ def smoke_registry_package(package_spec: str, *, retries: int, delay: float) ->
             env=env,
             context="registry smoke npm exec learn stats",
         )
+        assert_learning_recall_smoke(
+            lambda *args: npm_exec_cmd(package_spec, *args),
+            npx_root / "registry-recall-learning.json",
+            cwd=npx_root,
+            env=env,
+            context="registry smoke npm exec learn recall",
+        )
         assert_learning_audit_cleanup_smoke(
             lambda *args: npm_exec_cmd(package_spec, *args),
             npx_root / "registry-audit-learning.json",
@@ -6100,6 +6467,36 @@ def smoke_registry_package(package_spec: str, *, retries: int, delay: float) ->
             env=env,
             context="registry smoke npm exec learning relevance",
         )
+        assert_index_roundtrip_smoke(
+            lambda *args: npm_exec_cmd(package_spec, *args),
+            npx_root / "registry-index",
+            npx_root / "registry-index-learning.json",
+            cwd=npx_root,
+            env=env,
+            context="registry smoke npm exec index roundtrip",
+        )
+        assert_ranked_search_determinism_smoke(
+            lambda *args: npm_exec_cmd(package_spec, *args),
+            npx_root / "registry-ranked-search-index",
+            cwd=npx_root,
+            env=env,
+            context="registry smoke npm exec ranked search determinism",
+        )
+        assert_embeddings_off_by_default_smoke(
+            lambda *args: npm_exec_cmd(package_spec, *args),
+            npx_root / "registry-embeddings-off-index",
+            npx_root / "registry-embeddings-off-learning.json",
+            cwd=npx_root,
+            env=env,
+            context="registry smoke npm exec embeddings off by default",
+        )
+        assert_search_embeddings_no_provider_fallback_smoke(
+            lambda *args: npm_exec_cmd(package_spec, *args),
+            cwd=npx_root,
+            env=env,
+            context="registry smoke npm exec search embeddings no provider fallback",
+            config_file=npx_root / "registry-no-config-here" / "config.json",
+        )
         assert_update_dry_run_smoke(
             npm_exec_cmd(package_spec, "update", "--dry-run"),
             cwd=npx_root,
@@ -7981,7 +8378,7 @@ def run_self_test() -> None:
                 "",
                 "1. [accessibility] Prioritize keyboard accessibility details for Button component API specs",
                 "   learn-relevant · 2026-05-22T00:00:01.000Z",
-                "   score 4 · matched keyboard, accessibility · reason brief-match",
+                "   score 2.114533 · matched accessibility, keyboard · reason brief-match",
             ]),
             context="registry smoke self-test",
             cmd=learn_query_human_cmd,
@@ -7995,7 +8392,7 @@ def run_self_test() -> None:
                     "Limit: 2",
                     "Explain: selection score, matched tokens, and reason",
                     "[accessibility] Prioritize keyboard accessibility details for Button component API specs",
-                    "matched keyboard, accessibility",
+                    "matched accessibility, keyboard",
                 ]),
                 context="registry smoke self-test",
                 cmd=learn_query_human_cmd,
@@ -8297,6 +8694,38 @@ def run_self_test() -> None:
             scope="registry smoke",
         )
 
+        recall_payload = {
+            "recall": {
+                "query": EXPECTED_ROUTE_BRIEF,
+                "mode": "lexical",
+                "candidateCount": 42,
+                "selectedCount": 1,
+                "selected": [
+                    {
+                        "id": "knowledge/components/INDEX.md",
+                        "score": 9.5,
+                        "matchedTokens": ["button", "component"],
+                    },
+                ],
+                "markdown": (
+                    "## Recalled design knowledge\n\n"
+                    "- knowledge/components/INDEX.md\n"
+                    "  - Component index"
+                ),
+            },
+        }
+        recall_cmd = ["design-ai", "prompt", EXPECTED_ROUTE_BRIEF, "--with-recall", "--json"]
+        assert_recall_context(recall_payload, context="registry smoke self-test", cmd=recall_cmd)
+        expect_self_test_failure(
+            lambda: assert_recall_context(
+                {**recall_payload, "recall": {**recall_payload["recall"], "selectedCount": 0, "selected": []}},
+                context="registry smoke self-test",
+                cmd=recall_cmd,
+            ),
+            expected="recall should select at least one corpus file",
+            scope="registry smoke",
+        )
+
         learning_audit_path = tmp_root / "learning-audit.json"
         duplicate_command_args = [
             "design-ai",
@@ -8503,6 +8932,111 @@ def run_self_test() -> None:
             scope="registry smoke",
         )
 
+        index_dir = str(tmp_root / "registry-index-self-test")
+        index_build_cmd = ["design-ai", "index", "--build", "--json"]
+        assert_index_build_json(
+            passing_index_build_json(index_dir),
+            index_dir=index_dir,
+            context="registry smoke self-test",
+            cmd=index_build_cmd,
+        )
+        expect_self_test_failure(
+            lambda: assert_index_build_json(
+                "{",
+                index_dir=index_dir,
+                context="registry smoke self-test",
+                cmd=index_build_cmd,
+            ),
+            expected="failed to parse index build JSON",
+            scope="registry smoke",
+        )
+
+        index_status_cmd = ["design-ai", "index", "--status", "--json"]
+        assert_index_status_json(
+            passing_index_status_json(index_dir),
+            index_dir=index_dir,
+            context="registry smoke self-test",
+            cmd=index_status_cmd,
+        )
+        assert_index_status_json(
+            passing_index_status_json(index_dir, with_embeddings=True),
+            index_dir=index_dir,
+            context="registry smoke self-test embeddings",
+            cmd=index_status_cmd,
+            expect_embeddings=True,
+        )
+        expect_self_test_failure(
+            lambda: assert_index_status_json(
+                passing_index_status_json(index_dir, with_embeddings=True),
+                index_dir=index_dir,
+                context="registry smoke self-test embeddings unexpected",
+                cmd=index_status_cmd,
+                expect_embeddings=False,
+            ),
+            expected="embeddings is not null when embeddings were not built",
+            scope="registry smoke",
+        )
+
+        index_verify_cmd = ["design-ai", "index", "--verify", "--json"]
+        assert_index_verify_json(
+            passing_index_verify_json(index_dir),
+            index_dir=index_dir,
+            context="registry smoke self-test",
+            cmd=index_verify_cmd,
+        )
+        expect_self_test_failure(
+            lambda: assert_index_verify_json(
+                json.dumps({**json.loads(passing_index_verify_json(index_dir)), "ok": False}),
+                index_dir=index_dir,
+                context="registry smoke self-test",
+                cmd=index_verify_cmd,
+            ),
+            expected="ok is not true",
+            scope="registry smoke",
+        )
+
+        ranked_search_cmd = ["design-ai", "search", EXPECTED_CORPUS_SEARCH_QUERY, "--ranked", "--json"]
+        ranked_search_json = passing_ranked_search_json()
+        assert_ranked_search_json(
+            ranked_search_json,
+            context="registry smoke self-test",
+            cmd=ranked_search_cmd,
+        )
+        assert_ranked_search_determinism(
+            ranked_search_json,
+            ranked_search_json,
+            context="registry smoke self-test",
+            cmd=ranked_search_cmd,
+        )
+        expect_self_test_failure(
+            lambda: assert_ranked_search_determinism(
+                ranked_search_json,
+                passing_ranked_search_json(backend="lexical"),
+                context="registry smoke self-test",
+                cmd=ranked_search_cmd,
+            ),
+            expected="differs between identical runs",
+            scope="registry smoke",
+        )
+
+        ranked_search_fallback_json = passing_ranked_search_json(backend="lexical")
+        assert_ranked_search_json(
+            ranked_search_fallback_json,
+            context="registry smoke self-test fallback",
+            cmd=ranked_search_cmd,
+            expected_backend="lexical",
+        )
+        expect_self_test_failure(
+            lambda: assert_ranked_search_json(
+                passing_ranked_search_json(backend="embeddings"),
+                context="registry smoke self-test fallback mismatch",
+                cmd=ranked_search_cmd,
+                expected_backend="lexical",
+            ),
+            expected="backend differs from expected backend",
+            scope="registry smoke",
+        )
+
     expect_self_test_failure(
         lambda: smoke_registry_package("self-test-package", retries=0, delay=0),
         expected="--retries must be at least 1",