@design-ai/cli 4.55.0 → 4.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +109 -25
- package/CHANGELOG.md +75 -0
- package/README.ko.md +15 -32
- package/README.md +18 -31
- package/cli/bin/design-ai.mjs +1 -0
- package/cli/commands/help.mjs +7 -4
- package/cli/commands/index.mjs +292 -0
- package/cli/commands/learn-help.mjs +149 -0
- package/cli/commands/learn-print-profile.mjs +373 -0
- package/cli/commands/learn-print-restore.mjs +349 -0
- package/cli/commands/learn-print-signals.mjs +453 -0
- package/cli/commands/learn.mjs +60 -1281
- package/cli/commands/pack.mjs +7 -3
- package/cli/commands/prompt.mjs +7 -3
- package/cli/commands/search.mjs +94 -1
- package/cli/lib/dispatch.mjs +3 -0
- package/cli/lib/embedding-index.mjs +199 -0
- package/cli/lib/embedding-provider.mjs +121 -0
- package/cli/lib/embedding-rerank.mjs +52 -0
- package/cli/lib/learn-args.mjs +490 -0
- package/cli/lib/learn-backup.mjs +748 -0
- package/cli/lib/learn-curation.mjs +612 -0
- package/cli/lib/learn-eval.mjs +459 -0
- package/cli/lib/learn-profile.mjs +763 -0
- package/cli/lib/learn-select.mjs +223 -0
- package/cli/lib/learn-shared.mjs +158 -0
- package/cli/lib/learn-test-support.mjs +218 -0
- package/cli/lib/learn-usage.mjs +360 -0
- package/cli/lib/learn.mjs +83 -3694
- package/cli/lib/lexical.mjs +137 -0
- package/cli/lib/local-config.mjs +110 -0
- package/cli/lib/mcp-server.mjs +221 -13
- package/cli/lib/pack.mjs +31 -4
- package/cli/lib/prompt.mjs +42 -4
- package/cli/lib/recall.mjs +193 -0
- package/cli/lib/retrieval-index.mjs +0 -0
- package/cli/lib/search-ranked.mjs +170 -0
- package/cli/lib/search.mjs +11 -1
- package/cli/lib/signals-backlog-commands.mjs +673 -0
- package/cli/lib/signals-backlog.mjs +361 -0
- package/cli/lib/signals-eval.mjs +176 -0
- package/cli/lib/signals-readiness.mjs +247 -0
- package/cli/lib/signals-registry.mjs +377 -0
- package/cli/lib/signals-render.mjs +478 -0
- package/cli/lib/signals-shared.mjs +75 -0
- package/cli/lib/signals.mjs +16 -2337
- package/cli/lib/site-analysis.mjs +297 -0
- package/cli/lib/site-args.mjs +433 -0
- package/cli/lib/site-bundle-build.mjs +127 -0
- package/cli/lib/site-bundle-check.mjs +454 -0
- package/cli/lib/site-bundle-commands.mjs +95 -0
- package/cli/lib/site-bundle-compare.mjs +157 -0
- package/cli/lib/site-bundle-contract.mjs +79 -0
- package/cli/lib/site-bundle-files.mjs +87 -0
- package/cli/lib/site-bundle-handoff-expected.mjs +173 -0
- package/cli/lib/site-bundle-handoff-runbook-action-summary.mjs +332 -0
- package/cli/lib/site-bundle-handoff-runbook-actions.mjs +113 -0
- package/cli/lib/site-bundle-handoff-runbook-evidence-fields.mjs +164 -0
- package/cli/lib/site-bundle-handoff-runbook-evidence.mjs +334 -0
- package/cli/lib/site-bundle-handoff-runbook-format.mjs +31 -0
- package/cli/lib/site-bundle-handoff-runbook-human-lines.mjs +167 -0
- package/cli/lib/site-bundle-handoff-runbook-maps.mjs +238 -0
- package/cli/lib/site-bundle-handoff-runbook-next-step.mjs +278 -0
- package/cli/lib/site-bundle-handoff-runbook-stage-metadata.mjs +84 -0
- package/cli/lib/site-bundle-handoff-runbook.mjs +444 -0
- package/cli/lib/site-bundle-handoff-summary.mjs +183 -0
- package/cli/lib/site-bundle-handoff.mjs +271 -0
- package/cli/lib/site-bundle-readme.mjs +98 -0
- package/cli/lib/site-bundle-repair-report.mjs +143 -0
- package/cli/lib/site-bundle-repair.mjs +68 -0
- package/cli/lib/site-content.mjs +399 -0
- package/cli/lib/site-evidence.mjs +35 -0
- package/cli/lib/site-mcp-commands.mjs +28 -0
- package/cli/lib/site-mcp-probes.mjs +159 -0
- package/cli/lib/site-mcp-readiness.mjs +157 -0
- package/cli/lib/site-mcp-report.mjs +324 -0
- package/cli/lib/site-next-actions.mjs +333 -0
- package/cli/lib/site-options.mjs +104 -0
- package/cli/lib/site-prompts.mjs +332 -0
- package/cli/lib/site-starter.mjs +153 -0
- package/cli/lib/site-strings.mjs +23 -0
- package/cli/lib/site-tasks.mjs +93 -0
- package/cli/lib/site-test-support.mjs +68 -0
- package/cli/lib/site-workflow-graph.mjs +309 -0
- package/cli/lib/site-workspace.mjs +492 -0
- package/cli/lib/site.mjs +108 -6617
- package/cli/lib/skill-proposals-apply-commands.mjs +135 -0
- package/cli/lib/skill-proposals-apply-contract.mjs +750 -0
- package/cli/lib/skill-proposals-apply-plan.mjs +118 -0
- package/cli/lib/skill-proposals-generate.mjs +298 -0
- package/cli/lib/skill-proposals-render.mjs +532 -0
- package/cli/lib/skill-proposals-review.mjs +262 -0
- package/cli/lib/skill-proposals.mjs +15 -2046
- package/cli/lib/workspace-args.mjs +83 -0
- package/cli/lib/workspace-git.mjs +169 -0
- package/cli/lib/workspace-learning.mjs +483 -0
- package/cli/lib/workspace-repo.mjs +139 -0
- package/cli/lib/workspace-report.mjs +283 -0
- package/cli/lib/workspace-test-support.mjs +99 -0
- package/cli/lib/workspace.mjs +30 -1056
- package/docs/AI-LEARNING-PHASE2.md +215 -0
- package/docs/DISTRIBUTION.ko.md +35 -6
- package/docs/DISTRIBUTION.md +35 -8
- package/docs/NEXT-SURFACE-DECISION.md +125 -0
- package/docs/PRODUCT-READINESS.md +3 -3
- package/docs/RELEASE-CHECKLIST.md +20 -3
- package/docs/RELEASE-GATES.ko.md +35 -0
- package/docs/RELEASE-GATES.md +234 -0
- package/docs/ROADMAP.md +2232 -8
- package/docs/external-status.md +20 -7
- package/docs/inspection-20260630.md +169 -0
- package/docs/integrations/design-ai-mcp-server.md +35 -3
- package/docs/integrations/vscode-walkthrough.ko.md +3 -3
- package/docs/integrations/vscode-walkthrough.md +3 -3
- package/docs/reference/ant-design.md +413 -0
- package/docs/reference/awesome-design-md.md +439 -0
- package/docs/reference/mui.md +783 -0
- package/docs/reference/shadcn-ui.md +298 -0
- package/docs/site-overrides/main.html +1 -1
- package/examples/component-accordion-actions.md +5 -5
- package/examples/component-accordion-details.md +5 -5
- package/examples/component-accordion-summary.md +5 -5
- package/examples/component-accordion.md +3 -3
- package/examples/component-affix.md +1 -1
- package/examples/component-alert-dialog.md +1 -1
- package/examples/component-alert-title.md +1 -1
- package/examples/component-alert.md +3 -3
- package/examples/component-anchor.md +1 -1
- package/examples/component-app-bar.md +1 -1
- package/examples/component-aspect-ratio.md +1 -1
- package/examples/component-auto-complete.md +2 -2
- package/examples/component-avatar-group.md +5 -5
- package/examples/component-avatar.md +3 -3
- package/examples/component-back-top.md +1 -1
- package/examples/component-backdrop.md +1 -1
- package/examples/component-badge.md +3 -3
- package/examples/component-border-beam.md +3 -3
- package/examples/component-bottom-navigation.md +1 -1
- package/examples/component-box.md +1 -1
- package/examples/component-breadcrumb.md +3 -3
- package/examples/component-button-base.md +3 -3
- package/examples/component-button-group.md +2 -2
- package/examples/component-button.md +3 -3
- package/examples/component-calendar.md +2 -2
- package/examples/component-card-actions.md +1 -1
- package/examples/component-card-content.md +1 -1
- package/examples/component-card-header.md +1 -1
- package/examples/component-card-media.md +1 -1
- package/examples/component-card.md +3 -3
- package/examples/component-carousel.md +2 -2
- package/examples/component-cascader.md +1 -1
- package/examples/component-chart.md +1 -1
- package/examples/component-checkbox.md +3 -3
- package/examples/component-click-away-listener.md +1 -1
- package/examples/component-code.md +1 -1
- package/examples/component-collapsible.md +1 -1
- package/examples/component-color-picker.md +1 -1
- package/examples/component-combobox.md +1 -1
- package/examples/component-command.md +1 -1
- package/examples/component-config-provider.md +3 -3
- package/examples/component-context-menu.md +1 -1
- package/examples/component-css-baseline.md +3 -3
- package/examples/component-date-picker.md +2 -2
- package/examples/component-descriptions.md +1 -1
- package/examples/component-dialog-actions.md +1 -1
- package/examples/component-dialog-content-text.md +1 -1
- package/examples/component-dialog-content.md +1 -1
- package/examples/component-dialog-title.md +1 -1
- package/examples/component-dialog.md +2 -2
- package/examples/component-divider.md +3 -3
- package/examples/component-drawer.md +3 -3
- package/examples/component-dropdown.md +3 -3
- package/examples/component-empty.md +2 -2
- package/examples/component-fade.md +1 -1
- package/examples/component-field.md +1 -1
- package/examples/component-filled-input.md +1 -1
- package/examples/component-flex.md +1 -1
- package/examples/component-float-button.md +2 -2
- package/examples/component-form-control-label.md +1 -1
- package/examples/component-form-control.md +1 -1
- package/examples/component-form-controls.md +1 -1
- package/examples/component-form-group.md +1 -1
- package/examples/component-form-helper-text.md +1 -1
- package/examples/component-form-label.md +1 -1
- package/examples/component-form.md +2 -2
- package/examples/component-grid.md +2 -2
- package/examples/component-grow.md +1 -1
- package/examples/component-hover-card.md +1 -1
- package/examples/component-icon-button.md +1 -1
- package/examples/component-icon.md +1 -1
- package/examples/component-image-list.md +1 -1
- package/examples/component-image.md +1 -1
- package/examples/component-input-adornment.md +1 -1
- package/examples/component-input-base.md +1 -1
- package/examples/component-input-number.md +1 -1
- package/examples/component-input-otp.md +2 -2
- package/examples/component-input.md +3 -3
- package/examples/component-item.md +1 -1
- package/examples/component-label.md +1 -1
- package/examples/component-layout.md +1 -1
- package/examples/component-link.md +1 -1
- package/examples/component-list-item-avatar.md +1 -1
- package/examples/component-list-item-button.md +1 -1
- package/examples/component-list-item-icon.md +1 -1
- package/examples/component-list-item-text.md +1 -1
- package/examples/component-list-item.md +1 -1
- package/examples/component-list-subheader.md +1 -1
- package/examples/component-list.md +2 -2
- package/examples/component-masonry.md +1 -1
- package/examples/component-mentions.md +1 -1
- package/examples/component-menu-item.md +1 -1
- package/examples/component-menu-list.md +1 -1
- package/examples/component-menu.md +1 -1
- package/examples/component-menubar.md +1 -1
- package/examples/component-message.md +1 -1
- package/examples/component-mobile-stepper.md +1 -1
- package/examples/component-modal.md +5 -5
- package/examples/component-navigation-menu.md +1 -1
- package/examples/component-notification.md +1 -1
- package/examples/component-outlined-input.md +1 -1
- package/examples/component-pagination.md +3 -3
- package/examples/component-paper.md +1 -1
- package/examples/component-popconfirm.md +1 -1
- package/examples/component-popover.md +3 -3
- package/examples/component-popper.md +1 -1
- package/examples/component-progress.md +2 -2
- package/examples/component-qr-code.md +1 -1
- package/examples/component-radio.md +3 -3
- package/examples/component-rate.md +2 -2
- package/examples/component-resizable.md +1 -1
- package/examples/component-result.md +1 -1
- package/examples/component-scroll-area.md +1 -1
- package/examples/component-segmented.md +3 -3
- package/examples/component-select.md +3 -3
- package/examples/component-separator.md +1 -1
- package/examples/component-sheet.md +1 -1
- package/examples/component-sidebar.md +1 -1
- package/examples/component-skeleton.md +3 -3
- package/examples/component-slide.md +1 -1
- package/examples/component-slider.md +3 -3
- package/examples/component-snackbar-content.md +1 -1
- package/examples/component-snackbar.md +1 -1
- package/examples/component-sonner.md +1 -1
- package/examples/component-space.md +1 -1
- package/examples/component-speed-dial-action.md +1 -1
- package/examples/component-speed-dial.md +1 -1
- package/examples/component-spin.md +2 -2
- package/examples/component-spinner.md +1 -1
- package/examples/component-splitter.md +2 -2
- package/examples/component-stack.md +1 -1
- package/examples/component-statistic.md +1 -1
- package/examples/component-step-button.md +4 -4
- package/examples/component-step-connector.md +4 -4
- package/examples/component-step-content.md +1 -1
- package/examples/component-step-icon.md +1 -1
- package/examples/component-step-label.md +1 -1
- package/examples/component-step.md +2 -2
- package/examples/component-steps.md +2 -2
- package/examples/component-swipeable-drawer.md +1 -1
- package/examples/component-switch.md +3 -3
- package/examples/component-tab-scroll-button.md +5 -5
- package/examples/component-tab.md +1 -1
- package/examples/component-table-body.md +1 -1
- package/examples/component-table-cell.md +1 -1
- package/examples/component-table-container.md +1 -1
- package/examples/component-table-footer.md +1 -1
- package/examples/component-table-head.md +1 -1
- package/examples/component-table-pagination.md +1 -1
- package/examples/component-table-row.md +1 -1
- package/examples/component-table-sort-label.md +1 -1
- package/examples/component-table.md +2 -2
- package/examples/component-tabs.md +3 -3
- package/examples/component-tag-badge.md +2 -2
- package/examples/component-tag.md +1 -1
- package/examples/component-textarea-autosize.md +1 -1
- package/examples/component-textarea.md +1 -1
- package/examples/component-time-picker.md +1 -1
- package/examples/component-timeline.md +1 -1
- package/examples/component-toast.md +4 -4
- package/examples/component-toggle-button.md +1 -1
- package/examples/component-toggle.md +1 -1
- package/examples/component-toolbar.md +1 -1
- package/examples/component-tooltip.md +3 -3
- package/examples/component-tour.md +1 -1
- package/examples/component-transfer.md +1 -1
- package/examples/component-tree-select.md +1 -1
- package/examples/component-tree.md +1 -1
- package/examples/component-typography.md +2 -2
- package/examples/component-upload.md +1 -1
- package/examples/component-watermark.md +1 -1
- package/examples/component-zoom.md +1 -1
- package/knowledge/COVERAGE.md +4 -3
- package/knowledge/components/INDEX.md +202 -202
- package/knowledge/patterns/brand-references.md +72 -72
- package/package.json +1 -1
- package/tools/audit/local-ci.py +16 -1
- package/tools/audit/package-smoke.py +717 -9
- package/tools/audit/registry-smoke.py +912 -10
- package/tools/audit/release-metadata.py +47 -1
- package/tools/audit/smoke_assertions.py +998 -35
- package/tools/migrations/refs-links-to-reference-pages.py +179 -0
|
@@ -39,6 +39,8 @@ from smoke_assertions import (
|
|
|
39
39
|
EXPECTED_HELP_ALIASES,
|
|
40
40
|
EXPECTED_NUMERIC_VALUE_SMOKES,
|
|
41
41
|
EXPECTED_PACK_MAX_BYTES,
|
|
42
|
+
EXPECTED_RANKED_SEARCH_LIMIT,
|
|
43
|
+
EXPECTED_RANKED_SEARCH_NOT_BUILT_NOTICE,
|
|
42
44
|
EXPECTED_REPOSITORY_URL,
|
|
43
45
|
EXPECTED_ROUTE_BRIEF,
|
|
44
46
|
EXPECTED_ROUTE_ID,
|
|
@@ -55,11 +57,15 @@ from smoke_assertions import (
|
|
|
55
57
|
assert_check_examples_json_component_spec,
|
|
56
58
|
assert_check_stdin_json_component_spec,
|
|
57
59
|
assert_command_alias_output,
|
|
60
|
+
assert_design_ai_mcp_protocol_responses,
|
|
58
61
|
assert_examples_human_output,
|
|
59
62
|
assert_examples_json_route_hit,
|
|
60
63
|
assert_force_overwrite_replaced,
|
|
61
64
|
assert_functional_alias_smokes,
|
|
62
65
|
assert_help_topic_output,
|
|
66
|
+
assert_index_build_json,
|
|
67
|
+
assert_index_status_json,
|
|
68
|
+
assert_index_verify_json,
|
|
63
69
|
assert_install_doctor_lifecycle_output,
|
|
64
70
|
assert_list_catalog_output,
|
|
65
71
|
assert_list_catalog_json,
|
|
@@ -71,6 +77,8 @@ from smoke_assertions import (
|
|
|
71
77
|
assert_pack_json_component_spec,
|
|
72
78
|
assert_pack_markdown_body_component_spec,
|
|
73
79
|
assert_pack_markdown_component_spec,
|
|
80
|
+
assert_ranked_search_determinism,
|
|
81
|
+
assert_ranked_search_json,
|
|
74
82
|
assert_site_repair_apply_report_payload,
|
|
75
83
|
assert_site_repair_guidance_report_contract,
|
|
76
84
|
assert_site_repair_preview_report_payload,
|
|
@@ -127,9 +135,14 @@ from smoke_assertions import (
|
|
|
127
135
|
format_cmd,
|
|
128
136
|
help_alias_script,
|
|
129
137
|
help_topic_script,
|
|
138
|
+
mcp_smoke_input,
|
|
130
139
|
parse_help_topics,
|
|
131
140
|
passing_doctor_report_json,
|
|
132
141
|
passing_check_artifact_content,
|
|
142
|
+
passing_index_build_json,
|
|
143
|
+
passing_index_status_json,
|
|
144
|
+
passing_index_verify_json,
|
|
145
|
+
passing_ranked_search_json,
|
|
133
146
|
passing_site_json,
|
|
134
147
|
passing_site_mcp_check_json,
|
|
135
148
|
passing_site_mcp_check_probes_human,
|
|
@@ -154,6 +167,104 @@ from smoke_assertions import (
|
|
|
154
167
|
ROOT = Path(__file__).resolve().parents[2]
|
|
155
168
|
PACKAGE = json.loads((ROOT / "package.json").read_text(encoding="utf-8"))
|
|
156
169
|
DEFAULT_PACKAGE_SPEC = f"{PACKAGE['name']}@{PACKAGE['version']}"
|
|
170
|
+
EXPECTED_SITE_MCP_PROBE_COUNTS = {"count": 4, "pass": 4, "warn": 0, "fail": 0}
|
|
171
|
+
EXPECTED_SITE_BUNDLE_MCP_PROBES_KEYS = [
|
|
172
|
+
"enabled",
|
|
173
|
+
"mode",
|
|
174
|
+
"externalCalls",
|
|
175
|
+
"status",
|
|
176
|
+
"count",
|
|
177
|
+
"pass",
|
|
178
|
+
"warn",
|
|
179
|
+
"fail",
|
|
180
|
+
"items",
|
|
181
|
+
]
|
|
182
|
+
EXPECTED_SITE_BUNDLE_MCP_PROBE_ITEM_KEYS = [
|
|
183
|
+
"id",
|
|
184
|
+
"key",
|
|
185
|
+
"label",
|
|
186
|
+
"requestedStatus",
|
|
187
|
+
"level",
|
|
188
|
+
"passed",
|
|
189
|
+
"message",
|
|
190
|
+
"evidence",
|
|
191
|
+
"actions",
|
|
192
|
+
]
|
|
193
|
+
EXPECTED_SITE_BUNDLE_MCP_PROBE_IDS = [
|
|
194
|
+
"github-repo-reference",
|
|
195
|
+
"figma-url-reference",
|
|
196
|
+
"browser-smoke-target",
|
|
197
|
+
"deploy-provider-reference",
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def assert_site_mcp_probe_counts(
|
|
202
|
+
actual: object,
|
|
203
|
+
*,
|
|
204
|
+
context: str,
|
|
205
|
+
label: str,
|
|
206
|
+
) -> None:
|
|
207
|
+
if actual != EXPECTED_SITE_MCP_PROBE_COUNTS:
|
|
208
|
+
raise SystemExit(f"{label} after {context} MCP probe counts changed: {actual!r}")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def assert_smoke_json_keys(
|
|
212
|
+
payload: object,
|
|
213
|
+
expected_keys: list[str],
|
|
214
|
+
*,
|
|
215
|
+
label: str,
|
|
216
|
+
context: str,
|
|
217
|
+
command_label: str,
|
|
218
|
+
) -> dict:
|
|
219
|
+
if not isinstance(payload, dict):
|
|
220
|
+
raise SystemExit(f"{command_label} after {context} {label} must be an object")
|
|
221
|
+
keys = list(payload.keys())
|
|
222
|
+
if keys != expected_keys:
|
|
223
|
+
raise SystemExit(f"{command_label} after {context} {label} keys changed: {keys!r}")
|
|
224
|
+
return payload
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def assert_site_bundle_mcp_probes_payload(payload: object, *, context: str) -> None:
|
|
228
|
+
checked = assert_smoke_json_keys(
|
|
229
|
+
payload,
|
|
230
|
+
EXPECTED_SITE_BUNDLE_MCP_PROBES_KEYS,
|
|
231
|
+
label="mcp-probes.json",
|
|
232
|
+
context=context,
|
|
233
|
+
command_label="site bundle",
|
|
234
|
+
)
|
|
235
|
+
if checked.get("enabled") is not True or checked.get("mode") != "read-only-local":
|
|
236
|
+
raise SystemExit(f"site bundle after {context} mcp-probes.json mode changed")
|
|
237
|
+
if checked.get("externalCalls") is not False:
|
|
238
|
+
raise SystemExit(f"site bundle after {context} mcp-probes.json must remain read-only")
|
|
239
|
+
if checked.get("status") != "pass":
|
|
240
|
+
raise SystemExit(f"site bundle after {context} mcp-probes.json status changed")
|
|
241
|
+
assert_site_mcp_probe_counts(
|
|
242
|
+
{key: checked.get(key) for key in ("count", "pass", "warn", "fail")},
|
|
243
|
+
context=context,
|
|
244
|
+
label="site bundle mcp-probes.json",
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
items = checked.get("items")
|
|
248
|
+
if not isinstance(items, list) or len(items) != len(EXPECTED_SITE_BUNDLE_MCP_PROBE_IDS):
|
|
249
|
+
raise SystemExit(f"site bundle after {context} mcp-probes.json item count changed")
|
|
250
|
+
checked_ids = []
|
|
251
|
+
for item in items:
|
|
252
|
+
checked_item = assert_smoke_json_keys(
|
|
253
|
+
item,
|
|
254
|
+
EXPECTED_SITE_BUNDLE_MCP_PROBE_ITEM_KEYS,
|
|
255
|
+
label="mcp-probes.json item",
|
|
256
|
+
context=context,
|
|
257
|
+
command_label="site bundle",
|
|
258
|
+
)
|
|
259
|
+
checked_ids.append(checked_item.get("id"))
|
|
260
|
+
if checked_item.get("level") != "pass" or checked_item.get("passed") is not True:
|
|
261
|
+
raise SystemExit(f"site bundle after {context} mcp-probes.json item should pass: {checked_item.get('id')}")
|
|
262
|
+
if not isinstance(checked_item.get("evidence"), list) or not checked_item.get("evidence"):
|
|
263
|
+
raise SystemExit(f"site bundle after {context} mcp-probes.json item evidence missing")
|
|
264
|
+
if not isinstance(checked_item.get("actions"), list):
|
|
265
|
+
raise SystemExit(f"site bundle after {context} mcp-probes.json item actions must be an array")
|
|
266
|
+
if checked_ids != EXPECTED_SITE_BUNDLE_MCP_PROBE_IDS:
|
|
267
|
+
raise SystemExit(f"site bundle after {context} mcp-probes.json item order changed")
|
|
157
268
|
|
|
158
269
|
|
|
159
270
|
def npm_exec_cmd(package_spec: str, *args: str) -> list[str]:
|
|
@@ -169,6 +280,18 @@ def npm_exec_cmd(package_spec: str, *args: str) -> list[str]:
|
|
|
169
280
|
]
|
|
170
281
|
|
|
171
282
|
|
|
283
|
+
def npm_exec_mcp_cmd(package_spec: str) -> list[str]:
|
|
284
|
+
return [
|
|
285
|
+
"npm",
|
|
286
|
+
"exec",
|
|
287
|
+
"--yes",
|
|
288
|
+
"--package",
|
|
289
|
+
package_spec,
|
|
290
|
+
"--",
|
|
291
|
+
"design-ai-mcp",
|
|
292
|
+
]
|
|
293
|
+
|
|
294
|
+
|
|
172
295
|
def npm_exec_shell_cmd(package_spec: str, script: str) -> list[str]:
|
|
173
296
|
return [
|
|
174
297
|
"npm",
|
|
@@ -2093,6 +2216,102 @@ def assert_learning_redact_smoke(
|
|
|
2093
2216
|
)
|
|
2094
2217
|
|
|
2095
2218
|
|
|
2219
|
+
def assert_learning_recall_json(
|
|
2220
|
+
raw: str,
|
|
2221
|
+
*,
|
|
2222
|
+
context: str,
|
|
2223
|
+
cmd: list[str],
|
|
2224
|
+
) -> None:
|
|
2225
|
+
assert_no_ansi(raw, cmd)
|
|
2226
|
+
try:
|
|
2227
|
+
payload = json.loads(raw)
|
|
2228
|
+
except json.JSONDecodeError as error:
|
|
2229
|
+
raise SystemExit(f"{context}: failed to parse learn recall JSON") from error
|
|
2230
|
+
|
|
2231
|
+
require_registry_smoke(
|
|
2232
|
+
isinstance(payload, dict) and payload.get("query") == "korean mobile",
|
|
2233
|
+
context=context,
|
|
2234
|
+
cmd=cmd,
|
|
2235
|
+
message="learn recall JSON must echo the query",
|
|
2236
|
+
)
|
|
2237
|
+
corpus = payload.get("corpus")
|
|
2238
|
+
require_registry_smoke(
|
|
2239
|
+
isinstance(corpus, dict)
|
|
2240
|
+
and isinstance(corpus.get("candidateCount"), int)
|
|
2241
|
+
and corpus.get("candidateCount") > 0
|
|
2242
|
+
and isinstance(corpus.get("selectedCount"), int)
|
|
2243
|
+
and isinstance(corpus.get("selected"), list),
|
|
2244
|
+
context=context,
|
|
2245
|
+
cmd=cmd,
|
|
2246
|
+
message="learn recall corpus block shape changed",
|
|
2247
|
+
)
|
|
2248
|
+
learning = payload.get("learning")
|
|
2249
|
+
require_registry_smoke(
|
|
2250
|
+
isinstance(learning, dict)
|
|
2251
|
+
and learning.get("mode") == "brief-relevance"
|
|
2252
|
+
and learning.get("candidateCount") == 3
|
|
2253
|
+
and isinstance(learning.get("selected"), list),
|
|
2254
|
+
context=context,
|
|
2255
|
+
cmd=cmd,
|
|
2256
|
+
message="learn recall learning block shape changed",
|
|
2257
|
+
)
|
|
2258
|
+
selected = learning.get("selected")
|
|
2259
|
+
require_registry_smoke(
|
|
2260
|
+
len(selected) >= 1
|
|
2261
|
+
and isinstance(selected[0], dict)
|
|
2262
|
+
and selected[0].get("id") == "registry-korean"
|
|
2263
|
+
and selected[0].get("category") == "korean"
|
|
2264
|
+
and isinstance(selected[0].get("matchedTokens"), list),
|
|
2265
|
+
context=context,
|
|
2266
|
+
cmd=cmd,
|
|
2267
|
+
message="learn recall should rank the Korean learning entry for a Korean query",
|
|
2268
|
+
)
|
|
2269
|
+
|
|
2270
|
+
|
|
2271
|
+
def assert_learning_recall_smoke(
|
|
2272
|
+
command_factory,
|
|
2273
|
+
profile_path: Path,
|
|
2274
|
+
*,
|
|
2275
|
+
env: dict[str, str],
|
|
2276
|
+
cwd: Path | None = None,
|
|
2277
|
+
context: str,
|
|
2278
|
+
) -> None:
|
|
2279
|
+
write_learning_stats_fixture(profile_path)
|
|
2280
|
+
|
|
2281
|
+
json_cmd = command_factory(
|
|
2282
|
+
"learn",
|
|
2283
|
+
"--recall",
|
|
2284
|
+
"korean mobile",
|
|
2285
|
+
"--file",
|
|
2286
|
+
str(profile_path),
|
|
2287
|
+
"--json",
|
|
2288
|
+
)
|
|
2289
|
+
json_result = run_plain(json_cmd, cwd=cwd, env=env)
|
|
2290
|
+
assert_learning_recall_json(json_result.stdout, context=f"{context} JSON", cmd=json_cmd)
|
|
2291
|
+
|
|
2292
|
+
# Read-only: the profile file is unchanged after recall.
|
|
2293
|
+
before = write_learning_stats_fixture_snapshot(profile_path)
|
|
2294
|
+
human_cmd = command_factory("learn", "--recall", "korean mobile", "--file", str(profile_path))
|
|
2295
|
+
human_result = run_plain(human_cmd, cwd=cwd, env=env)
|
|
2296
|
+
require_registry_smoke(
|
|
2297
|
+
"Recall (read-only)" in human_result.stdout
|
|
2298
|
+
and "Privacy: recall is read-only" in human_result.stdout,
|
|
2299
|
+
context=f"{context} human",
|
|
2300
|
+
cmd=human_cmd,
|
|
2301
|
+
message="learn recall human output missing read-only markers",
|
|
2302
|
+
)
|
|
2303
|
+
require_registry_smoke(
|
|
2304
|
+
profile_path.read_text(encoding="utf-8") == before,
|
|
2305
|
+
context=f"{context} read-only",
|
|
2306
|
+
cmd=human_cmd,
|
|
2307
|
+
message="learn recall must not mutate the learning profile",
|
|
2308
|
+
)
|
|
2309
|
+
|
|
2310
|
+
|
|
2311
|
+
def write_learning_stats_fixture_snapshot(profile_path: Path) -> str:
|
|
2312
|
+
return profile_path.read_text(encoding="utf-8")
|
|
2313
|
+
|
|
2314
|
+
|
|
2096
2315
|
def assert_learning_stats_json(
|
|
2097
2316
|
raw: str,
|
|
2098
2317
|
*,
|
|
@@ -2621,6 +2840,45 @@ def assert_version_json_smoke(cmd: list[str], *, env: dict[str, str], cwd: Path
|
|
|
2621
2840
|
assert_version_json(result.stdout, context=context, cmd=cmd)
|
|
2622
2841
|
|
|
2623
2842
|
|
|
2843
|
+
def assert_design_ai_mcp_protocol_smoke(
|
|
2844
|
+
cmd: list[str],
|
|
2845
|
+
*,
|
|
2846
|
+
env: dict[str, str],
|
|
2847
|
+
cwd: Path | None,
|
|
2848
|
+
context: str,
|
|
2849
|
+
) -> None:
|
|
2850
|
+
print(f"$ {format_cmd(cmd)} < MCP smoke", flush=True)
|
|
2851
|
+
result = subprocess.run(
|
|
2852
|
+
cmd,
|
|
2853
|
+
cwd=cwd,
|
|
2854
|
+
env=env,
|
|
2855
|
+
input=mcp_smoke_input(),
|
|
2856
|
+
text=True,
|
|
2857
|
+
capture_output=True,
|
|
2858
|
+
timeout=15,
|
|
2859
|
+
)
|
|
2860
|
+
if result.stdout:
|
|
2861
|
+
print(result.stdout, end="")
|
|
2862
|
+
if result.stderr:
|
|
2863
|
+
print(result.stderr, end="", file=sys.stderr)
|
|
2864
|
+
|
|
2865
|
+
if result.returncode != 0:
|
|
2866
|
+
raise SystemExit(f"command failed with exit code {result.returncode}: {format_cmd(cmd)}")
|
|
2867
|
+
|
|
2868
|
+
output = f"{result.stdout}\n{result.stderr}"
|
|
2869
|
+
assert_no_ansi(output, cmd)
|
|
2870
|
+
|
|
2871
|
+
responses = []
|
|
2872
|
+
for line in result.stdout.splitlines():
|
|
2873
|
+
if not line.strip():
|
|
2874
|
+
continue
|
|
2875
|
+
try:
|
|
2876
|
+
responses.append(json.loads(line))
|
|
2877
|
+
except json.JSONDecodeError as error:
|
|
2878
|
+
raise SystemExit(f"{context}: MCP stdout line is not JSON: {error}") from error
|
|
2879
|
+
assert_design_ai_mcp_protocol_responses(responses, context=context, cmd=cmd)
|
|
2880
|
+
|
|
2881
|
+
|
|
2624
2882
|
def assert_workspace_json_smoke(cmd: list[str], *, env: dict[str, str], cwd: Path | None = None, context: str) -> None:
|
|
2625
2883
|
result = run_plain(cmd, cwd=cwd, env=env)
|
|
2626
2884
|
assert_workspace_json(result.stdout, context=context, cmd=cmd)
|
|
@@ -3031,6 +3289,7 @@ def assert_site_bundle_smoke(
|
|
|
3031
3289
|
"summary.json",
|
|
3032
3290
|
"website-workspace.tasks.json",
|
|
3033
3291
|
"mcp-check.json",
|
|
3292
|
+
"mcp-probes.json",
|
|
3034
3293
|
"mcp-action-plan.md",
|
|
3035
3294
|
"website-handoff.md",
|
|
3036
3295
|
"website-prompts.md",
|
|
@@ -3046,8 +3305,26 @@ def assert_site_bundle_smoke(
|
|
|
3046
3305
|
raise SystemExit(f"site bundle after {context} summary status changed: {summary.get('status')!r}")
|
|
3047
3306
|
if summary.get("taskGeneration", {}).get("totalTasks") != 3:
|
|
3048
3307
|
raise SystemExit(f"site bundle after {context} expected 3 generated/retained tasks")
|
|
3308
|
+
evidence_counts = summary.get("implementationEvidence")
|
|
3309
|
+
if (
|
|
3310
|
+
not isinstance(evidence_counts, dict)
|
|
3311
|
+
or evidence_counts.get("executedWork") != 0
|
|
3312
|
+
or evidence_counts.get("verificationResults") != 0
|
|
3313
|
+
or evidence_counts.get("remainingRisks") != 3
|
|
3314
|
+
or evidence_counts.get("nextActions") != 0
|
|
3315
|
+
):
|
|
3316
|
+
raise SystemExit(f"site bundle after {context} implementation evidence counts changed")
|
|
3049
3317
|
if summary.get("files") != expected_files:
|
|
3050
3318
|
raise SystemExit(f"site bundle after {context} file manifest changed")
|
|
3319
|
+
execution_checklist = summary.get("handoff", {}).get("executionChecklist")
|
|
3320
|
+
if not isinstance(execution_checklist, list) or [item.get("id") for item in execution_checklist] != [
|
|
3321
|
+
"confirm-target-repo",
|
|
3322
|
+
"inspect-architecture",
|
|
3323
|
+
"apply-focused-task",
|
|
3324
|
+
"verify-quality-gates",
|
|
3325
|
+
"record-handoff-evidence",
|
|
3326
|
+
]:
|
|
3327
|
+
raise SystemExit(f"site bundle after {context} handoff execution checklist changed")
|
|
3051
3328
|
checksums = summary.get("checksums", {})
|
|
3052
3329
|
checksum_files = checksums.get("files", {})
|
|
3053
3330
|
if checksums.get("algorithm") != "sha256":
|
|
@@ -3066,9 +3343,14 @@ def assert_site_bundle_smoke(
|
|
|
3066
3343
|
task_ids = [task.get("id") for task in tasks.get("refactorTasks", [])]
|
|
3067
3344
|
if task_ids != ["task-homepage-cta", "task-accessibility", "task-content-quality"]:
|
|
3068
3345
|
raise SystemExit(f"site bundle after {context} task ids changed: {task_ids!r}")
|
|
3346
|
+
evidence = tasks.get("implementationEvidence")
|
|
3347
|
+
if not isinstance(evidence, dict) or len(evidence.get("remainingRisks", [])) != 3:
|
|
3348
|
+
raise SystemExit(f"site bundle after {context} did not preserve implementationEvidence in workspace JSON")
|
|
3069
3349
|
|
|
3070
3350
|
mcp_check = json.loads((out_dir / "mcp-check.json").read_text(encoding="utf-8"))
|
|
3071
3351
|
assert_site_mcp_check_json(json.dumps(mcp_check), context=context, cmd=cmd)
|
|
3352
|
+
mcp_probes = json.loads((out_dir / "mcp-probes.json").read_text(encoding="utf-8"))
|
|
3353
|
+
assert_site_bundle_mcp_probes_payload(mcp_probes, context=context)
|
|
3072
3354
|
assert_site_mcp_plan_markdown((out_dir / "mcp-action-plan.md").read_text(encoding="utf-8"), context=context, cmd=cmd)
|
|
3073
3355
|
implementation_prompt = (out_dir / "codex-implementation.md").read_text(encoding="utf-8")
|
|
3074
3356
|
assert_no_ansi(implementation_prompt, cmd)
|
|
@@ -3083,6 +3365,8 @@ def assert_site_bundle_smoke(
|
|
|
3083
3365
|
readme = (out_dir / "README.md").read_text(encoding="utf-8")
|
|
3084
3366
|
if "Website improvement handoff bundle" not in readme or "does not call external MCPs" not in readme:
|
|
3085
3367
|
raise SystemExit(f"site bundle after {context} README missing bundle boundary guidance")
|
|
3368
|
+
if "Target Repo Execution Checklist" not in readme or "Run target repo quality gates" not in readme:
|
|
3369
|
+
raise SystemExit(f"site bundle after {context} README missing target repo execution checklist")
|
|
3086
3370
|
|
|
3087
3371
|
|
|
3088
3372
|
def assert_site_warning_bundle_smoke(
|
|
@@ -3141,22 +3425,82 @@ def assert_site_bundle_check_json_smoke(
|
|
|
3141
3425
|
env: dict[str, str],
|
|
3142
3426
|
cwd: Path | None = None,
|
|
3143
3427
|
context: str,
|
|
3428
|
+
expected_evidence_counts: dict[str, int] | None = None,
|
|
3144
3429
|
) -> None:
|
|
3430
|
+
if expected_evidence_counts is None:
|
|
3431
|
+
expected_evidence_counts = {
|
|
3432
|
+
"executedWork": 0,
|
|
3433
|
+
"verificationResults": 0,
|
|
3434
|
+
"remainingRisks": 3,
|
|
3435
|
+
"nextActions": 0,
|
|
3436
|
+
}
|
|
3145
3437
|
result = run_plain(cmd, cwd=cwd, env=env)
|
|
3146
3438
|
assert_no_ansi(result.stdout, cmd)
|
|
3147
3439
|
payload = json.loads(result.stdout)
|
|
3148
3440
|
if payload.get("status") != "pass" or payload.get("valid") is not True:
|
|
3149
3441
|
raise SystemExit(f"site bundle check after {context} expected pass/valid output")
|
|
3150
|
-
if payload.get("
|
|
3151
|
-
raise SystemExit(f"site bundle check after {context}
|
|
3152
|
-
|
|
3153
|
-
|
|
3442
|
+
if payload.get("externalCalls") is not False or payload.get("targetRepoMutation") is not False:
|
|
3443
|
+
raise SystemExit(f"site bundle check after {context} boundary flags changed")
|
|
3444
|
+
boundaries = payload.get("boundaries")
|
|
3445
|
+
if (
|
|
3446
|
+
not isinstance(boundaries, list)
|
|
3447
|
+
or "deterministic-local" not in boundaries
|
|
3448
|
+
or "no-external-mcp-calls" not in boundaries
|
|
3449
|
+
or "no-target-repo-mutation" not in boundaries
|
|
3450
|
+
):
|
|
3451
|
+
raise SystemExit(f"site bundle check after {context} boundary list changed: {boundaries!r}")
|
|
3452
|
+
if payload.get("counts", {}).get("presentFiles") != 9:
|
|
3453
|
+
raise SystemExit(f"site bundle check after {context} expected 9 present files")
|
|
3454
|
+
if payload.get("counts", {}).get("verifiedChecksumFiles") != 8:
|
|
3455
|
+
raise SystemExit(f"site bundle check after {context} expected 8 verified checksum files")
|
|
3154
3456
|
if payload.get("counts", {}).get("checksumFailures") != 0:
|
|
3155
3457
|
raise SystemExit(f"site bundle check after {context} expected no checksum failures")
|
|
3458
|
+
if payload.get("counts", {}).get("verifiedGeneratedFiles") != 8:
|
|
3459
|
+
raise SystemExit(f"site bundle check after {context} expected 8 current-contract generated files")
|
|
3460
|
+
if payload.get("counts", {}).get("generatedFailures") != 0:
|
|
3461
|
+
raise SystemExit(f"site bundle check after {context} expected no generated bundle contract failures")
|
|
3462
|
+
generated_contract = payload.get("generatedContract")
|
|
3463
|
+
if not isinstance(generated_contract, dict) or generated_contract.get("available") is not True:
|
|
3464
|
+
raise SystemExit(f"site bundle check after {context} generated contract diagnostics missing")
|
|
3465
|
+
if generated_contract.get("expectedFiles") != 8 or generated_contract.get("verifiedFiles") != 8:
|
|
3466
|
+
raise SystemExit(f"site bundle check after {context} generated contract file counts changed")
|
|
3467
|
+
if generated_contract.get("driftFiles") != []:
|
|
3468
|
+
raise SystemExit(f"site bundle check after {context} expected no generated contract drift files")
|
|
3469
|
+
generated_files = generated_contract.get("files")
|
|
3470
|
+
if not isinstance(generated_files, list) or len(generated_files) != 8:
|
|
3471
|
+
raise SystemExit(f"site bundle check after {context} expected 8 generated contract file diagnostics")
|
|
3472
|
+
for item in generated_files:
|
|
3473
|
+
if item.get("present") is not True or item.get("matches") is not True:
|
|
3474
|
+
raise SystemExit(f"site bundle check after {context} generated contract file did not match: {item!r}")
|
|
3475
|
+
for key in ("expectedDigest", "actualDigest"):
|
|
3476
|
+
digest = item.get(key)
|
|
3477
|
+
if not isinstance(digest, str) or len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
|
|
3478
|
+
raise SystemExit(f"site bundle check after {context} generated contract {key} is not a SHA-256 hex digest")
|
|
3479
|
+
repair_guidance = payload.get("repairGuidance")
|
|
3480
|
+
if not isinstance(repair_guidance, dict) or repair_guidance.get("available") is not True:
|
|
3481
|
+
raise SystemExit(f"site bundle check after {context} repair guidance missing")
|
|
3482
|
+
if repair_guidance.get("targetRepoMutation") is not False or repair_guidance.get("externalCalls") is not False:
|
|
3483
|
+
raise SystemExit(f"site bundle check after {context} repair guidance boundary flags changed")
|
|
3484
|
+
repair_command = repair_guidance.get("command")
|
|
3485
|
+
verify_command = repair_guidance.get("verifyCommand")
|
|
3486
|
+
if (
|
|
3487
|
+
not isinstance(repair_command, str)
|
|
3488
|
+
or "website-workspace.tasks.json --bundle --out " not in repair_command
|
|
3489
|
+
or " --force" not in repair_command
|
|
3490
|
+
):
|
|
3491
|
+
raise SystemExit(f"site bundle check after {context} repair command changed: {repair_command!r}")
|
|
3492
|
+
if not isinstance(verify_command, str) or "--bundle-check --strict --json" not in verify_command:
|
|
3493
|
+
raise SystemExit(f"site bundle check after {context} repair verify command changed: {verify_command!r}")
|
|
3156
3494
|
if payload.get("summary", {}).get("totalTasks") != 3:
|
|
3157
3495
|
raise SystemExit(f"site bundle check after {context} expected 3 tasks")
|
|
3158
3496
|
if payload.get("summary", {}).get("siteName") != "Korean SaaS marketing site":
|
|
3159
3497
|
raise SystemExit(f"site bundle check after {context} site name changed")
|
|
3498
|
+
evidence_counts = payload.get("summary", {}).get("implementationEvidence")
|
|
3499
|
+
if not isinstance(evidence_counts, dict):
|
|
3500
|
+
raise SystemExit(f"site bundle check after {context} implementationEvidence counts missing")
|
|
3501
|
+
for key, expected in expected_evidence_counts.items():
|
|
3502
|
+
if evidence_counts.get(key) != expected:
|
|
3503
|
+
raise SystemExit(f"site bundle check after {context} evidence count {key} changed: {evidence_counts.get(key)!r}")
|
|
3160
3504
|
if payload.get("summary", {}).get("checksumAlgorithm") != "sha256":
|
|
3161
3505
|
raise SystemExit(f"site bundle check after {context} checksum algorithm changed")
|
|
3162
3506
|
bundle_digest = payload.get("summary", {}).get("checksumBundleDigest")
|
|
@@ -3164,6 +3508,18 @@ def assert_site_bundle_check_json_smoke(
|
|
|
3164
3508
|
raise SystemExit(f"site bundle check after {context} bundle digest changed")
|
|
3165
3509
|
if payload.get("mcpStatus") != "pass":
|
|
3166
3510
|
raise SystemExit(f"site bundle check after {context} MCP status changed")
|
|
3511
|
+
if payload.get("mcpProbeStatus") != "pass":
|
|
3512
|
+
raise SystemExit(f"site bundle check after {context} MCP probe status changed")
|
|
3513
|
+
assert_site_mcp_probe_counts(
|
|
3514
|
+
payload.get("mcpProbeCounts"),
|
|
3515
|
+
context=context,
|
|
3516
|
+
label="site bundle check",
|
|
3517
|
+
)
|
|
3518
|
+
assert_site_mcp_probe_counts(
|
|
3519
|
+
payload.get("summary", {}).get("mcpProbeCounts"),
|
|
3520
|
+
context=context,
|
|
3521
|
+
label="site bundle check summary",
|
|
3522
|
+
)
|
|
3167
3523
|
issue_ids = [issue.get("id") for issue in payload.get("issues", [])]
|
|
3168
3524
|
if issue_ids != ["bundle-ready"]:
|
|
3169
3525
|
raise SystemExit(f"site bundle check after {context} expected bundle-ready only, got {issue_ids!r}")
|
|
@@ -3175,7 +3531,15 @@ def assert_site_bundle_compare_json_smoke(
|
|
|
3175
3531
|
env: dict[str, str],
|
|
3176
3532
|
cwd: Path | None = None,
|
|
3177
3533
|
context: str,
|
|
3534
|
+
expected_evidence_counts: dict[str, int] | None = None,
|
|
3178
3535
|
) -> None:
|
|
3536
|
+
if expected_evidence_counts is None:
|
|
3537
|
+
expected_evidence_counts = {
|
|
3538
|
+
"executedWork": 0,
|
|
3539
|
+
"verificationResults": 0,
|
|
3540
|
+
"remainingRisks": 3,
|
|
3541
|
+
"nextActions": 0,
|
|
3542
|
+
}
|
|
3179
3543
|
result = run_plain(cmd, cwd=cwd, env=env)
|
|
3180
3544
|
assert_no_ansi(result.stdout, cmd)
|
|
3181
3545
|
payload = json.loads(result.stdout)
|
|
@@ -3191,6 +3555,21 @@ def assert_site_bundle_compare_json_smoke(
|
|
|
3191
3555
|
raise SystemExit(f"site bundle compare after {context} {side} bundle digest changed")
|
|
3192
3556
|
if payload.get(side, {}).get("siteName") != "Korean SaaS marketing site":
|
|
3193
3557
|
raise SystemExit(f"site bundle compare after {context} {side} site name changed")
|
|
3558
|
+
assert_site_mcp_probe_counts(
|
|
3559
|
+
payload.get(side, {}).get("mcpProbeCounts"),
|
|
3560
|
+
context=context,
|
|
3561
|
+
label=f"site bundle compare {side}",
|
|
3562
|
+
)
|
|
3563
|
+
if payload.get(side, {}).get("verifiedGeneratedFiles") != 8 or payload.get(side, {}).get("generatedFailures") != 0:
|
|
3564
|
+
raise SystemExit(f"site bundle compare after {context} {side} generated bundle contract verification changed")
|
|
3565
|
+
if payload.get(side, {}).get("generatedDriftFiles") != []:
|
|
3566
|
+
raise SystemExit(f"site bundle compare after {context} {side} generated bundle contract drift changed")
|
|
3567
|
+
evidence_counts = payload.get(side, {}).get("implementationEvidence")
|
|
3568
|
+
if not isinstance(evidence_counts, dict):
|
|
3569
|
+
raise SystemExit(f"site bundle compare after {context} {side} implementationEvidence counts missing")
|
|
3570
|
+
for key, expected in expected_evidence_counts.items():
|
|
3571
|
+
if evidence_counts.get(key) != expected:
|
|
3572
|
+
raise SystemExit(f"site bundle compare after {context} {side} evidence count {key} changed: {evidence_counts.get(key)!r}")
|
|
3194
3573
|
issue_ids = [issue.get("id") for issue in payload.get("issues", [])]
|
|
3195
3574
|
if issue_ids != ["bundle-compare-identical"]:
|
|
3196
3575
|
raise SystemExit(f"site bundle compare after {context} expected bundle-compare-identical only, got {issue_ids!r}")
|
|
@@ -3202,17 +3581,72 @@ def assert_site_bundle_handoff_json_smoke(
|
|
|
3202
3581
|
env: dict[str, str],
|
|
3203
3582
|
cwd: Path | None = None,
|
|
3204
3583
|
context: str,
|
|
3584
|
+
expected_evidence_counts: dict[str, int] | None = None,
|
|
3205
3585
|
) -> None:
|
|
3586
|
+
if expected_evidence_counts is None:
|
|
3587
|
+
expected_evidence_counts = {
|
|
3588
|
+
"executedWork": 0,
|
|
3589
|
+
"verificationResults": 0,
|
|
3590
|
+
"remainingRisks": 3,
|
|
3591
|
+
"nextActions": 0,
|
|
3592
|
+
}
|
|
3206
3593
|
result = run_plain(cmd, cwd=cwd, env=env)
|
|
3207
3594
|
assert_no_ansi(result.stdout, cmd)
|
|
3208
3595
|
payload = json.loads(result.stdout)
|
|
3209
3596
|
if payload.get("status") != "pass" or payload.get("valid") is not True:
|
|
3210
3597
|
raise SystemExit(f"site bundle handoff after {context} expected pass/valid output")
|
|
3598
|
+
expected_boundaries = [
|
|
3599
|
+
"deterministic-local",
|
|
3600
|
+
"no-external-mcp-calls",
|
|
3601
|
+
"no-target-repo-mutation",
|
|
3602
|
+
"no-lighthouse-axe-visual-diff",
|
|
3603
|
+
"target-repo-work-after-handoff",
|
|
3604
|
+
]
|
|
3605
|
+
if payload.get("boundaries") != expected_boundaries:
|
|
3606
|
+
raise SystemExit(f"site bundle handoff after {context} boundary list changed: {payload.get('boundaries')!r}")
|
|
3607
|
+
if payload.get("externalCalls") is not False or payload.get("targetRepoMutation") is not False:
|
|
3608
|
+
raise SystemExit(f"site bundle handoff after {context} boundary flags changed")
|
|
3211
3609
|
bundle = payload.get("bundle", {})
|
|
3610
|
+
source_bundle = payload.get("sourceBundle")
|
|
3611
|
+
if not isinstance(source_bundle, dict):
|
|
3612
|
+
raise SystemExit(f"site bundle handoff after {context} source bundle provenance missing")
|
|
3613
|
+
if bundle.get("sourceBundle") != source_bundle:
|
|
3614
|
+
raise SystemExit(f"site bundle handoff after {context} source bundle provenance is not mirrored under bundle")
|
|
3615
|
+
if (
|
|
3616
|
+
source_bundle.get("status") != "pass"
|
|
3617
|
+
or source_bundle.get("valid") is not True
|
|
3618
|
+
or source_bundle.get("sourceWorkspace") != "stdin"
|
|
3619
|
+
or source_bundle.get("siteName") != "Korean SaaS marketing site"
|
|
3620
|
+
):
|
|
3621
|
+
raise SystemExit(f"site bundle handoff after {context} source bundle provenance summary changed: {source_bundle!r}")
|
|
3622
|
+
if (
|
|
3623
|
+
source_bundle.get("verifiedGeneratedFiles") != 8
|
|
3624
|
+
or source_bundle.get("expectedGeneratedFiles") != 8
|
|
3625
|
+
or source_bundle.get("verifiedChecksumFiles") != 8
|
|
3626
|
+
or source_bundle.get("expectedChecksumFiles") != 8
|
|
3627
|
+
):
|
|
3628
|
+
raise SystemExit(f"site bundle handoff after {context} source bundle provenance verification counts changed: {source_bundle!r}")
|
|
3212
3629
|
if bundle.get("siteName") != "Korean SaaS marketing site":
|
|
3213
3630
|
raise SystemExit(f"site bundle handoff after {context} site name changed")
|
|
3214
|
-
if bundle.get("
|
|
3631
|
+
if bundle.get("boundaries") != expected_boundaries:
|
|
3632
|
+
raise SystemExit(f"site bundle handoff after {context} bundle boundary list changed: {bundle.get('boundaries')!r}")
|
|
3633
|
+
if bundle.get("externalCalls") is not False or bundle.get("targetRepoMutation") is not False:
|
|
3634
|
+
raise SystemExit(f"site bundle handoff after {context} bundle boundary flags changed")
|
|
3635
|
+
if bundle.get("verifiedChecksumFiles") != 8 or bundle.get("checksumFailures") != 0:
|
|
3215
3636
|
raise SystemExit(f"site bundle handoff after {context} checksum verification changed")
|
|
3637
|
+
if bundle.get("verifiedGeneratedFiles") != 8 or bundle.get("generatedFailures") != 0:
|
|
3638
|
+
raise SystemExit(f"site bundle handoff after {context} generated bundle contract verification changed")
|
|
3639
|
+
assert_site_mcp_probe_counts(
|
|
3640
|
+
bundle.get("mcpProbeCounts"),
|
|
3641
|
+
context=context,
|
|
3642
|
+
label="site bundle handoff",
|
|
3643
|
+
)
|
|
3644
|
+
evidence_counts = bundle.get("implementationEvidence")
|
|
3645
|
+
if not isinstance(evidence_counts, dict):
|
|
3646
|
+
raise SystemExit(f"site bundle handoff after {context} implementationEvidence counts missing")
|
|
3647
|
+
for key, expected in expected_evidence_counts.items():
|
|
3648
|
+
if evidence_counts.get(key) != expected:
|
|
3649
|
+
raise SystemExit(f"site bundle handoff after {context} evidence count {key} changed: {evidence_counts.get(key)!r}")
|
|
3216
3650
|
digest = bundle.get("checksumBundleDigest")
|
|
3217
3651
|
if not isinstance(digest, str) or len(digest) != 64:
|
|
3218
3652
|
raise SystemExit(f"site bundle handoff after {context} bundle digest changed")
|
|
@@ -3221,9 +3655,9 @@ def assert_site_bundle_handoff_json_smoke(
|
|
|
3221
3655
|
raise SystemExit(f"site bundle handoff after {context} prompt missing")
|
|
3222
3656
|
for fragment in (
|
|
3223
3657
|
"Website improvement target-repo handoff prompt",
|
|
3224
|
-
"
|
|
3225
|
-
"
|
|
3226
|
-
"
|
|
3658
|
+
"SHA-256 bundle digest:",
|
|
3659
|
+
"Source bundle strict check command: `design-ai site",
|
|
3660
|
+
"You are Codex working in the target website repository, not in the design-ai repository.",
|
|
3227
3661
|
"# Codex implementation prompt",
|
|
3228
3662
|
):
|
|
3229
3663
|
if fragment not in prompt:
|
|
@@ -4085,6 +4519,56 @@ def assert_pack_stdin_smoke(
|
|
|
4085
4519
|
)
|
|
4086
4520
|
|
|
4087
4521
|
|
|
4522
|
+
def assert_recall_context(payload: dict[str, object], *, context: str, cmd: list[str]) -> None:
|
|
4523
|
+
recall = payload.get("recall")
|
|
4524
|
+
require_registry_smoke(
|
|
4525
|
+
isinstance(recall, dict),
|
|
4526
|
+
context=context,
|
|
4527
|
+
cmd=cmd,
|
|
4528
|
+
message="recall should be present when --with-recall is used",
|
|
4529
|
+
)
|
|
4530
|
+
require_registry_smoke(
|
|
4531
|
+
recall.get("mode") == "lexical",
|
|
4532
|
+
context=context,
|
|
4533
|
+
cmd=cmd,
|
|
4534
|
+
message="recall should use the deterministic lexical scorer",
|
|
4535
|
+
)
|
|
4536
|
+
selected_count = recall.get("selectedCount")
|
|
4537
|
+
require_registry_smoke(
|
|
4538
|
+
isinstance(selected_count, int) and selected_count >= 1,
|
|
4539
|
+
context=context,
|
|
4540
|
+
cmd=cmd,
|
|
4541
|
+
message="recall should select at least one corpus file for the Button brief",
|
|
4542
|
+
)
|
|
4543
|
+
selected = recall.get("selected")
|
|
4544
|
+
require_registry_smoke(
|
|
4545
|
+
isinstance(selected, list) and len(selected) == selected_count and len(selected) >= 1,
|
|
4546
|
+
context=context,
|
|
4547
|
+
cmd=cmd,
|
|
4548
|
+
message="recall selected list should match the reported selected count",
|
|
4549
|
+
)
|
|
4550
|
+
top = selected[0]
|
|
4551
|
+
require_registry_smoke(
|
|
4552
|
+
isinstance(top, dict) and isinstance(top.get("id"), str) and top.get("id"),
|
|
4553
|
+
context=context,
|
|
4554
|
+
cmd=cmd,
|
|
4555
|
+
message="recall selection should cite the corpus relPath as its id",
|
|
4556
|
+
)
|
|
4557
|
+
require_registry_smoke(
|
|
4558
|
+
type(top.get("score")) in (int, float) and top.get("score") > 0,
|
|
4559
|
+
context=context,
|
|
4560
|
+
cmd=cmd,
|
|
4561
|
+
message="recall selection should include a positive relevance score",
|
|
4562
|
+
)
|
|
4563
|
+
markdown = recall.get("markdown")
|
|
4564
|
+
require_registry_smoke(
|
|
4565
|
+
isinstance(markdown, str) and "## Recalled design knowledge" in markdown and top.get("id") in markdown,
|
|
4566
|
+
context=context,
|
|
4567
|
+
cmd=cmd,
|
|
4568
|
+
message="recall markdown should carry the Recalled design knowledge section and cite the top file",
|
|
4569
|
+
)
|
|
4570
|
+
|
|
4571
|
+
|
|
4088
4572
|
def assert_learning_relevance_context(payload: dict[str, object], *, context: str, cmd: list[str]) -> None:
|
|
4089
4573
|
learning_context = payload.get("learningContext")
|
|
4090
4574
|
require_registry_smoke(
|
|
@@ -4502,7 +4986,7 @@ def assert_learning_signals_report_smoke(
|
|
|
4502
4986
|
cwd: Path | None = None,
|
|
4503
4987
|
context: str,
|
|
4504
4988
|
) -> None:
|
|
4505
|
-
seed_force_overwrite_target(output_path, context=context, cmd=["design-ai", "learn", "--signals", "--report"])
|
|
4989
|
+
seed_force_overwrite_target(output_path, context=context, cmd=["design-ai", "learn", "--signals", "--report", "--force"])
|
|
4506
4990
|
cmd = command_factory(
|
|
4507
4991
|
"learn",
|
|
4508
4992
|
"--signals",
|
|
@@ -4536,7 +5020,7 @@ def assert_learning_agent_backlog_report_smoke(
|
|
|
4536
5020
|
cwd: Path | None = None,
|
|
4537
5021
|
context: str,
|
|
4538
5022
|
) -> None:
|
|
4539
|
-
seed_force_overwrite_target(output_path, context=context, cmd=["design-ai", "learn", "--agent-backlog", "--report"])
|
|
5023
|
+
seed_force_overwrite_target(output_path, context=context, cmd=["design-ai", "learn", "--agent-backlog", "--report", "--force"])
|
|
4540
5024
|
cmd = command_factory(
|
|
4541
5025
|
"learn",
|
|
4542
5026
|
"--agent-backlog",
|
|
@@ -4662,6 +5146,49 @@ def assert_learning_relevance_smoke(
|
|
|
4662
5146
|
require_registry_smoke(isinstance(plan, dict), context=f"{context} pack", cmd=pack_cmd, message="pack plan missing")
|
|
4663
5147
|
assert_learning_relevance_context(plan, context=f"{context} pack plan", cmd=pack_cmd)
|
|
4664
5148
|
|
|
5149
|
+
recall_prompt_cmd = command_factory(
|
|
5150
|
+
"prompt",
|
|
5151
|
+
EXPECTED_ROUTE_BRIEF,
|
|
5152
|
+
"--route",
|
|
5153
|
+
EXPECTED_ROUTE_ID,
|
|
5154
|
+
"--with-recall",
|
|
5155
|
+
"--recall-limit",
|
|
5156
|
+
"3",
|
|
5157
|
+
"--json",
|
|
5158
|
+
)
|
|
5159
|
+
recall_prompt_result = run_plain(recall_prompt_cmd, cwd=cwd, env=relevance_env)
|
|
5160
|
+
try:
|
|
5161
|
+
recall_prompt_payload = json.loads(recall_prompt_result.stdout)
|
|
5162
|
+
except json.JSONDecodeError as error:
|
|
5163
|
+
raise SystemExit(f"{context}: failed to parse prompt recall JSON") from error
|
|
5164
|
+
assert_recall_context(recall_prompt_payload, context=f"{context} prompt recall", cmd=recall_prompt_cmd)
|
|
5165
|
+
|
|
5166
|
+
recall_pack_cmd = command_factory(
|
|
5167
|
+
"pack",
|
|
5168
|
+
EXPECTED_ROUTE_BRIEF,
|
|
5169
|
+
"--route",
|
|
5170
|
+
EXPECTED_ROUTE_ID,
|
|
5171
|
+
"--with-recall",
|
|
5172
|
+
"--recall-limit",
|
|
5173
|
+
"3",
|
|
5174
|
+
"--max-bytes",
|
|
5175
|
+
str(EXPECTED_PACK_MAX_BYTES),
|
|
5176
|
+
"--json",
|
|
5177
|
+
)
|
|
5178
|
+
recall_pack_result = run_plain(recall_pack_cmd, cwd=cwd, env=relevance_env)
|
|
5179
|
+
try:
|
|
5180
|
+
recall_pack_payload = json.loads(recall_pack_result.stdout)
|
|
5181
|
+
except json.JSONDecodeError as error:
|
|
5182
|
+
raise SystemExit(f"{context}: failed to parse pack recall JSON") from error
|
|
5183
|
+
recall_plan = recall_pack_payload.get("plan")
|
|
5184
|
+
require_registry_smoke(
|
|
5185
|
+
isinstance(recall_plan, dict),
|
|
5186
|
+
context=f"{context} pack recall",
|
|
5187
|
+
cmd=recall_pack_cmd,
|
|
5188
|
+
message="pack recall plan missing",
|
|
5189
|
+
)
|
|
5190
|
+
assert_recall_context(recall_plan, context=f"{context} pack recall plan", cmd=recall_pack_cmd)
|
|
5191
|
+
|
|
4665
5192
|
eval_template_path = profile_path.with_name(f"{profile_path.stem}-eval-template.json")
|
|
4666
5193
|
eval_template_path.write_text("stale eval template output\n", encoding="utf-8")
|
|
4667
5194
|
eval_template_cmd = command_factory(
|
|
@@ -4728,6 +5255,166 @@ def assert_learning_relevance_smoke(
|
|
|
4728
5255
|
)
|
|
4729
5256
|
|
|
4730
5257
|
|
|
5258
|
+
def assert_index_roundtrip_smoke(
|
|
5259
|
+
command_factory,
|
|
5260
|
+
index_dir: Path,
|
|
5261
|
+
profile_path: Path,
|
|
5262
|
+
*,
|
|
5263
|
+
env: dict[str, str],
|
|
5264
|
+
cwd: Path | None = None,
|
|
5265
|
+
context: str,
|
|
5266
|
+
) -> None:
|
|
5267
|
+
"""`index --build` -> `index --status` -> `index --verify` round-trip against a
|
|
5268
|
+
temp index dir and learning profile, isolated from the ambient environment."""
|
|
5269
|
+
index_dir.mkdir(parents=True, exist_ok=True)
|
|
5270
|
+
write_learning_relevance_fixture(profile_path)
|
|
5271
|
+
index_env = env.copy()
|
|
5272
|
+
index_env["DESIGN_AI_INDEX_DIR"] = str(index_dir)
|
|
5273
|
+
index_env["DESIGN_AI_LEARNING_FILE"] = str(profile_path)
|
|
5274
|
+
|
|
5275
|
+
build_cmd = command_factory("index", "--build", "--json")
|
|
5276
|
+
build_result = run_plain(build_cmd, cwd=cwd, env=index_env)
|
|
5277
|
+
assert_index_build_json(
|
|
5278
|
+
build_result.stdout,
|
|
5279
|
+
index_dir=str(index_dir),
|
|
5280
|
+
context=f"{context} build",
|
|
5281
|
+
cmd=build_cmd,
|
|
5282
|
+
)
|
|
5283
|
+
|
|
5284
|
+
status_cmd = command_factory("index", "--status", "--json")
|
|
5285
|
+
status_result = run_plain(status_cmd, cwd=cwd, env=index_env)
|
|
5286
|
+
assert_index_status_json(
|
|
5287
|
+
status_result.stdout,
|
|
5288
|
+
index_dir=str(index_dir),
|
|
5289
|
+
context=f"{context} status",
|
|
5290
|
+
cmd=status_cmd,
|
|
5291
|
+
)
|
|
5292
|
+
|
|
5293
|
+
verify_cmd = command_factory("index", "--verify", "--json")
|
|
5294
|
+
verify_result = run_plain(verify_cmd, cwd=cwd, env=index_env)
|
|
5295
|
+
assert_index_verify_json(
|
|
5296
|
+
verify_result.stdout,
|
|
5297
|
+
index_dir=str(index_dir),
|
|
5298
|
+
context=f"{context} verify",
|
|
5299
|
+
cmd=verify_cmd,
|
|
5300
|
+
)
|
|
5301
|
+
|
|
5302
|
+
|
|
5303
|
+
def assert_ranked_search_determinism_smoke(
|
|
5304
|
+
command_factory,
|
|
5305
|
+
index_dir: Path,
|
|
5306
|
+
*,
|
|
5307
|
+
env: dict[str, str],
|
|
5308
|
+
cwd: Path | None = None,
|
|
5309
|
+
context: str,
|
|
5310
|
+
) -> None:
|
|
5311
|
+
"""Two identical `search --ranked --json` runs must produce byte-identical output."""
|
|
5312
|
+
index_dir.mkdir(parents=True, exist_ok=True)
|
|
5313
|
+
ranked_env = env.copy()
|
|
5314
|
+
ranked_env["DESIGN_AI_INDEX_DIR"] = str(index_dir)
|
|
5315
|
+
|
|
5316
|
+
ranked_cmd = command_factory(
|
|
5317
|
+
"search",
|
|
5318
|
+
EXPECTED_CORPUS_SEARCH_QUERY,
|
|
5319
|
+
"--dir",
|
|
5320
|
+
"knowledge",
|
|
5321
|
+
"--limit",
|
|
5322
|
+
str(EXPECTED_RANKED_SEARCH_LIMIT),
|
|
5323
|
+
"--ranked",
|
|
5324
|
+
"--json",
|
|
5325
|
+
)
|
|
5326
|
+
first_result = run_plain(ranked_cmd, cwd=cwd, env=ranked_env)
|
|
5327
|
+
second_result = run_plain(ranked_cmd, cwd=cwd, env=ranked_env)
|
|
5328
|
+
assert_ranked_search_json(
|
|
5329
|
+
first_result.stdout,
|
|
5330
|
+
context=f"{context} payload",
|
|
5331
|
+
cmd=ranked_cmd,
|
|
5332
|
+
expected_notice=EXPECTED_RANKED_SEARCH_NOT_BUILT_NOTICE,
|
|
5333
|
+
)
|
|
5334
|
+
assert_ranked_search_determinism(
|
|
5335
|
+
first_result.stdout,
|
|
5336
|
+
second_result.stdout,
|
|
5337
|
+
context=context,
|
|
5338
|
+
cmd=ranked_cmd,
|
|
5339
|
+
)
|
|
5340
|
+
|
|
5341
|
+
|
|
5342
|
+
def assert_embeddings_off_by_default_smoke(
|
|
5343
|
+
command_factory,
|
|
5344
|
+
index_dir: Path,
|
|
5345
|
+
profile_path: Path,
|
|
5346
|
+
*,
|
|
5347
|
+
env: dict[str, str],
|
|
5348
|
+
cwd: Path | None = None,
|
|
5349
|
+
context: str,
|
|
5350
|
+
) -> None:
|
|
5351
|
+
"""Phase B is opt-in: a plain `index --build` never touches embeddings, and
|
|
5352
|
+
`index --status` reports `embeddings: null` (the feature is simply unused)."""
|
|
5353
|
+
index_dir.mkdir(parents=True, exist_ok=True)
|
|
5354
|
+
write_learning_relevance_fixture(profile_path)
|
|
5355
|
+
index_env = env.copy()
|
|
5356
|
+
index_env["DESIGN_AI_INDEX_DIR"] = str(index_dir)
|
|
5357
|
+
index_env["DESIGN_AI_LEARNING_FILE"] = str(profile_path)
|
|
5358
|
+
# Point at a config file that is guaranteed not to exist, rather than relying on
|
|
5359
|
+
# the ambient environment lacking DESIGN_AI_CONFIG_FILE — a real ~/.design-ai/
|
|
5360
|
+
# config.json on the machine running this smoke must not leak into the test.
|
|
5361
|
+
index_env["DESIGN_AI_CONFIG_FILE"] = str(index_dir.parent / "no-config-here" / "config.json")
|
|
5362
|
+
|
|
5363
|
+
build_cmd = command_factory("index", "--build", "--json")
|
|
5364
|
+
build_result = run_plain(build_cmd, cwd=cwd, env=index_env)
|
|
5365
|
+
build_payload = json.loads(build_result.stdout)
|
|
5366
|
+
if "embeddings" in build_payload:
|
|
5367
|
+
raise SystemExit(f"index build JSON after {context} unexpectedly ran provider execution with plain --build")
|
|
5368
|
+
|
|
5369
|
+
status_cmd = command_factory("index", "--status", "--json")
|
|
5370
|
+
status_result = run_plain(status_cmd, cwd=cwd, env=index_env)
|
|
5371
|
+
assert_index_status_json(
|
|
5372
|
+
status_result.stdout,
|
|
5373
|
+
index_dir=str(index_dir),
|
|
5374
|
+
context=f"{context} status",
|
|
5375
|
+
cmd=status_cmd,
|
|
5376
|
+
expect_embeddings=False,
|
|
5377
|
+
)
|
|
5378
|
+
|
|
5379
|
+
|
|
5380
|
+
def assert_search_embeddings_no_provider_fallback_smoke(
|
|
5381
|
+
command_factory,
|
|
5382
|
+
*,
|
|
5383
|
+
env: dict[str, str],
|
|
5384
|
+
cwd: Path | None = None,
|
|
5385
|
+
context: str,
|
|
5386
|
+
config_file: Path,
|
|
5387
|
+
) -> None:
|
|
5388
|
+
"""`search --ranked --embeddings` with no provider configured (flag or config)
|
|
5389
|
+
degrades to the lexical backend with a notice and exit code 0."""
|
|
5390
|
+
fallback_env = env.copy()
|
|
5391
|
+
# Same isolation rationale as assert_embeddings_off_by_default_smoke: point at a
|
|
5392
|
+
# config path guaranteed not to exist rather than trusting the ambient environment.
|
|
5393
|
+
fallback_env["DESIGN_AI_CONFIG_FILE"] = str(config_file)
|
|
5394
|
+
|
|
5395
|
+
search_cmd = command_factory(
|
|
5396
|
+
"search",
|
|
5397
|
+
EXPECTED_CORPUS_SEARCH_QUERY,
|
|
5398
|
+
"--dir",
|
|
5399
|
+
"knowledge",
|
|
5400
|
+
"--limit",
|
|
5401
|
+
str(EXPECTED_RANKED_SEARCH_LIMIT),
|
|
5402
|
+
"--ranked",
|
|
5403
|
+
"--embeddings",
|
|
5404
|
+
"--json",
|
|
5405
|
+
)
|
|
5406
|
+
result = run_plain(search_cmd, cwd=cwd, env=fallback_env)
|
|
5407
|
+
assert_ranked_search_json(
|
|
5408
|
+
result.stdout,
|
|
5409
|
+
context=f"{context} payload",
|
|
5410
|
+
cmd=search_cmd,
|
|
5411
|
+
expected_backend="lexical",
|
|
5412
|
+
)
|
|
5413
|
+
payload = json.loads(result.stdout)
|
|
5414
|
+
if "embedding provider" not in payload.get("notice", "") and "no embedding provider configured" not in payload.get("notice", ""):
|
|
5415
|
+
raise SystemExit(f"search ranked JSON after {context} fallback notice does not mention the missing provider")
|
|
5416
|
+
|
|
5417
|
+
|
|
4731
5418
|
def wait_for_registry_package(
|
|
4732
5419
|
package_spec: str,
|
|
4733
5420
|
*,
|
|
@@ -4800,6 +5487,12 @@ def smoke_registry_package(package_spec: str, *, retries: int, delay: float) ->
|
|
|
4800
5487
|
env=env,
|
|
4801
5488
|
context="registry smoke npm exec version JSON",
|
|
4802
5489
|
)
|
|
5490
|
+
assert_design_ai_mcp_protocol_smoke(
|
|
5491
|
+
npm_exec_mcp_cmd(package_spec),
|
|
5492
|
+
cwd=npx_root,
|
|
5493
|
+
env=env,
|
|
5494
|
+
context="registry smoke npm exec design-ai-mcp protocol",
|
|
5495
|
+
)
|
|
4803
5496
|
assert_workspace_json_smoke(
|
|
4804
5497
|
npm_exec_cmd(package_spec, "workspace", "--json"),
|
|
4805
5498
|
cwd=npx_root,
|
|
@@ -5753,6 +6446,13 @@ def smoke_registry_package(package_spec: str, *, retries: int, delay: float) ->
|
|
|
5753
6446
|
env=env,
|
|
5754
6447
|
context="registry smoke npm exec learn stats",
|
|
5755
6448
|
)
|
|
6449
|
+
assert_learning_recall_smoke(
|
|
6450
|
+
lambda *args: npm_exec_cmd(package_spec, *args),
|
|
6451
|
+
npx_root / "registry-recall-learning.json",
|
|
6452
|
+
cwd=npx_root,
|
|
6453
|
+
env=env,
|
|
6454
|
+
context="registry smoke npm exec learn recall",
|
|
6455
|
+
)
|
|
5756
6456
|
assert_learning_audit_cleanup_smoke(
|
|
5757
6457
|
lambda *args: npm_exec_cmd(package_spec, *args),
|
|
5758
6458
|
npx_root / "registry-audit-learning.json",
|
|
@@ -5767,6 +6467,36 @@ def smoke_registry_package(package_spec: str, *, retries: int, delay: float) ->
|
|
|
5767
6467
|
env=env,
|
|
5768
6468
|
context="registry smoke npm exec learning relevance",
|
|
5769
6469
|
)
|
|
6470
|
+
assert_index_roundtrip_smoke(
|
|
6471
|
+
lambda *args: npm_exec_cmd(package_spec, *args),
|
|
6472
|
+
npx_root / "registry-index",
|
|
6473
|
+
npx_root / "registry-index-learning.json",
|
|
6474
|
+
cwd=npx_root,
|
|
6475
|
+
env=env,
|
|
6476
|
+
context="registry smoke npm exec index roundtrip",
|
|
6477
|
+
)
|
|
6478
|
+
assert_ranked_search_determinism_smoke(
|
|
6479
|
+
lambda *args: npm_exec_cmd(package_spec, *args),
|
|
6480
|
+
npx_root / "registry-ranked-search-index",
|
|
6481
|
+
cwd=npx_root,
|
|
6482
|
+
env=env,
|
|
6483
|
+
context="registry smoke npm exec ranked search determinism",
|
|
6484
|
+
)
|
|
6485
|
+
assert_embeddings_off_by_default_smoke(
|
|
6486
|
+
lambda *args: npm_exec_cmd(package_spec, *args),
|
|
6487
|
+
npx_root / "registry-embeddings-off-index",
|
|
6488
|
+
npx_root / "registry-embeddings-off-learning.json",
|
|
6489
|
+
cwd=npx_root,
|
|
6490
|
+
env=env,
|
|
6491
|
+
context="registry smoke npm exec embeddings off by default",
|
|
6492
|
+
)
|
|
6493
|
+
assert_search_embeddings_no_provider_fallback_smoke(
|
|
6494
|
+
lambda *args: npm_exec_cmd(package_spec, *args),
|
|
6495
|
+
cwd=npx_root,
|
|
6496
|
+
env=env,
|
|
6497
|
+
context="registry smoke npm exec search embeddings no provider fallback",
|
|
6498
|
+
config_file=npx_root / "registry-no-config-here" / "config.json",
|
|
6499
|
+
)
|
|
5770
6500
|
assert_update_dry_run_smoke(
|
|
5771
6501
|
npm_exec_cmd(package_spec, "update", "--dry-run"),
|
|
5772
6502
|
cwd=npx_root,
|
|
@@ -6192,6 +6922,41 @@ def run_self_test() -> None:
|
|
|
6192
6922
|
expected="without external calls",
|
|
6193
6923
|
scope="registry smoke",
|
|
6194
6924
|
)
|
|
6925
|
+
mcp_protocol_cmd = ["design-ai-mcp"]
|
|
6926
|
+
assert_design_ai_mcp_protocol_responses(
|
|
6927
|
+
[
|
|
6928
|
+
{"jsonrpc": "2.0", "id": 1, "result": {"serverInfo": {"name": "design-ai"}}},
|
|
6929
|
+
{"jsonrpc": "2.0", "id": 2, "result": {"tools": [{"name": "design_ai_route"}, {"name": "design_ai_search"}]}},
|
|
6930
|
+
{
|
|
6931
|
+
"jsonrpc": "2.0",
|
|
6932
|
+
"id": 3,
|
|
6933
|
+
"error": {"code": -32602, "message": "design_ai_search.limit must be an integer"},
|
|
6934
|
+
},
|
|
6935
|
+
],
|
|
6936
|
+
context="registry smoke self-test design-ai MCP protocol",
|
|
6937
|
+
cmd=mcp_protocol_cmd,
|
|
6938
|
+
)
|
|
6939
|
+
expect_self_test_failure(
|
|
6940
|
+
lambda: assert_design_ai_mcp_protocol_responses(
|
|
6941
|
+
[
|
|
6942
|
+
{"jsonrpc": "2.0", "id": 1, "result": {"serverInfo": {"name": "design-ai"}}},
|
|
6943
|
+
{
|
|
6944
|
+
"jsonrpc": "2.0",
|
|
6945
|
+
"id": 2,
|
|
6946
|
+
"result": {"tools": [{"name": "design_ai_route"}, {"name": "design_ai_search"}]},
|
|
6947
|
+
},
|
|
6948
|
+
{
|
|
6949
|
+
"jsonrpc": "2.0",
|
|
6950
|
+
"id": 3,
|
|
6951
|
+
"error": {"code": -32602, "message": "wrong validation message"},
|
|
6952
|
+
},
|
|
6953
|
+
],
|
|
6954
|
+
context="registry smoke self-test design-ai MCP protocol",
|
|
6955
|
+
cmd=mcp_protocol_cmd,
|
|
6956
|
+
),
|
|
6957
|
+
expected="MCP invalid argument response did not preserve invalid params validation",
|
|
6958
|
+
scope="registry smoke",
|
|
6959
|
+
)
|
|
6195
6960
|
assert_site_mcp_plan_markdown(
|
|
6196
6961
|
passing_site_mcp_plan_markdown(),
|
|
6197
6962
|
context="registry smoke self-test site mcp-plan",
|
|
@@ -7929,6 +8694,38 @@ def run_self_test() -> None:
|
|
|
7929
8694
|
scope="registry smoke",
|
|
7930
8695
|
)
|
|
7931
8696
|
|
|
8697
|
+
recall_payload = {
|
|
8698
|
+
"recall": {
|
|
8699
|
+
"query": EXPECTED_ROUTE_BRIEF,
|
|
8700
|
+
"mode": "lexical",
|
|
8701
|
+
"candidateCount": 42,
|
|
8702
|
+
"selectedCount": 1,
|
|
8703
|
+
"selected": [
|
|
8704
|
+
{
|
|
8705
|
+
"id": "knowledge/components/INDEX.md",
|
|
8706
|
+
"score": 9.5,
|
|
8707
|
+
"matchedTokens": ["button", "component"],
|
|
8708
|
+
},
|
|
8709
|
+
],
|
|
8710
|
+
"markdown": (
|
|
8711
|
+
"## Recalled design knowledge\n\n"
|
|
8712
|
+
"- knowledge/components/INDEX.md\n"
|
|
8713
|
+
" - Component index"
|
|
8714
|
+
),
|
|
8715
|
+
},
|
|
8716
|
+
}
|
|
8717
|
+
recall_cmd = ["design-ai", "prompt", EXPECTED_ROUTE_BRIEF, "--with-recall", "--json"]
|
|
8718
|
+
assert_recall_context(recall_payload, context="registry smoke self-test", cmd=recall_cmd)
|
|
8719
|
+
expect_self_test_failure(
|
|
8720
|
+
lambda: assert_recall_context(
|
|
8721
|
+
{**recall_payload, "recall": {**recall_payload["recall"], "selectedCount": 0, "selected": []}},
|
|
8722
|
+
context="registry smoke self-test",
|
|
8723
|
+
cmd=recall_cmd,
|
|
8724
|
+
),
|
|
8725
|
+
expected="recall should select at least one corpus file",
|
|
8726
|
+
scope="registry smoke",
|
|
8727
|
+
)
|
|
8728
|
+
|
|
7932
8729
|
learning_audit_path = tmp_root / "learning-audit.json"
|
|
7933
8730
|
duplicate_command_args = [
|
|
7934
8731
|
"design-ai",
|
|
@@ -8135,6 +8932,111 @@ def run_self_test() -> None:
|
|
|
8135
8932
|
scope="registry smoke",
|
|
8136
8933
|
)
|
|
8137
8934
|
|
|
8935
|
+
index_dir = str(tmp_root / "registry-index-self-test")
|
|
8936
|
+
index_build_cmd = ["design-ai", "index", "--build", "--json"]
|
|
8937
|
+
assert_index_build_json(
|
|
8938
|
+
passing_index_build_json(index_dir),
|
|
8939
|
+
index_dir=index_dir,
|
|
8940
|
+
context="registry smoke self-test",
|
|
8941
|
+
cmd=index_build_cmd,
|
|
8942
|
+
)
|
|
8943
|
+
expect_self_test_failure(
|
|
8944
|
+
lambda: assert_index_build_json(
|
|
8945
|
+
"{",
|
|
8946
|
+
index_dir=index_dir,
|
|
8947
|
+
context="registry smoke self-test",
|
|
8948
|
+
cmd=index_build_cmd,
|
|
8949
|
+
),
|
|
8950
|
+
expected="failed to parse index build JSON",
|
|
8951
|
+
scope="registry smoke",
|
|
8952
|
+
)
|
|
8953
|
+
|
|
8954
|
+
index_status_cmd = ["design-ai", "index", "--status", "--json"]
|
|
8955
|
+
assert_index_status_json(
|
|
8956
|
+
passing_index_status_json(index_dir),
|
|
8957
|
+
index_dir=index_dir,
|
|
8958
|
+
context="registry smoke self-test",
|
|
8959
|
+
cmd=index_status_cmd,
|
|
8960
|
+
)
|
|
8961
|
+
assert_index_status_json(
|
|
8962
|
+
passing_index_status_json(index_dir, with_embeddings=True),
|
|
8963
|
+
index_dir=index_dir,
|
|
8964
|
+
context="registry smoke self-test embeddings",
|
|
8965
|
+
cmd=index_status_cmd,
|
|
8966
|
+
expect_embeddings=True,
|
|
8967
|
+
)
|
|
8968
|
+
expect_self_test_failure(
|
|
8969
|
+
lambda: assert_index_status_json(
|
|
8970
|
+
passing_index_status_json(index_dir, with_embeddings=True),
|
|
8971
|
+
index_dir=index_dir,
|
|
8972
|
+
context="registry smoke self-test embeddings unexpected",
|
|
8973
|
+
cmd=index_status_cmd,
|
|
8974
|
+
expect_embeddings=False,
|
|
8975
|
+
),
|
|
8976
|
+
expected="embeddings is not null when embeddings were not built",
|
|
8977
|
+
scope="registry smoke",
|
|
8978
|
+
)
|
|
8979
|
+
|
|
8980
|
+
index_verify_cmd = ["design-ai", "index", "--verify", "--json"]
|
|
8981
|
+
assert_index_verify_json(
|
|
8982
|
+
passing_index_verify_json(index_dir),
|
|
8983
|
+
index_dir=index_dir,
|
|
8984
|
+
context="registry smoke self-test",
|
|
8985
|
+
cmd=index_verify_cmd,
|
|
8986
|
+
)
|
|
8987
|
+
expect_self_test_failure(
|
|
8988
|
+
lambda: assert_index_verify_json(
|
|
8989
|
+
json.dumps({**json.loads(passing_index_verify_json(index_dir)), "ok": False}),
|
|
8990
|
+
index_dir=index_dir,
|
|
8991
|
+
context="registry smoke self-test",
|
|
8992
|
+
cmd=index_verify_cmd,
|
|
8993
|
+
),
|
|
8994
|
+
expected="ok is not true",
|
|
8995
|
+
scope="registry smoke",
|
|
8996
|
+
)
|
|
8997
|
+
|
|
8998
|
+
ranked_search_cmd = ["design-ai", "search", EXPECTED_CORPUS_SEARCH_QUERY, "--ranked", "--json"]
|
|
8999
|
+
ranked_search_json = passing_ranked_search_json()
|
|
9000
|
+
assert_ranked_search_json(
|
|
9001
|
+
ranked_search_json,
|
|
9002
|
+
context="registry smoke self-test",
|
|
9003
|
+
cmd=ranked_search_cmd,
|
|
9004
|
+
)
|
|
9005
|
+
assert_ranked_search_determinism(
|
|
9006
|
+
ranked_search_json,
|
|
9007
|
+
ranked_search_json,
|
|
9008
|
+
context="registry smoke self-test",
|
|
9009
|
+
cmd=ranked_search_cmd,
|
|
9010
|
+
)
|
|
9011
|
+
expect_self_test_failure(
|
|
9012
|
+
lambda: assert_ranked_search_determinism(
|
|
9013
|
+
ranked_search_json,
|
|
9014
|
+
passing_ranked_search_json(backend="lexical"),
|
|
9015
|
+
context="registry smoke self-test",
|
|
9016
|
+
cmd=ranked_search_cmd,
|
|
9017
|
+
),
|
|
9018
|
+
expected="differs between identical runs",
|
|
9019
|
+
scope="registry smoke",
|
|
9020
|
+
)
|
|
9021
|
+
|
|
9022
|
+
ranked_search_fallback_json = passing_ranked_search_json(backend="lexical")
|
|
9023
|
+
assert_ranked_search_json(
|
|
9024
|
+
ranked_search_fallback_json,
|
|
9025
|
+
context="registry smoke self-test fallback",
|
|
9026
|
+
cmd=ranked_search_cmd,
|
|
9027
|
+
expected_backend="lexical",
|
|
9028
|
+
)
|
|
9029
|
+
expect_self_test_failure(
|
|
9030
|
+
lambda: assert_ranked_search_json(
|
|
9031
|
+
passing_ranked_search_json(backend="embeddings"),
|
|
9032
|
+
context="registry smoke self-test fallback mismatch",
|
|
9033
|
+
cmd=ranked_search_cmd,
|
|
9034
|
+
expected_backend="lexical",
|
|
9035
|
+
),
|
|
9036
|
+
expected="backend differs from expected backend",
|
|
9037
|
+
scope="registry smoke",
|
|
9038
|
+
)
|
|
9039
|
+
|
|
8138
9040
|
expect_self_test_failure(
|
|
8139
9041
|
lambda: smoke_registry_package("self-test-package", retries=0, delay=0),
|
|
8140
9042
|
expected="--retries must be at least 1",
|