mustflow 2.112.13 → 2.112.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/templates/default/i18n.toml +5 -5
- package/templates/default/locales/en/.mustflow/skills/INDEX.md +16 -13
- package/templates/default/locales/en/.mustflow/skills/llm-token-cost-control-review/SKILL.md +62 -24
- package/templates/default/locales/en/.mustflow/skills/rag-pipeline-triage/SKILL.md +77 -23
- package/templates/default/locales/en/.mustflow/skills/search-index-integrity-review/SKILL.md +64 -14
- package/templates/default/locales/en/.mustflow/skills/vector-search-integrity-review/SKILL.md +59 -32
- package/templates/default/manifest.toml +1 -1
package/package.json
CHANGED
|
@@ -62,7 +62,7 @@ translations = {}
|
|
|
62
62
|
[documents."skills.index"]
|
|
63
63
|
source = "locales/en/.mustflow/skills/INDEX.md"
|
|
64
64
|
source_locale = "en"
|
|
65
|
-
revision =
|
|
65
|
+
revision = 226
|
|
66
66
|
translations = {}
|
|
67
67
|
|
|
68
68
|
[documents."skill.adapter-boundary"]
|
|
@@ -565,19 +565,19 @@ translations = {}
|
|
|
565
565
|
[documents."skill.search-index-integrity-review"]
|
|
566
566
|
source = "locales/en/.mustflow/skills/search-index-integrity-review/SKILL.md"
|
|
567
567
|
source_locale = "en"
|
|
568
|
-
revision =
|
|
568
|
+
revision = 3
|
|
569
569
|
translations = {}
|
|
570
570
|
|
|
571
571
|
[documents."skill.vector-search-integrity-review"]
|
|
572
572
|
source = "locales/en/.mustflow/skills/vector-search-integrity-review/SKILL.md"
|
|
573
573
|
source_locale = "en"
|
|
574
|
-
revision =
|
|
574
|
+
revision = 4
|
|
575
575
|
translations = {}
|
|
576
576
|
|
|
577
577
|
[documents."skill.rag-pipeline-triage"]
|
|
578
578
|
source = "locales/en/.mustflow/skills/rag-pipeline-triage/SKILL.md"
|
|
579
579
|
source_locale = "en"
|
|
580
|
-
revision =
|
|
580
|
+
revision = 3
|
|
581
581
|
translations = {}
|
|
582
582
|
|
|
583
583
|
[documents."skill.dependency-injection"]
|
|
@@ -1272,7 +1272,7 @@ translations = {}
|
|
|
1272
1272
|
[documents."skill.llm-token-cost-control-review"]
|
|
1273
1273
|
source = "locales/en/.mustflow/skills/llm-token-cost-control-review/SKILL.md"
|
|
1274
1274
|
source_locale = "en"
|
|
1275
|
-
revision =
|
|
1275
|
+
revision = 4
|
|
1276
1276
|
translations = {}
|
|
1277
1277
|
|
|
1278
1278
|
[documents."skill.llm-response-latency-review"]
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
mustflow_doc: skills.index
|
|
3
3
|
locale: en
|
|
4
4
|
canonical: true
|
|
5
|
-
revision:
|
|
5
|
+
revision: 226
|
|
6
6
|
authority: router
|
|
7
7
|
lifecycle: mustflow-owned
|
|
8
8
|
---
|
|
@@ -76,7 +76,8 @@ refer to `AGENTS.md` and `.mustflow/config/commands.toml` to implement the most
|
|
|
76
76
|
citations, claim maps, evidence IDs, answerability states, retrieval thresholds, validators,
|
|
77
77
|
abstain behavior, or hallucination metrics need product-controlled grounding review.
|
|
78
78
|
- Use `llm-token-cost-control-review` as a primary route when LLM request assembly, token budgets,
|
|
79
|
-
prompt caching, chat history, RAG context size,
|
|
79
|
+
prompt caching, chat history, RAG context size, prompt packing, document metadata, source maps,
|
|
80
|
+
question-scoped compression, evidence cards, tool or schema payloads, model routing, reasoning
|
|
80
81
|
budget, retries, Batch, Flex, predicted outputs, or cost-per-success telemetry need token spend
|
|
81
82
|
review.
|
|
82
83
|
- Use `llm-response-latency-review` as a primary route when LLM response speed, time to first
|
|
@@ -424,16 +425,18 @@ refer to `AGENTS.md` and `.mustflow/config/commands.toml` to implement the most
|
|
|
424
425
|
index-defeating predicates, plan skew, or long transaction scope before live plan evidence.
|
|
425
426
|
- Use `search-index-integrity-review` as a primary route when keyword search, full-text search,
|
|
426
427
|
Elasticsearch, OpenSearch, Lucene-style indexing, aliases, bulk ingestion, refresh visibility,
|
|
427
|
-
analyzer,
|
|
428
|
-
|
|
428
|
+
analyzer, metadata taxonomy, negative metadata, exact keyword fields, source maps, synonym,
|
|
429
|
+
autocomplete, pagination, shard failure, search quality, or search performance needs
|
|
430
|
+
source-to-search and query-contract evidence.
|
|
429
431
|
- Use `vector-search-integrity-review` as a primary route when vector search, semantic search, RAG
|
|
430
|
-
retrieval, embeddings,
|
|
431
|
-
namespaces, tenants, hybrid search, reranking,
|
|
432
|
-
retrieval-contract review.
|
|
432
|
+
retrieval, embeddings, contextual headers, synthetic questions, chunk text variants, ANN indexes,
|
|
433
|
+
exact-versus-approximate search, filters, metadata, namespaces, tenants, hybrid search, reranking,
|
|
434
|
+
recall, latency, or golden-set behavior needs retrieval-contract review.
|
|
433
435
|
- Use `rag-pipeline-triage` as a primary route when a RAG, knowledge-base answer, grounded chat,
|
|
434
436
|
citation answer, document QA, or support bot failure is not yet localized to ingestion, parsing,
|
|
435
|
-
|
|
436
|
-
|
|
437
|
+
document metadata, frontmatter schema, source maps, heading paths, chunking, retrieval, filtering,
|
|
438
|
+
reranking, context assembly, prompt construction, generation, citation validation, answerability,
|
|
439
|
+
access control, latency, or cost.
|
|
437
440
|
- Use `database-json-modeling-review` as an adjunct when database review needs to decide whether
|
|
438
441
|
JSON, jsonb, metadata, settings, raw payload, or dynamic keys should become typed columns,
|
|
439
442
|
child tables, generated/computed columns, JSON indexes, schema versions, or key registries
|
|
@@ -533,8 +536,8 @@ routes. Event routes stay inactive until their event occurs.
|
|
|
533
536
|
| AI product features, AI Gateways, provider/model integrations, prompt/RAG/tool paths, AI caches, fallback paths, streaming paths, evaluation gates, user-data flows, model registries, AI observability, incident runbooks, or model portability plans are created, changed, reviewed, or reported before the risk is narrow enough for one specialist LLM skill | `.mustflow/skills/ai-product-readiness-review/SKILL.md` | Product role ledger, harm model, gateway ledger, authority and action ledger, data ledger, cost and cache ledger, eval ledger, fallback and streaming ledger, model portability ledger, observability ledger, changed files, and command contract entries | AI Gateway boundaries, provider adapters, model registries, prompt registries, policy engines, tool proposal gates, quota guards, token budgets, cache keys, redaction, retry and fallback states, streaming validators, eval fixtures, observability fields, incident runbooks, tests, docs, route metadata, and directly synchronized templates | model-as-product shortcut, automatic high-risk decision, direct client provider call, prompt-only security, RAG permission leak, unvalidated model output, missing policy engine, cost budget gap, app cache tenant leak, eval theater, fallback-as-second-model, partial streaming disclosure, raw AI log retention, scattered model names, stale provider assumption, or missing kill switch | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `prompt_cache_audit`, `docs_validate_fast`, `test_release`, `mustflow_check` | AI product-readiness surface reviewed, AI role and harm model, gateway and policy boundaries, prompt-injection/data/output/permission/side-effect controls, cost/cache/eval/fallback/streaming/observability/model-portability decisions, specialist skills applied or deferred, verification, and remaining AI product-readiness risk |
|
|
534
537
|
| Prompts, prompt builders, system or developer messages, RAG prompt assembly, few-shot examples, structured outputs, tool-use instructions, model selection, reasoning-effort settings, eval sets, refusal or fallback handling, prompt versioning, or AI feature completion criteria are created, changed, reviewed, or reported | `.mustflow/skills/prompt-contract-quality-review/SKILL.md` | Prompt contract ledger, input ledger, authority ledger, output schema, tool policy, model/runtime ledger, RAG evidence ledger, eval ledger, changed files, and command contract entries | Prompt builders, prompt templates, schemas, validators, eval fixtures, boundary examples, tool policies, fallback states, completion definitions, docs, tests, and directly synchronized templates | prompt-as-function gap, user input treated as authority, buried RAG evidence, happy-path-only examples, JSON-parse theater, unpinned production model, hidden prompt storage, raw chain-of-thought request, guessed tool parameters, missing failure state, unbounded reasoning/token cost, or vibe-based prompt improvement claim | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | Prompt contract reviewed, function boundary, authority and source separation, eval and semantic validation status, model/runtime policy, RAG/tool/failure/completion states, verification, and remaining prompt-contract risk |
|
|
535
538
|
| LLM answers, RAG responses, citations, source grounding, claim extraction, evidence IDs, answerability states, abstain behavior, retrieval thresholds, tool-backed facts, output validators, LLM judges, or hallucination-control metrics are created, changed, reviewed, or reported | `.mustflow/skills/llm-hallucination-control-review/SKILL.md` | Answer contract ledger, evidence ledger, claim ledger, tool ledger, validator ledger, eval ledger, observability ledger, changed files, and command contract entries | Answerability states, abstain states, missing-information states, source-coverage gates, claim maps, evidence-ID requirements, citation validators, retrieval thresholds, chunk metadata, tool-parameter ownership, deterministic calculators, domain validators, eval fixtures, tests, docs, route metadata, and directly synchronized templates | unsupported factual claim, fabricated citation, source ID invention, weak retrieval gate, noisy semantic-only retrieval, chunk context loss, summary-on-summary hallucination, guessed tool parameter, model arithmetic, source-priority conflict, LLM judge overtrust, low-temperature theater, missing abstain path, missing dirty eval, false citation metric gap, or unobservable grounding drift | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | Hallucination-control surface reviewed, answerability and abstain states, evidence IDs, claim map, citations, source coverage, validators, retrieval thresholds, tool ownership, evals, metrics, verification, and remaining hallucination-control risk |
|
|
536
|
-
| RAG, knowledge-base answer, grounded chat, citation answer, retrieval-augmented support bot, or document QA flow is wrong, stale, unsupported, slow, leaking data, over-refusing, or not yet localized to ingestion, parsing, chunking, retrieval, filtering, reranking, context assembly, prompt construction, generation, citation validation, or answerability boundaries | `.mustflow/skills/rag-pipeline-triage/SKILL.md` | Symptom classification, trace ledger, source ledger, comparison ledger, eval ledger, privacy ledger, changed files, and command contract entries | End-to-end trace preservation, source availability and parsed-text checks, chunk
|
|
537
|
-
| LLM API calls, prompt assembly, chat history, RAG context, tool schemas, structured output schemas, model routing, reasoning settings, token budgets, provider prompt caching, app-level response caching, retries, batch or flex processing, predicted outputs, image or file inputs, or LLM cost metrics are created, changed, reviewed, or reported | `.mustflow/skills/llm-token-cost-control-review/SKILL.md` | Cost surface ledger, request ledger, cache ledger, context ledger, output ledger, routing ledger, observability ledger, changed files, and command contract entries | Request builders, prompt prefix ordering, canonical serialization, prompt hashes, cache keys, token counters, budget guards, model routers, context trimming, RAG packing, tool and schema payloads, output patch formats, retry repair paths, metrics, logs, tests, docs, route metadata, and directly synchronized templates | prompt-cache prefix drift, volatile field before stable prefix, unmeasured token count, full transcript replay, RAG chunk bloat, oversized tool or JSON schema payload, expensive model default, unbounded reasoning, no visible output after token cap, full-output regeneration, full-context retry replay, app cache key leak, predicted-output cost confusion, image or file token surprise, or per-call cost hiding cost-per-success regression | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `prompt_cache_audit`, `docs_validate_fast`, `test_release`, `mustflow_check` | LLM token-cost surface reviewed, cost unit and measurement source, stable prefix and cache behavior, app cache/history/RAG/tool/schema/input choices, routing/reasoning/output/retry/Batch/Flex/prediction choices, observability, verification, and remaining token-cost risk |
|
|
539
|
+
| RAG, knowledge-base answer, grounded chat, citation answer, retrieval-augmented support bot, or document QA flow is wrong, stale, unsupported, slow, leaking data, over-refusing, or not yet localized to ingestion, parsing, document metadata, frontmatter schema, source maps, heading paths, chunking, retrieval, filtering, reranking, context assembly, prompt construction, generation, citation validation, or answerability boundaries | `.mustflow/skills/rag-pipeline-triage/SKILL.md` | Symptom classification, trace ledger, source ledger with original/index/prompt text, stable source/doc/chunk ids, document type/status/authority, routing summaries, document graph links, comparison ledger, eval ledger, privacy ledger, changed files, and command contract entries | End-to-end trace preservation, source availability and parsed-text checks, metadata, source maps, ACL prefilters, heading paths, semantic code chunks, table record shape, chunk graph, source-of-truth and supersession checks, duplicate and stale source checks, no-retrieval/current-context/gold-context comparison, keyword/vector/hybrid/retriever/reranker/context/prompt/generation/citation/answerability localization, safe synthetic fixtures, metrics, docs, and directly synchronized templates | model scapegoating, tuning top-k before source proof, original-document theater while parsed text is broken, generic heading coordinates, generated summary cited as source, missing source map, source/doc/chunk id collapse, stale source mixing, draft or meeting-note evidence outranking canonical documents, ACL inherited after retrieval, filter blamed as vector failure, fixed line-count code slicing, table number without row or column meaning, reranker candidate starvation, critical evidence truncated or buried, retrieved text treated as instruction, citation decoration, answerability missing state, private corpus dump, access filter bypass, or single satisfaction score hiding layer failure | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | RAG pipeline triaged, localized boundary, trace/source/comparison/eval/metric/privacy ledgers, metadata/frontmatter/source-map/ACL/chunk-graph/document-graph findings, fix or recommendation, evidence level, verification, and remaining RAG pipeline risk |
|
|
540
|
+
| LLM API calls, prompt assembly, chat history, RAG context, document metadata, chunk summaries, prompt packing, question-scoped compression, evidence cards, tool schemas, structured output schemas, model routing, reasoning settings, token budgets, provider prompt caching, app-level response caching, retries, batch or flex processing, predicted outputs, image or file inputs, or LLM cost metrics are created, changed, reviewed, or reported | `.mustflow/skills/llm-token-cost-control-review/SKILL.md` | Cost surface ledger, request ledger with authority lanes, cache ledger, context ledger with inclusion tests, block role tags, filter-only versus LLM-visible metadata, source maps, state cards, and evidence cards, output ledger, routing ledger, observability ledger, changed files, and command contract entries | Request builders, prompt prefix ordering, canonical serialization, prompt hashes, cache keys, token counters, budget guards, model routers, context trimming, RAG packing, question-scoped compression, slot records, state snapshots, original/index/prompt text separation, source-map references, tool and schema payloads, output patch formats, retry repair paths, metrics, logs, tests, docs, route metadata, and directly synchronized templates | prompt-cache prefix drift, volatile field before stable prefix, unmeasured token count, full transcript replay, state delta without baseline, answer-irrelevant context hoarding, RAG chunk bloat, generated summary treated as original evidence, search-only metadata treated as answer text, missing source coordinate, missing token_count, compression not evaluated, oversized tool or JSON schema payload, expensive model default, unbounded reasoning, no visible output after token cap, full-output regeneration, full-context retry replay, app cache key leak, predicted-output cost confusion, image or file token surprise, or per-call cost hiding cost-per-success regression | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `prompt_cache_audit`, `docs_validate_fast`, `test_release`, `mustflow_check` | LLM token-cost surface reviewed, cost unit and measurement source, stable prefix and cache behavior, authority lanes, app cache/history/RAG/metadata/source-map/prompt-packing/evidence-card/tool/schema/input choices, routing/reasoning/output/retry/Batch/Flex/prediction choices, observability, verification, and remaining token-cost risk |
|
|
538
541
|
| LLM response latency, time to first token, first useful output, streaming, output length, LLM round trips, tool-call wait, prompt-cache latency, model routing, speculative or parallel execution, realtime continuation, priority tiers, predicted outputs, or user-perceived AI speed are created, changed, reviewed, or reported | `.mustflow/skills/llm-response-latency-review/SKILL.md` | Latency target ledger, request timeline ledger, call graph ledger, output ledger, cache ledger, routing ledger, observability ledger, changed files, and command contract entries | Streaming paths, first-useful-output contracts, request timeline metrics, call graph simplification, parallel or speculative work, model routers, fallback cascades, output caps, schema shortening, prompt-cache prefix ordering, cache keys, realtime continuation, priority-tier routing, timeout and cancellation behavior, tests, docs, route metadata, and directly synchronized templates | slow first token, useless streamed preamble, extra sequential LLM round trip, tool wait blocking first output, cache-prefix drift, unmeasured cache miss, verbose output drift, long JSON key or enum overhead, RAG chunk bloat, router escalation loop, prediction-token mismatch, priority-tier misuse, unsafe speculative work, missing cancellation, or raw prompt telemetry leak | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | LLM response-latency surface reviewed, latency unit and request timeline, round trips, parallel/tool/stream/cancel behavior, output/schema/cache/history/RAG/routing/fallback/prediction/realtime/priority choices, observability, verification, and remaining response-latency risk |
|
|
539
542
|
| Autonomous or semi-autonomous LLM agents, agentic workflows, planners, executors, verifiers, tool contracts, tool-call gates, human approval or interrupt flows, durable agent state, handoffs, guardrails, loop budgets, retry policies, trace evaluation, or agent outcome metrics are created, changed, reviewed, or reported | `.mustflow/skills/agent-execution-control-review/SKILL.md` | Autonomy ledger, stage gate ledger, role separation ledger, tool contract ledger, effect ledger, state and resume ledger, memory and context ledger, handoff and guardrail ledger, loop, retry, and budget ledger, trace and eval outcome ledger, changed files, and command contract entries | Workflow-versus-agent routing, stage gates, planner/executor/verifier boundaries, tool contracts, tool argument ownership, draft/execute separation, idempotency keys, approval records, durable checkpoints, state schema versions, memory partitions, handoff filters, guardrails, loop budgets, retry classification, trace spans, eval fixtures, tests, docs, route metadata, and directly synchronized templates | unnecessary autonomous agent, one model self-certifying success, ungated bad plan, ambiguous tool contract, guessed tool argument, relative-path trap, external effect before approval, missing idempotency key, interrupt replay side effect, stale state schema, over-shared handoff, misplaced guardrail, repeated tool loop, blind retry, final-answer-only eval, or unsafe trace data | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | Agent execution-control surface reviewed, workflow-versus-agent decision, autonomy envelope, stage gates, role separation, tool contracts, approval and side-effect replay safety, state/resume/memory/handoff/guardrail/loop/retry/trace/eval checks, verification, and remaining agent execution-control risk |
|
|
540
543
|
| Browser automation, UI automation, Playwright, Selenium, Puppeteer, WebDriver, computer-use or browser-driving agents, visual browser verification, flaky selectors, page readiness, authentication state, CAPTCHA or anti-bot handling, rate limits, screenshot checks, retry, timeout, human approval, or browser automation observability is created, changed, reviewed, triaged, or reported | `.mustflow/skills/browser-automation-reliability-review/SKILL.md` | Automation intent ledger, state ledger, readiness ledger, selector and action ledger, auth and identity ledger, external pressure ledger, verification ledger, agent and approval ledger, changed files, and command contract entries | Browser automation state machines, locator contracts, test IDs, accessible names, readiness assertions, frame and popup handlers, input verification, auth fixtures, per-worker account isolation, retry classification, timeout hierarchy, idempotency checks, rate-limit handling, approval gates, manual fallback states, traces, screenshots, redaction, cleanup, tests, docs, route metadata, and directly synchronized templates | sleep-as-readiness, `networkidle` faith, flaky selector string patch, hidden duplicate DOM, skeleton-as-content, stale element handle, force-click default, shared mutable account, CAPTCHA bypass, anti-bot evasion, retry storm, non-idempotent replay, timeout layer mismatch, screenshot-as-business-proof, unredacted trace data, page prompt injection, stale approval resume, coordinate click drift, or unverified browser success claim | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | Browser automation surface reviewed, browser-versus-API boundary, automation owner, state/readiness/locator/actionability/auth/rate-limit/retry/timeout/idempotency decisions, screenshot and business-success evidence, agent page-content trust, approval and resume checks, verification, and remaining browser automation reliability risk |
|
|
@@ -660,8 +663,8 @@ routes. Event routes stay inactive until their event occurs.
|
|
|
660
663
|
| PostgreSQL-specific schema, query, transaction, migration, indexing, extension, role, row-level security, connection pooling, replication, backup, restore, managed Postgres, or Postgres runtime behavior is created, changed, reviewed, or reported | `.mustflow/skills/postgresql-code-change/SKILL.md` | PostgreSQL role, version, provider, extension inventory, topology, pooler, schema/type rules, query-plan evidence, transaction/retry rules, migration and recovery needs, changed files, and command contract entries | PostgreSQL schema, queries, migrations, generated SQL, connection setup, pool settings, roles, RLS policies, extensions, tests, docs, and directly synchronized templates | version drift, provider constraint miss, connection storm, lock or rewrite surprise, unsafe online DDL claim, bad pooler assumption, RLS bypass, search-path risk, extension drift, stale replica read, query-plan overclaim, or unverified restore | `changes_status`, `changes_diff_summary`, `test_related`, `test`, `lint`, `build`, `docs_validate_fast`, `test_release`, `mustflow_check` | PostgreSQL version/topology, pooling, lock/transaction, schema/type/RLS/role, query/index/statistics, backup/restore, verification, and remaining PostgreSQL risk |
|
|
661
664
|
| ClickHouse-specific schema, MergeTree engine configuration, partition or sorting keys, primary keys, projections, materialized views, dictionaries, ingest, async inserts, deduplication, mutations, joins, CTEs, aggregate states, arrays, maps, window functions, distributed queries, or query performance behavior is created, changed, reviewed, or reported | `.mustflow/skills/clickhouse-code-change/SKILL.md` | ClickHouse role, version or Cloud track, topology, engine, table shape, ingest shape, query shape, operational evidence, changed files, and command contract entries | ClickHouse DDL, SQL, query builders, ingest code, backfill code, materialized views, projections, dictionaries, settings, fixtures, tests, docs, and synchronized templates | OLTP-shaped table design, high-cardinality partition part explosion, primary-key uniqueness myth, bad sorting locality, tiny insert parts, async insert durability overclaim, block dedup retry drift, MV trigger misunderstanding, stale dictionary lookup, aggregate-state merge bug, `arrayJoin` row explosion, default window-frame bug, `FINAL` cost patch, `OPTIMIZE FINAL` routine, mutation write amplification, JOIN fan-out, CTE rerun surprise, projection backfill miss, skip-index cargo cult, or unverified query-plan claim | `changes_status`, `changes_diff_summary`, `test_related`, `test`, `lint`, `build`, `docs_validate_fast`, `test_release`, `mustflow_check` | ClickHouse role/version/topology, engine/storage, ingest/dedup/MV/backfill, aggregate/query/JOIN/distributed findings, evidence level, verification, and remaining ClickHouse risk |
|
|
662
665
|
| DuckDB-specific embedded OLAP database use, `.duckdb` file ownership, concurrency, language bindings, Appender usage, CSV/Parquet/JSON ingestion, query determinism, timestamp behavior, memory and temp spill settings, profiling, indexes, CTEs, macros, or DuckDB runtime behavior is created, changed, reviewed, or reported | `.mustflow/skills/duckdb-code-change/SKILL.md` | DuckDB role, version or track, binding, extension inventory, file and process ownership, ingest/export shape, query shape, memory and temp spill settings, profiling evidence, changed files, and command contract entries | DuckDB SQL, schemas, query builders, connection setup, binding-specific code, ingest/export code, Appender code, settings, fixtures, tests, docs, and synchronized templates | SQLite-like OLTP assumption, native file multi-process write bug, hidden global connection, thread/process confusion, Appender visibility overclaim, CSV sampling loss, `ignore_errors` data loss, schema-drift memory spike, `SELECT *` Parquet scan, partition file explosion, missing `ORDER BY`, order-sensitive aggregate drift, TIMESTAMPTZ timezone surprise, memory-limit overclaim, temp spill disk-full, overwritten profiling output, ART-index theater, CTE materialization surprise, window memory spike, unsafe macro input, or unverified query-plan claim | `changes_status`, `changes_diff_summary`, `test_related`, `test`, `lint`, `build`, `docs_validate_fast`, `test_release`, `mustflow_check` | DuckDB role/version/binding, file/process ownership, concurrency/Appender/import/export, deterministic SQL, memory/temp/profiling, query-plan evidence, verification, and remaining DuckDB risk |
|
|
663
|
-
| Keyword search, full-text search, Elasticsearch, OpenSearch, Lucene-style indexing, search APIs, indexing pipelines, aliases, bulk indexing, refresh visibility, analyzers, mappings, synonyms, autocomplete, pagination, shard failures, search quality, or search performance are created, changed, reviewed, or failing | `.mustflow/skills/search-index-integrity-review/SKILL.md` | Symptom classification, source-to-search ledger, query contract ledger, index contract ledger, quality ledger, performance ledger, privacy ledger, changed files, and command contract entries | Search canaries, indexing ledgers, bulk item error handling, alias checks, mapping and analyzer fixtures, exact-versus-full-text tests, tenant and permission filters, golden-set tests, synonym regression tests, pagination guards, query metrics, docs, and directly synchronized templates | cluster-green theater, batch-level bulk success, source/index count illusion, write alias drift, partial shard result, direct/API/UI mismatch, wrong keyword/text field, analyzer drift, synonym regression, rank eyeballing, profile misuse, query fingerprint leak, shard fan-out, cache-only benchmark, refresh overuse, segment merge backlog, disk watermark write block, deep pagination, oversized fetch, or private query/document leak | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | Search index integrity reviewed, source-to-search/query/index/quality/performance/privacy ledgers,
|
|
664
|
-
| Vector search, semantic search, RAG retrieval, embedding generation, preprocessing, chunking, vector schema, collection, namespace, tenant, named vector, metadata payload, filter, ANN index, exact-versus-approximate search, hybrid search, reranking, recall, latency, quantization, HNSW, IVF, pgvector, Qdrant, Milvus, Weaviate, OpenSearch kNN, or retrieval golden-set behavior is created, changed, reviewed, or failing | `.mustflow/skills/vector-search-integrity-review/SKILL.md` | Retrieval symptom, query contract ledger, ingestion ledger, quality ledger, performance ledger, privacy ledger, changed files, and command contract entries | Embedding and
|
|
666
|
+
| Keyword search, full-text search, Elasticsearch, OpenSearch, Lucene-style indexing, search APIs, indexing pipelines, source maps, metadata taxonomy, negative metadata, exact keyword fields, aliases, bulk indexing, refresh visibility, analyzers, mappings, synonyms, autocomplete, pagination, shard failures, search quality, or search performance are created, changed, reviewed, or failing | `.mustflow/skills/search-index-integrity-review/SKILL.md` | Symptom classification, source-to-search ledger, query contract ledger, index contract ledger with taxonomy/source-map fields, metadata key budget, stable source/doc/chunk ids, lifecycle and effective-date fields, quality ledger, performance ledger, privacy ledger, changed files, and command contract entries | Search canaries, indexing ledgers, bulk item error handling, alias checks, mapping and analyzer fixtures, metadata taxonomy checks, frontmatter schema checks, controlled-vocabulary fixtures, exact-keyword fixtures, negative-metadata filters, filter-only versus LLM-visible metadata checks, exact-versus-full-text tests, tenant and permission filters, golden-set tests, synonym regression tests, pagination guards, query and miss-log metrics, docs, and directly synchronized templates | cluster-green theater, batch-level bulk success, source/index count illusion, write alias drift, partial shard result, direct/API/UI mismatch, wrong keyword/text field, analyzer drift, uncontrolled tag vocabulary, unbounded metadata keys, file path treated as status, synonym regression, negative metadata ignored, missing source map, stale index hash, rank eyeballing, profile misuse, query fingerprint leak, shard fan-out, cache-only benchmark, refresh overuse, segment merge backlog, disk watermark write block, deep pagination, oversized fetch, or private query/document leak | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | Search index integrity reviewed, source-to-search/query/index/quality/performance/privacy ledgers, taxonomy/metadata-budget/exact-keyword/source-map/query-log findings, fix or recommendation, evidence level, verification, and remaining search-index risk |
|
|
667
|
+
| Vector search, semantic search, RAG retrieval, embedding generation, preprocessing, chunking, contextual headers, synthetic questions, chunk text variants, vector schema, collection, namespace, tenant, named vector, metadata payload, filter, ANN index, exact-versus-approximate search, hybrid search, reranking, recall, latency, quantization, HNSW, IVF, pgvector, Qdrant, Milvus, Weaviate, OpenSearch kNN, or retrieval golden-set behavior is created, changed, reviewed, or failing | `.mustflow/skills/vector-search-integrity-review/SKILL.md` | Retrieval symptom, query contract ledger, ingestion ledger with stable source/doc/chunk ids, parent document genealogy, text variants and hashes, quality ledger, performance ledger, privacy ledger, changed files, and command contract entries | Embedding, preprocessing, and chunker versioning, vector validation, deterministic ids, contextual retrieval headers, synthetic question fields, original/index/prompt/embedding text separation, namespace and tenant selection, metadata payload field typing, metadata indexes, ACL prefilters, filter construction, exact-search checks, ANN parameters, lexical safeguards for exact identifiers, hybrid score ledgers, RRF or MMR settings, reranker candidates, golden-set tests, synthetic fixtures, metrics, docs, and directly synchronized templates | vector-DB scapegoating, wrong embedding dimension, model revision drift, contextual retrieval text treated as source, synthetic question cited as evidence, unstable source/doc/chunk lineage, tiny chunk subject loss, huge chunk semantic averaging, overlap top-k duplication, filter post-candidate loss, metadata type drift, ACL applied after retrieval, tenant leak, duplicate chunk ids, stale deletes, content hash or embedding hash drift, metric or normalization mismatch, ANN tuning before exact-search proof, quantization recall loss, reranker candidate starvation, hybrid score misuse, exact identifiers smoothed into embedding-only text, deep ANN pagination, raw vector or document leak, or unmeasured p95 latency | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | Vector search integrity reviewed, retrieval/query/ingestion/quality/performance/privacy ledgers, exact-versus-ANN/filter/text-variant/contextual-header/lineage/hybrid/rerank findings, fix or recommendation, evidence level, verification, and remaining vector-search risk |
|
|
665
668
|
| Dependency versions, lockfiles, package-manager metadata, workspace constraints, runtime engines, peer dependencies, optional dependencies, security advisory fixes, vulnerability scanner alerts, generated dependency output, framework plugins, dev servers, test runners, TypeScript compiler tracks, CI actions, Docker base images, package manager behavior, or toolchain versions are upgraded, downgraded, pinned, widened, regenerated, reviewed, or reported | `.mustflow/skills/dependency-upgrade-review/SKILL.md` | Dependency name, old and new versions or ranges, direct or transitive path, ecosystem and package manager, declaration files, lockfiles, runtime or toolchain files, advisory or release-note evidence, exploit preconditions, generated outputs, callers, docs, package output, Docker, CI, dev-server or test-UI exposure, protocol role, or TypeScript compiler-track surfaces, and command contract entries | Package declarations, lockfiles, generated outputs, compatibility code, tests, docs, package metadata, Docker or CI files, dev-server/test-runner config notes, TypeScript compiler-track notes, and directly synchronized examples | lockfile churn, hidden transitive replacement, peer or engine break, module-format drift, native or optional package break, framework or generator output drift, devDependency advisory dismissed despite exposure, unsafe broad security update, weakened tests, Docker or CI runtime drift, protocol DoS misclassified, TS7 RC over-adoption, TS7 nightly over-adoption, or unreviewed supply-chain change | `changes_status`, `changes_diff_summary`, `lint`, `build`, `test_related`, `test`, `docs_validate_fast`, `test_release`, `mustflow_check` | Upgrade reason, ecosystem surface, direct and transitive graph changes, compatibility classification, runtime/peer/engine/module/feature/platform/generated-output/compiler-track risks, advisory exploit preconditions, synchronized surfaces, verification, and remaining dependency-upgrade risk |
|
|
666
669
|
| Dependency, package, runtime, framework, tool, command, plugin, service, platform capability, supported-version policy, security patch path, ecosystem maturity claim, maintainer-risk assumption, runtime portability claim, edge or serverless compatibility claim, critical-path library choice, package script, lifecycle hook, binary download, lockfile, audit result, or supply-chain-sensitive dependency surface is assumed, added, removed, imported, invoked, installed, audited, or documented | `.mustflow/skills/dependency-reality-check/SKILL.md` | Assumed dependency or capability, declaration files, version or feature expectation, role criticality, supported-version or end-of-life evidence, patchability expectation, runtime compatibility boundary, maintainer and ecosystem evidence when available, lockfile entry, package script or lifecycle hook, audit or provenance evidence, and relevant command intents | Package metadata, lockfiles, imports, scripts, command contracts, docs, tests, runtime policy notes, portability notes, and reports | unavailable dependency, hallucinated or lookalike package, fragile single-maintainer core dependency, experimental technology in a survival path, unsupported runtime, unclear security patch path, runtime-specific API leakage into core logic, stale version claim, lifecycle script risk, audit suppression, lockfile drift, or install guidance mismatch | `changes_status`, `changes_diff_summary`, `build`, `test_release`, `mustflow_check` | Dependency checked, ecosystem and maintainer-risk boundary reviewed, supported-version, patchability, and runtime-portability boundary reviewed, supply-chain surface reviewed, declarations synchronized, verification, and remaining dependency risk |
|
|
667
670
|
| Generated or edited code, configuration, CI workflows, package metadata, install instructions, examples, Docker images, framework setup, runtime declarations, toolchain declarations, TypeScript compiler-track references, Go release or framework references, Java/JDK GA, LTS, JEP, JVM, GC, or toolchain references, Rust release or MSRV references, or migration-sensitive snippets introduce explicit external version references, action refs, package ranges, runtime versions, framework majors, Docker image tags, or scaffold commands that may be stale | `.mustflow/skills/version-freshness-check/SKILL.md` | Versioned reference, owning files, repository version policy, approved freshness source, compatibility context, migration risk, TypeScript compiler track, Go toolchain/framework track, Java JDK/toolchain/bytecode/JEP track, or Rust MSRV/toolchain track when relevant, and command contract entries | Package metadata, lockfiles, CI workflows, Dockerfiles, runtime files, framework config, docs, examples, templates, tests, and version-decision reports | stale default version, false latest claim, accidental major migration, repository policy mismatch, unsupported generated example, TypeScript RC/nightly/API-track confusion, Java latest-GA/LTS/runtime/JEP/preview/incubator confusion, Rust stable/nightly/MSRV confusion, floating-tag drift, or unverified security/support claim | `changes_status`, `changes_diff_summary`, `build`, `test_related`, `docs_validate_fast`, `test_release`, `mustflow_check` | Versioned surfaces checked, repository policy and freshness source, selected version track, compatibility classification, TypeScript stable/RC/nightly/API-track, Go runtime/framework, Java GA/LTS/runtime/JEP/toolchain, and Rust stable/nightly/MSRV split when relevant, approval need, synchronized surfaces, verification, and remaining version-freshness risk |
|
package/templates/default/locales/en/.mustflow/skills/llm-token-cost-control-review/SKILL.md
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
mustflow_doc: skill.llm-token-cost-control-review
|
|
3
3
|
locale: en
|
|
4
4
|
canonical: true
|
|
5
|
-
revision:
|
|
5
|
+
revision: 4
|
|
6
6
|
lifecycle: mustflow-owned
|
|
7
7
|
authority: procedure
|
|
8
8
|
name: llm-token-cost-control-review
|
|
9
|
-
description: Apply this skill when LLM API calls, prompt assembly, chat history, RAG context, tool schemas, structured output schemas, model routing, reasoning settings, token budgets, provider prompt caching, app-level response caching, retries, batch or flex processing, predicted outputs, image or file inputs, or LLM cost metrics are created, changed, reviewed, or reported and the risk is token spend or cost-per-success drifting out of control.
|
|
9
|
+
description: Apply this skill when LLM API calls, prompt assembly, chat history, RAG context, document metadata, chunk summaries, prompt packing, tool schemas, structured output schemas, model routing, reasoning settings, token budgets, provider prompt caching, app-level response caching, retries, batch or flex processing, predicted outputs, image or file inputs, or LLM cost metrics are created, changed, reviewed, or reported and the risk is token spend or cost-per-success drifting out of control.
|
|
10
10
|
metadata:
|
|
11
11
|
mustflow_schema: "1"
|
|
12
12
|
mustflow_kind: procedure
|
|
@@ -35,7 +35,7 @@ Review LLM cost as a product and systems contract, not as prompt brevity. A cost
|
|
|
35
35
|
<!-- mustflow-section: use-when -->
|
|
36
36
|
## Use When
|
|
37
37
|
|
|
38
|
-
- A change adds, edits, reviews, or reports an LLM request builder, prompt prefix, system or developer message, chat history assembly, memory compaction, RAG context, retrieved chunk packing, few-shot examples, tool definitions, structured output schema, image or file input, model selector, reasoning-effort setting, output limit, retry path, streaming or prediction path, batch processing path, flex or low-priority processing path, cache key, token counter, usage logger, cost dashboard, or LLM quota guard.
|
|
38
|
+
- A change adds, edits, reviews, or reports an LLM request builder, prompt prefix, system or developer message, chat history assembly, memory compaction, RAG context, retrieved chunk packing, document metadata, source maps, summary layers, few-shot examples, tool definitions, structured output schema, image or file input, model selector, reasoning-effort setting, output limit, retry path, streaming or prediction path, batch processing path, flex or low-priority processing path, cache key, token counter, usage logger, cost dashboard, or LLM quota guard.
|
|
39
39
|
- A task asks to reduce token cost, improve provider prompt-cache hit rate, lower cost per successful task, shrink context, split realtime and offline LLM work, route small tasks away from expensive models, or prevent a token bill spike.
|
|
40
40
|
- The product sends repeated instructions, examples, tool schemas, output schemas, documents, policies, conversation history, screenshots, files, or retrieved context to an LLM.
|
|
41
41
|
- The system records total tokens but cannot explain which feature, endpoint, model, prompt version, tool schema version, retry, or validation failure caused the spend.
|
|
@@ -55,9 +55,15 @@ Review LLM cost as a product and systems contract, not as prompt brevity. A cost
|
|
|
55
55
|
## Required Inputs
|
|
56
56
|
|
|
57
57
|
- Cost surface ledger: feature, endpoint, caller, model, provider, sync or async path, expected traffic shape, success definition, and current or target budget.
|
|
58
|
-
- Request ledger: stable instructions, examples, tools, schemas,
|
|
58
|
+
- Request ledger: stable instructions, examples, tools, schemas, authority lane for each block,
|
|
59
|
+
user input, retrieved context, memory, history, files, images, runtime metadata, volatile IDs,
|
|
60
|
+
dates, locale, and personalization fields.
|
|
59
61
|
- Cache ledger: provider prompt-cache eligibility, stable prefix boundary, canonical serialization, prompt version hash, tool and schema version, cache key policy, app-level response cache key, TTL, invalidation rule, and permission boundary.
|
|
60
|
-
- Context ledger: conversation-window rule,
|
|
62
|
+
- Context ledger: conversation-window rule, state-card fields, delta rule, inclusion test,
|
|
63
|
+
block role tags, document ids, frontmatter or metadata fields split by filter-only versus
|
|
64
|
+
LLM-visible use, summary layers, source maps, question-specific evidence cards, RAG top-k, chunk
|
|
65
|
+
size, chunk token counts, chunk ordering, evidence span policy, deduplication, compression,
|
|
66
|
+
prompt-packing rule, and current input-token measurement.
|
|
61
67
|
- Output ledger: output schema size, repeated key length, patch versus full-output policy, `max_output_tokens`, reasoning budget, retry repair inputs, validator errors, and incomplete-response handling.
|
|
62
68
|
- Routing ledger: deterministic prefilters, small-model router, expensive-model escalation rule, batch or flex eligibility, predicted-output eligibility, image or file preprocessing, and fallback behavior.
|
|
63
69
|
- Observability ledger: input tokens, cached tokens when exposed by the provider, output tokens, reasoning tokens when exposed or billable, retry count, validation failure count, cache hit rate, cost per successful task, model, endpoint, prompt version, tool version, schema version, and budget breach events.
|
|
@@ -75,7 +81,7 @@ Review LLM cost as a product and systems contract, not as prompt brevity. A cost
|
|
|
75
81
|
## Allowed Edits
|
|
76
82
|
|
|
77
83
|
- Refactor request assembly so stable instructions, examples, tool schemas, and output schemas are serialized before volatile user input, timestamps, request IDs, search results, personalization, and runtime metadata.
|
|
78
|
-
- Add or refine prompt version hashes, canonical serialization, provider cache keys, app-level cache keys, token counters, budget guards, model routers, context trimming, RAG chunk packing, output patch formats, retry repair paths, metrics, logs, tests, docs, route metadata, and directly synchronized templates.
|
|
84
|
+
- Add or refine prompt version hashes, canonical serialization, provider cache keys, app-level cache keys, token counters, token-budget metadata, budget guards, model routers, context trimming, RAG chunk packing, source-map references, original/index/prompt text separation, question-scoped compression, slot records, canonical block references, state snapshots, output patch formats, retry repair paths, metrics, logs, tests, docs, route metadata, and directly synchronized templates.
|
|
79
85
|
- Move deterministic work such as validation, parsing, enum mapping, deduplication, sorting, formatting, arithmetic, date math, and permission checks out of LLM calls when code can perform it.
|
|
80
86
|
- Add focused fixtures for cost-sensitive boundaries: cache-prefix drift, history growth, RAG chunk bloat, retry replay, oversized schema, full-file regeneration, image input size, and expensive-model routing.
|
|
81
87
|
- Do not rely on "make the prompt shorter" as the primary fix when repeated static payload, history replay, RAG bloat, retries, or output shape causes the spend.
|
|
@@ -88,29 +94,61 @@ Review LLM cost as a product and systems contract, not as prompt brevity. A cost
|
|
|
88
94
|
|
|
89
95
|
1. Name the cost unit. Decide whether the system optimizes per request, per successful task, per workflow, per user session, per batch job, or per tenant.
|
|
90
96
|
2. Measure before cutting. Prefer provider token-counting APIs, usage fields, or repository-local token accounting over character estimates, especially when tools, schemas, images, files, or reasoning models are involved.
|
|
91
|
-
3.
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
97
|
+
3. Apply the output-changing test before packing context. If removing a document, transcript slice,
|
|
98
|
+
metadata field, example, or log would not change the answer, code, decision, citation, or
|
|
99
|
+
refusal state, keep it out of the model payload and preserve only a reference, count, or hash
|
|
100
|
+
when needed for traceability.
|
|
101
|
+
4. Keep authority lanes explicit. Stable rules, policies, and output contracts belong in
|
|
102
|
+
instruction blocks; user input, retrieved documents, examples, and tool observations belong in
|
|
103
|
+
tagged data blocks. Do not let retrieved text, examples, or summaries become equal-authority
|
|
104
|
+
instructions.
|
|
105
|
+
5. Split stable prefix from volatile suffix. Keep system and developer instructions, policy, examples, tool schemas, structured output schemas, and other repeated static context in a canonical stable order before user input, request IDs, dates, session IDs, retrieved snippets, and personalization.
|
|
106
|
+
6. Hash the expensive prefix. Record or derive a prompt version hash from canonical instructions, examples, tools, schemas, model settings, and output contract. If the hash varies per user without intent, treat it as cache-prefix drift.
|
|
107
|
+
7. Review provider prompt caching. Use provider cache keys only for requests that share the same stable prefix, distribute hot keys according to provider guidance, and log cached-token evidence when the provider exposes it.
|
|
108
|
+
8. Add app-level caching where identical normalized inputs can safely skip the model call. Include tenant, permission, locale, source version, policy version, and TTL in the cache key when they affect correctness.
|
|
109
|
+
9. Trim chat history by state, not by vibes. Keep recent turns, durable memory, active task state, confirmed user preferences, and unresolved tool results; avoid replaying full raw transcripts when a compact state is enough.
|
|
110
|
+
10. Shrink RAG by evidence span. Search broadly if needed, but pass the smallest source-backed spans that answer the question, deduplicate near-identical chunks, preserve source metadata, and use stable ordering when repeated workloads benefit from cache reuse.
|
|
111
|
+
Preserve conflicts with source, status, effective date, and authority instead of merging them into
|
|
112
|
+
one smooth but false summary.
|
|
113
|
+
11. Compress for the current question, not for the whole document. A refund-policy, implementation,
|
|
114
|
+
legal, and support answer may need different evidence from the same source; store the compression
|
|
115
|
+
goal, retained fields, dropped fields, and source coordinates.
|
|
116
|
+
12. Use evidence cards before raw chunks when the product can validate them. Include claim,
|
|
117
|
+
conditions, exceptions, numbers, source ids, section or line span, quote hash, and token count.
|
|
118
|
+
Expand the original source only when the answer, citation, or validator needs it.
|
|
119
|
+
13. Convert repetitive prose to structured slots. Prefer `condition`, `action`, `limit`,
|
|
120
|
+
`exception`, `source_ref`, and `confidence` fields over sentence-shaped padding when downstream
|
|
121
|
+
code or prompts consume rules.
|
|
122
|
+
14. Use state snapshots plus deltas. Carry the compact current state and the newest change rather
|
|
123
|
+
than replaying the full transcript or sending only a delta that has lost its baseline.
|
|
124
|
+
15. Put critical evidence where the model will see it. Do not bury the only decisive constraint in
|
|
125
|
+
the middle of a long context; put key evidence early and restate final hard constraints near the
|
|
126
|
+
end when the prompt shape is long.
|
|
127
|
+
16. Separate original, index, and prompt text. Keep original spans for evidence, enriched `index_text` for search recall, and compact `prompt_text` for model input. Do not let search-only synonyms or generated summaries look like original quoted evidence.
|
|
128
|
+
Keep search/filter metadata separate from LLM-visible metadata; tags, IDs, ACL fields, and
|
|
129
|
+
operational paths should reach the model only when they change the output or citation.
|
|
130
|
+
17. Pack by references before raw text. Prefer stable document ids, chunk ids, section anchors, source maps, block refs, and token counts when choosing context. Expand only the needed source spans instead of replaying a whole corpus or full document list.
|
|
131
|
+
18. Keep tool and schema payloads boring. Tool descriptions and JSON schemas should be long enough for correct routing and validation but not narrative prose. If permissions differ by user, keep schema stable and enforce permission at tool execution.
|
|
132
|
+
19. Route before calling the expensive model. Use deterministic code, regexes, database lookups, small models, or cheap classifiers for tasks that do not need a large reasoning model; escalate only ambiguous, high-value, or failed cases.
|
|
133
|
+
20. Budget reasoning and output together. Set reasoning effort and output limits according to task value; leave enough room for visible output, and handle incomplete responses instead of silently retrying the full expensive request.
|
|
134
|
+
21. Prefer patches over full regeneration when the product already owns most of the output. Use unified diff, JSON Patch, line-range replacement, IDs, labels, scores, or reason codes when downstream code can merge the result.
|
|
135
|
+
22. Repair failures without full replay. For parse failures, enum mismatches, missing fields, or validator errors, retry with previous output, validator error, and schema summary when safe instead of resending the entire original context.
|
|
136
|
+
23. Separate realtime and offline work. Move evals, bulk classification, enrichment, embeddings, report backfills, and log analysis to configured async, batch, or low-priority processing when user latency does not matter.
|
|
137
|
+
24. Treat predicted outputs as a latency tool unless current provider docs and usage evidence show cost behavior for the exact model and endpoint. Use `llm-response-latency-review` when the main goal is faster completion rather than cost control. Watch rejected prediction tokens or equivalent fields when exposed.
|
|
138
|
+
25. Reduce image and file input before the model. Crop screenshots, downsample where acceptable, extract DOM text or OCR first, and count the actual payload tokens when the provider supports it.
|
|
139
|
+
26. Evaluate compression as loss, not as style. Compare compressed and uncompressed answers on
|
|
140
|
+
answer exactness, citation recall, numeric accuracy, constraint violations, and correct refusals.
|
|
141
|
+
27. Instrument cost per success. Track endpoint, model, prompt version, tool version, schema version, corpus version, index version, source hash, input tokens, cached tokens, output tokens, reasoning tokens, retry count, validation failure rate, cache hit rate, and successful-task denominator.
|
|
142
|
+
28. When prompt-cache layout changes, run the configured prompt-cache audit intent if available and treat byte or token estimates as static layout evidence rather than provider billing proof.
|
|
143
|
+
29. Verify with the narrowest configured tests, fixtures, docs validation, release checks, and mustflow validation that cover request assembly, cache keys, budget guards, routing, retry repair, telemetry, and installed skill surfaces.
|
|
108
144
|
|
|
109
145
|
<!-- mustflow-section: postconditions -->
|
|
110
146
|
## Postconditions
|
|
111
147
|
|
|
112
148
|
- Stable LLM payload is separated from volatile data, hashed or versioned where useful, and cacheable by design when the provider supports prompt caching.
|
|
113
149
|
- Long context, chat history, RAG evidence, tools, schemas, files, images, reasoning, output, retries, and routing each have an explicit budget or reduction rule where relevant.
|
|
150
|
+
- Retrieved context distinguishes original evidence, enriched index text, prompt-ready text, question-specific evidence cards, source-map references, token counts, block references, and summary layers.
|
|
151
|
+
- Compression quality is evaluated against answer, citation, numeric, constraint, and refusal behavior rather than judged by shorter text alone.
|
|
114
152
|
- Deterministic work is handled outside the model unless language judgment is required.
|
|
115
153
|
- Metrics can explain cost per successful task, cache-prefix drift, retry cost, validation failures, and model routing decisions without leaking sensitive prompt or user data.
|
|
116
154
|
- Final reports distinguish proven cost-control evidence from assumed provider behavior, anecdotal token savings, and latency-only optimizations.
|
|
@@ -149,7 +187,7 @@ Use the narrowest configured fixture, unit, integration, telemetry, docs, packag
|
|
|
149
187
|
- LLM token-cost surface reviewed
|
|
150
188
|
- Cost unit, budget, and measurement source
|
|
151
189
|
- Stable prefix, volatile suffix, prompt hash, and provider cache behavior checked
|
|
152
|
-
- App cache, history, RAG, tool schema, structured schema, image or file input, and deterministic prefilter choices checked
|
|
190
|
+
- App cache, history, RAG, metadata, source-map, prompt-packing, tool schema, structured schema, image or file input, and deterministic prefilter choices checked
|
|
153
191
|
- Model routing, reasoning effort, output limit, patch-output, retry repair, Batch, Flex, and prediction choices checked
|
|
154
192
|
- Cost observability fields and sensitive-data redaction checked
|
|
155
193
|
- Files changed
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
mustflow_doc: skill.rag-pipeline-triage
|
|
3
3
|
locale: en
|
|
4
4
|
canonical: true
|
|
5
|
-
revision:
|
|
5
|
+
revision: 3
|
|
6
6
|
lifecycle: mustflow-owned
|
|
7
7
|
authority: procedure
|
|
8
8
|
name: rag-pipeline-triage
|
|
9
|
-
description: Apply this skill when a RAG, knowledge-base answer, grounded chat, citation answer, retrieval-augmented support bot, or document QA flow is wrong, stale, unsupported, slow, leaking data, over-refusing, or not yet localized to ingestion, parsing, chunking, retrieval, filtering, reranking, context assembly, prompt construction, generation, citation validation, or answerability boundaries.
|
|
9
|
+
description: Apply this skill when a RAG, knowledge-base answer, grounded chat, citation answer, retrieval-augmented support bot, or document QA flow is wrong, stale, unsupported, slow, leaking data, over-refusing, or not yet localized to ingestion, parsing, document metadata, source maps, chunking, retrieval, filtering, reranking, context assembly, prompt construction, generation, citation validation, or answerability boundaries.
|
|
10
10
|
metadata:
|
|
11
11
|
mustflow_schema: "1"
|
|
12
12
|
mustflow_kind: procedure
|
|
@@ -46,6 +46,9 @@ survive filtering and context assembly, and constrain the answer?"
|
|
|
46
46
|
generation, validators, citations, answerability, or access control.
|
|
47
47
|
- A review would otherwise tune the model, top-k, chunk size, reranker, or prompt before proving
|
|
48
48
|
which RAG layer failed.
|
|
49
|
+
- A document corpus, frontmatter contract, search index, chunk schema, ACL model, or prompt-packing
|
|
50
|
+
rule is being reviewed because token bloat, retrieval misses, stale context, or unsupported
|
|
51
|
+
citations may come from poor document structure rather than model behavior.
|
|
49
52
|
|
|
50
53
|
<!-- mustflow-section: do-not-use-when -->
|
|
51
54
|
## Do Not Use When
|
|
@@ -70,9 +73,14 @@ survive filtering and context assembly, and constrain the answer?"
|
|
|
70
73
|
context, filters, embedding model version, index version, candidate ids and scores, reranker
|
|
71
74
|
output, final context ids and order, prompt version, model version, answer, citations, validators,
|
|
72
75
|
latency, and cost when safe.
|
|
73
|
-
- Source ledger: authoritative source availability,
|
|
74
|
-
|
|
75
|
-
|
|
76
|
+
- Source ledger: authoritative source availability, original text, parsed text, index text,
|
|
77
|
+
prompt text, source id, stable doc id, chunk id, parent document genealogy, chunk boundaries,
|
|
78
|
+
frontmatter or metadata schema, document type, status, authority, source-of-truth flag, title,
|
|
79
|
+
aliases, exact keywords, synthetic questions, negative metadata, heading path, breadcrumb ids or
|
|
80
|
+
text, source map, parent and adjacent chunks, section or document summaries, routing summaries,
|
|
81
|
+
lifecycle owner, version, revision, supersedes or superseded-by links, content hash, index
|
|
82
|
+
freshness, published and effective dates, stale or deleted documents, duplicates, and
|
|
83
|
+
conflicting sources.
|
|
76
84
|
- Comparison ledger: no-retrieval answer, retrieved-context answer, human-selected gold-context
|
|
77
85
|
answer, exact or keyword search result, vector search result, hybrid result, and expected
|
|
78
86
|
answerability state.
|
|
@@ -93,10 +101,11 @@ survive filtering and context assembly, and constrain the answer?"
|
|
|
93
101
|
<!-- mustflow-section: allowed-edits -->
|
|
94
102
|
## Allowed Edits
|
|
95
103
|
|
|
96
|
-
- Add or tighten trace fields, fixture queries, parsing checks,
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
104
|
+
- Add or tighten trace fields, fixture queries, parsing checks, semantic or structure-aware chunking,
|
|
105
|
+
chunk metadata, frontmatter schemas, source maps, parent-child or adjacent chunk links,
|
|
106
|
+
ACL-prefilter checks, duplicate or stale source handling, retrieval comparisons, filter checks,
|
|
107
|
+
context-packing rules, prompt source separation, citation validators, answerability states, dirty
|
|
108
|
+
eval fixtures, metrics, docs, and directly synchronized templates.
|
|
100
109
|
- Add safe synthetic fixtures for missing-doc, correct-doc-unused, stale-doc, conflicting-doc,
|
|
101
110
|
unauthorized-doc, exact-id, keyword, vector, hybrid, reranker, citation, and abstain behavior.
|
|
102
111
|
- Do not change models, re-embed data, rebuild production indexes, widen access filters, disable
|
|
@@ -117,34 +126,74 @@ survive filtering and context assembly, and constrain the answer?"
|
|
|
117
126
|
headers, footers, and OCR can be broken even when the original file looks correct.
|
|
118
127
|
5. Inspect chunk boundaries and metadata. Verify title, parent section, version, dates, audience,
|
|
119
128
|
product, source authority, and neighboring context survive into chunks or parent retrieval.
|
|
120
|
-
6.
|
|
129
|
+
6. Check whether a single chunk can stand on its own. Each answerable chunk should carry enough
|
|
130
|
+
product, feature, version, lifecycle, and heading context that retrieval does not depend on the
|
|
131
|
+
model guessing the missing subject from a previous chunk.
|
|
132
|
+
7. Review headings as coordinates. Generic headings such as overview, details, or notes are weak
|
|
133
|
+
retrieval features; heading paths and breadcrumbs should name the product, workflow, rule, and
|
|
134
|
+
exception boundary that make the chunk answerable.
|
|
135
|
+
8. Separate original, index, and prompt text. The original span is evidence, enriched index text is
|
|
136
|
+
search bait, and prompt text is the compact model payload. Do not cite generated summaries,
|
|
137
|
+
aliases, or synonym-expanded text as if they were original source.
|
|
138
|
+
9. Check source maps and chunk graph. Verify file path or URL, section anchor, line or page span,
|
|
139
|
+
content hash, parent chunk, previous chunk, next chunk, and summary layer can lead back to the
|
|
140
|
+
source without loading the whole corpus.
|
|
141
|
+
10. Check document identity and lifecycle before merging evidence. Keep `source_id`, `doc_id`, and
|
|
142
|
+
`chunk_id` distinct; preserve rename-stable ids, `doc_type`, `status`, `authority`,
|
|
143
|
+
`source_of_truth`, `supersedes`, `superseded_by`, `valid_until`, and effective-date fields so
|
|
144
|
+
stale, draft, example, discussion, and canonical documents are not averaged into one answer.
|
|
145
|
+
11. Treat summaries as routers, not proof. A summary should carry what the document can answer,
|
|
146
|
+
what it cannot answer, entities, decisions, exceptions, related docs, deprecated content,
|
|
147
|
+
conditions, actions, numbers, and exact source coordinates; if it ages separately from the
|
|
148
|
+
source, classify the failure as stale entrance metadata before changing retrieval settings.
|
|
149
|
+
12. Check exact keywords and aliases. Error codes, SKU values, API paths, function names, legal
|
|
150
|
+
references, old product names, and user slang should be searchable without relying on embeddings.
|
|
151
|
+
13. For code corpora, preserve structure. Function, class, import, caller, callee, type, test, and
|
|
152
|
+
config-key boundaries should guide chunks; fixed line-count slicing that cuts through symbols is
|
|
153
|
+
retrieval damage, not neutral preprocessing.
|
|
154
|
+
14. For table, slide, and PDF corpora, preserve record shape. Row, column, unit, plan, page, and
|
|
155
|
+
figure context should be repeated in parsed/index text so a number is not retrieved without its
|
|
156
|
+
meaning.
|
|
157
|
+
15. Check rules beside exceptions. If prohibitions, limits, or jurisdiction exclusions live far away
|
|
158
|
+
from the rule they modify, context assembly may retrieve only the rule and miss the exception.
|
|
159
|
+
16. Check ACL before retrieval. Tenant, visibility, sensitivity, retention, and user or group access
|
|
160
|
+
must be inherited from documents to chunks and applied before candidate text reaches the model.
|
|
161
|
+
17. Compare source versions and deletes. Duplicates, obsolete documents, tombstones, and conflicting
|
|
121
162
|
effective dates must not be silently mixed into one answer.
|
|
122
|
-
|
|
163
|
+
Preserve conflicting sources with their priority, status, and effective date instead of
|
|
164
|
+
smoothing them into a synthesized rule.
|
|
165
|
+
18. Run the isolation comparison when evidence is available: no retrieval, current retrieved context,
|
|
123
166
|
and human-selected gold context. Gold-context failure points to generation or prompt; current
|
|
124
167
|
context failure with gold success points to retrieval or context assembly.
|
|
125
|
-
|
|
168
|
+
19. Compare keyword, vector, hybrid, and exact-id retrieval by data shape. IDs, error codes, SKUs,
|
|
126
169
|
names, dates, and numbers need exact or lexical safeguards; semantic questions may need vector or
|
|
127
170
|
hybrid retrieval.
|
|
128
|
-
|
|
171
|
+
20. Check filters before blaming embeddings. Record pre-filter candidate count, post-filter count,
|
|
129
172
|
tenant and permission filters, metadata types, time zones, empty arrays, case sensitivity, and
|
|
130
173
|
stale policy copies.
|
|
131
|
-
|
|
174
|
+
21. Check reranker candidate starvation. If the correct source never enters the candidate set, the
|
|
132
175
|
reranker cannot fix it. If it enters and then drops, inspect reranker inputs and scoring.
|
|
133
|
-
|
|
176
|
+
22. Check context assembly. Verify `top_k`, score thresholds, source order, truncation, deduping,
|
|
134
177
|
conflict handling, source authority, and whether important evidence is buried or cut off.
|
|
135
|
-
|
|
178
|
+
23. Check prompt construction. User input, retrieved text, examples, tool observations, and system or
|
|
136
179
|
developer instructions must remain separated. Retrieved text is data, not authority.
|
|
137
|
-
|
|
180
|
+
24. Check answerability and abstain behavior. Track no-evidence, low-confidence, conflicting-source,
|
|
138
181
|
stale-source, access-denied, tool-failed, and needs-human states separately.
|
|
139
|
-
|
|
182
|
+
25. Validate citations claim-by-claim. A citation id proves nothing unless the cited chunk supports
|
|
140
183
|
the specific generated claim.
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
184
|
+
26. Measure each layer separately. Track parsing success, metadata completeness, ACL inheritance,
|
|
185
|
+
index freshness, hit rate, Recall@k, MRR, precision, nDCG, rerank survival, context precision,
|
|
186
|
+
context recall, faithfulness, groundedness, answer relevance, context token budget, answer
|
|
187
|
+
accuracy, citation accuracy, abstain accuracy, access leaks, and retrieval/rerank/generation
|
|
188
|
+
latency and cost.
|
|
189
|
+
27. Check the document graph for multi-hop questions. PRDs, ADRs, issues, code modules, runbooks,
|
|
190
|
+
changelogs, incidents, and meeting notes should link decisions, implementation, ownership,
|
|
191
|
+
exceptions, and follow-up operations instead of forcing retrieval to infer relationships from
|
|
192
|
+
nearby words.
|
|
193
|
+
28. Use dirty eval cases from real failures. Include typos, abbreviations, multilingual questions,
|
|
145
194
|
unanswerable questions, date-sensitive questions, similar names, product codes, multi-hop
|
|
146
195
|
questions, unauthorized documents, stale documents, and conflicting documents.
|
|
147
|
-
|
|
196
|
+
29. Apply the smallest localized fix and switch to the narrower matching skill for retrieval,
|
|
148
197
|
hallucination control, prompt contract, token cost, latency, access control, or prompt-injection
|
|
149
198
|
defense once the boundary is known.
|
|
150
199
|
|
|
@@ -155,6 +204,11 @@ survive filtering and context assembly, and constrain the answer?"
|
|
|
155
204
|
reranking, context assembly, prompt construction, generation, citation validation, answerability,
|
|
156
205
|
access control, or a named evidence gap.
|
|
157
206
|
- Trace, source, comparison, eval, metric, and privacy ledgers are explicit where relevant.
|
|
207
|
+
- Document metadata, frontmatter schema, stable source/doc/chunk ids, document type, status,
|
|
208
|
+
authority, source-of-truth, effective dates, supersession links, heading paths, source maps, ACL
|
|
209
|
+
inheritance, original/index/prompt text separation, exact keywords, aliases, negative metadata,
|
|
210
|
+
routing summaries, summary layers, chunk adjacency, document graph links, and index freshness are
|
|
211
|
+
explicit where relevant.
|
|
158
212
|
- Model, prompt, chunk, top-k, reranker, or index changes are justified by layer evidence rather than
|
|
159
213
|
by general "RAG quality" claims.
|
|
160
214
|
|
package/templates/default/locales/en/.mustflow/skills/search-index-integrity-review/SKILL.md
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
mustflow_doc: skill.search-index-integrity-review
|
|
3
3
|
locale: en
|
|
4
4
|
canonical: true
|
|
5
|
-
revision:
|
|
5
|
+
revision: 3
|
|
6
6
|
lifecycle: mustflow-owned
|
|
7
7
|
authority: procedure
|
|
8
8
|
name: search-index-integrity-review
|
|
9
|
-
description: Apply this skill when keyword search, full-text search, Elasticsearch, OpenSearch, Lucene-style indexes, search APIs, indexing pipelines, aliases, bulk indexing, refresh visibility, analyzers, mappings, synonyms, autocomplete, pagination, shard failures, search quality, or search performance are created, changed, reviewed, or failing. Use vector-search-integrity-review first for vector-only or semantic retrieval mechanics, and use rag-pipeline-triage first when a RAG failure is not yet localized to search retrieval.
|
|
9
|
+
description: Apply this skill when keyword search, full-text search, Elasticsearch, OpenSearch, Lucene-style indexes, search APIs, indexing pipelines, source maps, metadata taxonomy, aliases, bulk indexing, refresh visibility, analyzers, mappings, synonyms, autocomplete, pagination, shard failures, search quality, or search performance are created, changed, reviewed, or failing. Use vector-search-integrity-review first for vector-only or semantic retrieval mechanics, and use rag-pipeline-triage first when a RAG failure is not yet localized to search retrieval.
|
|
10
10
|
metadata:
|
|
11
11
|
mustflow_schema: "1"
|
|
12
12
|
mustflow_kind: procedure
|
|
@@ -55,7 +55,13 @@ The core question is whether a source record can be changed, accepted by the ind
|
|
|
55
55
|
- Symptom classification: source not indexed, write not visible, wrong alias, partial shard result, wrong exact match, wrong full-text match, ranking drift, zero results, stale delete, tenant leak, autocomplete failure, deep-page failure, slow query, bulk rejection, mapping conflict, or UI/API mismatch.
|
|
56
56
|
- Source-to-search ledger: source id, tenant, category, status, update time, indexing request time, bulk item result, indexed document id, direct lookup result, first search-visible time, rank, delete or tombstone state, and visibility lag.
|
|
57
57
|
- Query contract ledger: user-facing API path, direct search request, API-transformed request, UI result shaping, index or alias, tenant and permission filters, analyzed fields, exact fields, sort, pagination mode, source fields, highlight fields, cache state, and expected result ids.
|
|
58
|
-
- Index contract ledger: read alias, write alias, index templates, mappings, analyzers, normalizers,
|
|
58
|
+
- Index contract ledger: read alias, write alias, index templates, mappings, analyzers, normalizers,
|
|
59
|
+
exact id fields, exact keyword fields, facet fields, filter-only metadata, embedding-header
|
|
60
|
+
metadata, citation metadata, negative metadata, taxonomy fields, controlled vocabulary, metadata
|
|
61
|
+
key budget, stable source id, document id, chunk id when applicable, alias or synonym tables,
|
|
62
|
+
source-map fields, published/effective/indexed/verified dates, version fields, content hash,
|
|
63
|
+
schema version, index version, dynamic mapping policy, refresh policy, shard and replica plan,
|
|
64
|
+
segment or merge state, disk watermark risk, and rollover or reindex status.
|
|
59
65
|
- Quality ledger: representative queries, expected results, acceptable alternatives, zero-result expectations, Precision@K, Recall@K, MRR or ranking metric, before/after comparison, click or behavior metric limitations, and golden-set fixture status.
|
|
60
66
|
- Performance ledger: cold and warm latency, p50, p95, p99, search server time, API time, UI time, query phase, fetch phase, response size, shard fan-out, thread-pool active/queue/rejected, slow-log or query-profile evidence, cache hit or miss, and retry behavior.
|
|
61
67
|
- Privacy ledger: raw query text, document text, tenant ids, user ids, behavior analytics, search logs, highlights, and whether evidence can be stored safely as synthetic fixtures, ids, hashes, summaries, or aggregate metrics.
|
|
@@ -71,7 +77,12 @@ The core question is whether a source record can be changed, accepted by the ind
|
|
|
71
77
|
<!-- mustflow-section: allowed-edits -->
|
|
72
78
|
## Allowed Edits
|
|
73
79
|
|
|
74
|
-
- Add or tighten search canaries, indexing ledgers, bulk item error handling, alias checks, mapping
|
|
80
|
+
- Add or tighten search canaries, indexing ledgers, bulk item error handling, alias checks, mapping
|
|
81
|
+
and analyzer fixtures, metadata taxonomy checks, frontmatter schema checks,
|
|
82
|
+
controlled-vocabulary fixtures, exact-keyword fixtures, negative-metadata filters,
|
|
83
|
+
exact-versus-full-text tests, tenant and permission filter tests, golden-set tests, synonym
|
|
84
|
+
regression tests, pagination guards, query and miss-log metrics, search latency metrics, docs,
|
|
85
|
+
and directly synchronized templates.
|
|
75
86
|
- Add focused synthetic fixtures that encode expected exact search, full-text search, analyzer behavior, synonym behavior, filtered search, tenant isolation, zero-result behavior, pagination stability, and ranking order.
|
|
76
87
|
- Do not change analyzer, synonym, refresh, alias, shard, cache, or ranking settings blindly. First preserve source-to-search, query-contract, quality, and performance evidence.
|
|
77
88
|
- Do not force every write to refresh immediately unless the product contract explicitly needs synchronous visibility and the indexing cost is accepted.
|
|
@@ -108,31 +119,64 @@ The core question is whether a source record can be changed, accepted by the ind
|
|
|
108
119
|
9. Inspect mappings and analyzers before changing queries.
|
|
109
120
|
- IDs, emails, status codes, tags, and exact filters usually need keyword-like fields.
|
|
110
121
|
- Human text usually needs text analysis. Use analyzer evidence or fixtures to prove tokenization for punctuation, case, hyphen, Korean spacing, product codes, and mixed alphanumeric text.
|
|
111
|
-
10. Review
|
|
122
|
+
10. Review metadata taxonomy and controlled vocabulary.
|
|
123
|
+
- Distinguish filter fields, facet fields, ranking fields, display fields, and source-map fields.
|
|
124
|
+
- Canonical keys, aliases, synonyms, old names, and related terms should be explicit so free-form tags do not become five spellings of the same concept.
|
|
125
|
+
- Treat tags as filter conditions, not vibes. Reject values such as important, misc, final, or
|
|
126
|
+
reference unless they have a controlled, queryable meaning.
|
|
127
|
+
- Keep the metadata key set small enough that writers can fill it consistently; separate a
|
|
128
|
+
core filter set from optional enrichment instead of accepting unbounded ad hoc keys.
|
|
129
|
+
11. Split metadata by job.
|
|
130
|
+
- Filter metadata should be typed and exact; context headers should help lexical and semantic
|
|
131
|
+
recall; citation metadata should point back to source spans.
|
|
132
|
+
- Do not put every metadata key into analyzed text. Select title, section, entity, date, type,
|
|
133
|
+
and other disambiguators; keep volatile, private, or filter-only fields out of recall text.
|
|
134
|
+
- Keep LLM-visible metadata separate from search/filter metadata when the index feeds RAG.
|
|
135
|
+
Operational paths, ACL fields, tenant IDs, and internal ids should not become answer text
|
|
136
|
+
unless they change citation, permission, or disambiguation.
|
|
137
|
+
12. Separate path hints from state fields.
|
|
138
|
+
- Paths may encode tenant, visibility, lifecycle, source, language, or date for operations, but
|
|
139
|
+
`status`, `is_current`, `effective_from`, `effective_to`, `published_at`, `indexed_at`,
|
|
140
|
+
`last_verified_at`, `version`, and `revision` should remain first-class filter fields.
|
|
141
|
+
- File names are human hints; stable ids are machine keys. Renames, storage moves, and reindex
|
|
142
|
+
jobs should not break citations, deletes, feedback logs, or eval sets.
|
|
143
|
+
13. Review negative metadata and lifecycle filters.
|
|
144
|
+
- Visibility, audience, jurisdiction exclusions, draft/deprecated state, retention, and security
|
|
145
|
+
class are search correctness gates, not decoration.
|
|
146
|
+
- A good search result that should not have been eligible is a security or product bug.
|
|
147
|
+
14. Preserve source maps and freshness fields.
|
|
148
|
+
- Store document id, chunk id when applicable, file or URL, section anchor, line or page span, content hash, schema or index version, and last indexed time so stale or unsupported results can be traced.
|
|
149
|
+
15. Review exact keyword fields separately from analyzed text.
|
|
150
|
+
- Error codes, API paths, legal references, ticket ids, SKU values, class or function names, and
|
|
151
|
+
deprecated names need exact or keyword search coverage even when semantic retrieval exists.
|
|
152
|
+
16. Review synonym and ranking changes against a golden set.
|
|
112
153
|
- Synonyms can improve one query and break many others.
|
|
113
154
|
- Compare representative queries before and after; include zero-result, typo, ambiguous, category, brand, long natural-language, and high-value queries.
|
|
114
|
-
|
|
155
|
+
17. Use query, miss, and click logs as improvement evidence without leaking raw private text.
|
|
156
|
+
- Track query family, normalized terms, filters, zero-result rate, clicked ids, used chunks, fallback reason, and answer success where safe.
|
|
157
|
+
- Do not turn private queries or highlights into fixtures unless they are redacted or synthetic.
|
|
158
|
+
18. Explain surprising ranks only for bounded cases.
|
|
115
159
|
- Inspect why one expected document ranked below another for a small failing sample.
|
|
116
160
|
- Do not enable expensive explain or profile behavior for broad production traffic.
|
|
117
|
-
|
|
161
|
+
19. Separate query, fetch, API, and UI latency.
|
|
118
162
|
- p50, p95, and p99 must be grouped by search type, index, route, role, tenant class, and query fingerprint where safe.
|
|
119
163
|
- Fetch time, source size, highlight fields, nested fields, and response shaping can dominate score calculation.
|
|
120
|
-
|
|
164
|
+
20. Fingerprint query shapes.
|
|
121
165
|
- Remove raw IDs, dates, user text, and tenant secrets before grouping.
|
|
122
166
|
- Store query family, index, role, shard count, cache state, source fields, sort, and filter shape so one feature cannot hide across millions of unique queries.
|
|
123
|
-
|
|
167
|
+
21. Check shard fan-out, thread pools, segments, cache, and disk.
|
|
124
168
|
- Search across hundreds of tiny shards, stale segments, merge backlog, cold caches, search queue rejections, hot shards, and disk watermarks can look like application bugs.
|
|
125
169
|
- Separate cold and warm measurements; cached benchmarks are not full search evidence.
|
|
126
|
-
|
|
170
|
+
22. Review refresh and visibility policy.
|
|
127
171
|
- Indexing success is not search visibility.
|
|
128
172
|
- Use synchronous visibility only for writes whose user contract needs it; otherwise define acceptable visibility lag and measure it.
|
|
129
|
-
|
|
173
|
+
23. Review pagination and result payloads.
|
|
130
174
|
- Avoid deep offset pagination as a default product contract.
|
|
131
175
|
- Use stable tie-breakers for cursor-like pagination, and limit source fields and highlights to what the UI needs.
|
|
132
|
-
|
|
176
|
+
24. Keep vector and hybrid boundaries explicit.
|
|
133
177
|
- If dense vectors, ANN, reranking, hybrid score fusion, or Recall@K drives the failure, switch to `vector-search-integrity-review` for that slice.
|
|
134
178
|
- If the bug is ordinary keyword indexing, alias, analyzer, source filtering, or shard partials, keep this skill as the primary route.
|
|
135
|
-
|
|
179
|
+
25. Define numeric SLOs and destructive drills when in scope.
|
|
136
180
|
- Useful examples include indexing-to-search-visible p95, search p99, zero-result rate, partial shard failure rate, top-result duplication, golden-set metric, bulk item failure rate, and reindex reconciliation lag.
|
|
137
181
|
- Live node kills, disk pressure, alias mistakes, mapping conflicts, and mass reindex drills are manual unless the command contract declares them.
|
|
138
182
|
|
|
@@ -140,6 +184,10 @@ The core question is whether a source record can be changed, accepted by the ind
|
|
|
140
184
|
## Postconditions
|
|
141
185
|
|
|
142
186
|
- The search symptom, source-to-search ledger, query contract, index contract, quality ledger, performance ledger, and privacy boundary are explicit.
|
|
187
|
+
- Metadata taxonomy, metadata key budget, schema versions, stable source/doc/chunk ids, canonical
|
|
188
|
+
keys, aliases, exact keyword fields, filter-only versus LLM-visible metadata, lifecycle and
|
|
189
|
+
effective-date fields, negative metadata, source maps, index version, content hash, query logs,
|
|
190
|
+
and miss logs are explicit where relevant.
|
|
143
191
|
- Bulk item errors, source/index reconciliation, direct lookup, exact search, full-text search, aliases, partial shards, API/UI transformation, mappings, analyzers, synonyms, ranking, pagination, source payload, shard fan-out, thread-pool pressure, cache state, refresh visibility, segments, disk, and SLO evidence are fixed or reported where relevant.
|
|
144
192
|
- Search claims are backed by configured tests, fixtures, golden-set evidence, static review, safe canary evidence, or labeled manual-only or missing.
|
|
145
193
|
|
|
@@ -173,7 +221,9 @@ Prefer the narrowest configured tests that cover search contract, tenant isolati
|
|
|
173
221
|
|
|
174
222
|
- Search index integrity reviewed
|
|
175
223
|
- Symptom, source-to-search ledger, query contract, index contract, quality ledger, performance ledger, and privacy boundary
|
|
176
|
-
- Bulk, reconciliation, lookup, exact/full-text, alias, shard, API/UI, mapping, analyzer,
|
|
224
|
+
- Bulk, reconciliation, lookup, exact/full-text, alias, shard, API/UI, mapping, analyzer, taxonomy,
|
|
225
|
+
source-map, synonym, ranking, pagination, payload, query-log, shard fan-out, thread-pool, cache,
|
|
226
|
+
refresh, segment, disk, and SLO findings
|
|
177
227
|
- Fix applied or recommended
|
|
178
228
|
- Evidence level: canary evidence, golden-set evidence, configured-test evidence, static review risk, manual-only, missing, or not applicable
|
|
179
229
|
- Command intents run
|
package/templates/default/locales/en/.mustflow/skills/vector-search-integrity-review/SKILL.md
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
mustflow_doc: skill.vector-search-integrity-review
|
|
3
3
|
locale: en
|
|
4
4
|
canonical: true
|
|
5
|
-
revision:
|
|
5
|
+
revision: 4
|
|
6
6
|
lifecycle: mustflow-owned
|
|
7
7
|
authority: procedure
|
|
8
8
|
name: vector-search-integrity-review
|
|
9
|
-
description: Apply this skill when vector search, semantic search, RAG retrieval mechanics, embeddings, vector databases, ANN indexes, exact versus approximate search, filters, metadata payloads, namespaces, tenants, named vectors, hybrid search, reranking, recall, latency, quantization, HNSW, IVF, pgvector, Qdrant, Milvus, Weaviate, OpenSearch kNN, or retrieval golden-set behavior is created, changed, reviewed, or failing. Use rag-pipeline-triage first when a RAG failure is not yet localized to retrieval versus parsing, context assembly, prompt, generation, citation, or answerability.
|
|
9
|
+
description: Apply this skill when vector search, semantic search, RAG retrieval mechanics, embeddings, vector databases, ANN indexes, exact versus approximate search, filters, metadata payloads, chunk text variants, namespaces, tenants, named vectors, hybrid search, reranking, recall, latency, quantization, HNSW, IVF, pgvector, Qdrant, Milvus, Weaviate, OpenSearch kNN, or retrieval golden-set behavior is created, changed, reviewed, or failing. Use rag-pipeline-triage first when a RAG failure is not yet localized to retrieval versus parsing, context assembly, prompt, generation, citation, or answerability.
|
|
10
10
|
metadata:
|
|
11
11
|
mustflow_schema: "1"
|
|
12
12
|
mustflow_kind: procedure
|
|
@@ -68,13 +68,16 @@ filters, reranking, and latency evidence all agree for the same query contract.
|
|
|
68
68
|
- Retrieval symptom classification: ingestion missing, write not visible, wrong results, empty
|
|
69
69
|
results, low recall, tenant leak, duplicate chunks, stale deletes, slow search, reranker drift, or
|
|
70
70
|
generated answer drift.
|
|
71
|
-
- Query contract ledger: query text or safe fixture id, embedding model and revision,
|
|
72
|
-
version,
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
71
|
+
- Query contract ledger: query text or safe fixture id, query family, embedding model and revision,
|
|
72
|
+
preprocessing version, chunker version, contextual header policy, synthetic question policy,
|
|
73
|
+
vector dimension, vector norm, metric, collection, namespace, tenant, named vector, metadata
|
|
74
|
+
payload fields and types, ACL prefilter, filters, `top_k`, candidate count, consistency level,
|
|
75
|
+
ANN parameters, hybrid weights, MMR or fusion settings, and reranker settings.
|
|
76
|
+
- Ingestion ledger: source id, stable doc id, chunk id, parent document genealogy, deterministic
|
|
77
|
+
vector id, original text hash, index text, prompt text, embedding text, contextual header, title,
|
|
78
|
+
heading path, document type, status, authority, aliases, exact keywords, synthetic questions,
|
|
79
|
+
embedding version, content hash, embedding hash, payload shape, write count, unique id count,
|
|
80
|
+
direct lookup count, indexed count, deleted or tombstoned count, and visibility lag.
|
|
78
81
|
- Quality ledger: golden queries, expected ids, acceptable alternatives, exact-search result,
|
|
79
82
|
ANN result, recall at k, MRR, empty rate, duplicate rate, filtered result count, and before/after
|
|
80
83
|
comparison.
|
|
@@ -95,10 +98,12 @@ filters, reranking, and latency evidence all agree for the same query contract.
|
|
|
95
98
|
<!-- mustflow-section: allowed-edits -->
|
|
96
99
|
## Allowed Edits
|
|
97
100
|
|
|
98
|
-
- Add or tighten embedding versioning, preprocessing versioning, vector validation,
|
|
99
|
-
IDs,
|
|
100
|
-
|
|
101
|
-
|
|
101
|
+
- Add or tighten embedding versioning, preprocessing and chunker versioning, vector validation,
|
|
102
|
+
deterministic IDs, contextual retrieval headers, synthetic question fields,
|
|
103
|
+
original/index/prompt/embedding text separation, namespace or tenant selection, metadata payload
|
|
104
|
+
field typing, metadata indexes, ACL prefilter construction, exact-search checks, ANN parameters,
|
|
105
|
+
hybrid score ledgers, RRF or MMR settings, reranker candidate counts, golden-set tests, metrics,
|
|
106
|
+
docs, fixtures, and retrieval contract tests.
|
|
102
107
|
- Add focused synthetic fixtures that encode expected retrieval behavior, filtered retrieval,
|
|
103
108
|
tenant separation, duplicate handling, and exact-versus-ANN comparison.
|
|
104
109
|
- Do not change embedding models, rebuild large indexes, tune ANN parameters, disable filters, widen
|
|
@@ -116,38 +121,56 @@ filters, reranking, and latency evidence all agree for the same query contract.
|
|
|
116
121
|
all-zero vectors, extreme norms, and model or preprocessing version mismatches.
|
|
117
122
|
4. Keep embedding model identity exact. Model name alone is not enough; include revision, tokenizer,
|
|
118
123
|
pooling, prefix policy, max length, HTML cleanup, chunking, and normalization.
|
|
119
|
-
5.
|
|
124
|
+
5. Separate original, index, prompt, and embedding text. Original text is the citation source,
|
|
125
|
+
index text may include titles, aliases, contextual retrieval summaries, and generated questions,
|
|
126
|
+
prompt text is the compact model payload, and embedding text is the vector input. Do not collapse
|
|
127
|
+
these fields unless the product intentionally accepts that tradeoff.
|
|
128
|
+
6. Check chunk graph and recursive retrieval shape. Source ids, parent document ids, parent chunk
|
|
129
|
+
ids, previous and next chunk ids, section paths, summary layers, and contextual retrieval
|
|
130
|
+
headers should be available when small chunks need larger context for answer generation. Stable
|
|
131
|
+
ids must survive renames, storage moves, and reindex jobs so citations, deletes, feedback logs,
|
|
132
|
+
and eval sets keep the same lineage.
|
|
133
|
+
7. Check chunk size and overlap as quality levers, not defaults. Tiny chunks that lose the subject,
|
|
134
|
+
huge chunks that average unrelated meanings, and large overlap that fills top-k with duplicates
|
|
135
|
+
each need separate recall and diversity evidence.
|
|
136
|
+
8. Validate contextual headers and synthetic questions. Headers should disambiguate product,
|
|
137
|
+
version, document status, authority, section, and entity; synthetic questions should reflect real
|
|
138
|
+
user phrasing without being cited as original source. Keep original, enriched retrieval text, and
|
|
139
|
+
LLM-visible metadata distinct so search bait does not become generated evidence.
|
|
140
|
+
9. Compare exact search with approximate search on the same query and filter. If exact search is
|
|
120
141
|
wrong, stop tuning ANN and inspect embeddings, preprocessing, metric, payload, and expectations.
|
|
121
|
-
|
|
142
|
+
10. Compare filter-free search, filter-only count, filtered exact search, and filtered ANN search.
|
|
122
143
|
Empty filtered results often mean filter construction, metadata typing, payload indexing, or
|
|
123
144
|
post-filter candidate loss rather than bad embeddings.
|
|
124
|
-
|
|
145
|
+
11. Confirm collection, namespace, tenant, alias, and named vector. Directly inspect the problem id
|
|
125
146
|
or safe synthetic id in the same search surface the app uses.
|
|
126
|
-
|
|
147
|
+
12. Check deterministic upsert ids. Source id, chunk index, tenant, and embedding version should not
|
|
127
148
|
accidentally duplicate chunks or overwrite all chunks with one id.
|
|
128
|
-
|
|
149
|
+
13. Separate write success from search visibility. Check direct lookup, exact search, ANN search,
|
|
129
150
|
indexed count, consistency, segment state, and visibility lag.
|
|
130
|
-
|
|
151
|
+
14. Check deletes and updates. Deletion marks, tombstones, compaction, vacuum, stale segments, and
|
|
131
152
|
old chunks can keep appearing after API success.
|
|
132
|
-
|
|
153
|
+
15. Review metric and normalization. Cosine, dot product, and L2 are different contracts unless
|
|
133
154
|
the vectors are intentionally normalized and the index operator matches.
|
|
134
|
-
|
|
155
|
+
16. Tune ANN only after the exact and filter contracts are correct. Evaluate recall and p95 latency
|
|
135
156
|
across parameter sweeps instead of changing one value blindly.
|
|
136
|
-
|
|
157
|
+
17. Treat build-time index parameters as recall ceilings. If search-time parameters cannot recover
|
|
137
158
|
recall, the index may need rebuild policy, not a larger query knob.
|
|
138
|
-
|
|
159
|
+
18. Review quantization and rescoring. Compare full-precision exact results with compressed-index
|
|
139
160
|
candidates and confirm enough candidates reach full-precision reranking.
|
|
140
|
-
|
|
141
|
-
and final score. Do not add incompatible raw scores directly
|
|
142
|
-
|
|
161
|
+
19. For hybrid search, store dense score, sparse or keyword score, normalized score, fusion method,
|
|
162
|
+
RRF or MMR settings when used, and final score. Do not add incompatible raw scores directly
|
|
163
|
+
without a deliberate combiner. Preserve exact product codes, API names, error codes, ticket IDs,
|
|
164
|
+
and dates in lexical fields instead of smoothing them into natural-language embedding text.
|
|
165
|
+
20. For reranking, record pre-rerank and post-rerank ids and ranks. If the right document never
|
|
143
166
|
enters the candidate set, the reranker cannot recover it.
|
|
144
|
-
|
|
167
|
+
21. Avoid deep ANN pagination as a product contract. Use cursor, filters, grouping, or ordinary
|
|
145
168
|
sorted indexes for deep browsing instead of pretending vector search has cheap random offsets.
|
|
146
|
-
|
|
169
|
+
22. Split cold and warm latency, server search time and client wait, vector DB time and reranker
|
|
147
170
|
time, and single-query latency from concurrent load.
|
|
148
|
-
|
|
171
|
+
23. Inspect shard, replica, segment, compaction, flush, indexing, memory, disk, and cache state when
|
|
149
172
|
p99 or intermittent quality varies across nodes.
|
|
150
|
-
|
|
173
|
+
24. Add a golden-set gate when the project has a test surface. Include easy, hard, filtered,
|
|
151
174
|
tenant-scoped, rare-name, synonym, short-query, long-query, and sparse-data cases.
|
|
152
175
|
|
|
153
176
|
<!-- mustflow-section: postconditions -->
|
|
@@ -155,6 +178,10 @@ filters, reranking, and latency evidence all agree for the same query contract.
|
|
|
155
178
|
|
|
156
179
|
- The retrieval symptom, query contract, ingestion contract, quality ledger, performance ledger, and
|
|
157
180
|
privacy boundary are explicit.
|
|
181
|
+
- Original/index/prompt/embedding text separation, stable source/doc/chunk ids, parent document
|
|
182
|
+
genealogy, payload field typing, ACL prefiltering, contextual headers, synthetic questions,
|
|
183
|
+
chunk graph, content hashes, embedding hashes, exact lexical safeguards, and hybrid or rerank
|
|
184
|
+
ledgers are explicit where relevant.
|
|
158
185
|
- Exact search, ANN, filters, metadata, namespaces, tenants, named vectors, IDs, deletes, metric,
|
|
159
186
|
normalization, quantization, hybrid search, reranking, shards, consistency, and latency are fixed
|
|
160
187
|
or reported where relevant.
|
|
@@ -199,8 +226,8 @@ of inventing live diagnostics.
|
|
|
199
226
|
- Vector search integrity reviewed
|
|
200
227
|
- Retrieval symptom, query contract, ingestion ledger, quality ledger, performance ledger, and
|
|
201
228
|
privacy boundary
|
|
202
|
-
- Exact versus ANN, filter, metadata, namespace, tenant, id, delete, metric,
|
|
203
|
-
quantization, hybrid, reranker, shard, consistency, and latency findings
|
|
229
|
+
- Exact versus ANN, filter, metadata, text-variant, namespace, tenant, id, delete, metric,
|
|
230
|
+
normalization, quantization, hybrid, MMR, reranker, shard, consistency, and latency findings
|
|
204
231
|
- Fix applied or recommended
|
|
205
232
|
- Evidence level: golden-set evidence, configured-test evidence, static review risk, manual-only,
|
|
206
233
|
missing, or not applicable
|