@zivis/mcp 0.1.8 → 0.1.10

package/dist/pattern-packs/zivis-public-0.2.0/capsules/prompting/self-consistency-no-isolation.yaml

@@ -0,0 +1,237 @@
+# Capsule: Self-consistency without context isolation
+# Spec: docs/specs/PATTERN-MATCHING-LIBRARY-V1.md
+# Plan: docs/plans/LOCAL-REPO-GRAPH-IMPLEMENTATION.md
+# Schema: schemas/pattern-detection-card.schema.json
+
+id: prompting.self-consistency-no-isolation
+slug: self-consistency-no-isolation
+version: 0.1.0
+category: prompting
+title: Self-Consistency Without Context Isolation
+headline: Vote Without Independence
+description: >
+  Detects code that aggregates multiple LLM samples (vote, consensus, ranking)
+  while every sample shares the same untrusted context. Majority voting only
+  filters random noise — when context is shared, correlated errors override
+  correctness and the vote count is mis-read as calibrated confidence.
+
+detection_tier: public
+scoring_tier: proprietary
+
+applicable_languages:
+  - typescript
+  - javascript
+  - python
+
+relevance_filter:
+  any_of:
+    - dependency: openai
+      direct_only: true
+    - dependency: "@anthropic-ai/sdk"
+      direct_only: true
+    - dependency: anthropic
+      direct_only: true
+    - dependency: "ai"
+      direct_only: true
+    - dependency: langchain
+      direct_only: true
+    - dependency: llama-index
+      direct_only: true
+
+strategies:
+  - code: AST
+    language: python
+    expression: |
+      rule:
+        kind: for_statement
+        has:
+          kind: call
+          any:
+            - pattern: $CLIENT.chat.completions.create($$$)
+            - pattern: $CLIENT.messages.create($$$)
+            - pattern: openai.ChatCompletion.create($$$)
+    evidence_fields: [file_path, line_span, symbol]
+    confidence_weight: 0.30
+
+  - code: AST
+    language: typescript
+    expression: |
+      rule:
+        any:
+          - pattern: $_.chat.completions.create($$$)
+          - pattern: generateText($$$)
+          - pattern: streamText($$$)
+        inside:
+          stopBy: end
+          any:
+            - pattern: Promise.all($$$)
+            - pattern: $X.map($$$)
+            - kind: for_statement
+            - kind: while_statement
+    evidence_fields: [file_path, line_span, symbol]
+    confidence_weight: 0.30
+
+  - code: GRAPH
+    expression: |
+      same_messages_argument_passed_to_n_completion_calls(n >= 2)
+      AND aggregator_observed(any_of: Counter, max(set(.), key=.count), majority_vote, np.argmax, statistics.mode)
+    evidence_fields: [graph_path, file_spans, node_ids]
+    confidence_weight: 0.30
+
+  - code: META
+    expression:
+      none_of_controls:
+        - per_sample_retrieval_diversification
+        - external_validator_on_facts
+        - human_approval_gate
+    evidence_fields: [missing_controls]
+    confidence_weight: 0.25
+
+  - code: INFER
+    expression: prompt_id:prompting.self-consistency-no-isolation.identify
+    evidence_fields: [llm_rationale]
+    confidence_weight: 0.15
+    trusted: false
+
+required_evidence_count: 2
+
+control_negations:
+  - "presence_of(per_sample_retrieval_diversification)"
+  - "presence_of(external_validator_on_facts)"
+  - "presence_of(human_approval_gate)"
+
+threats:
+  - title: Vote count mistaken for calibrated confidence
+    stride: [tampering, repudiation]
+    description: >
+      The application treats agreement across samples as independent confirmation,
+      when in fact every sample saw the same poisoned retrieval context or the
+      same prompt-injection payload. Downstream actions (auto-approve, send,
+      execute) fire under false high-confidence signal.
+    remediation: >
+      Diversify per-sample retrieval queries, temperatures, or seeds.
+      Add an external fact-validator. For high-impact outputs, require a human
+      approval gate. Never expose vote count as a confidence number to users
+      without disclosing the independence assumption.
+  - title: Correlated injection amplification
+    stride: [tampering]
+    description: >
+      A single prompt-injection payload in shared retrieved context biases all N
+      samples identically; majority vote then ratifies the attack rather than
+      filtering it.
+    remediation: >
+      Per-sample context isolation. Run independent retrieval (different queries
+      or different stores) per sample so an injection only contaminates a subset.
+
+fixtures:
+  positive:
+    - "fixtures/positive/self-consistency-no-isolation/openai_loop_vote_python"
+    - "fixtures/positive/self-consistency-no-isolation/anthropic_promise_all_typescript"
+  negative:
+    - "fixtures/negative/self-consistency-no-isolation/per_sample_diverse_retrieval"
+    - "fixtures/negative/self-consistency-no-isolation/single_completion_no_aggregation"
+
+lifecycle: draft
+owner: platform-threat-modeling
+tags:
+  - prompting
+  - self-consistency
+  - context-isolation
+  - prompt-injection-amplification
+
+sensitivity_level: customer_safe
+allowed_execution_modes:
+  - local_only
+  - local_model
+  - hybrid_cloud_eval
+
+safe_summary: >
+  Multiple LLM completions are aggregated by voting or consensus, but every
+  sample sees the same retrieved context or the same user message. Independence
+  is assumed and not enforced. A single prompt-injection or a single
+  hallucinated retrieval contaminates the vote uniformly; majority count is
+  mistaken for calibrated confidence and downstream actions fire under
+  false certainty.
+
+detect_when: >
+  Two or more LLM completions for the same user task are merged by voting,
+  consensus, or similarity scoring without independent context isolation —
+  no per-sample retrieval diversification, no external validator, no human gate.
+
+risk_hints:
+  - Shared retrieval/injection in the user message affects every sample the same way
+  - Majority vote amplifies systematic bias rather than filtering noise
+  - Vote count surfaced as a confidence number is a UX lie when context is shared
+  - Async Promise.all with the same messages array is the most common JS shape
+
+architectural_signals:
+  - Same `messages` argument passed to N completion calls inside a loop or Promise.all
+  - majority_vote, Counter, np.argmax, statistics.mode applied to the result array
+  - Single retrieval call upstream feeding all samples
+
+model_task_prompt_ref: prompts/prompting/self-consistency-no-isolation/identify.md
+
+expected_output_schema:
+  type: object
+  required: [independence_assessment, confidence_claim_risk, evidence_spans]
+  properties:
+    independence_assessment:
+      type: string
+      enum: [shared_context, isolated_context, unclear]
+    confidence_claim_risk:
+      type: string
+      enum: [high, medium, low]
+      description: How dangerous is the confidence claim users see, given the (lack of) independence
+    rationale:
+      type: string
+    recommended_controls:
+      type: array
+      items:
+        type: string
+        enum: [per_sample_retrieval_diversification, external_validator, human_approval, temperature_variation, seed_variation]
+    evidence_spans:
+      type: array
+      items:
+        type: object
+        required: [file]
+        properties:
+          file: { type: string }
+          start_line: { type: integer }
+          end_line: { type: integer }
+          note: { type: string }
+
+minimum_context_required:
+  lines: 80
+  symbols:
+    - chat.completions.create
+    - messages.create
+    - generateText
+    - Counter
+    - majority
+    - np.argmax
+
+related_patterns:
+  - prompting.chain-of-thought-leakage
+  - retrieval.shared-context-poisoning
+
+repair_contract:
+  allowed:
+    - Diversify retrieval queries per sample
+    - Vary temperature or seed across samples
+    - Add an external fact validator on aggregated output
+    - Add a human approval gate before high-impact actions
+  forbidden:
+    - Silently treating vote count as calibrated confidence
+    - Removing the aggregation entirely for performance reasons (does not address the architectural issue)
+    - Adding more samples without addressing context independence
+
+validation_contract:
+  requires_tests_pass: false
+  requires_static_rules:
+    - aggregator_remains_present_after_patch
+    - independence_mechanism_introduced
+
+unsafe_to_expose_fields:
+  - internal_rubric
+  - golden_transcripts
+  - exploit_templates

package/dist/pattern-packs/zivis-public-0.2.0/capsules/retrieval/context-injection-no-validator.yaml

@@ -0,0 +1,348 @@
+# Capsule: RAG context injection — retrieved content reaches the model without validation
+# Spec: docs/specs/PATTERN-MATCHING-LIBRARY-V1.md
+# Plan: docs/plans/LOCAL-REPO-GRAPH-IMPLEMENTATION.md
+# Schema: schemas/pattern-detection-card.schema.json
+
+id: retrieval.context-injection-no-validator
+slug: context-injection-no-validator
+version: 0.1.0
+category: retrieval
+title: Retrieval Context Injection — No Validator Between Retriever and Model
+headline: Retrieved Content Flows Into the LLM With Nothing Inspecting It First
+
+description: >
+  Detects retrieval-augmented generation (RAG) flows where the output of a
+  retriever (vector store, search index, document loader) is passed directly
+  into the model call with no fact-validator, source-attribution check, or
+  content sanitizer in between. Retrieval becomes a transparent injection
+  channel: anyone who can write to the knowledge base — directly, via a
+  scraping job, or via a tool the agent itself calls — can steer the model.
+
+detection_tier: public
+scoring_tier: proprietary
+
+applicable_languages:
+  - typescript
+  - javascript
+  - python
+
+relevance_filter:
+  any_of:
+    # LangChain / LlamaIndex RAG stacks
+    - dependency: langchain
+      direct_only: true
+    - dependency: "@langchain/core"
+      direct_only: true
+    - dependency: "@langchain/openai"
+      direct_only: true
+    - dependency: "@langchain/anthropic"
+      direct_only: true
+    - dependency: "@langchain/community"
+      direct_only: true
+    - dependency: llamaindex
+      direct_only: true
+    - dependency: "llama-index"
+      direct_only: true
+    # Manual RAG: any LLM SDK + a vector store / retrieval client
+    - dependency: openai
+      direct_only: true
+    - dependency: "@anthropic-ai/sdk"
+      direct_only: true
+    - dependency: anthropic
+      direct_only: true
+    - dependency: "@pinecone-database/pinecone"
+      direct_only: true
+    - dependency: pinecone-client
+      direct_only: true
+    - dependency: "@qdrant/js-client-rest"
+      direct_only: true
+    - dependency: qdrant-client
+      direct_only: true
+    - dependency: weaviate-ts-client
+      direct_only: true
+    - dependency: weaviate-client
+      direct_only: true
+    - dependency: chromadb
+      direct_only: true
+    - dependency: "@elastic/elasticsearch"
+      direct_only: true
+    - dependency: elasticsearch
+
+strategies:
+  # AST as architectural evidence (NOT a vulnerability claim) — does this
+  # codebase even have a retrieval→LLM call shape? If it doesn't, the
+  # pattern can't apply.
+  - code: AST
+    language: typescript
+    expression: |
+      rule:
+        any:
+          # LangChain idioms
+          - pattern: $RETRIEVER.invoke($$$)
+          - pattern: $RETRIEVER.getRelevantDocuments($$$)
+          - pattern: new RetrievalQAChain($$$)
+          - pattern: RetrievalQAChain.fromLLM($$$)
+          - pattern: createRetrievalChain($$$)
+          # LlamaIndex
+          - pattern: $INDEX.asQueryEngine($$$)
+          - pattern: $QUERY_ENGINE.query($$$)
+          # Manual RAG: vector-store similarity search feeding a model call
+          - pattern: $STORE.similaritySearch($$$)
+          - pattern: $STORE.similaritySearchWithScore($$$)
+    evidence_fields: [file_path, line_span, symbol]
+    confidence_weight: 0.40
+
+  - code: AST
+    language: python
+    expression: |
+      rule:
+        any:
+          # LangChain
+          - pattern: $RETRIEVER.invoke($$$)
+          - pattern: $RETRIEVER.get_relevant_documents($$$)
+          - pattern: RetrievalQA.from_chain_type($$$)
+          - pattern: create_retrieval_chain($$$)
+          # LlamaIndex
+          - pattern: $INDEX.as_query_engine($$$)
+          - pattern: $QUERY_ENGINE.query($$$)
+          # Manual RAG: vector-store similarity search
+          - pattern: $STORE.similarity_search($$$)
+          - pattern: $STORE.similarity_search_with_score($$$)
+    evidence_fields: [file_path, line_span, symbol]
+    confidence_weight: 0.40
+
+  # GRAPH strategy — forward-compatible. Matcher v1 doesn't evaluate
+  # graph predicates yet (per Phase 3 v1 in LOCAL-REPO-GRAPH-IMPLEMENTATION),
+  # but writing it now means the card "lights up" when the matcher catches
+  # up, without re-authoring.
+  - code: GRAPH
+    expression: |
+      retrieval_node_returns_documents
+      AND llm_call_node_consumes_documents
+      AND NO_NODE_BETWEEN(retrieval_node, llm_call_node) matches
+        (validator | fact_checker | source_attribution | content_sanitizer)
+    evidence_fields: [graph_path, file_spans, node_ids]
+    confidence_weight: 0.30
+
+  # META — does the codebase declare any of the controls that would make
+  # this pattern safe? Absence is what we care about.
+  - code: META
+    expression:
+      none_of_controls:
+        - fact_validator_present
+        - source_attribution_present
+    evidence_fields: [missing_controls]
+    confidence_weight: 0.30
+
+  # INFER — the host LLM (Claude in the IDE) gets the actual source spans
+  # and decides whether what looks like RAG truly has no validator. This is
+  # the lane we want most matches to land in: ZIVIS surfaces the candidate,
+  # the frontier model validates against the code.
+  - code: INFER
+    expression: prompt_id:retrieval.context-injection-no-validator.identify
+    evidence_fields: [llm_rationale]
+    confidence_weight: 0.20
+    trusted: false
+
+# Two deterministic strategies must fire for a trusted-lane match. AST
+# alone won't promote — it's just "there is RAG-shaped code." AST + META
+# (controls absent) WILL promote. GRAPH lights up later for free.
+required_evidence_count: 2
+
+control_negations:
+  - "presence_of(fact_validator_present)"
+  - "presence_of(source_attribution_present)"
+
+threats:
+  - title: Indirect prompt injection via retrieved corpus
+    stride: [tampering, elevation_of_privilege]
+    description: >
+      An attacker who can write to the retrieval corpus — a public web page
+      indexed by a scraper, a Confluence space synced into the vector store,
+      a customer-uploaded document, or a tool the agent uses to fetch
+      content — embeds instructions that the model treats as authoritative
+      because they arrive inside the "trusted" retrieval channel. The model
+      executes those instructions or leaks data based on them. With no
+      validator inspecting the retrieved text, the system has no defense
+      between the corpus and the model.
+    remediation: >
+      Insert a validation step between retriever and model that (a) checks
+      retrieved content for prompt-injection patterns, (b) requires
+      source-attribution metadata before content can influence answers, and
+      (c) rate-limits or content-flags low-trust sources. Patterns like
+      guardrails-ai, NeMo Guardrails, Lakera Guard, or a custom
+      `validateRetrievedContent()` boundary are all valid; the requirement
+      is that *some* boundary exists between corpus and model.
+
+  - title: Sensitive-data exfiltration via crafted retrieval queries
+    stride: [information_disclosure]
+    description: >
+      An attacker who can influence the retrieval query — directly or via a
+      preceding LLM step that re-writes the query — pulls documents the
+      requesting user should not see (other tenants' data, internal-only
+      pages indexed by mistake, PII-laden support tickets). The model then
+      surfaces that content in the response. Without source-attribution and
+      access checks at the retrieval boundary, the model becomes an
+      authorization bypass.
+    remediation: >
+      Enforce access control at the retrieval layer, not just at the API
+      layer. Tag every indexed document with the access scope it inherits
+      from its source system, and filter retrieval by the requesting
+      user's scope before content reaches the model. Treat the retriever
+      as a privileged data path — same scrutiny as any other query.
+
+  - title: Long-running drift from poisoned corpus
+    stride: [tampering, repudiation]
+    description: >
+      A single poisoned document indexed months ago continues to surface in
+      relevant queries indefinitely. The model's behavior drifts in ways
+      that are hard to attribute to a specific bug because the cause is in
+      the corpus, not the code. With no audit log of which retrieved
+      documents influenced which responses, post-incident investigation
+      cannot trace bad behavior back to its source.
+    remediation: >
+      Log every retrieval — query, returned document IDs, document hashes,
+      source attribution — alongside the LLM call it fed. Periodically
+      revalidate corpus contents against current trust assumptions
+      (re-classify documents from sources that have been compromised or
+      whose access policy changed).
+
+fixtures:
+  positive:
+    - "fixtures/positive/context-injection-no-validator/langchain_rag_chain_no_validator_typescript"
+    - "fixtures/positive/context-injection-no-validator/llamaindex_query_engine_no_validator_python"
+    - "fixtures/positive/context-injection-no-validator/manual_pinecone_to_openai_typescript"
+  negative:
+    - "fixtures/negative/context-injection-no-validator/guardrails_validator_present_python"
+    - "fixtures/negative/context-injection-no-validator/source_attribution_check_typescript"
+
+lifecycle: draft
+owner: platform-threat-modeling
+tags:
+  - retrieval
+  - rag
+  - prompt-injection
+  - context-injection
+  - corpus-poisoning
+  - agentic-ai
+
+sensitivity_level: customer_safe
+allowed_execution_modes:
+  - local_only
+  - local_model
+  - hybrid_cloud_eval
+
+safe_summary: >
+  The codebase has a retrieval-augmented generation (RAG) shape — content
+  is pulled from a vector store, search index, or document loader and
+  passed to the LLM — but there's no validator, sanitizer, or
+  source-attribution check between the retriever and the model. Anyone
+  who can write to the retrieval corpus can steer the model: an indexed
+  web page, a synced Confluence space, a customer-uploaded document, or
+  a tool the agent itself fetches from. The fix is to put a boundary
+  between corpus and model — a guardrails library, a custom validator,
+  or per-document source-attribution that the model is required to
+  surface in its answers.
+
+detect_when: >
+  A retriever call (`retriever.invoke`, `similaritySearch`, query engine,
+  retrieval chain) is followed by an LLM call within the same handler or
+  function, and the path between them contains no recognizable validator
+  or source-attribution step. Most common in chat / Q&A / "ask the docs"
+  applications.
+
+risk_hints:
+  - "Retrieval is a privileged data path — anything in the corpus is implicitly trusted by the model"
+  - "Prompt injection embedded in a web page or shared doc survives indefinitely until the document is re-indexed"
+  - "Query rewriting (HyDE, multi-query) compounds the problem — the rewritten query is itself an LLM output"
+  - "Access control at the API layer doesn't help if the retriever pulls from a corpus indexed under different rules"
+  - "Logging the LLM input/output without logging the retrieved documents leaves the actual injection invisible"
+
+architectural_signals:
+  - "Retriever / vector store / search index call returns documents"
+  - "Documents are concatenated into a system or user prompt slot"
+  - "No intermediate function whose name suggests validation, sanitization, or fact-checking"
+  - "No source-attribution requirement in the prompt template"
+
+model_task_prompt_ref: prompts/retrieval/context-injection-no-validator/identify.md
+
+expected_output_schema:
+  type: object
+  required: [verdict, validator_location, evidence_spans]
+  properties:
+    verdict:
+      type: string
+      enum: [no_validator_present, validator_present, partial_validator, unclear]
+    validator_location:
+      type: string
+      enum: [pre_retrieval, between_retriever_and_model, post_model, none, multiple]
+    rationale:
+      type: string
+    corpus_sources:
+      type: array
+      description: "Best-effort enumeration of where the retrieved content comes from"
+      items:
+        type: string
+    blast_radius:
+      type: string
+      enum: [single_endpoint, route_family, service_wide, agent_wide, unclear]
+    recommended_controls:
+      type: array
+      items:
+        type: string
+        enum:
+          - guardrails_library
+          - custom_content_validator
+          - source_attribution_in_prompt
+          - per_document_access_control
+          - retrieval_audit_log
+          - none
+    evidence_spans:
+      type: array
+      items:
+        type: object
+        required: [file]
+        properties:
+          file: { type: string }
+          start_line: { type: integer }
+          end_line: { type: integer }
+          note: { type: string }
+
+minimum_context_required:
+  lines: 120
+  symbols:
+    - retriever
+    - invoke
+    - similarity_search
+    - similaritySearch
+    - query_engine
+    - asQueryEngine
+    - RetrievalQA
+    - create_retrieval_chain
+
+related_patterns:
+  - prompting.user-input-in-system-prompt
+  - agents.tool-dispatch-from-llm-output
+
+repair_contract:
+  allowed:
+    - "Insert a validator step between retriever and model call"
+    - "Add source-attribution requirement to the prompt template"
+    - "Adopt guardrails-ai / NeMo Guardrails / Lakera Guard or equivalent"
+    - "Implement per-document access control at the retrieval boundary"
+    - "Add structured logging of retrieved-document IDs alongside LLM calls"
+  forbidden:
+    - "Removing retrieval entirely without first replacing the answering path"
+    - "Validating only LLM output instead of retrieved content (treats the wrong stage)"
+    - "Hand-rolling regex injection filters as the sole defense (well-known to be bypassable)"
+
+validation_contract:
+  requires_tests_pass: false
+  requires_static_rules:
+    - validator_node_present_between_retrieval_and_llm
+    - retrieval_audit_log_present
+
+unsafe_to_expose_fields:
+  - internal_rubric
+  - golden_transcripts
+  - exploit_templates