@pentatonic-ai/ai-agent-sdk 0.9.5 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +3 -3
  2. package/bin/cli.js +1 -1
  3. package/bin/commands/config.js +1 -1
  4. package/dist/index.cjs +39 -72
  5. package/dist/index.js +36 -69
  6. package/package.json +10 -3
  7. package/packages/doctor/src/checks/local-memory.js +2 -2
  8. package/packages/memory/README.md +2 -2
  9. package/packages/memory/openclaw-plugin/README.md +2 -2
  10. package/packages/memory/openclaw-plugin/openclaw.plugin.json +1 -1
  11. package/packages/memory/package-lock.json +49 -33
  12. package/packages/memory/package.json +4 -1
  13. package/packages/memory/src/__tests__/engine.test.js +40 -5
  14. package/packages/memory/src/engine.js +38 -3
  15. package/packages/memory/src/server.js +2 -2
  16. package/packages/memory-engine-v2/.env.example +30 -0
  17. package/packages/memory-engine-v2/README.md +125 -0
  18. package/packages/memory-engine-v2/compat/Dockerfile +11 -0
  19. package/packages/memory-engine-v2/compat/requirements.txt +6 -0
  20. package/packages/memory-engine-v2/compat/server.py +1047 -0
  21. package/packages/memory-engine-v2/docker-compose.aws.yml +78 -0
  22. package/packages/memory-engine-v2/docker-compose.yml +206 -0
  23. package/packages/memory-engine-v2/extractor-async/Dockerfile +14 -0
  24. package/packages/memory-engine-v2/extractor-async/confidence.py +62 -0
  25. package/packages/memory-engine-v2/extractor-async/noise_filter.py +144 -0
  26. package/packages/memory-engine-v2/extractor-async/requirements.txt +2 -0
  27. package/packages/memory-engine-v2/extractor-async/test_confidence.py +76 -0
  28. package/packages/memory-engine-v2/extractor-async/test_noise_filter.py +177 -0
  29. package/packages/memory-engine-v2/extractor-async/worker.py +797 -0
  30. package/packages/memory-engine-v2/extractor-sync/Dockerfile +11 -0
  31. package/packages/memory-engine-v2/extractor-sync/requirements.txt +4 -0
  32. package/packages/memory-engine-v2/extractor-sync/server.py +424 -0
  33. package/packages/memory-engine-v2/org-model/migrations/001_init.sql +390 -0
  34. package/packages/memory-engine-v2/tests/e2e_smoke.py +356 -0
  35. package/packages/memory-engine-v2/tests/fixtures/generate_synthetic_corpus.py +758 -0
  36. package/packages/memory-engine/.env.example +0 -13
  37. package/packages/memory-engine/MIGRATION.md +0 -219
  38. package/packages/memory-engine/README.md +0 -145
  39. package/packages/memory-engine/bench/README.md +0 -99
  40. package/packages/memory-engine/bench/scorecards-engine/agent-coding__pentatonic-baseline__20260427-142523.json +0 -1115
  41. package/packages/memory-engine/bench/scorecards-engine/chat-recall__pentatonic-baseline__20260427-142648.json +0 -819
  42. package/packages/memory-engine/bench/scorecards-engine/circular-economy__pentatonic-baseline__20260427-142757.json +0 -1278
  43. package/packages/memory-engine/bench/scorecards-engine/customer-support__pentatonic-baseline__20260427-142900.json +0 -1018
  44. package/packages/memory-engine/bench/scorecards-engine/marketplace-ops__pentatonic-baseline__20260427-142957.json +0 -1038
  45. package/packages/memory-engine/bench/scorecards-engine/product-catalogue__pentatonic-baseline__20260427-143122.json +0 -961
  46. package/packages/memory-engine/bench/scorecards-engine-via-docker/agent-coding__pentatonic-memory__20260427-161812.json +0 -1115
  47. package/packages/memory-engine/bench/scorecards-engine-via-docker/chat-recall__pentatonic-memory__20260427-161701.json +0 -819
  48. package/packages/memory-engine/bench/scorecards-engine-via-docker/circular-economy__pentatonic-memory__20260427-161713.json +0 -1278
  49. package/packages/memory-engine/bench/scorecards-engine-via-docker/customer-support__pentatonic-memory__20260427-161723.json +0 -1018
  50. package/packages/memory-engine/bench/scorecards-engine-via-docker/marketplace-ops__pentatonic-memory__20260427-161732.json +0 -1038
  51. package/packages/memory-engine/bench/scorecards-engine-via-docker/product-catalogue__pentatonic-memory__20260427-161741.json +0 -937
  52. package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/agent-coding__pentatonic-memory__20260427-184718.json +0 -1115
  53. package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/chat-recall__pentatonic-memory__20260427-184614.json +0 -819
  54. package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/circular-economy__pentatonic-memory__20260427-184809.json +0 -1278
  55. package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/customer-support__pentatonic-memory__20260427-184854.json +0 -1018
  56. package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/marketplace-ops__pentatonic-memory__20260427-184929.json +0 -1038
  57. package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/product-catalogue__pentatonic-memory__20260427-185015.json +0 -961
  58. package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/agent-coding__pentatonic-memory__20260427-175252.json +0 -1115
  59. package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/chat-recall__pentatonic-memory__20260427-175312.json +0 -819
  60. package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/circular-economy__pentatonic-memory__20260427-175335.json +0 -1278
  61. package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/customer-support__pentatonic-memory__20260427-175355.json +0 -1018
  62. package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/marketplace-ops__pentatonic-memory__20260427-175413.json +0 -1038
  63. package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/product-catalogue__pentatonic-memory__20260427-175430.json +0 -883
  64. package/packages/memory-engine/bench/scorecards-engine-via-shim/agent-coding__pentatonic-memory__20260427-155409.json +0 -1115
  65. package/packages/memory-engine/bench/scorecards-engine-via-shim/chat-recall__pentatonic-memory__20260427-155421.json +0 -819
  66. package/packages/memory-engine/bench/scorecards-engine-via-shim/circular-economy__pentatonic-memory__20260427-155433.json +0 -1278
  67. package/packages/memory-engine/bench/scorecards-engine-via-shim/customer-support__pentatonic-memory__20260427-155443.json +0 -1018
  68. package/packages/memory-engine/bench/scorecards-engine-via-shim/marketplace-ops__pentatonic-memory__20260427-155453.json +0 -1038
  69. package/packages/memory-engine/bench/scorecards-engine-via-shim/product-catalogue__pentatonic-memory__20260427-155503.json +0 -937
  70. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/agent-coding__pentatonic-memory-latest__20260427-145103.json +0 -1115
  71. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/agent-coding__pentatonic-memory__20260427-144909.json +0 -1115
  72. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/chat-recall__pentatonic-memory-latest__20260427-145153.json +0 -819
  73. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/chat-recall__pentatonic-memory__20260427-145120.json +0 -542
  74. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/circular-economy__pentatonic-memory-latest__20260427-145313.json +0 -1278
  75. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/circular-economy__pentatonic-memory__20260427-145207.json +0 -894
  76. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/customer-support__pentatonic-memory-latest__20260427-145412.json +0 -1018
  77. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/customer-support__pentatonic-memory__20260427-145327.json +0 -680
  78. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/marketplace-ops__pentatonic-memory-latest__20260427-145517.json +0 -1038
  79. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/marketplace-ops__pentatonic-memory__20260427-145422.json +0 -693
  80. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/product-catalogue__pentatonic-memory-latest__20260427-145616.json +0 -961
  81. package/packages/memory-engine/bench/scorecards-pentatonic-baseline/product-catalogue__pentatonic-memory__20260427-145528.json +0 -727
  82. package/packages/memory-engine/compat/Dockerfile +0 -22
  83. package/packages/memory-engine/compat/server.py +0 -1255
  84. package/packages/memory-engine/docker-compose.test.yml +0 -59
  85. package/packages/memory-engine/docker-compose.yml +0 -240
  86. package/packages/memory-engine/engine/README.md +0 -52
  87. package/packages/memory-engine/engine/l2-hybridrag-proxy.py +0 -1543
  88. package/packages/memory-engine/engine/l5-comms-layer.py +0 -663
  89. package/packages/memory-engine/engine/l6-document-store.py +0 -1018
  90. package/packages/memory-engine/engine/services/_shared/__init__.py +0 -1
  91. package/packages/memory-engine/engine/services/_shared/embed_provider.py +0 -468
  92. package/packages/memory-engine/engine/services/l2/Dockerfile +0 -50
  93. package/packages/memory-engine/engine/services/l2/init_databases.py +0 -81
  94. package/packages/memory-engine/engine/services/l2/l2-hybridrag-proxy.py +0 -2721
  95. package/packages/memory-engine/engine/services/l5/Dockerfile +0 -11
  96. package/packages/memory-engine/engine/services/l5/l5-comms-layer.py +0 -808
  97. package/packages/memory-engine/engine/services/l6/Dockerfile +0 -30
  98. package/packages/memory-engine/engine/services/l6/l6-document-store.py +0 -1221
  99. package/packages/memory-engine/engine/services/nv-embed/Dockerfile +0 -28
  100. package/packages/memory-engine/engine/services/nv-embed/server.py +0 -152
  101. package/packages/memory-engine/pme_memory/__init__.py +0 -0
  102. package/packages/memory-engine/pme_memory/__main__.py +0 -129
  103. package/packages/memory-engine/pme_memory/artifacts.py +0 -95
  104. package/packages/memory-engine/pme_memory/embed.py +0 -74
  105. package/packages/memory-engine/pme_memory/health.py +0 -36
  106. package/packages/memory-engine/pme_memory/hygiene.py +0 -159
  107. package/packages/memory-engine/pme_memory/indexer.py +0 -200
  108. package/packages/memory-engine/pme_memory/needs.py +0 -55
  109. package/packages/memory-engine/pme_memory/provenance.py +0 -80
  110. package/packages/memory-engine/pme_memory/scoring.py +0 -168
  111. package/packages/memory-engine/pme_memory/search.py +0 -52
  112. package/packages/memory-engine/pme_memory/store.py +0 -86
  113. package/packages/memory-engine/pme_memory/synthesis.py +0 -114
  114. package/packages/memory-engine/pyproject.toml +0 -65
  115. package/packages/memory-engine/scripts/kg-extractor.py +0 -557
  116. package/packages/memory-engine/scripts/kg-preflexor-v2.py +0 -738
  117. package/packages/memory-engine/scripts/wipe-legacy-l3-entities.py +0 -128
  118. package/packages/memory-engine/tests/e2e_arena.sh +0 -259
  119. package/packages/memory-engine/tests/embed_stub/Dockerfile +0 -13
  120. package/packages/memory-engine/tests/embed_stub/server.py +0 -80
  121. package/packages/memory-engine/tests/test_aggregate.py +0 -333
  122. package/packages/memory-engine/tests/test_api_contract.sh +0 -57
  123. package/packages/memory-engine/tests/test_arena_safety.py +0 -232
  124. package/packages/memory-engine/tests/test_channel_stat_reader.py +0 -437
  125. package/packages/memory-engine/tests/test_channel_stat_rollups.py +0 -308
  126. package/packages/memory-engine/tests/test_compat_nv_embed_probe.py +0 -48
  127. package/packages/memory-engine/tests/test_embed_provider.py +0 -492
  128. package/packages/memory-engine/tests/test_l2_qmd_vec_search.py +0 -280
  129. package/packages/memory-engine/tests/test_l3_arena_isolation.py +0 -412
  130. package/packages/memory-engine/tests/test_l6_module_load.py +0 -84
  131. package/packages/memory-engine/tests/test_people_list_reader.py +0 -432
@@ -1,1255 +0,0 @@
1
- """
2
- pentatonic-memory-engine compatibility shim.
3
-
4
- Exposes the same HTTP API as `pentatonic-memory` v0.5.x (POST /store,
5
- POST /search, GET /health) plus the v0.6 regression-fix endpoints
6
- (POST /store-batch, POST /forget). Internally routes every call through
7
- the 7-layer hybrid retrieval engine running in sibling containers
8
- (L0 BM25, L1 core files, L2 HybridRAG orchestrator, L3 Knowledge Graph,
9
- L4 vec, L5 Milvus, L6 doc-store).
10
-
11
- Drop-in replacement: change a single base URL in your existing
12
- pentatonic-memory SDK client and you get ~5x retrieval accuracy
13
- without touching anything else.
14
-
15
- Endpoints:
16
-
17
- POST /store { content, metadata } → { id, content, layerId }
18
- POST /store-batch { records: [{ id?, content, metadata }] } → { inserted, ids[], embed_ms, insert_ms }
19
- POST /search { query, limit, min_score } → { results: [...] }
20
- POST /forget { metadata_contains } | { id } → { deleted: N }
21
- GET /health → { status, layers: {...}, memories }
22
-
23
- Environment:
24
-
25
- L0_URL default http://l0:8030
26
- L2_PROXY_URL default http://l2:8031
27
- L3_KG_URL default http://l3:8047
28
- L5_MILVUS_URL default http://l5:8035
29
- L6_DOC_URL default http://l6:8037
30
- NV_EMBED_URL default http://nv-embed:8041/v1/embeddings
31
- PORT default 8099 (matches pentatonic-memory v0.5)
32
- """
33
-
34
- import hashlib
35
- import os
36
- import sys
37
- import time
38
- from datetime import datetime, timezone
39
- from typing import Any, Optional
40
-
41
- import httpx
42
- from fastapi import FastAPI, HTTPException
43
- from pydantic import BaseModel, Field
44
-
45
- # Reach into the engine/services tree so we can reuse EmbedClient. The
46
- # tree isn't a real installed package; layer services and the compat
47
- # shim both side-load it the same way. Keeps the chunking + auto-detect
48
- # behaviour identical between the shim's pre-embed and the per-layer
49
- # embeds that previously did the same work N times.
50
- sys.path.insert(
51
- 0,
52
- os.path.join(os.path.dirname(__file__), "..", "engine", "services"),
53
- )
54
- from _shared.embed_provider import EmbedClient # noqa: E402
55
-
56
- # ----------------------------------------------------------------------
57
- # Config
58
- # ----------------------------------------------------------------------
59
-
60
- L0_URL = os.environ.get("L0_URL", "http://l0:8030")
61
- L2_PROXY_URL = os.environ.get("L2_PROXY_URL", "http://l2:8031")
62
- L3_KG_URL = os.environ.get("L3_KG_URL", "http://l3:8047")
63
- L5_MILVUS_URL = os.environ.get("L5_MILVUS_URL", "http://l5:8035")
64
- L6_DOC_URL = os.environ.get("L6_DOC_URL", "http://l6:8037")
65
- NV_EMBED_URL = os.environ.get("NV_EMBED_URL", "http://nv-embed:8041/v1/embeddings")
66
-
67
- # Neo4j has no /health endpoint, so the shim probes the HTTP transactional
68
- # API with a trivial RETURN 1 — that confirms Neo4j is actually answering
69
- # Cypher, not just serving HTTP. Auth shape is the same as L2 / docker-compose:
70
- # "user/pass" string. Default matches the local-dev compose default.
71
- NEO4J_AUTH = os.environ.get("NEO4J_AUTH", "neo4j/local-dev-pw")
72
- NEO4J_DB = os.environ.get("NEO4J_DB", "neo4j")
73
-
74
- PORT = int(os.environ.get("PORT", "8099"))
75
-
76
- # Shared-embed mode. When on, /store-batch computes embeddings once at
77
- # the shim level and forwards them to each layer's /index-batch so the
78
- # layer skips its own embed call. Cuts gateway RPC count by ~4× (L4 +
79
- # L5 + L6 + L2-internal all did the same embed work independently).
80
- # Default ON because all layers in this engine use the same NV-Embed
81
- # model; disable if you ever wire up per-layer differentiated embedders
82
- # (e.g. cohere on L5, openai on L4).
83
- SHARE_EMBEDDINGS = os.environ.get("PME_SHARE_EMBEDDINGS", "true").lower() == "true"
84
-
85
- _embed_client: EmbedClient | None = None
86
-
87
-
88
- def _get_embed_client() -> EmbedClient:
89
- """Lazy-init the shim's EmbedClient using PME_-prefixed env vars
90
- (matches L2's pattern). Cached for the process lifetime so the
91
- auto-detect handshake only happens once."""
92
- global _embed_client
93
- if _embed_client is None:
94
- _embed_client = EmbedClient.from_env(
95
- prefix="PME_",
96
- default_url=NV_EMBED_URL,
97
- )
98
- return _embed_client
99
-
100
-
101
- # Layer types we surface as the SDK 4-layer projection. Engine stores
102
- # everything as chunks tagged with arena + layer_type metadata; this
103
- # helper renders the legacy `ml_<arena>_<type>` layer-id from the
104
- # per-row arena, so the response reflects the actual data not a
105
- # deployment-wide constant. Falls back to "episodic" when arena or
106
- # layer_type is missing.
107
- def _layer_id(arena: Optional[str], layer_type: Optional[str] = None) -> str:
108
- a = arena or "general"
109
- t = layer_type or "episodic"
110
- return f"ml_{a}_{t}"
111
-
112
- # Test/isolated mode: bypass the L2 HybridRAG orchestrator and query L6 directly.
113
- # Useful for bench harnesses where you want to validate the ingest+search
114
- # round-trip against an isolated test L6 instance, without the L2 proxy
115
- # pulling in production data from other layers.
116
- BYPASS_L2 = os.environ.get("BYPASS_L2_PROXY", "0") in ("1", "true", "yes")
117
-
118
- VERSION = "0.1.0"
119
-
120
-
121
- # ----------------------------------------------------------------------
122
- # Request / response models (match pentatonic-memory v0.5 wire format)
123
- # ----------------------------------------------------------------------
124
-
125
- class StoreRequest(BaseModel):
126
- content: str
127
- metadata: Optional[dict[str, Any]] = None
128
-
129
-
130
- class StoreBatchRequest(BaseModel):
131
- records: list[dict[str, Any]] = Field(default_factory=list)
132
- arena: Optional[str] = "general"
133
-
134
-
135
- class SearchRequest(BaseModel):
136
- query: str
137
- limit: Optional[int] = 10
138
- min_score: Optional[float] = 0.001
139
- # Tenant scope (single arena). Back-compat shape — single-arena callers
140
- # can keep sending this. Treated as a one-element `arenas` list.
141
- arena: Optional[str] = None
142
- # Multi-arena scope. Used by callers that want to span both a tenant-
143
- # wide arena ("acme") and a user-scoped arena ("acme:user-42") in one
144
- # search — the SDK helper composes this list automatically when a
145
- # `userId` is supplied. Authoritative when both `arena` and `arenas`
146
- # are present; engine treats it as a set.
147
- arenas: Optional[list[str]] = None
148
- # Arbitrary metadata equality filters, applied as a post-filter on
149
- # the shim. Useful for `kind`, `layer_type`, `source_repo`, etc.
150
- # Keys not present on a result's metadata are treated as no-match.
151
- # Each pair is exact string equality. Engine doesn't currently
152
- # forward these to underlying stores, so over-fetch happens; the
153
- # shim trims to the requested limit after filtering.
154
- metadata_filter: Optional[dict[str, Any]] = None
155
-
156
-
157
- class ForgetRequest(BaseModel):
158
- metadata_contains: Optional[dict[str, Any]] = None
159
- id: Optional[str] = None
160
-
161
-
162
- class AggregateRequest(BaseModel):
163
- """Public-facing /aggregate request.
164
-
165
- Mirrors /aggregate-internal on the L2 proxy with arena scoping
166
- enforced at the shim layer (multi-arena spans aren't allowed
167
- here — pick a single arena per call). The relationships UI
168
- queries this directly via TES; future callers will too.
169
- """
170
-
171
- arena: str
172
- contact_email: Optional[str] = None
173
- contact_name: Optional[str] = None
174
- group_by: Optional[list[str]] = None
175
-
176
-
177
- class PeopleListRequest(BaseModel):
178
- """Public-facing /people-list request.
179
-
180
- Corpus-level "all people across these arenas" projection backing
181
- the Pip Relationships UI list page. Distinct from /aggregate
182
- (which is per-person): /people-list returns one row per Person
183
- across the whole arena set, with per-channel breakdown nested.
184
-
185
- Multi-arena is supported here (unlike /aggregate) because the
186
- UI use case — "people known by Philip OR Jeanne" — is a
187
- legitimate cross-user view. The L2 proxy does the dedup by
188
- person_email.
189
-
190
- `emails`: batched-mode filter. Return only these emails.
191
- `search`: case-insensitive substring on person_name and
192
- person_email.
193
- `order_by`: whitelisted sort key; see the L2 proxy for the
194
- allowlist.
195
- """
196
-
197
- arenas: list[str]
198
- emails: Optional[list[str]] = None
199
- search: Optional[str] = None
200
- limit: int = 200
201
- offset: int = 0
202
- order_by: str = "last_seen_desc"
203
-
204
-
205
- # ----------------------------------------------------------------------
206
- # Engine clients (one per layer)
207
- # ----------------------------------------------------------------------
208
-
209
- _http: Optional[httpx.AsyncClient] = None
210
-
211
- # In-memory metadata sidecar — per-id stash so arbitrary client metadata
212
- # (e.g. {"bench_tag": "...", "doc_id": "..."}) survives the round-trip
213
- # even when the underlying L5/L6 schemas don't carry a JSON metadata column.
214
- # Bounded to the most recent 100k entries to avoid leaking memory in long-
215
- # running deployments. Resets on shim restart.
216
- from collections import OrderedDict
217
- _META_CACHE: "OrderedDict[str, dict[str, Any]]" = OrderedDict()
218
- _META_CACHE_MAX = 100_000
219
-
220
- def _stash_meta(rid: str, meta: dict[str, Any] | None) -> None:
221
- if not rid:
222
- return
223
- _META_CACHE[rid] = meta or {}
224
- while len(_META_CACHE) > _META_CACHE_MAX:
225
- _META_CACHE.popitem(last=False)
226
-
227
- def _lookup_meta(rid: str) -> dict[str, Any]:
228
- return _META_CACHE.get(rid, {}) if rid else {}
229
-
230
-
231
- def _stash_all_keys(rid: str, meta: dict[str, Any], arena: str = "general") -> None:
232
- """Stash meta under every id-shape any of the 7 layers might echo back.
233
-
234
- L4 sidecar: rid (and `<rid>.md`)
235
- L5 comms: rid (path may be `.pentatonic/chats/<rid>.jsonl`)
236
- L6 doc-store: `l6:<rid>:0`, `<rid>.md` (chunk_id, source_file)
237
- L2 internal L0/L4-qmd: `bench/<arena>/<rid>.md`,
238
- `bench/<arena>/<doc_id>.md`
239
- L3 graph chunk: rid (Chunk.id) and doc_id
240
- """
241
- if not rid:
242
- return
243
- meta = meta or {}
244
- keys = {
245
- rid,
246
- f"{rid}.md",
247
- f"l6:{rid}:0",
248
- f"bench/{arena}/{rid}.md",
249
- f"bench/{arena}/{rid}",
250
- }
251
- doc_id = meta.get("doc_id")
252
- if doc_id:
253
- keys.update({
254
- doc_id,
255
- f"{doc_id}.md",
256
- f"l6:{doc_id}:0",
257
- f"bench/{arena}/{doc_id}.md",
258
- f"bench/{arena}/{doc_id}",
259
- })
260
- path = meta.get("path")
261
- if path:
262
- keys.add(path)
263
- keys.add(path.rsplit(".", 1)[0])
264
- for k in keys:
265
- if k:
266
- _stash_meta(k, meta)
267
-
268
-
269
- def _client() -> httpx.AsyncClient:
270
- global _http
271
- if _http is None:
272
- _http = httpx.AsyncClient(timeout=60.0)
273
- return _http
274
-
275
-
276
- async def _embed_batch(texts: list[str]) -> list[list[float]]:
277
- """One NV-Embed call for many texts. Returns vectors in input order."""
278
- if not texts:
279
- return []
280
- resp = await _client().post(
281
- NV_EMBED_URL,
282
- json={"input": texts, "model": "nv-embed-v2"},
283
- timeout=120.0,
284
- )
285
- resp.raise_for_status()
286
- return [d["embedding"] for d in resp.json()["data"]]
287
-
288
-
289
- async def _index_l5(
290
- records: list[dict[str, Any]],
291
- arena: str = "general",
292
- embeddings: list[list[float]] | None = None,
293
- ) -> int:
294
- """Index records into the L5 Milvus comms layer (chats collection).
295
-
296
- arena is forwarded as a Milvus dynamic field so /search can filter
297
- by arena natively (vs the shim's defence-in-depth post-filter).
298
-
299
- When `embeddings` is supplied (parallel to records), L5 skips its
300
- own embed call — the shim pre-computes vectors once at /store-batch
301
- level and threads them through each layer to avoid 3× redundant
302
- embed RPCs (L5 + L6 + L2-internal otherwise each re-embed the same
303
- texts in parallel).
304
- """
305
- payload: dict[str, Any] = {
306
- "collection": "chats",
307
- "records": [
308
- {
309
- "id": r.get("id") or hashlib.sha1(r["content"].encode()).hexdigest()[:32],
310
- "text": r["content"],
311
- "source": (r.get("metadata") or {}).get("source", "shim"),
312
- "channel": "pentatonic-memory",
313
- "contact": (r.get("metadata") or {}).get("user", ""),
314
- "arena": (r.get("metadata") or {}).get("arena") or arena,
315
- }
316
- for r in records
317
- ],
318
- }
319
- if embeddings is not None:
320
- payload["embeddings"] = embeddings
321
- try:
322
- resp = await _client().post(f"{L5_MILVUS_URL}/index-batch", json=payload, timeout=60.0)
323
- resp.raise_for_status()
324
- return resp.json().get("inserted", 0)
325
- except Exception as exc:
326
- # Best-effort: L5 is one of five redundant layers; failure here
327
- # doesn't mean the record is unsearchable. L0 BM25 + L4 QMD +
328
- # L6 doc-store all carry it independently.
329
- print(f"[shim] L5 index-batch failed: {exc}")
330
- return 0
331
-
332
-
333
- async def _index_l6(
334
- records: list[dict[str, Any]],
335
- arena: str = "general",
336
- embeddings: list[list[float]] | None = None,
337
- ) -> int:
338
- """Index records into the L6 document store.
339
-
340
- When `embeddings` is supplied (parallel to records), L6 skips its
341
- own embed call — the shim pre-computes vectors once at /store-batch
342
- level and threads them through each layer.
343
- """
344
- payload: dict[str, Any] = {
345
- "arena": arena,
346
- "records": [
347
- {
348
- "id": r.get("id") or hashlib.sha1(r["content"].encode()).hexdigest()[:32],
349
- "text": r["content"],
350
- "source_file": (r.get("metadata") or {}).get("path") or f"{r.get('id', 'doc')}.md",
351
- "doc_type": (r.get("metadata") or {}).get("doc_type", "general"),
352
- "heading": (r.get("metadata") or {}).get("heading", ""),
353
- }
354
- for r in records
355
- ],
356
- }
357
- if embeddings is not None:
358
- payload["embeddings"] = embeddings
359
- try:
360
- resp = await _client().post(f"{L6_DOC_URL}/index-batch", json=payload, timeout=120.0)
361
- resp.raise_for_status()
362
- return resp.json().get("inserted", 0)
363
- except Exception as exc:
364
- print(f"[shim] L6 index-batch failed: {exc}")
365
- return 0
366
-
367
-
368
- async def _index_l2_internal(
369
- records: list[dict[str, Any]],
370
- arena: str = "general",
371
- embeddings: list[list[float]] | None = None,
372
- ) -> dict:
373
- """Populate L2's internal stores: L0 BM25 + L4 QMD vec + L3 Neo4j KG.
374
-
375
- Without this, L2's RRF fusion runs over empty L0/L4-qmd/L3 layers and
376
- those zero-result rank lists pollute the score. The L2 proxy exposes
377
- /index-internal-batch which writes to all three in one round-trip.
378
-
379
- When `embeddings` is supplied (parallel to records), L2's internal
380
- embed call (used for L4-QMD population) is skipped — the shim
381
- pre-computes vectors once at /store-batch level and threads them
382
- through to L4_QMD via this endpoint.
383
- """
384
- payload: dict[str, Any] = {
385
- "arena": arena,
386
- "records": [
387
- {
388
- "id": r.get("id") or hashlib.sha1(r["content"].encode()).hexdigest()[:32],
389
- "content": r["content"],
390
- "metadata": r.get("metadata") or {},
391
- }
392
- for r in records
393
- ],
394
- }
395
- if embeddings is not None:
396
- payload["embeddings"] = embeddings
397
- try:
398
- resp = await _client().post(f"{L2_PROXY_URL}/index-internal-batch",
399
- json=payload, timeout=180.0)
400
- resp.raise_for_status()
401
- return resp.json()
402
- except Exception as exc:
403
- print(f"[shim] L2 internal index-batch failed: {exc}")
404
- return {"l0": 0, "l4_qmd": 0, "l3_entities": 0, "l3_chunks": 0}
405
-
406
-
407
- # ----------------------------------------------------------------------
408
- # FastAPI app
409
- # ----------------------------------------------------------------------
410
-
411
- app = FastAPI(
412
- title="pentatonic-memory-engine compat shim",
413
- version=VERSION,
414
- description="Drop-in API compat for pentatonic-memory v0.5; routed through the 7-layer engine.",
415
- )
416
-
417
-
418
- def _interpret_body_status(body: Any) -> str | None:
419
- """Pull a layer's self-reported status out of its /health body.
420
-
421
- Layers don't all use the same vocabulary — L4 says "ok"/"degraded",
422
- L2 says "healthy"/"unavailable", some return nothing. Normalize to
423
- "ok" or a short failure reason; None means the body didn't carry
424
- a status field, in which case the HTTP code is the source of truth.
425
- """
426
- if not isinstance(body, dict):
427
- return None
428
- raw = body.get("status")
429
- if raw is None:
430
- return None
431
- s = str(raw).lower()
432
- if s in ("ok", "healthy"):
433
- return "ok"
434
- err = body.get("error") or body.get("reason") or ""
435
- return f"{s}: {str(err)[:80]}" if err else s
436
-
437
-
438
- async def _probe(url: str) -> str:
439
- """Probe a layer /health endpoint and return a single-string verdict
440
- that surfaces both transport-level failure and self-reported status."""
441
- try:
442
- r = await _client().get(url, timeout=3.0)
443
- except Exception as exc:
444
- return f"unreachable: {type(exc).__name__}"
445
- if r.status_code != 200:
446
- return f"http {r.status_code}"
447
- try:
448
- body_status = _interpret_body_status(r.json())
449
- except Exception:
450
- body_status = None
451
- return body_status or "ok"
452
-
453
-
454
- async def _probe_l3() -> str:
455
- """Real Neo4j probe — POST a trivial Cypher via the HTTP transactional
456
- API and require a 200 response. Confirms Neo4j is actually answering
457
- queries, not just serving the Browser HTML on :7474.
458
- """
459
- user, _, password = NEO4J_AUTH.partition("/")
460
- url = f"{L3_KG_URL}/db/{NEO4J_DB}/tx/commit"
461
- try:
462
- r = await _client().post(
463
- url,
464
- json={"statements": [{"statement": "RETURN 1"}]},
465
- auth=(user, password),
466
- timeout=3.0,
467
- )
468
- except Exception as exc:
469
- return f"unreachable: {type(exc).__name__}"
470
- if r.status_code != 200:
471
- return f"http {r.status_code}"
472
- try:
473
- body = r.json()
474
- # Neo4j tx/commit returns {"results":[...], "errors":[...]}.
475
- # Any errors here means the DB is up but rejecting queries.
476
- errs = body.get("errors") or []
477
- if errs:
478
- return f"cypher error: {str(errs[0])[:80]}"
479
- except Exception:
480
- return "non-json response"
481
- return "ok"
482
-
483
-
484
- @app.get("/health")
485
- async def health():
486
- """Aggregate health across all 7 layers.
487
-
488
- Each layer's verdict is honest: it reflects whether the layer can
489
- actually do its job, not just whether its HTTP server answers. The
490
- shim reads the layer's body.status (when present) and degrades when
491
- the layer self-reports a problem. L3 uses a real Cypher probe since
492
- Neo4j has no /health route.
493
- """
494
- out = {
495
- "status": "ok",
496
- "version": VERSION,
497
- "engine": "pentatonic-memory-engine",
498
- "layers": {},
499
- }
500
- # NV-Embed (or the upstream gateway) exposes /health at the host root.
501
- # Use urlparse so we swap *just* the path component instead of doing a
502
- # string replace — that breaks the moment NV_EMBED_URL is /v1/embed,
503
- # /v1/embeddings, or bare-host. The probe is informational only; gateways
504
- # that return non-200 on root-/health don't block engine operation.
505
- from urllib.parse import urlparse, urlunparse
506
- _u = urlparse(NV_EMBED_URL)
507
- nv_embed_health = urlunparse((_u.scheme, _u.netloc, "/health", "", "", ""))
508
-
509
- import asyncio
510
- l2_v, l5_v, l6_v, nv_v, l3_v = await asyncio.gather(
511
- _probe(f"{L2_PROXY_URL}/health"),
512
- _probe(f"{L5_MILVUS_URL}/health"),
513
- _probe(f"{L6_DOC_URL}/health"),
514
- _probe(nv_embed_health),
515
- _probe_l3(),
516
- )
517
-
518
- # L0 BM25 (FTS5), L1 (always-loaded core files) and L4 QMD vec are
519
- # all in-process inside the L2 proxy — L0+L1 in workspace.db / core
520
- # files; L4 in qmd.sqlite which L2 opens directly. No separate runtime;
521
- # if L2 is healthy, all three layers are usable. Tie their status to L2.
522
- l2_ok = l2_v == "ok"
523
- out["layers"] = {
524
- "l0": "ok" if l2_ok else l2_v,
525
- "l1": "ok" if l2_ok else l2_v,
526
- "l2": l2_v,
527
- "l3": l3_v,
528
- "l4": "ok" if l2_ok else l2_v,
529
- "l5": l5_v,
530
- "l6": l6_v,
531
- "nv_embed": nv_v,
532
- }
533
- failures = sum(1 for v in out["layers"].values() if v != "ok")
534
- if failures:
535
- out["status"] = "degraded" if failures < 3 else "down"
536
-
537
- # Per-layer chunk counts. Replaces the previous single `memories` int
538
- # which only reflected L6's vector chunk count — misleading because
539
- # L0/L4/L5 hold different (overlapping) projections of the corpus.
540
- # Each layer is independently probed; transient failure on one layer
541
- # leaves its slot null rather than zeroing the whole field.
542
- memories: dict[str, int | None] = {
543
- "l0_bm25_chunks": None,
544
- "l4_vectors": None,
545
- "l5_chats_chunks": None,
546
- "l6_vector_chunks": None,
547
- "l6_fts_chunks": None,
548
- }
549
- # L0 and L4 both live inside L2 (workspace.db + qmd.sqlite directly
550
- # opened by the L2 proxy). L2 exposes /index-internal-stats with both
551
- # counts in one round-trip.
552
- try:
553
- r = await _client().get(f"{L2_PROXY_URL}/index-internal-stats", timeout=3.0)
554
- if r.status_code == 200:
555
- stats = r.json()
556
- memories["l0_bm25_chunks"] = int(stats.get("l0_chunks") or 0)
557
- memories["l4_vectors"] = int(stats.get("l4_qmd_chunks") or 0)
558
- except Exception:
559
- pass
560
- # L5 reports per-collection counts on /health. We surface chats —
561
- # the only collection currently populated; emails/contacts/memory
562
- # collections coming back online with L5 collection bootstrap.
563
- try:
564
- r = await _client().get(f"{L5_MILVUS_URL}/health", timeout=3.0)
565
- if r.status_code == 200:
566
- colls = r.json().get("collections") or {}
567
- chats = colls.get("chats") if isinstance(colls, dict) else None
568
- if isinstance(chats, dict):
569
- memories["l5_chats_chunks"] = int(chats.get("count") or 0)
570
- except Exception:
571
- pass
572
- # L6 exposes vector vs fts splits on /stats.
573
- try:
574
- r = await _client().get(f"{L6_DOC_URL}/stats", timeout=3.0)
575
- if r.status_code == 200:
576
- stats = r.json()
577
- memories["l6_vector_chunks"] = int(stats.get("vector_chunks") or 0)
578
- memories["l6_fts_chunks"] = int(stats.get("fts_chunks") or 0)
579
- except Exception:
580
- pass
581
- out["memories"] = memories
582
- return out
583
-
584
-
585
- @app.get("/health/deep")
586
- async def health_deep():
587
- """Aggregate functional probe — fans out to each layer's /health/deep
588
- and reports per-layer ok/status.
589
-
590
- Where /health checks "process up + port answering", /health/deep
591
- actually exercises embed → write → search round-trips on each layer
592
- that supports it. Catches the class of bug that masquerades as
593
- "healthy" — request handlers 500'ing while the process stays up.
594
-
595
- Slower than /health (~1–2s); intended for ops/monitoring/cron use,
596
- not the deploy gate or compose healthcheck.
597
- """
598
- import asyncio
599
- out: dict[str, Any] = {"status": "ok", "ok": True, "layers": {}}
600
-
601
- async def _probe_deep(name: str, url: str) -> tuple[str, dict]:
602
- try:
603
- r = await _client().get(url, timeout=15.0)
604
- if r.status_code != 200:
605
- return name, {"ok": False, "status": f"http {r.status_code}"}
606
- return name, r.json()
607
- except Exception as exc:
608
- return name, {"ok": False, "status": f"unreachable: {type(exc).__name__}"}
609
-
610
- # L4 is in-process inside L2 (qmd.sqlite direct-read) — its deep
611
- # round-trip is covered by L2's /health/deep, no separate probe needed.
612
- results = await asyncio.gather(
613
- _probe_deep("l5", f"{L5_MILVUS_URL}/health/deep"),
614
- _probe_deep("l6", f"{L6_DOC_URL}/health/deep"),
615
- )
616
- for name, body in results:
617
- out["layers"][name] = body
618
- if not body.get("ok", False):
619
- out["ok"] = False
620
-
621
- if not out["ok"]:
622
- out["status"] = "degraded"
623
- return out
624
-
625
-
626
- def _arena_scoped_rid(arena: str, content: str) -> str:
627
- """Stable, arena-scoped record id.
628
-
629
- Same content in different arenas hashes to *different* ids — so two
630
- tenants storing the byte-identical chunk get distinct primary keys
631
- in the underlying stores (L4 sqlite-vec, L5 Milvus, L6 Milvus).
632
- Without this, arena=tenant-b's write would overwrite arena=tenant-a's
633
- row of the same content, silently breaking multi-tenant isolation
634
- for any duplicated phrase.
635
- """
636
- return hashlib.sha1(f"{arena}:{content}".encode()).hexdigest()[:32]
637
-
638
-
639
- @app.post("/store")
640
- async def store(req: StoreRequest):
641
- """Single-record ingest. Same wire format as pentatonic-memory v0.5."""
642
- arena = (req.metadata or {}).get("arena", "general")
643
- rid = (req.metadata or {}).get("id") or _arena_scoped_rid(arena, req.content)
644
- record = {"id": rid, "content": req.content, "metadata": req.metadata or {}}
645
-
646
- # Stash the full metadata under every key shape any layer could echo back.
647
- # L5/L6 use derivatives of rid; L2-internal returns paths shaped like
648
- # bench/<arena>/<id>.md (and <id> may be the SHA1 rid OR the caller's doc_id
649
- # depending on which one was supplied).
650
- _stash_all_keys(rid, req.metadata or {}, arena)
651
-
652
- # Fan out to L5 + L6 + L2-internal (L0+L4qmd+L3) in parallel.
653
- import asyncio
654
- l5_count, l6_count, l2_internal = await asyncio.gather(
655
- _index_l5([record], arena=arena),
656
- _index_l6([record], arena=arena),
657
- _index_l2_internal([record], arena=arena),
658
- )
659
-
660
- l4_qmd_count = l2_internal.get("l4_qmd", 0)
661
- return {
662
- "id": rid,
663
- "content": req.content,
664
- "layerId": _layer_id(arena, (req.metadata or {}).get("layer_type")),
665
- "engine": {
666
- "l0": l2_internal.get("l0", 0),
667
- "l3_chunks": l2_internal.get("l3_chunks", 0),
668
- "l3_entities": l2_internal.get("l3_entities", 0),
669
- "l4_qmd": l4_qmd_count,
670
- # `l4` is aliased to L4_QMD now that the standalone L4 sqlite-vec
671
- # sidecar has been dropped. Kept in the response for wire-format
672
- # back-compat with callers that read engine.l4.
673
- "l4": l4_qmd_count,
674
- "l5": l5_count,
675
- "l6": l6_count,
676
- },
677
- }
678
-
679
-
680
- @app.post("/store-batch")
681
- async def store_batch(req: StoreBatchRequest):
682
- """Batch ingest — 30-50× faster than calling /store N times."""
683
- if not req.records:
684
- return {"inserted": 0, "ids": []}
685
-
686
- # Normalise each record to {id, content, metadata}.
687
- arena = req.arena or "general"
688
- normalised = []
689
- for r in req.records:
690
- content = r.get("content") or r.get("text") or ""
691
- if not content:
692
- continue
693
- rid = r.get("id") or _arena_scoped_rid(arena, content)
694
- normalised.append({"id": rid, "content": content, "metadata": r.get("metadata") or {}})
695
-
696
- # Stash metadata for every record so /search can re-attach it.
697
- for r in normalised:
698
- _stash_all_keys(r["id"], r.get("metadata") or {}, arena)
699
-
700
- t0 = time.perf_counter()
701
- import asyncio
702
-
703
- # Shared-embed mode: compute embeddings ONCE here, pass them down to
704
- # every layer so they skip their own embed call. Previously L5 + L6
705
- # + L2-internal each re-embedded the same texts in parallel, which
706
- # fanned 3× the gateway RPCs. The gateway throttles at K≈10 concurrent
707
- # requests, so 30-way fan-out serialised into ~3 rounds of ~850ms
708
- # each = ~2.5s of pure embed time per /store-batch. With shared
709
- # embeddings we issue one chunked embed pass (10 sub-calls for N=50
710
- # records) and skip the per-layer redundant work entirely.
711
- # Disabled via PME_SHARE_EMBEDDINGS=false for operators wiring up
712
- # per-layer differentiated embedders.
713
- shared_embeddings: list[list[float]] | None = None
714
- embed_ms = 0.0
715
- if SHARE_EMBEDDINGS and normalised:
716
- texts = [r["content"] for r in normalised]
717
- embed_t0 = time.perf_counter()
718
- try:
719
- shared_embeddings = await _get_embed_client().embed_batch_async(texts)
720
- except Exception as exc:
721
- # Fall back to per-layer embedding rather than failing the
722
- # whole batch. The layers' /index-batch still works when
723
- # `embeddings` is absent.
724
- print(f"[shim] shared embed failed, falling back to per-layer: {exc}")
725
- shared_embeddings = None
726
- embed_ms = (time.perf_counter() - embed_t0) * 1000.0
727
-
728
- l5_count, l6_count, l2_internal = await asyncio.gather(
729
- _index_l5(normalised, arena=req.arena or "general", embeddings=shared_embeddings),
730
- _index_l6(normalised, arena=req.arena or "general", embeddings=shared_embeddings),
731
- _index_l2_internal(normalised, arena=req.arena or "general", embeddings=shared_embeddings),
732
- )
733
- dur_ms = (time.perf_counter() - t0) * 1000.0
734
-
735
- l4_qmd_count = l2_internal.get("l4_qmd", 0)
736
- return {
737
- "status": "ok",
738
- "inserted": max(l4_qmd_count, l5_count, l6_count),
739
- "ids": [r["id"] for r in normalised],
740
- "engine": {
741
- "l0": l2_internal.get("l0", 0),
742
- "l3_chunks": l2_internal.get("l3_chunks", 0),
743
- "l3_entities": l2_internal.get("l3_entities", 0),
744
- "l4_qmd": l4_qmd_count,
745
- # `l4` aliased to L4_QMD — sidecar dropped, see /store handler.
746
- "l4": l4_qmd_count,
747
- "l5": l5_count,
748
- "l6": l6_count,
749
- },
750
- "duration_ms": round(dur_ms, 1),
751
- }
752
-
753
-
754
- def _arenas_for(req: SearchRequest) -> list[str]:
755
- """Normalize req's single-arena + multi-arena fields into one list.
756
-
757
- `arenas` is authoritative when set; otherwise `arena` is treated as
758
- a one-element list; otherwise empty (= search is unscoped, dev/test).
759
- """
760
- if req.arenas:
761
- return [a for a in req.arenas if a]
762
- if req.arena:
763
- return [req.arena]
764
- return []
765
-
766
-
767
- def _apply_metadata_filters(results: list[dict[str, Any]], req: SearchRequest) -> list[dict[str, Any]]:
768
- """Post-filter results by arena set + arbitrary metadata equality.
769
-
770
- Many layer searches don't yet honour arena/metadata at the storage
771
- level, so the shim enforces tenant isolation here as defence in
772
- depth. Even if the underlying layer leaks across arenas, the shim
773
- drops cross-arena rows before returning.
774
-
775
- Multi-arena rule: a row passes if its arena tag is in the request's
776
- arena set. So a user-scoped search (arenas=[acme, acme:u-42]) sees
777
- both tenant-wide rows (arena=acme) and that user's own user-scoped
778
- rows (arena=acme:u-42), but never another user's user-scoped rows
779
- (arena=acme:u-99).
780
- """
781
- arenas = _arenas_for(req)
782
- extra = req.metadata_filter or {}
783
- if not arenas and not extra:
784
- return results
785
- arena_set = set(arenas)
786
- out: list[dict[str, Any]] = []
787
- for item in results:
788
- meta = item.get("metadata") or {}
789
- if arena_set:
790
- row_arena = meta.get("arena") or item.get("arena")
791
- if row_arena and row_arena not in arena_set:
792
- continue
793
- # If row has no arena tag at all, drop on multi-tenant
794
- # safety: a row without arena predates the multi-tenant
795
- # plumbing and could belong to anyone.
796
- if not row_arena:
797
- continue
798
- ok = True
799
- for k, v in extra.items():
800
- if str(meta.get(k, "")) != str(v):
801
- ok = False
802
- break
803
- if ok:
804
- out.append(item)
805
- return out
806
-
807
-
808
- def _search_overfetch(req: SearchRequest) -> int:
809
- """Decide how many results to over-fetch from layers.
810
-
811
- Post-filtering can drop many rows; we ask layers for more than the
812
- user's limit so we have headroom after filtering. 5x is a balance
813
- between accuracy and latency.
814
- """
815
- base = req.limit or 10
816
- return base * 5 if (_arenas_for(req) or req.metadata_filter) else base * 3
817
-
818
-
819
- @app.post("/search")
820
- async def search(req: SearchRequest):
821
- """
822
- Hybrid retrieval over all 7 layers via the L2 HybridRAG proxy. The proxy
823
- queries L0 BM25, L4 vec, L5 Milvus, L6 doc-store in parallel and fuses
824
- the results with Reciprocal Rank Fusion. L3 KG adds entity-aware
825
- boosting for graph queries.
826
-
827
- Multi-tenancy: pass `arena` to scope results to a single tenant.
828
- Underlying layers may or may not honour arena natively (L6 does;
829
- L2/L4/L5 don't yet — engine TODO); the shim applies arena as a
830
- post-filter regardless, so cross-tenant leakage is prevented even
831
- when a layer is non-compliant.
832
- """
833
- if not req.query:
834
- return {"results": []}
835
-
836
- # The L2 proxy exposes hybrid search via GET /search?q=... and a strict
837
- # OpenAI-compatible POST /v1/search. Try GET first (lower overhead, no
838
- # JSON parsing on the proxy side); fall back to /v1/search; then to L6.
839
- #
840
- # When BYPASS_L2_PROXY is set, skip the proxy entirely and query L6
841
- # directly. Useful for isolated bench/test runs.
842
- data: dict[str, Any] | None = None
843
- last_err: Exception | None = None
844
- if BYPASS_L2:
845
- # L6-only path: L6 already does vector + BM25 + cross-encoder
846
- # reranker. Adding L4's pure vector via RRF actively hurt on
847
- # product-catalogue (-5.6pp on the 84.6% baseline) by diluting
848
- # the rerank ordering. Stick to L6 for now; the proper next
849
- # step is wiring up the L2 7-layer proxy.
850
- import asyncio
851
- async def _q_l6(query: str):
852
- try:
853
- params: list = [
854
- ("q", query),
855
- ("limit", str(_search_overfetch(req))),
856
- ("method", "hybrid"),
857
- ]
858
- # L6 supports arena natively (l6-document-store.py).
859
- # Forward all arenas in the search scope; L6 expands the
860
- # filter to `arena IN (...)`. Multiple `arenas` query
861
- # params on the wire = list-shaped server side.
862
- for a in _arenas_for(req):
863
- params.append(("arenas", a))
864
- r = await _client().get(
865
- f"{L6_DOC_URL}/search",
866
- params=params,
867
- timeout=30.0,
868
- )
869
- r.raise_for_status()
870
- return [{"layer": "L6", **item} for item in r.json().get("results", [])]
871
- except Exception as exc:
872
- print(f"[shim] L6 search failed for {query!r}: {exc}")
873
- return []
874
-
875
- # Optional HyDE: if HYDE_ENABLED, also generate 2 hypothetical
876
- # answers via the LLM, embed each, and run them as additional
877
- # queries that we RRF-fuse with the original. Off by default;
878
- # set HYDE_ENABLED=1 to try it. Runs in parallel with the main
879
- # query so latency only grows by the LLM call (1-2s).
880
- l6_hits = await _q_l6(req.query)
881
- l4_hits: list[dict[str, Any]] = [] # kept empty intentionally
882
-
883
- # Reciprocal Rank Fusion (RRF) — k=60 is the standard constant
884
- # from Cormack et al. 2009. Score = 1/(k + rank). For each unique
885
- # doc id we sum contributions from each layer that returned it.
886
- #
887
- # Critical: layers return items with different id shapes
888
- # (L6 uses "l6:<rid>:0" chunk ids and "<rid>.md" source_file,
889
- # L4 uses the raw rid as path), so we resolve a *canonical* id
890
- # by walking the metadata cache for each candidate id form.
891
- # That makes the same doc collapse into one rank entry across
892
- # layers and lets RRF actually fuse instead of double-listing.
893
- def _canonical_key(item: dict[str, Any]) -> str:
894
- candidates = [
895
- item.get("id"),
896
- item.get("chunk_id"),
897
- item.get("source_file"),
898
- item.get("source_file", "").rsplit(".md", 1)[0] if item.get("source_file") else None,
899
- item.get("path"),
900
- ]
901
- for cid in candidates:
902
- if not cid:
903
- continue
904
- m = _META_CACHE.get(cid)
905
- if m and m.get("doc_id"):
906
- return m["doc_id"]
907
- # Fallback: use first non-empty candidate as key.
908
- for cid in candidates:
909
- if cid:
910
- return cid
911
- return hashlib.sha1((item.get("text") or item.get("content") or "").encode()).hexdigest()[:32]
912
-
913
- K = 60
914
- rrf_scores: dict[str, float] = {}
915
- first_item: dict[str, dict[str, Any]] = {}
916
- layer_provenance: dict[str, list[str]] = {}
917
- for hits in (l6_hits, l4_hits):
918
- for rank, item in enumerate(hits, start=1):
919
- key = _canonical_key(item)
920
- rrf_scores[key] = rrf_scores.get(key, 0.0) + 1.0 / (K + rank)
921
- layer_provenance.setdefault(key, []).append(item.get("layer", "?"))
922
- # Keep the richest version of the doc (prefer L6 — it
923
- # carries cross-encoder reranker scores plus content).
924
- if key not in first_item or item.get("layer") == "L6":
925
- first_item[key] = item
926
-
927
- # Sort by fused score, take top-N.
928
- ranked_keys = sorted(rrf_scores.keys(), key=lambda k: -rrf_scores[k])
929
- top_keys = ranked_keys[: req.limit or 10]
930
-
931
- out_results = []
932
- for key in top_keys:
933
- item = first_item[key]
934
- attached_meta = _lookup_meta(key)
935
- if not attached_meta:
936
- # The canonical key may itself be a derived form; walk
937
- # all known id shapes one more time as a safety net.
938
- for cid in (item.get("id"), item.get("chunk_id"),
939
- item.get("source_file"), item.get("path")):
940
- if cid:
941
- m = _lookup_meta(cid)
942
- if m:
943
- attached_meta = m
944
- break
945
- # Some layers surface arena/kind/layer_type as top-level
946
- # fields on the raw item rather than nested under metadata.
947
- # Pull them through into the formatted metadata so the
948
- # post-filter can see them even on cache miss (e.g., after
949
- # a compat container restart wipes _META_CACHE).
950
- raw_top_level = {
951
- k: item[k]
952
- for k in ("arena", "kind", "layer_type", "doc_type", "source_file")
953
- if item.get(k)
954
- }
955
- merged_meta = {**raw_top_level, **(attached_meta or item.get("metadata") or {})}
956
- row_arena = merged_meta.get("arena")
957
- row_layer_type = merged_meta.get("layer_type")
958
- out_results.append({
959
- "id": key,
960
- "content": item.get("text") or item.get("content") or item.get("snippet") or "",
961
- "metadata": merged_meta,
962
- "similarity": float(rrf_scores[key]),
963
- "layer_id": _layer_id(row_arena, row_layer_type),
964
- "source": item.get("source_file") or item.get("path") or "",
965
- "engine_layer": "+".join(sorted(set(layer_provenance.get(key, [])))),
966
- })
967
- # Defense-in-depth post-filter (arena + arbitrary metadata),
968
- # then trim to the requested limit.
969
- out_results = _apply_metadata_filters(out_results, req)
970
- return {"results": out_results[: req.limit or 10]}
971
- arenas = _arenas_for(req)
972
- try:
973
- get_params: list = [
974
- ("q", req.query),
975
- ("limit", str(_search_overfetch(req))),
976
- ]
977
- for a in arenas:
978
- get_params.append(("arenas", a))
979
- r = await _client().get(
980
- f"{L2_PROXY_URL}/search",
981
- params=get_params,
982
- timeout=30.0,
983
- )
984
- r.raise_for_status()
985
- data = r.json()
986
- except Exception as exc:
987
- last_err = exc
988
- try:
989
- post_body: dict[str, Any] = {
990
- "query": req.query,
991
- "limit": _search_overfetch(req),
992
- "min_score": req.min_score or 0.001,
993
- }
994
- if arenas:
995
- post_body["arenas"] = arenas
996
- r = await _client().post(
997
- f"{L2_PROXY_URL}/v1/search",
998
- json=post_body,
999
- timeout=30.0,
1000
- )
1001
- r.raise_for_status()
1002
- data = r.json()
1003
- except Exception as exc2:
1004
- last_err = exc2
1005
- try:
1006
- params: list = [
1007
- ("q", req.query),
1008
- ("limit", str(_search_overfetch(req))),
1009
- ]
1010
- # L6 supports arena natively; forward all in the search
1011
- # scope on the last-resort fallback path too.
1012
- for a in arenas:
1013
- params.append(("arenas", a))
1014
- r = await _client().get(
1015
- f"{L6_DOC_URL}/search",
1016
- params=params,
1017
- timeout=10.0,
1018
- )
1019
- r.raise_for_status()
1020
- data = r.json()
1021
- except Exception as exc3:
1022
- raise HTTPException(status_code=502,
1023
- detail=f"engine unreachable: {last_err}; L6 fallback: {exc3}")
1024
- if data is None:
1025
- raise HTTPException(status_code=502, detail=f"engine returned no data: {last_err}")
1026
-
1027
- # Normalise to pentatonic-memory v0.5 result shape. Re-attach the
1028
- # client-supplied metadata via the in-memory cache (same logic as
1029
- # the BYPASS_L2 path). Bench adapters filter results by
1030
- # metadata.bench_tag, so the metadata MUST survive the L2 round-trip
1031
- # even though L2's response shape doesn't carry an arbitrary JSON
1032
- # metadata column.
1033
- out_results = []
1034
- for item in data.get("results", []):
1035
- candidate_ids = [
1036
- item.get("id"),
1037
- item.get("doc_id"),
1038
- item.get("path"),
1039
- item.get("source_file"),
1040
- item.get("chunk_id"),
1041
- item.get("source"),
1042
- # L5 returns paths like ".pentatonic/chats/<rid>.jsonl" — strip suffix
1043
- item.get("path", "").rsplit(".", 1)[0] if item.get("path") else None,
1044
- item.get("source_file", "").rsplit(".md", 1)[0] if item.get("source_file") else None,
1045
- ]
1046
- attached_meta: dict[str, Any] = {}
1047
- chosen_id = ""
1048
- for cid in candidate_ids:
1049
- if cid:
1050
- m = _lookup_meta(cid)
1051
- if m:
1052
- attached_meta = m
1053
- chosen_id = m.get("doc_id") or cid
1054
- break
1055
- if not chosen_id:
1056
- chosen_id = (item.get("id") or item.get("doc_id")
1057
- or item.get("path") or item.get("source_file") or "")
1058
- # Pull arena/kind/layer_type from raw item top-level if present
1059
- # (L6 returns these as top-level columns; cache may be empty).
1060
- raw_top_level = {
1061
- k: item[k]
1062
- for k in ("arena", "kind", "layer_type", "doc_type", "source_file")
1063
- if item.get(k)
1064
- }
1065
- merged_meta = {**raw_top_level, **(attached_meta or item.get("metadata") or {})}
1066
- row_arena = merged_meta.get("arena")
1067
- row_layer_type = merged_meta.get("layer_type")
1068
- out_results.append({
1069
- "id": chosen_id,
1070
- "content": item.get("text") or item.get("content") or item.get("snippet") or "",
1071
- "metadata": merged_meta,
1072
- "similarity": float(item.get("score") or item.get("similarity") or 0.0),
1073
- "layer_id": _layer_id(row_arena, row_layer_type),
1074
- "source": item.get("source", item.get("source_file", "")),
1075
- "engine_layer": item.get("layer", item.get("source_layer", "")),
1076
- })
1077
- # Defense-in-depth post-filter (arena + arbitrary metadata) on L2/L6
1078
- # fallback paths. Same logic as the BYPASS branch above.
1079
- out_results = _apply_metadata_filters(out_results, req)
1080
- return {"results": out_results[: req.limit or 10]}
1081
-
1082
-
1083
- @app.post("/forget")
1084
- async def forget(req: ForgetRequest):
1085
- """
1086
- Delete records by id or metadata filter. Restored from v0.4.x — was
1087
- removed in v0.5.x, causing test/bench pollution and blocking GDPR
1088
- deletion workflows.
1089
- """
1090
- if not req.id and not req.metadata_contains:
1091
- raise HTTPException(status_code=400, detail="provide id or metadata_contains")
1092
-
1093
- deleted_total = 0
1094
- # Forward to layers that support deletion. L6 doc-store supports both.
1095
- try:
1096
- if req.id:
1097
- r = await _client().delete(
1098
- f"{L6_DOC_URL}/purge",
1099
- params={"source_file": req.id},
1100
- timeout=10.0,
1101
- )
1102
- if r.status_code == 200:
1103
- deleted_total += int(r.json().get("deleted", 1))
1104
- elif req.metadata_contains:
1105
- r = await _client().post(
1106
- f"{L6_DOC_URL}/forget",
1107
- json={"metadata_contains": req.metadata_contains},
1108
- timeout=10.0,
1109
- )
1110
- if r.status_code == 200:
1111
- deleted_total += int(r.json().get("deleted", 0))
1112
- except Exception as exc:
1113
- print(f"[shim] L6 /forget failed: {exc}")
1114
-
1115
- # Also wipe L0 BM25 + L4 QMD + L3 KG so bench resets fully.
1116
- # No per-id forget for these — bench harness uses /forget once at
1117
- # start of each run with empty filters to reset state.
1118
- #
1119
- # Three forwarding rules:
1120
- # - metadata_contains has `arena` → tenant-scoped delete (forwards
1121
- # the arena to the internal endpoint, which only wipes that
1122
- # tenant's L3 data).
1123
- # - metadata_contains is set, no arena → targeted L6-only delete;
1124
- # do NOT call the internal wipe (we don't have arena scope and
1125
- # a global wipe would be wildly wrong here).
1126
- # - empty filters → bench-reset semantics; explicitly request the
1127
- # unsafe global wipe via the new `confirm: GLOBAL_WIPE` gate.
1128
- arena_for_internal = None
1129
- if isinstance(req.metadata_contains, dict):
1130
- meta_arena = req.metadata_contains.get("arena")
1131
- if isinstance(meta_arena, str) and meta_arena:
1132
- arena_for_internal = meta_arena
1133
- try:
1134
- if arena_for_internal:
1135
- payload = {"arena": arena_for_internal}
1136
- elif req.metadata_contains:
1137
- payload = None # skip — targeted delete shouldn't wipe shared layers
1138
- else:
1139
- payload = {"confirm": "GLOBAL_WIPE"}
1140
- if payload is not None:
1141
- r = await _client().post(
1142
- f"{L2_PROXY_URL}/forget-internal", json=payload, timeout=15.0,
1143
- )
1144
- if r.status_code == 200:
1145
- d = r.json().get("deleted", {})
1146
- deleted_total += sum(int(v or 0) for v in d.values())
1147
- except Exception as exc:
1148
- print(f"[shim] L2 /forget-internal failed: {exc}")
1149
-
1150
- return {"deleted": deleted_total, "engine": "pentatonic-memory-engine"}
1151
-
1152
-
1153
- @app.post("/aggregate")
1154
- async def aggregate(req: AggregateRequest) -> dict[str, Any]:
1155
- """Aggregate communications history with one person.
1156
-
1157
- Pass-through to the L2 proxy's /aggregate-internal which runs a
1158
- single Cypher query against L3's typed-Person graph
1159
- ((:Person)-[:COMMUNICATED]->(:Chunk)). Used by TES personFacets
1160
- once we cut over from the over-fetch v1.
1161
-
1162
- The shim's job here is shape validation + arena enforcement; the
1163
- real aggregation lives in L3. No layer fan-out, no metadata
1164
- filter scanning — when the typed-Person nodes don't exist for
1165
- this contact (older memories, tenants pre-#28), the response is
1166
- `total: 0` and the caller falls back to whatever it had before.
1167
- """
1168
- arena = (req.arena or "").strip()
1169
- if not arena:
1170
- raise HTTPException(status_code=400, detail="arena is required")
1171
- if not (req.contact_email or req.contact_name):
1172
- raise HTTPException(
1173
- status_code=400,
1174
- detail="provide contact_email and/or contact_name to identify the Person",
1175
- )
1176
- payload: dict[str, Any] = {
1177
- "arena": arena,
1178
- "group_by": req.group_by or ["channel"],
1179
- }
1180
- if req.contact_email:
1181
- payload["contact_email"] = req.contact_email
1182
- if req.contact_name:
1183
- payload["contact_name"] = req.contact_name
1184
- try:
1185
- r = await _client().post(
1186
- f"{L2_PROXY_URL}/aggregate-internal", json=payload, timeout=15.0,
1187
- )
1188
- if r.status_code != 200:
1189
- raise HTTPException(
1190
- status_code=r.status_code,
1191
- detail=f"aggregate failed: {r.text[:200]}",
1192
- )
1193
- return r.json()
1194
- except HTTPException:
1195
- raise
1196
- except Exception as exc:
1197
- raise HTTPException(status_code=502, detail=f"aggregate upstream: {exc}")
1198
-
1199
-
1200
- @app.post("/people-list")
1201
- async def people_list(req: PeopleListRequest) -> dict[str, Any]:
1202
- """List all people across one or more arenas.
1203
-
1204
- Pass-through to the L2 proxy's /people-list-internal which runs a
1205
- single Cypher pass over the ChannelStat denorm keyed on
1206
- (arena, person_email, channel). Returns paginated, filtered,
1207
- sorted rows with per-channel breakdown.
1208
-
1209
- Shim's job: shape validation + arena enforcement. The real
1210
- aggregation lives in L3 over ChannelStat — see PRs #28-33 for
1211
- the writer side that maintains those nodes.
1212
-
1213
- Multi-arena: legitimate here (unlike /aggregate). The UI
1214
- fetches "people known by Philip OR Jeanne" by passing both
1215
- arenas; the L2 proxy collapses duplicates by person_email.
1216
- """
1217
- arenas = [a.strip() for a in (req.arenas or []) if a and a.strip()]
1218
- if not arenas:
1219
- raise HTTPException(status_code=400, detail="at least one arena is required")
1220
- payload: dict[str, Any] = {
1221
- "arenas": arenas,
1222
- "limit": req.limit,
1223
- "offset": req.offset,
1224
- "order_by": req.order_by,
1225
- }
1226
- if req.emails:
1227
- # Lowercase here so the L2 path can rely on exact-match; the
1228
- # writer side also lowercases person_email on the ChannelStat
1229
- # node, so this stays consistent.
1230
- payload["emails"] = [e.strip().lower() for e in req.emails if e and e.strip()]
1231
- if req.search:
1232
- payload["search"] = req.search.strip()
1233
- try:
1234
- r = await _client().post(
1235
- f"{L2_PROXY_URL}/people-list-internal", json=payload, timeout=15.0,
1236
- )
1237
- if r.status_code != 200:
1238
- raise HTTPException(
1239
- status_code=r.status_code,
1240
- detail=f"people-list failed: {r.text[:200]}",
1241
- )
1242
- return r.json()
1243
- except HTTPException:
1244
- raise
1245
- except Exception as exc:
1246
- raise HTTPException(status_code=502, detail=f"people-list upstream: {exc}")
1247
-
1248
-
1249
- # ----------------------------------------------------------------------
1250
- # Entrypoint
1251
- # ----------------------------------------------------------------------
1252
-
1253
- if __name__ == "__main__":
1254
- import uvicorn
1255
- uvicorn.run("server:app", host="0.0.0.0", port=PORT, log_level="info")