@pentatonic-ai/ai-agent-sdk 0.9.5 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/bin/cli.js +1 -1
- package/bin/commands/config.js +1 -1
- package/dist/index.cjs +39 -72
- package/dist/index.js +36 -69
- package/package.json +10 -3
- package/packages/doctor/src/checks/local-memory.js +2 -2
- package/packages/memory/README.md +2 -2
- package/packages/memory/openclaw-plugin/README.md +2 -2
- package/packages/memory/openclaw-plugin/openclaw.plugin.json +1 -1
- package/packages/memory/package-lock.json +49 -33
- package/packages/memory/package.json +4 -1
- package/packages/memory/src/__tests__/engine.test.js +40 -5
- package/packages/memory/src/engine.js +38 -3
- package/packages/memory/src/server.js +2 -2
- package/packages/memory-engine-v2/.env.example +30 -0
- package/packages/memory-engine-v2/README.md +125 -0
- package/packages/memory-engine-v2/compat/Dockerfile +11 -0
- package/packages/memory-engine-v2/compat/requirements.txt +6 -0
- package/packages/memory-engine-v2/compat/server.py +1047 -0
- package/packages/memory-engine-v2/docker-compose.aws.yml +78 -0
- package/packages/memory-engine-v2/docker-compose.yml +206 -0
- package/packages/memory-engine-v2/extractor-async/Dockerfile +14 -0
- package/packages/memory-engine-v2/extractor-async/confidence.py +62 -0
- package/packages/memory-engine-v2/extractor-async/noise_filter.py +144 -0
- package/packages/memory-engine-v2/extractor-async/requirements.txt +2 -0
- package/packages/memory-engine-v2/extractor-async/test_confidence.py +76 -0
- package/packages/memory-engine-v2/extractor-async/test_noise_filter.py +177 -0
- package/packages/memory-engine-v2/extractor-async/worker.py +797 -0
- package/packages/memory-engine-v2/extractor-sync/Dockerfile +11 -0
- package/packages/memory-engine-v2/extractor-sync/requirements.txt +4 -0
- package/packages/memory-engine-v2/extractor-sync/server.py +424 -0
- package/packages/memory-engine-v2/org-model/migrations/001_init.sql +390 -0
- package/packages/memory-engine-v2/tests/e2e_smoke.py +356 -0
- package/packages/memory-engine-v2/tests/fixtures/generate_synthetic_corpus.py +758 -0
- package/packages/memory-engine/.env.example +0 -13
- package/packages/memory-engine/MIGRATION.md +0 -219
- package/packages/memory-engine/README.md +0 -145
- package/packages/memory-engine/bench/README.md +0 -99
- package/packages/memory-engine/bench/scorecards-engine/agent-coding__pentatonic-baseline__20260427-142523.json +0 -1115
- package/packages/memory-engine/bench/scorecards-engine/chat-recall__pentatonic-baseline__20260427-142648.json +0 -819
- package/packages/memory-engine/bench/scorecards-engine/circular-economy__pentatonic-baseline__20260427-142757.json +0 -1278
- package/packages/memory-engine/bench/scorecards-engine/customer-support__pentatonic-baseline__20260427-142900.json +0 -1018
- package/packages/memory-engine/bench/scorecards-engine/marketplace-ops__pentatonic-baseline__20260427-142957.json +0 -1038
- package/packages/memory-engine/bench/scorecards-engine/product-catalogue__pentatonic-baseline__20260427-143122.json +0 -961
- package/packages/memory-engine/bench/scorecards-engine-via-docker/agent-coding__pentatonic-memory__20260427-161812.json +0 -1115
- package/packages/memory-engine/bench/scorecards-engine-via-docker/chat-recall__pentatonic-memory__20260427-161701.json +0 -819
- package/packages/memory-engine/bench/scorecards-engine-via-docker/circular-economy__pentatonic-memory__20260427-161713.json +0 -1278
- package/packages/memory-engine/bench/scorecards-engine-via-docker/customer-support__pentatonic-memory__20260427-161723.json +0 -1018
- package/packages/memory-engine/bench/scorecards-engine-via-docker/marketplace-ops__pentatonic-memory__20260427-161732.json +0 -1038
- package/packages/memory-engine/bench/scorecards-engine-via-docker/product-catalogue__pentatonic-memory__20260427-161741.json +0 -937
- package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/agent-coding__pentatonic-memory__20260427-184718.json +0 -1115
- package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/chat-recall__pentatonic-memory__20260427-184614.json +0 -819
- package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/circular-economy__pentatonic-memory__20260427-184809.json +0 -1278
- package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/customer-support__pentatonic-memory__20260427-184854.json +0 -1018
- package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/marketplace-ops__pentatonic-memory__20260427-184929.json +0 -1038
- package/packages/memory-engine/bench/scorecards-engine-via-l2-7-layer-populated/product-catalogue__pentatonic-memory__20260427-185015.json +0 -961
- package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/agent-coding__pentatonic-memory__20260427-175252.json +0 -1115
- package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/chat-recall__pentatonic-memory__20260427-175312.json +0 -819
- package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/circular-economy__pentatonic-memory__20260427-175335.json +0 -1278
- package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/customer-support__pentatonic-memory__20260427-175355.json +0 -1018
- package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/marketplace-ops__pentatonic-memory__20260427-175413.json +0 -1038
- package/packages/memory-engine/bench/scorecards-engine-via-l2-empty-layers/product-catalogue__pentatonic-memory__20260427-175430.json +0 -883
- package/packages/memory-engine/bench/scorecards-engine-via-shim/agent-coding__pentatonic-memory__20260427-155409.json +0 -1115
- package/packages/memory-engine/bench/scorecards-engine-via-shim/chat-recall__pentatonic-memory__20260427-155421.json +0 -819
- package/packages/memory-engine/bench/scorecards-engine-via-shim/circular-economy__pentatonic-memory__20260427-155433.json +0 -1278
- package/packages/memory-engine/bench/scorecards-engine-via-shim/customer-support__pentatonic-memory__20260427-155443.json +0 -1018
- package/packages/memory-engine/bench/scorecards-engine-via-shim/marketplace-ops__pentatonic-memory__20260427-155453.json +0 -1038
- package/packages/memory-engine/bench/scorecards-engine-via-shim/product-catalogue__pentatonic-memory__20260427-155503.json +0 -937
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/agent-coding__pentatonic-memory-latest__20260427-145103.json +0 -1115
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/agent-coding__pentatonic-memory__20260427-144909.json +0 -1115
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/chat-recall__pentatonic-memory-latest__20260427-145153.json +0 -819
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/chat-recall__pentatonic-memory__20260427-145120.json +0 -542
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/circular-economy__pentatonic-memory-latest__20260427-145313.json +0 -1278
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/circular-economy__pentatonic-memory__20260427-145207.json +0 -894
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/customer-support__pentatonic-memory-latest__20260427-145412.json +0 -1018
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/customer-support__pentatonic-memory__20260427-145327.json +0 -680
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/marketplace-ops__pentatonic-memory-latest__20260427-145517.json +0 -1038
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/marketplace-ops__pentatonic-memory__20260427-145422.json +0 -693
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/product-catalogue__pentatonic-memory-latest__20260427-145616.json +0 -961
- package/packages/memory-engine/bench/scorecards-pentatonic-baseline/product-catalogue__pentatonic-memory__20260427-145528.json +0 -727
- package/packages/memory-engine/compat/Dockerfile +0 -22
- package/packages/memory-engine/compat/server.py +0 -1255
- package/packages/memory-engine/docker-compose.test.yml +0 -59
- package/packages/memory-engine/docker-compose.yml +0 -240
- package/packages/memory-engine/engine/README.md +0 -52
- package/packages/memory-engine/engine/l2-hybridrag-proxy.py +0 -1543
- package/packages/memory-engine/engine/l5-comms-layer.py +0 -663
- package/packages/memory-engine/engine/l6-document-store.py +0 -1018
- package/packages/memory-engine/engine/services/_shared/__init__.py +0 -1
- package/packages/memory-engine/engine/services/_shared/embed_provider.py +0 -468
- package/packages/memory-engine/engine/services/l2/Dockerfile +0 -50
- package/packages/memory-engine/engine/services/l2/init_databases.py +0 -81
- package/packages/memory-engine/engine/services/l2/l2-hybridrag-proxy.py +0 -2721
- package/packages/memory-engine/engine/services/l5/Dockerfile +0 -11
- package/packages/memory-engine/engine/services/l5/l5-comms-layer.py +0 -808
- package/packages/memory-engine/engine/services/l6/Dockerfile +0 -30
- package/packages/memory-engine/engine/services/l6/l6-document-store.py +0 -1221
- package/packages/memory-engine/engine/services/nv-embed/Dockerfile +0 -28
- package/packages/memory-engine/engine/services/nv-embed/server.py +0 -152
- package/packages/memory-engine/pme_memory/__init__.py +0 -0
- package/packages/memory-engine/pme_memory/__main__.py +0 -129
- package/packages/memory-engine/pme_memory/artifacts.py +0 -95
- package/packages/memory-engine/pme_memory/embed.py +0 -74
- package/packages/memory-engine/pme_memory/health.py +0 -36
- package/packages/memory-engine/pme_memory/hygiene.py +0 -159
- package/packages/memory-engine/pme_memory/indexer.py +0 -200
- package/packages/memory-engine/pme_memory/needs.py +0 -55
- package/packages/memory-engine/pme_memory/provenance.py +0 -80
- package/packages/memory-engine/pme_memory/scoring.py +0 -168
- package/packages/memory-engine/pme_memory/search.py +0 -52
- package/packages/memory-engine/pme_memory/store.py +0 -86
- package/packages/memory-engine/pme_memory/synthesis.py +0 -114
- package/packages/memory-engine/pyproject.toml +0 -65
- package/packages/memory-engine/scripts/kg-extractor.py +0 -557
- package/packages/memory-engine/scripts/kg-preflexor-v2.py +0 -738
- package/packages/memory-engine/scripts/wipe-legacy-l3-entities.py +0 -128
- package/packages/memory-engine/tests/e2e_arena.sh +0 -259
- package/packages/memory-engine/tests/embed_stub/Dockerfile +0 -13
- package/packages/memory-engine/tests/embed_stub/server.py +0 -80
- package/packages/memory-engine/tests/test_aggregate.py +0 -333
- package/packages/memory-engine/tests/test_api_contract.sh +0 -57
- package/packages/memory-engine/tests/test_arena_safety.py +0 -232
- package/packages/memory-engine/tests/test_channel_stat_reader.py +0 -437
- package/packages/memory-engine/tests/test_channel_stat_rollups.py +0 -308
- package/packages/memory-engine/tests/test_compat_nv_embed_probe.py +0 -48
- package/packages/memory-engine/tests/test_embed_provider.py +0 -492
- package/packages/memory-engine/tests/test_l2_qmd_vec_search.py +0 -280
- package/packages/memory-engine/tests/test_l3_arena_isolation.py +0 -412
- package/packages/memory-engine/tests/test_l6_module_load.py +0 -84
- package/packages/memory-engine/tests/test_people_list_reader.py +0 -432
|
@@ -1,1255 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
pentatonic-memory-engine compatibility shim.
|
|
3
|
-
|
|
4
|
-
Exposes the same HTTP API as `pentatonic-memory` v0.5.x (POST /store,
|
|
5
|
-
POST /search, GET /health) plus the v0.6 regression-fix endpoints
|
|
6
|
-
(POST /store-batch, POST /forget). Internally routes every call through
|
|
7
|
-
the 7-layer hybrid retrieval engine running in sibling containers
|
|
8
|
-
(L0 BM25, L1 core files, L2 HybridRAG orchestrator, L3 Knowledge Graph,
|
|
9
|
-
L4 vec, L5 Milvus, L6 doc-store).
|
|
10
|
-
|
|
11
|
-
Drop-in replacement: change a single base URL in your existing
|
|
12
|
-
pentatonic-memory SDK client and you get ~5x retrieval accuracy
|
|
13
|
-
without touching anything else.
|
|
14
|
-
|
|
15
|
-
Endpoints:
|
|
16
|
-
|
|
17
|
-
POST /store { content, metadata } → { id, content, layerId }
|
|
18
|
-
POST /store-batch { records: [{ id?, content, metadata }] } → { inserted, ids[], embed_ms, insert_ms }
|
|
19
|
-
POST /search { query, limit, min_score } → { results: [...] }
|
|
20
|
-
POST /forget { metadata_contains } | { id } → { deleted: N }
|
|
21
|
-
GET /health → { status, layers: {...}, memories }
|
|
22
|
-
|
|
23
|
-
Environment:
|
|
24
|
-
|
|
25
|
-
L0_URL default http://l0:8030
|
|
26
|
-
L2_PROXY_URL default http://l2:8031
|
|
27
|
-
L3_KG_URL default http://l3:8047
|
|
28
|
-
L5_MILVUS_URL default http://l5:8035
|
|
29
|
-
L6_DOC_URL default http://l6:8037
|
|
30
|
-
NV_EMBED_URL default http://nv-embed:8041/v1/embeddings
|
|
31
|
-
PORT default 8099 (matches pentatonic-memory v0.5)
|
|
32
|
-
"""
|
|
33
|
-
|
|
34
|
-
import hashlib
|
|
35
|
-
import os
|
|
36
|
-
import sys
|
|
37
|
-
import time
|
|
38
|
-
from datetime import datetime, timezone
|
|
39
|
-
from typing import Any, Optional
|
|
40
|
-
|
|
41
|
-
import httpx
|
|
42
|
-
from fastapi import FastAPI, HTTPException
|
|
43
|
-
from pydantic import BaseModel, Field
|
|
44
|
-
|
|
45
|
-
# Reach into the engine/services tree so we can reuse EmbedClient. The
|
|
46
|
-
# tree isn't a real installed package; layer services and the compat
|
|
47
|
-
# shim both side-load it the same way. Keeps the chunking + auto-detect
|
|
48
|
-
# behaviour identical between the shim's pre-embed and the per-layer
|
|
49
|
-
# embeds that previously did the same work N times.
|
|
50
|
-
sys.path.insert(
|
|
51
|
-
0,
|
|
52
|
-
os.path.join(os.path.dirname(__file__), "..", "engine", "services"),
|
|
53
|
-
)
|
|
54
|
-
from _shared.embed_provider import EmbedClient # noqa: E402
|
|
55
|
-
|
|
56
|
-
# ----------------------------------------------------------------------
|
|
57
|
-
# Config
|
|
58
|
-
# ----------------------------------------------------------------------
|
|
59
|
-
|
|
60
|
-
L0_URL = os.environ.get("L0_URL", "http://l0:8030")
|
|
61
|
-
L2_PROXY_URL = os.environ.get("L2_PROXY_URL", "http://l2:8031")
|
|
62
|
-
L3_KG_URL = os.environ.get("L3_KG_URL", "http://l3:8047")
|
|
63
|
-
L5_MILVUS_URL = os.environ.get("L5_MILVUS_URL", "http://l5:8035")
|
|
64
|
-
L6_DOC_URL = os.environ.get("L6_DOC_URL", "http://l6:8037")
|
|
65
|
-
NV_EMBED_URL = os.environ.get("NV_EMBED_URL", "http://nv-embed:8041/v1/embeddings")
|
|
66
|
-
|
|
67
|
-
# Neo4j has no /health endpoint, so the shim probes the HTTP transactional
|
|
68
|
-
# API with a trivial RETURN 1 — that confirms Neo4j is actually answering
|
|
69
|
-
# Cypher, not just serving HTTP. Auth shape is the same as L2 / docker-compose:
|
|
70
|
-
# "user/pass" string. Default matches the local-dev compose default.
|
|
71
|
-
NEO4J_AUTH = os.environ.get("NEO4J_AUTH", "neo4j/local-dev-pw")
|
|
72
|
-
NEO4J_DB = os.environ.get("NEO4J_DB", "neo4j")
|
|
73
|
-
|
|
74
|
-
PORT = int(os.environ.get("PORT", "8099"))
|
|
75
|
-
|
|
76
|
-
# Shared-embed mode. When on, /store-batch computes embeddings once at
|
|
77
|
-
# the shim level and forwards them to each layer's /index-batch so the
|
|
78
|
-
# layer skips its own embed call. Cuts gateway RPC count by ~4× (L4 +
|
|
79
|
-
# L5 + L6 + L2-internal all did the same embed work independently).
|
|
80
|
-
# Default ON because all layers in this engine use the same NV-Embed
|
|
81
|
-
# model; disable if you ever wire up per-layer differentiated embedders
|
|
82
|
-
# (e.g. cohere on L5, openai on L4).
|
|
83
|
-
SHARE_EMBEDDINGS = os.environ.get("PME_SHARE_EMBEDDINGS", "true").lower() == "true"
|
|
84
|
-
|
|
85
|
-
_embed_client: EmbedClient | None = None
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
def _get_embed_client() -> EmbedClient:
|
|
89
|
-
"""Lazy-init the shim's EmbedClient using PME_-prefixed env vars
|
|
90
|
-
(matches L2's pattern). Cached for the process lifetime so the
|
|
91
|
-
auto-detect handshake only happens once."""
|
|
92
|
-
global _embed_client
|
|
93
|
-
if _embed_client is None:
|
|
94
|
-
_embed_client = EmbedClient.from_env(
|
|
95
|
-
prefix="PME_",
|
|
96
|
-
default_url=NV_EMBED_URL,
|
|
97
|
-
)
|
|
98
|
-
return _embed_client
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
# Layer types we surface as the SDK 4-layer projection. Engine stores
|
|
102
|
-
# everything as chunks tagged with arena + layer_type metadata; this
|
|
103
|
-
# helper renders the legacy `ml_<arena>_<type>` layer-id from the
|
|
104
|
-
# per-row arena, so the response reflects the actual data not a
|
|
105
|
-
# deployment-wide constant. Falls back to "episodic" when arena or
|
|
106
|
-
# layer_type is missing.
|
|
107
|
-
def _layer_id(arena: Optional[str], layer_type: Optional[str] = None) -> str:
|
|
108
|
-
a = arena or "general"
|
|
109
|
-
t = layer_type or "episodic"
|
|
110
|
-
return f"ml_{a}_{t}"
|
|
111
|
-
|
|
112
|
-
# Test/isolated mode: bypass the L2 HybridRAG orchestrator and query L6 directly.
|
|
113
|
-
# Useful for bench harnesses where you want to validate the ingest+search
|
|
114
|
-
# round-trip against an isolated test L6 instance, without the L2 proxy
|
|
115
|
-
# pulling in production data from other layers.
|
|
116
|
-
BYPASS_L2 = os.environ.get("BYPASS_L2_PROXY", "0") in ("1", "true", "yes")
|
|
117
|
-
|
|
118
|
-
VERSION = "0.1.0"
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
# ----------------------------------------------------------------------
|
|
122
|
-
# Request / response models (match pentatonic-memory v0.5 wire format)
|
|
123
|
-
# ----------------------------------------------------------------------
|
|
124
|
-
|
|
125
|
-
class StoreRequest(BaseModel):
|
|
126
|
-
content: str
|
|
127
|
-
metadata: Optional[dict[str, Any]] = None
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
class StoreBatchRequest(BaseModel):
|
|
131
|
-
records: list[dict[str, Any]] = Field(default_factory=list)
|
|
132
|
-
arena: Optional[str] = "general"
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
class SearchRequest(BaseModel):
|
|
136
|
-
query: str
|
|
137
|
-
limit: Optional[int] = 10
|
|
138
|
-
min_score: Optional[float] = 0.001
|
|
139
|
-
# Tenant scope (single arena). Back-compat shape — single-arena callers
|
|
140
|
-
# can keep sending this. Treated as a one-element `arenas` list.
|
|
141
|
-
arena: Optional[str] = None
|
|
142
|
-
# Multi-arena scope. Used by callers that want to span both a tenant-
|
|
143
|
-
# wide arena ("acme") and a user-scoped arena ("acme:user-42") in one
|
|
144
|
-
# search — the SDK helper composes this list automatically when a
|
|
145
|
-
# `userId` is supplied. Authoritative when both `arena` and `arenas`
|
|
146
|
-
# are present; engine treats it as a set.
|
|
147
|
-
arenas: Optional[list[str]] = None
|
|
148
|
-
# Arbitrary metadata equality filters, applied as a post-filter on
|
|
149
|
-
# the shim. Useful for `kind`, `layer_type`, `source_repo`, etc.
|
|
150
|
-
# Keys not present on a result's metadata are treated as no-match.
|
|
151
|
-
# Each pair is exact string equality. Engine doesn't currently
|
|
152
|
-
# forward these to underlying stores, so over-fetch happens; the
|
|
153
|
-
# shim trims to the requested limit after filtering.
|
|
154
|
-
metadata_filter: Optional[dict[str, Any]] = None
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
class ForgetRequest(BaseModel):
|
|
158
|
-
metadata_contains: Optional[dict[str, Any]] = None
|
|
159
|
-
id: Optional[str] = None
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
class AggregateRequest(BaseModel):
|
|
163
|
-
"""Public-facing /aggregate request.
|
|
164
|
-
|
|
165
|
-
Mirrors /aggregate-internal on the L2 proxy with arena scoping
|
|
166
|
-
enforced at the shim layer (multi-arena spans aren't allowed
|
|
167
|
-
here — pick a single arena per call). The relationships UI
|
|
168
|
-
queries this directly via TES; future callers will too.
|
|
169
|
-
"""
|
|
170
|
-
|
|
171
|
-
arena: str
|
|
172
|
-
contact_email: Optional[str] = None
|
|
173
|
-
contact_name: Optional[str] = None
|
|
174
|
-
group_by: Optional[list[str]] = None
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
class PeopleListRequest(BaseModel):
|
|
178
|
-
"""Public-facing /people-list request.
|
|
179
|
-
|
|
180
|
-
Corpus-level "all people across these arenas" projection backing
|
|
181
|
-
the Pip Relationships UI list page. Distinct from /aggregate
|
|
182
|
-
(which is per-person): /people-list returns one row per Person
|
|
183
|
-
across the whole arena set, with per-channel breakdown nested.
|
|
184
|
-
|
|
185
|
-
Multi-arena is supported here (unlike /aggregate) because the
|
|
186
|
-
UI use case — "people known by Philip OR Jeanne" — is a
|
|
187
|
-
legitimate cross-user view. The L2 proxy does the dedup by
|
|
188
|
-
person_email.
|
|
189
|
-
|
|
190
|
-
`emails`: batched-mode filter. Return only these emails.
|
|
191
|
-
`search`: case-insensitive substring on person_name and
|
|
192
|
-
person_email.
|
|
193
|
-
`order_by`: whitelisted sort key; see the L2 proxy for the
|
|
194
|
-
allowlist.
|
|
195
|
-
"""
|
|
196
|
-
|
|
197
|
-
arenas: list[str]
|
|
198
|
-
emails: Optional[list[str]] = None
|
|
199
|
-
search: Optional[str] = None
|
|
200
|
-
limit: int = 200
|
|
201
|
-
offset: int = 0
|
|
202
|
-
order_by: str = "last_seen_desc"
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
# ----------------------------------------------------------------------
|
|
206
|
-
# Engine clients (one per layer)
|
|
207
|
-
# ----------------------------------------------------------------------
|
|
208
|
-
|
|
209
|
-
_http: Optional[httpx.AsyncClient] = None
|
|
210
|
-
|
|
211
|
-
# In-memory metadata sidecar — per-id stash so arbitrary client metadata
|
|
212
|
-
# (e.g. {"bench_tag": "...", "doc_id": "..."}) survives the round-trip
|
|
213
|
-
# even when the underlying L5/L6 schemas don't carry a JSON metadata column.
|
|
214
|
-
# Bounded to the most recent 100k entries to avoid leaking memory in long-
|
|
215
|
-
# running deployments. Resets on shim restart.
|
|
216
|
-
from collections import OrderedDict
|
|
217
|
-
_META_CACHE: "OrderedDict[str, dict[str, Any]]" = OrderedDict()
|
|
218
|
-
_META_CACHE_MAX = 100_000
|
|
219
|
-
|
|
220
|
-
def _stash_meta(rid: str, meta: dict[str, Any] | None) -> None:
|
|
221
|
-
if not rid:
|
|
222
|
-
return
|
|
223
|
-
_META_CACHE[rid] = meta or {}
|
|
224
|
-
while len(_META_CACHE) > _META_CACHE_MAX:
|
|
225
|
-
_META_CACHE.popitem(last=False)
|
|
226
|
-
|
|
227
|
-
def _lookup_meta(rid: str) -> dict[str, Any]:
|
|
228
|
-
return _META_CACHE.get(rid, {}) if rid else {}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
def _stash_all_keys(rid: str, meta: dict[str, Any], arena: str = "general") -> None:
|
|
232
|
-
"""Stash meta under every id-shape any of the 7 layers might echo back.
|
|
233
|
-
|
|
234
|
-
L4 sidecar: rid (and `<rid>.md`)
|
|
235
|
-
L5 comms: rid (path may be `.pentatonic/chats/<rid>.jsonl`)
|
|
236
|
-
L6 doc-store: `l6:<rid>:0`, `<rid>.md` (chunk_id, source_file)
|
|
237
|
-
L2 internal L0/L4-qmd: `bench/<arena>/<rid>.md`,
|
|
238
|
-
`bench/<arena>/<doc_id>.md`
|
|
239
|
-
L3 graph chunk: rid (Chunk.id) and doc_id
|
|
240
|
-
"""
|
|
241
|
-
if not rid:
|
|
242
|
-
return
|
|
243
|
-
meta = meta or {}
|
|
244
|
-
keys = {
|
|
245
|
-
rid,
|
|
246
|
-
f"{rid}.md",
|
|
247
|
-
f"l6:{rid}:0",
|
|
248
|
-
f"bench/{arena}/{rid}.md",
|
|
249
|
-
f"bench/{arena}/{rid}",
|
|
250
|
-
}
|
|
251
|
-
doc_id = meta.get("doc_id")
|
|
252
|
-
if doc_id:
|
|
253
|
-
keys.update({
|
|
254
|
-
doc_id,
|
|
255
|
-
f"{doc_id}.md",
|
|
256
|
-
f"l6:{doc_id}:0",
|
|
257
|
-
f"bench/{arena}/{doc_id}.md",
|
|
258
|
-
f"bench/{arena}/{doc_id}",
|
|
259
|
-
})
|
|
260
|
-
path = meta.get("path")
|
|
261
|
-
if path:
|
|
262
|
-
keys.add(path)
|
|
263
|
-
keys.add(path.rsplit(".", 1)[0])
|
|
264
|
-
for k in keys:
|
|
265
|
-
if k:
|
|
266
|
-
_stash_meta(k, meta)
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
def _client() -> httpx.AsyncClient:
|
|
270
|
-
global _http
|
|
271
|
-
if _http is None:
|
|
272
|
-
_http = httpx.AsyncClient(timeout=60.0)
|
|
273
|
-
return _http
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
async def _embed_batch(texts: list[str]) -> list[list[float]]:
|
|
277
|
-
"""One NV-Embed call for many texts. Returns vectors in input order."""
|
|
278
|
-
if not texts:
|
|
279
|
-
return []
|
|
280
|
-
resp = await _client().post(
|
|
281
|
-
NV_EMBED_URL,
|
|
282
|
-
json={"input": texts, "model": "nv-embed-v2"},
|
|
283
|
-
timeout=120.0,
|
|
284
|
-
)
|
|
285
|
-
resp.raise_for_status()
|
|
286
|
-
return [d["embedding"] for d in resp.json()["data"]]
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
async def _index_l5(
|
|
290
|
-
records: list[dict[str, Any]],
|
|
291
|
-
arena: str = "general",
|
|
292
|
-
embeddings: list[list[float]] | None = None,
|
|
293
|
-
) -> int:
|
|
294
|
-
"""Index records into the L5 Milvus comms layer (chats collection).
|
|
295
|
-
|
|
296
|
-
arena is forwarded as a Milvus dynamic field so /search can filter
|
|
297
|
-
by arena natively (vs the shim's defence-in-depth post-filter).
|
|
298
|
-
|
|
299
|
-
When `embeddings` is supplied (parallel to records), L5 skips its
|
|
300
|
-
own embed call — the shim pre-computes vectors once at /store-batch
|
|
301
|
-
level and threads them through each layer to avoid 3× redundant
|
|
302
|
-
embed RPCs (L5 + L6 + L2-internal otherwise each re-embed the same
|
|
303
|
-
texts in parallel).
|
|
304
|
-
"""
|
|
305
|
-
payload: dict[str, Any] = {
|
|
306
|
-
"collection": "chats",
|
|
307
|
-
"records": [
|
|
308
|
-
{
|
|
309
|
-
"id": r.get("id") or hashlib.sha1(r["content"].encode()).hexdigest()[:32],
|
|
310
|
-
"text": r["content"],
|
|
311
|
-
"source": (r.get("metadata") or {}).get("source", "shim"),
|
|
312
|
-
"channel": "pentatonic-memory",
|
|
313
|
-
"contact": (r.get("metadata") or {}).get("user", ""),
|
|
314
|
-
"arena": (r.get("metadata") or {}).get("arena") or arena,
|
|
315
|
-
}
|
|
316
|
-
for r in records
|
|
317
|
-
],
|
|
318
|
-
}
|
|
319
|
-
if embeddings is not None:
|
|
320
|
-
payload["embeddings"] = embeddings
|
|
321
|
-
try:
|
|
322
|
-
resp = await _client().post(f"{L5_MILVUS_URL}/index-batch", json=payload, timeout=60.0)
|
|
323
|
-
resp.raise_for_status()
|
|
324
|
-
return resp.json().get("inserted", 0)
|
|
325
|
-
except Exception as exc:
|
|
326
|
-
# Best-effort: L5 is one of five redundant layers; failure here
|
|
327
|
-
# doesn't mean the record is unsearchable. L0 BM25 + L4 QMD +
|
|
328
|
-
# L6 doc-store all carry it independently.
|
|
329
|
-
print(f"[shim] L5 index-batch failed: {exc}")
|
|
330
|
-
return 0
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
async def _index_l6(
|
|
334
|
-
records: list[dict[str, Any]],
|
|
335
|
-
arena: str = "general",
|
|
336
|
-
embeddings: list[list[float]] | None = None,
|
|
337
|
-
) -> int:
|
|
338
|
-
"""Index records into the L6 document store.
|
|
339
|
-
|
|
340
|
-
When `embeddings` is supplied (parallel to records), L6 skips its
|
|
341
|
-
own embed call — the shim pre-computes vectors once at /store-batch
|
|
342
|
-
level and threads them through each layer.
|
|
343
|
-
"""
|
|
344
|
-
payload: dict[str, Any] = {
|
|
345
|
-
"arena": arena,
|
|
346
|
-
"records": [
|
|
347
|
-
{
|
|
348
|
-
"id": r.get("id") or hashlib.sha1(r["content"].encode()).hexdigest()[:32],
|
|
349
|
-
"text": r["content"],
|
|
350
|
-
"source_file": (r.get("metadata") or {}).get("path") or f"{r.get('id', 'doc')}.md",
|
|
351
|
-
"doc_type": (r.get("metadata") or {}).get("doc_type", "general"),
|
|
352
|
-
"heading": (r.get("metadata") or {}).get("heading", ""),
|
|
353
|
-
}
|
|
354
|
-
for r in records
|
|
355
|
-
],
|
|
356
|
-
}
|
|
357
|
-
if embeddings is not None:
|
|
358
|
-
payload["embeddings"] = embeddings
|
|
359
|
-
try:
|
|
360
|
-
resp = await _client().post(f"{L6_DOC_URL}/index-batch", json=payload, timeout=120.0)
|
|
361
|
-
resp.raise_for_status()
|
|
362
|
-
return resp.json().get("inserted", 0)
|
|
363
|
-
except Exception as exc:
|
|
364
|
-
print(f"[shim] L6 index-batch failed: {exc}")
|
|
365
|
-
return 0
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
async def _index_l2_internal(
|
|
369
|
-
records: list[dict[str, Any]],
|
|
370
|
-
arena: str = "general",
|
|
371
|
-
embeddings: list[list[float]] | None = None,
|
|
372
|
-
) -> dict:
|
|
373
|
-
"""Populate L2's internal stores: L0 BM25 + L4 QMD vec + L3 Neo4j KG.
|
|
374
|
-
|
|
375
|
-
Without this, L2's RRF fusion runs over empty L0/L4-qmd/L3 layers and
|
|
376
|
-
those zero-result rank lists pollute the score. The L2 proxy exposes
|
|
377
|
-
/index-internal-batch which writes to all three in one round-trip.
|
|
378
|
-
|
|
379
|
-
When `embeddings` is supplied (parallel to records), L2's internal
|
|
380
|
-
embed call (used for L4-QMD population) is skipped — the shim
|
|
381
|
-
pre-computes vectors once at /store-batch level and threads them
|
|
382
|
-
through to L4_QMD via this endpoint.
|
|
383
|
-
"""
|
|
384
|
-
payload: dict[str, Any] = {
|
|
385
|
-
"arena": arena,
|
|
386
|
-
"records": [
|
|
387
|
-
{
|
|
388
|
-
"id": r.get("id") or hashlib.sha1(r["content"].encode()).hexdigest()[:32],
|
|
389
|
-
"content": r["content"],
|
|
390
|
-
"metadata": r.get("metadata") or {},
|
|
391
|
-
}
|
|
392
|
-
for r in records
|
|
393
|
-
],
|
|
394
|
-
}
|
|
395
|
-
if embeddings is not None:
|
|
396
|
-
payload["embeddings"] = embeddings
|
|
397
|
-
try:
|
|
398
|
-
resp = await _client().post(f"{L2_PROXY_URL}/index-internal-batch",
|
|
399
|
-
json=payload, timeout=180.0)
|
|
400
|
-
resp.raise_for_status()
|
|
401
|
-
return resp.json()
|
|
402
|
-
except Exception as exc:
|
|
403
|
-
print(f"[shim] L2 internal index-batch failed: {exc}")
|
|
404
|
-
return {"l0": 0, "l4_qmd": 0, "l3_entities": 0, "l3_chunks": 0}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
# ----------------------------------------------------------------------
|
|
408
|
-
# FastAPI app
|
|
409
|
-
# ----------------------------------------------------------------------
|
|
410
|
-
|
|
411
|
-
app = FastAPI(
|
|
412
|
-
title="pentatonic-memory-engine compat shim",
|
|
413
|
-
version=VERSION,
|
|
414
|
-
description="Drop-in API compat for pentatonic-memory v0.5; routed through the 7-layer engine.",
|
|
415
|
-
)
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
def _interpret_body_status(body: Any) -> str | None:
|
|
419
|
-
"""Pull a layer's self-reported status out of its /health body.
|
|
420
|
-
|
|
421
|
-
Layers don't all use the same vocabulary — L4 says "ok"/"degraded",
|
|
422
|
-
L2 says "healthy"/"unavailable", some return nothing. Normalize to
|
|
423
|
-
"ok" or a short failure reason; None means the body didn't carry
|
|
424
|
-
a status field, in which case the HTTP code is the source of truth.
|
|
425
|
-
"""
|
|
426
|
-
if not isinstance(body, dict):
|
|
427
|
-
return None
|
|
428
|
-
raw = body.get("status")
|
|
429
|
-
if raw is None:
|
|
430
|
-
return None
|
|
431
|
-
s = str(raw).lower()
|
|
432
|
-
if s in ("ok", "healthy"):
|
|
433
|
-
return "ok"
|
|
434
|
-
err = body.get("error") or body.get("reason") or ""
|
|
435
|
-
return f"{s}: {str(err)[:80]}" if err else s
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
async def _probe(url: str) -> str:
|
|
439
|
-
"""Probe a layer /health endpoint and return a single-string verdict
|
|
440
|
-
that surfaces both transport-level failure and self-reported status."""
|
|
441
|
-
try:
|
|
442
|
-
r = await _client().get(url, timeout=3.0)
|
|
443
|
-
except Exception as exc:
|
|
444
|
-
return f"unreachable: {type(exc).__name__}"
|
|
445
|
-
if r.status_code != 200:
|
|
446
|
-
return f"http {r.status_code}"
|
|
447
|
-
try:
|
|
448
|
-
body_status = _interpret_body_status(r.json())
|
|
449
|
-
except Exception:
|
|
450
|
-
body_status = None
|
|
451
|
-
return body_status or "ok"
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
async def _probe_l3() -> str:
|
|
455
|
-
"""Real Neo4j probe — POST a trivial Cypher via the HTTP transactional
|
|
456
|
-
API and require a 200 response. Confirms Neo4j is actually answering
|
|
457
|
-
queries, not just serving the Browser HTML on :7474.
|
|
458
|
-
"""
|
|
459
|
-
user, _, password = NEO4J_AUTH.partition("/")
|
|
460
|
-
url = f"{L3_KG_URL}/db/{NEO4J_DB}/tx/commit"
|
|
461
|
-
try:
|
|
462
|
-
r = await _client().post(
|
|
463
|
-
url,
|
|
464
|
-
json={"statements": [{"statement": "RETURN 1"}]},
|
|
465
|
-
auth=(user, password),
|
|
466
|
-
timeout=3.0,
|
|
467
|
-
)
|
|
468
|
-
except Exception as exc:
|
|
469
|
-
return f"unreachable: {type(exc).__name__}"
|
|
470
|
-
if r.status_code != 200:
|
|
471
|
-
return f"http {r.status_code}"
|
|
472
|
-
try:
|
|
473
|
-
body = r.json()
|
|
474
|
-
# Neo4j tx/commit returns {"results":[...], "errors":[...]}.
|
|
475
|
-
# Any errors here means the DB is up but rejecting queries.
|
|
476
|
-
errs = body.get("errors") or []
|
|
477
|
-
if errs:
|
|
478
|
-
return f"cypher error: {str(errs[0])[:80]}"
|
|
479
|
-
except Exception:
|
|
480
|
-
return "non-json response"
|
|
481
|
-
return "ok"
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
@app.get("/health")
|
|
485
|
-
async def health():
|
|
486
|
-
"""Aggregate health across all 7 layers.
|
|
487
|
-
|
|
488
|
-
Each layer's verdict is honest: it reflects whether the layer can
|
|
489
|
-
actually do its job, not just whether its HTTP server answers. The
|
|
490
|
-
shim reads the layer's body.status (when present) and degrades when
|
|
491
|
-
the layer self-reports a problem. L3 uses a real Cypher probe since
|
|
492
|
-
Neo4j has no /health route.
|
|
493
|
-
"""
|
|
494
|
-
out = {
|
|
495
|
-
"status": "ok",
|
|
496
|
-
"version": VERSION,
|
|
497
|
-
"engine": "pentatonic-memory-engine",
|
|
498
|
-
"layers": {},
|
|
499
|
-
}
|
|
500
|
-
# NV-Embed (or the upstream gateway) exposes /health at the host root.
|
|
501
|
-
# Use urlparse so we swap *just* the path component instead of doing a
|
|
502
|
-
# string replace — that breaks the moment NV_EMBED_URL is /v1/embed,
|
|
503
|
-
# /v1/embeddings, or bare-host. The probe is informational only; gateways
|
|
504
|
-
# that return non-200 on root-/health don't block engine operation.
|
|
505
|
-
from urllib.parse import urlparse, urlunparse
|
|
506
|
-
_u = urlparse(NV_EMBED_URL)
|
|
507
|
-
nv_embed_health = urlunparse((_u.scheme, _u.netloc, "/health", "", "", ""))
|
|
508
|
-
|
|
509
|
-
import asyncio
|
|
510
|
-
l2_v, l5_v, l6_v, nv_v, l3_v = await asyncio.gather(
|
|
511
|
-
_probe(f"{L2_PROXY_URL}/health"),
|
|
512
|
-
_probe(f"{L5_MILVUS_URL}/health"),
|
|
513
|
-
_probe(f"{L6_DOC_URL}/health"),
|
|
514
|
-
_probe(nv_embed_health),
|
|
515
|
-
_probe_l3(),
|
|
516
|
-
)
|
|
517
|
-
|
|
518
|
-
# L0 BM25 (FTS5), L1 (always-loaded core files) and L4 QMD vec are
|
|
519
|
-
# all in-process inside the L2 proxy — L0+L1 in workspace.db / core
|
|
520
|
-
# files; L4 in qmd.sqlite which L2 opens directly. No separate runtime;
|
|
521
|
-
# if L2 is healthy, all three layers are usable. Tie their status to L2.
|
|
522
|
-
l2_ok = l2_v == "ok"
|
|
523
|
-
out["layers"] = {
|
|
524
|
-
"l0": "ok" if l2_ok else l2_v,
|
|
525
|
-
"l1": "ok" if l2_ok else l2_v,
|
|
526
|
-
"l2": l2_v,
|
|
527
|
-
"l3": l3_v,
|
|
528
|
-
"l4": "ok" if l2_ok else l2_v,
|
|
529
|
-
"l5": l5_v,
|
|
530
|
-
"l6": l6_v,
|
|
531
|
-
"nv_embed": nv_v,
|
|
532
|
-
}
|
|
533
|
-
failures = sum(1 for v in out["layers"].values() if v != "ok")
|
|
534
|
-
if failures:
|
|
535
|
-
out["status"] = "degraded" if failures < 3 else "down"
|
|
536
|
-
|
|
537
|
-
# Per-layer chunk counts. Replaces the previous single `memories` int
|
|
538
|
-
# which only reflected L6's vector chunk count — misleading because
|
|
539
|
-
# L0/L4/L5 hold different (overlapping) projections of the corpus.
|
|
540
|
-
# Each layer is independently probed; transient failure on one layer
|
|
541
|
-
# leaves its slot null rather than zeroing the whole field.
|
|
542
|
-
memories: dict[str, int | None] = {
|
|
543
|
-
"l0_bm25_chunks": None,
|
|
544
|
-
"l4_vectors": None,
|
|
545
|
-
"l5_chats_chunks": None,
|
|
546
|
-
"l6_vector_chunks": None,
|
|
547
|
-
"l6_fts_chunks": None,
|
|
548
|
-
}
|
|
549
|
-
# L0 and L4 both live inside L2 (workspace.db + qmd.sqlite directly
|
|
550
|
-
# opened by the L2 proxy). L2 exposes /index-internal-stats with both
|
|
551
|
-
# counts in one round-trip.
|
|
552
|
-
try:
|
|
553
|
-
r = await _client().get(f"{L2_PROXY_URL}/index-internal-stats", timeout=3.0)
|
|
554
|
-
if r.status_code == 200:
|
|
555
|
-
stats = r.json()
|
|
556
|
-
memories["l0_bm25_chunks"] = int(stats.get("l0_chunks") or 0)
|
|
557
|
-
memories["l4_vectors"] = int(stats.get("l4_qmd_chunks") or 0)
|
|
558
|
-
except Exception:
|
|
559
|
-
pass
|
|
560
|
-
# L5 reports per-collection counts on /health. We surface chats —
|
|
561
|
-
# the only collection currently populated; emails/contacts/memory
|
|
562
|
-
# collections coming back online with L5 collection bootstrap.
|
|
563
|
-
try:
|
|
564
|
-
r = await _client().get(f"{L5_MILVUS_URL}/health", timeout=3.0)
|
|
565
|
-
if r.status_code == 200:
|
|
566
|
-
colls = r.json().get("collections") or {}
|
|
567
|
-
chats = colls.get("chats") if isinstance(colls, dict) else None
|
|
568
|
-
if isinstance(chats, dict):
|
|
569
|
-
memories["l5_chats_chunks"] = int(chats.get("count") or 0)
|
|
570
|
-
except Exception:
|
|
571
|
-
pass
|
|
572
|
-
# L6 exposes vector vs fts splits on /stats.
|
|
573
|
-
try:
|
|
574
|
-
r = await _client().get(f"{L6_DOC_URL}/stats", timeout=3.0)
|
|
575
|
-
if r.status_code == 200:
|
|
576
|
-
stats = r.json()
|
|
577
|
-
memories["l6_vector_chunks"] = int(stats.get("vector_chunks") or 0)
|
|
578
|
-
memories["l6_fts_chunks"] = int(stats.get("fts_chunks") or 0)
|
|
579
|
-
except Exception:
|
|
580
|
-
pass
|
|
581
|
-
out["memories"] = memories
|
|
582
|
-
return out
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
@app.get("/health/deep")
|
|
586
|
-
async def health_deep():
|
|
587
|
-
"""Aggregate functional probe — fans out to each layer's /health/deep
|
|
588
|
-
and reports per-layer ok/status.
|
|
589
|
-
|
|
590
|
-
Where /health checks "process up + port answering", /health/deep
|
|
591
|
-
actually exercises embed → write → search round-trips on each layer
|
|
592
|
-
that supports it. Catches the class of bug that masquerades as
|
|
593
|
-
"healthy" — request handlers 500'ing while the process stays up.
|
|
594
|
-
|
|
595
|
-
Slower than /health (~1–2s); intended for ops/monitoring/cron use,
|
|
596
|
-
not the deploy gate or compose healthcheck.
|
|
597
|
-
"""
|
|
598
|
-
import asyncio
|
|
599
|
-
out: dict[str, Any] = {"status": "ok", "ok": True, "layers": {}}
|
|
600
|
-
|
|
601
|
-
async def _probe_deep(name: str, url: str) -> tuple[str, dict]:
|
|
602
|
-
try:
|
|
603
|
-
r = await _client().get(url, timeout=15.0)
|
|
604
|
-
if r.status_code != 200:
|
|
605
|
-
return name, {"ok": False, "status": f"http {r.status_code}"}
|
|
606
|
-
return name, r.json()
|
|
607
|
-
except Exception as exc:
|
|
608
|
-
return name, {"ok": False, "status": f"unreachable: {type(exc).__name__}"}
|
|
609
|
-
|
|
610
|
-
# L4 is in-process inside L2 (qmd.sqlite direct-read) — its deep
|
|
611
|
-
# round-trip is covered by L2's /health/deep, no separate probe needed.
|
|
612
|
-
results = await asyncio.gather(
|
|
613
|
-
_probe_deep("l5", f"{L5_MILVUS_URL}/health/deep"),
|
|
614
|
-
_probe_deep("l6", f"{L6_DOC_URL}/health/deep"),
|
|
615
|
-
)
|
|
616
|
-
for name, body in results:
|
|
617
|
-
out["layers"][name] = body
|
|
618
|
-
if not body.get("ok", False):
|
|
619
|
-
out["ok"] = False
|
|
620
|
-
|
|
621
|
-
if not out["ok"]:
|
|
622
|
-
out["status"] = "degraded"
|
|
623
|
-
return out
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
def _arena_scoped_rid(arena: str, content: str) -> str:
|
|
627
|
-
"""Stable, arena-scoped record id.
|
|
628
|
-
|
|
629
|
-
Same content in different arenas hashes to *different* ids — so two
|
|
630
|
-
tenants storing the byte-identical chunk get distinct primary keys
|
|
631
|
-
in the underlying stores (L4 sqlite-vec, L5 Milvus, L6 Milvus).
|
|
632
|
-
Without this, arena=tenant-b's write would overwrite arena=tenant-a's
|
|
633
|
-
row of the same content, silently breaking multi-tenant isolation
|
|
634
|
-
for any duplicated phrase.
|
|
635
|
-
"""
|
|
636
|
-
return hashlib.sha1(f"{arena}:{content}".encode()).hexdigest()[:32]
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
@app.post("/store")
|
|
640
|
-
async def store(req: StoreRequest):
|
|
641
|
-
"""Single-record ingest. Same wire format as pentatonic-memory v0.5."""
|
|
642
|
-
arena = (req.metadata or {}).get("arena", "general")
|
|
643
|
-
rid = (req.metadata or {}).get("id") or _arena_scoped_rid(arena, req.content)
|
|
644
|
-
record = {"id": rid, "content": req.content, "metadata": req.metadata or {}}
|
|
645
|
-
|
|
646
|
-
# Stash the full metadata under every key shape any layer could echo back.
|
|
647
|
-
# L5/L6 use derivatives of rid; L2-internal returns paths shaped like
|
|
648
|
-
# bench/<arena>/<id>.md (and <id> may be the SHA1 rid OR the caller's doc_id
|
|
649
|
-
# depending on which one was supplied).
|
|
650
|
-
_stash_all_keys(rid, req.metadata or {}, arena)
|
|
651
|
-
|
|
652
|
-
# Fan out to L5 + L6 + L2-internal (L0+L4qmd+L3) in parallel.
|
|
653
|
-
import asyncio
|
|
654
|
-
l5_count, l6_count, l2_internal = await asyncio.gather(
|
|
655
|
-
_index_l5([record], arena=arena),
|
|
656
|
-
_index_l6([record], arena=arena),
|
|
657
|
-
_index_l2_internal([record], arena=arena),
|
|
658
|
-
)
|
|
659
|
-
|
|
660
|
-
l4_qmd_count = l2_internal.get("l4_qmd", 0)
|
|
661
|
-
return {
|
|
662
|
-
"id": rid,
|
|
663
|
-
"content": req.content,
|
|
664
|
-
"layerId": _layer_id(arena, (req.metadata or {}).get("layer_type")),
|
|
665
|
-
"engine": {
|
|
666
|
-
"l0": l2_internal.get("l0", 0),
|
|
667
|
-
"l3_chunks": l2_internal.get("l3_chunks", 0),
|
|
668
|
-
"l3_entities": l2_internal.get("l3_entities", 0),
|
|
669
|
-
"l4_qmd": l4_qmd_count,
|
|
670
|
-
# `l4` is aliased to L4_QMD now that the standalone L4 sqlite-vec
|
|
671
|
-
# sidecar has been dropped. Kept in the response for wire-format
|
|
672
|
-
# back-compat with callers that read engine.l4.
|
|
673
|
-
"l4": l4_qmd_count,
|
|
674
|
-
"l5": l5_count,
|
|
675
|
-
"l6": l6_count,
|
|
676
|
-
},
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
@app.post("/store-batch")
|
|
681
|
-
async def store_batch(req: StoreBatchRequest):
|
|
682
|
-
"""Batch ingest — 30-50× faster than calling /store N times."""
|
|
683
|
-
if not req.records:
|
|
684
|
-
return {"inserted": 0, "ids": []}
|
|
685
|
-
|
|
686
|
-
# Normalise each record to {id, content, metadata}.
|
|
687
|
-
arena = req.arena or "general"
|
|
688
|
-
normalised = []
|
|
689
|
-
for r in req.records:
|
|
690
|
-
content = r.get("content") or r.get("text") or ""
|
|
691
|
-
if not content:
|
|
692
|
-
continue
|
|
693
|
-
rid = r.get("id") or _arena_scoped_rid(arena, content)
|
|
694
|
-
normalised.append({"id": rid, "content": content, "metadata": r.get("metadata") or {}})
|
|
695
|
-
|
|
696
|
-
# Stash metadata for every record so /search can re-attach it.
|
|
697
|
-
for r in normalised:
|
|
698
|
-
_stash_all_keys(r["id"], r.get("metadata") or {}, arena)
|
|
699
|
-
|
|
700
|
-
t0 = time.perf_counter()
|
|
701
|
-
import asyncio
|
|
702
|
-
|
|
703
|
-
# Shared-embed mode: compute embeddings ONCE here, pass them down to
|
|
704
|
-
# every layer so they skip their own embed call. Previously L5 + L6
|
|
705
|
-
# + L2-internal each re-embedded the same texts in parallel, which
|
|
706
|
-
# fanned 3× the gateway RPCs. The gateway throttles at K≈10 concurrent
|
|
707
|
-
# requests, so 30-way fan-out serialised into ~3 rounds of ~850ms
|
|
708
|
-
# each = ~2.5s of pure embed time per /store-batch. With shared
|
|
709
|
-
# embeddings we issue one chunked embed pass (10 sub-calls for N=50
|
|
710
|
-
# records) and skip the per-layer redundant work entirely.
|
|
711
|
-
# Disabled via PME_SHARE_EMBEDDINGS=false for operators wiring up
|
|
712
|
-
# per-layer differentiated embedders.
|
|
713
|
-
shared_embeddings: list[list[float]] | None = None
|
|
714
|
-
embed_ms = 0.0
|
|
715
|
-
if SHARE_EMBEDDINGS and normalised:
|
|
716
|
-
texts = [r["content"] for r in normalised]
|
|
717
|
-
embed_t0 = time.perf_counter()
|
|
718
|
-
try:
|
|
719
|
-
shared_embeddings = await _get_embed_client().embed_batch_async(texts)
|
|
720
|
-
except Exception as exc:
|
|
721
|
-
# Fall back to per-layer embedding rather than failing the
|
|
722
|
-
# whole batch. The layers' /index-batch still works when
|
|
723
|
-
# `embeddings` is absent.
|
|
724
|
-
print(f"[shim] shared embed failed, falling back to per-layer: {exc}")
|
|
725
|
-
shared_embeddings = None
|
|
726
|
-
embed_ms = (time.perf_counter() - embed_t0) * 1000.0
|
|
727
|
-
|
|
728
|
-
l5_count, l6_count, l2_internal = await asyncio.gather(
|
|
729
|
-
_index_l5(normalised, arena=req.arena or "general", embeddings=shared_embeddings),
|
|
730
|
-
_index_l6(normalised, arena=req.arena or "general", embeddings=shared_embeddings),
|
|
731
|
-
_index_l2_internal(normalised, arena=req.arena or "general", embeddings=shared_embeddings),
|
|
732
|
-
)
|
|
733
|
-
dur_ms = (time.perf_counter() - t0) * 1000.0
|
|
734
|
-
|
|
735
|
-
l4_qmd_count = l2_internal.get("l4_qmd", 0)
|
|
736
|
-
return {
|
|
737
|
-
"status": "ok",
|
|
738
|
-
"inserted": max(l4_qmd_count, l5_count, l6_count),
|
|
739
|
-
"ids": [r["id"] for r in normalised],
|
|
740
|
-
"engine": {
|
|
741
|
-
"l0": l2_internal.get("l0", 0),
|
|
742
|
-
"l3_chunks": l2_internal.get("l3_chunks", 0),
|
|
743
|
-
"l3_entities": l2_internal.get("l3_entities", 0),
|
|
744
|
-
"l4_qmd": l4_qmd_count,
|
|
745
|
-
# `l4` aliased to L4_QMD — sidecar dropped, see /store handler.
|
|
746
|
-
"l4": l4_qmd_count,
|
|
747
|
-
"l5": l5_count,
|
|
748
|
-
"l6": l6_count,
|
|
749
|
-
},
|
|
750
|
-
"duration_ms": round(dur_ms, 1),
|
|
751
|
-
}
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
def _arenas_for(req: SearchRequest) -> list[str]:
|
|
755
|
-
"""Normalize req's single-arena + multi-arena fields into one list.
|
|
756
|
-
|
|
757
|
-
`arenas` is authoritative when set; otherwise `arena` is treated as
|
|
758
|
-
a one-element list; otherwise empty (= search is unscoped, dev/test).
|
|
759
|
-
"""
|
|
760
|
-
if req.arenas:
|
|
761
|
-
return [a for a in req.arenas if a]
|
|
762
|
-
if req.arena:
|
|
763
|
-
return [req.arena]
|
|
764
|
-
return []
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
def _apply_metadata_filters(results: list[dict[str, Any]], req: SearchRequest) -> list[dict[str, Any]]:
|
|
768
|
-
"""Post-filter results by arena set + arbitrary metadata equality.
|
|
769
|
-
|
|
770
|
-
Many layer searches don't yet honour arena/metadata at the storage
|
|
771
|
-
level, so the shim enforces tenant isolation here as defence in
|
|
772
|
-
depth. Even if the underlying layer leaks across arenas, the shim
|
|
773
|
-
drops cross-arena rows before returning.
|
|
774
|
-
|
|
775
|
-
Multi-arena rule: a row passes if its arena tag is in the request's
|
|
776
|
-
arena set. So a user-scoped search (arenas=[acme, acme:u-42]) sees
|
|
777
|
-
both tenant-wide rows (arena=acme) and that user's own user-scoped
|
|
778
|
-
rows (arena=acme:u-42), but never another user's user-scoped rows
|
|
779
|
-
(arena=acme:u-99).
|
|
780
|
-
"""
|
|
781
|
-
arenas = _arenas_for(req)
|
|
782
|
-
extra = req.metadata_filter or {}
|
|
783
|
-
if not arenas and not extra:
|
|
784
|
-
return results
|
|
785
|
-
arena_set = set(arenas)
|
|
786
|
-
out: list[dict[str, Any]] = []
|
|
787
|
-
for item in results:
|
|
788
|
-
meta = item.get("metadata") or {}
|
|
789
|
-
if arena_set:
|
|
790
|
-
row_arena = meta.get("arena") or item.get("arena")
|
|
791
|
-
if row_arena and row_arena not in arena_set:
|
|
792
|
-
continue
|
|
793
|
-
# If row has no arena tag at all, drop on multi-tenant
|
|
794
|
-
# safety: a row without arena predates the multi-tenant
|
|
795
|
-
# plumbing and could belong to anyone.
|
|
796
|
-
if not row_arena:
|
|
797
|
-
continue
|
|
798
|
-
ok = True
|
|
799
|
-
for k, v in extra.items():
|
|
800
|
-
if str(meta.get(k, "")) != str(v):
|
|
801
|
-
ok = False
|
|
802
|
-
break
|
|
803
|
-
if ok:
|
|
804
|
-
out.append(item)
|
|
805
|
-
return out
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
def _search_overfetch(req: SearchRequest) -> int:
|
|
809
|
-
"""Decide how many results to over-fetch from layers.
|
|
810
|
-
|
|
811
|
-
Post-filtering can drop many rows; we ask layers for more than the
|
|
812
|
-
user's limit so we have headroom after filtering. 5x is a balance
|
|
813
|
-
between accuracy and latency.
|
|
814
|
-
"""
|
|
815
|
-
base = req.limit or 10
|
|
816
|
-
return base * 5 if (_arenas_for(req) or req.metadata_filter) else base * 3
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
@app.post("/search")
|
|
820
|
-
async def search(req: SearchRequest):
|
|
821
|
-
"""
|
|
822
|
-
Hybrid retrieval over all 7 layers via the L2 HybridRAG proxy. The proxy
|
|
823
|
-
queries L0 BM25, L4 vec, L5 Milvus, L6 doc-store in parallel and fuses
|
|
824
|
-
the results with Reciprocal Rank Fusion. L3 KG adds entity-aware
|
|
825
|
-
boosting for graph queries.
|
|
826
|
-
|
|
827
|
-
Multi-tenancy: pass `arena` to scope results to a single tenant.
|
|
828
|
-
Underlying layers may or may not honour arena natively (L6 does;
|
|
829
|
-
L2/L4/L5 don't yet — engine TODO); the shim applies arena as a
|
|
830
|
-
post-filter regardless, so cross-tenant leakage is prevented even
|
|
831
|
-
when a layer is non-compliant.
|
|
832
|
-
"""
|
|
833
|
-
if not req.query:
|
|
834
|
-
return {"results": []}
|
|
835
|
-
|
|
836
|
-
# The L2 proxy exposes hybrid search via GET /search?q=... and a strict
|
|
837
|
-
# OpenAI-compatible POST /v1/search. Try GET first (lower overhead, no
|
|
838
|
-
# JSON parsing on the proxy side); fall back to /v1/search; then to L6.
|
|
839
|
-
#
|
|
840
|
-
# When BYPASS_L2_PROXY is set, skip the proxy entirely and query L6
|
|
841
|
-
# directly. Useful for isolated bench/test runs.
|
|
842
|
-
data: dict[str, Any] | None = None
|
|
843
|
-
last_err: Exception | None = None
|
|
844
|
-
if BYPASS_L2:
|
|
845
|
-
# L6-only path: L6 already does vector + BM25 + cross-encoder
|
|
846
|
-
# reranker. Adding L4's pure vector via RRF actively hurt on
|
|
847
|
-
# product-catalogue (-5.6pp on the 84.6% baseline) by diluting
|
|
848
|
-
# the rerank ordering. Stick to L6 for now; the proper next
|
|
849
|
-
# step is wiring up the L2 7-layer proxy.
|
|
850
|
-
import asyncio
|
|
851
|
-
async def _q_l6(query: str):
|
|
852
|
-
try:
|
|
853
|
-
params: list = [
|
|
854
|
-
("q", query),
|
|
855
|
-
("limit", str(_search_overfetch(req))),
|
|
856
|
-
("method", "hybrid"),
|
|
857
|
-
]
|
|
858
|
-
# L6 supports arena natively (l6-document-store.py).
|
|
859
|
-
# Forward all arenas in the search scope; L6 expands the
|
|
860
|
-
# filter to `arena IN (...)`. Multiple `arenas` query
|
|
861
|
-
# params on the wire = list-shaped server side.
|
|
862
|
-
for a in _arenas_for(req):
|
|
863
|
-
params.append(("arenas", a))
|
|
864
|
-
r = await _client().get(
|
|
865
|
-
f"{L6_DOC_URL}/search",
|
|
866
|
-
params=params,
|
|
867
|
-
timeout=30.0,
|
|
868
|
-
)
|
|
869
|
-
r.raise_for_status()
|
|
870
|
-
return [{"layer": "L6", **item} for item in r.json().get("results", [])]
|
|
871
|
-
except Exception as exc:
|
|
872
|
-
print(f"[shim] L6 search failed for {query!r}: {exc}")
|
|
873
|
-
return []
|
|
874
|
-
|
|
875
|
-
# Optional HyDE: if HYDE_ENABLED, also generate 2 hypothetical
|
|
876
|
-
# answers via the LLM, embed each, and run them as additional
|
|
877
|
-
# queries that we RRF-fuse with the original. Off by default;
|
|
878
|
-
# set HYDE_ENABLED=1 to try it. Runs in parallel with the main
|
|
879
|
-
# query so latency only grows by the LLM call (1-2s).
|
|
880
|
-
l6_hits = await _q_l6(req.query)
|
|
881
|
-
l4_hits: list[dict[str, Any]] = [] # kept empty intentionally
|
|
882
|
-
|
|
883
|
-
# Reciprocal Rank Fusion (RRF) — k=60 is the standard constant
|
|
884
|
-
# from Cormack et al. 2009. Score = 1/(k + rank). For each unique
|
|
885
|
-
# doc id we sum contributions from each layer that returned it.
|
|
886
|
-
#
|
|
887
|
-
# Critical: layers return items with different id shapes
|
|
888
|
-
# (L6 uses "l6:<rid>:0" chunk ids and "<rid>.md" source_file,
|
|
889
|
-
# L4 uses the raw rid as path), so we resolve a *canonical* id
|
|
890
|
-
# by walking the metadata cache for each candidate id form.
|
|
891
|
-
# That makes the same doc collapse into one rank entry across
|
|
892
|
-
# layers and lets RRF actually fuse instead of double-listing.
|
|
893
|
-
def _canonical_key(item: dict[str, Any]) -> str:
|
|
894
|
-
candidates = [
|
|
895
|
-
item.get("id"),
|
|
896
|
-
item.get("chunk_id"),
|
|
897
|
-
item.get("source_file"),
|
|
898
|
-
item.get("source_file", "").rsplit(".md", 1)[0] if item.get("source_file") else None,
|
|
899
|
-
item.get("path"),
|
|
900
|
-
]
|
|
901
|
-
for cid in candidates:
|
|
902
|
-
if not cid:
|
|
903
|
-
continue
|
|
904
|
-
m = _META_CACHE.get(cid)
|
|
905
|
-
if m and m.get("doc_id"):
|
|
906
|
-
return m["doc_id"]
|
|
907
|
-
# Fallback: use first non-empty candidate as key.
|
|
908
|
-
for cid in candidates:
|
|
909
|
-
if cid:
|
|
910
|
-
return cid
|
|
911
|
-
return hashlib.sha1((item.get("text") or item.get("content") or "").encode()).hexdigest()[:32]
|
|
912
|
-
|
|
913
|
-
K = 60
|
|
914
|
-
rrf_scores: dict[str, float] = {}
|
|
915
|
-
first_item: dict[str, dict[str, Any]] = {}
|
|
916
|
-
layer_provenance: dict[str, list[str]] = {}
|
|
917
|
-
for hits in (l6_hits, l4_hits):
|
|
918
|
-
for rank, item in enumerate(hits, start=1):
|
|
919
|
-
key = _canonical_key(item)
|
|
920
|
-
rrf_scores[key] = rrf_scores.get(key, 0.0) + 1.0 / (K + rank)
|
|
921
|
-
layer_provenance.setdefault(key, []).append(item.get("layer", "?"))
|
|
922
|
-
# Keep the richest version of the doc (prefer L6 — it
|
|
923
|
-
# carries cross-encoder reranker scores plus content).
|
|
924
|
-
if key not in first_item or item.get("layer") == "L6":
|
|
925
|
-
first_item[key] = item
|
|
926
|
-
|
|
927
|
-
# Sort by fused score, take top-N.
|
|
928
|
-
ranked_keys = sorted(rrf_scores.keys(), key=lambda k: -rrf_scores[k])
|
|
929
|
-
top_keys = ranked_keys[: req.limit or 10]
|
|
930
|
-
|
|
931
|
-
out_results = []
|
|
932
|
-
for key in top_keys:
|
|
933
|
-
item = first_item[key]
|
|
934
|
-
attached_meta = _lookup_meta(key)
|
|
935
|
-
if not attached_meta:
|
|
936
|
-
# The canonical key may itself be a derived form; walk
|
|
937
|
-
# all known id shapes one more time as a safety net.
|
|
938
|
-
for cid in (item.get("id"), item.get("chunk_id"),
|
|
939
|
-
item.get("source_file"), item.get("path")):
|
|
940
|
-
if cid:
|
|
941
|
-
m = _lookup_meta(cid)
|
|
942
|
-
if m:
|
|
943
|
-
attached_meta = m
|
|
944
|
-
break
|
|
945
|
-
# Some layers surface arena/kind/layer_type as top-level
|
|
946
|
-
# fields on the raw item rather than nested under metadata.
|
|
947
|
-
# Pull them through into the formatted metadata so the
|
|
948
|
-
# post-filter can see them even on cache miss (e.g., after
|
|
949
|
-
# a compat container restart wipes _META_CACHE).
|
|
950
|
-
raw_top_level = {
|
|
951
|
-
k: item[k]
|
|
952
|
-
for k in ("arena", "kind", "layer_type", "doc_type", "source_file")
|
|
953
|
-
if item.get(k)
|
|
954
|
-
}
|
|
955
|
-
merged_meta = {**raw_top_level, **(attached_meta or item.get("metadata") or {})}
|
|
956
|
-
row_arena = merged_meta.get("arena")
|
|
957
|
-
row_layer_type = merged_meta.get("layer_type")
|
|
958
|
-
out_results.append({
|
|
959
|
-
"id": key,
|
|
960
|
-
"content": item.get("text") or item.get("content") or item.get("snippet") or "",
|
|
961
|
-
"metadata": merged_meta,
|
|
962
|
-
"similarity": float(rrf_scores[key]),
|
|
963
|
-
"layer_id": _layer_id(row_arena, row_layer_type),
|
|
964
|
-
"source": item.get("source_file") or item.get("path") or "",
|
|
965
|
-
"engine_layer": "+".join(sorted(set(layer_provenance.get(key, [])))),
|
|
966
|
-
})
|
|
967
|
-
# Defense-in-depth post-filter (arena + arbitrary metadata),
|
|
968
|
-
# then trim to the requested limit.
|
|
969
|
-
out_results = _apply_metadata_filters(out_results, req)
|
|
970
|
-
return {"results": out_results[: req.limit or 10]}
|
|
971
|
-
arenas = _arenas_for(req)
|
|
972
|
-
try:
|
|
973
|
-
get_params: list = [
|
|
974
|
-
("q", req.query),
|
|
975
|
-
("limit", str(_search_overfetch(req))),
|
|
976
|
-
]
|
|
977
|
-
for a in arenas:
|
|
978
|
-
get_params.append(("arenas", a))
|
|
979
|
-
r = await _client().get(
|
|
980
|
-
f"{L2_PROXY_URL}/search",
|
|
981
|
-
params=get_params,
|
|
982
|
-
timeout=30.0,
|
|
983
|
-
)
|
|
984
|
-
r.raise_for_status()
|
|
985
|
-
data = r.json()
|
|
986
|
-
except Exception as exc:
|
|
987
|
-
last_err = exc
|
|
988
|
-
try:
|
|
989
|
-
post_body: dict[str, Any] = {
|
|
990
|
-
"query": req.query,
|
|
991
|
-
"limit": _search_overfetch(req),
|
|
992
|
-
"min_score": req.min_score or 0.001,
|
|
993
|
-
}
|
|
994
|
-
if arenas:
|
|
995
|
-
post_body["arenas"] = arenas
|
|
996
|
-
r = await _client().post(
|
|
997
|
-
f"{L2_PROXY_URL}/v1/search",
|
|
998
|
-
json=post_body,
|
|
999
|
-
timeout=30.0,
|
|
1000
|
-
)
|
|
1001
|
-
r.raise_for_status()
|
|
1002
|
-
data = r.json()
|
|
1003
|
-
except Exception as exc2:
|
|
1004
|
-
last_err = exc2
|
|
1005
|
-
try:
|
|
1006
|
-
params: list = [
|
|
1007
|
-
("q", req.query),
|
|
1008
|
-
("limit", str(_search_overfetch(req))),
|
|
1009
|
-
]
|
|
1010
|
-
# L6 supports arena natively; forward all in the search
|
|
1011
|
-
# scope on the last-resort fallback path too.
|
|
1012
|
-
for a in arenas:
|
|
1013
|
-
params.append(("arenas", a))
|
|
1014
|
-
r = await _client().get(
|
|
1015
|
-
f"{L6_DOC_URL}/search",
|
|
1016
|
-
params=params,
|
|
1017
|
-
timeout=10.0,
|
|
1018
|
-
)
|
|
1019
|
-
r.raise_for_status()
|
|
1020
|
-
data = r.json()
|
|
1021
|
-
except Exception as exc3:
|
|
1022
|
-
raise HTTPException(status_code=502,
|
|
1023
|
-
detail=f"engine unreachable: {last_err}; L6 fallback: {exc3}")
|
|
1024
|
-
if data is None:
|
|
1025
|
-
raise HTTPException(status_code=502, detail=f"engine returned no data: {last_err}")
|
|
1026
|
-
|
|
1027
|
-
# Normalise to pentatonic-memory v0.5 result shape. Re-attach the
|
|
1028
|
-
# client-supplied metadata via the in-memory cache (same logic as
|
|
1029
|
-
# the BYPASS_L2 path). Bench adapters filter results by
|
|
1030
|
-
# metadata.bench_tag, so the metadata MUST survive the L2 round-trip
|
|
1031
|
-
# even though L2's response shape doesn't carry an arbitrary JSON
|
|
1032
|
-
# metadata column.
|
|
1033
|
-
out_results = []
|
|
1034
|
-
for item in data.get("results", []):
|
|
1035
|
-
candidate_ids = [
|
|
1036
|
-
item.get("id"),
|
|
1037
|
-
item.get("doc_id"),
|
|
1038
|
-
item.get("path"),
|
|
1039
|
-
item.get("source_file"),
|
|
1040
|
-
item.get("chunk_id"),
|
|
1041
|
-
item.get("source"),
|
|
1042
|
-
# L5 returns paths like ".pentatonic/chats/<rid>.jsonl" — strip suffix
|
|
1043
|
-
item.get("path", "").rsplit(".", 1)[0] if item.get("path") else None,
|
|
1044
|
-
item.get("source_file", "").rsplit(".md", 1)[0] if item.get("source_file") else None,
|
|
1045
|
-
]
|
|
1046
|
-
attached_meta: dict[str, Any] = {}
|
|
1047
|
-
chosen_id = ""
|
|
1048
|
-
for cid in candidate_ids:
|
|
1049
|
-
if cid:
|
|
1050
|
-
m = _lookup_meta(cid)
|
|
1051
|
-
if m:
|
|
1052
|
-
attached_meta = m
|
|
1053
|
-
chosen_id = m.get("doc_id") or cid
|
|
1054
|
-
break
|
|
1055
|
-
if not chosen_id:
|
|
1056
|
-
chosen_id = (item.get("id") or item.get("doc_id")
|
|
1057
|
-
or item.get("path") or item.get("source_file") or "")
|
|
1058
|
-
# Pull arena/kind/layer_type from raw item top-level if present
|
|
1059
|
-
# (L6 returns these as top-level columns; cache may be empty).
|
|
1060
|
-
raw_top_level = {
|
|
1061
|
-
k: item[k]
|
|
1062
|
-
for k in ("arena", "kind", "layer_type", "doc_type", "source_file")
|
|
1063
|
-
if item.get(k)
|
|
1064
|
-
}
|
|
1065
|
-
merged_meta = {**raw_top_level, **(attached_meta or item.get("metadata") or {})}
|
|
1066
|
-
row_arena = merged_meta.get("arena")
|
|
1067
|
-
row_layer_type = merged_meta.get("layer_type")
|
|
1068
|
-
out_results.append({
|
|
1069
|
-
"id": chosen_id,
|
|
1070
|
-
"content": item.get("text") or item.get("content") or item.get("snippet") or "",
|
|
1071
|
-
"metadata": merged_meta,
|
|
1072
|
-
"similarity": float(item.get("score") or item.get("similarity") or 0.0),
|
|
1073
|
-
"layer_id": _layer_id(row_arena, row_layer_type),
|
|
1074
|
-
"source": item.get("source", item.get("source_file", "")),
|
|
1075
|
-
"engine_layer": item.get("layer", item.get("source_layer", "")),
|
|
1076
|
-
})
|
|
1077
|
-
# Defense-in-depth post-filter (arena + arbitrary metadata) on L2/L6
|
|
1078
|
-
# fallback paths. Same logic as the BYPASS branch above.
|
|
1079
|
-
out_results = _apply_metadata_filters(out_results, req)
|
|
1080
|
-
return {"results": out_results[: req.limit or 10]}
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
@app.post("/forget")
|
|
1084
|
-
async def forget(req: ForgetRequest):
|
|
1085
|
-
"""
|
|
1086
|
-
Delete records by id or metadata filter. Restored from v0.4.x — was
|
|
1087
|
-
removed in v0.5.x, causing test/bench pollution and blocking GDPR
|
|
1088
|
-
deletion workflows.
|
|
1089
|
-
"""
|
|
1090
|
-
if not req.id and not req.metadata_contains:
|
|
1091
|
-
raise HTTPException(status_code=400, detail="provide id or metadata_contains")
|
|
1092
|
-
|
|
1093
|
-
deleted_total = 0
|
|
1094
|
-
# Forward to layers that support deletion. L6 doc-store supports both.
|
|
1095
|
-
try:
|
|
1096
|
-
if req.id:
|
|
1097
|
-
r = await _client().delete(
|
|
1098
|
-
f"{L6_DOC_URL}/purge",
|
|
1099
|
-
params={"source_file": req.id},
|
|
1100
|
-
timeout=10.0,
|
|
1101
|
-
)
|
|
1102
|
-
if r.status_code == 200:
|
|
1103
|
-
deleted_total += int(r.json().get("deleted", 1))
|
|
1104
|
-
elif req.metadata_contains:
|
|
1105
|
-
r = await _client().post(
|
|
1106
|
-
f"{L6_DOC_URL}/forget",
|
|
1107
|
-
json={"metadata_contains": req.metadata_contains},
|
|
1108
|
-
timeout=10.0,
|
|
1109
|
-
)
|
|
1110
|
-
if r.status_code == 200:
|
|
1111
|
-
deleted_total += int(r.json().get("deleted", 0))
|
|
1112
|
-
except Exception as exc:
|
|
1113
|
-
print(f"[shim] L6 /forget failed: {exc}")
|
|
1114
|
-
|
|
1115
|
-
# Also wipe L0 BM25 + L4 QMD + L3 KG so bench resets fully.
|
|
1116
|
-
# No per-id forget for these — bench harness uses /forget once at
|
|
1117
|
-
# start of each run with empty filters to reset state.
|
|
1118
|
-
#
|
|
1119
|
-
# Three forwarding rules:
|
|
1120
|
-
# - metadata_contains has `arena` → tenant-scoped delete (forwards
|
|
1121
|
-
# the arena to the internal endpoint, which only wipes that
|
|
1122
|
-
# tenant's L3 data).
|
|
1123
|
-
# - metadata_contains is set, no arena → targeted L6-only delete;
|
|
1124
|
-
# do NOT call the internal wipe (we don't have arena scope and
|
|
1125
|
-
# a global wipe would be wildly wrong here).
|
|
1126
|
-
# - empty filters → bench-reset semantics; explicitly request the
|
|
1127
|
-
# unsafe global wipe via the new `confirm: GLOBAL_WIPE` gate.
|
|
1128
|
-
arena_for_internal = None
|
|
1129
|
-
if isinstance(req.metadata_contains, dict):
|
|
1130
|
-
meta_arena = req.metadata_contains.get("arena")
|
|
1131
|
-
if isinstance(meta_arena, str) and meta_arena:
|
|
1132
|
-
arena_for_internal = meta_arena
|
|
1133
|
-
try:
|
|
1134
|
-
if arena_for_internal:
|
|
1135
|
-
payload = {"arena": arena_for_internal}
|
|
1136
|
-
elif req.metadata_contains:
|
|
1137
|
-
payload = None # skip — targeted delete shouldn't wipe shared layers
|
|
1138
|
-
else:
|
|
1139
|
-
payload = {"confirm": "GLOBAL_WIPE"}
|
|
1140
|
-
if payload is not None:
|
|
1141
|
-
r = await _client().post(
|
|
1142
|
-
f"{L2_PROXY_URL}/forget-internal", json=payload, timeout=15.0,
|
|
1143
|
-
)
|
|
1144
|
-
if r.status_code == 200:
|
|
1145
|
-
d = r.json().get("deleted", {})
|
|
1146
|
-
deleted_total += sum(int(v or 0) for v in d.values())
|
|
1147
|
-
except Exception as exc:
|
|
1148
|
-
print(f"[shim] L2 /forget-internal failed: {exc}")
|
|
1149
|
-
|
|
1150
|
-
return {"deleted": deleted_total, "engine": "pentatonic-memory-engine"}
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
@app.post("/aggregate")
|
|
1154
|
-
async def aggregate(req: AggregateRequest) -> dict[str, Any]:
|
|
1155
|
-
"""Aggregate communications history with one person.
|
|
1156
|
-
|
|
1157
|
-
Pass-through to the L2 proxy's /aggregate-internal which runs a
|
|
1158
|
-
single Cypher query against L3's typed-Person graph
|
|
1159
|
-
((:Person)-[:COMMUNICATED]->(:Chunk)). Used by TES personFacets
|
|
1160
|
-
once we cut over from the over-fetch v1.
|
|
1161
|
-
|
|
1162
|
-
The shim's job here is shape validation + arena enforcement; the
|
|
1163
|
-
real aggregation lives in L3. No layer fan-out, no metadata
|
|
1164
|
-
filter scanning — when the typed-Person nodes don't exist for
|
|
1165
|
-
this contact (older memories, tenants pre-#28), the response is
|
|
1166
|
-
`total: 0` and the caller falls back to whatever it had before.
|
|
1167
|
-
"""
|
|
1168
|
-
arena = (req.arena or "").strip()
|
|
1169
|
-
if not arena:
|
|
1170
|
-
raise HTTPException(status_code=400, detail="arena is required")
|
|
1171
|
-
if not (req.contact_email or req.contact_name):
|
|
1172
|
-
raise HTTPException(
|
|
1173
|
-
status_code=400,
|
|
1174
|
-
detail="provide contact_email and/or contact_name to identify the Person",
|
|
1175
|
-
)
|
|
1176
|
-
payload: dict[str, Any] = {
|
|
1177
|
-
"arena": arena,
|
|
1178
|
-
"group_by": req.group_by or ["channel"],
|
|
1179
|
-
}
|
|
1180
|
-
if req.contact_email:
|
|
1181
|
-
payload["contact_email"] = req.contact_email
|
|
1182
|
-
if req.contact_name:
|
|
1183
|
-
payload["contact_name"] = req.contact_name
|
|
1184
|
-
try:
|
|
1185
|
-
r = await _client().post(
|
|
1186
|
-
f"{L2_PROXY_URL}/aggregate-internal", json=payload, timeout=15.0,
|
|
1187
|
-
)
|
|
1188
|
-
if r.status_code != 200:
|
|
1189
|
-
raise HTTPException(
|
|
1190
|
-
status_code=r.status_code,
|
|
1191
|
-
detail=f"aggregate failed: {r.text[:200]}",
|
|
1192
|
-
)
|
|
1193
|
-
return r.json()
|
|
1194
|
-
except HTTPException:
|
|
1195
|
-
raise
|
|
1196
|
-
except Exception as exc:
|
|
1197
|
-
raise HTTPException(status_code=502, detail=f"aggregate upstream: {exc}")
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
@app.post("/people-list")
|
|
1201
|
-
async def people_list(req: PeopleListRequest) -> dict[str, Any]:
|
|
1202
|
-
"""List all people across one or more arenas.
|
|
1203
|
-
|
|
1204
|
-
Pass-through to the L2 proxy's /people-list-internal which runs a
|
|
1205
|
-
single Cypher pass over the ChannelStat denorm keyed on
|
|
1206
|
-
(arena, person_email, channel). Returns paginated, filtered,
|
|
1207
|
-
sorted rows with per-channel breakdown.
|
|
1208
|
-
|
|
1209
|
-
Shim's job: shape validation + arena enforcement. The real
|
|
1210
|
-
aggregation lives in L3 over ChannelStat — see PRs #28-33 for
|
|
1211
|
-
the writer side that maintains those nodes.
|
|
1212
|
-
|
|
1213
|
-
Multi-arena: legitimate here (unlike /aggregate). The UI
|
|
1214
|
-
fetches "people known by Philip OR Jeanne" by passing both
|
|
1215
|
-
arenas; the L2 proxy collapses duplicates by person_email.
|
|
1216
|
-
"""
|
|
1217
|
-
arenas = [a.strip() for a in (req.arenas or []) if a and a.strip()]
|
|
1218
|
-
if not arenas:
|
|
1219
|
-
raise HTTPException(status_code=400, detail="at least one arena is required")
|
|
1220
|
-
payload: dict[str, Any] = {
|
|
1221
|
-
"arenas": arenas,
|
|
1222
|
-
"limit": req.limit,
|
|
1223
|
-
"offset": req.offset,
|
|
1224
|
-
"order_by": req.order_by,
|
|
1225
|
-
}
|
|
1226
|
-
if req.emails:
|
|
1227
|
-
# Lowercase here so the L2 path can rely on exact-match; the
|
|
1228
|
-
# writer side also lowercases person_email on the ChannelStat
|
|
1229
|
-
# node, so this stays consistent.
|
|
1230
|
-
payload["emails"] = [e.strip().lower() for e in req.emails if e and e.strip()]
|
|
1231
|
-
if req.search:
|
|
1232
|
-
payload["search"] = req.search.strip()
|
|
1233
|
-
try:
|
|
1234
|
-
r = await _client().post(
|
|
1235
|
-
f"{L2_PROXY_URL}/people-list-internal", json=payload, timeout=15.0,
|
|
1236
|
-
)
|
|
1237
|
-
if r.status_code != 200:
|
|
1238
|
-
raise HTTPException(
|
|
1239
|
-
status_code=r.status_code,
|
|
1240
|
-
detail=f"people-list failed: {r.text[:200]}",
|
|
1241
|
-
)
|
|
1242
|
-
return r.json()
|
|
1243
|
-
except HTTPException:
|
|
1244
|
-
raise
|
|
1245
|
-
except Exception as exc:
|
|
1246
|
-
raise HTTPException(status_code=502, detail=f"people-list upstream: {exc}")
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
# ----------------------------------------------------------------------
|
|
1250
|
-
# Entrypoint
|
|
1251
|
-
# ----------------------------------------------------------------------
|
|
1252
|
-
|
|
1253
|
-
if __name__ == "__main__":
|
|
1254
|
-
import uvicorn
|
|
1255
|
-
uvicorn.run("server:app", host="0.0.0.0", port=PORT, log_level="info")
|