@100xprompt/chitta 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +203 -0
  3. package/assets/rules/claude-md.md +9 -0
  4. package/assets/skill/SKILL.md +47 -0
  5. package/package.json +48 -0
  6. package/src/README.md +124 -0
  7. package/src/arango-client.ts +67 -0
  8. package/src/arango-graph-provider.ts +364 -0
  9. package/src/bin.ts +27 -0
  10. package/src/config-env.ts +53 -0
  11. package/src/embedded/authorizer.ts +89 -0
  12. package/src/embedded/cli.ts +86 -0
  13. package/src/embedded/code-extractor.ts +9 -0
  14. package/src/embedded/demo.ts +36 -0
  15. package/src/embedded/extract.ts +12 -0
  16. package/src/embedded/extractors/code.ts +308 -0
  17. package/src/embedded/extractors/deterministic.ts +63 -0
  18. package/src/embedded/extractors/llm.ts +151 -0
  19. package/src/embedded/extractors/text-hygiene.ts +54 -0
  20. package/src/embedded/extractors/types.ts +34 -0
  21. package/src/embedded/graph/acl-paths.ts +96 -0
  22. package/src/embedded/graph/adjacency.ts +61 -0
  23. package/src/embedded/graph/centrality.ts +23 -0
  24. package/src/embedded/graph/communities.ts +46 -0
  25. package/src/embedded/graph/cypher.ts +17 -0
  26. package/src/embedded/graph/impact.ts +24 -0
  27. package/src/embedded/graph/knowledge-graph.ts +108 -0
  28. package/src/embedded/graph/pagerank.ts +57 -0
  29. package/src/embedded/graph/sql-access.ts +13 -0
  30. package/src/embedded/graph/traversal.ts +73 -0
  31. package/src/embedded/graph/types.ts +35 -0
  32. package/src/embedded/graph-query.ts +126 -0
  33. package/src/embedded/index.ts +171 -0
  34. package/src/embedded/ingest.ts +262 -0
  35. package/src/embedded/kgqa/answer-paths.ts +197 -0
  36. package/src/embedded/kgqa/entity-link.ts +13 -0
  37. package/src/embedded/kgqa/intent.ts +14 -0
  38. package/src/embedded/kgqa/predicates.ts +9 -0
  39. package/src/embedded/kgqa/preference.ts +20 -0
  40. package/src/embedded/kgqa/select.ts +99 -0
  41. package/src/embedded/kgqa/text.ts +16 -0
  42. package/src/embedded/kgqa/types.ts +6 -0
  43. package/src/embedded/kgqa-service.ts +122 -0
  44. package/src/embedded/llm-extractor.ts +10 -0
  45. package/src/embedded/local-embeddings.ts +36 -0
  46. package/src/embedded/personal.ts +100 -0
  47. package/src/embedded/reranker.ts +62 -0
  48. package/src/embedded/retrieval/decay-stage.ts +59 -0
  49. package/src/embedded/retrieval/diversity.ts +37 -0
  50. package/src/embedded/retrieval/fuse.ts +52 -0
  51. package/src/embedded/retrieval/graph-stage.ts +45 -0
  52. package/src/embedded/retrieval/hybrid-retriever.ts +80 -0
  53. package/src/embedded/retrieval/keyword-stage.ts +27 -0
  54. package/src/embedded/retrieval/passage.ts +44 -0
  55. package/src/embedded/retrieval/rerank-stage.ts +31 -0
  56. package/src/embedded/retrieval/trace.ts +31 -0
  57. package/src/embedded/retrieval/vector-stage.ts +15 -0
  58. package/src/embedded/sqlite-graph-provider.ts +119 -0
  59. package/src/embedded/sqlite-store.ts +95 -0
  60. package/src/embedded/sqlite-vec-service.ts +122 -0
  61. package/src/embedded/store/chunks.ts +61 -0
  62. package/src/embedded/store/fts.ts +50 -0
  63. package/src/embedded/store/nodes-edges.ts +112 -0
  64. package/src/embedded/store/salience.ts +37 -0
  65. package/src/embedded/store/schema.ts +109 -0
  66. package/src/embedded/transformers-embeddings.ts +100 -0
  67. package/src/embeddings.ts +51 -0
  68. package/src/eval/goldset.ts +46 -0
  69. package/src/eval/harness.ts +65 -0
  70. package/src/eval/metrics.ts +38 -0
  71. package/src/http/server.ts +93 -0
  72. package/src/index.ts +44 -0
  73. package/src/install/index.ts +139 -0
  74. package/src/install/platforms.ts +126 -0
  75. package/src/install/skill.ts +46 -0
  76. package/src/install/writers.ts +82 -0
  77. package/src/mcp/backend.ts +129 -0
  78. package/src/mcp/server.ts +83 -0
  79. package/src/mcp/tools/context-about.ts +69 -0
  80. package/src/mcp/tools/context-graph.ts +23 -0
  81. package/src/mcp/tools/context-ingest.ts +88 -0
  82. package/src/mcp/tools/context-rebuild.ts +22 -0
  83. package/src/mcp/tools/context-relate.ts +88 -0
  84. package/src/mcp/tools/get-context.ts +52 -0
  85. package/src/mcp/tools/index.ts +40 -0
  86. package/src/mcp/tools/types.ts +33 -0
  87. package/src/permission.ts +72 -0
  88. package/src/provider.ts +65 -0
  89. package/src/qdrant-vector.ts +76 -0
  90. package/src/retrieval.ts +218 -0
  91. package/src/service.ts +40 -0
  92. package/src/types.ts +91 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Nipurn Agarwal
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,203 @@
1
+ # Chitta
2
+
3
+ <!-- LANG-PICKER-START -->
4
+ <p align="center">
5
+ <b>English</b> ·
6
+ <a href="docs/translations/README.zh-CN.md">简体中文</a> ·
7
+ <a href="docs/translations/README.zh-TW.md">繁體中文</a> ·
8
+ <a href="docs/translations/README.ja-JP.md">日本語</a> ·
9
+ <a href="docs/translations/README.ko-KR.md">한국어</a> ·
10
+ <a href="docs/translations/README.hi-IN.md">हिन्दी</a> ·
11
+ <a href="docs/translations/README.bn-IN.md">বাংলা</a> ·
12
+ <a href="docs/translations/README.es-ES.md">Español</a> ·
13
+ <a href="docs/translations/README.fr-FR.md">Français</a> ·
14
+ <a href="docs/translations/README.de-DE.md">Deutsch</a> ·
15
+ <a href="docs/translations/README.pt-BR.md">Português</a> ·
16
+ <a href="docs/translations/README.ru-RU.md">Русский</a> ·
17
+ <a href="docs/translations/README.ar-SA.md">العربية</a> ·
18
+ <a href="docs/translations/README.fa-IR.md">فارسی</a> ·
19
+ <a href="docs/translations/README.it-IT.md">Italiano</a> ·
20
+ <a href="docs/translations/README.tr-TR.md">Türkçe</a> ·
21
+ <a href="docs/translations/README.vi-VN.md">Tiếng Việt</a> ·
22
+ <a href="docs/translations/README.id-ID.md">Bahasa Indonesia</a> ·
23
+ <a href="docs/translations/README.pl-PL.md">Polski</a> ·
24
+ <a href="docs/translations/README.uk-UA.md">Українська</a> ·
25
+ <a href="docs/translations/README.nl-NL.md">Nederlands</a> ·
26
+ <a href="docs/translations/README.th-TH.md">ภาษาไทย</a>
27
+ </p>
28
+ <!-- LANG-PICKER-END -->
29
+
30
+ <p>
31
+ <img src="https://img.shields.io/badge/license-MIT-green" alt="MIT License"/>
32
+ <img src="https://img.shields.io/badge/tests-124%20passing-brightgreen" alt="Tests"/>
33
+ <img src="https://img.shields.io/badge/runtime-Bun-black?logo=bun" alt="Bun"/>
34
+ <img src="https://img.shields.io/badge/protocol-MCP-blue" alt="MCP"/>
35
+ </p>
36
+
37
+ <p align="center">
38
+ <a href="docs/assets/chitta-graph.mp4"><img src="docs/assets/chitta-graph.gif" width="640" alt="Chitta knowledge graph - a rotating 3D constellation of concepts, colored by type and linked by relationships"/></a>
39
+ </p>
40
+ <p align="center"><sub>A real Chitta knowledge graph - 285 concepts, 291 relationships, colored by type, labeled hubs. <a href="docs/assets/chitta-graph.mp4">▶ full-quality MP4</a></sub></p>
41
+
42
+ ***Chitta*** (चित्त) - in Indian philosophy, the mind's storehouse where every impression is
43
+ kept. Permission-aware **memory for AI agents**, by **[100xprompt](https://github.com/Nipurn123)**.
44
+
45
+ Permission-aware **knowledge graph + vector memory**, shipped as a standalone **MCP server**.
46
+ Any MCP client (Claude Code, 100xprompt, Claude Desktop, Cursor, IDEs) uses it via config - no code changes.
47
+
48
+ Point your AI assistant at it once, and every conversation can **store, recall, and reason over**
49
+ your team's knowledge - with each user seeing only what their permissions allow.
50
+
51
+ > The part every other memory layer treats as an afterthought: who is allowed to remember what.
52
+
53
+ > **Architecture & internals:** see [ARCHITECTURE.md](ARCHITECTURE.md).
54
+
55
+ - **Local mode (default):** one SQLite file. Ingest, extract a knowledge graph, retrieve - no servers.
56
+ - **Central-office mode:** point it at a shared backend (ArangoDB + Qdrant + embeddings) via env; the
57
+ whole org shares one graph, each user sees only what their ACL permits.
58
+
59
+ ## See it in 30 seconds
60
+
61
+ Two users, one store, three documents - each user sees only what they're allowed to:
62
+
63
+ ```bash
64
+ bun install
65
+ ./examples/permission-aware-retrieval/run.sh
66
+ ```
67
+
68
+ ```
69
+ ALICE (org handbook + her roadmap; NOT comp):
70
+ • [Company Handbook] Acme builds privacy-first AI infrastructure…
71
+ • [Eng Roadmap] Q3 roadmap: ship the permission-aware retrieval engine…
72
+
73
+ BOB (org handbook + his comp; NOT roadmap):
74
+ • [Company Handbook] Acme builds privacy-first AI infrastructure…
75
+ • [Comp Bands] Compensation bands for 2026. Senior engineers: 180-220k…
76
+ ```
77
+
78
+ Same query, different results - because the ACL graph produces the candidate set *before*
79
+ the vector index is touched. Full walkthrough: [examples/permission-aware-retrieval](examples/permission-aware-retrieval/).
80
+
81
+ **Benchmark:** on a small permission-scoped knowledge base, retrieval delivers
82
+ **7.4× fewer tokens** than dumping the whole corpus into context (more as the corpus
83
+ grows) - with **zero cross-user leak**. Reproducible: [examples/token-reduction](examples/token-reduction/).
84
+
85
+ ## Install
86
+
87
+ Chitta runs on [Bun](https://bun.sh) (install once: `curl -fsSL https://bun.sh/install | bash`).
88
+ One command then wires it into your AI tools - as an **MCP server** (everywhere) and a
89
+ **Skill** (where supported):
90
+
91
+ ```bash
92
+ bunx @100xprompt/chitta install # auto-detect installed tools
93
+ bunx @100xprompt/chitta install --all # every supported tool
94
+ bunx @100xprompt/chitta install --platform cursor,claude-code
95
+ bunx @100xprompt/chitta install --print # just print the MCP config to paste anywhere
96
+ ```
97
+
98
+ Options: `--project` (write project-scoped config instead of global) · `--user-id <id> --org-id <id>`
99
+ (bake identity into the config) · `--list` (show all tools) · `uninstall`.
100
+
101
+ **Supported tools (15):** Claude Code, Claude Desktop, Cursor, VS Code (Copilot), Windsurf,
102
+ Zed, Cline, Roo Code, Codex CLI, Gemini CLI, opencode, Kiro, Amp, Factory Droid, Kilo Code.
103
+ Skill (not just MCP) is installed for the ones that support it (Claude Code, Cursor, Gemini,
104
+ opencode, Kiro, Amp, Factory, Kilo, Trae). Any other MCP client: `--print` and paste.
105
+
106
+ > Published to npm as `@100xprompt/chitta` and run via `bunx` (the Bun runtime ships SQLite +
107
+ > the vector index in-process, so there are no native build steps for users).
108
+
109
+ ## Tools exposed over MCP
110
+
111
+ | Tool | Does |
112
+ |---|---|
113
+ | `context_ingest` | Store text → record node + **permission edges** (ACL) + **vector chunks** + **extracted concept graph** |
114
+ | `get_context` | Retrieve ranked, cited, permission-filtered snippets |
115
+ | `context_graph` | Return the knowledge graph (concepts + relationships) the user can access |
116
+
117
+ ## Run it
118
+
119
+ ```bash
120
+ bun install
121
+ bun start # boots the MCP server (stdio)
122
+ bun test # 124 tests
123
+ bun run build # → dist/chitta (single binary)
124
+ ```
125
+
126
+ ## Use it from any MCP client
127
+
128
+ ```jsonc
129
+ {
130
+ "mcp": {
131
+ "context": {
132
+ "type": "local",
133
+ "command": ["bun", "run", "/path/to/chitta/src/mcp/server.ts"],
134
+ "environment": { "CONTEXT_USER_ID": "alice", "CONTEXT_ORG_ID": "acme" }
135
+ }
136
+ }
137
+ }
138
+ ```
139
+
140
+ **Central office:** add the shared backend URLs so everyone queries one graph with per-user ACL:
141
+
142
+ ```jsonc
143
+ "environment": {
144
+ "CONTEXT_USER_ID": "alice", "CONTEXT_ORG_ID": "acme",
145
+ "CONTEXT_ARANGO_URL": "https://office.internal/arango",
146
+ "CONTEXT_QDRANT_URL": "https://office.internal/qdrant",
147
+ "CONTEXT_EMBED_URL": "https://office.internal/embed",
148
+ "CONTEXT_COLLECTION": "records"
149
+ }
150
+ ```
151
+
152
+ ## How it works
153
+
154
+ ```
155
+ ingest → record node + permission edges (ACL) ┐
156
+ → chunk + embed → vectors ├─ all in one SQLite file (local)
157
+ → extract entities + relations → graph ┘
158
+ query → ACL: which records may this user see? (graph traversal)
159
+ → vector search restricted to those records
160
+ → cross-connector leak guard → cited snippets
161
+ ```
162
+
163
+ ## Storage (local mode)
164
+
165
+ One file at `$CONTEXT_DB` or `~/.local/share/100xprompt/context.db`:
166
+ - `nodes` - graph vertices (records, users, orgs, **entities**)
167
+ - `edges` - relationships (`permissions`, `belongsTo`, `mentions`, `relates_to`)
168
+ - `chunks` - text + embedding vectors
169
+ - `vec_chunks` - **sqlite-vec ANN index** (when available - see below)
170
+
171
+ ## Vector search - adaptive (sqlite-vec, in-process)
172
+
173
+ If an extension-capable SQLite is present, the store loads **[sqlite-vec](https://github.com/asg017/sqlite-vec)**
174
+ and keeps a `vec0` ANN index *in the same file* - the TS-native, Python-free equivalent of
175
+ zvec ("the SQLite of vector DBs"). ~16× faster than brute-force at 3k vectors, more at scale.
176
+ Otherwise it **transparently falls back to brute-force cosine** - same results, same interface,
177
+ fully portable for the single-binary path.
178
+
179
+ `bun:sqlite` disables extension loading by default; to enable the ANN fast path, point it at an
180
+ extension-capable SQLite (e.g. `brew install sqlite`, auto-detected at common paths). No config
181
+ needed - `store.vecEnabled` reflects which path is active.
182
+
183
+ ## Status
184
+
185
+ Implemented: ACL graph, vector store, retrieval + leak guard, **knowledge-graph extraction**, MCP server
186
+ (local + central), fetch-based Arango/Qdrant/embedding adapters. All dependency-free except the MCP SDK.
187
+
188
+ Next (swap-in, same interfaces): real embeddings (transformers.js ONNX `bge-*`) for semantic ranking;
189
+ GraphRAG retrieval (expand results along `relates_to` edges); LLM-based entity extraction for higher recall.
190
+
191
+ See [ARCHITECTURE.md](ARCHITECTURE.md) for module-by-module internals and the security invariant.
192
+
193
+ ## Docs
194
+
195
+ - [ARCHITECTURE.md](ARCHITECTURE.md) - pipeline, module map, security invariant, extending it
196
+ - [examples/](examples/) - runnable demos
197
+ - [CONTRIBUTING.md](CONTRIBUTING.md) - dev setup and workflow
198
+ - [SECURITY.md](SECURITY.md) - security model and how to report issues
199
+ - [CHANGELOG.md](CHANGELOG.md) - notable changes
200
+
201
+ ## License
202
+
203
+ [MIT](LICENSE) © 2026 Nipurn Agarwal
@@ -0,0 +1,9 @@
1
+ <!-- chitta:start -->
2
+ ## Memory (Chitta)
3
+
4
+ This project has **Chitta** — permission-aware long-term memory — available over MCP.
5
+ - Before answering anything that may depend on earlier work, call `get_context` to recall.
6
+ - When the user shares a durable fact, decision, or preference, call `context_ingest` to store it.
7
+ - For "how are X and Y related", call `context_graph`.
8
+ Cite recalled snippets. Don't store secrets or throwaway state.
9
+ <!-- chitta:end -->
@@ -0,0 +1,47 @@
1
+ ---
2
+ name: chitta
3
+ description: Permission-aware long-term memory for AI agents. Use to recall prior context before answering, store durable facts/decisions/preferences, and query how concepts relate. Backed by Chitta's MCP tools (context_ingest, get_context, context_graph) with a CLI fallback. Trigger whenever a task may depend on earlier work, when the user shares something worth remembering, or when asked how things connect.
4
+ ---
5
+
6
+ # Chitta — memory for this agent
7
+
8
+ Chitta (चित्त, "the mind's storehouse") gives you persistent, **permission-aware** memory: a
9
+ knowledge graph + vector store exposed over MCP. Each user only ever sees what their
10
+ permissions allow. Use it proactively — memory is only useful if you reach for it.
11
+
12
+ ## When to use it
13
+
14
+ 1. **Recall before answering.** If a request might depend on prior decisions, context, or
15
+ facts ("what did we decide about X", "continue the Y work", anything project-specific),
16
+ call **`get_context`** first with the question. It returns ranked, cited,
17
+ permission-filtered snippets. Cite what you use.
18
+ 2. **Store durable knowledge.** When the user states a lasting fact, decision, preference, or
19
+ you produce an artifact worth remembering, call **`context_ingest`** with the text (and a
20
+ short `recordName`). Don't store secrets, throwaway chatter, or transient state.
21
+ 3. **Reason over connections.** For "how are X and Y related" or to map a topic, call
22
+ **`context_graph`** to get the concepts + relationships the user can access.
23
+
24
+ ## The MCP tools
25
+
26
+ | Tool | Use |
27
+ |---|---|
28
+ | `get_context` | Retrieve ranked, cited, permission-filtered snippets for a query |
29
+ | `context_ingest` | Store text → record + permission edges + vector chunks + extracted concept graph |
30
+ | `context_graph` | Return the accessible knowledge graph (concepts + relationships) |
31
+
32
+ ## CLI fallback (no MCP)
33
+
34
+ If MCP tools aren't available in this environment, shell out:
35
+
36
+ ```bash
37
+ bunx @100xprompt/chitta query "<question>" # recall
38
+ bunx @100xprompt/chitta ingest --text "<fact>" --name "<title>" # store
39
+ ```
40
+
41
+ ## Guardrails
42
+
43
+ - Respect permissions: never try to surface content a user isn't entitled to; Chitta enforces
44
+ this, but don't work around it.
45
+ - Prefer recalling over guessing. If `get_context` returns nothing relevant, say so rather
46
+ than inventing.
47
+ - Keep stored entries concise and factual; one idea per ingest.
package/package.json ADDED
@@ -0,0 +1,48 @@
1
+ {
2
+ "name": "@100xprompt/chitta",
3
+ "version": "0.1.0",
4
+ "description": "Chitta - permission-aware memory for AI agents: a knowledge-graph + vector memory MCP server with per-user access control. Runs on Bun. By 100xprompt.",
5
+ "type": "module",
6
+ "license": "MIT",
7
+ "engines": {
8
+ "bun": ">=1.0.0"
9
+ },
10
+ "bin": {
11
+ "chitta": "./src/bin.ts"
12
+ },
13
+ "files": [
14
+ "src",
15
+ "assets",
16
+ "README.md",
17
+ "LICENSE"
18
+ ],
19
+ "keywords": [
20
+ "mcp", "mcp-server", "model-context-protocol", "ai-memory", "agent-memory",
21
+ "knowledge-graph", "graph-rag", "rag", "vector-database", "permission-aware",
22
+ "rbac", "access-control", "ai-agents"
23
+ ],
24
+ "publishConfig": {
25
+ "access": "public"
26
+ },
27
+ "scripts": {
28
+ "start": "bun run src/bin.ts",
29
+ "test": "bun test test/",
30
+ "typecheck": "tsc --noEmit",
31
+ "build": "bun build src/bin.ts --compile --outfile dist/chitta",
32
+ "install:tools": "bun run src/bin.ts install",
33
+ "cli": "bun run src/embedded/cli.ts"
34
+ },
35
+ "dependencies": {
36
+ "@modelcontextprotocol/sdk": "^1.0.0",
37
+ "sqlite-vec": "^0.1.9",
38
+ "tree-sitter-wasms": "^0.1.13",
39
+ "web-tree-sitter": "0.24.7"
40
+ },
41
+ "optionalDependencies": {
42
+ "@huggingface/transformers": "^4.2.0"
43
+ },
44
+ "devDependencies": {
45
+ "@types/bun": "latest",
46
+ "typescript": "^5.6.0"
47
+ }
48
+ }
package/src/README.md ADDED
@@ -0,0 +1,124 @@
1
+ # `src/context` - the permission-aware retrieval moat (native TS)
2
+
3
+ This is the **Phase-2 port** from the reuse blueprint: PipesHub's ~800-LOC
4
+ retrieval + ACL layer, rewritten natively in TypeScript so we own and control the
5
+ moat without a Python dependency. The heavy/commodity parts (connectors, parsers,
6
+ embeddings server, the Arango/Qdrant engines) stay as the backend; only the
7
+ *logic that decides who sees what* lives here.
8
+
9
+ ## Provenance
10
+
11
+ | This file | Ported from (PipesHub) |
12
+ |---|---|
13
+ | `permission.ts` | `backend/python/app/models/permission.py` |
14
+ | `arango-graph-provider.ts` | `services/graph_db/arango/arango_http_provider.py` (`get_accessible_virtual_record_ids`, `_get_virtual_ids_for_connector`, `_get_kb_virtual_ids`, `_get_user_app_ids`) |
15
+ | `retrieval.ts` | `modules/retrieval/retrieval_service.py::search_with_filters` |
16
+ | `provider.ts`, `types.ts` | the interfaces those depend on |
17
+
18
+ ## The security invariant (do not break)
19
+
20
+ `RetrievalService.searchWithFilters` enforces, in order:
21
+
22
+ 1. **ACL first** - `getAccessibleVirtualRecordIds(user, org, filters)` computes the
23
+ `{ virtualRecordId -> recordId }` map of everything the user may access, by
24
+ traversing permission edges in the graph. Eight permission paths (direct,
25
+ group×2, org×2, record-group inheritance×2, anyone) plus two KB paths.
26
+ 2. **Restrict the search** - the vector query is filtered to those virtual ids.
27
+ The model never sees a chunk outside this set.
28
+ 3. **Cross-connector leak guard** - every hit resolves its `recordId` *from the
29
+ accessible map*, never from the vector payload. If two connectors share a
30
+ `virtualRecordId`, only the record this user may see is returned.
31
+
32
+ The AQL in `arango-graph-provider.ts` is preserved verbatim from the source. Each
33
+ path is a legitimate access route - dropping one silently denies access; loosening
34
+ one silently leaks data. Treat changes here as security-critical.
35
+
36
+ ## The seams + adapters (now built - fetch-based, zero deps)
37
+
38
+ The three backend seams (`provider.ts`) each have a concrete **fetch-based** adapter -
39
+ no SDKs, no Node-only APIs, nothing leaves the boundary when the endpoints are local:
40
+
41
+ | Seam | Adapter | Talks to |
42
+ |---|---|---|
43
+ | `ArangoClient` | `arango-client.ts` | ArangoDB HTTP cursor API (`/_api/cursor`, drains multi-batch) |
44
+ | `VectorDBService` | `qdrant-vector.ts` | Qdrant REST (`/points/query/batch`, builds the filter) |
45
+ | `EmbeddingProvider` | `embeddings.ts` | OpenAI-compatible `/v1/embeddings` + optional sparse endpoint |
46
+
47
+ Wire it all from config in one call:
48
+
49
+ ```ts
50
+ import { buildContextService } from "@/context/service"
51
+
52
+ const { retrieval } = buildContextService({
53
+ arango: { url: "http://localhost:8529", database: "_system", username, password },
54
+ qdrant: { url: "http://localhost:6333", apiKey },
55
+ embeddings: { denseEndpoint: "http://localhost:8002", denseModel: "BAAI/bge-small-en-v1.5", sparseEndpoint },
56
+ collectionName: "records",
57
+ })
58
+
59
+ const res = await retrieval.searchWithFilters({
60
+ queries: [userQuery],
61
+ userId, // the asking user (env v0; → packages/identity later)
62
+ orgId,
63
+ filterGroups: { kb: [...], apps: [...] },
64
+ limit: 20,
65
+ })
66
+ // res.searchResults are already ACL-filtered + cited.
67
+ ```
68
+
69
+ Config comes from env in v0 (`config-env.ts`: `CONTEXT_ARANGO_URL`, `CONTEXT_QDRANT_URL`,
70
+ `CONTEXT_EMBED_URL`, `CONTEXT_COLLECTION`, `CONTEXT_USER_ID`, `CONTEXT_ORG_ID`, …).
71
+
72
+ ## Exposed to the agent
73
+
74
+ `src/tool/context.ts` registers the **`get_context`** tool (in `src/tool/registry.ts`).
75
+ The agent calls it with a query; it returns ranked, cited, ACL-filtered snippets.
76
+
77
+ ## Two deployment tiers (same moat, swapped adapters)
78
+
79
+ The ports-and-adapters design means the ACL + retrieval logic is identical across both;
80
+ only the backend adapters differ.
81
+
82
+ **Tier 1 - server-backed** (`arango-client.ts`, `qdrant-vector.ts`, `embeddings.ts`):
83
+ scales out, uses ArangoDB + Qdrant + an embedding server. Wired by `buildContextService`.
84
+
85
+ **Tier 2 - embedded / single-binary** (`embedded/`): one SQLite file + in-process
86
+ embeddings, **zero servers, zero Python**. Wired by `buildEmbeddedContext`:
87
+ - `embedded/sqlite-store.ts` - node/edge/chunk schema in one `.db`.
88
+ - `embedded/sqlite-graph-provider.ts` - the ACL traversal **ported from AQL to recursive SQL** (same access semantics; same `GraphProvider` interface).
89
+ - `embedded/sqlite-vec-service.ts` - brute-force cosine honoring the must/should ACL filter (swap in sqlite-vec for scale).
90
+ - `embedded/local-embeddings.ts` - deterministic in-process embedder (swap in transformers.js / fastembed ONNX `bge-*` for real semantic quality - same `EmbeddingProvider` interface).
91
+
92
+ ```ts
93
+ import { buildEmbeddedContext } from "@/context/embedded"
94
+ const ctx = buildEmbeddedContext({ path: "knowledge.db" }) // one file, no servers
95
+ const res = await ctx.retrieval.searchWithFilters({ queries: [q], userId, orgId })
96
+ ```
97
+
98
+ `bun build src/context/embedded/demo.ts --compile --outfile ctx` produces a single
99
+ self-contained ~59 MB executable. **Note (distribution):** bun-compiled binaries on
100
+ macOS arm64 need a code-signing/notarization step before they'll launch (the kernel
101
+ SIGKILLs unsigned ones); finalize that in the release packaging. The logic itself runs
102
+ identically via `bun run`.
103
+
104
+ ## Verification
105
+
106
+ - `bun test` - 22 passing (ACL traversal, retrieval enforcement, adapter HTTP shaping, config).
107
+ - strict `tsc` (`strict` + `noUnusedLocals/Parameters` + `noImplicitOverride`) - 0 errors across all 11 production files.
108
+ - The module is **dependency-free** (relative imports + web `fetch` only) - no Python, no SDKs.
109
+
110
+ ## Intentionally omitted (port later if needed)
111
+
112
+ - Cosmetic file/mail `webUrl` + mime fallback enrichment (`retrieval_service.py`
113
+ 462-532) - presentation, not access control.
114
+ - Embedding-model config/caching, BGE query prefixing - wire to our model config.
115
+
116
+ ## Next steps (still net-new, per the blueprint)
117
+
118
+ - **Graph-level ACL propagation** - PipesHub enforces at the *record* level; push
119
+ `acl_ref` down to extracted entities/edges (edge = intersection of endpoints).
120
+ - **Bi-temporal edges** - `valid_at`/`invalid_at` for non-destructive update/delete.
121
+ - **Late-binding verify** - re-check the top-K against live source perms.
122
+ - Wire `searchWithFilters` into `src/tool/context.ts` (`get_context`) and
123
+ `src/cli/cmd/context.ts`, with `userId` supplied by `packages/identity`.
124
+ </content>
@@ -0,0 +1,67 @@
1
+ // ArangoClient adapter over ArangoDB's HTTP cursor API (no SDK dependency).
2
+ // Implements the single seam the ACL traversal needs: executeAql(query, bindVars),
3
+ // transparently following the cursor when results span multiple batches.
4
+
5
+ import type { ArangoClient } from "./provider"
6
+
7
+ export interface ArangoConfig {
8
+ /** e.g. http://localhost:8529 */
9
+ url: string
10
+ database: string
11
+ username?: string
12
+ password?: string
13
+ /** rows per cursor batch */
14
+ batchSize?: number
15
+ fetchImpl?: typeof fetch
16
+ }
17
+
18
+ interface CursorResponse {
19
+ result: any[]
20
+ hasMore: boolean
21
+ id?: string
22
+ error?: boolean
23
+ errorMessage?: string
24
+ }
25
+
26
+ export class ArangoHttpClient implements ArangoClient {
27
+ private readonly fetch: typeof fetch
28
+ constructor(private readonly cfg: ArangoConfig) {
29
+ this.fetch = cfg.fetchImpl ?? fetch
30
+ }
31
+
32
+ private headers(): Record<string, string> {
33
+ const h: Record<string, string> = { "content-type": "application/json" }
34
+ if (this.cfg.username != null) {
35
+ const basic = btoa(`${this.cfg.username}:${this.cfg.password ?? ""}`)
36
+ h["authorization"] = `Basic ${basic}`
37
+ }
38
+ return h
39
+ }
40
+
41
+ private base(): string {
42
+ return `${this.cfg.url.replace(/\/$/, "")}/_db/${encodeURIComponent(this.cfg.database)}`
43
+ }
44
+
45
+ async executeAql(query: string, bindVars: Record<string, unknown>): Promise<any[]> {
46
+ const res = await this.fetch(`${this.base()}/_api/cursor`, {
47
+ method: "POST",
48
+ headers: this.headers(),
49
+ body: JSON.stringify({ query, bindVars, batchSize: this.cfg.batchSize ?? 1000 }),
50
+ })
51
+ let body = (await res.json()) as CursorResponse
52
+ if (body.error) throw new Error(`arango: ${body.errorMessage ?? res.status}`)
53
+
54
+ const rows: any[] = [...body.result]
55
+ // Drain the cursor - ACL queries can legitimately return many records.
56
+ while (body.hasMore && body.id) {
57
+ const next = await this.fetch(`${this.base()}/_api/cursor/${body.id}`, {
58
+ method: "PUT",
59
+ headers: this.headers(),
60
+ })
61
+ body = (await next.json()) as CursorResponse
62
+ if (body.error) throw new Error(`arango cursor: ${body.errorMessage ?? next.status}`)
63
+ rows.push(...body.result)
64
+ }
65
+ return rows
66
+ }
67
+ }