@memtensor/memos-local-openclaw-plugin 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +11 -0
- package/README.md +251 -0
- package/SKILL.md +43 -0
- package/dist/capture/index.d.ts +16 -0
- package/dist/capture/index.d.ts.map +1 -0
- package/dist/capture/index.js +80 -0
- package/dist/capture/index.js.map +1 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +96 -0
- package/dist/config.js.map +1 -0
- package/dist/embedding/index.d.ts +12 -0
- package/dist/embedding/index.d.ts.map +1 -0
- package/dist/embedding/index.js +75 -0
- package/dist/embedding/index.js.map +1 -0
- package/dist/embedding/local.d.ts +3 -0
- package/dist/embedding/local.d.ts.map +1 -0
- package/dist/embedding/local.js +65 -0
- package/dist/embedding/local.js.map +1 -0
- package/dist/embedding/providers/cohere.d.ts +4 -0
- package/dist/embedding/providers/cohere.d.ts.map +1 -0
- package/dist/embedding/providers/cohere.js +57 -0
- package/dist/embedding/providers/cohere.js.map +1 -0
- package/dist/embedding/providers/gemini.d.ts +3 -0
- package/dist/embedding/providers/gemini.d.ts.map +1 -0
- package/dist/embedding/providers/gemini.js +31 -0
- package/dist/embedding/providers/gemini.js.map +1 -0
- package/dist/embedding/providers/mistral.d.ts +3 -0
- package/dist/embedding/providers/mistral.d.ts.map +1 -0
- package/dist/embedding/providers/mistral.js +25 -0
- package/dist/embedding/providers/mistral.js.map +1 -0
- package/dist/embedding/providers/openai.d.ts +3 -0
- package/dist/embedding/providers/openai.d.ts.map +1 -0
- package/dist/embedding/providers/openai.js +35 -0
- package/dist/embedding/providers/openai.js.map +1 -0
- package/dist/embedding/providers/voyage.d.ts +3 -0
- package/dist/embedding/providers/voyage.d.ts.map +1 -0
- package/dist/embedding/providers/voyage.js +25 -0
- package/dist/embedding/providers/voyage.js.map +1 -0
- package/dist/index.d.ts +44 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +75 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/chunker.d.ts +15 -0
- package/dist/ingest/chunker.d.ts.map +1 -0
- package/dist/ingest/chunker.js +193 -0
- package/dist/ingest/chunker.js.map +1 -0
- package/dist/ingest/dedup.d.ts +11 -0
- package/dist/ingest/dedup.d.ts.map +1 -0
- package/dist/ingest/dedup.js +29 -0
- package/dist/ingest/dedup.js.map +1 -0
- package/dist/ingest/providers/anthropic.d.ts +3 -0
- package/dist/ingest/providers/anthropic.d.ts.map +1 -0
- package/dist/ingest/providers/anthropic.js +33 -0
- package/dist/ingest/providers/anthropic.js.map +1 -0
- package/dist/ingest/providers/bedrock.d.ts +8 -0
- package/dist/ingest/providers/bedrock.d.ts.map +1 -0
- package/dist/ingest/providers/bedrock.js +41 -0
- package/dist/ingest/providers/bedrock.js.map +1 -0
- package/dist/ingest/providers/gemini.d.ts +3 -0
- package/dist/ingest/providers/gemini.d.ts.map +1 -0
- package/dist/ingest/providers/gemini.js +31 -0
- package/dist/ingest/providers/gemini.js.map +1 -0
- package/dist/ingest/providers/index.d.ts +9 -0
- package/dist/ingest/providers/index.d.ts.map +1 -0
- package/dist/ingest/providers/index.js +68 -0
- package/dist/ingest/providers/index.js.map +1 -0
- package/dist/ingest/providers/openai.d.ts +3 -0
- package/dist/ingest/providers/openai.d.ts.map +1 -0
- package/dist/ingest/providers/openai.js +41 -0
- package/dist/ingest/providers/openai.js.map +1 -0
- package/dist/ingest/worker.d.ts +21 -0
- package/dist/ingest/worker.d.ts.map +1 -0
- package/dist/ingest/worker.js +111 -0
- package/dist/ingest/worker.js.map +1 -0
- package/dist/recall/engine.d.ts +23 -0
- package/dist/recall/engine.d.ts.map +1 -0
- package/dist/recall/engine.js +153 -0
- package/dist/recall/engine.js.map +1 -0
- package/dist/recall/mmr.d.ts +17 -0
- package/dist/recall/mmr.d.ts.map +1 -0
- package/dist/recall/mmr.js +51 -0
- package/dist/recall/mmr.js.map +1 -0
- package/dist/recall/recency.d.ts +20 -0
- package/dist/recall/recency.d.ts.map +1 -0
- package/dist/recall/recency.js +26 -0
- package/dist/recall/recency.js.map +1 -0
- package/dist/recall/rrf.d.ts +16 -0
- package/dist/recall/rrf.d.ts.map +1 -0
- package/dist/recall/rrf.js +15 -0
- package/dist/recall/rrf.js.map +1 -0
- package/dist/storage/sqlite.d.ts +34 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/storage/sqlite.js +274 -0
- package/dist/storage/sqlite.js.map +1 -0
- package/dist/storage/vector.d.ts +13 -0
- package/dist/storage/vector.d.ts.map +1 -0
- package/dist/storage/vector.js +33 -0
- package/dist/storage/vector.js.map +1 -0
- package/dist/tools/index.d.ts +4 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +10 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/memory-get.d.ts +4 -0
- package/dist/tools/memory-get.d.ts.map +1 -0
- package/dist/tools/memory-get.js +59 -0
- package/dist/tools/memory-get.js.map +1 -0
- package/dist/tools/memory-search.d.ts +4 -0
- package/dist/tools/memory-search.d.ts.map +1 -0
- package/dist/tools/memory-search.js +36 -0
- package/dist/tools/memory-search.js.map +1 -0
- package/dist/tools/memory-timeline.d.ts +4 -0
- package/dist/tools/memory-timeline.d.ts.map +1 -0
- package/dist/tools/memory-timeline.js +64 -0
- package/dist/tools/memory-timeline.js.map +1 -0
- package/dist/types.d.ts +158 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +25 -0
- package/dist/types.js.map +1 -0
- package/dist/viewer/html.d.ts +2 -0
- package/dist/viewer/html.d.ts.map +1 -0
- package/dist/viewer/html.js +686 -0
- package/dist/viewer/html.js.map +1 -0
- package/dist/viewer/server.d.ts +48 -0
- package/dist/viewer/server.d.ts.map +1 -0
- package/dist/viewer/server.js +470 -0
- package/dist/viewer/server.js.map +1 -0
- package/index.ts +357 -0
- package/openclaw.plugin.json +57 -0
- package/package.json +57 -0
- package/src/capture/index.ts +92 -0
- package/src/config.ts +67 -0
- package/src/embedding/index.ts +76 -0
- package/src/embedding/local.ts +35 -0
- package/src/embedding/providers/cohere.ts +69 -0
- package/src/embedding/providers/gemini.ts +41 -0
- package/src/embedding/providers/mistral.ts +32 -0
- package/src/embedding/providers/openai.ts +42 -0
- package/src/embedding/providers/voyage.ts +32 -0
- package/src/index.ts +106 -0
- package/src/ingest/chunker.ts +217 -0
- package/src/ingest/dedup.ts +37 -0
- package/src/ingest/providers/anthropic.ts +41 -0
- package/src/ingest/providers/bedrock.ts +50 -0
- package/src/ingest/providers/gemini.ts +41 -0
- package/src/ingest/providers/index.ts +67 -0
- package/src/ingest/providers/openai.ts +48 -0
- package/src/ingest/worker.ts +130 -0
- package/src/recall/engine.ts +182 -0
- package/src/recall/mmr.ts +60 -0
- package/src/recall/recency.ts +27 -0
- package/src/recall/rrf.ts +31 -0
- package/src/storage/sqlite.ts +305 -0
- package/src/storage/vector.ts +39 -0
- package/src/tools/index.ts +3 -0
- package/src/tools/memory-get.ts +68 -0
- package/src/tools/memory-search.ts +36 -0
- package/src/tools/memory-timeline.ts +73 -0
- package/src/types.ts +214 -0
- package/src/viewer/html.ts +682 -0
- package/src/viewer/server.ts +464 -0
- package/www/index.html +606 -0
package/.env.example
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Embedding API
|
|
2
|
+
# Use any OpenAI-compatible embedding service, or leave blank for local offline model
|
|
3
|
+
EMBEDDING_API_KEY=your-embedding-api-key
|
|
4
|
+
EMBEDDING_ENDPOINT=https://api.openai.com/v1
|
|
5
|
+
EMBEDDING_MODEL=text-embedding-3-small
|
|
6
|
+
|
|
7
|
+
# Summarizer API (OpenAI-compatible)
|
|
8
|
+
# Leave blank to use rule-based fallback (no LLM needed)
|
|
9
|
+
SUMMARIZER_API_KEY=your-summarizer-api-key
|
|
10
|
+
SUMMARIZER_ENDPOINT=https://api.openai.com/v1
|
|
11
|
+
SUMMARIZER_MODEL=gpt-4o-mini
|
package/README.md
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
# 🦞 MemOS Local — OpenClaw Memory Plugin
|
|
2
|
+
|
|
3
|
+
Persistent local conversation memory for [OpenClaw](https://github.com/nicepkg/openclaw) AI Agents. Every conversation is automatically captured, semantically indexed, and instantly recallable.
|
|
4
|
+
|
|
5
|
+
**Full-write | Hybrid Search (FTS5 + Vector) | RRF Fusion | MMR Diversity | Recency Decay**
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- **Auto-capture** — Stores user, assistant, and tool messages after each agent turn
|
|
10
|
+
- **Semantic chunking** — Preserves complete code blocks, function bodies, and paragraph boundaries
|
|
11
|
+
- **Hybrid retrieval** — FTS5 keyword + vector semantic dual-channel search with RRF fusion
|
|
12
|
+
- **LLM summarization** — One-sentence summary per chunk for fast browsing
|
|
13
|
+
- **Multi-provider** — OpenAI, Anthropic, Gemini, Cohere, Voyage, Mistral, Bedrock, or local offline
|
|
14
|
+
- **Web viewer** — Built-in dashboard at `http://127.0.0.1:18799` with CRUD, search, filters, pagination
|
|
15
|
+
- **Privacy first** — All data in local SQLite, no cloud uploads, password-protected viewer
|
|
16
|
+
|
|
17
|
+
## Quick Start
|
|
18
|
+
|
|
19
|
+
### 1. Install
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
openclaw plugins install memos-local
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Or from source:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
git clone https://github.com/MemTensor/MemOS.git
|
|
29
|
+
cd MemOS/apps/memos-local-openclaw-plugin
|
|
30
|
+
npm install && npm run build
|
|
31
|
+
openclaw plugins install .
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### 2. Configure
|
|
35
|
+
|
|
36
|
+
Add the plugin config to `~/.openclaw/openclaw.json`:
|
|
37
|
+
|
|
38
|
+
```jsonc
|
|
39
|
+
{
|
|
40
|
+
"plugins": {
|
|
41
|
+
"slots": {
|
|
42
|
+
"memory": "memos-local"
|
|
43
|
+
},
|
|
44
|
+
"entries": {
|
|
45
|
+
"memos-local": {
|
|
46
|
+
"enabled": true,
|
|
47
|
+
"config": {
|
|
48
|
+
"embedding": {
|
|
49
|
+
"provider": "openai_compatible",
|
|
50
|
+
"endpoint": "https://your-embedding-api.com/v1",
|
|
51
|
+
"apiKey": "your-embedding-key",
|
|
52
|
+
"model": "bge-m3"
|
|
53
|
+
},
|
|
54
|
+
"summarizer": {
|
|
55
|
+
"provider": "openai_compatible",
|
|
56
|
+
"endpoint": "https://api.openai.com/v1",
|
|
57
|
+
"apiKey": "sk-xxx",
|
|
58
|
+
"model": "gpt-4o-mini",
|
|
59
|
+
"temperature": 0
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
#### Embedding Provider Options
|
|
69
|
+
|
|
70
|
+
| Provider | `provider` value | Example `model` | Notes |
|
|
71
|
+
|---|---|---|---|
|
|
72
|
+
| OpenAI / compatible | `openai_compatible` | `bge-m3`, `text-embedding-3-small` | Any OpenAI-compatible API |
|
|
73
|
+
| Gemini | `gemini` | `text-embedding-004` | Requires `apiKey` |
|
|
74
|
+
| Cohere | `cohere` | `embed-english-v3.0` | Separates document/query embedding |
|
|
75
|
+
| Voyage | `voyage` | `voyage-2` | |
|
|
76
|
+
| Mistral | `mistral` | `mistral-embed` | |
|
|
77
|
+
| Local (offline) | `local` | — | Uses `Xenova/all-MiniLM-L6-v2`, no API needed |
|
|
78
|
+
|
|
79
|
+
> **No embedding config?** The plugin falls back to the local model automatically. You can start with zero configuration and add a cloud provider later for better quality.
|
|
80
|
+
|
|
81
|
+
#### Summarizer Provider Options
|
|
82
|
+
|
|
83
|
+
| Provider | `provider` value | Example `model` |
|
|
84
|
+
|---|---|---|
|
|
85
|
+
| OpenAI / compatible | `openai_compatible` | `gpt-4o-mini` |
|
|
86
|
+
| Anthropic | `anthropic` | `claude-3-haiku-20240307` |
|
|
87
|
+
| Gemini | `gemini` | `gemini-1.5-flash` |
|
|
88
|
+
| AWS Bedrock | `bedrock` | `anthropic.claude-3-haiku-20240307-v1:0` |
|
|
89
|
+
|
|
90
|
+
> **No summarizer config?** A rule-based fallback generates summaries from the first sentence + key entities. Good enough to start.
|
|
91
|
+
|
|
92
|
+
#### Environment Variable Support
|
|
93
|
+
|
|
94
|
+
Use `${ENV_VAR}` placeholders in config to avoid hardcoding keys:
|
|
95
|
+
|
|
96
|
+
```jsonc
|
|
97
|
+
{
|
|
98
|
+
"apiKey": "${OPENAI_API_KEY}"
|
|
99
|
+
}
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### 3. Restart Gateway
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
openclaw gateway stop
|
|
106
|
+
openclaw gateway install
|
|
107
|
+
openclaw gateway start
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
### 4. Verify Installation
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
# Check the gateway log
|
|
114
|
+
tail -20 ~/.openclaw/logs/gateway.log
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
You should see:
|
|
118
|
+
|
|
119
|
+
```
|
|
120
|
+
memos-local: initialized (db: ~/.openclaw/memos-local/memos.db)
|
|
121
|
+
memos-local: started (embedding: openai_compatible)
|
|
122
|
+
╔══════════════════════════════════════════╗
|
|
123
|
+
║ MemOS Memory Viewer ║
|
|
124
|
+
║ → http://127.0.0.1:18799 ║
|
|
125
|
+
║ Open in browser to manage memories ║
|
|
126
|
+
╚══════════════════════════════════════════╝
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### 5. Verify Memory is Working
|
|
130
|
+
|
|
131
|
+
**Step A** — Have a conversation with your OpenClaw agent about anything.
|
|
132
|
+
|
|
133
|
+
**Step B** — Open the Memory Viewer at `http://127.0.0.1:18799` and check that the conversation appears.
|
|
134
|
+
|
|
135
|
+
**Step C** — In a new conversation, ask the agent to recall what you discussed:
|
|
136
|
+
|
|
137
|
+
```
|
|
138
|
+
You: Do you remember what we talked about?
|
|
139
|
+
Agent: (calls memory_search) Yes, we discussed...
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
**Step D** — Check the gateway log for ingest activity:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
grep -E "Stored chunk|Chunked turn|Dedup" ~/.openclaw/logs/gateway.log | tail -10
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
You should see lines like:
|
|
149
|
+
|
|
150
|
+
```
|
|
151
|
+
Chunked turn=1772459198930-839rr3 into 3 chunks
|
|
152
|
+
Stored chunk=667f289e kind=paragraph len=392 hasVec=true
|
|
153
|
+
Stored chunk=107d7f32 kind=tool_result role=tool len=210 hasVec=true
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### 6. Run the Smoke Test (Optional)
|
|
157
|
+
|
|
158
|
+
For a comprehensive end-to-end test with your actual API:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
# Create a .env file with your keys
|
|
162
|
+
cp .env.example .env
|
|
163
|
+
# Edit .env with your actual API keys
|
|
164
|
+
|
|
165
|
+
# Run the smoke test
|
|
166
|
+
npx tsx scripts/smoke-test.ts
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
The smoke test writes test conversations, searches for them, verifies timeline and get operations, and checks anti-writeback protection.
|
|
170
|
+
|
|
171
|
+
## Agent Tools
|
|
172
|
+
|
|
173
|
+
The plugin registers 4 tools that your agent can use:
|
|
174
|
+
|
|
175
|
+
| Tool | Purpose |
|
|
176
|
+
|---|---|
|
|
177
|
+
| `memory_search` | Search memories by natural language query |
|
|
178
|
+
| `memory_timeline` | Get surrounding context for a search hit |
|
|
179
|
+
| `memory_get` | Get full original text of a memory chunk |
|
|
180
|
+
| `memory_viewer` | Get the URL of the web dashboard |
|
|
181
|
+
|
|
182
|
+
The agent uses these automatically via the SKILL.md prompt guide.
|
|
183
|
+
|
|
184
|
+
## Memory Viewer
|
|
185
|
+
|
|
186
|
+
Open `http://127.0.0.1:18799` in your browser:
|
|
187
|
+
|
|
188
|
+
- First visit: set a password (min 4 chars)
|
|
189
|
+
- Browse, search, create, edit, delete memories
|
|
190
|
+
- Filter by role (user/assistant/tool), type, time range (down to seconds)
|
|
191
|
+
- Pagination (30 per page)
|
|
192
|
+
|
|
193
|
+
**Forgot password?** Click "Forgot password?" on the login page and use the reset token from the gateway log:
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
grep "reset token" ~/.openclaw/logs/gateway.log | tail -1
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
## Advanced Configuration
|
|
200
|
+
|
|
201
|
+
All optional — shown with defaults:
|
|
202
|
+
|
|
203
|
+
```jsonc
|
|
204
|
+
{
|
|
205
|
+
"config": {
|
|
206
|
+
"recall": {
|
|
207
|
+
"maxResultsDefault": 6, // Default search results
|
|
208
|
+
"maxResultsMax": 20, // Max search results
|
|
209
|
+
"minScoreDefault": 0.45, // Default min score threshold
|
|
210
|
+
"minScoreFloor": 0.35, // Lowest allowed min score
|
|
211
|
+
"rrfK": 60, // RRF fusion constant
|
|
212
|
+
"mmrLambda": 0.7, // MMR relevance vs diversity (0-1)
|
|
213
|
+
"recencyHalfLifeDays": 14 // Time decay half-life
|
|
214
|
+
},
|
|
215
|
+
"dedup": {
|
|
216
|
+
"similarityThreshold": 0.93 // Cosine similarity for dedup
|
|
217
|
+
},
|
|
218
|
+
"viewerPort": 18799 // Memory Viewer port
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## How It Works
|
|
224
|
+
|
|
225
|
+
**Write path** (automatic on every agent turn):
|
|
226
|
+
|
|
227
|
+
```
|
|
228
|
+
Conversation → Capture (filter roles) → Semantic Chunk → LLM Summarize
|
|
229
|
+
→ Embed → Dedup Check → Store (SQLite + FTS5 + Vector)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
**Read path** (when agent calls `memory_search`):
|
|
233
|
+
|
|
234
|
+
```
|
|
235
|
+
Query → FTS5 + Vector dual recall → RRF Fusion → MMR Rerank
|
|
236
|
+
→ Recency Decay → Score Filter → Top-K Results
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
See [full documentation](www/docs/) for detailed architecture and algorithm explanations.
|
|
240
|
+
|
|
241
|
+
## Data Location
|
|
242
|
+
|
|
243
|
+
| File | Path |
|
|
244
|
+
|---|---|
|
|
245
|
+
| Database | `~/.openclaw/memos-local/memos.db` |
|
|
246
|
+
| Viewer auth | `~/.openclaw/memos-local/viewer-auth.json` |
|
|
247
|
+
| Gateway log | `~/.openclaw/logs/gateway.log` |
|
|
248
|
+
|
|
249
|
+
## License
|
|
250
|
+
|
|
251
|
+
MIT
|
package/SKILL.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Memory Recall Skill — memos-local
|
|
2
|
+
|
|
3
|
+
You have access to long-term conversation memory through three tools:
|
|
4
|
+
`memory_search`, `memory_timeline`, `memory_get`.
|
|
5
|
+
|
|
6
|
+
## When to Search
|
|
7
|
+
|
|
8
|
+
- **DO search** when the user asks about a previous conversation, a specific detail (command, error, parameter, decision) you don't have in the current context, or when the topic clearly references past interactions.
|
|
9
|
+
- **DO NOT search** when the current context already contains enough information to answer accurately.
|
|
10
|
+
|
|
11
|
+
## Progressive Recall Chain
|
|
12
|
+
|
|
13
|
+
Follow this 3-step chain. Stop as soon as you have sufficient evidence.
|
|
14
|
+
|
|
15
|
+
### Step 1 — `memory_search`
|
|
16
|
+
|
|
17
|
+
Start with the default call (no extra parameters → top 6 results, minScore 0.45).
|
|
18
|
+
|
|
19
|
+
- If results are insufficient, **refine the query first** (add entity names, exact commands, error keywords).
|
|
20
|
+
- If still insufficient, increase `maxResults` to 12, then 20.
|
|
21
|
+
- As a last resort, lower `minScore` to 0.35.
|
|
22
|
+
- **Never repeat the exact same query with the same parameters** — vary query wording or adjust parameters.
|
|
23
|
+
|
|
24
|
+
### Step 2 — `memory_timeline`
|
|
25
|
+
|
|
26
|
+
When a search hit looks relevant but the `original_excerpt` is too short to confirm, call `memory_timeline` with the hit's `ref` to get surrounding context (±2 turns by default).
|
|
27
|
+
|
|
28
|
+
### Step 3 — `memory_get`
|
|
29
|
+
|
|
30
|
+
When you need the exact original text (to verify a command, a code snippet, an error stack), call `memory_get` with the `ref` and request up to 2000 characters (max 8000).
|
|
31
|
+
|
|
32
|
+
## How to Answer
|
|
33
|
+
|
|
34
|
+
1. **Evidence-based**: Only state facts backed by `original_excerpt`, timeline entries, or `memory_get` content. Do not fabricate details.
|
|
35
|
+
2. **Cite sources**: When referencing a memory, mention the approximate time and context (e.g., "In our conversation about deploying the API…").
|
|
36
|
+
3. **Acknowledge gaps**: If memory search returns no relevant results, say so honestly rather than guessing.
|
|
37
|
+
|
|
38
|
+
## Anti-Patterns to Avoid
|
|
39
|
+
|
|
40
|
+
- Searching on every single turn (only search when needed).
|
|
41
|
+
- Repeating the same failed query without modification.
|
|
42
|
+
- Ignoring `original_excerpt` and only using `summary` — the excerpt is the primary evidence.
|
|
43
|
+
- Making claims based solely on `summary` without checking the original text when details matter.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { ConversationMessage, Logger } from "../types";
|
|
2
|
+
/**
|
|
3
|
+
* Filter and extract writable messages from a conversation turn.
|
|
4
|
+
*
|
|
5
|
+
* - Keep user, assistant, and tool messages
|
|
6
|
+
* - Skip system prompts
|
|
7
|
+
* - Skip tool results from our own memory tools (prevents memory loop)
|
|
8
|
+
* - Truncate long tool results to avoid storage bloat
|
|
9
|
+
* - Strip injected evidence blocks wrapped in [STORED_MEMORY]...[/STORED_MEMORY]
|
|
10
|
+
*/
|
|
11
|
+
export declare function captureMessages(messages: Array<{
|
|
12
|
+
role: string;
|
|
13
|
+
content: string;
|
|
14
|
+
toolName?: string;
|
|
15
|
+
}>, sessionKey: string, turnId: string, evidenceTag: string, log: Logger): ConversationMessage[];
|
|
16
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/capture/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAQ,MAAM,EAAE,MAAM,UAAU,CAAC;AAYlE;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,EACrE,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,MAAM,GACV,mBAAmB,EAAE,CA+CvB"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.captureMessages = captureMessages;
|
|
4
|
+
const types_1 = require("../types");
|
|
5
|
+
const SKIP_ROLES = new Set(["system"]);
|
|
6
|
+
const SELF_TOOLS = new Set([
|
|
7
|
+
"memory_search",
|
|
8
|
+
"memory_timeline",
|
|
9
|
+
"memory_get",
|
|
10
|
+
"memory_viewer",
|
|
11
|
+
]);
|
|
12
|
+
/**
|
|
13
|
+
* Filter and extract writable messages from a conversation turn.
|
|
14
|
+
*
|
|
15
|
+
* - Keep user, assistant, and tool messages
|
|
16
|
+
* - Skip system prompts
|
|
17
|
+
* - Skip tool results from our own memory tools (prevents memory loop)
|
|
18
|
+
* - Truncate long tool results to avoid storage bloat
|
|
19
|
+
* - Strip injected evidence blocks wrapped in [STORED_MEMORY]...[/STORED_MEMORY]
|
|
20
|
+
*/
|
|
21
|
+
function captureMessages(messages, sessionKey, turnId, evidenceTag, log) {
|
|
22
|
+
const now = Date.now();
|
|
23
|
+
const result = [];
|
|
24
|
+
for (const msg of messages) {
|
|
25
|
+
const role = msg.role;
|
|
26
|
+
if (SKIP_ROLES.has(role))
|
|
27
|
+
continue;
|
|
28
|
+
if (!msg.content || msg.content.trim().length === 0)
|
|
29
|
+
continue;
|
|
30
|
+
if (role === "tool") {
|
|
31
|
+
if (msg.toolName && SELF_TOOLS.has(msg.toolName)) {
|
|
32
|
+
log.debug(`Skipping self-tool result: ${msg.toolName}`);
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
let content = msg.content.trim();
|
|
36
|
+
const maxChars = types_1.DEFAULTS.toolResultMaxChars;
|
|
37
|
+
if (content.length > maxChars) {
|
|
38
|
+
content = content.slice(0, maxChars) + `\n\n[truncated — original ${content.length} chars]`;
|
|
39
|
+
}
|
|
40
|
+
const toolLabel = msg.toolName ? `[tool:${msg.toolName}] ` : "[tool] ";
|
|
41
|
+
result.push({
|
|
42
|
+
role: "tool",
|
|
43
|
+
content: toolLabel + content,
|
|
44
|
+
timestamp: now,
|
|
45
|
+
turnId,
|
|
46
|
+
sessionKey,
|
|
47
|
+
toolName: msg.toolName,
|
|
48
|
+
});
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
const cleaned = stripEvidenceBlocks(msg.content, evidenceTag);
|
|
52
|
+
if (cleaned.trim().length === 0)
|
|
53
|
+
continue;
|
|
54
|
+
result.push({
|
|
55
|
+
role,
|
|
56
|
+
content: cleaned,
|
|
57
|
+
timestamp: now,
|
|
58
|
+
turnId,
|
|
59
|
+
sessionKey,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
log.debug(`Captured ${result.length}/${messages.length} messages for session=${sessionKey} turn=${turnId}`);
|
|
63
|
+
return result;
|
|
64
|
+
}
|
|
65
|
+
function stripEvidenceBlocks(text, tag) {
|
|
66
|
+
const openTag = `[${tag}]`;
|
|
67
|
+
const closeTag = `[/${tag}]`;
|
|
68
|
+
let result = text;
|
|
69
|
+
let safety = 0;
|
|
70
|
+
while (result.includes(openTag) && result.includes(closeTag) && safety < 50) {
|
|
71
|
+
const start = result.indexOf(openTag);
|
|
72
|
+
const end = result.indexOf(closeTag, start);
|
|
73
|
+
if (end === -1)
|
|
74
|
+
break;
|
|
75
|
+
result = result.slice(0, start) + result.slice(end + closeTag.length);
|
|
76
|
+
safety++;
|
|
77
|
+
}
|
|
78
|
+
return result;
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/capture/index.ts"],"names":[],"mappings":";;AAqBA,0CAqDC;AAzED,oCAAoC;AAEpC,MAAM,UAAU,GAAc,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAElD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,eAAe;IACf,iBAAiB;IACjB,YAAY;IACZ,eAAe;CAChB,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,SAAgB,eAAe,CAC7B,QAAqE,EACrE,UAAkB,EAClB,MAAc,EACd,WAAmB,EACnB,GAAW;IAEX,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAY,CAAC;QAC9B,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QACnC,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE9D,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjD,GAAG,CAAC,KAAK,CAAC,8BAA8B,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACxD,SAAS;YACX,CAAC;YAED,IAAI,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,gBAAQ,CAAC,kBAAkB,CAAC;YAC7C,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBAC9B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,6BAA6B,OAAO,CAAC,MAAM,SAAS,CAAC;YAC9F,CAAC;YAED,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,SAAS,GAAG,OAAO;gBAC5B,SAAS,EAAE,GAAG;gBACd,MAAM;gBACN,UAAU;gBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;aACvB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE1C,MAAM,CAAC,IAAI,CAAC;YACV,IAAI;YACJ,OAAO,EAAE,OAAO;YAChB,SAAS,EAAE,GAAG;YACd,MAAM;YACN,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,yBAAyB,UAAU,SAAS,MAAM,EAAE,CAAC,CAAC;IAC5G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,GAAW;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,GAAG,CAAC;IAC3B,MAAM,QAAQ,GAAG,KAAK,GAAG,GAAG,CAAC;IAC7B,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;QAC5E,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAM;QACtB,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtE,MAAM,EAAE,CAAC;IACX,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type MemosLocalConfig, type PluginContext, type Logger } from "./types";
|
|
2
|
+
export declare function resolveConfig(raw: Partial<MemosLocalConfig> | undefined, stateDir: string): MemosLocalConfig;
|
|
3
|
+
export declare function buildContext(stateDir: string, workspaceDir: string, rawConfig: Partial<MemosLocalConfig> | undefined, log?: Logger): PluginContext;
|
|
4
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EAAY,KAAK,gBAAgB,EAAE,KAAK,aAAa,EAAE,KAAK,MAAM,EAAE,MAAM,SAAS,CAAC;AAqB3F,wBAAgB,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,CAuB5G;AAED,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,EAChD,GAAG,CAAC,EAAE,MAAM,GACX,aAAa,CAcf"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.resolveConfig = resolveConfig;
|
|
37
|
+
exports.buildContext = buildContext;
|
|
38
|
+
const path = __importStar(require("path"));
|
|
39
|
+
const types_1 = require("./types");
|
|
40
|
+
const ENV_RE = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
|
|
41
|
+
function resolveEnvVars(value) {
|
|
42
|
+
return value.replace(ENV_RE, (_, name) => process.env[name] ?? "");
|
|
43
|
+
}
|
|
44
|
+
function deepResolveEnv(obj) {
|
|
45
|
+
if (typeof obj === "string")
|
|
46
|
+
return resolveEnvVars(obj);
|
|
47
|
+
if (Array.isArray(obj))
|
|
48
|
+
return obj.map(deepResolveEnv);
|
|
49
|
+
if (obj && typeof obj === "object") {
|
|
50
|
+
const out = {};
|
|
51
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
52
|
+
out[k] = deepResolveEnv(v);
|
|
53
|
+
}
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
56
|
+
return obj;
|
|
57
|
+
}
|
|
58
|
+
function resolveConfig(raw, stateDir) {
|
|
59
|
+
const cfg = deepResolveEnv(raw ?? {});
|
|
60
|
+
return {
|
|
61
|
+
...cfg,
|
|
62
|
+
storage: {
|
|
63
|
+
dbPath: cfg.storage?.dbPath ?? path.join(stateDir, "memos-local", "memos.db"),
|
|
64
|
+
},
|
|
65
|
+
recall: {
|
|
66
|
+
maxResultsDefault: cfg.recall?.maxResultsDefault ?? types_1.DEFAULTS.maxResultsDefault,
|
|
67
|
+
maxResultsMax: cfg.recall?.maxResultsMax ?? types_1.DEFAULTS.maxResultsMax,
|
|
68
|
+
minScoreDefault: cfg.recall?.minScoreDefault ?? types_1.DEFAULTS.minScoreDefault,
|
|
69
|
+
minScoreFloor: cfg.recall?.minScoreFloor ?? types_1.DEFAULTS.minScoreFloor,
|
|
70
|
+
rrfK: cfg.recall?.rrfK ?? types_1.DEFAULTS.rrfK,
|
|
71
|
+
mmrLambda: cfg.recall?.mmrLambda ?? types_1.DEFAULTS.mmrLambda,
|
|
72
|
+
recencyHalfLifeDays: cfg.recall?.recencyHalfLifeDays ?? types_1.DEFAULTS.recencyHalfLifeDays,
|
|
73
|
+
},
|
|
74
|
+
dedup: {
|
|
75
|
+
similarityThreshold: cfg.dedup?.similarityThreshold ?? types_1.DEFAULTS.dedupSimilarityThreshold,
|
|
76
|
+
},
|
|
77
|
+
capture: {
|
|
78
|
+
evidenceWrapperTag: cfg.capture?.evidenceWrapperTag ?? types_1.DEFAULTS.evidenceWrapperTag,
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
function buildContext(stateDir, workspaceDir, rawConfig, log) {
|
|
83
|
+
const defaultLog = {
|
|
84
|
+
debug: (...args) => console.debug("[memos-local]", ...args),
|
|
85
|
+
info: (...args) => console.info("[memos-local]", ...args),
|
|
86
|
+
warn: (...args) => console.warn("[memos-local]", ...args),
|
|
87
|
+
error: (...args) => console.error("[memos-local]", ...args),
|
|
88
|
+
};
|
|
89
|
+
return {
|
|
90
|
+
stateDir,
|
|
91
|
+
workspaceDir,
|
|
92
|
+
config: resolveConfig(rawConfig, stateDir),
|
|
93
|
+
log: log ?? defaultLog,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBA,sCAuBC;AAED,oCAmBC;AAlED,2CAA6B;AAC7B,mCAA2F;AAE3F,MAAM,MAAM,GAAG,2BAA2B,CAAC;AAE3C,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,cAAc,CAAI,GAAM;IAC/B,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,cAAc,CAAC,GAAG,CAAiB,CAAC;IACxE,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;IACvE,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,GAAQ,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAgB,aAAa,CAAC,GAA0C,EAAE,QAAgB;IACxF,MAAM,GAAG,GAAG,cAAc,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IACtC,OAAO;QACL,GAAG,GAAG;QACN,OAAO,EAAE;YACP,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,CAAC;SAC9E;QACD,MAAM,EAAE;YACN,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,iBAAiB,IAAI,gBAAQ,CAAC,iBAAiB;YAC9E,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,IAAI,gBAAQ,CAAC,aAAa;YAClE,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,IAAI,gBAAQ,CAAC,eAAe;YACxE,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,IAAI,gBAAQ,CAAC,aAAa;YAClE,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,IAAI,gBAAQ,CAAC,IAAI;YACvC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,IAAI,gBAAQ,CAAC,SAAS;YACtD,mBAAmB,EAAE,GAAG,CAAC,MAAM,EAAE,mBAAmB,IAAI,gBAAQ,CAAC,mBAAmB;SACrF;QACD,KAAK,EAAE;YACL,mBAAmB,EAAE,GAAG,CAAC,KAAK,EAAE,mBAAmB,IAAI,gBAAQ,CAAC,wBAAwB;SACzF;QACD,OAAO,EAAE;YACP,kBAAkB,EAAE,GAAG,CAAC,OAAO,EAAE,kBAAkB,IAAI,gBAAQ,CAAC,kBAAkB;SACnF;KACF,CAAC;AACJ,CAAC;AAED,SAAgB,YAAY,CAC1B,QAAgB,EAChB,YAAoB,EACpB,SAAgD,EAChD,GAAY;IAEZ,MAAM,UAAU,GAAW;QACzB,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC;QAC3D,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC;QACzD,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC;QACzD,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC;KAC5D,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,YAAY;QACZ,MAAM,EAAE,aAAa,CAAC,SAAS,EAAE,QAAQ,CAAC;QAC1C,GAAG,EAAE,GAAG,IAAI,UAAU;KACvB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { EmbeddingConfig, Logger } from "../types";
|
|
2
|
+
export declare class Embedder {
|
|
3
|
+
private cfg;
|
|
4
|
+
private log;
|
|
5
|
+
constructor(cfg: EmbeddingConfig | undefined, log: Logger);
|
|
6
|
+
get provider(): string;
|
|
7
|
+
get dimensions(): number;
|
|
8
|
+
embed(texts: string[]): Promise<number[][]>;
|
|
9
|
+
embedQuery(text: string): Promise<number[]>;
|
|
10
|
+
private embedBatch;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAQxD,qBAAa,QAAQ;IAEjB,OAAO,CAAC,GAAG;IACX,OAAO,CAAC,GAAG;gBADH,GAAG,EAAE,eAAe,GAAG,SAAS,EAChC,GAAG,EAAE,MAAM;IAGrB,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,UAAU,IAAI,MAAM,CAGvB;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAa3C,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAQnC,UAAU;CA+BzB"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Embedder = void 0;
|
|
4
|
+
const openai_1 = require("./providers/openai");
|
|
5
|
+
const gemini_1 = require("./providers/gemini");
|
|
6
|
+
const cohere_1 = require("./providers/cohere");
|
|
7
|
+
const voyage_1 = require("./providers/voyage");
|
|
8
|
+
const mistral_1 = require("./providers/mistral");
|
|
9
|
+
const local_1 = require("./local");
|
|
10
|
+
class Embedder {
|
|
11
|
+
cfg;
|
|
12
|
+
log;
|
|
13
|
+
constructor(cfg, log) {
|
|
14
|
+
this.cfg = cfg;
|
|
15
|
+
this.log = log;
|
|
16
|
+
}
|
|
17
|
+
get provider() {
|
|
18
|
+
return this.cfg?.provider ?? "local";
|
|
19
|
+
}
|
|
20
|
+
get dimensions() {
|
|
21
|
+
if (this.provider === "local")
|
|
22
|
+
return 384;
|
|
23
|
+
return this.cfg?.dimensions ?? 1536;
|
|
24
|
+
}
|
|
25
|
+
async embed(texts) {
|
|
26
|
+
const batchSize = this.cfg?.batchSize ?? 32;
|
|
27
|
+
const results = [];
|
|
28
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
29
|
+
const batch = texts.slice(i, i + batchSize);
|
|
30
|
+
const vecs = await this.embedBatch(batch);
|
|
31
|
+
results.push(...vecs);
|
|
32
|
+
}
|
|
33
|
+
return results;
|
|
34
|
+
}
|
|
35
|
+
async embedQuery(text) {
|
|
36
|
+
if (this.provider === "cohere" && this.cfg) {
|
|
37
|
+
return (0, cohere_1.embedCohereQuery)(text, this.cfg, this.log);
|
|
38
|
+
}
|
|
39
|
+
const vecs = await this.embedBatch([text]);
|
|
40
|
+
return vecs[0];
|
|
41
|
+
}
|
|
42
|
+
async embedBatch(texts) {
|
|
43
|
+
const provider = this.provider;
|
|
44
|
+
const cfg = this.cfg;
|
|
45
|
+
try {
|
|
46
|
+
switch (provider) {
|
|
47
|
+
case "openai":
|
|
48
|
+
case "openai_compatible":
|
|
49
|
+
return await (0, openai_1.embedOpenAI)(texts, cfg, this.log);
|
|
50
|
+
case "gemini":
|
|
51
|
+
return await (0, gemini_1.embedGemini)(texts, cfg, this.log);
|
|
52
|
+
case "azure_openai":
|
|
53
|
+
return await (0, openai_1.embedOpenAI)(texts, cfg, this.log);
|
|
54
|
+
case "cohere":
|
|
55
|
+
return await (0, cohere_1.embedCohere)(texts, cfg, this.log);
|
|
56
|
+
case "mistral":
|
|
57
|
+
return await (0, mistral_1.embedMistral)(texts, cfg, this.log);
|
|
58
|
+
case "voyage":
|
|
59
|
+
return await (0, voyage_1.embedVoyage)(texts, cfg, this.log);
|
|
60
|
+
case "local":
|
|
61
|
+
default:
|
|
62
|
+
return await (0, local_1.embedLocal)(texts, this.log);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
if (provider !== "local") {
|
|
67
|
+
this.log.warn(`Embedding provider '${provider}' failed, falling back to local: ${err}`);
|
|
68
|
+
return await (0, local_1.embedLocal)(texts, this.log);
|
|
69
|
+
}
|
|
70
|
+
throw err;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
exports.Embedder = Embedder;
|
|
75
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":";;;AACA,+CAAiD;AACjD,+CAAiD;AACjD,+CAAmE;AACnE,+CAAiD;AACjD,iDAAmD;AACnD,mCAAqC;AAErC,MAAa,QAAQ;IAET;IACA;IAFV,YACU,GAAgC,EAChC,GAAW;QADX,QAAG,GAAH,GAAG,CAA6B;QAChC,QAAG,GAAH,GAAG,CAAQ;IAClB,CAAC;IAEJ,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,GAAG,EAAE,QAAQ,IAAI,OAAO,CAAC;IACvC,CAAC;IAED,IAAI,UAAU;QACZ,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO;YAAE,OAAO,GAAG,CAAC;QAC1C,OAAO,IAAI,CAAC,GAAG,EAAE,UAAU,IAAI,IAAI,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,SAAS,IAAI,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QACxB,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3C,OAAO,IAAA,yBAAgB,EAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAe;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QAErB,IAAI,CAAC;YACH,QAAQ,QAAQ,EAAE,CAAC;gBACjB,KAAK,QAAQ,CAAC;gBACd,KAAK,mBAAmB;oBACtB,OAAO,MAAM,IAAA,oBAAW,EAAC,KAAK,EAAE,GAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClD,KAAK,QAAQ;oBACX,OAAO,MAAM,IAAA,oBAAW,EAAC,KAAK,EAAE,GAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClD,KAAK,cAAc;oBACjB,OAAO,MAAM,IAAA,oBAAW,EAAC,KAAK,EAAE,GAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClD,KAAK,QAAQ;oBACX,OAAO,MAAM,IAAA,oBAAW,EAAC,KAAK,EAAE,GAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClD,KAAK,SAAS;oBACZ,OAAO,MAAM,IAAA,sBAAY,EAAC,KAAK,EAAE,GAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnD,KAAK,QAAQ;oBACX,OAAO,MAAM,IAAA,oBAAW,EAAC,KAAK,EAAE,GAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClD,KAAK,OAAO,CAAC;gBACb;oBACE,OAAO,MAAM,IAAA,kBAAU,EAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,QAAQ,oCAAoC,GAAG,EAAE,CAAC,CAAC;gBACxF,OAAO,MAAM,IAAA,kBAAU,EAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAnED,4BAmEC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/embedding/local.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAwBvC,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAUlF"}
|