obedding 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -163
- package/dist/cli.js +6 -2
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +78 -23
- package/dist/index.js.map +1 -1
- package/dist/lmstudio.d.ts +57 -0
- package/dist/lmstudio.d.ts.map +1 -0
- package/dist/lmstudio.js +109 -0
- package/dist/lmstudio.js.map +1 -0
- package/dist/preprocessor.d.ts +34 -0
- package/dist/preprocessor.d.ts.map +1 -0
- package/dist/preprocessor.js +146 -0
- package/dist/preprocessor.js.map +1 -0
- package/dist/search.d.ts +1 -0
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +31 -7
- package/dist/search.js.map +1 -1
- package/dist/storage.d.ts +1 -1
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +8 -1
- package/dist/storage.js.map +1 -1
- package/docs/API.md +290 -0
- package/docs/ARCHITECTURE.md +229 -0
- package/docs/BACKENDS.md +230 -0
- package/package.json +5 -2
package/README.md
CHANGED
|
@@ -1,223 +1,152 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
Semantic search for your Obsidian vault using local
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
1
|
+
# obedding
|
|
2
|
+
|
|
3
|
+
> Semantic search for your Obsidian vault using local embeddings
|
|
4
|
+
|
|
5
|
+
```text _____________ ___________
|
|
6
|
+
│ │
|
|
7
|
+
▼ ▼
|
|
8
|
+
┌─────────────┐ ┌──────────────┐
|
|
9
|
+
│ Obsidian │ │ obedding │
|
|
10
|
+
│ Vault │─────▶│ │
|
|
11
|
+
│ │ │ ● Index │
|
|
12
|
+
│ *.md files │ │ ● Search │
|
|
13
|
+
│ │ │ ● Stats │
|
|
14
|
+
└─────────────┘ └──────┬───────┘
|
|
15
|
+
│
|
|
16
|
+
┌───────────────┼───────────────┐
|
|
17
|
+
│ │ │
|
|
18
|
+
┌─────▼─────┐ ┌─────▼─────┐ ┌────▼─────┐
|
|
19
|
+
│ LM Studio │ │ Ollama │ │ MLX │
|
|
20
|
+
│ (DEFAULT) │ │ │ │ (NOT REC)│
|
|
21
|
+
│ :1234 │ │ :11434 │ │ :28100 │
|
|
22
|
+
└───────────┘ └───────────┘ └──────────┘
|
|
23
23
|
```
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
```bash
|
|
28
|
-
git clone https://github.com/tuannvm/obedding.git
|
|
29
|
-
cd obedging
|
|
30
|
-
npm install
|
|
31
|
-
npm run build
|
|
32
|
-
npm link
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
## Prerequisites
|
|
36
|
-
|
|
37
|
-
1. **MLX Embedding Server** running at `http://localhost:28100` with model `mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ`
|
|
38
|
-
|
|
39
|
-
2. **Node.js** 18+ installed
|
|
40
|
-
|
|
41
|
-
3. **Obsidian vault** at:
|
|
42
|
-
```
|
|
43
|
-
~/.obsidian/Projects
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
## Usage
|
|
25
|
+
## ✨ Features
|
|
47
26
|
|
|
48
|
-
|
|
27
|
+
- **🔍 Semantic Search** — Find notes by meaning, not just keywords
|
|
28
|
+
- **🏠 Local & Private** — Everything runs on your machine, no external API calls
|
|
29
|
+
- **🔄 Multiple Backends** — LM Studio (default), Ollama, or MLX
|
|
30
|
+
- **⚡ Incremental Indexing** — Only re-index changed files
|
|
31
|
+
- **📝 YAML Support** — Extracts metadata from frontmatter
|
|
49
32
|
|
|
50
|
-
|
|
33
|
+
## 🚀 Quick Start
|
|
51
34
|
|
|
52
|
-
|
|
53
|
-
obedding index --vault ~/.obsidian/Projects
|
|
54
|
-
```
|
|
35
|
+
### 1. Install a Backend
|
|
55
36
|
|
|
56
|
-
|
|
37
|
+
Choose one — LM Studio is recommended:
|
|
57
38
|
|
|
39
|
+
**LM Studio** (GUI, easy):
|
|
58
40
|
```bash
|
|
59
|
-
|
|
41
|
+
# Download from https://lmstudio.ai/
|
|
42
|
+
# Load an embedding model and start the server
|
|
60
43
|
```
|
|
61
44
|
|
|
62
|
-
|
|
63
|
-
|
|
45
|
+
**Ollama** (CLI, simple):
|
|
64
46
|
```bash
|
|
65
|
-
|
|
47
|
+
curl -fsSL https://ollama.ai/install.sh | sh
|
|
48
|
+
ollama pull qwen3-embedding:0.6b
|
|
49
|
+
ollama serve
|
|
66
50
|
```
|
|
67
51
|
|
|
68
|
-
|
|
52
|
+
### 2. Index Your Notes
|
|
69
53
|
|
|
70
54
|
```bash
|
|
71
|
-
obedding
|
|
55
|
+
npx obedding index --vault ~/.obsidian/Projects
|
|
72
56
|
```
|
|
73
57
|
|
|
74
|
-
|
|
58
|
+
### 3. Search
|
|
75
59
|
|
|
76
60
|
```bash
|
|
77
|
-
obedding search "
|
|
61
|
+
npx obedding search "what did we decide about caching?"
|
|
78
62
|
```
|
|
79
63
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
Show statistics:
|
|
64
|
+
## 📖 Usage
|
|
83
65
|
|
|
84
66
|
```bash
|
|
85
|
-
|
|
86
|
-
|
|
67
|
+
# Index notes
|
|
68
|
+
npx obedding index --vault ~/.obsidian/Projects
|
|
87
69
|
|
|
88
|
-
|
|
70
|
+
# Incremental indexing (faster)
|
|
71
|
+
npx obedding index --vault ~/.obsidian/Projects --incremental
|
|
89
72
|
|
|
90
|
-
|
|
91
|
-
obedding
|
|
92
|
-
```
|
|
73
|
+
# Semantic search
|
|
74
|
+
npx obedding search "database optimization"
|
|
93
75
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
### Index Command
|
|
97
|
-
|
|
98
|
-
| Option | Description | Default |
|
|
99
|
-
|--------|-------------|---------|
|
|
100
|
-
| `-v, --vault <path>` | Path to Obsidian vault | `~/.obsidian` |
|
|
101
|
-
| `-s, --storage <path>` | Path to storage file | `~/.claude/data/obsidian-embeddings.json` |
|
|
102
|
-
| `-i, --incremental` | Only index new/modified files | false |
|
|
103
|
-
| `-m, --model <name>` | MLX model to use | `mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ` |
|
|
104
|
-
|
|
105
|
-
### Search Command
|
|
106
|
-
|
|
107
|
-
| Option | Description | Default |
|
|
108
|
-
|--------|-------------|---------|
|
|
109
|
-
| `-k, --top-k <number>` | Number of results | 10 |
|
|
110
|
-
| `-s, --storage <path>` | Path to storage file | `~/.claude/data/obsidian-embeddings.json` |
|
|
111
|
-
| `-m, --model <name>` | MLX model to use | `mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ` |
|
|
112
|
-
| `--min-score <number>` | Minimum similarity (0-1) | 0 |
|
|
113
|
-
| `--json` | Output as JSON | false |
|
|
114
|
-
|
|
115
|
-
## Data Structure
|
|
116
|
-
|
|
117
|
-
Embeddings are stored in `~/.claude/data/obsidian-embeddings.json`:
|
|
118
|
-
|
|
119
|
-
```json
|
|
120
|
-
{
|
|
121
|
-
"version": "1.0",
|
|
122
|
-
"model": "mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ",
|
|
123
|
-
"dimensions": 2048,
|
|
124
|
-
"indexed_at": "2026-03-08T10:00:00Z",
|
|
125
|
-
"notes": [
|
|
126
|
-
{
|
|
127
|
-
"path": "Projects/cli/my-repo/feature-x/note.md",
|
|
128
|
-
"vault_path": "/full/path/to/note.md",
|
|
129
|
-
"embedding": [0.0123, -0.0456, ...],
|
|
130
|
-
"metadata": {
|
|
131
|
-
"type": "cli",
|
|
132
|
-
"repo": "my-repo",
|
|
133
|
-
"context": "feature-x",
|
|
134
|
-
"tags": ["decision", "architecture"],
|
|
135
|
-
"title": "Note Title"
|
|
136
|
-
},
|
|
137
|
-
"excerpt": "First 200 chars...",
|
|
138
|
-
"indexed_at": "2026-03-08T10:00:00Z",
|
|
139
|
-
"hash": "sha256..."
|
|
140
|
-
}
|
|
141
|
-
]
|
|
142
|
-
}
|
|
143
|
-
```
|
|
76
|
+
# Top 5 results
|
|
77
|
+
npx obedding search "API design" --top-k 5
|
|
144
78
|
|
|
145
|
-
|
|
79
|
+
# Filter by relevance
|
|
80
|
+
npx obedding search "redis" --min-score 0.6
|
|
146
81
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
- **Index speed**: ~1-2 notes/sec (depends on hardware)
|
|
150
|
-
- **Search latency**: ~50-100ms for 100 notes
|
|
82
|
+
# JSON output
|
|
83
|
+
npx obedding search "architecture" --json
|
|
151
84
|
|
|
152
|
-
|
|
85
|
+
# Show statistics
|
|
86
|
+
npx obedding stats
|
|
153
87
|
|
|
154
|
-
|
|
88
|
+
# Clear all embeddings
|
|
89
|
+
npx obedding clear --force
|
|
90
|
+
```
|
|
155
91
|
|
|
156
|
-
|
|
157
|
-
- "Find notes about..."
|
|
158
|
-
- "What did we decide about..."
|
|
159
|
-
- "Show me notes related to..."
|
|
92
|
+
## 🎯 Backend Options
|
|
160
93
|
|
|
161
|
-
|
|
94
|
+
| Backend | Server | Model | Dims | Notes |
|
|
95
|
+
|---------|--------|-------|------|-------|
|
|
96
|
+
| **LM Studio** `--backend lmstudio` | `:1234` | `text-embedding-qwen3-embedding-0.6b` | 1024 | ✅ Default, GUI |
|
|
97
|
+
| **Ollama** `--backend ollama` | `:11434` | `qwen3-embedding:0.6b` | 768 | ✅ Stable |
|
|
98
|
+
| **MLX** `--backend mlx` | `:28100` | `Qwen3-Embedding-0.6B-4bit-DWQ` | 2048 | ⚠️ Known issues |
|
|
162
99
|
|
|
163
|
-
|
|
100
|
+
## 📁 YAML Frontmatter
|
|
164
101
|
|
|
165
102
|
```yaml
|
|
166
103
|
---
|
|
167
104
|
type: cli
|
|
168
105
|
repo: my-repo
|
|
169
106
|
context: feature-x
|
|
170
|
-
|
|
171
|
-
tags: [decision, architecture, cache]
|
|
107
|
+
tags: [decision, architecture]
|
|
172
108
|
title: Cache Strategy Decision
|
|
173
109
|
---
|
|
174
110
|
```
|
|
175
111
|
|
|
176
|
-
##
|
|
112
|
+
## 📚 Documentation
|
|
177
113
|
|
|
178
|
-
|
|
114
|
+
- **[Architecture](docs/ARCHITECTURE.md)** — System design and components
|
|
115
|
+
- **[Backends](docs/BACKENDS.md)** — Backend setup and comparison
|
|
116
|
+
- **[API Reference](docs/API.md)** — Complete CLI documentation
|
|
117
|
+
- **[CLAUDE.md](CLAUDE.md)** — Project context for Claude Code
|
|
179
118
|
|
|
180
|
-
|
|
181
|
-
# Check if MLX server is running
|
|
182
|
-
curl http://localhost:28100/
|
|
183
|
-
|
|
184
|
-
# Test embedding endpoint
|
|
185
|
-
curl -X POST http://localhost:28100/v1/embeddings \
|
|
186
|
-
-H "Content-Type: application/json" \
|
|
187
|
-
-d '{"input": ["test"], "model": "mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ"}'
|
|
188
|
-
```
|
|
189
|
-
|
|
190
|
-
### Vault path not found
|
|
119
|
+
## 🛠️ Development
|
|
191
120
|
|
|
192
121
|
```bash
|
|
193
|
-
|
|
194
|
-
|
|
122
|
+
git clone https://github.com/tuannvm/obedding.git
|
|
123
|
+
cd obedding
|
|
124
|
+
npm install
|
|
125
|
+
npm run build
|
|
126
|
+
npm link
|
|
127
|
+
obedding search "test"
|
|
195
128
|
```
|
|
196
129
|
|
|
197
|
-
|
|
130
|
+
## ❓ Troubleshooting
|
|
198
131
|
|
|
132
|
+
**Server not running?**
|
|
199
133
|
```bash
|
|
200
|
-
#
|
|
201
|
-
|
|
134
|
+
# LM Studio
|
|
135
|
+
curl http://localhost:1234/v1/models
|
|
202
136
|
|
|
203
|
-
#
|
|
204
|
-
|
|
137
|
+
# Ollama
|
|
138
|
+
curl http://localhost:11434/api/tags
|
|
205
139
|
```
|
|
206
140
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
- **Variable-length embeddings**: MLX server returns variable-length embeddings based on input content. The client truncates to 2048 dimensions as a workaround.
|
|
210
|
-
|
|
211
|
-
## Development
|
|
212
|
-
|
|
141
|
+
**No results?**
|
|
213
142
|
```bash
|
|
214
|
-
#
|
|
215
|
-
|
|
143
|
+
# Check stats
|
|
144
|
+
npx obedding stats
|
|
216
145
|
|
|
217
|
-
#
|
|
218
|
-
|
|
146
|
+
# Re-index
|
|
147
|
+
npx obedding index --vault ~/.obsidian/Projects
|
|
219
148
|
```
|
|
220
149
|
|
|
221
|
-
## License
|
|
150
|
+
## 📄 License
|
|
222
151
|
|
|
223
152
|
MIT
|
package/dist/cli.js
CHANGED
|
@@ -16,7 +16,8 @@ program
|
|
|
16
16
|
.option('-v, --vault <path>', `Path to Obsidian vault`, DEFAULT_VAULT_PATH)
|
|
17
17
|
.option('-s, --storage <path>', `Path to storage file`, DEFAULT_STORAGE_PATH)
|
|
18
18
|
.option('-i, --incremental', 'Only index new or modified files')
|
|
19
|
-
.option('-
|
|
19
|
+
.option('-b, --backend <type>', 'Embedding backend: lmstudio (default), ollama, or mlx', 'lmstudio')
|
|
20
|
+
.option('-m, --model <name>', 'Model to use (default: qwen3-embedding:0.6b for ollama, text-embedding-qwen3-embedding-0.6b for lmstudio, Qwen3-Embedding-0.6B-4bit-DWQ for mlx)')
|
|
20
21
|
.action(async (options) => {
|
|
21
22
|
try {
|
|
22
23
|
const vaultPath = path.resolve(options.vault);
|
|
@@ -26,6 +27,7 @@ program
|
|
|
26
27
|
vaultPath,
|
|
27
28
|
storagePath,
|
|
28
29
|
incremental: options.incremental,
|
|
30
|
+
backend: options.backend,
|
|
29
31
|
model: options.model
|
|
30
32
|
});
|
|
31
33
|
console.log('\n✓ Indexing complete!');
|
|
@@ -43,7 +45,8 @@ program
|
|
|
43
45
|
.option('-v, --vault <path>', `Path to Obsidian vault (for reference)`, DEFAULT_VAULT_PATH)
|
|
44
46
|
.option('-s, --storage <path>', `Path to storage file`, DEFAULT_STORAGE_PATH)
|
|
45
47
|
.option('-k, --top-k <number>', 'Number of results to return', '10')
|
|
46
|
-
.option('-
|
|
48
|
+
.option('-b, --backend <type>', 'Embedding backend: lmstudio (default), ollama, or mlx', 'lmstudio')
|
|
49
|
+
.option('-m, --model <name>', 'Model to use')
|
|
47
50
|
.option('--min-score <number>', 'Minimum similarity score (0-1)', '0')
|
|
48
51
|
.option('--json', 'Output results as JSON')
|
|
49
52
|
.action(async (query, options) => {
|
|
@@ -65,6 +68,7 @@ program
|
|
|
65
68
|
query,
|
|
66
69
|
storagePath,
|
|
67
70
|
topK,
|
|
71
|
+
backend: options.backend,
|
|
68
72
|
model: options.model,
|
|
69
73
|
minScore
|
|
70
74
|
});
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAE/D,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CACrC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EACtB,gEAAgE,CACjE,CAAC;AACF,MAAM,oBAAoB,GAAG,IAAI,CAAC,OAAO,CACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EACtB,uCAAuC,CACxC,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,iBAAiB,CAAC;KACvB,WAAW,CAAC,yDAAyD,CAAC;KACtE,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,8CAA8C,CAAC;KAC3D,MAAM,CAAC,oBAAoB,EAAE,wBAAwB,EAAE,kBAAkB,CAAC;KAC1E,MAAM,CAAC,sBAAsB,EAAE,sBAAsB,EAAE,oBAAoB,CAAC;KAC5E,MAAM,CAAC,mBAAmB,EAAE,kCAAkC,CAAC;KAC/D,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAE/D,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CACrC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EACtB,gEAAgE,CACjE,CAAC;AACF,MAAM,oBAAoB,GAAG,IAAI,CAAC,OAAO,CACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EACtB,uCAAuC,CACxC,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,iBAAiB,CAAC;KACvB,WAAW,CAAC,yDAAyD,CAAC;KACtE,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,8CAA8C,CAAC;KAC3D,MAAM,CAAC,oBAAoB,EAAE,wBAAwB,EAAE,kBAAkB,CAAC;KAC1E,MAAM,CAAC,sBAAsB,EAAE,sBAAsB,EAAE,oBAAoB,CAAC;KAC5E,MAAM,CAAC,mBAAmB,EAAE,kCAAkC,CAAC;KAC/D,MAAM,CAAC,sBAAsB,EAAE,uDAAuD,EAAE,UAAU,CAAC;KACnG,MAAM,CAAC,oBAAoB,EAAE,kJAAkJ,CAAC;KAChL,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAElD,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAElD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,SAAS;YACT,WAAW;YACX,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,cAAc,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAe,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,wCAAwC,CAAC;KACrD,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;KACnC,MAAM,CAAC,oBAAoB,EAAE,wCAAwC,EAAE,kBAAkB,CAAC;KAC1F,MAAM,CAAC,sBAAsB,EAAE,sBAAsB,EAAE,oBAAoB,CAAC;KAC5E,MAAM,CAAC,sBAAsB,EAAE,6BAA6B,EAAE,IAAI,CAAC;KACnE,MAAM,CAAC,sBAAsB,EAAE,uDAAuD,EAAE,UAAU,CAAC;KACnG,MAAM,CAAC,oBAAoB,EAAE,cAAc,CAAC;KAC5C,MAAM,CAAC,sBAAsB,EAAE,gCAAgC,EAAE,GAAG,CAAC;KACrE,MAAM,CAAC,QAAQ,EAAE,wBAAwB,CAAC;KAC1C,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAC/B,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,2BAA2B;QAC3B,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpD,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAE7C,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,KAAK;YACL,WAAW;YACX,IAAI;YACJ,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ;SACT,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAe,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,yBAAyB,CAAC;KACtC,MAAM,CAAC,sBAAsB,EAAE,sBAAsB,EAAE,oBAAoB,CAAC;KAC5E,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;QAEtD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC;QAChD,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAE3B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QAEjC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,cAAc,WAAW,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAe,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,6BAA6B,CAAC;KAC1C,MAAM,CAAC,sBAAsB,EAAE,sBAAsB,EAAE,oBAAoB,CAAC;KAC5E,MAAM,CAAC,aAAa,EAAE,mBAAmB,CAAC;KAC1C,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAElD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;YAC1C,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;gBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE;gBACnD,EAAE,CAAC,QAAQ,CACT,oDAAoD,WAAW,cAAc,EAC7E,OAAO,CACR,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,KAAK,EAAE,CAAC;YAEX,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,KAAK,EAAE,CAAC;gBACnC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBAC1B,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC;QAChD,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC3B,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEtB,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAe,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IAC/C,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,UAAU,CAAC;IACxC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACrE;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAiN5E"}
|
package/dist/index.js
CHANGED
|
@@ -1,27 +1,56 @@
|
|
|
1
1
|
import { scanObsidianVault, extractMetadata, getNoteHash } from './scanner.js';
|
|
2
|
-
import { generateEmbeddings,
|
|
2
|
+
import { generateEmbeddings as generateOllamaEmbeddings, checkOllamaConnection, getModelInfo as getOllamaModelInfo } from './ollama.js';
|
|
3
|
+
import { generateEmbeddings as generateMLXEmbeddings, checkMLXConnection, getModelInfo as getMLXModelInfo } from './mlx.js';
|
|
4
|
+
import { generateEmbeddings as generateLMStudioEmbeddings, checkLMStudioConnection, getModelInfo as getLMStudioModelInfo } from './lmstudio.js';
|
|
3
5
|
import { StorageManager } from './storage.js';
|
|
4
6
|
import { truncateToTokens, generateExcerpt, formatDuration, formatFileSize } from './utils.js';
|
|
7
|
+
import { preprocessNote, checkEmbeddingVariance } from './preprocessor.js';
|
|
5
8
|
/**
|
|
6
9
|
* Index Obsidian notes and generate embeddings
|
|
7
10
|
*/
|
|
8
11
|
export async function indexNotes(options) {
|
|
9
12
|
const startTime = Date.now();
|
|
10
|
-
const { vaultPath, storagePath, incremental = false,
|
|
11
|
-
//
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
const { vaultPath, storagePath, incremental = false, backend = 'lmstudio', model } = options;
|
|
14
|
+
// Determine model and connection check based on backend
|
|
15
|
+
let effectiveModel = model;
|
|
16
|
+
let isConnected = false;
|
|
17
|
+
let connectionType = '';
|
|
18
|
+
let modelInfo = null;
|
|
19
|
+
if (backend === 'ollama') {
|
|
20
|
+
console.log('Checking Ollama connection...');
|
|
21
|
+
isConnected = await checkOllamaConnection();
|
|
22
|
+
connectionType = 'Ollama';
|
|
23
|
+
effectiveModel = effectiveModel || 'qwen3-embedding:0.6b';
|
|
24
|
+
modelInfo = await getOllamaModelInfo(effectiveModel);
|
|
25
|
+
}
|
|
26
|
+
else if (backend === 'lmstudio') {
|
|
27
|
+
console.log('Checking LM Studio connection...');
|
|
28
|
+
isConnected = await checkLMStudioConnection();
|
|
29
|
+
connectionType = 'LM Studio';
|
|
30
|
+
effectiveModel = effectiveModel || 'text-embedding-qwen3-embedding-0.6b';
|
|
31
|
+
modelInfo = await getLMStudioModelInfo(effectiveModel);
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
console.log('Checking MLX embedding server connection...');
|
|
35
|
+
isConnected = await checkMLXConnection();
|
|
36
|
+
connectionType = 'MLX';
|
|
37
|
+
effectiveModel = effectiveModel || 'mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ';
|
|
38
|
+
modelInfo = await getMLXModelInfo(effectiveModel);
|
|
39
|
+
}
|
|
40
|
+
const serverUrls = {
|
|
41
|
+
ollama: 'http://localhost:11434',
|
|
42
|
+
lmstudio: 'http://localhost:1234',
|
|
43
|
+
mlx: 'http://localhost:28100'
|
|
44
|
+
};
|
|
14
45
|
if (!isConnected) {
|
|
15
|
-
throw new Error(
|
|
46
|
+
throw new Error(`Cannot connect to ${connectionType} embedding server. Make sure it is running at ${serverUrls[backend] || serverUrls.ollama}`);
|
|
16
47
|
}
|
|
17
|
-
console.log(
|
|
18
|
-
// Get model info
|
|
19
|
-
const modelInfo = await getModelInfo(model);
|
|
48
|
+
console.log(`✓ ${connectionType} embedding server is running`);
|
|
20
49
|
if (modelInfo) {
|
|
21
|
-
console.log(`✓ Using model: ${modelInfo.name} (${modelInfo.dimensions} dimensions)`);
|
|
50
|
+
console.log(`✓ Using model: ${modelInfo.name}${modelInfo.dimensions ? ` (${modelInfo.dimensions} dimensions)` : ''}`);
|
|
22
51
|
}
|
|
23
52
|
else {
|
|
24
|
-
console.log(`✓ Using model: ${
|
|
53
|
+
console.log(`✓ Using model: ${effectiveModel}`);
|
|
25
54
|
}
|
|
26
55
|
// Initialize storage
|
|
27
56
|
const storage = new StorageManager(storagePath);
|
|
@@ -48,8 +77,10 @@ export async function indexNotes(options) {
|
|
|
48
77
|
const textsToEmbed = [];
|
|
49
78
|
const noteMapping = [];
|
|
50
79
|
for (const note of notesToIndex) {
|
|
51
|
-
//
|
|
52
|
-
|
|
80
|
+
// Preprocess content to remove structural patterns
|
|
81
|
+
// This helps the embedding model focus on semantic content rather than markdown structure
|
|
82
|
+
const preprocessed = preprocessNote(note);
|
|
83
|
+
const contentForEmbedding = preprocessed.combined;
|
|
53
84
|
// Truncate to avoid token limits
|
|
54
85
|
const truncated = truncateToTokens(contentForEmbedding, 8000);
|
|
55
86
|
textsToEmbed.push(truncated);
|
|
@@ -65,18 +96,42 @@ export async function indexNotes(options) {
|
|
|
65
96
|
skippedNotes: notes.length,
|
|
66
97
|
failedNotes: 0,
|
|
67
98
|
duration: (Date.now() - startTime) / 1000,
|
|
68
|
-
model,
|
|
99
|
+
model: effectiveModel,
|
|
69
100
|
storageSize: stats.totalSize
|
|
70
101
|
};
|
|
71
102
|
}
|
|
72
|
-
// Generate embeddings
|
|
103
|
+
// Generate embeddings using the selected backend
|
|
73
104
|
console.log(`\nGenerating embeddings for ${textsToEmbed.length} notes...`);
|
|
74
|
-
const embeddings =
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
105
|
+
const embeddings = backend === 'ollama'
|
|
106
|
+
? await generateOllamaEmbeddings(textsToEmbed, effectiveModel, (current, total) => {
|
|
107
|
+
const percent = ((current / total) * 100).toFixed(1);
|
|
108
|
+
const note = noteMapping[current - 1]?.note.relativePath || 'unknown';
|
|
109
|
+
console.log(` [${percent}%] ${note}`);
|
|
110
|
+
options.onProgress?.(current, total, note);
|
|
111
|
+
})
|
|
112
|
+
: backend === 'lmstudio'
|
|
113
|
+
? await generateLMStudioEmbeddings(textsToEmbed, effectiveModel, (current, total) => {
|
|
114
|
+
const percent = ((current / total) * 100).toFixed(1);
|
|
115
|
+
const note = noteMapping[current - 1]?.note.relativePath || 'unknown';
|
|
116
|
+
console.log(` [${percent}%] ${note}`);
|
|
117
|
+
options.onProgress?.(current, total, note);
|
|
118
|
+
})
|
|
119
|
+
: await generateMLXEmbeddings(textsToEmbed, effectiveModel, (current, total) => {
|
|
120
|
+
const percent = ((current / total) * 100).toFixed(1);
|
|
121
|
+
const note = noteMapping[current - 1]?.note.relativePath || 'unknown';
|
|
122
|
+
console.log(` [${percent}%] ${note}`);
|
|
123
|
+
options.onProgress?.(current, total, note);
|
|
124
|
+
});
|
|
125
|
+
// Check embedding variance to detect potential issues
|
|
126
|
+
const varianceCheck = checkEmbeddingVariance(embeddings);
|
|
127
|
+
if (varianceCheck.isTooLow) {
|
|
128
|
+
console.warn(`\n⚠️ Warning: Embeddings have low variance (${(varianceCheck.variance * 100).toFixed(1)}%)`);
|
|
129
|
+
console.warn(' This may indicate the embedding model is struggling with the content structure.');
|
|
130
|
+
console.warn(' Consider: (1) Using a different model, or (2) Reviewing note content patterns');
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
console.log(`\n✓ Embedding variance: ${(varianceCheck.variance * 100).toFixed(1)}% (good)`);
|
|
134
|
+
}
|
|
80
135
|
// Store embeddings
|
|
81
136
|
console.log('\nStoring embeddings...');
|
|
82
137
|
let indexedCount = 0;
|
|
@@ -101,7 +156,7 @@ export async function indexNotes(options) {
|
|
|
101
156
|
excerpt,
|
|
102
157
|
indexed_at: new Date().toISOString(),
|
|
103
158
|
hash
|
|
104
|
-
});
|
|
159
|
+
}, effectiveModel, embedding.length);
|
|
105
160
|
indexedCount++;
|
|
106
161
|
}
|
|
107
162
|
catch (error) {
|
|
@@ -129,7 +184,7 @@ export async function indexNotes(options) {
|
|
|
129
184
|
skippedNotes: notes.length - indexedCount,
|
|
130
185
|
failedNotes: failedCount,
|
|
131
186
|
duration,
|
|
132
|
-
model,
|
|
187
|
+
model: effectiveModel,
|
|
133
188
|
storageSize: stats.totalSize
|
|
134
189
|
};
|
|
135
190
|
}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,WAAW,EAAY,MAAM,cAAc,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,WAAW,EAAY,MAAM,cAAc,CAAC;AACzF,OAAO,EAAE,kBAAkB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,YAAY,IAAI,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACxI,OAAO,EAAE,kBAAkB,IAAI,qBAAqB,EAAE,kBAAkB,EAAE,YAAY,IAAI,eAAe,EAAE,MAAM,UAAU,CAAC;AAC5H,OAAO,EAAE,kBAAkB,IAAI,0BAA0B,EAAE,uBAAuB,EAAE,YAAY,IAAI,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAChJ,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC/F,OAAO,EAAE,cAAc,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAsB3E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAqB;IACpD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,GAAG,KAAK,EAAE,OAAO,GAAG,UAAU,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IAE7F,wDAAwD;IACxD,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,IAAI,SAAS,GAAiD,IAAI,CAAC;IAEnE,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,WAAW,GAAG,MAAM,qBAAqB,EAAE,CAAC;QAC5C,cAAc,GAAG,QAAQ,CAAC;QAC1B,cAAc,GAAG,cAAc,IAAI,sBAAsB,CAAC;QAC1D,SAAS,GAAG,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;IACvD,CAAC;SAAM,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,WAAW,GAAG,MAAM,uBAAuB,EAAE,CAAC;QAC9C,cAAc,GAAG,WAAW,CAAC;QAC7B,cAAc,GAAG,cAAc,IAAI,qCAAqC,CAAC;QACzE,SAAS,GAAG,MAAM,oBAAoB,CAAC,cAAc,CAAC,CAAC;IACzD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,WAAW,GAAG,MAAM,kBAAkB,EAAE,CAAC;QACzC,cAAc,GAAG,KAAK,CAAC;QACvB,cAAc,GAAG,cAAc,IAAI,6CAA6C,CAAC;QACjF,SAAS,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,UAAU,GAA2B;QACzC,MAAM,EAAE,wBAAwB;QAChC,QAAQ,EAAE,uBAAuB;QACjC,GAAG,EAAE,wBAAwB;KAC9B,CAAC;IAEF,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,qBAAqB,cAAc,iDAAiD,UAAU,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAClJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,cAAc,8BAA8B,CAAC,CAAC;IAE/D,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,kBAAkB,SAAS,CAAC,IAAI,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,UAAU,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,kBAAkB,cAAc,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,qBAAqB;IACrB,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAE3B,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,qBAAqB,SAAS,EAAE,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,iBAAiB,CAAC,CAAC;IAEpD,wCAAwC;IACxC,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAElE,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YACjC,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;YAExE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3D,CAAC;YAED,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,cAAc,YAAY,CAAC,MAAM,qBAAqB,CAAC,CAAC;IACtE,CAAC;IAED,8BAA8B;IAC9B,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,WAAW,GAA4C,EAAE,CAAC;IAEhE,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,mDAAmD;QACnD,0FAA0F;QAC1F,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,mBAAmB,GAAG,YAAY,CAAC,QAAQ,CAAC;QAElD,iCAAiC;QACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;QAC9D,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE7B,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/B,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;QAEpD,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QACjC,OAAO;YACL,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,KAAK,CAAC,MAAM;YAC1B,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI;YACzC,KAAK,EAAE,cAAc;YACrB,WAAW,EAAE,KAAK,CAAC,SAAS;SAC7B,CAAC;IACJ,CAAC;IAED,iDAAiD;IACjD,OAAO,CAAC,GAAG,CAAC,+BAA+B,YAAY,CAAC,MAAM,WAAW,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,OAAO,KAAK,QAAQ;QACrC,CAAC,CAAC,MAAM,wBAAwB,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAC9E,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,MAAM,OAAO,MAAM,IAAI,EAAE,CAAC,CAAC;YACvC,OAAO,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QAC7C,CAAC,CAAC;QACJ,CAAC,CAAC,OAAO,KAAK,UAAU;YACxB,CAAC,CAAC,MAAM,0BAA0B,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;gBAChF,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC;gBACtE,OAAO,CAAC,GAAG,CAAC,MAAM,OAAO,MAAM,IAAI,EAAE,CAAC,CAAC;gBACvC,OAAO,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YAC7C,CAAC,CAAC;YACJ,CAAC,CAAC,MAAM,qBAAqB,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;gBAC3E,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC;gBACtE,OAAO,CAAC,GAAG,CAAC,MAAM,OAAO,MAAM,IAAI,EAAE,CAAC,CAAC;gBACvC,OAAO,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC;IAEP,sDAAsD;IACtD,MAAM,aAAa,GAAG,sBAAsB,CAAC,UAAU,CAAC,CAAC;IACzD,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,aAAa,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5G,OAAO,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;QACnG,OAAO,CAAC,IAAI,CAAC,kFAAkF,CAAC,CAAC;IACnG,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,aAAa,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAC9F,CAAC;IAED,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAEvC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE9C,qBAAqB;YACrB,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,kCAAkC,IAAI,CAAC,YAAY,YAAY,CAAC,CAAC;gBAC9E,WAAW,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YAED,MAAM,OAAO,CAAC,UAAU,CAAC;gBACvB,IAAI,EAAE,IAAI,CAAC,YAAY;gBACvB,UAAU,EAAE,IAAI,CAAC,QAAQ;gBACzB,SAAS;gBACT,QAAQ;gBACR,OAAO;gBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,IAAI;aACL,EAAE,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YAErC,YAAY,EAAE,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,GAAG,CAAC,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9E,WAAW,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,eAAe;IACf,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IAErB,eAAe;IACf,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IAErB,eAAe;IACf,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IAErB,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAEjC,OAAO,CAAC,GAAG,CAAC,eAAe,YAAY,aAAa,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAChF,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,CAAC,MAAM,GAAG,YAAY,QAAQ,CAAC,CAAC;IAC/D,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,aAAa,WAAW,QAAQ,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,mBAAmB,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAElE,OAAO;QACL,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,YAAY,EAAE,YAAY;QAC1B,YAAY,EAAE,KAAK,CAAC,MAAM,GAAG,YAAY;QACzC,WAAW,EAAE,WAAW;QACxB,QAAQ;QACR,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,KAAK,CAAC,SAAS;KAC7B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LM Studio API client for generating embeddings using GGUF models
|
|
3
|
+
*
|
|
4
|
+
* LM Studio provides an OpenAI-compatible API for local models.
|
|
5
|
+
* Default port: 1234
|
|
6
|
+
* Default model: Qwen3-Embedding-0.6B-GGUF (from Hugging Face)
|
|
7
|
+
*/
|
|
8
|
+
export interface EmbeddingResponse {
|
|
9
|
+
object: string;
|
|
10
|
+
data: Array<{
|
|
11
|
+
object: string;
|
|
12
|
+
index: number;
|
|
13
|
+
embedding: number[];
|
|
14
|
+
}>;
|
|
15
|
+
model: string;
|
|
16
|
+
usage: {
|
|
17
|
+
prompt_tokens: number;
|
|
18
|
+
total_tokens: number;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export interface ModelInfo {
|
|
22
|
+
id: string;
|
|
23
|
+
object: string;
|
|
24
|
+
created: number;
|
|
25
|
+
owned_by: string;
|
|
26
|
+
}
|
|
27
|
+
export interface ModelsResponse {
|
|
28
|
+
object: string;
|
|
29
|
+
data: ModelInfo[];
|
|
30
|
+
}
|
|
31
|
+
export interface LMStudioError {
|
|
32
|
+
error: {
|
|
33
|
+
message: string;
|
|
34
|
+
type: string;
|
|
35
|
+
code?: string;
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Check if LM Studio is running
|
|
40
|
+
*/
|
|
41
|
+
export declare function checkLMStudioConnection(): Promise<boolean>;
|
|
42
|
+
/**
|
|
43
|
+
* Generate embedding for text using LM Studio
|
|
44
|
+
*/
|
|
45
|
+
export declare function generateEmbedding(text: string, model?: string): Promise<number[]>;
|
|
46
|
+
/**
|
|
47
|
+
* Batch generate embeddings for multiple texts
|
|
48
|
+
*/
|
|
49
|
+
export declare function generateEmbeddings(texts: string[], model?: string, onProgress?: (current: number, total: number) => void): Promise<number[][]>;
|
|
50
|
+
/**
|
|
51
|
+
* Get model information
|
|
52
|
+
*/
|
|
53
|
+
export declare function getModelInfo(model?: string): Promise<{
|
|
54
|
+
name: string;
|
|
55
|
+
dimensions?: number;
|
|
56
|
+
} | null>;
|
|
57
|
+
//# sourceMappingURL=lmstudio.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lmstudio.d.ts","sourceRoot":"","sources":["../src/lmstudio.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,KAAK,CAAC;QACV,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC,CAAC;IACH,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE;QACL,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,OAAO,CAAC,CAUhE;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,KAAK,GAAE,MAAsB,GAC5B,OAAO,CAAC,MAAM,EAAE,CAAC,CA0CnB;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,MAAM,EAAE,EACf,KAAK,GAAE,MAAsB,EAC7B,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACpD,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAarB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,KAAK,GAAE,MAAsB,GAAG,OAAO,CAAC;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,IAAI,CAAC,CA4BR"}
|