@mrplex/embedder 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -11
- package/embedder.mjs +6 -11
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# @mrplex/embedder
|
|
2
2
|
|
|
3
3
|
A ready-to-use local embedding provider for [mrplex](https://github.com/usergenic/mrplex)'s
|
|
4
|
-
`--
|
|
4
|
+
`--embedder` hook. It runs `bge-small-en-v1.5` (384-dim) on CPU via ONNX — no GPU, no
|
|
5
5
|
separate service. mrplex spawns it once and keeps it resident, so the model loads a single
|
|
6
6
|
time and each batch is one JSON line in / one JSON line out.
|
|
7
7
|
|
|
@@ -30,34 +30,34 @@ cd packages/embedder && npm install
|
|
|
30
30
|
|
|
31
31
|
## Use with mrplex
|
|
32
32
|
|
|
33
|
-
After a global install
|
|
33
|
+
After a global install:
|
|
34
34
|
|
|
35
35
|
```bash
|
|
36
36
|
# Serve with the hook — every write auto-embeds.
|
|
37
|
-
mrplex serve --unsafe --
|
|
37
|
+
mrplex serve --unsafe --embedder mrplex-embedder
|
|
38
38
|
|
|
39
39
|
# One-off backfill of an existing repo.
|
|
40
|
-
mrplex embed backfill -r notes --
|
|
40
|
+
mrplex embed backfill -r notes --embedder mrplex-embedder
|
|
41
41
|
|
|
42
|
-
# Or make it ambient
|
|
43
|
-
export
|
|
42
|
+
# Or make it ambient for every mrplex command:
|
|
43
|
+
export MRPLEX_EMBEDDER=mrplex-embedder
|
|
44
|
+
# mrplex config set-embedder mrplex-embedder # same thing, persisted
|
|
44
45
|
```
|
|
45
46
|
|
|
46
47
|
**Monorepo / local checkout** (no global install):
|
|
47
48
|
|
|
48
49
|
```bash
|
|
49
|
-
mrplex serve --unsafe
|
|
50
|
-
--embed-cmd "node packages/embedder/embedder.mjs --stdio"
|
|
50
|
+
mrplex serve --unsafe --embedder "node packages/embedder/embedder.mjs"
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
### Different models
|
|
54
54
|
|
|
55
55
|
```bash
|
|
56
56
|
# Stronger, slower (768-dim).
|
|
57
|
-
export
|
|
57
|
+
export MRPLEX_EMBEDDER="mrplex-embedder --model fast-bge-base-en-v1.5"
|
|
58
58
|
|
|
59
59
|
# Matryoshka truncation (re-embed after changing --dim).
|
|
60
|
-
export
|
|
60
|
+
export MRPLEX_EMBEDDER="mrplex-embedder --dim 256"
|
|
61
61
|
```
|
|
62
62
|
|
|
63
63
|
List supported model keys:
|
|
@@ -68,13 +68,14 @@ mrplex-embedder --list-models
|
|
|
68
68
|
|
|
69
69
|
## Flags
|
|
70
70
|
|
|
71
|
-
- `--stdio` — required transport (one JSON line per batch over stdin/stdout).
|
|
72
71
|
- `--model KEY` — any `fastembed` model key. Default `fast-bge-small-en-v1.5`.
|
|
73
72
|
Others include `fast-all-MiniLM-L6-v2`, `fast-bge-base-en-v1.5`,
|
|
74
73
|
`fast-multilingual-e5-large`. Run `--list-models` for the full set.
|
|
75
74
|
- `--dim N` — truncate + re-normalize each vector to `N` dims (for Matryoshka
|
|
76
75
|
models). The truncation is encoded into the reported model string
|
|
77
76
|
(`<key>@<N>`) so mrplex treats a dim change as a model change and re-embeds.
|
|
77
|
+
- `--stdio` — optional no-op; stdio is the default transport (one JSON line
|
|
78
|
+
per batch over stdin/stdout).
|
|
78
79
|
- `--list-models` — print supported `--model` keys and exit.
|
|
79
80
|
- `--help`, `--version` — usage and version.
|
|
80
81
|
|
package/embedder.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* Local embedding provider for mrplex's `--
|
|
3
|
+
* Local embedding provider for mrplex's `--embedder` hook.
|
|
4
4
|
*
|
|
5
5
|
* Speaks the subprocess contract (design §5.3): one JSON line in on stdin
|
|
6
6
|
* `{ chunks: string[] }`, one JSON line out on stdout
|
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
* ONCE and reuses it for every batch (src/embed/cmd-hook.ts), so the model
|
|
9
9
|
* loads a single time and stays resident — no per-call cold start.
|
|
10
10
|
*
|
|
11
|
-
* mrplex serve --unsafe --
|
|
12
|
-
* mrplex embed backfill -r notes --
|
|
11
|
+
* mrplex serve --unsafe --embedder mrplex-embedder
|
|
12
|
+
* mrplex embed backfill -r notes --embedder mrplex-embedder
|
|
13
13
|
*
|
|
14
14
|
* Model: bge-small-en-v1.5 (384-dim) by default — strong retrieval quality for
|
|
15
15
|
* its size, and small vectors keep mrplex's brute-force cosine scan fast. Runs
|
|
@@ -35,13 +35,13 @@ const has = (name) => argv.includes(name);
|
|
|
35
35
|
|
|
36
36
|
function printHelp() {
|
|
37
37
|
console.error(`mrplex-embedder ${VERSION}
|
|
38
|
-
Local CPU embedding provider for mrplex's --
|
|
38
|
+
Local CPU embedding provider for mrplex's --embedder hook.
|
|
39
39
|
|
|
40
40
|
Usage:
|
|
41
|
-
mrplex-embedder
|
|
41
|
+
mrplex-embedder [--model KEY] [--dim N]
|
|
42
42
|
|
|
43
43
|
Options:
|
|
44
|
-
--stdio
|
|
44
|
+
--stdio optional; stdio is the default (one JSON line per batch over stdin/stdout)
|
|
45
45
|
--model KEY fastembed model key (default: fast-bge-small-en-v1.5)
|
|
46
46
|
--dim N truncate + re-normalize to N dims (Matryoshka models only)
|
|
47
47
|
--list-models print supported --model keys and exit
|
|
@@ -89,11 +89,6 @@ if (has("--list-models")) {
|
|
|
89
89
|
process.exit(0);
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
if (!has("--stdio")) {
|
|
93
|
-
printHelp();
|
|
94
|
-
process.exit(2);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
92
|
// fastembed keys its models by an enum whose values are the string keys above.
|
|
98
93
|
const model = Object.values(EmbeddingModel).find((v) => v === modelKey);
|
|
99
94
|
if (!model) {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mrplex/embedder",
|
|
3
|
-
"version": "0.1.
|
|
4
|
-
"description": "Local CPU embedding provider for mrplex's --
|
|
3
|
+
"version": "0.1.2",
|
|
4
|
+
"description": "Local CPU embedding provider for mrplex's --embedder hook (bge-small-en-v1.5 via fastembed). Isolated from mrplex core so fastembed/tar stay out of the core dependency graph.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"author": "Brendan Baldwin <brendan@usergenic.com>",
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"README.md"
|
|
34
34
|
],
|
|
35
35
|
"scripts": {
|
|
36
|
-
"start": "node embedder.mjs
|
|
36
|
+
"start": "node embedder.mjs",
|
|
37
37
|
"test": "node --test test/*.test.mjs"
|
|
38
38
|
},
|
|
39
39
|
"engines": {
|