@factiii/runner 0.24.0 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +174 -0
- package/README.md +87 -12
- package/dist/cli.js +83814 -40252
- package/dist/vault-client.js +876 -0
- package/dist/vault-db.js +858 -0
- package/embedder/app.py +331 -0
- package/embedder/clip_tokenizer.py +137 -0
- package/embedder/convert_clip.py +119 -0
- package/embedder/convert_onnx.py +129 -0
- package/embedder/convert_text.py +148 -0
- package/embedder/onnx_backend.py +48 -0
- package/embedder/overrides-onnx.txt +3 -0
- package/embedder/requirements-onnx.txt +22 -0
- package/embedder/requirements.txt +20 -0
- package/embedder/serve.py +241 -0
- package/embedder/text_tokenizer.py +34 -0
- package/native/simhid.m +7 -0
- package/native/simstream.m +7 -0
- package/package.json +28 -14
- package/plugin/factiii/bin/factiii-board-mcp +206 -0
- package/plugin/factiii/bin/factiii-secrets +182 -18
- package/plugin/factiii/bin/report-state.sh +19 -1
- package/plugin/factiii/bin/session-context.sh +76 -0
- package/plugin/factiii/hooks/hooks.json +22 -0
- package/plugin/factiii/skills/board-sessions/SKILL.md +74 -0
- package/plugin/factiii/skills/factiii-secrets/SKILL.md +21 -3
- package/plugin/factiii/skills/runner-onboarding/SKILL.md +67 -0
- package/plugin/factiii/skills/scratchpad/SKILL.md +1 -0
- package/plugin/factiii/skills/share-a-port/SKILL.md +11 -6
- package/index/image-package.json +0 -8
- package/index/index.js +0 -105341
- package/native/screenhost.m +0 -500
- package/plugin/factiii/skills/factiii-workspace/SKILL.md +0 -69
- package/vendor/frpc-darwin-amd64 +0 -0
- package/vendor/frpc-darwin-arm64 +0 -0
- package/vendor/frpc-linux-amd64 +0 -0
- package/vendor/frpc-linux-arm64 +0 -0
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Export the Vault's two search models to ONNX, for the runner on Linux.
|
|
2
|
+
|
|
3
|
+
The Linux twin of the cloud's convert_text.py / convert_clip.py: the same
|
|
4
|
+
torch wrappers (imported from them), the same checkpoints and the same parity
|
|
5
|
+
bar (cosine with torch above 0.99, printed as theirs are), so a vector made
|
|
6
|
+
on Linux is in the same space as the Mac's and the cloud's. Kept in fp32, so
|
|
7
|
+
parity is near 1; fp16 or int8 would move vectors off the cloud's.
|
|
8
|
+
|
|
9
|
+
python convert_onnx.py text # text.onnx + text_tokenizer.json
|
|
10
|
+
python convert_onnx.py clip # clip_image.onnx + clip_text.onnx + clip_vocab.txt.gz
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import shutil
|
|
15
|
+
import sys
|
|
16
|
+
import types
|
|
17
|
+
|
|
18
|
+
from onnx_backend import OnnxModel, stub_coremltools
|
|
19
|
+
|
|
20
|
+
stub_coremltools()
|
|
21
|
+
|
|
22
|
+
import torch # noqa: E402
|
|
23
|
+
import torch.nn.functional as F # noqa: E402
|
|
24
|
+
|
|
25
|
+
import convert_clip # noqa: E402
|
|
26
|
+
import convert_text # noqa: E402
|
|
27
|
+
|
|
28
|
+
MIN_PARITY = 0.99
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def export(wrapper, example: tuple, names: list[str], path: str):
|
|
32
|
+
with torch.no_grad():
|
|
33
|
+
# dynamo=False: the TorchScript tracer, as convert_text.py uses for
|
|
34
|
+
# bge (torch.export trips on BERT's new_ones).
|
|
35
|
+
torch.onnx.export(
|
|
36
|
+
wrapper,
|
|
37
|
+
example,
|
|
38
|
+
path,
|
|
39
|
+
input_names=names,
|
|
40
|
+
output_names=["embedding"],
|
|
41
|
+
opset_version=17,
|
|
42
|
+
dynamo=False,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def parity(path: str, feeds: dict, ref: torch.Tensor, name: str) -> bool:
|
|
47
|
+
vec = torch.tensor(OnnxModel(path).predict(feeds)["embedding"]).flatten().float()
|
|
48
|
+
cos = F.cosine_similarity(vec, ref.flatten().float(), dim=0).item()
|
|
49
|
+
print(f"{name}: cosine(onnx, torch) = {cos:.5f} (dim={vec.numel()})", flush=True)
|
|
50
|
+
return cos > MIN_PARITY
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def text() -> bool:
|
|
54
|
+
from transformers import AutoModel, AutoTokenizer
|
|
55
|
+
|
|
56
|
+
tok = AutoTokenizer.from_pretrained(convert_text.MODEL)
|
|
57
|
+
model = AutoModel.from_pretrained(convert_text.MODEL, attn_implementation="eager").float().eval()
|
|
58
|
+
|
|
59
|
+
# convert_text.py's padding mask (-1e4), so the graph is the Mac's.
|
|
60
|
+
def _safe_ext_mask(self, attention_mask, input_shape, device=None, dtype=None):
|
|
61
|
+
if dtype is None:
|
|
62
|
+
dtype = self.dtype
|
|
63
|
+
ext = attention_mask[:, None, None, :].to(dtype=dtype)
|
|
64
|
+
return (1.0 - ext) * -1.0e4
|
|
65
|
+
|
|
66
|
+
model.get_extended_attention_mask = types.MethodType(_safe_ext_mask, model)
|
|
67
|
+
wrapper = convert_text.TextEncoder(model, convert_text.MAX_LEN).eval()
|
|
68
|
+
|
|
69
|
+
def enc(sentence: str):
|
|
70
|
+
e = tok(sentence, padding="max_length", truncation=True, max_length=convert_text.MAX_LEN, return_tensors="pt")
|
|
71
|
+
return e["input_ids"], e["attention_mask"]
|
|
72
|
+
|
|
73
|
+
print("exporting text encoder...", flush=True)
|
|
74
|
+
export(wrapper, enc("a short example sentence"), ["input_ids", "attention_mask"], "text.onnx")
|
|
75
|
+
ok = True
|
|
76
|
+
# convert_text.py's sentence, then a longer one with less padding.
|
|
77
|
+
for sentence in ("a short example sentence", "Quarterly report: revenue rose 12% on strong subscription sales. " * 6):
|
|
78
|
+
ids, mask = enc(sentence)
|
|
79
|
+
with torch.no_grad():
|
|
80
|
+
ref = wrapper(ids, mask)
|
|
81
|
+
feeds = {"input_ids": ids.numpy(), "attention_mask": mask.numpy()}
|
|
82
|
+
ok = parity("text.onnx", feeds, ref, "text") and ok
|
|
83
|
+
if ref.numel() != convert_text.DIM:
|
|
84
|
+
ok = False
|
|
85
|
+
if ok:
|
|
86
|
+
tok.backend_tokenizer.save("text_tokenizer.json")
|
|
87
|
+
return ok
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def clip() -> bool:
|
|
91
|
+
import open_clip
|
|
92
|
+
from open_clip import tokenizer as _tk
|
|
93
|
+
|
|
94
|
+
model, _, _ = open_clip.create_model_and_transforms(convert_clip.MODEL, pretrained=convert_clip.PRETRAINED)
|
|
95
|
+
model.eval()
|
|
96
|
+
tok = open_clip.get_tokenizer(convert_clip.MODEL)
|
|
97
|
+
shutil.copy(_tk.default_bpe(), "clip_vocab.txt.gz")
|
|
98
|
+
|
|
99
|
+
torch.manual_seed(0)
|
|
100
|
+
img_ex = torch.randn(1, 3, 224, 224)
|
|
101
|
+
txt_ex = tok(["a photo of a dog"]) # int64 [1, 77]
|
|
102
|
+
with torch.no_grad():
|
|
103
|
+
img_ref = model.encode_image(img_ex)
|
|
104
|
+
txt_ref = model.encode_text(txt_ex)
|
|
105
|
+
|
|
106
|
+
print("exporting CLIP image encoder...", flush=True)
|
|
107
|
+
export(convert_clip.ImageEncoder(model).eval(), (img_ex,), ["image"], "clip_image.onnx")
|
|
108
|
+
ok_i = parity("clip_image.onnx", {"image": img_ex.numpy()}, img_ref, "image")
|
|
109
|
+
print("exporting CLIP text encoder...", flush=True)
|
|
110
|
+
export(convert_clip.TextEncoder(model).eval(), (txt_ex,), ["tokens"], "clip_text.onnx")
|
|
111
|
+
ok_t = parity("clip_text.onnx", {"tokens": txt_ex.numpy()}, txt_ref, "text")
|
|
112
|
+
return ok_i and ok_t
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def main() -> int:
|
|
116
|
+
which = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
117
|
+
if which not in ("text", "clip"):
|
|
118
|
+
print("usage: convert_onnx.py text|clip")
|
|
119
|
+
return 2
|
|
120
|
+
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
|
121
|
+
if not (text() if which == "text" else clip()):
|
|
122
|
+
print("PARITY FAILED")
|
|
123
|
+
return 1
|
|
124
|
+
print("done")
|
|
125
|
+
return 0
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
if __name__ == "__main__":
|
|
129
|
+
sys.exit(main())
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Convert a text-embedding model (bge-base-en-v1.5, 768-d) to Core ML (fp16) so
|
|
2
|
+
the text↔text semantic search path runs torch-free on the ANE, like CLIP/DINOv2.
|
|
3
|
+
|
|
4
|
+
bge-base-en-v1.5 is a BERT encoder + CLS pooling — stronger discrimination than
|
|
5
|
+
gte-small (less anisotropic, better retrieval). We use it symmetrically (no query
|
|
6
|
+
prefix); the pooling + L2 normalize are baked into the graph, so the runtime just
|
|
7
|
+
feeds tokens (text_tokenizer.py) and reads a ready-to-compare unit vector.
|
|
8
|
+
|
|
9
|
+
Verifies parity vs torch and precompiles/warms text.mlmodelc. Also writes
|
|
10
|
+
text_tokenizer.json for the torch-free runtime tokenizer.
|
|
11
|
+
|
|
12
|
+
./.venv/bin/python convert_text.py
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import os
|
|
16
|
+
import shutil
|
|
17
|
+
import sys
|
|
18
|
+
import types
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import torch
|
|
22
|
+
import torch.nn.functional as F
|
|
23
|
+
|
|
24
|
+
import coremltools as ct
|
|
25
|
+
from coremltools.models import CompiledMLModel
|
|
26
|
+
from coremltools.models.utils import compile_model
|
|
27
|
+
from transformers import AutoModel, AutoTokenizer
|
|
28
|
+
|
|
29
|
+
# transformers pinned < 4.48 (see setup.sh): newer BERT attention emits ops the
|
|
30
|
+
# Core ML TorchScript frontend can't convert.
|
|
31
|
+
|
|
32
|
+
MODEL = "BAAI/bge-base-en-v1.5"
|
|
33
|
+
DIM = 768
|
|
34
|
+
# Fixed sequence length — must match MAX_LEN in text_tokenizer.py. Core ML runs a
|
|
35
|
+
# fixed shape (like CLIP's 77); 256 covers bios/descriptions and post snippets.
|
|
36
|
+
MAX_LEN = 256
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class TextEncoder(torch.nn.Module):
|
|
40
|
+
"""BERT encoder + CLS pooling + L2 normalize.
|
|
41
|
+
|
|
42
|
+
position_ids and token_type_ids are passed as fixed-length constant buffers so
|
|
43
|
+
BERT's embeddings never compute them dynamically (those size()/expand paths are
|
|
44
|
+
what emit the un-convertible aten::Int / new_ones ops). Combined with eager
|
|
45
|
+
attention (no SDPA mask-prep), the graph stays Core ML convertible.
|
|
46
|
+
|
|
47
|
+
BGE pools the [CLS] token (position 0), not the mean — using mean here would
|
|
48
|
+
silently produce wrong embeddings.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(self, m, max_len):
|
|
52
|
+
super().__init__()
|
|
53
|
+
self.m = m
|
|
54
|
+
self.register_buffer("position_ids", torch.arange(max_len).unsqueeze(0))
|
|
55
|
+
self.register_buffer(
|
|
56
|
+
"token_type_ids", torch.zeros((1, max_len), dtype=torch.long)
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def forward(self, input_ids, attention_mask):
|
|
60
|
+
hidden = self.m(
|
|
61
|
+
input_ids=input_ids,
|
|
62
|
+
attention_mask=attention_mask,
|
|
63
|
+
token_type_ids=self.token_type_ids,
|
|
64
|
+
position_ids=self.position_ids,
|
|
65
|
+
)[0]
|
|
66
|
+
return F.normalize(hidden[:, 0], p=2, dim=1) # [CLS] pooling
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def main() -> int:
|
|
70
|
+
tok = AutoTokenizer.from_pretrained(MODEL)
|
|
71
|
+
# Eager attention avoids SDPA ops the converter rejects; .float() forces fp32 weights
|
|
72
|
+
# (the fp16 checkpoint trips a layer_norm dtype check at the frontend).
|
|
73
|
+
model = AutoModel.from_pretrained(MODEL, attn_implementation="eager").float().eval()
|
|
74
|
+
|
|
75
|
+
# Mask padding with -1e4, not fp32-min: the latter overflows to -inf under fp16 and
|
|
76
|
+
# 0*-inf = nan poisons every token. softmax still drives masked positions to ~0.
|
|
77
|
+
def _safe_ext_mask(self, attention_mask, input_shape, device=None, dtype=None):
|
|
78
|
+
if dtype is None:
|
|
79
|
+
dtype = self.dtype
|
|
80
|
+
ext = attention_mask[:, None, None, :].to(dtype=dtype)
|
|
81
|
+
return (1.0 - ext) * -1.0e4
|
|
82
|
+
|
|
83
|
+
model.get_extended_attention_mask = types.MethodType(_safe_ext_mask, model)
|
|
84
|
+
wrapper = TextEncoder(model, MAX_LEN).eval()
|
|
85
|
+
|
|
86
|
+
enc = tok(
|
|
87
|
+
"a short example sentence",
|
|
88
|
+
padding="max_length",
|
|
89
|
+
truncation=True,
|
|
90
|
+
max_length=MAX_LEN,
|
|
91
|
+
return_tensors="pt",
|
|
92
|
+
)
|
|
93
|
+
# Keep int64 for export (casting to int32 here makes jit.trace emit an aten::Int
|
|
94
|
+
# coremltools can't lower); Core ML inputs are declared int32 below.
|
|
95
|
+
ids_ex = enc["input_ids"]
|
|
96
|
+
mask_ex = enc["attention_mask"]
|
|
97
|
+
|
|
98
|
+
with torch.no_grad():
|
|
99
|
+
ref = wrapper(ids_ex, mask_ex)
|
|
100
|
+
|
|
101
|
+
print("tracing + converting text encoder...")
|
|
102
|
+
# jit.trace (TorchScript frontend) tolerates new_ones, which the export frontend rejects.
|
|
103
|
+
traced = torch.jit.trace(wrapper, (ids_ex, mask_ex))
|
|
104
|
+
mlmodel = ct.convert(
|
|
105
|
+
traced,
|
|
106
|
+
inputs=[
|
|
107
|
+
ct.TensorType(name="input_ids", shape=(1, MAX_LEN), dtype=np.int32),
|
|
108
|
+
ct.TensorType(name="attention_mask", shape=(1, MAX_LEN), dtype=np.int32),
|
|
109
|
+
],
|
|
110
|
+
outputs=[ct.TensorType(name="embedding")],
|
|
111
|
+
compute_units=ct.ComputeUnit.ALL,
|
|
112
|
+
# fp16 except layer_norm (its fp32 epsilon trips a dtype check); matmuls stay fp16.
|
|
113
|
+
compute_precision=ct.transform.FP16ComputePrecision(
|
|
114
|
+
op_selector=lambda op: op.op_type != "layer_norm"
|
|
115
|
+
),
|
|
116
|
+
minimum_deployment_target=ct.target.macOS14,
|
|
117
|
+
)
|
|
118
|
+
pkg = "text.mlpackage"
|
|
119
|
+
mlmodel.save(pkg)
|
|
120
|
+
|
|
121
|
+
out = mlmodel.predict(
|
|
122
|
+
{
|
|
123
|
+
"input_ids": ids_ex.numpy().astype(np.int32),
|
|
124
|
+
"attention_mask": mask_ex.numpy().astype(np.int32),
|
|
125
|
+
}
|
|
126
|
+
)
|
|
127
|
+
vec = torch.tensor(out["embedding"]).flatten().float()
|
|
128
|
+
cos = F.cosine_similarity(vec, ref.flatten().float(), dim=0).item()
|
|
129
|
+
print(f"text: cosine(coreml, torch) = {cos:.5f} (dim={vec.numel()})")
|
|
130
|
+
if cos <= 0.99 or vec.numel() != DIM:
|
|
131
|
+
print("PARITY FAILED")
|
|
132
|
+
return 1
|
|
133
|
+
|
|
134
|
+
# Bundle the WordPiece vocab/config for the torch-free runtime tokenizer.
|
|
135
|
+
tok.backend_tokenizer.save("text_tokenizer.json")
|
|
136
|
+
|
|
137
|
+
print("warming ANE caches...")
|
|
138
|
+
compiled = "text.mlmodelc"
|
|
139
|
+
if os.path.exists(compiled):
|
|
140
|
+
shutil.rmtree(compiled)
|
|
141
|
+
compile_model(pkg, compiled)
|
|
142
|
+
CompiledMLModel(compiled, compute_units=ct.ComputeUnit.CPU_AND_NE)
|
|
143
|
+
print("done")
|
|
144
|
+
return 0
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
if __name__ == "__main__":
|
|
148
|
+
sys.exit(main())
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""The Vault's embedder on Linux: ONNX Runtime on the CPU in place of Core ML.
|
|
2
|
+
|
|
3
|
+
The cloud's app.py (preprocessing, tokenizers, pooling) and its converters
|
|
4
|
+
(the torch wrappers) are used as they are. They import Core ML at the top,
|
|
5
|
+
which does not exist on Linux, so a stand-in is put in its place first; on
|
|
6
|
+
Linux nothing ever calls it.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import sys
|
|
10
|
+
import types
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def stub_coremltools():
|
|
16
|
+
"""Lets app.py, convert_text.py and convert_clip.py import on Linux."""
|
|
17
|
+
ct = types.ModuleType("coremltools")
|
|
18
|
+
ct.ComputeUnit = types.SimpleNamespace(CPU_AND_NE=None, CPU_AND_GPU=None, CPU_ONLY=None, ALL=None)
|
|
19
|
+
models = types.ModuleType("coremltools.models")
|
|
20
|
+
models.CompiledMLModel = None
|
|
21
|
+
utils = types.ModuleType("coremltools.models.utils")
|
|
22
|
+
utils.compile_model = None
|
|
23
|
+
models.utils = utils
|
|
24
|
+
ct.models = models
|
|
25
|
+
sys.modules.update({"coremltools": ct, "coremltools.models": models, "coremltools.models.utils": utils})
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class OnnxModel:
|
|
29
|
+
"""A Core ML model's predict(): named arrays in, {"embedding"} out, so
|
|
30
|
+
app.py's embed_* functions run on it unchanged."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, path: str):
|
|
33
|
+
import onnxruntime as ort
|
|
34
|
+
|
|
35
|
+
options = ort.SessionOptions()
|
|
36
|
+
# No arena: it keeps the largest activations' memory after a call,
|
|
37
|
+
# and an idle embedder should hold little more than its weights.
|
|
38
|
+
options.enable_cpu_mem_arena = False
|
|
39
|
+
self.session = ort.InferenceSession(path, options, providers=["CPUExecutionProvider"])
|
|
40
|
+
# app.py's tokenizers hand int32 (Core ML's inputs); the export takes
|
|
41
|
+
# torch's int64.
|
|
42
|
+
self.inputs = {
|
|
43
|
+
i.name: np.int64 if i.type == "tensor(int64)" else np.float32 for i in self.session.get_inputs()
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
def predict(self, feeds: dict) -> dict:
|
|
47
|
+
args = {name: np.asarray(feeds[name]).astype(dtype, copy=False) for name, dtype in self.inputs.items()}
|
|
48
|
+
return {"embedding": self.session.run(["embedding"], args)[0]}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# The embedder's serving packages on Linux (onnx_backend.py): the cloud's
|
|
2
|
+
# requirements.txt without Core ML and pyobjc, which are macOS-only, plus
|
|
3
|
+
# ONNX Runtime, a PDF reader and OCR. Keep the shared pins in step with
|
|
4
|
+
# services/embedder/requirements.txt: Pillow resizes the photos, so a
|
|
5
|
+
# different one makes different CLIP vectors.
|
|
6
|
+
onnxruntime>=1.20
|
|
7
|
+
numpy>=2.0
|
|
8
|
+
ftfy>=6.0
|
|
9
|
+
regex>=2024.0
|
|
10
|
+
tokenizers>=0.20.0
|
|
11
|
+
fastapi==0.115.6
|
|
12
|
+
uvicorn[standard]==0.34.0
|
|
13
|
+
pillow==13.0.0
|
|
14
|
+
pydantic==2.10.4
|
|
15
|
+
# A PDF's text layer, and its pages drawn for OCR (PDFKit's and Quartz's job
|
|
16
|
+
# on the Mac).
|
|
17
|
+
pypdfium2>=4.30
|
|
18
|
+
# OCR for scans (Apple Vision's job on the Mac): PaddleOCR's models on ONNX
|
|
19
|
+
# Runtime, shipped inside the wheel. Its opencv-python needs libGL, which a
|
|
20
|
+
# server often lacks: the headless build stands in (overrides-onnx.txt).
|
|
21
|
+
rapidocr_onnxruntime==1.4.4
|
|
22
|
+
opencv-python-headless>=4.8,<5
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Serving deps — the running service imports only these (no torch). ftfy+regex
|
|
2
|
+
# back the torch-free CLIP tokenizer (clip_tokenizer.py); tokenizers backs the
|
|
3
|
+
# torch-free WordPiece tokenizer for gte-small (text_tokenizer.py). torch/
|
|
4
|
+
# torchvision/open_clip/transformers are installed separately by setup.sh for the
|
|
5
|
+
# one-time Core ML conversions (convert*.py) and are NOT imported at serve time.
|
|
6
|
+
coremltools>=9.0
|
|
7
|
+
numpy>=2.0
|
|
8
|
+
ftfy>=6.0
|
|
9
|
+
regex>=2024.0
|
|
10
|
+
tokenizers>=0.20.0
|
|
11
|
+
python-dotenv>=1.0
|
|
12
|
+
fastapi==0.115.6
|
|
13
|
+
uvicorn[standard]==0.34.0
|
|
14
|
+
pillow==13.0.0
|
|
15
|
+
pydantic==2.10.4
|
|
16
|
+
# OCR for scanned PDFs (/extract): CoreGraphics rasterization + Apple Vision text
|
|
17
|
+
# recognition via pyobjc. macOS-only; the embedder already runs natively on the
|
|
18
|
+
# Mac Mini. Vision pulls in CoreML/Quartz frameworks it needs transitively.
|
|
19
|
+
pyobjc-framework-Quartz>=10.0
|
|
20
|
+
pyobjc-framework-Vision>=10.0
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""The Vault's embedder on the runner (.specs/runner-vault.md "Embedder").
|
|
2
|
+
|
|
3
|
+
The cloud's embedder (services/embedder/app.py) with the vault's two search
|
|
4
|
+
models only: bge-base-en-v1.5 (768-d text) and MobileCLIP2-B (512-d CLIP).
|
|
5
|
+
Everything that makes a vector is imported from app.py as it is, so a file
|
|
6
|
+
embedded here lands in the same space as the cloud's index and the two can be
|
|
7
|
+
ranked together. DINOv2 (meme grouping) is not loaded. On a Mac the models
|
|
8
|
+
are Core ML; on Linux they are ONNX on the CPU (onnx_backend.py, made by
|
|
9
|
+
convert_onnx.py), run through the same app.py functions.
|
|
10
|
+
|
|
11
|
+
Routes:
|
|
12
|
+
- POST /embed {text} | {clipText} | {clipImage: base64} -> {vector}. The
|
|
13
|
+
same body as the cloud embedder's, so a vector here and one there compare.
|
|
14
|
+
- POST /read {file_b64, ext} -> {pages: [{text, source}]}. A PDF's text
|
|
15
|
+
layer (PDFKit on a Mac, pdfium on Linux), else OCR (Apple Vision by
|
|
16
|
+
app.ocr_pdf on a Mac, RapidOCR on Linux); a docx's paragraphs.
|
|
17
|
+
- GET /health
|
|
18
|
+
|
|
19
|
+
Started by the runner (vault/embedder.ts) on 127.0.0.1 with a key in
|
|
20
|
+
VAULT_EMBEDDER_KEY, from the folder that holds the models.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import base64
|
|
24
|
+
import hmac
|
|
25
|
+
import importlib.util
|
|
26
|
+
import io
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
import sys
|
|
30
|
+
import threading
|
|
31
|
+
import zipfile
|
|
32
|
+
from contextlib import asynccontextmanager
|
|
33
|
+
from xml.etree import ElementTree
|
|
34
|
+
|
|
35
|
+
from fastapi import Depends, FastAPI, Header, HTTPException
|
|
36
|
+
from PIL import Image, ImageOps
|
|
37
|
+
from pydantic import BaseModel
|
|
38
|
+
|
|
39
|
+
# Linux has no Core ML: app.py imports with a stand-in and serves ONNX.
|
|
40
|
+
ONNX = sys.platform != "darwin"
|
|
41
|
+
if ONNX:
|
|
42
|
+
from onnx_backend import OnnxModel, stub_coremltools
|
|
43
|
+
|
|
44
|
+
stub_coremltools()
|
|
45
|
+
|
|
46
|
+
import app as cloud # noqa: E402
|
|
47
|
+
from clip_tokenizer import ClipTokenizer
|
|
48
|
+
from text_tokenizer import TextTokenizer
|
|
49
|
+
|
|
50
|
+
KEY = os.environ.get("VAULT_EMBEDDER_KEY", "")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def require_key(authorization: str | None = Header(default=None)):
|
|
54
|
+
sent = (authorization or "").removeprefix("Bearer ").strip()
|
|
55
|
+
if not KEY or not hmac.compare_digest(sent, KEY):
|
|
56
|
+
raise HTTPException(status_code=401, detail="invalid or missing key")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _exit_with_runner():
|
|
60
|
+
"""The runner holds this process's stdin: when the runner goes, so does
|
|
61
|
+
the embedder, even if the runner was killed and could not stop it."""
|
|
62
|
+
try:
|
|
63
|
+
while sys.stdin.buffer.read(65536):
|
|
64
|
+
pass
|
|
65
|
+
finally:
|
|
66
|
+
os._exit(0)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@asynccontextmanager
|
|
70
|
+
async def lifespan(_: FastAPI):
|
|
71
|
+
if os.environ.get("VAULT_EMBEDDER_WATCH_STDIN") == "1":
|
|
72
|
+
threading.Thread(target=_exit_with_runner, daemon=True).start()
|
|
73
|
+
# app.py's own loader and state, so its embed_* functions find the models.
|
|
74
|
+
load = (lambda name: OnnxModel(f"{name}.onnx")) if ONNX else cloud._load
|
|
75
|
+
cloud.state["clip_image"] = load("clip_image")
|
|
76
|
+
cloud.state["clip_text"] = load("clip_text")
|
|
77
|
+
cloud.state["clip_tok"] = ClipTokenizer()
|
|
78
|
+
cloud.state["text"] = load("text")
|
|
79
|
+
cloud.state["text_tok"] = TextTokenizer()
|
|
80
|
+
yield
|
|
81
|
+
cloud.state.clear()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
api = FastAPI(lifespan=lifespan)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class EmbedBody(BaseModel):
|
|
88
|
+
text: str | None = None
|
|
89
|
+
clipText: str | None = None
|
|
90
|
+
clipImage: str | None = None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class ReadBody(BaseModel):
|
|
94
|
+
file_b64: str
|
|
95
|
+
ext: str
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _bytes(b64: str) -> bytes:
|
|
99
|
+
try:
|
|
100
|
+
return base64.b64decode(b64)
|
|
101
|
+
except Exception:
|
|
102
|
+
raise HTTPException(status_code=400, detail="invalid base64")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _upright(raw: bytes) -> bytes:
|
|
106
|
+
"""A photo turned the way its EXIF says, as the cloud's sharp().rotate()."""
|
|
107
|
+
img = ImageOps.exif_transpose(Image.open(io.BytesIO(raw)))
|
|
108
|
+
out = io.BytesIO()
|
|
109
|
+
img.convert("RGB").save(out, format="JPEG", quality=90)
|
|
110
|
+
return out.getvalue()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@api.get("/health")
|
|
114
|
+
def health():
|
|
115
|
+
return {
|
|
116
|
+
"status": "ok",
|
|
117
|
+
"text": "bge-base-en-v1.5",
|
|
118
|
+
"text_dim": cloud.TEXT_DIM,
|
|
119
|
+
"clip": "MobileCLIP2-B",
|
|
120
|
+
"clip_dim": cloud.CLIP_DIM,
|
|
121
|
+
"runtime": "onnx" if ONNX else "coreml",
|
|
122
|
+
"ocr": _ocr_ok(),
|
|
123
|
+
"loaded": "text" in cloud.state,
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@api.post("/embed", dependencies=[Depends(require_key)])
|
|
128
|
+
def embed(body: EmbedBody):
|
|
129
|
+
# The server route's order: clipImage, clipText, text.
|
|
130
|
+
try:
|
|
131
|
+
if body.clipImage:
|
|
132
|
+
return {"vector": cloud.embed_clip_image(_upright(_bytes(body.clipImage)))}
|
|
133
|
+
if body.clipText and body.clipText.strip():
|
|
134
|
+
return {"vector": cloud.embed_clip_text(body.clipText)}
|
|
135
|
+
if body.text and body.text.strip():
|
|
136
|
+
return {"vector": cloud.embed_text(body.text)}
|
|
137
|
+
except HTTPException:
|
|
138
|
+
raise
|
|
139
|
+
except Exception as exc:
|
|
140
|
+
raise HTTPException(status_code=422, detail=f"could not embed: {exc}")
|
|
141
|
+
raise HTTPException(status_code=400, detail="text, clipText, or clipImage required")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _pdf_text_layer(raw: bytes) -> list[str]:
|
|
145
|
+
if ONNX:
|
|
146
|
+
import pypdfium2 as pdfium
|
|
147
|
+
|
|
148
|
+
doc = pdfium.PdfDocument(raw)
|
|
149
|
+
# pdfium ends lines with \r\n; PDFKit with \n.
|
|
150
|
+
return [re.sub(r"\r\n?", "\n", page.get_textpage().get_text_bounded()).strip() for page in doc]
|
|
151
|
+
from Foundation import NSData
|
|
152
|
+
from Quartz import PDFDocument
|
|
153
|
+
|
|
154
|
+
doc = PDFDocument.alloc().initWithData_(NSData.dataWithBytes_length_(raw, len(raw)))
|
|
155
|
+
if doc is None:
|
|
156
|
+
raise ValueError("not a valid PDF")
|
|
157
|
+
pages = []
|
|
158
|
+
for i in range(doc.pageCount()):
|
|
159
|
+
text = doc.pageAtIndex_(i).string()
|
|
160
|
+
pages.append(str(text).strip() if text else "")
|
|
161
|
+
return pages
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
_rapid = None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _ocr_ok() -> bool:
|
|
168
|
+
if not ONNX:
|
|
169
|
+
return cloud._VISION_OK
|
|
170
|
+
return importlib.util.find_spec("rapidocr_onnxruntime") is not None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _ocr_pdf(raw: bytes) -> list[str]:
|
|
174
|
+
"""Every page's text, as app.ocr_pdf: drawn at its 200 DPI, lines top to
|
|
175
|
+
bottom. RapidOCR on Linux, loaded on the first scan only."""
|
|
176
|
+
if not ONNX:
|
|
177
|
+
return cloud.ocr_pdf(raw)
|
|
178
|
+
global _rapid
|
|
179
|
+
import numpy as np
|
|
180
|
+
import pypdfium2 as pdfium
|
|
181
|
+
|
|
182
|
+
if _rapid is None:
|
|
183
|
+
from rapidocr_onnxruntime import RapidOCR
|
|
184
|
+
|
|
185
|
+
_rapid = RapidOCR()
|
|
186
|
+
pages = []
|
|
187
|
+
for page in pdfium.PdfDocument(raw):
|
|
188
|
+
image = page.render(scale=cloud.OCR_DPI / 72).to_pil()
|
|
189
|
+
with cloud._INFER_LOCK:
|
|
190
|
+
found, _ = _rapid(np.asarray(image.convert("RGB")))
|
|
191
|
+
pages.append("\n".join(line[1] for line in found or []))
|
|
192
|
+
return pages
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _docx_text(raw: bytes) -> str:
|
|
199
|
+
"""Paragraph text, a blank line between paragraphs (mammoth's raw text)."""
|
|
200
|
+
with zipfile.ZipFile(io.BytesIO(raw)) as z:
|
|
201
|
+
root = ElementTree.fromstring(z.read("word/document.xml"))
|
|
202
|
+
paragraphs = []
|
|
203
|
+
for p in root.iter(f"{_W}p"):
|
|
204
|
+
parts = []
|
|
205
|
+
for node in p.iter():
|
|
206
|
+
if node.tag == f"{_W}t" and node.text:
|
|
207
|
+
parts.append(node.text)
|
|
208
|
+
elif node.tag == f"{_W}tab":
|
|
209
|
+
parts.append("\t")
|
|
210
|
+
elif node.tag in (f"{_W}br", f"{_W}cr"):
|
|
211
|
+
parts.append("\n")
|
|
212
|
+
paragraphs.append("".join(parts))
|
|
213
|
+
return re.sub(r"\n{3,}", "\n\n", "\n\n".join(paragraphs)).strip()
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@api.post("/read", dependencies=[Depends(require_key)])
|
|
217
|
+
def read(body: ReadBody):
|
|
218
|
+
ext = body.ext.lower().lstrip(".")
|
|
219
|
+
raw = _bytes(body.file_b64)
|
|
220
|
+
try:
|
|
221
|
+
if ext == "pdf":
|
|
222
|
+
pages = _pdf_text_layer(raw)
|
|
223
|
+
if any(pages):
|
|
224
|
+
return {"pages": [{"text": t, "source": "TEXT_LAYER"} for t in pages]}
|
|
225
|
+
# No text layer: a scan. OCR reads every page, as the cloud does.
|
|
226
|
+
if not _ocr_ok():
|
|
227
|
+
raise HTTPException(status_code=422, detail="PDF has no text layer and OCR is unavailable")
|
|
228
|
+
ocr = [t.strip() for t in _ocr_pdf(raw)]
|
|
229
|
+
if not any(ocr):
|
|
230
|
+
raise HTTPException(status_code=422, detail="PDF has no recoverable text (OCR found nothing)")
|
|
231
|
+
return {"pages": [{"text": t, "source": "OCR"} for t in ocr]}
|
|
232
|
+
if ext == "docx":
|
|
233
|
+
text = _docx_text(raw)
|
|
234
|
+
if not text:
|
|
235
|
+
raise HTTPException(status_code=422, detail="docx contains no text")
|
|
236
|
+
return {"pages": [{"text": text, "source": "TEXT_LAYER"}]}
|
|
237
|
+
except HTTPException:
|
|
238
|
+
raise
|
|
239
|
+
except Exception as exc:
|
|
240
|
+
raise HTTPException(status_code=422, detail=f"could not read {ext}: {exc}")
|
|
241
|
+
raise HTTPException(status_code=415, detail=f"cannot read type: {ext}")
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Torch-free WordPiece tokenizer for the text-embedding model (bge-base-en-v1.5).
|
|
2
|
+
Loads the model's `text_tokenizer.json` via the Rust `tokenizers` lib (no torch,
|
|
3
|
+
no transformers) so the running service stays lean — same idea as the CLIP
|
|
4
|
+
tokenizer (clip_tokenizer.py), but BERT WordPiece instead of CLIP BPE.
|
|
5
|
+
|
|
6
|
+
Returns fixed-length int32 `input_ids` + `attention_mask` (padded/truncated to
|
|
7
|
+
MAX_LEN) matching the Core ML model's input shapes. The json is written by
|
|
8
|
+
convert_text.py from the HF tokenizer.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
from tokenizers import Tokenizer
|
|
15
|
+
|
|
16
|
+
TOKENIZER_JSON = os.path.join(
|
|
17
|
+
os.path.dirname(os.path.abspath(__file__)), "text_tokenizer.json"
|
|
18
|
+
)
|
|
19
|
+
# Must match MAX_LEN in convert_text.py (the Core ML model's fixed sequence len).
|
|
20
|
+
MAX_LEN = 256
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class TextTokenizer:
|
|
24
|
+
def __init__(self, path: str = TOKENIZER_JSON, max_len: int = MAX_LEN):
|
|
25
|
+
self.tok = Tokenizer.from_file(path)
|
|
26
|
+
self.tok.enable_truncation(max_length=max_len)
|
|
27
|
+
self.tok.enable_padding(length=max_len)
|
|
28
|
+
self.max_len = max_len
|
|
29
|
+
|
|
30
|
+
def __call__(self, text: str) -> tuple[np.ndarray, np.ndarray]:
|
|
31
|
+
enc = self.tok.encode(text)
|
|
32
|
+
ids = np.asarray(enc.ids, dtype=np.int32)[None] # [1, max_len]
|
|
33
|
+
mask = np.asarray(enc.attention_mask, dtype=np.int32)[None] # [1, max_len]
|
|
34
|
+
return ids, mask
|
package/native/simhid.m
CHANGED
|
@@ -76,6 +76,13 @@ static BOOL loadPrivateFrameworks(NSString *devDir, void **simulatorKit) {
|
|
|
76
76
|
NSString *kitPath = [devDir stringByAppendingString:
|
|
77
77
|
@"/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit"];
|
|
78
78
|
void *kit = dlopen(kitPath.UTF8String, RTLD_NOW);
|
|
79
|
+
if (!kit) {
|
|
80
|
+
// Xcode 27 moved SimulatorKit out of Developer/Library/PrivateFrameworks
|
|
81
|
+
// into Contents/SharedFrameworks, beside the Developer directory.
|
|
82
|
+
NSString *shared = [[devDir stringByDeletingLastPathComponent]
|
|
83
|
+
stringByAppendingString:@"/SharedFrameworks/SimulatorKit.framework/SimulatorKit"];
|
|
84
|
+
kit = dlopen(shared.UTF8String, RTLD_NOW);
|
|
85
|
+
}
|
|
79
86
|
if (simulatorKit) *simulatorKit = kit;
|
|
80
87
|
if (!loadedCore || !kit) {
|
|
81
88
|
fprintf(stderr, "Could not load the simulator frameworks from %s\n", devDir.UTF8String);
|
package/native/simstream.m
CHANGED
|
@@ -80,6 +80,13 @@ static BOOL loadPrivateFrameworks(NSString *devDir, void **simulatorKit) {
|
|
|
80
80
|
NSString *kitPath = [devDir stringByAppendingString:
|
|
81
81
|
@"/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit"];
|
|
82
82
|
void *kit = dlopen(kitPath.UTF8String, RTLD_NOW);
|
|
83
|
+
if (!kit) {
|
|
84
|
+
// Xcode 27 moved SimulatorKit out of Developer/Library/PrivateFrameworks
|
|
85
|
+
// into Contents/SharedFrameworks, beside the Developer directory.
|
|
86
|
+
NSString *shared = [[devDir stringByDeletingLastPathComponent]
|
|
87
|
+
stringByAppendingString:@"/SharedFrameworks/SimulatorKit.framework/SimulatorKit"];
|
|
88
|
+
kit = dlopen(shared.UTF8String, RTLD_NOW);
|
|
89
|
+
}
|
|
83
90
|
if (simulatorKit) *simulatorKit = kit;
|
|
84
91
|
if (!loadedCore || !kit) {
|
|
85
92
|
fprintf(stderr, "Could not load the simulator frameworks from %s\n", devDir.UTF8String);
|