@factiii/runner 0.27.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +174 -0
- package/README.md +36 -16
- package/dist/cli.js +84643 -50754
- package/dist/vault-client.js +876 -0
- package/dist/vault-db.js +858 -0
- package/embedder/app.py +331 -0
- package/embedder/clip_tokenizer.py +137 -0
- package/embedder/convert_clip.py +119 -0
- package/embedder/convert_onnx.py +129 -0
- package/embedder/convert_text.py +148 -0
- package/embedder/onnx_backend.py +48 -0
- package/embedder/overrides-onnx.txt +3 -0
- package/embedder/requirements-onnx.txt +22 -0
- package/embedder/requirements.txt +20 -0
- package/embedder/serve.py +241 -0
- package/embedder/text_tokenizer.py +34 -0
- package/native/simhid.m +7 -0
- package/native/simstream.m +7 -0
- package/package.json +29 -16
- package/plugin/factiii/bin/factiii-board-mcp +11 -3
- package/plugin/factiii/bin/factiii-secrets +88 -21
- package/plugin/factiii/bin/session-context.sh +55 -2
- package/plugin/factiii/skills/board-sessions/SKILL.md +74 -0
- package/plugin/factiii/skills/factiii-secrets/SKILL.md +21 -7
- package/plugin/factiii/skills/runner-onboarding/SKILL.md +67 -0
- package/plugin/factiii/skills/scratchpad/SKILL.md +1 -0
- package/plugin/factiii/skills/share-a-port/SKILL.md +12 -6
- package/index/image-package.json +0 -8
- package/index/index.js +0 -105575
- package/native/screenhost.m +0 -500
- package/vendor/frpc-darwin-amd64 +0 -0
- package/vendor/frpc-darwin-arm64 +0 -0
- package/vendor/frpc-linux-amd64 +0 -0
- package/vendor/frpc-linux-arm64 +0 -0
package/embedder/app.py
ADDED
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import io
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
from contextlib import asynccontextmanager
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
from fastapi import Depends, FastAPI, Header, HTTPException, Request
|
|
11
|
+
from PIL import Image
|
|
12
|
+
from pydantic import BaseModel
|
|
13
|
+
|
|
14
|
+
import coremltools as ct
|
|
15
|
+
from clip_tokenizer import ClipTokenizer
|
|
16
|
+
from coremltools.models import CompiledMLModel
|
|
17
|
+
from text_tokenizer import TextTokenizer
|
|
18
|
+
|
|
19
|
+
# OCR for scanned PDFs (file-vault /extract) uses native macOS frameworks via
|
|
20
|
+
# pyobjc — CoreGraphics rasterizes each PDF page, Apple Vision recognizes text.
|
|
21
|
+
# Guarded so the embedder still serves embeddings if pyobjc isn't installed;
|
|
22
|
+
# /extract then returns 503 rather than crashing startup.
|
|
23
|
+
try:
|
|
24
|
+
import Quartz
|
|
25
|
+
import Vision
|
|
26
|
+
from Foundation import NSData
|
|
27
|
+
|
|
28
|
+
_VISION_OK = True
|
|
29
|
+
except Exception: # noqa: BLE001 — any import failure disables OCR, not the service
|
|
30
|
+
_VISION_OK = False
|
|
31
|
+
|
|
32
|
+
# Config comes from the environment only. start.sh copies this service's two
|
|
33
|
+
# keys out of the monorepo root .env (.specs/env.md). This file never opens a
|
|
34
|
+
# .env itself: the runner ships it in its npm package (apps/runner embedder/),
|
|
35
|
+
# and npm pulled runner 0.28.0 and 0.33.0 as malware while it read one.
|
|
36
|
+
|
|
37
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
38
|
+
logger = logging.getLogger("embedder")
|
|
39
|
+
|
|
40
|
+
# Shared secret: the server sends it as `x-api-key`. When set, requests without
|
|
41
|
+
# it are rejected so only our server (which has the key) can use the embedder.
|
|
42
|
+
API_KEY = os.environ.get("EMBEDDER_API_KEY")
|
|
43
|
+
if not API_KEY:
|
|
44
|
+
logger.warning("EMBEDDER_API_KEY not set — embedder is UNSECURED (open to anyone).")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def require_key(x_api_key: str | None = Header(default=None)):
|
|
48
|
+
if API_KEY and x_api_key != API_KEY:
|
|
49
|
+
raise HTTPException(status_code=401, detail="invalid or missing api key")
|
|
50
|
+
|
|
51
|
+
# Three models, all Core ML / Apple Neural Engine, torch-free (~0.5GB serving):
|
|
52
|
+
# DINOv2 (768-d) grouping, MobileCLIP2-B (512-d) text<->image, bge-base-en-v1.5 (768-d) text<->text.
|
|
53
|
+
|
|
54
|
+
DIM = 768
|
|
55
|
+
CLIP_DIM = 512
|
|
56
|
+
TEXT_DIM = 768
|
|
57
|
+
|
|
58
|
+
_UNITS = {
|
|
59
|
+
"ne": ct.ComputeUnit.CPU_AND_NE, # Neural Engine (default — fastest, leanest)
|
|
60
|
+
"gpu": ct.ComputeUnit.CPU_AND_GPU,
|
|
61
|
+
"cpu": ct.ComputeUnit.CPU_ONLY,
|
|
62
|
+
"all": ct.ComputeUnit.ALL,
|
|
63
|
+
}
|
|
64
|
+
COMPUTE = os.environ.get("COREML_UNITS", "ne")
|
|
65
|
+
|
|
66
|
+
# DINOv2 preprocessing: resize shorter edge to 256 (bilinear), center-crop 224,
|
|
67
|
+
# scale to [0,1], ImageNet normalize.
|
|
68
|
+
_DINO_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
|
|
69
|
+
_DINO_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
|
|
70
|
+
# CLIP preprocessing: 224 bilinear, center-crop, scale to [0,1]. MobileCLIP2-B/dfndr2b
|
|
71
|
+
# uses NO normalization (mean 0/std 1) — wrong values yield garbage image vectors.
|
|
72
|
+
_CLIP_MEAN = np.array([0.0, 0.0, 0.0], dtype=np.float32)
|
|
73
|
+
_CLIP_STD = np.array([1.0, 1.0, 1.0], dtype=np.float32)
|
|
74
|
+
|
|
75
|
+
state: dict = {}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _load(name: str):
|
|
79
|
+
units = _UNITS[COMPUTE]
|
|
80
|
+
if os.path.isdir(f"{name}.mlmodelc"):
|
|
81
|
+
return CompiledMLModel(f"{name}.mlmodelc", compute_units=units)
|
|
82
|
+
# Fallback to the raw package (slow start; run convert*.py to precompile).
|
|
83
|
+
return ct.models.MLModel(f"{name}.mlpackage", compute_units=units)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@asynccontextmanager
|
|
87
|
+
async def lifespan(_: FastAPI):
|
|
88
|
+
state["dino"] = _load("model")
|
|
89
|
+
state["clip_image"] = _load("clip_image")
|
|
90
|
+
state["clip_text"] = _load("clip_text")
|
|
91
|
+
state["clip_tok"] = ClipTokenizer()
|
|
92
|
+
state["text"] = _load("text")
|
|
93
|
+
state["text_tok"] = TextTokenizer()
|
|
94
|
+
yield
|
|
95
|
+
state.clear()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
app = FastAPI(lifespan=lifespan)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# Log every request: method, path, status, latency.
|
|
102
|
+
@app.middleware("http")
|
|
103
|
+
async def log_requests(request: Request, call_next):
|
|
104
|
+
start = time.perf_counter()
|
|
105
|
+
response = await call_next(request)
|
|
106
|
+
took = (time.perf_counter() - start) * 1000
|
|
107
|
+
logger.info(
|
|
108
|
+
"%s %s -> %s %.0fms",
|
|
109
|
+
request.method,
|
|
110
|
+
request.url.path,
|
|
111
|
+
response.status_code,
|
|
112
|
+
took
|
|
113
|
+
)
|
|
114
|
+
return response
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class EmbedRequest(BaseModel):
|
|
118
|
+
image: str # base64-encoded image bytes
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class EmbedResponse(BaseModel):
|
|
122
|
+
vector: list[float]
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class TextRequest(BaseModel):
|
|
126
|
+
text: str
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def preprocess(raw: bytes, resize: int, mean: np.ndarray, std: np.ndarray) -> np.ndarray:
|
|
130
|
+
img = Image.open(io.BytesIO(raw)).convert("RGB")
|
|
131
|
+
w, h = img.size
|
|
132
|
+
scale = resize / min(w, h)
|
|
133
|
+
# Both image models (DINOv2 and MobileCLIP2-B) use bilinear resize.
|
|
134
|
+
img = img.resize((round(w * scale), round(h * scale)), Image.BILINEAR)
|
|
135
|
+
nw, nh = img.size
|
|
136
|
+
crop = 224
|
|
137
|
+
left, top = (nw - crop) // 2, (nh - crop) // 2
|
|
138
|
+
img = img.crop((left, top, left + crop, top + crop))
|
|
139
|
+
arr = (np.asarray(img, dtype=np.float32) / 255.0 - mean) / std # HWC
|
|
140
|
+
return arr.transpose(2, 0, 1)[None].astype(np.float32) # 1,C,H,W
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def l2(arr: np.ndarray) -> list[float]:
|
|
144
|
+
vec = np.asarray(arr, dtype=np.float32).flatten()
|
|
145
|
+
vec /= np.linalg.norm(vec) or 1.0
|
|
146
|
+
return vec.tolist()
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# Serialize ALL CoreML inference. Sync endpoints run in Starlette's threadpool, so
|
|
150
|
+
# concurrent requests would call .predict() on the shared MLModels from multiple
|
|
151
|
+
# threads at once — the ANE's MLE5 execution stream is not safe under that and
|
|
152
|
+
# segfaults during stream teardown (CoreML -[MLE5ExecutionStream resetAfterLingering]).
|
|
153
|
+
# One ANE serializes the real work anyway, so a single global lock costs ~nothing.
|
|
154
|
+
_INFER_LOCK = threading.Lock()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def embed_dino(raw: bytes) -> list[float]:
|
|
158
|
+
batch = preprocess(raw, 256, _DINO_MEAN, _DINO_STD)
|
|
159
|
+
with _INFER_LOCK:
|
|
160
|
+
return l2(state["dino"].predict({"image": batch})["embedding"])
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def embed_clip_image(raw: bytes) -> list[float]:
|
|
164
|
+
batch = preprocess(raw, 224, _CLIP_MEAN, _CLIP_STD)
|
|
165
|
+
with _INFER_LOCK:
|
|
166
|
+
return l2(state["clip_image"].predict({"image": batch})["embedding"])
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def embed_clip_text(text: str) -> list[float]:
|
|
170
|
+
tokens = state["clip_tok"](text)
|
|
171
|
+
with _INFER_LOCK:
|
|
172
|
+
return l2(state["clip_text"].predict({"tokens": tokens})["embedding"])
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def embed_text(text: str) -> list[float]:
|
|
176
|
+
ids, mask = state["text_tok"](text)
|
|
177
|
+
with _INFER_LOCK:
|
|
178
|
+
out = state["text"].predict({"input_ids": ids, "attention_mask": mask})
|
|
179
|
+
# The graph already CLS-pools + normalizes (bge); l2 is a harmless re-normalize.
|
|
180
|
+
return l2(out["embedding"])
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
# --- OCR (Apple Vision) for scanned PDFs -------------------------------------
|
|
184
|
+
# Render at 200 DPI: enough resolution for Vision to read body text on a scan
|
|
185
|
+
# without ballooning bitmap memory (a US-Letter page is ~1700x2200 px here).
|
|
186
|
+
OCR_DPI = 200
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _render_pdf_page(page, scale: float):
|
|
190
|
+
"""CGPDFPage -> white-backed RGB CGImage at `scale`× its native size."""
|
|
191
|
+
rect = Quartz.CGPDFPageGetBoxRect(page, Quartz.kCGPDFMediaBox)
|
|
192
|
+
width = max(1, int(rect.size.width * scale))
|
|
193
|
+
height = max(1, int(rect.size.height * scale))
|
|
194
|
+
color_space = Quartz.CGColorSpaceCreateDeviceRGB()
|
|
195
|
+
ctx = Quartz.CGBitmapContextCreate(
|
|
196
|
+
None, width, height, 8, 0, color_space,
|
|
197
|
+
Quartz.kCGImageAlphaPremultipliedLast,
|
|
198
|
+
)
|
|
199
|
+
# Scans assume paper: fill white first, else transparent areas read as black.
|
|
200
|
+
Quartz.CGContextSetRGBFillColor(ctx, 1.0, 1.0, 1.0, 1.0)
|
|
201
|
+
Quartz.CGContextFillRect(ctx, Quartz.CGRectMake(0, 0, width, height))
|
|
202
|
+
Quartz.CGContextScaleCTM(ctx, scale, scale)
|
|
203
|
+
Quartz.CGContextDrawPDFPage(ctx, page)
|
|
204
|
+
return Quartz.CGBitmapContextCreateImage(ctx)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _ocr_cgimage(cgimage) -> str:
|
|
208
|
+
"""Recognize text in a CGImage; lines joined top-to-bottom (reading order)."""
|
|
209
|
+
handler = Vision.VNImageRequestHandler.alloc().initWithCGImage_options_(
|
|
210
|
+
cgimage, None
|
|
211
|
+
)
|
|
212
|
+
request = Vision.VNRecognizeTextRequest.alloc().init()
|
|
213
|
+
request.setRecognitionLevel_(Vision.VNRequestTextRecognitionLevelAccurate)
|
|
214
|
+
request.setUsesLanguageCorrection_(True)
|
|
215
|
+
# Serialize with CoreML inference — one ANE, and keeping Vision off the
|
|
216
|
+
# MLModel threadpool avoids contending the same accelerator mid-embed.
|
|
217
|
+
with _INFER_LOCK:
|
|
218
|
+
ok, _ = handler.performRequests_error_([request], None)
|
|
219
|
+
if not ok:
|
|
220
|
+
return ""
|
|
221
|
+
lines = []
|
|
222
|
+
for obs in request.results() or []:
|
|
223
|
+
candidate = obs.topCandidates_(1)
|
|
224
|
+
if candidate and len(candidate):
|
|
225
|
+
lines.append(candidate[0].string())
|
|
226
|
+
return "\n".join(lines)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def ocr_pdf(raw: bytes, dpi: int = OCR_DPI) -> list[str]:
|
|
230
|
+
"""Rasterize + OCR every page of a PDF. Returns one text string per page."""
|
|
231
|
+
data = NSData.dataWithBytes_length_(raw, len(raw))
|
|
232
|
+
provider = Quartz.CGDataProviderCreateWithCFData(data)
|
|
233
|
+
doc = Quartz.CGPDFDocumentCreateWithProvider(provider)
|
|
234
|
+
if doc is None:
|
|
235
|
+
raise ValueError("not a valid PDF")
|
|
236
|
+
scale = dpi / 72.0 # PDF user space is 72 dpi
|
|
237
|
+
pages: list[str] = []
|
|
238
|
+
for i in range(1, Quartz.CGPDFDocumentGetNumberOfPages(doc) + 1):
|
|
239
|
+
page = Quartz.CGPDFDocumentGetPage(doc, i)
|
|
240
|
+
pages.append(_ocr_cgimage(_render_pdf_page(page, scale)))
|
|
241
|
+
return pages
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
@app.get("/health")
|
|
245
|
+
def health():
|
|
246
|
+
return {
|
|
247
|
+
"status": "ok",
|
|
248
|
+
"runtime": "coreml",
|
|
249
|
+
"compute": COMPUTE,
|
|
250
|
+
"model": "dinov2_vitb14",
|
|
251
|
+
"dim": DIM,
|
|
252
|
+
"clip": "MobileCLIP2-B",
|
|
253
|
+
"clip_dim": CLIP_DIM,
|
|
254
|
+
"text": "bge-base-en-v1.5",
|
|
255
|
+
"text_dim": TEXT_DIM,
|
|
256
|
+
"model_loaded": "dino" in state,
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _decode(image: str) -> bytes:
|
|
261
|
+
try:
|
|
262
|
+
return base64.b64decode(image)
|
|
263
|
+
except Exception:
|
|
264
|
+
raise HTTPException(status_code=400, detail="invalid base64")
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
@app.post("/embed", response_model=EmbedResponse, dependencies=[Depends(require_key)])
|
|
268
|
+
def embed(req: EmbedRequest):
|
|
269
|
+
try:
|
|
270
|
+
return EmbedResponse(vector=embed_dino(_decode(req.image)))
|
|
271
|
+
except HTTPException:
|
|
272
|
+
raise
|
|
273
|
+
except Exception as exc:
|
|
274
|
+
raise HTTPException(status_code=422, detail=f"could not embed image: {exc}")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
@app.post("/clip/image", response_model=EmbedResponse, dependencies=[Depends(require_key)])
|
|
278
|
+
def clip_image(req: EmbedRequest):
|
|
279
|
+
try:
|
|
280
|
+
return EmbedResponse(vector=embed_clip_image(_decode(req.image)))
|
|
281
|
+
except HTTPException:
|
|
282
|
+
raise
|
|
283
|
+
except Exception as exc:
|
|
284
|
+
raise HTTPException(status_code=422, detail=f"could not embed image: {exc}")
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
@app.post("/clip/text", response_model=EmbedResponse, dependencies=[Depends(require_key)])
|
|
288
|
+
def clip_text(req: TextRequest):
|
|
289
|
+
return EmbedResponse(vector=embed_clip_text(req.text))
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
@app.post("/text", response_model=EmbedResponse, dependencies=[Depends(require_key)])
|
|
293
|
+
def text(req: TextRequest):
|
|
294
|
+
return EmbedResponse(vector=embed_text(req.text))
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
class ExtractRequest(BaseModel):
|
|
298
|
+
file_b64: str # base64-encoded file bytes
|
|
299
|
+
ext: str # file extension (lowercase, no dot) — "pdf" for now
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
class ExtractPage(BaseModel):
|
|
303
|
+
text: str
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
class ExtractResponse(BaseModel):
|
|
307
|
+
pages: list[ExtractPage]
|
|
308
|
+
engine: str # which OCR path produced the text ("vision")
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
@app.post("/extract", response_model=ExtractResponse, dependencies=[Depends(require_key)])
|
|
312
|
+
def extract(req: ExtractRequest):
|
|
313
|
+
"""OCR a document the server couldn't read a text layer from (scanned PDF)."""
|
|
314
|
+
if not _VISION_OK:
|
|
315
|
+
raise HTTPException(
|
|
316
|
+
status_code=503,
|
|
317
|
+
detail="OCR unavailable: pyobjc Vision/Quartz not installed",
|
|
318
|
+
)
|
|
319
|
+
ext = req.ext.lower().lstrip(".")
|
|
320
|
+
if ext != "pdf":
|
|
321
|
+
raise HTTPException(status_code=415, detail=f"cannot OCR type: {ext}")
|
|
322
|
+
raw = _decode(req.file_b64)
|
|
323
|
+
try:
|
|
324
|
+
pages = ocr_pdf(raw)
|
|
325
|
+
except HTTPException:
|
|
326
|
+
raise
|
|
327
|
+
except Exception as exc:
|
|
328
|
+
raise HTTPException(status_code=422, detail=f"could not OCR pdf: {exc}")
|
|
329
|
+
return ExtractResponse(
|
|
330
|
+
pages=[ExtractPage(text=t) for t in pages], engine="vision"
|
|
331
|
+
)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Torch-free CLIP BPE tokenizer — a verbatim port of open_clip's
|
|
2
|
+
SimpleTokenizer (clean='lower', context_length 77), returning an int32 numpy
|
|
3
|
+
array instead of a torch tensor. Lets the service tokenize search queries
|
|
4
|
+
without importing open_clip/torch. Vocab file (clip_vocab.txt.gz) is copied from
|
|
5
|
+
open_clip by convert_clip.py.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import gzip
|
|
9
|
+
import html
|
|
10
|
+
import os
|
|
11
|
+
from functools import lru_cache
|
|
12
|
+
|
|
13
|
+
import ftfy
|
|
14
|
+
import numpy as np
|
|
15
|
+
import regex as re
|
|
16
|
+
|
|
17
|
+
VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "clip_vocab.txt.gz")
|
|
18
|
+
CONTEXT_LENGTH = 77
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@lru_cache()
|
|
22
|
+
def bytes_to_unicode():
|
|
23
|
+
bs = (
|
|
24
|
+
list(range(ord("!"), ord("~") + 1))
|
|
25
|
+
+ list(range(ord("¡"), ord("¬") + 1))
|
|
26
|
+
+ list(range(ord("®"), ord("ÿ") + 1))
|
|
27
|
+
)
|
|
28
|
+
cs = bs[:]
|
|
29
|
+
n = 0
|
|
30
|
+
for b in range(2**8):
|
|
31
|
+
if b not in bs:
|
|
32
|
+
bs.append(b)
|
|
33
|
+
cs.append(2**8 + n)
|
|
34
|
+
n += 1
|
|
35
|
+
cs = [chr(n) for n in cs]
|
|
36
|
+
return dict(zip(bs, cs))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def get_pairs(word):
|
|
40
|
+
pairs = set()
|
|
41
|
+
prev_char = word[0]
|
|
42
|
+
for char in word[1:]:
|
|
43
|
+
pairs.add((prev_char, char))
|
|
44
|
+
prev_char = char
|
|
45
|
+
return pairs
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def basic_clean(text):
|
|
49
|
+
text = ftfy.fix_text(text)
|
|
50
|
+
text = html.unescape(html.unescape(text))
|
|
51
|
+
return text.strip()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def whitespace_clean(text):
|
|
55
|
+
return " ".join(text.split()).strip()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ClipTokenizer:
|
|
59
|
+
def __init__(self, bpe_path: str = VOCAB, context_length: int = CONTEXT_LENGTH):
|
|
60
|
+
self.byte_encoder = bytes_to_unicode()
|
|
61
|
+
merges = gzip.open(bpe_path).read().decode("utf-8").split("\n")
|
|
62
|
+
merges = merges[1 : 49152 - 256 - 2 + 1]
|
|
63
|
+
merges = [tuple(merge.split()) for merge in merges]
|
|
64
|
+
vocab = list(bytes_to_unicode().values())
|
|
65
|
+
vocab = vocab + [v + "</w>" for v in vocab]
|
|
66
|
+
for merge in merges:
|
|
67
|
+
vocab.append("".join(merge))
|
|
68
|
+
special_tokens = ["<start_of_text>", "<end_of_text>"]
|
|
69
|
+
vocab.extend(special_tokens)
|
|
70
|
+
self.encoder = dict(zip(vocab, range(len(vocab))))
|
|
71
|
+
self.bpe_ranks = dict(zip(merges, range(len(merges))))
|
|
72
|
+
self.cache = {t: t for t in special_tokens}
|
|
73
|
+
special = "|".join(special_tokens)
|
|
74
|
+
self.pat = re.compile(
|
|
75
|
+
special
|
|
76
|
+
+ r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
|
|
77
|
+
re.IGNORECASE,
|
|
78
|
+
)
|
|
79
|
+
self.sot_token_id = self.encoder[special_tokens[0]]
|
|
80
|
+
self.eot_token_id = self.encoder[special_tokens[1]]
|
|
81
|
+
self.context_length = context_length
|
|
82
|
+
|
|
83
|
+
def bpe(self, token):
|
|
84
|
+
if token in self.cache:
|
|
85
|
+
return self.cache[token]
|
|
86
|
+
word = tuple(token[:-1]) + (token[-1] + "</w>",)
|
|
87
|
+
pairs = get_pairs(word)
|
|
88
|
+
if not pairs:
|
|
89
|
+
return token + "</w>"
|
|
90
|
+
while True:
|
|
91
|
+
bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
|
|
92
|
+
if bigram not in self.bpe_ranks:
|
|
93
|
+
break
|
|
94
|
+
first, second = bigram
|
|
95
|
+
new_word = []
|
|
96
|
+
i = 0
|
|
97
|
+
while i < len(word):
|
|
98
|
+
try:
|
|
99
|
+
j = word.index(first, i)
|
|
100
|
+
new_word.extend(word[i:j])
|
|
101
|
+
i = j
|
|
102
|
+
except Exception:
|
|
103
|
+
new_word.extend(word[i:])
|
|
104
|
+
break
|
|
105
|
+
if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
|
|
106
|
+
new_word.append(first + second)
|
|
107
|
+
i += 2
|
|
108
|
+
else:
|
|
109
|
+
new_word.append(word[i])
|
|
110
|
+
i += 1
|
|
111
|
+
new_word = tuple(new_word)
|
|
112
|
+
word = new_word
|
|
113
|
+
if len(word) == 1:
|
|
114
|
+
break
|
|
115
|
+
pairs = get_pairs(word)
|
|
116
|
+
word = " ".join(word)
|
|
117
|
+
self.cache[token] = word
|
|
118
|
+
return word
|
|
119
|
+
|
|
120
|
+
def encode(self, text):
|
|
121
|
+
bpe_tokens = []
|
|
122
|
+
text = whitespace_clean(basic_clean(text)).lower()
|
|
123
|
+
for token in re.findall(self.pat, text):
|
|
124
|
+
token = "".join(self.byte_encoder[b] for b in token.encode("utf-8"))
|
|
125
|
+
bpe_tokens.extend(
|
|
126
|
+
self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")
|
|
127
|
+
)
|
|
128
|
+
return bpe_tokens
|
|
129
|
+
|
|
130
|
+
def __call__(self, text: str) -> np.ndarray:
|
|
131
|
+
tokens = [self.sot_token_id] + self.encode(text) + [self.eot_token_id]
|
|
132
|
+
result = np.zeros(self.context_length, dtype=np.int32)
|
|
133
|
+
if len(tokens) > self.context_length:
|
|
134
|
+
tokens = tokens[: self.context_length]
|
|
135
|
+
tokens[-1] = self.eot_token_id
|
|
136
|
+
result[: len(tokens)] = tokens
|
|
137
|
+
return result[None] # [1, context_length]
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Convert CLIP (open_clip MobileCLIP2-B) image + text encoders to Core ML (fp16)
|
|
2
|
+
so the search path runs torch-free on the ANE, like DINOv2. Verifies parity vs
|
|
3
|
+
torch and precompiles/warms the .mlmodelc. MobileCLIP2-B is a drop-in for ViT-B-32
|
|
4
|
+
(same 512-d / 224px / BPE) but stronger; swapping it means re-embedding media.
|
|
5
|
+
|
|
6
|
+
./.venv/bin/python convert_clip.py
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import shutil
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import torch
|
|
15
|
+
import torch.nn.functional as F
|
|
16
|
+
|
|
17
|
+
import coremltools as ct
|
|
18
|
+
import open_clip
|
|
19
|
+
from coremltools.models import CompiledMLModel
|
|
20
|
+
from coremltools.models.utils import compile_model
|
|
21
|
+
|
|
22
|
+
MODEL, PRETRAINED = "MobileCLIP2-B", "dfndr2b"
|
|
23
|
+
CTX = 77
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ImageEncoder(torch.nn.Module):
|
|
27
|
+
def __init__(self, m):
|
|
28
|
+
super().__init__()
|
|
29
|
+
self.m = m
|
|
30
|
+
|
|
31
|
+
def forward(self, x):
|
|
32
|
+
return self.m.encode_image(x)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TextEncoder(torch.nn.Module):
|
|
36
|
+
def __init__(self, m):
|
|
37
|
+
super().__init__()
|
|
38
|
+
self.m = m
|
|
39
|
+
|
|
40
|
+
def forward(self, t):
|
|
41
|
+
return self.m.encode_text(t)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def to_coreml(wrapper, example, name, input_spec):
|
|
45
|
+
exported = torch.export.export(wrapper, (example,)).run_decompositions({})
|
|
46
|
+
mlmodel = ct.convert(
|
|
47
|
+
exported,
|
|
48
|
+
inputs=[input_spec],
|
|
49
|
+
outputs=[ct.TensorType(name="embedding")],
|
|
50
|
+
compute_units=ct.ComputeUnit.ALL,
|
|
51
|
+
compute_precision=ct.precision.FLOAT16,
|
|
52
|
+
minimum_deployment_target=ct.target.macOS14,
|
|
53
|
+
)
|
|
54
|
+
pkg = f"{name}.mlpackage"
|
|
55
|
+
mlmodel.save(pkg)
|
|
56
|
+
return mlmodel, pkg
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def parity(mlmodel, input_np, ref, name):
|
|
60
|
+
out = mlmodel.predict(
|
|
61
|
+
{mlmodel.get_spec().description.input[0].name: input_np}
|
|
62
|
+
)
|
|
63
|
+
vec = torch.tensor(out["embedding"]).flatten().float()
|
|
64
|
+
cos = F.cosine_similarity(vec, ref.flatten().float(), dim=0).item()
|
|
65
|
+
print(f"{name}: cosine(coreml, torch) = {cos:.5f}")
|
|
66
|
+
return cos
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def warm(pkg, name):
|
|
70
|
+
compiled = f"{name}.mlmodelc"
|
|
71
|
+
if os.path.exists(compiled):
|
|
72
|
+
shutil.rmtree(compiled)
|
|
73
|
+
compile_model(pkg, compiled)
|
|
74
|
+
CompiledMLModel(compiled, compute_units=ct.ComputeUnit.CPU_AND_NE)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def main() -> int:
|
|
78
|
+
model, _, _ = open_clip.create_model_and_transforms(MODEL, pretrained=PRETRAINED)
|
|
79
|
+
model.eval()
|
|
80
|
+
tok = open_clip.get_tokenizer(MODEL)
|
|
81
|
+
|
|
82
|
+
# Bundle the BPE vocab for the torch-free runtime tokenizer (clip_tokenizer.py).
|
|
83
|
+
from open_clip import tokenizer as _tk
|
|
84
|
+
|
|
85
|
+
shutil.copy(_tk.default_bpe(), "clip_vocab.txt.gz")
|
|
86
|
+
|
|
87
|
+
img_ex = torch.randn(1, 3, 224, 224)
|
|
88
|
+
txt_ex = tok(["a photo of a dog"]) # int64 [1, 77]
|
|
89
|
+
|
|
90
|
+
with torch.no_grad():
|
|
91
|
+
img_ref = model.encode_image(img_ex)
|
|
92
|
+
txt_ref = model.encode_text(txt_ex)
|
|
93
|
+
|
|
94
|
+
print("converting CLIP image encoder...")
|
|
95
|
+
img_ml, img_pkg = to_coreml(
|
|
96
|
+
ImageEncoder(model), img_ex, "clip_image",
|
|
97
|
+
ct.TensorType(name="image", shape=(1, 3, 224, 224)),
|
|
98
|
+
)
|
|
99
|
+
ok_i = parity(img_ml, img_ex.numpy(), img_ref, "image") > 0.99
|
|
100
|
+
|
|
101
|
+
print("converting CLIP text encoder...")
|
|
102
|
+
txt_ml, txt_pkg = to_coreml(
|
|
103
|
+
TextEncoder(model), txt_ex, "clip_text",
|
|
104
|
+
ct.TensorType(name="tokens", shape=(1, CTX), dtype=np.int32),
|
|
105
|
+
)
|
|
106
|
+
ok_t = parity(txt_ml, txt_ex.numpy().astype(np.int32), txt_ref, "text") > 0.99
|
|
107
|
+
|
|
108
|
+
if not (ok_i and ok_t):
|
|
109
|
+
print("PARITY FAILED")
|
|
110
|
+
return 1
|
|
111
|
+
print("warming ANE caches...")
|
|
112
|
+
warm(img_pkg, "clip_image")
|
|
113
|
+
warm(txt_pkg, "clip_text")
|
|
114
|
+
print("done")
|
|
115
|
+
return 0
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
if __name__ == "__main__":
|
|
119
|
+
sys.exit(main())
|