@factiii/runner 0.27.0 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,331 @@
1
+ import base64
2
+ import io
3
+ import logging
4
+ import os
5
+ import threading
6
+ import time
7
+ from contextlib import asynccontextmanager
8
+
9
+ import numpy as np
10
+ from fastapi import Depends, FastAPI, Header, HTTPException, Request
11
+ from PIL import Image
12
+ from pydantic import BaseModel
13
+
14
+ import coremltools as ct
15
+ from clip_tokenizer import ClipTokenizer
16
+ from coremltools.models import CompiledMLModel
17
+ from text_tokenizer import TextTokenizer
18
+
19
+ # OCR for scanned PDFs (file-vault /extract) uses native macOS frameworks via
20
+ # pyobjc — CoreGraphics rasterizes each PDF page, Apple Vision recognizes text.
21
+ # Guarded so the embedder still serves embeddings if pyobjc isn't installed;
22
+ # /extract then returns 503 rather than crashing startup.
23
+ try:
24
+ import Quartz
25
+ import Vision
26
+ from Foundation import NSData
27
+
28
+ _VISION_OK = True
29
+ except Exception: # noqa: BLE001 — any import failure disables OCR, not the service
30
+ _VISION_OK = False
31
+
32
+ # Config comes from the environment only. start.sh copies this service's two
33
+ # keys out of the monorepo root .env (.specs/env.md). This file never opens a
34
+ # .env itself: the runner ships it in its npm package (apps/runner embedder/),
35
+ # and npm pulled runner 0.28.0 and 0.33.0 as malware while it read one.
36
+
37
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
38
+ logger = logging.getLogger("embedder")
39
+
40
+ # Shared secret: the server sends it as `x-api-key`. When set, requests without
41
+ # it are rejected so only our server (which has the key) can use the embedder.
42
+ API_KEY = os.environ.get("EMBEDDER_API_KEY")
43
+ if not API_KEY:
44
+ logger.warning("EMBEDDER_API_KEY not set — embedder is UNSECURED (open to anyone).")
45
+
46
+
47
+ def require_key(x_api_key: str | None = Header(default=None)):
48
+ if API_KEY and x_api_key != API_KEY:
49
+ raise HTTPException(status_code=401, detail="invalid or missing api key")
50
+
51
+ # Three models, all Core ML / Apple Neural Engine, torch-free (~0.5GB serving):
52
+ # DINOv2 (768-d) grouping, MobileCLIP2-B (512-d) text<->image, bge-base-en-v1.5 (768-d) text<->text.
53
+
54
+ DIM = 768
55
+ CLIP_DIM = 512
56
+ TEXT_DIM = 768
57
+
58
+ _UNITS = {
59
+ "ne": ct.ComputeUnit.CPU_AND_NE, # Neural Engine (default — fastest, leanest)
60
+ "gpu": ct.ComputeUnit.CPU_AND_GPU,
61
+ "cpu": ct.ComputeUnit.CPU_ONLY,
62
+ "all": ct.ComputeUnit.ALL,
63
+ }
64
+ COMPUTE = os.environ.get("COREML_UNITS", "ne")
65
+
66
+ # DINOv2 preprocessing: resize shorter edge to 256 (bilinear), center-crop 224,
67
+ # scale to [0,1], ImageNet normalize.
68
+ _DINO_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
69
+ _DINO_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
70
+ # CLIP preprocessing: 224 bilinear, center-crop, scale to [0,1]. MobileCLIP2-B/dfndr2b
71
+ # uses NO normalization (mean 0/std 1) — wrong values yield garbage image vectors.
72
+ _CLIP_MEAN = np.array([0.0, 0.0, 0.0], dtype=np.float32)
73
+ _CLIP_STD = np.array([1.0, 1.0, 1.0], dtype=np.float32)
74
+
75
+ state: dict = {}
76
+
77
+
78
+ def _load(name: str):
79
+ units = _UNITS[COMPUTE]
80
+ if os.path.isdir(f"{name}.mlmodelc"):
81
+ return CompiledMLModel(f"{name}.mlmodelc", compute_units=units)
82
+ # Fallback to the raw package (slow start; run convert*.py to precompile).
83
+ return ct.models.MLModel(f"{name}.mlpackage", compute_units=units)
84
+
85
+
86
+ @asynccontextmanager
87
+ async def lifespan(_: FastAPI):
88
+ state["dino"] = _load("model")
89
+ state["clip_image"] = _load("clip_image")
90
+ state["clip_text"] = _load("clip_text")
91
+ state["clip_tok"] = ClipTokenizer()
92
+ state["text"] = _load("text")
93
+ state["text_tok"] = TextTokenizer()
94
+ yield
95
+ state.clear()
96
+
97
+
98
+ app = FastAPI(lifespan=lifespan)
99
+
100
+
101
+ # Log every request: method, path, status, latency.
102
+ @app.middleware("http")
103
+ async def log_requests(request: Request, call_next):
104
+ start = time.perf_counter()
105
+ response = await call_next(request)
106
+ took = (time.perf_counter() - start) * 1000
107
+ logger.info(
108
+ "%s %s -> %s %.0fms",
109
+ request.method,
110
+ request.url.path,
111
+ response.status_code,
112
+ took
113
+ )
114
+ return response
115
+
116
+
117
+ class EmbedRequest(BaseModel):
118
+ image: str # base64-encoded image bytes
119
+
120
+
121
+ class EmbedResponse(BaseModel):
122
+ vector: list[float]
123
+
124
+
125
+ class TextRequest(BaseModel):
126
+ text: str
127
+
128
+
129
+ def preprocess(raw: bytes, resize: int, mean: np.ndarray, std: np.ndarray) -> np.ndarray:
130
+ img = Image.open(io.BytesIO(raw)).convert("RGB")
131
+ w, h = img.size
132
+ scale = resize / min(w, h)
133
+ # Both image models (DINOv2 and MobileCLIP2-B) use bilinear resize.
134
+ img = img.resize((round(w * scale), round(h * scale)), Image.BILINEAR)
135
+ nw, nh = img.size
136
+ crop = 224
137
+ left, top = (nw - crop) // 2, (nh - crop) // 2
138
+ img = img.crop((left, top, left + crop, top + crop))
139
+ arr = (np.asarray(img, dtype=np.float32) / 255.0 - mean) / std # HWC
140
+ return arr.transpose(2, 0, 1)[None].astype(np.float32) # 1,C,H,W
141
+
142
+
143
+ def l2(arr: np.ndarray) -> list[float]:
144
+ vec = np.asarray(arr, dtype=np.float32).flatten()
145
+ vec /= np.linalg.norm(vec) or 1.0
146
+ return vec.tolist()
147
+
148
+
149
+ # Serialize ALL CoreML inference. Sync endpoints run in Starlette's threadpool, so
150
+ # concurrent requests would call .predict() on the shared MLModels from multiple
151
+ # threads at once — the ANE's MLE5 execution stream is not safe under that and
152
+ # segfaults during stream teardown (CoreML -[MLE5ExecutionStream resetAfterLingering]).
153
+ # One ANE serializes the real work anyway, so a single global lock costs ~nothing.
154
+ _INFER_LOCK = threading.Lock()
155
+
156
+
157
+ def embed_dino(raw: bytes) -> list[float]:
158
+ batch = preprocess(raw, 256, _DINO_MEAN, _DINO_STD)
159
+ with _INFER_LOCK:
160
+ return l2(state["dino"].predict({"image": batch})["embedding"])
161
+
162
+
163
+ def embed_clip_image(raw: bytes) -> list[float]:
164
+ batch = preprocess(raw, 224, _CLIP_MEAN, _CLIP_STD)
165
+ with _INFER_LOCK:
166
+ return l2(state["clip_image"].predict({"image": batch})["embedding"])
167
+
168
+
169
+ def embed_clip_text(text: str) -> list[float]:
170
+ tokens = state["clip_tok"](text)
171
+ with _INFER_LOCK:
172
+ return l2(state["clip_text"].predict({"tokens": tokens})["embedding"])
173
+
174
+
175
+ def embed_text(text: str) -> list[float]:
176
+ ids, mask = state["text_tok"](text)
177
+ with _INFER_LOCK:
178
+ out = state["text"].predict({"input_ids": ids, "attention_mask": mask})
179
+ # The graph already CLS-pools + normalizes (bge); l2 is a harmless re-normalize.
180
+ return l2(out["embedding"])
181
+
182
+
183
+ # --- OCR (Apple Vision) for scanned PDFs -------------------------------------
184
+ # Render at 200 DPI: enough resolution for Vision to read body text on a scan
185
+ # without ballooning bitmap memory (a US-Letter page is ~1700x2200 px here).
186
+ OCR_DPI = 200
187
+
188
+
189
+ def _render_pdf_page(page, scale: float):
190
+ """CGPDFPage -> white-backed RGB CGImage at `scale`× its native size."""
191
+ rect = Quartz.CGPDFPageGetBoxRect(page, Quartz.kCGPDFMediaBox)
192
+ width = max(1, int(rect.size.width * scale))
193
+ height = max(1, int(rect.size.height * scale))
194
+ color_space = Quartz.CGColorSpaceCreateDeviceRGB()
195
+ ctx = Quartz.CGBitmapContextCreate(
196
+ None, width, height, 8, 0, color_space,
197
+ Quartz.kCGImageAlphaPremultipliedLast,
198
+ )
199
+ # Scans assume paper: fill white first, else transparent areas read as black.
200
+ Quartz.CGContextSetRGBFillColor(ctx, 1.0, 1.0, 1.0, 1.0)
201
+ Quartz.CGContextFillRect(ctx, Quartz.CGRectMake(0, 0, width, height))
202
+ Quartz.CGContextScaleCTM(ctx, scale, scale)
203
+ Quartz.CGContextDrawPDFPage(ctx, page)
204
+ return Quartz.CGBitmapContextCreateImage(ctx)
205
+
206
+
207
+ def _ocr_cgimage(cgimage) -> str:
208
+ """Recognize text in a CGImage; lines joined top-to-bottom (reading order)."""
209
+ handler = Vision.VNImageRequestHandler.alloc().initWithCGImage_options_(
210
+ cgimage, None
211
+ )
212
+ request = Vision.VNRecognizeTextRequest.alloc().init()
213
+ request.setRecognitionLevel_(Vision.VNRequestTextRecognitionLevelAccurate)
214
+ request.setUsesLanguageCorrection_(True)
215
+ # Serialize with CoreML inference — one ANE, and keeping Vision off the
216
+ # MLModel threadpool avoids contending the same accelerator mid-embed.
217
+ with _INFER_LOCK:
218
+ ok, _ = handler.performRequests_error_([request], None)
219
+ if not ok:
220
+ return ""
221
+ lines = []
222
+ for obs in request.results() or []:
223
+ candidate = obs.topCandidates_(1)
224
+ if candidate and len(candidate):
225
+ lines.append(candidate[0].string())
226
+ return "\n".join(lines)
227
+
228
+
229
+ def ocr_pdf(raw: bytes, dpi: int = OCR_DPI) -> list[str]:
230
+ """Rasterize + OCR every page of a PDF. Returns one text string per page."""
231
+ data = NSData.dataWithBytes_length_(raw, len(raw))
232
+ provider = Quartz.CGDataProviderCreateWithCFData(data)
233
+ doc = Quartz.CGPDFDocumentCreateWithProvider(provider)
234
+ if doc is None:
235
+ raise ValueError("not a valid PDF")
236
+ scale = dpi / 72.0 # PDF user space is 72 dpi
237
+ pages: list[str] = []
238
+ for i in range(1, Quartz.CGPDFDocumentGetNumberOfPages(doc) + 1):
239
+ page = Quartz.CGPDFDocumentGetPage(doc, i)
240
+ pages.append(_ocr_cgimage(_render_pdf_page(page, scale)))
241
+ return pages
242
+
243
+
244
+ @app.get("/health")
245
+ def health():
246
+ return {
247
+ "status": "ok",
248
+ "runtime": "coreml",
249
+ "compute": COMPUTE,
250
+ "model": "dinov2_vitb14",
251
+ "dim": DIM,
252
+ "clip": "MobileCLIP2-B",
253
+ "clip_dim": CLIP_DIM,
254
+ "text": "bge-base-en-v1.5",
255
+ "text_dim": TEXT_DIM,
256
+ "model_loaded": "dino" in state,
257
+ }
258
+
259
+
260
+ def _decode(image: str) -> bytes:
261
+ try:
262
+ return base64.b64decode(image)
263
+ except Exception:
264
+ raise HTTPException(status_code=400, detail="invalid base64")
265
+
266
+
267
+ @app.post("/embed", response_model=EmbedResponse, dependencies=[Depends(require_key)])
268
+ def embed(req: EmbedRequest):
269
+ try:
270
+ return EmbedResponse(vector=embed_dino(_decode(req.image)))
271
+ except HTTPException:
272
+ raise
273
+ except Exception as exc:
274
+ raise HTTPException(status_code=422, detail=f"could not embed image: {exc}")
275
+
276
+
277
+ @app.post("/clip/image", response_model=EmbedResponse, dependencies=[Depends(require_key)])
278
+ def clip_image(req: EmbedRequest):
279
+ try:
280
+ return EmbedResponse(vector=embed_clip_image(_decode(req.image)))
281
+ except HTTPException:
282
+ raise
283
+ except Exception as exc:
284
+ raise HTTPException(status_code=422, detail=f"could not embed image: {exc}")
285
+
286
+
287
+ @app.post("/clip/text", response_model=EmbedResponse, dependencies=[Depends(require_key)])
288
+ def clip_text(req: TextRequest):
289
+ return EmbedResponse(vector=embed_clip_text(req.text))
290
+
291
+
292
+ @app.post("/text", response_model=EmbedResponse, dependencies=[Depends(require_key)])
293
+ def text(req: TextRequest):
294
+ return EmbedResponse(vector=embed_text(req.text))
295
+
296
+
297
+ class ExtractRequest(BaseModel):
298
+ file_b64: str # base64-encoded file bytes
299
+ ext: str # file extension (lowercase, no dot) — "pdf" for now
300
+
301
+
302
+ class ExtractPage(BaseModel):
303
+ text: str
304
+
305
+
306
+ class ExtractResponse(BaseModel):
307
+ pages: list[ExtractPage]
308
+ engine: str # which OCR path produced the text ("vision")
309
+
310
+
311
+ @app.post("/extract", response_model=ExtractResponse, dependencies=[Depends(require_key)])
312
+ def extract(req: ExtractRequest):
313
+ """OCR a document the server couldn't read a text layer from (scanned PDF)."""
314
+ if not _VISION_OK:
315
+ raise HTTPException(
316
+ status_code=503,
317
+ detail="OCR unavailable: pyobjc Vision/Quartz not installed",
318
+ )
319
+ ext = req.ext.lower().lstrip(".")
320
+ if ext != "pdf":
321
+ raise HTTPException(status_code=415, detail=f"cannot OCR type: {ext}")
322
+ raw = _decode(req.file_b64)
323
+ try:
324
+ pages = ocr_pdf(raw)
325
+ except HTTPException:
326
+ raise
327
+ except Exception as exc:
328
+ raise HTTPException(status_code=422, detail=f"could not OCR pdf: {exc}")
329
+ return ExtractResponse(
330
+ pages=[ExtractPage(text=t) for t in pages], engine="vision"
331
+ )
@@ -0,0 +1,137 @@
1
+ """Torch-free CLIP BPE tokenizer — a verbatim port of open_clip's
2
+ SimpleTokenizer (clean='lower', context_length 77), returning an int32 numpy
3
+ array instead of a torch tensor. Lets the service tokenize search queries
4
+ without importing open_clip/torch. Vocab file (clip_vocab.txt.gz) is copied from
5
+ open_clip by convert_clip.py.
6
+ """
7
+
8
+ import gzip
9
+ import html
10
+ import os
11
+ from functools import lru_cache
12
+
13
+ import ftfy
14
+ import numpy as np
15
+ import regex as re
16
+
17
+ VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "clip_vocab.txt.gz")
18
+ CONTEXT_LENGTH = 77
19
+
20
+
21
+ @lru_cache()
22
+ def bytes_to_unicode():
23
+ bs = (
24
+ list(range(ord("!"), ord("~") + 1))
25
+ + list(range(ord("¡"), ord("¬") + 1))
26
+ + list(range(ord("®"), ord("ÿ") + 1))
27
+ )
28
+ cs = bs[:]
29
+ n = 0
30
+ for b in range(2**8):
31
+ if b not in bs:
32
+ bs.append(b)
33
+ cs.append(2**8 + n)
34
+ n += 1
35
+ cs = [chr(n) for n in cs]
36
+ return dict(zip(bs, cs))
37
+
38
+
39
+ def get_pairs(word):
40
+ pairs = set()
41
+ prev_char = word[0]
42
+ for char in word[1:]:
43
+ pairs.add((prev_char, char))
44
+ prev_char = char
45
+ return pairs
46
+
47
+
48
+ def basic_clean(text):
49
+ text = ftfy.fix_text(text)
50
+ text = html.unescape(html.unescape(text))
51
+ return text.strip()
52
+
53
+
54
+ def whitespace_clean(text):
55
+ return " ".join(text.split()).strip()
56
+
57
+
58
+ class ClipTokenizer:
59
+ def __init__(self, bpe_path: str = VOCAB, context_length: int = CONTEXT_LENGTH):
60
+ self.byte_encoder = bytes_to_unicode()
61
+ merges = gzip.open(bpe_path).read().decode("utf-8").split("\n")
62
+ merges = merges[1 : 49152 - 256 - 2 + 1]
63
+ merges = [tuple(merge.split()) for merge in merges]
64
+ vocab = list(bytes_to_unicode().values())
65
+ vocab = vocab + [v + "</w>" for v in vocab]
66
+ for merge in merges:
67
+ vocab.append("".join(merge))
68
+ special_tokens = ["<start_of_text>", "<end_of_text>"]
69
+ vocab.extend(special_tokens)
70
+ self.encoder = dict(zip(vocab, range(len(vocab))))
71
+ self.bpe_ranks = dict(zip(merges, range(len(merges))))
72
+ self.cache = {t: t for t in special_tokens}
73
+ special = "|".join(special_tokens)
74
+ self.pat = re.compile(
75
+ special
76
+ + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
77
+ re.IGNORECASE,
78
+ )
79
+ self.sot_token_id = self.encoder[special_tokens[0]]
80
+ self.eot_token_id = self.encoder[special_tokens[1]]
81
+ self.context_length = context_length
82
+
83
+ def bpe(self, token):
84
+ if token in self.cache:
85
+ return self.cache[token]
86
+ word = tuple(token[:-1]) + (token[-1] + "</w>",)
87
+ pairs = get_pairs(word)
88
+ if not pairs:
89
+ return token + "</w>"
90
+ while True:
91
+ bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
92
+ if bigram not in self.bpe_ranks:
93
+ break
94
+ first, second = bigram
95
+ new_word = []
96
+ i = 0
97
+ while i < len(word):
98
+ try:
99
+ j = word.index(first, i)
100
+ new_word.extend(word[i:j])
101
+ i = j
102
+ except Exception:
103
+ new_word.extend(word[i:])
104
+ break
105
+ if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
106
+ new_word.append(first + second)
107
+ i += 2
108
+ else:
109
+ new_word.append(word[i])
110
+ i += 1
111
+ new_word = tuple(new_word)
112
+ word = new_word
113
+ if len(word) == 1:
114
+ break
115
+ pairs = get_pairs(word)
116
+ word = " ".join(word)
117
+ self.cache[token] = word
118
+ return word
119
+
120
+ def encode(self, text):
121
+ bpe_tokens = []
122
+ text = whitespace_clean(basic_clean(text)).lower()
123
+ for token in re.findall(self.pat, text):
124
+ token = "".join(self.byte_encoder[b] for b in token.encode("utf-8"))
125
+ bpe_tokens.extend(
126
+ self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")
127
+ )
128
+ return bpe_tokens
129
+
130
+ def __call__(self, text: str) -> np.ndarray:
131
+ tokens = [self.sot_token_id] + self.encode(text) + [self.eot_token_id]
132
+ result = np.zeros(self.context_length, dtype=np.int32)
133
+ if len(tokens) > self.context_length:
134
+ tokens = tokens[: self.context_length]
135
+ tokens[-1] = self.eot_token_id
136
+ result[: len(tokens)] = tokens
137
+ return result[None] # [1, context_length]
@@ -0,0 +1,119 @@
1
+ """Convert CLIP (open_clip MobileCLIP2-B) image + text encoders to Core ML (fp16)
2
+ so the search path runs torch-free on the ANE, like DINOv2. Verifies parity vs
3
+ torch and precompiles/warms the .mlmodelc. MobileCLIP2-B is a drop-in for ViT-B-32
4
+ (same 512-d / 224px / BPE) but stronger; swapping it means re-embedding media.
5
+
6
+ ./.venv/bin/python convert_clip.py
7
+ """
8
+
9
+ import os
10
+ import shutil
11
+ import sys
12
+
13
+ import numpy as np
14
+ import torch
15
+ import torch.nn.functional as F
16
+
17
+ import coremltools as ct
18
+ import open_clip
19
+ from coremltools.models import CompiledMLModel
20
+ from coremltools.models.utils import compile_model
21
+
22
+ MODEL, PRETRAINED = "MobileCLIP2-B", "dfndr2b"
23
+ CTX = 77
24
+
25
+
26
+ class ImageEncoder(torch.nn.Module):
27
+ def __init__(self, m):
28
+ super().__init__()
29
+ self.m = m
30
+
31
+ def forward(self, x):
32
+ return self.m.encode_image(x)
33
+
34
+
35
+ class TextEncoder(torch.nn.Module):
36
+ def __init__(self, m):
37
+ super().__init__()
38
+ self.m = m
39
+
40
+ def forward(self, t):
41
+ return self.m.encode_text(t)
42
+
43
+
44
+ def to_coreml(wrapper, example, name, input_spec):
45
+ exported = torch.export.export(wrapper, (example,)).run_decompositions({})
46
+ mlmodel = ct.convert(
47
+ exported,
48
+ inputs=[input_spec],
49
+ outputs=[ct.TensorType(name="embedding")],
50
+ compute_units=ct.ComputeUnit.ALL,
51
+ compute_precision=ct.precision.FLOAT16,
52
+ minimum_deployment_target=ct.target.macOS14,
53
+ )
54
+ pkg = f"{name}.mlpackage"
55
+ mlmodel.save(pkg)
56
+ return mlmodel, pkg
57
+
58
+
59
+ def parity(mlmodel, input_np, ref, name):
60
+ out = mlmodel.predict(
61
+ {mlmodel.get_spec().description.input[0].name: input_np}
62
+ )
63
+ vec = torch.tensor(out["embedding"]).flatten().float()
64
+ cos = F.cosine_similarity(vec, ref.flatten().float(), dim=0).item()
65
+ print(f"{name}: cosine(coreml, torch) = {cos:.5f}")
66
+ return cos
67
+
68
+
69
+ def warm(pkg, name):
70
+ compiled = f"{name}.mlmodelc"
71
+ if os.path.exists(compiled):
72
+ shutil.rmtree(compiled)
73
+ compile_model(pkg, compiled)
74
+ CompiledMLModel(compiled, compute_units=ct.ComputeUnit.CPU_AND_NE)
75
+
76
+
77
+ def main() -> int:
78
+ model, _, _ = open_clip.create_model_and_transforms(MODEL, pretrained=PRETRAINED)
79
+ model.eval()
80
+ tok = open_clip.get_tokenizer(MODEL)
81
+
82
+ # Bundle the BPE vocab for the torch-free runtime tokenizer (clip_tokenizer.py).
83
+ from open_clip import tokenizer as _tk
84
+
85
+ shutil.copy(_tk.default_bpe(), "clip_vocab.txt.gz")
86
+
87
+ img_ex = torch.randn(1, 3, 224, 224)
88
+ txt_ex = tok(["a photo of a dog"]) # int64 [1, 77]
89
+
90
+ with torch.no_grad():
91
+ img_ref = model.encode_image(img_ex)
92
+ txt_ref = model.encode_text(txt_ex)
93
+
94
+ print("converting CLIP image encoder...")
95
+ img_ml, img_pkg = to_coreml(
96
+ ImageEncoder(model), img_ex, "clip_image",
97
+ ct.TensorType(name="image", shape=(1, 3, 224, 224)),
98
+ )
99
+ ok_i = parity(img_ml, img_ex.numpy(), img_ref, "image") > 0.99
100
+
101
+ print("converting CLIP text encoder...")
102
+ txt_ml, txt_pkg = to_coreml(
103
+ TextEncoder(model), txt_ex, "clip_text",
104
+ ct.TensorType(name="tokens", shape=(1, CTX), dtype=np.int32),
105
+ )
106
+ ok_t = parity(txt_ml, txt_ex.numpy().astype(np.int32), txt_ref, "text") > 0.99
107
+
108
+ if not (ok_i and ok_t):
109
+ print("PARITY FAILED")
110
+ return 1
111
+ print("warming ANE caches...")
112
+ warm(img_pkg, "clip_image")
113
+ warm(txt_pkg, "clip_text")
114
+ print("done")
115
+ return 0
116
+
117
+
118
+ if __name__ == "__main__":
119
+ sys.exit(main())