@melaya/runner 1.0.38 → 1.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/localRagIngest.py +13 -0
- package/localRagIngest.py +13 -0
- package/package.json +1 -1
package/dist/localRagIngest.py
CHANGED
|
@@ -113,6 +113,15 @@ async def _ingest(args) -> int:
|
|
|
113
113
|
dimensions=768,
|
|
114
114
|
)
|
|
115
115
|
embedder.client.base_url = "http://localhost:1234/v1"
|
|
116
|
+
# LM Studio's /v1/embeddings rejects the OpenAI-specific
|
|
117
|
+
# `dimensions` kwarg (only OpenAI text-embedding-3 supports
|
|
118
|
+
# output-dim reduction). Strip it before each call so the
|
|
119
|
+
# request is shaped exactly as LMS expects.
|
|
120
|
+
_orig_create = embedder.client.embeddings.create
|
|
121
|
+
async def _lms_create(**kw):
|
|
122
|
+
kw.pop("dimensions", None)
|
|
123
|
+
return await _orig_create(**kw)
|
|
124
|
+
embedder.client.embeddings.create = _lms_create # type: ignore[assignment]
|
|
116
125
|
else:
|
|
117
126
|
_emit("error", reason=(
|
|
118
127
|
f"refused embedder provider '{args.embedder_provider}' for Mode B "
|
|
@@ -173,6 +182,10 @@ async def _ingest(args) -> int:
|
|
|
173
182
|
ext = f.suffix.lower()
|
|
174
183
|
if ext not in SUPPORTED_EXTS:
|
|
175
184
|
continue
|
|
185
|
+
# macOS AppleDouble resource forks (._foo.docx, ._.DS_Store, …) —
|
|
186
|
+
# they're not real documents and every extractor will choke on them.
|
|
187
|
+
if f.name.startswith("._") or f.name == ".DS_Store":
|
|
188
|
+
continue
|
|
176
189
|
try:
|
|
177
190
|
raw = f.read_bytes()
|
|
178
191
|
except Exception as exc:
|
package/localRagIngest.py
CHANGED
|
@@ -113,6 +113,15 @@ async def _ingest(args) -> int:
|
|
|
113
113
|
dimensions=768,
|
|
114
114
|
)
|
|
115
115
|
embedder.client.base_url = "http://localhost:1234/v1"
|
|
116
|
+
# LM Studio's /v1/embeddings rejects the OpenAI-specific
|
|
117
|
+
# `dimensions` kwarg (only OpenAI text-embedding-3 supports
|
|
118
|
+
# output-dim reduction). Strip it before each call so the
|
|
119
|
+
# request is shaped exactly as LMS expects.
|
|
120
|
+
_orig_create = embedder.client.embeddings.create
|
|
121
|
+
async def _lms_create(**kw):
|
|
122
|
+
kw.pop("dimensions", None)
|
|
123
|
+
return await _orig_create(**kw)
|
|
124
|
+
embedder.client.embeddings.create = _lms_create # type: ignore[assignment]
|
|
116
125
|
else:
|
|
117
126
|
_emit("error", reason=(
|
|
118
127
|
f"refused embedder provider '{args.embedder_provider}' for Mode B "
|
|
@@ -173,6 +182,10 @@ async def _ingest(args) -> int:
|
|
|
173
182
|
ext = f.suffix.lower()
|
|
174
183
|
if ext not in SUPPORTED_EXTS:
|
|
175
184
|
continue
|
|
185
|
+
# macOS AppleDouble resource forks (._foo.docx, ._.DS_Store, …) —
|
|
186
|
+
# they're not real documents and every extractor will choke on them.
|
|
187
|
+
if f.name.startswith("._") or f.name == ".DS_Store":
|
|
188
|
+
continue
|
|
176
189
|
try:
|
|
177
190
|
raw = f.read_bytes()
|
|
178
191
|
except Exception as exc:
|