@cloudgrid-io/mcp 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/corpus/capability-map.md +44 -15
- package/src/corpus/install.md +27 -0
- package/src/corpus/templates/semantic-search/AGENTS.md +216 -0
- package/src/corpus/templates/semantic-search/README.md +75 -0
- package/src/corpus/templates/semantic-search/cloudgrid.yaml +219 -0
- package/src/corpus/templates/semantic-search/index.md +108 -0
- package/src/corpus/templates/semantic-search/services/backend/requirements.txt +7 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/__init__.py +0 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/answer.py +90 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/auth.py +65 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/config.py +110 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/db.py +89 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/embeddings.py +44 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/indexing.py +223 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/main.py +259 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/search.py +222 -0
- package/src/corpus/templates/semantic-search/services/backend/src/app/source.py +259 -0
- package/src/corpus/templates/semantic-search/services/backend/src/main.py +12 -0
- package/src/corpus/templates/semantic-search/services/refresh/README.md +42 -0
- package/src/corpus/templates/semantic-search/services/refresh/requirements.txt +9 -0
- package/src/corpus/templates/semantic-search/services/refresh/src/app/__init__.py +0 -0
- package/src/corpus/templates/semantic-search/services/refresh/src/app/config.py +110 -0
- package/src/corpus/templates/semantic-search/services/refresh/src/app/db.py +89 -0
- package/src/corpus/templates/semantic-search/services/refresh/src/app/embeddings.py +44 -0
- package/src/corpus/templates/semantic-search/services/refresh/src/app/indexing.py +223 -0
- package/src/corpus/templates/semantic-search/services/refresh/src/app/source.py +259 -0
- package/src/corpus/templates/semantic-search/services/refresh/src/main.py +45 -0
- package/src/corpus/templates/semantic-search/services/web/index.html +12 -0
- package/src/corpus/templates/semantic-search/services/web/package-lock.json +1680 -0
- package/src/corpus/templates/semantic-search/services/web/package.json +19 -0
- package/src/corpus/templates/semantic-search/services/web/src/App.jsx +55 -0
- package/src/corpus/templates/semantic-search/services/web/src/api.js +125 -0
- package/src/corpus/templates/semantic-search/services/web/src/components/Dashboard.jsx +61 -0
- package/src/corpus/templates/semantic-search/services/web/src/components/ManagerLogin.jsx +39 -0
- package/src/corpus/templates/semantic-search/services/web/src/components/SearchView.jsx +166 -0
- package/src/corpus/templates/semantic-search/services/web/src/components/Settings.jsx +148 -0
- package/src/corpus/templates/semantic-search/services/web/src/main.jsx +10 -0
- package/src/corpus/templates/semantic-search/services/web/src/styles.css +78 -0
- package/src/corpus/templates/semantic-search/services/web/vite.config.js +10 -0
- package/src/corpus/workflows/semantic-search.md +142 -0
- package/src/tools.js +76 -14
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""CloudGrid-native hybrid search over Mongo.
|
|
2
|
+
|
|
3
|
+
Modes blended in one query:
|
|
4
|
+
- lexical/exact : Mongo $text index over chunks.content (+ synonym expansion)
|
|
5
|
+
- semantic : in-app cosine similarity (NumPy) over chunk embedding arrays
|
|
6
|
+
- metadata : Mongo filters on documents (date, collection)
|
|
7
|
+
|
|
8
|
+
Results aggregate to the document level; each result carries its single best
|
|
9
|
+
matching passage with a highlight. Counting questions use countDocuments.
|
|
10
|
+
|
|
11
|
+
Everything degrades gracefully:
|
|
12
|
+
- no embeddings / no query vector -> semantic score = 0, lexical carries it
|
|
13
|
+
- no $text index yet (empty catalog) -> lexical contributes nothing, no 500
|
|
14
|
+
- empty catalog -> empty results + a clear reason
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
from pymongo.errors import OperationFailure
|
|
22
|
+
|
|
23
|
+
from . import config, db, embeddings
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _synonyms() -> dict[str, list[str]]:
|
|
27
|
+
doc = db.settings().find_one({"_id": "synonyms"}) or {}
|
|
28
|
+
return doc.get("map", {})
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _expand_query(q: str) -> str:
|
|
32
|
+
syn = _synonyms()
|
|
33
|
+
extra: list[str] = []
|
|
34
|
+
for token in q.split():
|
|
35
|
+
for alt in syn.get(token, []):
|
|
36
|
+
extra.append(alt)
|
|
37
|
+
return q if not extra else q + " " + " ".join(extra)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _metadata_filter(collection: str | None, date_from: str | None, date_to: str | None) -> dict:
|
|
41
|
+
f: dict = {}
|
|
42
|
+
if collection:
|
|
43
|
+
f["collection"] = collection
|
|
44
|
+
date_clause: dict = {}
|
|
45
|
+
if date_from:
|
|
46
|
+
date_clause["$gte"] = date_from
|
|
47
|
+
if date_to:
|
|
48
|
+
date_clause["$lte"] = date_to
|
|
49
|
+
if date_clause:
|
|
50
|
+
f["date"] = date_clause
|
|
51
|
+
return f
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _cosine(query_vec: np.ndarray, mat: np.ndarray) -> np.ndarray:
|
|
55
|
+
if mat.size == 0:
|
|
56
|
+
return np.array([])
|
|
57
|
+
qn = query_vec / (np.linalg.norm(query_vec) + 1e-9)
|
|
58
|
+
norms = np.linalg.norm(mat, axis=1) + 1e-9
|
|
59
|
+
return (mat @ qn) / norms
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _highlight(content: str, query: str) -> str:
|
|
63
|
+
terms = [t for t in re.split(r"\s+", query.strip()) if len(t) >= 2]
|
|
64
|
+
highlighted = content
|
|
65
|
+
for t in sorted(set(terms), key=len, reverse=True):
|
|
66
|
+
highlighted = re.sub(
|
|
67
|
+
f"({re.escape(t)})", r"«\1»", highlighted, flags=re.IGNORECASE
|
|
68
|
+
)
|
|
69
|
+
return highlighted
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def count_documents(collection: str | None = None,
|
|
73
|
+
date_from: str | None = None,
|
|
74
|
+
date_to: str | None = None) -> int:
|
|
75
|
+
"""Direct count from the DB (countDocuments), not estimated from results."""
|
|
76
|
+
return db.documents().count_documents(_metadata_filter(collection, date_from, date_to))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def search(query: str,
|
|
80
|
+
mode: str = "hybrid",
|
|
81
|
+
collection: str | None = None,
|
|
82
|
+
date_from: str | None = None,
|
|
83
|
+
date_to: str | None = None,
|
|
84
|
+
limit: int = 20) -> dict:
|
|
85
|
+
"""Blended search. mode in {hybrid, lexical, semantic, metadata}."""
|
|
86
|
+
meta_filter = _metadata_filter(collection, date_from, date_to)
|
|
87
|
+
|
|
88
|
+
# Candidate documents after metadata pre-filter (shrinks the semantic set).
|
|
89
|
+
doc_ids = None
|
|
90
|
+
if meta_filter:
|
|
91
|
+
doc_ids = [d["_id"] for d in db.documents().find(meta_filter, {"_id": 1})]
|
|
92
|
+
if not doc_ids:
|
|
93
|
+
return {"results": [], "total_matching_documents": 0, "mode": mode,
|
|
94
|
+
"note": "No documents match the metadata filter."}
|
|
95
|
+
|
|
96
|
+
# Pure metadata browse (no text query).
|
|
97
|
+
if mode == "metadata" or not query.strip():
|
|
98
|
+
return _metadata_only(meta_filter, limit)
|
|
99
|
+
|
|
100
|
+
chunk_scope: dict = {}
|
|
101
|
+
if doc_ids is not None:
|
|
102
|
+
chunk_scope["document_id"] = {"$in": doc_ids}
|
|
103
|
+
|
|
104
|
+
# ── lexical score via $text ──────────────────────────────────────────────
|
|
105
|
+
lexical: dict = {} # chunk_id -> (score, content, document_id)
|
|
106
|
+
if mode in ("hybrid", "lexical"):
|
|
107
|
+
expanded = _expand_query(query)
|
|
108
|
+
try:
|
|
109
|
+
cursor = db.chunks().find(
|
|
110
|
+
{**chunk_scope, "$text": {"$search": expanded}},
|
|
111
|
+
{"content": 1, "document_id": 1, "score": {"$meta": "textScore"}},
|
|
112
|
+
).sort([("score", {"$meta": "textScore"})]).limit(500)
|
|
113
|
+
for c in cursor:
|
|
114
|
+
lexical[c["_id"]] = (float(c.get("score", 0.0)), c["content"], c["document_id"])
|
|
115
|
+
except OperationFailure:
|
|
116
|
+
# No text index yet (empty catalog, not indexed) — lexical contributes
|
|
117
|
+
# nothing rather than 500ing. ensure_indexes() at startup normally
|
|
118
|
+
# prevents this; this is belt-and-suspenders.
|
|
119
|
+
pass
|
|
120
|
+
|
|
121
|
+
# ── semantic score via in-app cosine ─────────────────────────────────────
|
|
122
|
+
semantic: dict = {} # chunk_id -> (score, content, document_id)
|
|
123
|
+
semantic_note = None
|
|
124
|
+
if mode in ("hybrid", "semantic"):
|
|
125
|
+
if not config.embeddings_configured():
|
|
126
|
+
semantic_note = "Semantic search unavailable (embeddings not configured)."
|
|
127
|
+
else:
|
|
128
|
+
try:
|
|
129
|
+
qvec = np.array(embeddings.embed_one(query), dtype=np.float32)
|
|
130
|
+
candidate = db.chunks().find(
|
|
131
|
+
{**chunk_scope, "embedding": {"$ne": None}},
|
|
132
|
+
{"content": 1, "document_id": 1, "embedding": 1},
|
|
133
|
+
).limit(50000)
|
|
134
|
+
ids, contents, dids, mat = [], [], [], []
|
|
135
|
+
for c in candidate:
|
|
136
|
+
ids.append(c["_id"])
|
|
137
|
+
contents.append(c["content"])
|
|
138
|
+
dids.append(c["document_id"])
|
|
139
|
+
mat.append(c["embedding"])
|
|
140
|
+
if mat:
|
|
141
|
+
sims = _cosine(qvec, np.array(mat, dtype=np.float32))
|
|
142
|
+
for cid, content, did, s in zip(ids, contents, dids, sims):
|
|
143
|
+
semantic[cid] = (float(s), content, did)
|
|
144
|
+
except Exception as exc: # noqa: BLE001
|
|
145
|
+
semantic_note = f"Semantic search degraded: {exc}"
|
|
146
|
+
|
|
147
|
+
# ── normalize + blend at the chunk level ─────────────────────────────────
|
|
148
|
+
def _normalize(d: dict) -> dict:
|
|
149
|
+
if not d:
|
|
150
|
+
return {}
|
|
151
|
+
vals = [v[0] for v in d.values()]
|
|
152
|
+
lo, hi = min(vals), max(vals)
|
|
153
|
+
span = (hi - lo) or 1.0
|
|
154
|
+
return {k: (v[0] - lo) / span for k, v in d.items()}
|
|
155
|
+
|
|
156
|
+
lex_n = _normalize(lexical)
|
|
157
|
+
sem_n = _normalize(semantic)
|
|
158
|
+
sw, kw = config.semantic_weight(), config.keyword_weight()
|
|
159
|
+
if mode == "lexical":
|
|
160
|
+
sw, kw = 0.0, 1.0
|
|
161
|
+
elif mode == "semantic":
|
|
162
|
+
sw, kw = 1.0, 0.0
|
|
163
|
+
|
|
164
|
+
blended: dict = {} # chunk_id -> (score, content, document_id)
|
|
165
|
+
for cid in set(lex_n) | set(sem_n):
|
|
166
|
+
content, did = (lexical.get(cid) or semantic.get(cid))[1:3]
|
|
167
|
+
score = sw * sem_n.get(cid, 0.0) + kw * lex_n.get(cid, 0.0)
|
|
168
|
+
blended[cid] = (score, content, did)
|
|
169
|
+
|
|
170
|
+
# ── aggregate to document level: keep best passage per document ──────────
|
|
171
|
+
best_per_doc: dict = {} # document_id -> (score, content, chunk_id)
|
|
172
|
+
for cid, (score, content, did) in blended.items():
|
|
173
|
+
cur = best_per_doc.get(did)
|
|
174
|
+
if cur is None or score > cur[0]:
|
|
175
|
+
best_per_doc[did] = (score, content, cid)
|
|
176
|
+
|
|
177
|
+
ranked = sorted(best_per_doc.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
|
178
|
+
|
|
179
|
+
results = []
|
|
180
|
+
for did, (score, content, cid) in ranked:
|
|
181
|
+
doc = db.documents().find_one({"_id": did})
|
|
182
|
+
if not doc:
|
|
183
|
+
continue
|
|
184
|
+
results.append({
|
|
185
|
+
"document_id": str(did),
|
|
186
|
+
"title": doc.get("title"),
|
|
187
|
+
"collection": doc.get("collection"),
|
|
188
|
+
"date": doc.get("date"),
|
|
189
|
+
"has_media": doc.get("has_media", False),
|
|
190
|
+
"score": round(score, 4),
|
|
191
|
+
"best_passage": _highlight(content, query),
|
|
192
|
+
"source_filename": doc.get("source_filename"),
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
return {
|
|
196
|
+
"results": results,
|
|
197
|
+
"total_matching_documents": len(best_per_doc),
|
|
198
|
+
"mode": mode,
|
|
199
|
+
"note": semantic_note,
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _metadata_only(meta_filter: dict, limit: int) -> dict:
|
|
204
|
+
cursor = db.documents().find(meta_filter).sort("date", -1).limit(limit)
|
|
205
|
+
results = []
|
|
206
|
+
for doc in cursor:
|
|
207
|
+
results.append({
|
|
208
|
+
"document_id": str(doc["_id"]),
|
|
209
|
+
"title": doc.get("title"),
|
|
210
|
+
"collection": doc.get("collection"),
|
|
211
|
+
"date": doc.get("date"),
|
|
212
|
+
"has_media": doc.get("has_media", False),
|
|
213
|
+
"score": None,
|
|
214
|
+
"best_passage": None,
|
|
215
|
+
"source_filename": doc.get("source_filename"),
|
|
216
|
+
})
|
|
217
|
+
return {
|
|
218
|
+
"results": results,
|
|
219
|
+
"total_matching_documents": db.documents().count_documents(meta_filter),
|
|
220
|
+
"mode": "metadata",
|
|
221
|
+
"note": None,
|
|
222
|
+
}
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Pluggable document-source adapter, selected by SOURCE_TYPE.
|
|
2
|
+
|
|
3
|
+
The source is where the raw documents *live*, not the query database. Three
|
|
4
|
+
adapters ship here:
|
|
5
|
+
|
|
6
|
+
- dropbox : read-only Dropbox via the offline refresh-token flow.
|
|
7
|
+
- local : a directory on the container filesystem (e.g. a `needs: disk`
|
|
8
|
+
mount at DISK_PATH, or a bundled folder via LOCAL_SOURCE_PATH).
|
|
9
|
+
- url : a manifest of document URLs (URL_SOURCE_MANIFEST) fetched over HTTP.
|
|
10
|
+
|
|
11
|
+
Add your own by subclassing Source and returning it from get_source(). Never read
|
|
12
|
+
source secrets at import time; a missing config surfaces as SourceUnavailable,
|
|
13
|
+
handled gracefully by the indexing pipeline (it records a report and exits 0).
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import io
|
|
18
|
+
import os
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
|
|
21
|
+
import httpx
|
|
22
|
+
|
|
23
|
+
from . import config
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SourceUnavailable(RuntimeError):
|
|
27
|
+
"""Raised when the configured source is not usable (missing secrets, etc)."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# File classes we handle.
|
|
31
|
+
TEXT_EXTS = {".txt", ".md"}
|
|
32
|
+
PDF_EXTS = {".pdf"}
|
|
33
|
+
DOCX_EXTS = {".docx", ".doc"}
|
|
34
|
+
AUDIO_EXTS = {".mp3", ".m4a", ".wav", ".aac", ".ogg"}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class SourceFile:
|
|
39
|
+
path: str # source-internal path/URL (never exposed to clients)
|
|
40
|
+
name: str # filename only
|
|
41
|
+
rev: str # content revision / hash for incremental sync
|
|
42
|
+
size: int
|
|
43
|
+
is_audio: bool
|
|
44
|
+
is_text: bool # text-extractable (txt/md/pdf/docx)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _classify(name: str) -> tuple[bool, bool]:
|
|
48
|
+
ext = os.path.splitext(name)[1].lower()
|
|
49
|
+
return (ext in AUDIO_EXTS, ext in (TEXT_EXTS | PDF_EXTS | DOCX_EXTS))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Source:
|
|
53
|
+
def list_files(self) -> list[SourceFile]:
|
|
54
|
+
raise NotImplementedError
|
|
55
|
+
|
|
56
|
+
def download(self, path: str) -> bytes:
|
|
57
|
+
raise NotImplementedError
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# ── Dropbox adapter ───────────────────────────────────────────────────────────
|
|
61
|
+
|
|
62
|
+
class DropboxSource(Source):
|
|
63
|
+
"""Read-only Dropbox via the offline refresh-token flow."""
|
|
64
|
+
|
|
65
|
+
def __init__(self) -> None:
|
|
66
|
+
c = config.dropbox_config()
|
|
67
|
+
if not (c["app_key"] and c["app_secret"] and c["refresh_token"]):
|
|
68
|
+
raise SourceUnavailable(
|
|
69
|
+
"Dropbox is not configured — set DROPBOX_APP_KEY, "
|
|
70
|
+
"DROPBOX_APP_SECRET and DROPBOX_REFRESH_TOKEN."
|
|
71
|
+
)
|
|
72
|
+
self._app_key = c["app_key"]
|
|
73
|
+
self._app_secret = c["app_secret"]
|
|
74
|
+
self._refresh_token = c["refresh_token"]
|
|
75
|
+
self._folder = (c["folder_path"] or "").rstrip("/")
|
|
76
|
+
self._token: str | None = None
|
|
77
|
+
|
|
78
|
+
def _access_token(self) -> str:
|
|
79
|
+
if self._token:
|
|
80
|
+
return self._token
|
|
81
|
+
resp = httpx.post(
|
|
82
|
+
"https://api.dropbox.com/oauth2/token",
|
|
83
|
+
data={
|
|
84
|
+
"grant_type": "refresh_token",
|
|
85
|
+
"refresh_token": self._refresh_token,
|
|
86
|
+
},
|
|
87
|
+
auth=(self._app_key, self._app_secret),
|
|
88
|
+
timeout=30.0,
|
|
89
|
+
)
|
|
90
|
+
resp.raise_for_status()
|
|
91
|
+
self._token = resp.json()["access_token"]
|
|
92
|
+
return self._token
|
|
93
|
+
|
|
94
|
+
def _headers(self) -> dict[str, str]:
|
|
95
|
+
return {"Authorization": f"Bearer {self._access_token()}"}
|
|
96
|
+
|
|
97
|
+
def list_files(self) -> list[SourceFile]:
|
|
98
|
+
out: list[SourceFile] = []
|
|
99
|
+
url = "https://api.dropboxapi.com/2/files/list_folder"
|
|
100
|
+
body: dict = {"path": self._folder, "recursive": True}
|
|
101
|
+
while True:
|
|
102
|
+
resp = httpx.post(url, headers=self._headers(), json=body, timeout=60.0)
|
|
103
|
+
resp.raise_for_status()
|
|
104
|
+
payload = resp.json()
|
|
105
|
+
for entry in payload.get("entries", []):
|
|
106
|
+
if entry.get(".tag") != "file":
|
|
107
|
+
continue
|
|
108
|
+
out.append(self._to_source_file(entry))
|
|
109
|
+
if payload.get("has_more"):
|
|
110
|
+
url = "https://api.dropboxapi.com/2/files/list_folder/continue"
|
|
111
|
+
body = {"cursor": payload["cursor"]}
|
|
112
|
+
else:
|
|
113
|
+
break
|
|
114
|
+
return out
|
|
115
|
+
|
|
116
|
+
@staticmethod
|
|
117
|
+
def _to_source_file(entry: dict) -> SourceFile:
|
|
118
|
+
name = entry["name"]
|
|
119
|
+
is_audio, is_text = _classify(name)
|
|
120
|
+
return SourceFile(
|
|
121
|
+
path=entry["path_lower"],
|
|
122
|
+
name=name,
|
|
123
|
+
rev=entry.get("content_hash") or entry.get("rev", ""),
|
|
124
|
+
size=int(entry.get("size", 0)),
|
|
125
|
+
is_audio=is_audio,
|
|
126
|
+
is_text=is_text,
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
def download(self, path: str) -> bytes:
|
|
130
|
+
import json
|
|
131
|
+
|
|
132
|
+
resp = httpx.post(
|
|
133
|
+
"https://content.dropboxapi.com/2/files/download",
|
|
134
|
+
headers={
|
|
135
|
+
**self._headers(),
|
|
136
|
+
"Dropbox-API-Arg": json.dumps({"path": path}),
|
|
137
|
+
},
|
|
138
|
+
timeout=120.0,
|
|
139
|
+
)
|
|
140
|
+
resp.raise_for_status()
|
|
141
|
+
return resp.content
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# ── Local-filesystem adapter ──────────────────────────────────────────────────
|
|
145
|
+
|
|
146
|
+
class LocalSource(Source):
|
|
147
|
+
"""Read documents from a directory on the container filesystem.
|
|
148
|
+
|
|
149
|
+
Point LOCAL_SOURCE_PATH (or a `needs: disk` mount at DISK_PATH) at a folder
|
|
150
|
+
of .txt/.md/.pdf/.docx files. `rev` is the file mtime+size, so changed files
|
|
151
|
+
re-index on the next sync.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
def __init__(self) -> None:
|
|
155
|
+
root = config.local_source_path()
|
|
156
|
+
if not root:
|
|
157
|
+
raise SourceUnavailable(
|
|
158
|
+
"Local source is not configured — set LOCAL_SOURCE_PATH "
|
|
159
|
+
"(or mount a disk and it will default to DISK_PATH)."
|
|
160
|
+
)
|
|
161
|
+
if not os.path.isdir(root):
|
|
162
|
+
raise SourceUnavailable(f"LOCAL_SOURCE_PATH does not exist: {root!r}")
|
|
163
|
+
self._root = root
|
|
164
|
+
|
|
165
|
+
def list_files(self) -> list[SourceFile]:
|
|
166
|
+
out: list[SourceFile] = []
|
|
167
|
+
for dirpath, _dirs, files in os.walk(self._root):
|
|
168
|
+
for name in files:
|
|
169
|
+
full = os.path.join(dirpath, name)
|
|
170
|
+
is_audio, is_text = _classify(name)
|
|
171
|
+
try:
|
|
172
|
+
st = os.stat(full)
|
|
173
|
+
except OSError:
|
|
174
|
+
continue
|
|
175
|
+
out.append(SourceFile(
|
|
176
|
+
path=full,
|
|
177
|
+
name=name,
|
|
178
|
+
rev=f"{int(st.st_mtime)}-{st.st_size}",
|
|
179
|
+
size=st.st_size,
|
|
180
|
+
is_audio=is_audio,
|
|
181
|
+
is_text=is_text,
|
|
182
|
+
))
|
|
183
|
+
return out
|
|
184
|
+
|
|
185
|
+
def download(self, path: str) -> bytes:
|
|
186
|
+
with open(path, "rb") as fh:
|
|
187
|
+
return fh.read()
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# ── URL-manifest adapter ──────────────────────────────────────────────────────
|
|
191
|
+
|
|
192
|
+
class UrlSource(Source):
|
|
193
|
+
"""Fetch documents from a manifest of URLs.
|
|
194
|
+
|
|
195
|
+
URL_SOURCE_MANIFEST is a newline- or comma-separated list of document URLs.
|
|
196
|
+
The filename is the URL's last path segment; `rev` is the URL itself
|
|
197
|
+
(override by pointing the manifest at content-hashed URLs).
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
def __init__(self) -> None:
|
|
201
|
+
manifest = config.url_source_manifest()
|
|
202
|
+
if not manifest:
|
|
203
|
+
raise SourceUnavailable(
|
|
204
|
+
"URL source is not configured — set URL_SOURCE_MANIFEST to a "
|
|
205
|
+
"newline- or comma-separated list of document URLs."
|
|
206
|
+
)
|
|
207
|
+
raw = manifest.replace(",", "\n").splitlines()
|
|
208
|
+
self._urls = [u.strip() for u in raw if u.strip()]
|
|
209
|
+
|
|
210
|
+
def list_files(self) -> list[SourceFile]:
|
|
211
|
+
out: list[SourceFile] = []
|
|
212
|
+
for u in self._urls:
|
|
213
|
+
name = u.rstrip("/").rsplit("/", 1)[-1] or "document"
|
|
214
|
+
is_audio, is_text = _classify(name)
|
|
215
|
+
out.append(SourceFile(
|
|
216
|
+
path=u,
|
|
217
|
+
name=name,
|
|
218
|
+
rev=u,
|
|
219
|
+
size=0,
|
|
220
|
+
is_audio=is_audio,
|
|
221
|
+
is_text=is_text,
|
|
222
|
+
))
|
|
223
|
+
return out
|
|
224
|
+
|
|
225
|
+
def download(self, path: str) -> bytes:
|
|
226
|
+
resp = httpx.get(path, timeout=120.0, follow_redirects=True)
|
|
227
|
+
resp.raise_for_status()
|
|
228
|
+
return resp.content
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def get_source() -> Source:
|
|
232
|
+
"""Build the configured source adapter. Raises SourceUnavailable if unusable."""
|
|
233
|
+
st = config.source_type()
|
|
234
|
+
if st == "dropbox":
|
|
235
|
+
return DropboxSource()
|
|
236
|
+
if st == "local":
|
|
237
|
+
return LocalSource()
|
|
238
|
+
if st == "url":
|
|
239
|
+
return UrlSource()
|
|
240
|
+
raise SourceUnavailable(f"Unknown SOURCE_TYPE: {st!r}")
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# ── Text extraction (used by the pipeline) ───────────────────────────────────
|
|
244
|
+
|
|
245
|
+
def extract_text(name: str, raw: bytes) -> str:
|
|
246
|
+
ext = os.path.splitext(name)[1].lower()
|
|
247
|
+
if ext in TEXT_EXTS:
|
|
248
|
+
return raw.decode("utf-8", errors="replace")
|
|
249
|
+
if ext in PDF_EXTS:
|
|
250
|
+
from pypdf import PdfReader
|
|
251
|
+
|
|
252
|
+
reader = PdfReader(io.BytesIO(raw))
|
|
253
|
+
return "\n".join((page.extract_text() or "") for page in reader.pages)
|
|
254
|
+
if ext in DOCX_EXTS:
|
|
255
|
+
from docx import Document
|
|
256
|
+
|
|
257
|
+
doc = Document(io.BytesIO(raw))
|
|
258
|
+
return "\n".join(p.text for p in doc.paragraphs)
|
|
259
|
+
return ""
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""CloudGrid python-service entry point (services/backend/src/main.py).
|
|
2
|
+
|
|
3
|
+
Launches the FastAPI app (app/main.py) with uvicorn, binding to the platform's
|
|
4
|
+
injected PORT (default 8080). No secret is read here or at import time.
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
import uvicorn
|
|
9
|
+
|
|
10
|
+
if __name__ == "__main__":
|
|
11
|
+
port = int(os.environ.get("PORT", "8080"))
|
|
12
|
+
uvicorn.run("app.main:app", host="0.0.0.0", port=port, workers=1)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# services/refresh — scheduled cron reindex
|
|
2
|
+
|
|
3
|
+
A `type: cron`, `run: job` Python service that reruns the incremental indexing
|
|
4
|
+
pipeline on a schedule (default daily `0 3 * * *`, UTC). It is the scheduled
|
|
5
|
+
sibling of the manager-only `POST /backend/admin/refresh` endpoint — both call the
|
|
6
|
+
exact same `app.indexing.run_sync`.
|
|
7
|
+
|
|
8
|
+
## How it works
|
|
9
|
+
|
|
10
|
+
- `src/main.py` is the batch entry point. CloudGrid runs it to completion once per
|
|
11
|
+
schedule and exits; there is no HTTP server. It calls
|
|
12
|
+
`app.indexing.run_sync(triggered_by="cron")`.
|
|
13
|
+
- It connects DIRECTLY to the grid-injected `DATABASE_MONGODB_URL` (from the
|
|
14
|
+
app-level `needs: { database: true }`, inherited by every service including
|
|
15
|
+
cron). It does NOT call the backend over HTTP — there is no injected self-URL or
|
|
16
|
+
documented service-to-service DNS, so a network hop would be fragile.
|
|
17
|
+
- Graceful no-op: with no source / embeddings secrets set, `run_sync` records a
|
|
18
|
+
report (`status: skipped`, `source_configured: false`, 0 documents) and returns
|
|
19
|
+
WITHOUT raising. The run exits 0. Only a genuine pipeline error exits non-zero.
|
|
20
|
+
|
|
21
|
+
## Why the code is vendored (and the drift note)
|
|
22
|
+
|
|
23
|
+
Each CloudGrid service builds in its OWN isolated container from its own folder, so
|
|
24
|
+
a cross-folder import of `services/backend/src/app` does not resolve at build time.
|
|
25
|
+
The reuse mechanism is therefore a **vendored copy**: the backend modules that
|
|
26
|
+
`run_sync` transitively needs are copied verbatim into `services/refresh/src/app/`:
|
|
27
|
+
|
|
28
|
+
config.py db.py source.py embeddings.py indexing.py __init__.py
|
|
29
|
+
|
|
30
|
+
These are exact copies of `services/backend/src/app/*`. **Drift consideration:** if
|
|
31
|
+
you change the indexing pipeline, chunking, source adapters, embeddings, or the DB
|
|
32
|
+
schema in the backend, re-copy the changed module(s) here so the scheduled run
|
|
33
|
+
matches the on-demand endpoint. Keep the two in lockstep. (`answer.py`, `search.py`,
|
|
34
|
+
`auth.py`, `main.py` are NOT needed — the cron only indexes, it never serves or
|
|
35
|
+
searches — so they are intentionally not vendored.)
|
|
36
|
+
|
|
37
|
+
## Secrets it uses
|
|
38
|
+
|
|
39
|
+
Same secrets as the backend, read lazily at run time (never at import):
|
|
40
|
+
`SOURCE_TYPE` + its source creds (e.g. `DROPBOX_*` / `LOCAL_SOURCE_PATH` /
|
|
41
|
+
`URL_SOURCE_MANIFEST`), and `EMBEDDINGS_API_KEY` (+ `EMBEDDINGS_MODEL`,
|
|
42
|
+
`EMBEDDINGS_BASE_URL`). With none set, the run is a clean no-op.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Cron-job deps — the subset of the backend's requirements that the vendored
|
|
2
|
+
# indexing pipeline (app/config,db,source,embeddings,indexing) actually needs.
|
|
3
|
+
# No fastapi/uvicorn (this is a batch job, not a server) and no numpy (in-app
|
|
4
|
+
# cosine ranking lives in the backend's search.py, which the cron does not run).
|
|
5
|
+
# pypdf / python-docx are imported lazily by source.extract_text for pdf/docx.
|
|
6
|
+
pymongo==4.8.0
|
|
7
|
+
httpx==0.27.0
|
|
8
|
+
pypdf==4.2.0
|
|
9
|
+
python-docx==1.1.2
|
|
File without changes
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Central configuration and secret gating.
|
|
2
|
+
|
|
3
|
+
CRITICAL CloudGrid rule: NEVER read a secret at module import time. Every value
|
|
4
|
+
here is resolved lazily inside a function, so the process starts and passes
|
|
5
|
+
`GET /backend/health` even when NO secret is set. Search and indexing degrade
|
|
6
|
+
gracefully (empty results + a clear "not indexed / missing secret" note) rather
|
|
7
|
+
than crashing.
|
|
8
|
+
|
|
9
|
+
The Mongo connection is the ONLY hard dependency for health, and it is injected
|
|
10
|
+
automatically by CloudGrid (`needs: database`) as DATABASE_MONGODB_URL.
|
|
11
|
+
|
|
12
|
+
Two pluggable config points:
|
|
13
|
+
- SOURCE_TYPE selects the document-source adapter (dropbox | local | url).
|
|
14
|
+
- EMBEDDINGS_* selects the embeddings provider (OpenAI-compatible by default).
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# ── Mongo (injected by the grid — the only health-critical config) ──────────
|
|
22
|
+
|
|
23
|
+
def mongo_url() -> str | None:
|
|
24
|
+
"""Injected by CloudGrid `needs: database`. Never set by hand."""
|
|
25
|
+
return os.environ.get("DATABASE_MONGODB_URL") or os.environ.get("MONGODB_URL")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# ── Embeddings (one config point, gated) ─────────────────────────────────────
|
|
29
|
+
|
|
30
|
+
def embeddings_api_key() -> str | None:
|
|
31
|
+
return os.environ.get("EMBEDDINGS_API_KEY")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def embeddings_model() -> str:
|
|
35
|
+
# Changing the model changes vector dimensions and requires a full reindex.
|
|
36
|
+
return os.environ.get("EMBEDDINGS_MODEL", "text-embedding-3-small")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def embeddings_base_url() -> str:
|
|
40
|
+
# Any OpenAI-compatible endpoint. Default is OpenAI.
|
|
41
|
+
return os.environ.get("EMBEDDINGS_BASE_URL", "https://api.openai.com/v1")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def embeddings_configured() -> bool:
|
|
45
|
+
return bool(embeddings_api_key())
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def answer_model() -> str:
|
|
49
|
+
return os.environ.get("ANSWER_MODEL", "gpt-4o-mini")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# ── Source adapter (one config point, gated) ─────────────────────────────────
|
|
53
|
+
|
|
54
|
+
def source_type() -> str:
|
|
55
|
+
"""dropbox | local | url. Selects the document-source adapter."""
|
|
56
|
+
return os.environ.get("SOURCE_TYPE", "dropbox")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def dropbox_config() -> dict[str, str | None]:
|
|
60
|
+
return {
|
|
61
|
+
"app_key": os.environ.get("DROPBOX_APP_KEY"),
|
|
62
|
+
"app_secret": os.environ.get("DROPBOX_APP_SECRET"),
|
|
63
|
+
"refresh_token": os.environ.get("DROPBOX_REFRESH_TOKEN"),
|
|
64
|
+
"folder_path": os.environ.get("DROPBOX_FOLDER_PATH", ""),
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def local_source_path() -> str:
|
|
69
|
+
"""Directory the local adapter reads. Point at a `needs: disk` mount
|
|
70
|
+
(DISK_PATH) or a bundled folder."""
|
|
71
|
+
return os.environ.get("LOCAL_SOURCE_PATH", os.environ.get("DISK_PATH", ""))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def url_source_manifest() -> str | None:
|
|
75
|
+
"""A newline- or comma-separated list of document URLs the url adapter
|
|
76
|
+
fetches. Set as a secret or plain env var."""
|
|
77
|
+
return os.environ.get("URL_SOURCE_MANIFEST")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def source_configured() -> bool:
|
|
81
|
+
st = source_type()
|
|
82
|
+
if st == "dropbox":
|
|
83
|
+
c = dropbox_config()
|
|
84
|
+
return bool(c["app_key"] and c["app_secret"] and c["refresh_token"])
|
|
85
|
+
if st == "local":
|
|
86
|
+
return bool(local_source_path())
|
|
87
|
+
if st == "url":
|
|
88
|
+
return bool(url_source_manifest())
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# ── Manager auth (gated) ──────────────────────────────────────────────────────
|
|
93
|
+
|
|
94
|
+
def manager_password_hash() -> str | None:
|
|
95
|
+
"""A sha256 hex digest of the manager password. Compared server-side."""
|
|
96
|
+
return os.environ.get("MANAGER_PASSWORD_HASH")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def manager_auth_configured() -> bool:
|
|
100
|
+
return bool(manager_password_hash())
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# ── Search blend weights ─────────────────────────────────────────────────────
|
|
104
|
+
|
|
105
|
+
def semantic_weight() -> float:
|
|
106
|
+
return float(os.environ.get("SEMANTIC_WEIGHT", "0.55"))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def keyword_weight() -> float:
|
|
110
|
+
return float(os.environ.get("KEYWORD_WEIGHT", "0.45"))
|