@cloudgrid-io/mcp 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/package.json +1 -1
  2. package/src/corpus/capability-map.md +44 -15
  3. package/src/corpus/install.md +27 -0
  4. package/src/corpus/templates/semantic-search/AGENTS.md +216 -0
  5. package/src/corpus/templates/semantic-search/README.md +75 -0
  6. package/src/corpus/templates/semantic-search/cloudgrid.yaml +219 -0
  7. package/src/corpus/templates/semantic-search/index.md +108 -0
  8. package/src/corpus/templates/semantic-search/services/backend/requirements.txt +7 -0
  9. package/src/corpus/templates/semantic-search/services/backend/src/app/__init__.py +0 -0
  10. package/src/corpus/templates/semantic-search/services/backend/src/app/answer.py +90 -0
  11. package/src/corpus/templates/semantic-search/services/backend/src/app/auth.py +65 -0
  12. package/src/corpus/templates/semantic-search/services/backend/src/app/config.py +110 -0
  13. package/src/corpus/templates/semantic-search/services/backend/src/app/db.py +89 -0
  14. package/src/corpus/templates/semantic-search/services/backend/src/app/embeddings.py +44 -0
  15. package/src/corpus/templates/semantic-search/services/backend/src/app/indexing.py +223 -0
  16. package/src/corpus/templates/semantic-search/services/backend/src/app/main.py +259 -0
  17. package/src/corpus/templates/semantic-search/services/backend/src/app/search.py +222 -0
  18. package/src/corpus/templates/semantic-search/services/backend/src/app/source.py +259 -0
  19. package/src/corpus/templates/semantic-search/services/backend/src/main.py +12 -0
  20. package/src/corpus/templates/semantic-search/services/refresh/README.md +42 -0
  21. package/src/corpus/templates/semantic-search/services/refresh/requirements.txt +9 -0
  22. package/src/corpus/templates/semantic-search/services/refresh/src/app/__init__.py +0 -0
  23. package/src/corpus/templates/semantic-search/services/refresh/src/app/config.py +110 -0
  24. package/src/corpus/templates/semantic-search/services/refresh/src/app/db.py +89 -0
  25. package/src/corpus/templates/semantic-search/services/refresh/src/app/embeddings.py +44 -0
  26. package/src/corpus/templates/semantic-search/services/refresh/src/app/indexing.py +223 -0
  27. package/src/corpus/templates/semantic-search/services/refresh/src/app/source.py +259 -0
  28. package/src/corpus/templates/semantic-search/services/refresh/src/main.py +45 -0
  29. package/src/corpus/templates/semantic-search/services/web/index.html +12 -0
  30. package/src/corpus/templates/semantic-search/services/web/package-lock.json +1680 -0
  31. package/src/corpus/templates/semantic-search/services/web/package.json +19 -0
  32. package/src/corpus/templates/semantic-search/services/web/src/App.jsx +55 -0
  33. package/src/corpus/templates/semantic-search/services/web/src/api.js +125 -0
  34. package/src/corpus/templates/semantic-search/services/web/src/components/Dashboard.jsx +61 -0
  35. package/src/corpus/templates/semantic-search/services/web/src/components/ManagerLogin.jsx +39 -0
  36. package/src/corpus/templates/semantic-search/services/web/src/components/SearchView.jsx +166 -0
  37. package/src/corpus/templates/semantic-search/services/web/src/components/Settings.jsx +148 -0
  38. package/src/corpus/templates/semantic-search/services/web/src/main.jsx +10 -0
  39. package/src/corpus/templates/semantic-search/services/web/src/styles.css +78 -0
  40. package/src/corpus/templates/semantic-search/services/web/vite.config.js +10 -0
  41. package/src/corpus/workflows/semantic-search.md +142 -0
  42. package/src/tools.js +76 -14
@@ -0,0 +1,222 @@
1
+ """CloudGrid-native hybrid search over Mongo.
2
+
3
+ Modes blended in one query:
4
+ - lexical/exact : Mongo $text index over chunks.content (+ synonym expansion)
5
+ - semantic : in-app cosine similarity (NumPy) over chunk embedding arrays
6
+ - metadata : Mongo filters on documents (date, collection)
7
+
8
+ Results aggregate to the document level; each result carries its single best
9
+ matching passage with a highlight. Counting questions use countDocuments.
10
+
11
+ Everything degrades gracefully:
12
+ - no embeddings / no query vector -> semantic score = 0, lexical carries it
13
+ - no $text index yet (empty catalog) -> lexical contributes nothing, no 500
14
+ - empty catalog -> empty results + a clear reason
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import re
19
+
20
+ import numpy as np
21
+ from pymongo.errors import OperationFailure
22
+
23
+ from . import config, db, embeddings
24
+
25
+
26
+ def _synonyms() -> dict[str, list[str]]:
27
+ doc = db.settings().find_one({"_id": "synonyms"}) or {}
28
+ return doc.get("map", {})
29
+
30
+
31
+ def _expand_query(q: str) -> str:
32
+ syn = _synonyms()
33
+ extra: list[str] = []
34
+ for token in q.split():
35
+ for alt in syn.get(token, []):
36
+ extra.append(alt)
37
+ return q if not extra else q + " " + " ".join(extra)
38
+
39
+
40
+ def _metadata_filter(collection: str | None, date_from: str | None, date_to: str | None) -> dict:
41
+ f: dict = {}
42
+ if collection:
43
+ f["collection"] = collection
44
+ date_clause: dict = {}
45
+ if date_from:
46
+ date_clause["$gte"] = date_from
47
+ if date_to:
48
+ date_clause["$lte"] = date_to
49
+ if date_clause:
50
+ f["date"] = date_clause
51
+ return f
52
+
53
+
54
+ def _cosine(query_vec: np.ndarray, mat: np.ndarray) -> np.ndarray:
55
+ if mat.size == 0:
56
+ return np.array([])
57
+ qn = query_vec / (np.linalg.norm(query_vec) + 1e-9)
58
+ norms = np.linalg.norm(mat, axis=1) + 1e-9
59
+ return (mat @ qn) / norms
60
+
61
+
62
+ def _highlight(content: str, query: str) -> str:
63
+ terms = [t for t in re.split(r"\s+", query.strip()) if len(t) >= 2]
64
+ highlighted = content
65
+ for t in sorted(set(terms), key=len, reverse=True):
66
+ highlighted = re.sub(
67
+ f"({re.escape(t)})", r"«\1»", highlighted, flags=re.IGNORECASE
68
+ )
69
+ return highlighted
70
+
71
+
72
+ def count_documents(collection: str | None = None,
73
+ date_from: str | None = None,
74
+ date_to: str | None = None) -> int:
75
+ """Direct count from the DB (countDocuments), not estimated from results."""
76
+ return db.documents().count_documents(_metadata_filter(collection, date_from, date_to))
77
+
78
+
79
+ def search(query: str,
80
+ mode: str = "hybrid",
81
+ collection: str | None = None,
82
+ date_from: str | None = None,
83
+ date_to: str | None = None,
84
+ limit: int = 20) -> dict:
85
+ """Blended search. mode in {hybrid, lexical, semantic, metadata}."""
86
+ meta_filter = _metadata_filter(collection, date_from, date_to)
87
+
88
+ # Candidate documents after metadata pre-filter (shrinks the semantic set).
89
+ doc_ids = None
90
+ if meta_filter:
91
+ doc_ids = [d["_id"] for d in db.documents().find(meta_filter, {"_id": 1})]
92
+ if not doc_ids:
93
+ return {"results": [], "total_matching_documents": 0, "mode": mode,
94
+ "note": "No documents match the metadata filter."}
95
+
96
+ # Pure metadata browse (no text query).
97
+ if mode == "metadata" or not query.strip():
98
+ return _metadata_only(meta_filter, limit)
99
+
100
+ chunk_scope: dict = {}
101
+ if doc_ids is not None:
102
+ chunk_scope["document_id"] = {"$in": doc_ids}
103
+
104
+ # ── lexical score via $text ──────────────────────────────────────────────
105
+ lexical: dict = {} # chunk_id -> (score, content, document_id)
106
+ if mode in ("hybrid", "lexical"):
107
+ expanded = _expand_query(query)
108
+ try:
109
+ cursor = db.chunks().find(
110
+ {**chunk_scope, "$text": {"$search": expanded}},
111
+ {"content": 1, "document_id": 1, "score": {"$meta": "textScore"}},
112
+ ).sort([("score", {"$meta": "textScore"})]).limit(500)
113
+ for c in cursor:
114
+ lexical[c["_id"]] = (float(c.get("score", 0.0)), c["content"], c["document_id"])
115
+ except OperationFailure:
116
+ # No text index yet (empty catalog, not indexed) — lexical contributes
117
+ # nothing rather than 500ing. ensure_indexes() at startup normally
118
+ # prevents this; this is belt-and-suspenders.
119
+ pass
120
+
121
+ # ── semantic score via in-app cosine ─────────────────────────────────────
122
+ semantic: dict = {} # chunk_id -> (score, content, document_id)
123
+ semantic_note = None
124
+ if mode in ("hybrid", "semantic"):
125
+ if not config.embeddings_configured():
126
+ semantic_note = "Semantic search unavailable (embeddings not configured)."
127
+ else:
128
+ try:
129
+ qvec = np.array(embeddings.embed_one(query), dtype=np.float32)
130
+ candidate = db.chunks().find(
131
+ {**chunk_scope, "embedding": {"$ne": None}},
132
+ {"content": 1, "document_id": 1, "embedding": 1},
133
+ ).limit(50000)
134
+ ids, contents, dids, mat = [], [], [], []
135
+ for c in candidate:
136
+ ids.append(c["_id"])
137
+ contents.append(c["content"])
138
+ dids.append(c["document_id"])
139
+ mat.append(c["embedding"])
140
+ if mat:
141
+ sims = _cosine(qvec, np.array(mat, dtype=np.float32))
142
+ for cid, content, did, s in zip(ids, contents, dids, sims):
143
+ semantic[cid] = (float(s), content, did)
144
+ except Exception as exc: # noqa: BLE001
145
+ semantic_note = f"Semantic search degraded: {exc}"
146
+
147
+ # ── normalize + blend at the chunk level ─────────────────────────────────
148
+ def _normalize(d: dict) -> dict:
149
+ if not d:
150
+ return {}
151
+ vals = [v[0] for v in d.values()]
152
+ lo, hi = min(vals), max(vals)
153
+ span = (hi - lo) or 1.0
154
+ return {k: (v[0] - lo) / span for k, v in d.items()}
155
+
156
+ lex_n = _normalize(lexical)
157
+ sem_n = _normalize(semantic)
158
+ sw, kw = config.semantic_weight(), config.keyword_weight()
159
+ if mode == "lexical":
160
+ sw, kw = 0.0, 1.0
161
+ elif mode == "semantic":
162
+ sw, kw = 1.0, 0.0
163
+
164
+ blended: dict = {} # chunk_id -> (score, content, document_id)
165
+ for cid in set(lex_n) | set(sem_n):
166
+ content, did = (lexical.get(cid) or semantic.get(cid))[1:3]
167
+ score = sw * sem_n.get(cid, 0.0) + kw * lex_n.get(cid, 0.0)
168
+ blended[cid] = (score, content, did)
169
+
170
+ # ── aggregate to document level: keep best passage per document ──────────
171
+ best_per_doc: dict = {} # document_id -> (score, content, chunk_id)
172
+ for cid, (score, content, did) in blended.items():
173
+ cur = best_per_doc.get(did)
174
+ if cur is None or score > cur[0]:
175
+ best_per_doc[did] = (score, content, cid)
176
+
177
+ ranked = sorted(best_per_doc.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
178
+
179
+ results = []
180
+ for did, (score, content, cid) in ranked:
181
+ doc = db.documents().find_one({"_id": did})
182
+ if not doc:
183
+ continue
184
+ results.append({
185
+ "document_id": str(did),
186
+ "title": doc.get("title"),
187
+ "collection": doc.get("collection"),
188
+ "date": doc.get("date"),
189
+ "has_media": doc.get("has_media", False),
190
+ "score": round(score, 4),
191
+ "best_passage": _highlight(content, query),
192
+ "source_filename": doc.get("source_filename"),
193
+ })
194
+
195
+ return {
196
+ "results": results,
197
+ "total_matching_documents": len(best_per_doc),
198
+ "mode": mode,
199
+ "note": semantic_note,
200
+ }
201
+
202
+
203
+ def _metadata_only(meta_filter: dict, limit: int) -> dict:
204
+ cursor = db.documents().find(meta_filter).sort("date", -1).limit(limit)
205
+ results = []
206
+ for doc in cursor:
207
+ results.append({
208
+ "document_id": str(doc["_id"]),
209
+ "title": doc.get("title"),
210
+ "collection": doc.get("collection"),
211
+ "date": doc.get("date"),
212
+ "has_media": doc.get("has_media", False),
213
+ "score": None,
214
+ "best_passage": None,
215
+ "source_filename": doc.get("source_filename"),
216
+ })
217
+ return {
218
+ "results": results,
219
+ "total_matching_documents": db.documents().count_documents(meta_filter),
220
+ "mode": "metadata",
221
+ "note": None,
222
+ }
@@ -0,0 +1,259 @@
1
+ """Pluggable document-source adapter, selected by SOURCE_TYPE.
2
+
3
+ The source is where the raw documents *live*, not the query database. Three
4
+ adapters ship here:
5
+
6
+ - dropbox : read-only Dropbox via the offline refresh-token flow.
7
+ - local : a directory on the container filesystem (e.g. a `needs: disk`
8
+ mount at DISK_PATH, or a bundled folder via LOCAL_SOURCE_PATH).
9
+ - url : a manifest of document URLs (URL_SOURCE_MANIFEST) fetched over HTTP.
10
+
11
+ Add your own by subclassing Source and returning it from get_source(). Never read
12
+ source secrets at import time; a missing config surfaces as SourceUnavailable,
13
+ handled gracefully by the indexing pipeline (it records a report and exits 0).
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import io
18
+ import os
19
+ from dataclasses import dataclass
20
+
21
+ import httpx
22
+
23
+ from . import config
24
+
25
+
26
+ class SourceUnavailable(RuntimeError):
27
+ """Raised when the configured source is not usable (missing secrets, etc)."""
28
+
29
+
30
+ # File classes we handle.
31
+ TEXT_EXTS = {".txt", ".md"}
32
+ PDF_EXTS = {".pdf"}
33
+ DOCX_EXTS = {".docx", ".doc"}
34
+ AUDIO_EXTS = {".mp3", ".m4a", ".wav", ".aac", ".ogg"}
35
+
36
+
37
+ @dataclass
38
+ class SourceFile:
39
+ path: str # source-internal path/URL (never exposed to clients)
40
+ name: str # filename only
41
+ rev: str # content revision / hash for incremental sync
42
+ size: int
43
+ is_audio: bool
44
+ is_text: bool # text-extractable (txt/md/pdf/docx)
45
+
46
+
47
+ def _classify(name: str) -> tuple[bool, bool]:
48
+ ext = os.path.splitext(name)[1].lower()
49
+ return (ext in AUDIO_EXTS, ext in (TEXT_EXTS | PDF_EXTS | DOCX_EXTS))
50
+
51
+
52
+ class Source:
53
+ def list_files(self) -> list[SourceFile]:
54
+ raise NotImplementedError
55
+
56
+ def download(self, path: str) -> bytes:
57
+ raise NotImplementedError
58
+
59
+
60
+ # ── Dropbox adapter ───────────────────────────────────────────────────────────
61
+
62
+ class DropboxSource(Source):
63
+ """Read-only Dropbox via the offline refresh-token flow."""
64
+
65
+ def __init__(self) -> None:
66
+ c = config.dropbox_config()
67
+ if not (c["app_key"] and c["app_secret"] and c["refresh_token"]):
68
+ raise SourceUnavailable(
69
+ "Dropbox is not configured — set DROPBOX_APP_KEY, "
70
+ "DROPBOX_APP_SECRET and DROPBOX_REFRESH_TOKEN."
71
+ )
72
+ self._app_key = c["app_key"]
73
+ self._app_secret = c["app_secret"]
74
+ self._refresh_token = c["refresh_token"]
75
+ self._folder = (c["folder_path"] or "").rstrip("/")
76
+ self._token: str | None = None
77
+
78
+ def _access_token(self) -> str:
79
+ if self._token:
80
+ return self._token
81
+ resp = httpx.post(
82
+ "https://api.dropbox.com/oauth2/token",
83
+ data={
84
+ "grant_type": "refresh_token",
85
+ "refresh_token": self._refresh_token,
86
+ },
87
+ auth=(self._app_key, self._app_secret),
88
+ timeout=30.0,
89
+ )
90
+ resp.raise_for_status()
91
+ self._token = resp.json()["access_token"]
92
+ return self._token
93
+
94
+ def _headers(self) -> dict[str, str]:
95
+ return {"Authorization": f"Bearer {self._access_token()}"}
96
+
97
+ def list_files(self) -> list[SourceFile]:
98
+ out: list[SourceFile] = []
99
+ url = "https://api.dropboxapi.com/2/files/list_folder"
100
+ body: dict = {"path": self._folder, "recursive": True}
101
+ while True:
102
+ resp = httpx.post(url, headers=self._headers(), json=body, timeout=60.0)
103
+ resp.raise_for_status()
104
+ payload = resp.json()
105
+ for entry in payload.get("entries", []):
106
+ if entry.get(".tag") != "file":
107
+ continue
108
+ out.append(self._to_source_file(entry))
109
+ if payload.get("has_more"):
110
+ url = "https://api.dropboxapi.com/2/files/list_folder/continue"
111
+ body = {"cursor": payload["cursor"]}
112
+ else:
113
+ break
114
+ return out
115
+
116
+ @staticmethod
117
+ def _to_source_file(entry: dict) -> SourceFile:
118
+ name = entry["name"]
119
+ is_audio, is_text = _classify(name)
120
+ return SourceFile(
121
+ path=entry["path_lower"],
122
+ name=name,
123
+ rev=entry.get("content_hash") or entry.get("rev", ""),
124
+ size=int(entry.get("size", 0)),
125
+ is_audio=is_audio,
126
+ is_text=is_text,
127
+ )
128
+
129
+ def download(self, path: str) -> bytes:
130
+ import json
131
+
132
+ resp = httpx.post(
133
+ "https://content.dropboxapi.com/2/files/download",
134
+ headers={
135
+ **self._headers(),
136
+ "Dropbox-API-Arg": json.dumps({"path": path}),
137
+ },
138
+ timeout=120.0,
139
+ )
140
+ resp.raise_for_status()
141
+ return resp.content
142
+
143
+
144
+ # ── Local-filesystem adapter ──────────────────────────────────────────────────
145
+
146
+ class LocalSource(Source):
147
+ """Read documents from a directory on the container filesystem.
148
+
149
+ Point LOCAL_SOURCE_PATH (or a `needs: disk` mount at DISK_PATH) at a folder
150
+ of .txt/.md/.pdf/.docx files. `rev` is the file mtime+size, so changed files
151
+ re-index on the next sync.
152
+ """
153
+
154
+ def __init__(self) -> None:
155
+ root = config.local_source_path()
156
+ if not root:
157
+ raise SourceUnavailable(
158
+ "Local source is not configured — set LOCAL_SOURCE_PATH "
159
+ "(or mount a disk and it will default to DISK_PATH)."
160
+ )
161
+ if not os.path.isdir(root):
162
+ raise SourceUnavailable(f"LOCAL_SOURCE_PATH does not exist: {root!r}")
163
+ self._root = root
164
+
165
+ def list_files(self) -> list[SourceFile]:
166
+ out: list[SourceFile] = []
167
+ for dirpath, _dirs, files in os.walk(self._root):
168
+ for name in files:
169
+ full = os.path.join(dirpath, name)
170
+ is_audio, is_text = _classify(name)
171
+ try:
172
+ st = os.stat(full)
173
+ except OSError:
174
+ continue
175
+ out.append(SourceFile(
176
+ path=full,
177
+ name=name,
178
+ rev=f"{int(st.st_mtime)}-{st.st_size}",
179
+ size=st.st_size,
180
+ is_audio=is_audio,
181
+ is_text=is_text,
182
+ ))
183
+ return out
184
+
185
+ def download(self, path: str) -> bytes:
186
+ with open(path, "rb") as fh:
187
+ return fh.read()
188
+
189
+
190
+ # ── URL-manifest adapter ──────────────────────────────────────────────────────
191
+
192
+ class UrlSource(Source):
193
+ """Fetch documents from a manifest of URLs.
194
+
195
+ URL_SOURCE_MANIFEST is a newline- or comma-separated list of document URLs.
196
+ The filename is the URL's last path segment; `rev` is the URL itself
197
+ (override by pointing the manifest at content-hashed URLs).
198
+ """
199
+
200
+ def __init__(self) -> None:
201
+ manifest = config.url_source_manifest()
202
+ if not manifest:
203
+ raise SourceUnavailable(
204
+ "URL source is not configured — set URL_SOURCE_MANIFEST to a "
205
+ "newline- or comma-separated list of document URLs."
206
+ )
207
+ raw = manifest.replace(",", "\n").splitlines()
208
+ self._urls = [u.strip() for u in raw if u.strip()]
209
+
210
+ def list_files(self) -> list[SourceFile]:
211
+ out: list[SourceFile] = []
212
+ for u in self._urls:
213
+ name = u.rstrip("/").rsplit("/", 1)[-1] or "document"
214
+ is_audio, is_text = _classify(name)
215
+ out.append(SourceFile(
216
+ path=u,
217
+ name=name,
218
+ rev=u,
219
+ size=0,
220
+ is_audio=is_audio,
221
+ is_text=is_text,
222
+ ))
223
+ return out
224
+
225
+ def download(self, path: str) -> bytes:
226
+ resp = httpx.get(path, timeout=120.0, follow_redirects=True)
227
+ resp.raise_for_status()
228
+ return resp.content
229
+
230
+
231
+ def get_source() -> Source:
232
+ """Build the configured source adapter. Raises SourceUnavailable if unusable."""
233
+ st = config.source_type()
234
+ if st == "dropbox":
235
+ return DropboxSource()
236
+ if st == "local":
237
+ return LocalSource()
238
+ if st == "url":
239
+ return UrlSource()
240
+ raise SourceUnavailable(f"Unknown SOURCE_TYPE: {st!r}")
241
+
242
+
243
+ # ── Text extraction (used by the pipeline) ───────────────────────────────────
244
+
245
+ def extract_text(name: str, raw: bytes) -> str:
246
+ ext = os.path.splitext(name)[1].lower()
247
+ if ext in TEXT_EXTS:
248
+ return raw.decode("utf-8", errors="replace")
249
+ if ext in PDF_EXTS:
250
+ from pypdf import PdfReader
251
+
252
+ reader = PdfReader(io.BytesIO(raw))
253
+ return "\n".join((page.extract_text() or "") for page in reader.pages)
254
+ if ext in DOCX_EXTS:
255
+ from docx import Document
256
+
257
+ doc = Document(io.BytesIO(raw))
258
+ return "\n".join(p.text for p in doc.paragraphs)
259
+ return ""
@@ -0,0 +1,12 @@
1
+ """CloudGrid python-service entry point (services/backend/src/main.py).
2
+
3
+ Launches the FastAPI app (app/main.py) with uvicorn, binding to the platform's
4
+ injected PORT (default 8080). No secret is read here or at import time.
5
+ """
6
+ import os
7
+
8
+ import uvicorn
9
+
10
+ if __name__ == "__main__":
11
+ port = int(os.environ.get("PORT", "8080"))
12
+ uvicorn.run("app.main:app", host="0.0.0.0", port=port, workers=1)
@@ -0,0 +1,42 @@
1
+ # services/refresh — scheduled cron reindex
2
+
3
+ A `type: cron`, `run: job` Python service that reruns the incremental indexing
4
+ pipeline on a schedule (default daily `0 3 * * *`, UTC). It is the scheduled
5
+ sibling of the manager-only `POST /backend/admin/refresh` endpoint — both call the
6
+ exact same `app.indexing.run_sync`.
7
+
8
+ ## How it works
9
+
10
+ - `src/main.py` is the batch entry point. CloudGrid runs it to completion once per
11
+ schedule and exits; there is no HTTP server. It calls
12
+ `app.indexing.run_sync(triggered_by="cron")`.
13
+ - It connects DIRECTLY to the grid-injected `DATABASE_MONGODB_URL` (from the
14
+ app-level `needs: { database: true }`, inherited by every service including
15
+ cron). It does NOT call the backend over HTTP — there is no injected self-URL or
16
+ documented service-to-service DNS, so a network hop would be fragile.
17
+ - Graceful no-op: with no source / embeddings secrets set, `run_sync` records a
18
+ report (`status: skipped`, `source_configured: false`, 0 documents) and returns
19
+ WITHOUT raising. The run exits 0. Only a genuine pipeline error exits non-zero.
20
+
21
+ ## Why the code is vendored (and the drift note)
22
+
23
+ Each CloudGrid service builds in its OWN isolated container from its own folder, so
24
+ a cross-folder import of `services/backend/src/app` does not resolve at build time.
25
+ The reuse mechanism is therefore a **vendored copy**: the backend modules that
26
+ `run_sync` transitively needs are copied verbatim into `services/refresh/src/app/`:
27
+
28
+ config.py db.py source.py embeddings.py indexing.py __init__.py
29
+
30
+ These are exact copies of `services/backend/src/app/*`. **Drift consideration:** if
31
+ you change the indexing pipeline, chunking, source adapters, embeddings, or the DB
32
+ schema in the backend, re-copy the changed module(s) here so the scheduled run
33
+ matches the on-demand endpoint. Keep the two in lockstep. (`answer.py`, `search.py`,
34
+ `auth.py`, `main.py` are NOT needed — the cron only indexes, it never serves or
35
+ searches — so they are intentionally not vendored.)
36
+
37
+ ## Secrets it uses
38
+
39
+ Same secrets as the backend, read lazily at run time (never at import):
40
+ `SOURCE_TYPE` + its source creds (e.g. `DROPBOX_*` / `LOCAL_SOURCE_PATH` /
41
+ `URL_SOURCE_MANIFEST`), and `EMBEDDINGS_API_KEY` (+ `EMBEDDINGS_MODEL`,
42
+ `EMBEDDINGS_BASE_URL`). With none set, the run is a clean no-op.
@@ -0,0 +1,9 @@
1
+ # Cron-job deps — the subset of the backend's requirements that the vendored
2
+ # indexing pipeline (app/config,db,source,embeddings,indexing) actually needs.
3
+ # No fastapi/uvicorn (this is a batch job, not a server) and no numpy (in-app
4
+ # cosine ranking lives in the backend's search.py, which the cron does not run).
5
+ # pypdf / python-docx are imported lazily by source.extract_text for pdf/docx.
6
+ pymongo==4.8.0
7
+ httpx==0.27.0
8
+ pypdf==4.2.0
9
+ python-docx==1.1.2
@@ -0,0 +1,110 @@
1
+ """Central configuration and secret gating.
2
+
3
+ CRITICAL CloudGrid rule: NEVER read a secret at module import time. Every value
4
+ here is resolved lazily inside a function, so the process starts and passes
5
+ `GET /backend/health` even when NO secret is set. Search and indexing degrade
6
+ gracefully (empty results + a clear "not indexed / missing secret" note) rather
7
+ than crashing.
8
+
9
+ The Mongo connection is the ONLY hard dependency for health, and it is injected
10
+ automatically by CloudGrid (`needs: database`) as DATABASE_MONGODB_URL.
11
+
12
+ Two pluggable config points:
13
+ - SOURCE_TYPE selects the document-source adapter (dropbox | local | url).
14
+ - EMBEDDINGS_* selects the embeddings provider (OpenAI-compatible by default).
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import os
19
+
20
+
21
+ # ── Mongo (injected by the grid — the only health-critical config) ──────────
22
+
23
+ def mongo_url() -> str | None:
24
+ """Injected by CloudGrid `needs: database`. Never set by hand."""
25
+ return os.environ.get("DATABASE_MONGODB_URL") or os.environ.get("MONGODB_URL")
26
+
27
+
28
+ # ── Embeddings (one config point, gated) ─────────────────────────────────────
29
+
30
+ def embeddings_api_key() -> str | None:
31
+ return os.environ.get("EMBEDDINGS_API_KEY")
32
+
33
+
34
+ def embeddings_model() -> str:
35
+ # Changing the model changes vector dimensions and requires a full reindex.
36
+ return os.environ.get("EMBEDDINGS_MODEL", "text-embedding-3-small")
37
+
38
+
39
+ def embeddings_base_url() -> str:
40
+ # Any OpenAI-compatible endpoint. Default is OpenAI.
41
+ return os.environ.get("EMBEDDINGS_BASE_URL", "https://api.openai.com/v1")
42
+
43
+
44
+ def embeddings_configured() -> bool:
45
+ return bool(embeddings_api_key())
46
+
47
+
48
+ def answer_model() -> str:
49
+ return os.environ.get("ANSWER_MODEL", "gpt-4o-mini")
50
+
51
+
52
+ # ── Source adapter (one config point, gated) ─────────────────────────────────
53
+
54
+ def source_type() -> str:
55
+ """dropbox | local | url. Selects the document-source adapter."""
56
+ return os.environ.get("SOURCE_TYPE", "dropbox")
57
+
58
+
59
+ def dropbox_config() -> dict[str, str | None]:
60
+ return {
61
+ "app_key": os.environ.get("DROPBOX_APP_KEY"),
62
+ "app_secret": os.environ.get("DROPBOX_APP_SECRET"),
63
+ "refresh_token": os.environ.get("DROPBOX_REFRESH_TOKEN"),
64
+ "folder_path": os.environ.get("DROPBOX_FOLDER_PATH", ""),
65
+ }
66
+
67
+
68
+ def local_source_path() -> str:
69
+ """Directory the local adapter reads. Point at a `needs: disk` mount
70
+ (DISK_PATH) or a bundled folder."""
71
+ return os.environ.get("LOCAL_SOURCE_PATH", os.environ.get("DISK_PATH", ""))
72
+
73
+
74
+ def url_source_manifest() -> str | None:
75
+ """A newline- or comma-separated list of document URLs the url adapter
76
+ fetches. Set as a secret or plain env var."""
77
+ return os.environ.get("URL_SOURCE_MANIFEST")
78
+
79
+
80
+ def source_configured() -> bool:
81
+ st = source_type()
82
+ if st == "dropbox":
83
+ c = dropbox_config()
84
+ return bool(c["app_key"] and c["app_secret"] and c["refresh_token"])
85
+ if st == "local":
86
+ return bool(local_source_path())
87
+ if st == "url":
88
+ return bool(url_source_manifest())
89
+ return False
90
+
91
+
92
+ # ── Manager auth (gated) ──────────────────────────────────────────────────────
93
+
94
+ def manager_password_hash() -> str | None:
95
+ """A sha256 hex digest of the manager password. Compared server-side."""
96
+ return os.environ.get("MANAGER_PASSWORD_HASH")
97
+
98
+
99
+ def manager_auth_configured() -> bool:
100
+ return bool(manager_password_hash())
101
+
102
+
103
+ # ── Search blend weights ─────────────────────────────────────────────────────
104
+
105
+ def semantic_weight() -> float:
106
+ return float(os.environ.get("SEMANTIC_WEIGHT", "0.55"))
107
+
108
+
109
+ def keyword_weight() -> float:
110
+ return float(os.environ.get("KEYWORD_WEIGHT", "0.45"))