@cairnkeep/cli 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/CONTRIBUTING.md +6 -1
- package/README.md +43 -0
- package/docs/building-an-overlay.md +9 -0
- package/docs/domain-knowledge.md +14 -10
- package/docs/operating.md +2 -0
- package/docs/overlay-distributions.md +97 -0
- package/docs/privacy-and-data-flow.md +49 -0
- package/docs/releasing.md +33 -0
- package/docs/storage.md +24 -0
- package/examples/anythingllm/README.md +66 -0
- package/examples/anythingllm/anythingllm-projects.example.json +30 -0
- package/examples/anythingllm/sync_to_anythingllm.py +596 -0
- package/examples/overlay-distribution/cairnkeep.overlay.json +25 -0
- package/mcp-memory-server/dist/index.js +3 -4
- package/package.json +7 -3
- package/schemas/cairnkeep-overlay.schema.json +79 -0
- package/scripts/probe-memory-server.mjs +17 -5
- package/scripts/smoke-mcp-boot.sh +6 -13
- package/scripts/test-overlay-schema.sh +41 -0
- package/scripts/test-package-install.sh +4 -0
- package/templates/env.example.template +6 -0
|
@@ -0,0 +1,596 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Multi-project incremental sync into AnythingLLM.
|
|
4
|
+
|
|
5
|
+
Each project repo gets its own AnythingLLM workspace. Projects are configured
|
|
6
|
+
in the file selected by CAIRN_ANYTHINGLLM_PROJECTS_FILE, the legacy file next to
|
|
7
|
+
this script when present, or the XDG user config directory.
|
|
8
|
+
|
|
9
|
+
This is the bundled reference implementation wired to Cairnkeep's
|
|
10
|
+
`domain_knowledge_sync` tool. See ../../docs/domain-knowledge.md.
|
|
11
|
+
|
|
12
|
+
Usage:
|
|
13
|
+
# Sync all configured projects (normal workflow after git pull):
|
|
14
|
+
python3 sync_to_anythingllm.py
|
|
15
|
+
|
|
16
|
+
# Only sync one project:
|
|
17
|
+
python3 sync_to_anythingllm.py --project my-project
|
|
18
|
+
|
|
19
|
+
# Preview without making changes:
|
|
20
|
+
python3 sync_to_anythingllm.py --dry-run
|
|
21
|
+
|
|
22
|
+
# First-time upload for a new project:
|
|
23
|
+
python3 sync_to_anythingllm.py --full --project my-project
|
|
24
|
+
|
|
25
|
+
# Replace stale embedded workspace docs, then upload current configured docs:
|
|
26
|
+
python3 sync_to_anythingllm.py --replace --project my-project
|
|
27
|
+
|
|
28
|
+
# Rebuild state from existing store (recovery, no re-upload):
|
|
29
|
+
python3 sync_to_anythingllm.py --rebuild-state --project my-project
|
|
30
|
+
|
|
31
|
+
# List all configured projects and file counts:
|
|
32
|
+
python3 sync_to_anythingllm.py --list
|
|
33
|
+
|
|
34
|
+
Config: ${XDG_CONFIG_HOME:-~/.config}/cairnkeep/anythingllm-projects.json
|
|
35
|
+
State: ${XDG_STATE_HOME:-~/.local/state}/cairnkeep/anythingllm-sync.json
|
|
36
|
+
|
|
37
|
+
State file format:
|
|
38
|
+
{
|
|
39
|
+
"project-a": { "docs/foo.md": {"location": "custom-documents/foo-uuid.json", "sha256": "..."} },
|
|
40
|
+
"project-b": { ... },
|
|
41
|
+
...
|
|
42
|
+
}
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
import argparse
|
|
46
|
+
import fnmatch
|
|
47
|
+
import hashlib
|
|
48
|
+
import json
|
|
49
|
+
import os
|
|
50
|
+
import sys
|
|
51
|
+
import time
|
|
52
|
+
from pathlib import Path
|
|
53
|
+
|
|
54
|
+
import requests
|
|
55
|
+
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
# Configuration
|
|
58
|
+
# ---------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
BASE_URL = f"{os.environ.get('ANYTHINGLLM_BASE_URL', 'http://localhost:3001').rstrip('/')}/api/v1"
|
|
61
|
+
API_KEY = os.environ.get("ANYTHINGLLM_API_KEY", "")
|
|
62
|
+
|
|
63
|
+
SCRIPT_DIR = Path(__file__).parent
|
|
64
|
+
LEGACY_CONFIG_FILE = SCRIPT_DIR / "anythingllm-projects.json"
|
|
65
|
+
CONFIG_HOME = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config"))
|
|
66
|
+
STATE_HOME = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state"))
|
|
67
|
+
|
|
68
|
+
if os.environ.get("CAIRN_ANYTHINGLLM_PROJECTS_FILE"):
|
|
69
|
+
CONFIG_FILE = Path(os.environ["CAIRN_ANYTHINGLLM_PROJECTS_FILE"]).expanduser()
|
|
70
|
+
elif LEGACY_CONFIG_FILE.exists():
|
|
71
|
+
CONFIG_FILE = LEGACY_CONFIG_FILE
|
|
72
|
+
else:
|
|
73
|
+
CONFIG_FILE = CONFIG_HOME / "cairnkeep" / "anythingllm-projects.json"
|
|
74
|
+
|
|
75
|
+
if os.environ.get("CAIRN_ANYTHINGLLM_STATE_FILE"):
|
|
76
|
+
STATE_FILE = Path(os.environ["CAIRN_ANYTHINGLLM_STATE_FILE"]).expanduser()
|
|
77
|
+
elif CONFIG_FILE == LEGACY_CONFIG_FILE:
|
|
78
|
+
STATE_FILE = SCRIPT_DIR / ".anythingllm-sync.json"
|
|
79
|
+
else:
|
|
80
|
+
STATE_FILE = STATE_HOME / "cairnkeep" / "anythingllm-sync.json"
|
|
81
|
+
|
|
82
|
+
HEADERS = {"Authorization": f"Bearer {API_KEY}"}
|
|
83
|
+
JSON_HEADERS = {**HEADERS, "Content-Type": "application/json"}
|
|
84
|
+
|
|
85
|
+
EMBED_BATCH_SIZE = 5
|
|
86
|
+
EMBED_TIMEOUT = 600 # seconds per embed batch
|
|
87
|
+
|
|
88
|
+
WORKSPACE_SYSTEM_PROMPT = (
|
|
89
|
+
"You are an assistant for the {display_name} documentation. "
|
|
90
|
+
"Answer the user question using ONLY the context documents provided below. "
|
|
91
|
+
"Do not rely on prior knowledge or make assumptions beyond what is explicitly "
|
|
92
|
+
"stated in the documents.\n\n"
|
|
93
|
+
"Rules:\n"
|
|
94
|
+
"- If the answer is clearly present in the context, provide a precise and structured answer.\n"
|
|
95
|
+
"- If the context does not contain enough information to answer confidently, respond: "
|
|
96
|
+
'"This information is not available in the provided documentation."\n'
|
|
97
|
+
"- Never fabricate component names, port numbers, version numbers, or procedures "
|
|
98
|
+
"not in the context.\n"
|
|
99
|
+
"- Cite which document or section your answer comes from when possible."
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def require_api_key() -> None:
|
|
104
|
+
if API_KEY:
|
|
105
|
+
return
|
|
106
|
+
print("Error: ANYTHINGLLM_API_KEY is not set.")
|
|
107
|
+
sys.exit(1)
|
|
108
|
+
|
|
109
|
+
# ---------------------------------------------------------------------------
|
|
110
|
+
# Config loading
|
|
111
|
+
# ---------------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
def load_config() -> list[dict]:
|
|
114
|
+
if not CONFIG_FILE.exists():
|
|
115
|
+
print(f"Error: Config file not found: {CONFIG_FILE}")
|
|
116
|
+
sys.exit(1)
|
|
117
|
+
data = json.loads(CONFIG_FILE.read_text())
|
|
118
|
+
return data["projects"]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
# State file helpers
|
|
123
|
+
# ---------------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
def load_state() -> dict:
|
|
126
|
+
"""
|
|
127
|
+
Load full state: {project_slug: {relative_path: {location, sha256}}}
|
|
128
|
+
|
|
129
|
+
Automatically migrates a legacy flat (single-project) state file into the
|
|
130
|
+
nested multi-project format, keyed under the "default" slug.
|
|
131
|
+
"""
|
|
132
|
+
if not STATE_FILE.exists():
|
|
133
|
+
return {}
|
|
134
|
+
raw = json.loads(STATE_FILE.read_text())
|
|
135
|
+
if not raw:
|
|
136
|
+
return {}
|
|
137
|
+
# Detect old flat format: top-level values are {location, sha256} dicts
|
|
138
|
+
first_val = next(iter(raw.values()))
|
|
139
|
+
if isinstance(first_val, dict) and "location" in first_val:
|
|
140
|
+
print("[INFO] Migrating state file from flat to multi-project format...")
|
|
141
|
+
return {"default": raw}
|
|
142
|
+
return raw
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def save_state(state: dict) -> None:
|
|
146
|
+
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
147
|
+
STATE_FILE.write_text(json.dumps(state, indent=2))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def file_sha256(path: Path) -> str:
|
|
151
|
+
h = hashlib.sha256()
|
|
152
|
+
h.update(path.read_bytes())
|
|
153
|
+
return h.hexdigest()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# ---------------------------------------------------------------------------
|
|
157
|
+
# File collection
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
|
|
160
|
+
def is_excluded(rel: str, excludes: list[str]) -> bool:
|
|
161
|
+
"""Return True if the relative path matches any exclude pattern."""
|
|
162
|
+
name = os.path.basename(rel)
|
|
163
|
+
for exc in excludes:
|
|
164
|
+
if fnmatch.fnmatch(rel, exc):
|
|
165
|
+
return True
|
|
166
|
+
if fnmatch.fnmatch(name, exc):
|
|
167
|
+
return True
|
|
168
|
+
return False
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def collect_files(project: dict) -> dict[str, Path]:
|
|
172
|
+
"""
|
|
173
|
+
Return {relative_path_str: absolute_Path} for all files matching the
|
|
174
|
+
project's include patterns minus any exclude patterns. Paths are relative
|
|
175
|
+
to the project root.
|
|
176
|
+
"""
|
|
177
|
+
root = Path(project["path"])
|
|
178
|
+
patterns = project.get("patterns", ["docs/**/*.md"])
|
|
179
|
+
excludes = project.get("exclude_patterns", [])
|
|
180
|
+
|
|
181
|
+
result: dict[str, Path] = {}
|
|
182
|
+
for pattern in patterns:
|
|
183
|
+
for p in root.glob(pattern):
|
|
184
|
+
if not p.is_file():
|
|
185
|
+
continue
|
|
186
|
+
rel = str(p.relative_to(root))
|
|
187
|
+
if not is_excluded(rel, excludes):
|
|
188
|
+
result[rel] = p
|
|
189
|
+
return result
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
# ---------------------------------------------------------------------------
|
|
193
|
+
# AnythingLLM API helpers
|
|
194
|
+
# ---------------------------------------------------------------------------
|
|
195
|
+
|
|
196
|
+
def ensure_workspace(slug: str, display_name: str) -> str:
|
|
197
|
+
"""
|
|
198
|
+
Create the workspace if it doesn't exist, then apply standard RAG settings.
|
|
199
|
+
Returns the actual workspace slug to use for all subsequent API calls,
|
|
200
|
+
or empty string on failure.
|
|
201
|
+
|
|
202
|
+
Uses the project slug as the workspace name so AnythingLLM generates
|
|
203
|
+
a matching slug (e.g. name='my-project' -> slug='my-project').
|
|
204
|
+
"""
|
|
205
|
+
resp = requests.get(f"{BASE_URL}/workspaces", headers=HEADERS, timeout=10)
|
|
206
|
+
if resp.status_code == 200:
|
|
207
|
+
for ws in resp.json().get("workspaces", []):
|
|
208
|
+
if ws["slug"] == slug:
|
|
209
|
+
update_workspace_settings(slug, display_name)
|
|
210
|
+
return slug
|
|
211
|
+
|
|
212
|
+
# Create -- use slug as name so AnythingLLM generates the exact slug we want
|
|
213
|
+
resp = requests.post(
|
|
214
|
+
f"{BASE_URL}/workspace/new",
|
|
215
|
+
headers=JSON_HEADERS,
|
|
216
|
+
data=json.dumps({"name": slug}),
|
|
217
|
+
timeout=10,
|
|
218
|
+
)
|
|
219
|
+
if resp.status_code != 200:
|
|
220
|
+
print(f" [WARN] Could not create workspace '{slug}': {resp.status_code} {resp.text[:200]}")
|
|
221
|
+
return ""
|
|
222
|
+
|
|
223
|
+
actual_slug = resp.json().get("workspace", {}).get("slug", slug)
|
|
224
|
+
|
|
225
|
+
update_workspace_settings(actual_slug, display_name)
|
|
226
|
+
print(f" Created workspace: {actual_slug}")
|
|
227
|
+
return actual_slug
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def update_workspace_settings(slug: str, display_name: str) -> None:
|
|
231
|
+
"""Apply standard RAG settings and project-specific system prompt."""
|
|
232
|
+
requests.post(
|
|
233
|
+
f"{BASE_URL}/workspace/{slug}/update",
|
|
234
|
+
headers=JSON_HEADERS,
|
|
235
|
+
data=json.dumps({
|
|
236
|
+
"openAiTemp": 0,
|
|
237
|
+
"topN": 6,
|
|
238
|
+
"similarityThreshold": 0.20,
|
|
239
|
+
"openAiPrompt": WORKSPACE_SYSTEM_PROMPT.format(display_name=display_name),
|
|
240
|
+
"queryRefusalResponse": f"This information is not available in the provided {display_name} documentation.",
|
|
241
|
+
"chatMode": "automatic",
|
|
242
|
+
}),
|
|
243
|
+
timeout=10,
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def upload_file(filepath: Path) -> str | None:
|
|
248
|
+
"""Upload a single markdown file. Returns the location string or None."""
|
|
249
|
+
with open(filepath, "rb") as f:
|
|
250
|
+
resp = requests.post(
|
|
251
|
+
f"{BASE_URL}/document/upload",
|
|
252
|
+
headers=HEADERS,
|
|
253
|
+
files={"file": (filepath.name, f, "text/markdown")},
|
|
254
|
+
timeout=60,
|
|
255
|
+
)
|
|
256
|
+
if resp.status_code == 200:
|
|
257
|
+
docs = resp.json().get("documents", [])
|
|
258
|
+
if docs:
|
|
259
|
+
return docs[0].get("location")
|
|
260
|
+
print(f" [WARN] Upload failed for {filepath.name}: {resp.status_code} {resp.text[:200]}")
|
|
261
|
+
return None
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def embed_locations(slug: str, locations: list[str]) -> bool:
|
|
265
|
+
"""Embed document locations into a workspace in batches."""
|
|
266
|
+
for i in range(0, len(locations), EMBED_BATCH_SIZE):
|
|
267
|
+
batch = locations[i: i + EMBED_BATCH_SIZE]
|
|
268
|
+
resp = requests.post(
|
|
269
|
+
f"{BASE_URL}/workspace/{slug}/update-embeddings",
|
|
270
|
+
headers=JSON_HEADERS,
|
|
271
|
+
data=json.dumps({"adds": batch, "deletes": []}),
|
|
272
|
+
timeout=EMBED_TIMEOUT,
|
|
273
|
+
)
|
|
274
|
+
if resp.status_code != 200:
|
|
275
|
+
print(f" [WARN] Embed failed: {resp.status_code} {resp.text[:200]}")
|
|
276
|
+
return False
|
|
277
|
+
time.sleep(1)
|
|
278
|
+
return True
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def unembed_locations(slug: str, locations: list[str]) -> bool:
|
|
282
|
+
"""Remove document locations from a workspace (un-embed only, keeps store)."""
|
|
283
|
+
if not locations:
|
|
284
|
+
return True
|
|
285
|
+
resp = requests.post(
|
|
286
|
+
f"{BASE_URL}/workspace/{slug}/update-embeddings",
|
|
287
|
+
headers=JSON_HEADERS,
|
|
288
|
+
data=json.dumps({"adds": [], "deletes": locations}),
|
|
289
|
+
timeout=60,
|
|
290
|
+
)
|
|
291
|
+
return resp.status_code == 200
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def get_workspace_locations(slug: str) -> list[str]:
|
|
295
|
+
"""Return document locations currently embedded in the workspace."""
|
|
296
|
+
resp = requests.get(f"{BASE_URL}/workspace/{slug}", headers=HEADERS, timeout=30)
|
|
297
|
+
if resp.status_code != 200:
|
|
298
|
+
print(f" [WARN] Could not inspect workspace '{slug}': {resp.status_code} {resp.text[:200]}")
|
|
299
|
+
return []
|
|
300
|
+
|
|
301
|
+
data = resp.json()
|
|
302
|
+
workspaces = data.get("workspace", [])
|
|
303
|
+
workspace = workspaces[0] if workspaces else {}
|
|
304
|
+
return [
|
|
305
|
+
doc.get("docpath", "")
|
|
306
|
+
for doc in workspace.get("documents", [])
|
|
307
|
+
if doc.get("docpath")
|
|
308
|
+
]
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def delete_from_store(locations: list[str]) -> bool:
|
|
312
|
+
"""Permanently delete documents from the global document store."""
|
|
313
|
+
if not locations:
|
|
314
|
+
return True
|
|
315
|
+
resp = requests.delete(
|
|
316
|
+
f"{BASE_URL}/system/remove-documents",
|
|
317
|
+
headers=JSON_HEADERS,
|
|
318
|
+
data=json.dumps({"names": locations}),
|
|
319
|
+
timeout=60,
|
|
320
|
+
)
|
|
321
|
+
return resp.status_code == 200
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
# ---------------------------------------------------------------------------
|
|
325
|
+
# Diff logic
|
|
326
|
+
# ---------------------------------------------------------------------------
|
|
327
|
+
|
|
328
|
+
def compute_diff(
|
|
329
|
+
project_state: dict, files: dict[str, Path]
|
|
330
|
+
) -> tuple[list[str], list[str], list[str]]:
|
|
331
|
+
"""Return (added, modified, deleted) lists of relative path strings."""
|
|
332
|
+
added = [rel for rel in files if rel not in project_state]
|
|
333
|
+
deleted = [rel for rel in project_state if rel not in files]
|
|
334
|
+
modified = [
|
|
335
|
+
rel for rel, path in files.items()
|
|
336
|
+
if rel in project_state and file_sha256(path) != project_state[rel].get("sha256")
|
|
337
|
+
]
|
|
338
|
+
return added, modified, deleted
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
# ---------------------------------------------------------------------------
|
|
342
|
+
# Per-project operations
|
|
343
|
+
# ---------------------------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
def full_sync_project(project: dict, state: dict, dry_run: bool) -> None:
|
|
346
|
+
slug = project["slug"]
|
|
347
|
+
display_name = project.get("display_name", slug)
|
|
348
|
+
files = collect_files(project)
|
|
349
|
+
|
|
350
|
+
print(f"\n=== Full sync: {display_name} ({len(files)} files) ===")
|
|
351
|
+
|
|
352
|
+
if dry_run:
|
|
353
|
+
for rel in sorted(files):
|
|
354
|
+
print(f" would upload: {rel}")
|
|
355
|
+
return
|
|
356
|
+
|
|
357
|
+
actual_slug = ensure_workspace(slug, display_name)
|
|
358
|
+
if not actual_slug:
|
|
359
|
+
return
|
|
360
|
+
|
|
361
|
+
project_state: dict = {}
|
|
362
|
+
locations: list[str] = []
|
|
363
|
+
|
|
364
|
+
print(" Uploading...")
|
|
365
|
+
for i, (rel, filepath) in enumerate(sorted(files.items()), 1):
|
|
366
|
+
print(f" [{i}/{len(files)}] {rel}")
|
|
367
|
+
loc = upload_file(filepath)
|
|
368
|
+
if loc:
|
|
369
|
+
project_state[rel] = {"location": loc, "sha256": file_sha256(filepath)}
|
|
370
|
+
locations.append(loc)
|
|
371
|
+
time.sleep(0.1)
|
|
372
|
+
|
|
373
|
+
print(f" Embedding {len(locations)} docs...")
|
|
374
|
+
ok = embed_locations(actual_slug, locations)
|
|
375
|
+
if not ok:
|
|
376
|
+
print(" [WARN] Some embed batches failed. State saved; retry with --full to re-embed.")
|
|
377
|
+
|
|
378
|
+
state[slug] = project_state
|
|
379
|
+
save_state(state)
|
|
380
|
+
print(f" Done. {len(locations)} docs uploaded to workspace '{actual_slug}'")
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def replace_sync_project(project: dict, state: dict, dry_run: bool) -> None:
|
|
384
|
+
"""Remove current workspace docs, then full-sync configured project docs."""
|
|
385
|
+
slug = project["slug"]
|
|
386
|
+
display_name = project.get("display_name", slug)
|
|
387
|
+
files = collect_files(project)
|
|
388
|
+
|
|
389
|
+
print(f"\n=== Replace sync: {display_name} ({len(files)} files) ===")
|
|
390
|
+
|
|
391
|
+
if dry_run:
|
|
392
|
+
print(f" would remove embedded docs from workspace: {slug}")
|
|
393
|
+
for rel in sorted(files):
|
|
394
|
+
print(f" would upload: {rel}")
|
|
395
|
+
return
|
|
396
|
+
|
|
397
|
+
actual_slug = ensure_workspace(slug, display_name)
|
|
398
|
+
if not actual_slug:
|
|
399
|
+
return
|
|
400
|
+
|
|
401
|
+
existing_locations = get_workspace_locations(actual_slug)
|
|
402
|
+
if existing_locations:
|
|
403
|
+
print(f" Removing {len(existing_locations)} existing embedded doc(s)...")
|
|
404
|
+
unembed_locations(actual_slug, existing_locations)
|
|
405
|
+
delete_from_store(existing_locations)
|
|
406
|
+
|
|
407
|
+
state[slug] = {}
|
|
408
|
+
save_state(state)
|
|
409
|
+
full_sync_project(project, state, dry_run=False)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def incremental_sync_project(project: dict, state: dict, dry_run: bool) -> None:
|
|
413
|
+
slug = project["slug"]
|
|
414
|
+
display_name = project.get("display_name", slug)
|
|
415
|
+
actual_slug = slug if dry_run else (ensure_workspace(slug, display_name) or slug)
|
|
416
|
+
project_state = state.get(slug, {})
|
|
417
|
+
|
|
418
|
+
if not project_state:
|
|
419
|
+
print(
|
|
420
|
+
f"\n[{display_name}] No sync state found. "
|
|
421
|
+
f"Run --full --project {slug} for initial upload."
|
|
422
|
+
)
|
|
423
|
+
return
|
|
424
|
+
|
|
425
|
+
files = collect_files(project)
|
|
426
|
+
added, modified, deleted = compute_diff(project_state, files)
|
|
427
|
+
|
|
428
|
+
if not added and not modified and not deleted:
|
|
429
|
+
print(f"[{display_name}] Up to date.")
|
|
430
|
+
return
|
|
431
|
+
|
|
432
|
+
print(f"\n=== [{display_name}] Changes: +{len(added)} ~{len(modified)} -{len(deleted)} ===")
|
|
433
|
+
|
|
434
|
+
if dry_run:
|
|
435
|
+
for rel in sorted(added):
|
|
436
|
+
print(f" + {rel}")
|
|
437
|
+
for rel in sorted(modified):
|
|
438
|
+
print(f" ~ {rel}")
|
|
439
|
+
for rel in sorted(deleted):
|
|
440
|
+
print(f" - {rel}")
|
|
441
|
+
return
|
|
442
|
+
|
|
443
|
+
if deleted:
|
|
444
|
+
old_locs = [project_state[rel]["location"] for rel in deleted if "location" in project_state[rel]]
|
|
445
|
+
unembed_locations(actual_slug, old_locs)
|
|
446
|
+
delete_from_store(old_locs)
|
|
447
|
+
for rel in deleted:
|
|
448
|
+
del project_state[rel]
|
|
449
|
+
print(f" [DEL] {rel}")
|
|
450
|
+
state[slug] = project_state
|
|
451
|
+
save_state(state)
|
|
452
|
+
|
|
453
|
+
if modified:
|
|
454
|
+
old_locs = [project_state[rel]["location"] for rel in modified if "location" in project_state[rel]]
|
|
455
|
+
unembed_locations(actual_slug, old_locs)
|
|
456
|
+
delete_from_store(old_locs)
|
|
457
|
+
new_locs: list[str] = []
|
|
458
|
+
for rel in modified:
|
|
459
|
+
filepath = files[rel]
|
|
460
|
+
print(f" [MOD] {rel}")
|
|
461
|
+
loc = upload_file(filepath)
|
|
462
|
+
if loc:
|
|
463
|
+
project_state[rel] = {"location": loc, "sha256": file_sha256(filepath)}
|
|
464
|
+
new_locs.append(loc)
|
|
465
|
+
time.sleep(0.1)
|
|
466
|
+
if new_locs:
|
|
467
|
+
embed_locations(actual_slug, new_locs)
|
|
468
|
+
state[slug] = project_state
|
|
469
|
+
save_state(state)
|
|
470
|
+
|
|
471
|
+
if added:
|
|
472
|
+
new_locs = []
|
|
473
|
+
for rel in sorted(added):
|
|
474
|
+
filepath = files[rel]
|
|
475
|
+
print(f" [ADD] {rel}")
|
|
476
|
+
loc = upload_file(filepath)
|
|
477
|
+
if loc:
|
|
478
|
+
project_state[rel] = {"location": loc, "sha256": file_sha256(filepath)}
|
|
479
|
+
new_locs.append(loc)
|
|
480
|
+
time.sleep(0.1)
|
|
481
|
+
if new_locs:
|
|
482
|
+
embed_locations(actual_slug, new_locs)
|
|
483
|
+
state[slug] = project_state
|
|
484
|
+
save_state(state)
|
|
485
|
+
|
|
486
|
+
print(f" Done.")
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def rebuild_state_project(project: dict, state: dict) -> None:
|
|
490
|
+
"""
|
|
491
|
+
Rebuild state by matching files against documents already in the store.
|
|
492
|
+
Useful for recovery when the state file is lost after docs are already uploaded.
|
|
493
|
+
Note: matches by filename (title); may be ambiguous if two projects share filenames.
|
|
494
|
+
"""
|
|
495
|
+
slug = project["slug"]
|
|
496
|
+
display_name = project.get("display_name", slug)
|
|
497
|
+
|
|
498
|
+
resp = requests.get(f"{BASE_URL}/documents", headers=HEADERS, timeout=30)
|
|
499
|
+
resp.raise_for_status()
|
|
500
|
+
|
|
501
|
+
store_docs: dict[str, dict] = {}
|
|
502
|
+
for folder in resp.json().get("localFiles", {}).get("items", []):
|
|
503
|
+
folder_name = folder.get("name", "")
|
|
504
|
+
for item in folder.get("items", []):
|
|
505
|
+
title = item.get("title", "")
|
|
506
|
+
loc = f"{folder_name}/{item['name']}"
|
|
507
|
+
if title not in store_docs or item.get("published", "") > store_docs[title]["published"]:
|
|
508
|
+
store_docs[title] = {"location": loc, "published": item.get("published", "")}
|
|
509
|
+
|
|
510
|
+
files = collect_files(project)
|
|
511
|
+
project_state: dict = {}
|
|
512
|
+
matched = 0
|
|
513
|
+
unmatched: list[str] = []
|
|
514
|
+
|
|
515
|
+
for rel, filepath in files.items():
|
|
516
|
+
title = filepath.name
|
|
517
|
+
if title in store_docs:
|
|
518
|
+
project_state[rel] = {
|
|
519
|
+
"location": store_docs[title]["location"],
|
|
520
|
+
"sha256": file_sha256(filepath),
|
|
521
|
+
}
|
|
522
|
+
matched += 1
|
|
523
|
+
else:
|
|
524
|
+
unmatched.append(rel)
|
|
525
|
+
|
|
526
|
+
state[slug] = project_state
|
|
527
|
+
save_state(state)
|
|
528
|
+
print(f"[{display_name}] Rebuilt: {matched} matched, {len(unmatched)} unmatched")
|
|
529
|
+
if unmatched:
|
|
530
|
+
for u in unmatched:
|
|
531
|
+
print(f" unmatched: {u}")
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
# ---------------------------------------------------------------------------
|
|
535
|
+
# Entry point
|
|
536
|
+
# ---------------------------------------------------------------------------
|
|
537
|
+
|
|
538
|
+
def main() -> None:
|
|
539
|
+
parser = argparse.ArgumentParser(
|
|
540
|
+
description="Multi-project AnythingLLM sync",
|
|
541
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
542
|
+
epilog=(
|
|
543
|
+
"examples:\n"
|
|
544
|
+
" sync all projects after git pull:\n"
|
|
545
|
+
" python3 sync_to_anythingllm.py\n\n"
|
|
546
|
+
" first-time upload for a new project:\n"
|
|
547
|
+
" python3 sync_to_anythingllm.py --full --project my-project\n\n"
|
|
548
|
+
" preview what would change:\n"
|
|
549
|
+
" python3 sync_to_anythingllm.py --dry-run\n"
|
|
550
|
+
),
|
|
551
|
+
)
|
|
552
|
+
parser.add_argument(
|
|
553
|
+
"--project", metavar="SLUG",
|
|
554
|
+
help="Only operate on this project (slug from anythingllm-projects.json).",
|
|
555
|
+
)
|
|
556
|
+
group = parser.add_mutually_exclusive_group()
|
|
557
|
+
group.add_argument("--full", action="store_true", help="Full re-upload and re-embed")
|
|
558
|
+
group.add_argument("--replace", action="store_true", help="Remove currently embedded workspace docs, then full re-upload and re-embed")
|
|
559
|
+
group.add_argument("--rebuild-state", action="store_true", help="Rebuild state from existing store (no upload)")
|
|
560
|
+
group.add_argument("--list", action="store_true", help="List configured projects and file counts")
|
|
561
|
+
parser.add_argument("--dry-run", action="store_true", help="Show what would change without doing it")
|
|
562
|
+
args = parser.parse_args()
|
|
563
|
+
|
|
564
|
+
projects = load_config()
|
|
565
|
+
slug_map = {p["slug"]: p for p in projects}
|
|
566
|
+
|
|
567
|
+
if args.list:
|
|
568
|
+
print(f"{'SLUG':<25} {'FILES':>6} PATH")
|
|
569
|
+
print("-" * 70)
|
|
570
|
+
for p in projects:
|
|
571
|
+
files = collect_files(p)
|
|
572
|
+
print(f"{p['slug']:<25} {len(files):>6} {p['path']}")
|
|
573
|
+
return
|
|
574
|
+
|
|
575
|
+
require_api_key()
|
|
576
|
+
|
|
577
|
+
if args.project and args.project not in slug_map:
|
|
578
|
+
print(f"Error: Unknown project '{args.project}'. Known: {', '.join(slug_map)}")
|
|
579
|
+
sys.exit(1)
|
|
580
|
+
|
|
581
|
+
target_projects = [slug_map[args.project]] if args.project else projects
|
|
582
|
+
state = load_state()
|
|
583
|
+
|
|
584
|
+
for project in target_projects:
|
|
585
|
+
if args.rebuild_state:
|
|
586
|
+
rebuild_state_project(project, state)
|
|
587
|
+
elif args.replace:
|
|
588
|
+
replace_sync_project(project, state, dry_run=args.dry_run)
|
|
589
|
+
elif args.full:
|
|
590
|
+
full_sync_project(project, state, dry_run=args.dry_run)
|
|
591
|
+
else:
|
|
592
|
+
incremental_sync_project(project, state, dry_run=args.dry_run)
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
if __name__ == "__main__":
|
|
596
|
+
main()
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "../../schemas/cairnkeep-overlay.schema.json",
|
|
3
|
+
"schemaVersion": 1,
|
|
4
|
+
"id": "team-managed",
|
|
5
|
+
"name": "Example managed Cairnkeep distribution",
|
|
6
|
+
"version": "1.0.0",
|
|
7
|
+
"core": {
|
|
8
|
+
"package": "@cairnkeep/cli",
|
|
9
|
+
"version": "1.1.2"
|
|
10
|
+
},
|
|
11
|
+
"policy": {
|
|
12
|
+
"memory": {
|
|
13
|
+
"transport": "stdio",
|
|
14
|
+
"storage": "local",
|
|
15
|
+
"baseDir": "${HOME}/.cairnkeep"
|
|
16
|
+
},
|
|
17
|
+
"rag": {"mode": "disabled"},
|
|
18
|
+
"gitProvider": "none"
|
|
19
|
+
},
|
|
20
|
+
"bootstrap": {
|
|
21
|
+
"defaultMode": "untracked",
|
|
22
|
+
"launcher": ".ai/start-managed-agent.sh",
|
|
23
|
+
"profileLock": ".ai/cairnkeep-profile.json"
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -17,12 +17,11 @@ class ClientContextError extends Error {
|
|
|
17
17
|
}
|
|
18
18
|
const moduleDir = dirname(fileURLToPath(import.meta.url));
|
|
19
19
|
const infraRoot = resolve(moduleDir, "..", "..");
|
|
20
|
-
// Path to the
|
|
21
|
-
//
|
|
22
|
-
// (the integration scripts often live in the wrapper, not the core).
|
|
20
|
+
// Path to the bundled deployment-neutral sync script. An overlay can replace it
|
|
21
|
+
// with a deployment-specific implementation.
|
|
23
22
|
const anythingllmSyncScript = process.env.CAIRN_ANYTHINGLLM_SYNC_SCRIPT
|
|
24
23
|
? resolve(expandHome(process.env.CAIRN_ANYTHINGLLM_SYNC_SCRIPT))
|
|
25
|
-
: join(infraRoot, "anythingllm", "sync_to_anythingllm.py");
|
|
24
|
+
: join(infraRoot, "examples", "anythingllm", "sync_to_anythingllm.py");
|
|
26
25
|
const HISTORY_NAMESPACE = "__history__";
|
|
27
26
|
const PROJECT_ID_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/;
|
|
28
27
|
const WORKSPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cairnkeep/cli",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.2",
|
|
4
4
|
"description": "A durable, harness-agnostic memory + context layer for coding agents (Claude Code, OpenCode).",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -16,7 +16,11 @@
|
|
|
16
16
|
"templates/",
|
|
17
17
|
"claude/",
|
|
18
18
|
"opencode/",
|
|
19
|
+
"examples/anythingllm/",
|
|
20
|
+
"examples/overlay-distribution/",
|
|
21
|
+
"schemas/",
|
|
19
22
|
"docs/",
|
|
23
|
+
"CHANGELOG.md",
|
|
20
24
|
"CONTRIBUTING.md",
|
|
21
25
|
"SECURITY.md",
|
|
22
26
|
"mcp-memory-server/dist/",
|
|
@@ -36,8 +40,8 @@
|
|
|
36
40
|
"fix:perms": "git ls-files -s | awk '$1==\"100755\"{print $4}' | xargs chmod 755",
|
|
37
41
|
"prepack": "npm run build:server && npm run fix:perms",
|
|
38
42
|
"test": "for test in scripts/test-*.sh; do \"$test\" || exit 1; done",
|
|
39
|
-
"check:public": "scripts/verify-no-private-references.sh && scripts/verify-docs-parity.sh && npm test",
|
|
40
|
-
"prepublishOnly": "npm run check:public && npm
|
|
43
|
+
"check:public": "scripts/verify-no-private-references.sh && scripts/verify-docs-parity.sh && npm run build:server && npm test",
|
|
44
|
+
"prepublishOnly": "npm run check:public && npm --prefix mcp-memory-server test --global=false --dry-run=false"
|
|
41
45
|
},
|
|
42
46
|
"repository": {
|
|
43
47
|
"type": "git",
|