@bcmyguest/openmem 0.0.0-stage → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. package/LICENSE +190 -0
  2. package/README.md +136 -2
  3. package/bin/hindsight-project-index +530 -0
  4. package/cli/coding-agent/access-policy.mjs +38 -0
  5. package/cli/coding-agent/agents.mjs +34 -0
  6. package/cli/coding-agent/bank-from-cwd.mjs +140 -0
  7. package/cli/coding-agent/capabilities.mjs +23 -0
  8. package/cli/coding-agent/hook-harnesses.mjs +87 -0
  9. package/cli/coding-agent/hook-runner.mjs +260 -0
  10. package/cli/coding-agent/install-access.mjs +93 -0
  11. package/cli/coding-agent/install-shim.mjs +97 -0
  12. package/cli/coding-agent/opencode-config.mjs +91 -0
  13. package/cli/coding-agent/opencode-plugin.mjs +96 -0
  14. package/cli/coding-agent/runtime.mjs +39 -0
  15. package/cli/coding-agent/session-facade.mjs +62 -0
  16. package/cli/cognee-mcp.mjs +260 -0
  17. package/cli/feed-sync.mjs +11 -0
  18. package/cli/install-coding-agent.mjs +165 -0
  19. package/cli/install-memory-mcp.mjs +426 -0
  20. package/cli/is-main.mjs +18 -0
  21. package/cli/mcp/context.mjs +61 -0
  22. package/cli/mcp/export/cognee-dataset-name.mjs +12 -0
  23. package/cli/mcp/export/cognee.mjs +119 -0
  24. package/cli/mcp/export/hindsight.mjs +48 -0
  25. package/cli/mcp/export/index.mjs +26 -0
  26. package/cli/mcp/export/mem0.mjs +31 -0
  27. package/cli/mcp/llm-synthesize.mjs +50 -0
  28. package/cli/mcp/mem0-scope-id.mjs +22 -0
  29. package/cli/mcp/memory-facade.mjs +455 -0
  30. package/cli/mcp/memory-reflect-consolidate.mjs +120 -0
  31. package/cli/mcp/notebooklm-banks.mjs +21 -0
  32. package/cli/mcp/notebooklm-hindsight-rest.mjs +45 -0
  33. package/cli/mcp/notebooklm-provider.mjs +225 -0
  34. package/cli/mcp/pmm-records.mjs +23 -0
  35. package/cli/mcp/portability.mjs +158 -0
  36. package/cli/mcp/stdio.mjs +65 -0
  37. package/cli/mcp/unified-mcp-install.mjs +65 -0
  38. package/cli/mem0-mcp.mjs +108 -0
  39. package/cli/notebooklm-mcp.mjs +236 -0
  40. package/cli/openmem-mcp.mjs +35 -0
  41. package/cli/openmem.mjs +205 -0
  42. package/cli/roots.mjs +85 -0
  43. package/cli/stack.mjs +436 -0
  44. package/cli/switch-memory-provider.mjs +358 -0
  45. package/cli/wiring-report.mjs +125 -0
  46. package/compose.bench.yaml +12 -0
  47. package/compose.yaml +164 -0
  48. package/config/coding-agent.json +13 -0
  49. package/config/hindsight.env.example +24 -0
  50. package/config/index.yml +54 -0
  51. package/config/litellm.lemond.yaml +40 -0
  52. package/config/litellm.ollama.yaml +33 -0
  53. package/config/notebooklm.yml +33 -0
  54. package/config/openmem.toml.example +62 -0
  55. package/docs/OPENMEM.md +23 -0
  56. package/docs/architecture.md +219 -0
  57. package/docs/assets/README.md +10 -0
  58. package/docs/assets/setup-demo.gif +0 -0
  59. package/docs/coding-agent-hooks.md +51 -0
  60. package/docs/coding-agent-scopes.md +22 -0
  61. package/docs/coding-agents.md +51 -0
  62. package/docs/cognee.md +37 -0
  63. package/docs/concepts.md +112 -0
  64. package/docs/hindsight.md +74 -0
  65. package/docs/index-sources-roadmap.md +38 -0
  66. package/docs/index-sources.md +39 -0
  67. package/docs/indexer-sinks.md +122 -0
  68. package/docs/indexing.md +57 -0
  69. package/docs/manual-stack-setup.md +74 -0
  70. package/docs/mcp-install-guide.md +4 -0
  71. package/docs/mcp-setup.md +101 -0
  72. package/docs/mem0.md +103 -0
  73. package/docs/memory-portability.md +73 -0
  74. package/docs/memory-providers.md +246 -0
  75. package/docs/model-configuration.md +84 -0
  76. package/docs/notebooklm.md +83 -0
  77. package/docs/npx-install.md +170 -0
  78. package/docs/openmem-config.md +194 -0
  79. package/docs/provider-wizard-hooks.md +222 -0
  80. package/docs/quickstart.md +101 -0
  81. package/docs/setup-wizard.md +116 -0
  82. package/docs/sources-feeds.md +52 -0
  83. package/docs/sources-readwise.md +23 -0
  84. package/docs/sources-zotero.md +22 -0
  85. package/glama.json +4 -0
  86. package/indexer/Dockerfile +38 -0
  87. package/indexer/code_outline.py +146 -0
  88. package/indexer/feed_sync.py +107 -0
  89. package/indexer/index_config.py +290 -0
  90. package/indexer/pytest.ini +5 -0
  91. package/indexer/sinks/__init__.py +148 -0
  92. package/indexer/sinks/_http.py +44 -0
  93. package/indexer/sinks/cognee.py +329 -0
  94. package/indexer/sinks/hindsight.py +107 -0
  95. package/indexer/sinks/mem0.py +247 -0
  96. package/indexer/watch.py +240 -0
  97. package/package.json +95 -4
  98. package/providers/_template/compose.yaml +52 -0
  99. package/providers/cognee/OPERATOR.md +82 -0
  100. package/providers/cognee/compose.yaml +89 -0
  101. package/providers/cognee/lib/clack.mjs +157 -0
  102. package/providers/cognee/lib/coding-agent.mjs +26 -0
  103. package/providers/cognee/lib/cognee-mcp-install.mjs +153 -0
  104. package/providers/cognee/lib/config.mjs +98 -0
  105. package/providers/cognee/lib/env-file.mjs +194 -0
  106. package/providers/cognee/lib/env-setup.mjs +83 -0
  107. package/providers/cognee/lib/llm-env.mjs +382 -0
  108. package/providers/cognee/lib/ports.mjs +25 -0
  109. package/providers/cognee/lib/smoke.mjs +246 -0
  110. package/providers/cognee/lib/stack-env.mjs +134 -0
  111. package/providers/cognee/lib/state.mjs +101 -0
  112. package/providers/cognee/lib/summary.mjs +56 -0
  113. package/providers/cognee/lib/validate.mjs +97 -0
  114. package/providers/cognee/manifest.json +56 -0
  115. package/providers/cognee/wizard.mjs +134 -0
  116. package/providers/graphiti/manifest.json +15 -0
  117. package/providers/hindsight/lib/clack.mjs +245 -0
  118. package/providers/hindsight/lib/coding-agent.mjs +204 -0
  119. package/providers/hindsight/lib/config.mjs +73 -0
  120. package/providers/hindsight/lib/env-setup.mjs +71 -0
  121. package/providers/hindsight/lib/litellm-routes.mjs +187 -0
  122. package/providers/hindsight/lib/model-backend.mjs +214 -0
  123. package/providers/hindsight/lib/notebooklm-mcp-install.mjs +127 -0
  124. package/providers/hindsight/lib/ollama-compose.mjs +51 -0
  125. package/providers/hindsight/lib/ports.mjs +30 -0
  126. package/providers/hindsight/lib/probe.mjs +33 -0
  127. package/providers/hindsight/lib/stack-env.mjs +132 -0
  128. package/providers/hindsight/lib/state.mjs +75 -0
  129. package/providers/hindsight/lib/summary.mjs +43 -0
  130. package/providers/hindsight/lib/validate.mjs +75 -0
  131. package/providers/hindsight/manifest.json +58 -0
  132. package/providers/hindsight/wizard.mjs +149 -0
  133. package/providers/letta/manifest.json +29 -0
  134. package/providers/mem0/OPERATOR.md +124 -0
  135. package/providers/mem0/compose.yaml +57 -0
  136. package/providers/mem0/lib/coding-agent.mjs +58 -0
  137. package/providers/mem0/lib/config.mjs +45 -0
  138. package/providers/mem0/lib/configure.mjs +135 -0
  139. package/providers/mem0/lib/dotenv.mjs +71 -0
  140. package/providers/mem0/lib/env-file.mjs +110 -0
  141. package/providers/mem0/lib/http.mjs +69 -0
  142. package/providers/mem0/lib/mcp-install.mjs +165 -0
  143. package/providers/mem0/lib/secret.mjs +30 -0
  144. package/providers/mem0/lib/smoke.mjs +179 -0
  145. package/providers/mem0/lib/stack-env.mjs +216 -0
  146. package/providers/mem0/lib/state.mjs +67 -0
  147. package/providers/mem0/lib/summary.mjs +60 -0
  148. package/providers/mem0/lib/url.mjs +106 -0
  149. package/providers/mem0/lib/validate.mjs +158 -0
  150. package/providers/mem0/manifest.json +47 -0
  151. package/providers/mem0/wizard.mjs +133 -0
  152. package/providers/zep/manifest.json +29 -0
  153. package/scripts/check-pack.mjs +218 -0
  154. package/scripts/gen-masks.mjs +83 -0
  155. package/scripts/gen-masks.sh +4 -0
  156. package/scripts/promotion-provider-smoke.mjs +34 -0
  157. package/scripts/run-setup.sh +30 -0
  158. package/scripts/setup.mjs +94 -0
  159. package/scripts/setup.sh +5 -0
  160. package/scripts/spin-up.sh +45 -0
  161. package/scripts/wizard.sh +5 -0
  162. package/server.json +34 -0
  163. package/skills/openmem/SKILL.md +29 -0
  164. package/sync/Dockerfile +27 -0
  165. package/sync/memory_backend.py +87 -0
  166. package/sync/notebooklm_sync.py +664 -0
  167. package/wizard/bin/pm.mjs +97 -0
  168. package/wizard/bin/setup.mjs +33 -0
  169. package/wizard/lib/compose-version.mjs +32 -0
  170. package/wizard/lib/compose.mjs +70 -0
  171. package/wizard/lib/curl.mjs +103 -0
  172. package/wizard/lib/docker-build.mjs +148 -0
  173. package/wizard/lib/exec.mjs +41 -0
  174. package/wizard/lib/indexer.mjs +6 -0
  175. package/wizard/lib/load-machine-env.mjs +4 -0
  176. package/wizard/lib/log.mjs +32 -0
  177. package/wizard/lib/notebooklm-agent-access.mjs +92 -0
  178. package/wizard/lib/notebooklm.mjs +214 -0
  179. package/wizard/lib/parse-args.mjs +87 -0
  180. package/wizard/lib/paths.mjs +39 -0
  181. package/wizard/lib/pm-config.mjs +163 -0
  182. package/wizard/lib/ports.mjs +94 -0
  183. package/wizard/lib/prompts.mjs +57 -0
  184. package/wizard/lib/provider-wizard/helpers.mjs +98 -0
  185. package/wizard/lib/provider-wizard/run-steps.mjs +129 -0
  186. package/wizard/lib/providers/registry.mjs +262 -0
  187. package/wizard/lib/run-clack.mjs +108 -0
  188. package/wizard/lib/run-wizard.mjs +755 -0
  189. package/wizard/lib/validate-setup.mjs +62 -0
  190. package/wizard/lib/wizard-action.mjs +34 -0
  191. package/wizard/lib/wizard-conf.mjs +71 -0
@@ -0,0 +1,530 @@
1
+ #!/usr/bin/env python3
2
+ """Index tracked project docs, tree-sitter code outlines and Obsidian vaults into a memory provider (Hindsight by default)."""
3
+
4
+ import argparse
5
+ import fcntl
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import re
10
+ import subprocess
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ import code_outline
15
+ import index_config
16
+ import sinks
17
+ from sinks.hindsight import DEFAULT_URL, HttpError # noqa: F401 (HttpError stays importable from here)
18
+
19
+
20
+ SKIP_NAMES = {"auth.json", "credentials.json", "id_rsa", "id_ed25519"}
21
+ VAULT_SKIP_DIRS = {".obsidian", ".trash"}
22
+ DOC_SUFFIXES = {".md", ".mdx"}
23
+ HEADING = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
24
+ WIKILINK = re.compile(r"!?\[\[([^\]|#\n]+)[^\]\n]*\]\]")
25
+ INLINE_TAG = re.compile(r"(?<![\w/#&])#([A-Za-z][\w/-]*)")
26
+ DEFAULT_STATE = Path.home() / ".cache/hindsight-project-index/state.json"
27
+ AGENT_CONFIG = Path.home() / ".hindsight/coding-agent.json"
28
+ PERSONAL_MEMORY_AGENT_CONFIG = Path.home() / ".config/personal-memory/coding-agent.json"
29
+
30
+
31
+ def scan_root(root: Path, depth: int, skip_dirs: frozenset[str], want_vaults: bool) -> tuple[list[Path], list[Path]]:
32
+ """Return the outermost git repositories and, when want_vaults, the .obsidian folders within depth levels of root."""
33
+ repos: list[Path] = []
34
+ vaults: list[Path] = []
35
+ if not root.is_dir():
36
+ print(f"{root}: root not found, skipped", file=sys.stderr)
37
+ return repos, vaults
38
+ for child in root.iterdir():
39
+ if not child.is_dir() or child.name.startswith("."):
40
+ continue
41
+ for current, dirs, _files in os.walk(child):
42
+ here = Path(current)
43
+ if len(here.relative_to(root).parts) > depth:
44
+ dirs[:] = []
45
+ continue
46
+ is_repo = (here / ".git").exists()
47
+ is_vault = want_vaults and (here / ".obsidian").is_dir()
48
+ if is_repo:
49
+ repos.append(here)
50
+ if is_vault:
51
+ vaults.append(here)
52
+ if is_repo or is_vault:
53
+ dirs[:] = []
54
+ continue
55
+ dirs[:] = [name for name in dirs if name not in skip_dirs and not name.startswith(".")]
56
+ return repos, vaults
57
+
58
+
59
+ def is_git_repo(path: Path) -> bool:
60
+ return subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=path, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
61
+
62
+
63
+ def discover(config: index_config.Config) -> tuple[list[Path], list[Path], list[tuple[str, Path]]]:
64
+ """Return repositories, Obsidian vaults, and markdown directory sources (kind, path)."""
65
+ repos: set[Path] = set(config.repos)
66
+ vaults: set[Path] = set(config.vault_paths)
67
+ markdown: list[tuple[str, Path]] = list(config.markdown_sources())
68
+ for root in config.roots:
69
+ found_repos, found_vaults = scan_root(root.path, root.depth, config.skip_dirs, config.vault_auto)
70
+ repos.update(found_repos)
71
+ vaults.update(found_vaults)
72
+ selected_repos = []
73
+ for repo in sorted(repos):
74
+ if is_git_repo(repo):
75
+ selected_repos.append(repo)
76
+ else:
77
+ print(f"{repo}: not a git repository, skipped", file=sys.stderr)
78
+ selected_vaults = []
79
+ for vault in sorted(vaults):
80
+ if vault.is_dir():
81
+ selected_vaults.append(vault)
82
+ else:
83
+ print(f"{vault}: vault not found, skipped", file=sys.stderr)
84
+ selected_markdown: list[tuple[str, Path]] = []
85
+ for kind, path in markdown:
86
+ if path.is_dir():
87
+ selected_markdown.append((kind, path))
88
+ else:
89
+ print(f"{path}: {kind} path not found, skipped", file=sys.stderr)
90
+ selected_markdown.sort(key=lambda item: (item[0], item[1]))
91
+ return selected_repos, selected_vaults, selected_markdown
92
+
93
+
94
+ def _config(config: index_config.Config | None) -> index_config.Config:
95
+ return config if config is not None else index_config.parse(None)
96
+
97
+
98
+ def tracked_files(repo: Path, config: index_config.Config | None = None) -> list[Path]:
99
+ config = _config(config)
100
+ raw = subprocess.run(["git", "ls-files", "-z"], cwd=repo, capture_output=True, check=True).stdout
101
+ paths = []
102
+ for item in raw.split(b"\0"):
103
+ if not item:
104
+ continue
105
+ rel = Path(os.fsdecode(item))
106
+ if any(part in config.skip_dirs for part in rel.parts) or rel.name in SKIP_NAMES:
107
+ continue
108
+ if rel.name.startswith(".env") or any(word in rel.name.lower() for word in ("secret", "credential", "private_key")):
109
+ continue
110
+ path = repo / rel
111
+ if path.is_file() and not path.is_symlink() and path.stat().st_size <= config.max_file_bytes:
112
+ paths.append(rel)
113
+ return paths
114
+
115
+
116
+ def doc_chunks(text: str, limit: int = 2400) -> list[tuple[str, str]]:
117
+ sections = []
118
+ stack: list[str] = []
119
+ body: list[str] = []
120
+
121
+ def flush() -> None:
122
+ content = "\n".join(body).strip()
123
+ if content:
124
+ sections.append((" / ".join(stack) or "Introduction", content))
125
+
126
+ for line in text.splitlines():
127
+ match = HEADING.match(line)
128
+ if match:
129
+ flush()
130
+ body = []
131
+ depth = len(match.group(1))
132
+ stack = stack[: depth - 1] + [match.group(2)]
133
+ else:
134
+ body.append(line)
135
+ flush()
136
+ chunks = []
137
+ for heading, content in sections:
138
+ paragraphs = re.split(r"\n\s*\n", content)
139
+ part = ""
140
+ for paragraph in paragraphs:
141
+ for segment in (paragraph[i : i + limit] for i in range(0, len(paragraph), limit)):
142
+ if part and len(part) + len(segment) + 2 > limit:
143
+ chunks.append((heading, part))
144
+ part = ""
145
+ part = f"{part}\n\n{segment}".strip()
146
+ if part:
147
+ chunks.append((heading, part))
148
+ return chunks
149
+
150
+
151
+ def source_documents(repo: Path, config: index_config.Config | None = None) -> dict[str, dict]:
152
+ """Return the repository's prose chunks and tree-sitter code outlines, keyed by stable document ID."""
153
+ documents = {}
154
+ for rel in tracked_files(repo, config):
155
+ kind = "doc" if rel.suffix.lower() in DOC_SUFFIXES else "code" if code_outline.language_for(rel.as_posix()) else None
156
+ if kind is None:
157
+ continue
158
+ if kind == "code":
159
+ outline = code_outline.outline(rel.as_posix(), (repo / rel).read_bytes())
160
+ if not outline:
161
+ continue
162
+ parts = [("outline", outline)]
163
+ else:
164
+ try:
165
+ parts = doc_chunks((repo / rel).read_text(encoding="utf-8"))
166
+ except UnicodeError:
167
+ continue
168
+ for index, (heading, content) in enumerate(parts):
169
+ label = f"{kind}:{repo.as_posix()}:{rel.as_posix()}:{heading}:{index}"
170
+ doc_id = "project-index:v1:" + hashlib.sha256(label.encode()).hexdigest()[:32]
171
+ payload = f"Project: {repo.name}\nSource: {rel.as_posix()}\nSection: {heading}\n\n{content}"
172
+ documents[doc_id] = {
173
+ "content": payload,
174
+ "path": rel.as_posix(),
175
+ "kind": kind,
176
+ "digest": hashlib.sha256(payload.encode()).hexdigest(),
177
+ "context": f"Project {repo.name} {kind} source",
178
+ "metadata": {"path": rel.as_posix(), "kind": kind},
179
+ }
180
+ return documents
181
+
182
+
183
+ def slugify(name: str) -> str:
184
+ return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
185
+
186
+
187
+ def vault_bank(vault: Path, config: index_config.Config | None = None) -> str:
188
+ return _config(config).bank_for(vault) or "obsidian::" + slugify(vault.name)
189
+
190
+
191
+ def markdown_bank(kind: str, root: Path, config: index_config.Config | None = None) -> str:
192
+ cfg = _config(config)
193
+ mapped = cfg.bank_for(root)
194
+ if mapped:
195
+ return mapped
196
+ prefix = {"notes": "notes", "papers": "papers", "bookmarks": "bookmarks", "feed": "feed"}.get(kind, "notes")
197
+ return f"{prefix}::{slugify(root.name)}"
198
+
199
+
200
+ def split_frontmatter(text: str) -> tuple[list[str], str]:
201
+ if not text.startswith("---\n"):
202
+ return [], text
203
+ end = text.find("\n---", 3)
204
+ if end == -1:
205
+ return [], text
206
+ block = text[4:end].splitlines()
207
+ body = text[end + 4 :].lstrip("\n")
208
+ tags: list[str] = []
209
+ for index, line in enumerate(block):
210
+ if not line.startswith("tags:"):
211
+ continue
212
+ value = line[len("tags:") :].strip()
213
+ if value:
214
+ tags = re.split(r"[\[\],]", value)
215
+ else:
216
+ for following in block[index + 1 :]:
217
+ match = re.match(r"^\s*-\s+(.+)$", following)
218
+ if not match:
219
+ break
220
+ tags.append(match.group(1))
221
+ break
222
+ cleaned = [tag.strip().strip("\"'").lstrip("#").strip() for tag in tags]
223
+ return [tag for tag in cleaned if tag], body
224
+
225
+
226
+ MARKDOWN_SUFFIXES = {
227
+ "notes": (".md",),
228
+ "papers": (".md", ".txt"),
229
+ "bookmarks": (".md", ".html", ".htm", ".txt"),
230
+ "feed": (".md",),
231
+ }
232
+
233
+
234
+ def vault_documents(vault: Path, config: index_config.Config | None = None, *, kind: str = "notes") -> dict[str, dict]:
235
+ config = _config(config)
236
+ suffixes = MARKDOWN_SUFFIXES.get(kind, (".md",))
237
+ documents = {}
238
+ for current, dirs, files in os.walk(vault):
239
+ dirs[:] = [name for name in dirs if name not in VAULT_SKIP_DIRS]
240
+ for name in files:
241
+ path = Path(current) / name
242
+ if path.suffix.lower() not in suffixes or path.is_symlink():
243
+ continue
244
+ if name.startswith(".env") or any(word in name.lower() for word in ("secret", "credential", "private_key")):
245
+ continue
246
+ if path.stat().st_size > config.max_file_bytes:
247
+ continue
248
+ try:
249
+ text = path.read_text(encoding="utf-8")
250
+ except UnicodeError:
251
+ continue
252
+ if path.suffix.lower() in (".html", ".htm"):
253
+ text = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", text)
254
+ text = re.sub(r"<[^>]+>", " ", text)
255
+ text = re.sub(r"\s+", " ", text).strip()
256
+ rel = path.relative_to(vault).as_posix()
257
+ frontmatter_tags, body = split_frontmatter(text)
258
+ tags = list(dict.fromkeys(frontmatter_tags + INLINE_TAG.findall(body)))
259
+ links = sorted({target.strip() for target in WIKILINK.findall(body) if target.strip()})
260
+ for index, (heading, content) in enumerate(doc_chunks(body)):
261
+ label = f"note:{vault.as_posix()}:{rel}:{heading}:{index}"
262
+ doc_id = "obsidian-index:v1:" + hashlib.sha256(label.encode()).hexdigest()[:32]
263
+ payload = f"Vault: {vault.name}\nSource: {rel}\nSection: {heading}\n\n{content}"
264
+ metadata = {"path": rel, "kind": "note"}
265
+ if links:
266
+ metadata["links"] = ", ".join(links)
267
+ documents[doc_id] = {
268
+ "content": payload,
269
+ "path": rel,
270
+ "kind": "note",
271
+ "digest": hashlib.sha256(payload.encode()).hexdigest(),
272
+ "context": f"Obsidian vault {vault.name} note",
273
+ "metadata": metadata,
274
+ "tags": tags,
275
+ }
276
+ return documents
277
+
278
+
279
+ def acquire_state_lock(state_path: Path):
280
+ state_path.parent.mkdir(parents=True, exist_ok=True)
281
+ handle = state_path.with_name(state_path.name + ".lock").open("w")
282
+ fcntl.flock(handle, fcntl.LOCK_EX)
283
+ return handle
284
+
285
+
286
+ def save_state(path: Path, state: dict) -> None:
287
+ path.parent.mkdir(parents=True, exist_ok=True)
288
+ temp = path.with_suffix(".tmp")
289
+ temp.write_text(json.dumps(state, indent=2, sort_keys=True))
290
+ temp.replace(path)
291
+
292
+
293
+ def bank_id(repo: Path, config: index_config.Config | None = None) -> str:
294
+ # Order: index.yml banks, openmem mapping, legacy Hindsight mapping, repository name.
295
+ configured = _config(config).bank_for(repo)
296
+ if configured:
297
+ return configured
298
+ for config_path in (PERSONAL_MEMORY_AGENT_CONFIG, AGENT_CONFIG):
299
+ agent = json.loads(config_path.read_text()) if config_path.exists() else {}
300
+ mappings = agent.get("mapPathToBank", {})
301
+ resolved = sorted(
302
+ ((Path(prefix).expanduser().resolve(), bank) for prefix, bank in mappings.items()),
303
+ key=lambda item: len(str(item[0])),
304
+ reverse=True,
305
+ )
306
+ for path, bank in resolved:
307
+ if repo == path or path in repo.parents:
308
+ return bank
309
+ return "coding-agent::" + project_name(repo)
310
+
311
+
312
+ def project_name(repo: Path) -> str:
313
+ """Return the shared project name for linked worktrees, otherwise the repo directory name."""
314
+ root = repo.resolve()
315
+ git_entry = root / ".git"
316
+ try:
317
+ if git_entry.is_dir():
318
+ git_dir = git_entry
319
+ elif git_entry.is_file():
320
+ match = re.match(r"^gitdir:\s*(.+?)\s*$", git_entry.read_text())
321
+ if not match:
322
+ return root.name
323
+ git_dir = Path(match.group(1))
324
+ if not git_dir.is_absolute():
325
+ git_dir = (root / git_dir).resolve()
326
+ else:
327
+ return root.name
328
+
329
+ common_file = git_dir / "commondir"
330
+ common_dir = (git_dir / common_file.read_text().strip()).resolve() if common_file.exists() else git_dir
331
+ if git_dir.parent.name == "worktrees" and common_dir.name == ".git":
332
+ return common_dir.parent.name
333
+ except OSError:
334
+ pass
335
+ return root.name
336
+
337
+
338
+ def kinds_key(source: Path) -> str:
339
+ return f"{source}::kinds"
340
+
341
+
342
+ # Banks whose sink flush() has not completed. A distinct top-level key (source keys are absolute paths, so it
343
+ # cannot collide). It exists only while a flush is pending and only for sinks that define flush(), so a state
344
+ # file written for a sink without flush() (Hindsight) never contains it.
345
+ PENDING_FLUSH_KEY = "::flush-pending"
346
+
347
+
348
+ def pending_flush_banks(state: dict) -> set[str]:
349
+ return set(state.get(PENDING_FLUSH_KEY, []))
350
+
351
+
352
+ def set_pending_flush(state: dict, state_path: Path, banks: set[str]) -> None:
353
+ if banks:
354
+ state[PENDING_FLUSH_KEY] = sorted(banks)
355
+ else:
356
+ state.pop(PENDING_FLUSH_KEY, None)
357
+ save_state(state_path, state)
358
+
359
+
360
+ def sync(source: Path, bank: str, documents: dict[str, dict], sink: sinks.Sink, state: dict, state_path: Path, apply: bool, limit: int | None, kind: str | None) -> tuple[int, int]:
361
+ if kind:
362
+ documents = {key: value for key, value in documents.items() if value["kind"] == kind}
363
+ old = state.get(str(source), {})
364
+ kinds = state.get(kinds_key(source), {})
365
+ changed = {key: value for key, value in documents.items() if old.get(key) != value["digest"]}
366
+ if limit is not None:
367
+ changed = dict(list(changed.items())[:limit])
368
+ stale = set(old) - set(documents) if limit is None and kind is None else set()
369
+ if apply:
370
+ # Optional flush hook (async-ingest sinks). A flush still pending from an earlier run is retried here.
371
+ flushable = callable(getattr(sink, "flush", None))
372
+ flush_due = flushable and bank in pending_flush_banks(state)
373
+ if changed:
374
+ sink.prepare(bank)
375
+ if flushable and (changed or stale) and not flush_due:
376
+ # Recorded before the first write: the per-document saves below persist digests, so without the
377
+ # marker a failed flush would never be retried.
378
+ set_pending_flush(state, state_path, pending_flush_banks(state) | {bank})
379
+ flush_due = True
380
+ for doc_id, value in changed.items():
381
+ sink.retain(bank, doc_id, value["content"], tags=value.get("tags"), metadata=value["metadata"], context=value["context"])
382
+ old[doc_id] = value["digest"]
383
+ kinds[doc_id] = value["kind"]
384
+ state[str(source)] = old
385
+ state[kinds_key(source)] = kinds
386
+ save_state(state_path, state)
387
+ for doc_id in sorted(stale):
388
+ # A document already gone (never retained, or deleted server-side) is not an error; drop it from the manifest.
389
+ sink.delete(bank, doc_id)
390
+ old.pop(doc_id)
391
+ kinds.pop(doc_id, None)
392
+ save_state(state_path, state)
393
+ if flush_due:
394
+ # Once per bank batch, not per document. The marker is cleared only after a successful flush.
395
+ sinks.flush_sink(sink, bank)
396
+ set_pending_flush(state, state_path, pending_flush_banks(state) - {bank})
397
+ state[str(source)] = old
398
+ state[kinds_key(source)] = kinds
399
+ return len(changed), len(stale)
400
+
401
+
402
+ def sync_repo(repo: Path, sink: sinks.Sink, state: dict, state_path: Path, apply: bool, limit: int | None, kind: str | None, config: index_config.Config | None = None) -> tuple[int, int]:
403
+ return sync(repo, bank_id(repo, config), source_documents(repo, config), sink, state, state_path, apply, limit, kind)
404
+
405
+
406
+ def sync_vault(vault: Path, sink: sinks.Sink, state: dict, state_path: Path, apply: bool, limit: int | None, config: index_config.Config | None = None) -> tuple[int, int]:
407
+ return sync(vault, vault_bank(vault, config), vault_documents(vault, config), sink, state, state_path, apply, limit, None)
408
+
409
+
410
+ def sync_markdown(kind: str, root: Path, sink: sinks.Sink, state: dict, state_path: Path, apply: bool, limit: int | None, config: index_config.Config | None = None) -> tuple[int, int]:
411
+ return sync(root, markdown_bank(kind, root, config), vault_documents(root, config, kind=kind), sink, state, state_path, apply, limit, None)
412
+
413
+
414
+ def load_opt_in_paths(config: Path) -> list[Path] | None:
415
+ if not config.exists():
416
+ print(f"warning: {config} not found; indexing every source under --home", file=sys.stderr)
417
+ return None
418
+ paths = json.loads(config.read_text()).get("optInPaths")
419
+ if paths is None:
420
+ return None
421
+ return [Path(item).expanduser().resolve() for item in paths]
422
+
423
+
424
+ def opted_in(path: Path, allowed: list[Path] | None) -> bool:
425
+ return allowed is None or any(path == root or root in path.parents for root in allowed)
426
+
427
+
428
+ def selected_sources(args: argparse.Namespace, config: index_config.Config) -> list[tuple[str, Path]]:
429
+ # Explicit --repo or --vault narrows the run to exactly those sources.
430
+ if args.repo or args.vault or args.notes or args.papers or args.bookmarks or args.feed:
431
+ return (
432
+ [("repo", path) for path in args.repo or []]
433
+ + [("vault", path) for path in args.vault or []]
434
+ + [("notes", path) for path in args.notes or []]
435
+ + [("papers", path) for path in args.papers or []]
436
+ + [("bookmarks", path) for path in args.bookmarks or []]
437
+ + [("feed", path) for path in args.feed or []]
438
+ )
439
+ repos, vaults, markdown = discover(config)
440
+ return (
441
+ [("repo", repo) for repo in repos]
442
+ + [("vault", vault) for vault in vaults]
443
+ + [(kind, path) for kind, path in markdown]
444
+ )
445
+
446
+
447
+ def resolve_url(sink_name: str, explicit: str | None) -> str | None:
448
+ """--url wins. Otherwise $HINDSIGHT_URL for the hindsight sink (then its default), $MEMORY_API_URL for any other."""
449
+ if explicit:
450
+ return explicit
451
+ if sink_name == "hindsight":
452
+ return os.environ.get("HINDSIGHT_URL") or DEFAULT_URL
453
+ return os.environ.get("MEMORY_API_URL") or None
454
+
455
+
456
+ def main() -> int:
457
+ parser = argparse.ArgumentParser(description=__doc__)
458
+ parser.add_argument("--config", type=Path, help="index.yml to use (default: $HINDSIGHT_INDEX_CONFIG, ~/.config/hindsight/index.yml, then config/index.yml)")
459
+ parser.add_argument("--repo", type=Path, action="append", help="index only these repositories (repeatable)")
460
+ parser.add_argument("--vault", type=Path, action="append", help="index an Obsidian vault (repeatable)")
461
+ parser.add_argument("--notes", type=Path, action="append", help="index a markdown notes folder (repeatable)")
462
+ parser.add_argument("--papers", type=Path, action="append", help="index a papers library folder (markdown exports; repeatable)")
463
+ parser.add_argument("--bookmarks", type=Path, action="append", help="index a bookmarks export folder (repeatable)")
464
+ parser.add_argument("--feed", type=Path, action="append", help="index feed-sync markdown cache (repeatable)")
465
+ parser.add_argument("--sink", default=os.environ.get("MEMORY_SINK") or sinks.DEFAULT_SINK, help="memory provider to write to (default: $MEMORY_SINK, then %(default)s); registered: " + ", ".join(sinks.registered_sinks()))
466
+ parser.add_argument("--url", help="provider API base URL (default: $HINDSIGHT_URL for the hindsight sink, $MEMORY_API_URL for other sinks, then the sink's own default)")
467
+ parser.add_argument("--state", type=Path, default=DEFAULT_STATE)
468
+ parser.add_argument("--limit", type=int, help="at most N changed documents per source; disables deletion")
469
+ parser.add_argument("--kind", choices=("doc", "code"), help="index only documentation or code in repositories; disables deletion")
470
+ parser.add_argument("--apply", action="store_true", help="write to the memory provider")
471
+ parser.add_argument("--list", action="store_true", help="print the selected sources as 'kind<TAB>path' and exit")
472
+ args = parser.parse_args()
473
+
474
+ if args.sink not in sinks.registered_sinks():
475
+ print(f"unknown sink {args.sink!r}; registered sinks: {', '.join(sinks.registered_sinks())}", file=sys.stderr)
476
+ return 2
477
+ try:
478
+ config = index_config.load(args.config)
479
+ except index_config.ConfigError as error:
480
+ print(f"config error: {error}", file=sys.stderr)
481
+ return 2
482
+ allowed = load_opt_in_paths(AGENT_CONFIG)
483
+ sources = []
484
+ for kind, path in selected_sources(args, config):
485
+ path = path.expanduser().resolve()
486
+ if not opted_in(path, allowed):
487
+ print(f"{path}: skipped (outside optInPaths in {AGENT_CONFIG})")
488
+ elif config.is_excluded(path):
489
+ print(f"{path}: skipped (matches exclude in {config.source})", file=sys.stderr)
490
+ else:
491
+ sources.append((kind, path))
492
+ if args.list:
493
+ for kind, path in sources:
494
+ print(f"{kind}\t{path}")
495
+ return 0
496
+
497
+ sink = None
498
+ if args.apply:
499
+ try:
500
+ sink = sinks.get_sink(args.sink, url=resolve_url(args.sink, args.url))
501
+ except ValueError as error: # includes sinks.UnknownSinkError
502
+ print(f"sink error: {error}", file=sys.stderr)
503
+ return 2
504
+ # Held until exit: the watcher and manual runs must not overwrite each other's state file.
505
+ _state_lock = acquire_state_lock(args.state) if args.apply else None # held until exit
506
+ state = json.loads(args.state.read_text()) if args.state.exists() else {}
507
+ failed = 0
508
+ for kind, path in sources:
509
+ if kind in ("vault", "notes", "papers", "bookmarks", "feed") and args.kind:
510
+ continue
511
+ try:
512
+ if kind == "repo":
513
+ changed, stale = sync_repo(path, sink, state, args.state, args.apply, args.limit, args.kind, config)
514
+ elif kind in ("notes", "papers", "bookmarks", "feed"):
515
+ changed, stale = sync_markdown(kind, path, sink, state, args.state, args.apply, args.limit, config)
516
+ else:
517
+ changed, stale = sync_vault(path, sink, state, args.state, args.apply, args.limit, config)
518
+ print(f"{path}: {changed} to retain, {stale} stale" + (" (applied)" if args.apply else " (dry run)"))
519
+ if args.apply:
520
+ save_state(args.state, state)
521
+ except Exception as error: # one failing source must not abort the others
522
+ failed += 1
523
+ print(f"{path}: {type(error).__name__}: {error}", file=sys.stderr)
524
+ if failed:
525
+ print(f"{failed} source(s) failed", file=sys.stderr)
526
+ return 1 if failed else 0
527
+
528
+
529
+ if __name__ == "__main__":
530
+ raise SystemExit(main())
@@ -0,0 +1,38 @@
1
+ import fs from "node:fs";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+
5
+ export const accessConfigPath = (home) => path.join(home, ".config", "personal-memory", "coding-agent.json");
6
+
7
+ /** Read policy config. A missing file keeps the historical device-wide default; all other read/parse errors deny. */
8
+ export function readAccessConfig(home) {
9
+ try {
10
+ const config = JSON.parse(fs.readFileSync(accessConfigPath(home), "utf8"));
11
+ if (!config || typeof config !== "object" || Array.isArray(config)) return null;
12
+ if ("disabled" in config && typeof config.disabled !== "boolean") return null;
13
+ if ("optInOnly" in config && typeof config.optInOnly !== "boolean") return null;
14
+ if ("optInPaths" in config && (!Array.isArray(config.optInPaths)
15
+ || config.optInPaths.some((p) => typeof p !== "string" || !p.trim() || !path.isAbsolute(p)))) return null;
16
+ if ("mapPathToBank" in config && (!config.mapPathToBank || typeof config.mapPathToBank !== "object"
17
+ || Array.isArray(config.mapPathToBank)
18
+ || Object.entries(config.mapPathToBank).some(([prefix, bank]) => !prefix.trim() || !path.isAbsolute(prefix)
19
+ || typeof bank !== "string" || !bank.trim()))) return null;
20
+ return config;
21
+ } catch (error) {
22
+ return error?.code === "ENOENT" ? {} : null;
23
+ }
24
+ }
25
+
26
+ export function accessAllowed(cwd, config) {
27
+ if (!config || typeof cwd !== "string" || !cwd.trim() || !path.isAbsolute(cwd) || config.disabled === true) return false;
28
+ if (config.optInOnly !== true) return true;
29
+ const dir = path.resolve(cwd);
30
+ return (config.optInPaths || []).some((configured) => {
31
+ const allowed = path.resolve(configured);
32
+ return dir === allowed || dir.startsWith(`${allowed}${path.sep}`);
33
+ });
34
+ }
35
+
36
+ export function currentHome() {
37
+ return process.env.HOME || os.homedir();
38
+ }
@@ -0,0 +1,34 @@
1
+ /** Supported coding-agent targets for install-coding-agent. */
2
+
3
+ export const CODING_AGENTS = Object.freeze([
4
+ { id: "claude-code", label: "Claude Code", hindsightInstall: true },
5
+ { id: "codex", label: "Codex CLI", hindsightInstall: true },
6
+ { id: "opencode", label: "OpenCode", hindsightInstall: true },
7
+ { id: "cursor-cli", label: "Cursor CLI", hindsightInstall: true },
8
+ {
9
+ id: "hermes",
10
+ label: "Hermes",
11
+ hindsightInstall: false,
12
+ sessionPath: "Unified openmem MCP (memory_*) or agent-native MCP; no bundled Hermes plugin in this repo.",
13
+ },
14
+ {
15
+ id: "gemini-cli",
16
+ label: "Gemini CLI",
17
+ hindsightInstall: false,
18
+ sessionPath: "Use install-mcp; configure Gemini MCP per Google docs.",
19
+ },
20
+ {
21
+ id: "amp",
22
+ label: "Amp (Sourcegraph)",
23
+ hindsightInstall: false,
24
+ sessionPath: "Unified MCP via install-mcp; follow Amp MCP configuration.",
25
+ },
26
+ {
27
+ id: "cline",
28
+ label: "Cline",
29
+ hindsightInstall: false,
30
+ sessionPath: "Unified MCP in Cursor/VS Code; install-mcp for memory_* tools.",
31
+ },
32
+ ]);
33
+
34
+ export const DEFAULT_AGENT_IDS = CODING_AGENTS.filter((a) => a.hindsightInstall).map((a) => a.id);