digital-brain 0.1.3 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -7
- package/bin/digital-brain.js +275 -38
- package/docs/AUTOMATIONS.md +13 -4
- package/docs/INTEGRATIONS.md +72 -0
- package/docs/PRIVACY.md +3 -1
- package/docs/SETUP.md +78 -0
- package/examples/sample-vault/{04 People/Interpreted Relationships/Close Friend.md → 06 AI Memory/Generated Relationship Drafts/Close Friend (WhatsApp).md } +4 -3
- package/examples/sample-vault/{04 People/Interpreted Relationships/Mom.md → 06 AI Memory/Generated Relationship Drafts/Mom (WhatsApp).md } +4 -3
- package/examples/sample-vault/{08 Sources/WhatsApp/Analysis/Interpreted/Project Team.md → 06 AI Memory/Generated Relationship Drafts/Project Team (WhatsApp).md } +4 -3
- package/examples/sample-vault/06 AI Memory/Interpreted Relationship Memory.md +3 -3
- package/examples/sample-vault/06 AI Memory/Person Context Index.md +26 -0
- package/examples/sample-vault/06 AI Memory/Person Reply Context.md +26 -0
- package/examples/sample-vault/08 Sources/{WhatsApp/Analysis/Interpreted/Close Friend.md → Analysis/Interpreted/Close Friend (WhatsApp).md } +4 -3
- package/examples/sample-vault/08 Sources/{WhatsApp/Analysis/Interpreted/Mom.md → Analysis/Interpreted/Mom (WhatsApp).md } +4 -3
- package/examples/sample-vault/{04 People/Interpreted Relationships/Project Team.md → 08 Sources/Analysis/Interpreted/Project Team (WhatsApp).md } +4 -3
- package/examples/sample-vault/08 Sources/Analysis/Relationship Map.md +38 -0
- package/examples/sample-vault/08 Sources/Analysis/interpreted_relationship_models.json +175 -0
- package/examples/sample-vault/08 Sources/Analysis/person_identity_map.json +78 -0
- package/examples/sample-vault/08 Sources/Analysis/relationship_profiles.json +122 -0
- package/examples/sample-vault/08 Sources/WhatsApp/Analysis/Interpreted/Close Friend (WhatsApp).md +44 -0
- package/examples/sample-vault/08 Sources/WhatsApp/Analysis/Interpreted/Mom (WhatsApp).md +45 -0
- package/examples/sample-vault/08 Sources/WhatsApp/Analysis/Interpreted/Project Team (WhatsApp).md +45 -0
- package/examples/sample-vault/08 Sources/WhatsApp/Analysis/Relationship Map.md +9 -3
- package/examples/sample-vault/08 Sources/WhatsApp/Analysis/interpreted_relationship_models.json +18 -0
- package/examples/sample-vault/08 Sources/WhatsApp/Analysis/person_identity_map.json +78 -0
- package/examples/sample-vault/08 Sources/WhatsApp/Analysis/relationship_profiles.json +18 -0
- package/examples/sample-vault/08 Sources/WhatsApp/Raw/2026-01-01.jsonl +6 -6
- package/lib/fs.js +33 -0
- package/package.json +2 -1
- package/scripts/digital_brain_imessage_sync.py +175 -0
- package/scripts/digital_brain_linkedin_export_import.py +214 -0
- package/scripts/digital_brain_relationship_extractor.py +189 -12
- package/scripts/digital_brain_relationship_interpreter.py +104 -15
- package/scripts/digital_brain_slack_export_import.py +181 -0
- package/scripts/digital_brain_whatsapp_mac_sync.py +37 -8
- package/templates/vault/00 Home/How AI Should Use This Vault.md +1 -1
- package/templates/vault/00 Home/Start Here.md +2 -1
- package/templates/vault/04 People/Relationship Overrides.md +2 -1
- package/templates/vault/06 AI Memory/Generated Relationship Drafts/README.md +5 -0
- package/templates/vault/06 AI Memory/Interpreted Relationship Memory.md +1 -2
- package/templates/vault/06 AI Memory/Person Context Index.md +4 -0
- package/templates/vault/06 AI Memory/Person Reply Context.md +4 -0
- package/templates/vault/08 Sources/README.md +5 -0
- package/templates/vault/08 Sources/WhatsApp/Outbound/README.md +2 -2
- package/templates/vault/AGENTS.md +5 -1
- package/templates/vault/CLAUDE.md +3 -0
- package/templates/vault/GEMINI.md +4 -0
- package/whatsapp-web/send.mjs +32 -5
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
import argparse
|
|
3
3
|
import json
|
|
4
|
+
import time
|
|
4
5
|
from datetime import datetime, timezone
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
|
|
@@ -19,26 +20,38 @@ ROLE_KEYWORDS = [
|
|
|
19
20
|
def main():
|
|
20
21
|
args = parse_args()
|
|
21
22
|
vault = args.vault.resolve()
|
|
22
|
-
|
|
23
|
-
|
|
23
|
+
sources = vault / "08 Sources"
|
|
24
|
+
whatsapp = sources / "WhatsApp"
|
|
25
|
+
analysis = sources / "Analysis"
|
|
26
|
+
profile_path = analysis / "relationship_profiles.json"
|
|
27
|
+
if not profile_path.exists():
|
|
28
|
+
profile_path = whatsapp / "Analysis" / "relationship_profiles.json"
|
|
24
29
|
profiles = json.loads(profile_path.read_text(encoding="utf-8"))
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
30
|
+
people_path = analysis / "person_identity_map.json"
|
|
31
|
+
people = load_json(people_path, [])
|
|
32
|
+
overrides = load_overrides(sources)
|
|
33
|
+
out_dir = analysis / "Interpreted"
|
|
34
|
+
legacy_out_dir = whatsapp / "Analysis" / "Interpreted"
|
|
35
|
+
drafts_dir = vault / "06 AI Memory" / "Generated Relationship Drafts"
|
|
28
36
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
29
|
-
|
|
37
|
+
legacy_out_dir.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
drafts_dir.mkdir(parents=True, exist_ok=True)
|
|
30
39
|
|
|
31
40
|
models = []
|
|
32
41
|
for profile in profiles:
|
|
33
|
-
|
|
42
|
+
override = overrides.get(profile_key(profile), overrides.get(profile["chatName"], {}))
|
|
43
|
+
model = build_model(profile, override)
|
|
34
44
|
note = render_note(model)
|
|
35
|
-
filename = safe_filename(profile["chatName"]) + ".md"
|
|
36
|
-
(out_dir / filename
|
|
37
|
-
(
|
|
45
|
+
filename = safe_filename(profile.get("displayName") or profile["chatName"]) + ".md"
|
|
46
|
+
write_text_atomic(out_dir / filename, note)
|
|
47
|
+
write_text_atomic(legacy_out_dir / filename, note)
|
|
48
|
+
write_text_atomic(drafts_dir / filename, note)
|
|
38
49
|
models.append(model)
|
|
39
50
|
|
|
40
|
-
(
|
|
51
|
+
write_json_atomic(analysis / "interpreted_relationship_models.json", models)
|
|
52
|
+
write_json_atomic(whatsapp / "Analysis" / "interpreted_relationship_models.json", models)
|
|
41
53
|
write_index(vault / "06 AI Memory" / "Interpreted Relationship Memory.md", models)
|
|
54
|
+
write_person_reply_index(vault / "06 AI Memory" / "Person Reply Context.md", people, models)
|
|
42
55
|
print(f"Wrote interpreted notes: {len(models)}")
|
|
43
56
|
|
|
44
57
|
|
|
@@ -130,16 +143,17 @@ def infer_boundaries(role, difficulty):
|
|
|
130
143
|
|
|
131
144
|
|
|
132
145
|
def render_note(model):
|
|
133
|
-
return f"""# {model['chatName']}
|
|
146
|
+
return f"""# {model.get('displayName') or model['chatName']}
|
|
134
147
|
|
|
135
148
|
Generated: {datetime.now(timezone.utc).isoformat()}
|
|
149
|
+
Source: {model.get('sourceSystem', 'Unknown')}
|
|
136
150
|
Role: {model['role']}
|
|
137
151
|
Role confidence: {model['roleConfidence']}
|
|
138
152
|
Closeness: {model['closeness']}
|
|
139
153
|
Conversation difficulty: {model['conversationDifficulty']}
|
|
140
154
|
Typing style: {model['typingStyle'].get('signature', 'unknown')}
|
|
141
155
|
|
|
142
|
-
These are private working notes. Edit them where wrong.
|
|
156
|
+
Generated draft, not truth. These are private working notes. Edit them where wrong.
|
|
143
157
|
|
|
144
158
|
## Role / Relationship Label
|
|
145
159
|
- {model['role']} ({model['roleConfidence']} confidence).
|
|
@@ -173,8 +187,62 @@ def write_index(path, models):
|
|
|
173
187
|
lines = ["# Interpreted Relationship Memory", "", "Generated working notes. Treat as editable, not truth.", ""]
|
|
174
188
|
for model in models:
|
|
175
189
|
style = model.get("typingStyle", {}).get("signature", "unknown style")
|
|
176
|
-
|
|
177
|
-
|
|
190
|
+
display = model.get("displayName") or model["chatName"]
|
|
191
|
+
lines.append(f"- [[{safe_filename(display)}]]: {model['role']} ({model['roleConfidence']}), closeness {model['closeness']}, difficulty {model['conversationDifficulty']}, style {style}")
|
|
192
|
+
write_text_atomic(path, "\n".join(lines) + "\n")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def write_person_reply_index(path, people, models):
|
|
196
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
197
|
+
models_by_key = {}
|
|
198
|
+
for model in models:
|
|
199
|
+
key = model.get("canonicalPersonKey")
|
|
200
|
+
if key:
|
|
201
|
+
models_by_key.setdefault(key, []).append(model)
|
|
202
|
+
if not people:
|
|
203
|
+
people = synthesize_people(models_by_key)
|
|
204
|
+
lines = [
|
|
205
|
+
"# Person Reply Context",
|
|
206
|
+
"",
|
|
207
|
+
"Use this first when responding to a specific person. It merges confirmed-looking matches across sources while keeping each source visible.",
|
|
208
|
+
"",
|
|
209
|
+
]
|
|
210
|
+
for person in people:
|
|
211
|
+
key = person.get("canonicalPersonKey")
|
|
212
|
+
linked_models = sorted(models_by_key.get(key, []), key=lambda model: (model["messageCount"], model["lastSeen"]), reverse=True)
|
|
213
|
+
if not linked_models:
|
|
214
|
+
continue
|
|
215
|
+
lines.extend([
|
|
216
|
+
f"## {person.get('displayName') or linked_models[0].get('identityName') or linked_models[0]['chatName']}",
|
|
217
|
+
"",
|
|
218
|
+
f"- Canonical key: `{key}`",
|
|
219
|
+
f"- Aliases: {', '.join(person.get('aliases') or [])}",
|
|
220
|
+
f"- Sources: {', '.join(sorted({model['sourceSystem'] for model in linked_models}))}",
|
|
221
|
+
f"- Total messages: {sum(model['messageCount'] for model in linked_models)}",
|
|
222
|
+
"- Source-specific guidance:",
|
|
223
|
+
])
|
|
224
|
+
for model in linked_models:
|
|
225
|
+
lines.extend([
|
|
226
|
+
f" - {model['sourceSystem']} / {model['chatName']}: {model['role']} ({model['roleConfidence']}), closeness {model['closeness']}, difficulty {model['conversationDifficulty']}.",
|
|
227
|
+
f" Style: {model.get('typingStyle', {}).get('signature', 'unknown')}.",
|
|
228
|
+
f" Reply: {' '.join(model.get('replyStyle', [])[:2])}",
|
|
229
|
+
])
|
|
230
|
+
lines.append("")
|
|
231
|
+
write_text_atomic(path, "\n".join(lines) + "\n")
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def synthesize_people(models_by_key):
|
|
235
|
+
people = []
|
|
236
|
+
for key, models in models_by_key.items():
|
|
237
|
+
if key.startswith("group::"):
|
|
238
|
+
continue
|
|
239
|
+
names = [model.get("identityName") or model["chatName"] for model in models]
|
|
240
|
+
people.append({
|
|
241
|
+
"canonicalPersonKey": key,
|
|
242
|
+
"displayName": names[0],
|
|
243
|
+
"aliases": sorted(set(names)),
|
|
244
|
+
})
|
|
245
|
+
return people
|
|
178
246
|
|
|
179
247
|
|
|
180
248
|
def bullets(items):
|
|
@@ -229,6 +297,27 @@ def load_json(path, fallback):
|
|
|
229
297
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
230
298
|
|
|
231
299
|
|
|
300
|
+
def write_json_atomic(path, data):
|
|
301
|
+
write_text_atomic(path, json.dumps(data, indent=2, ensure_ascii=False))
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def write_text_atomic(path, content):
|
|
305
|
+
temp = path.with_name(f"{path.name}.{time.time_ns()}.tmp")
|
|
306
|
+
temp.write_text(content, encoding="utf-8")
|
|
307
|
+
temp.replace(path)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def load_overrides(sources):
|
|
311
|
+
merged = {}
|
|
312
|
+
for path in sorted(sources.glob("*/relationship_overrides.json")):
|
|
313
|
+
merged.update(load_json(path, {}))
|
|
314
|
+
return merged
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def profile_key(profile):
|
|
318
|
+
return f"{profile.get('sourceSystem') or 'Unknown'}::{profile.get('chatName') or 'Unknown Chat'}"
|
|
319
|
+
|
|
320
|
+
|
|
232
321
|
def parse_args():
|
|
233
322
|
parser = argparse.ArgumentParser()
|
|
234
323
|
parser.add_argument("--vault", type=Path, required=True)
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import tempfile
|
|
5
|
+
import zipfile
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def main():
|
|
11
|
+
args = parse_args()
|
|
12
|
+
with unpack(args.input) as source:
|
|
13
|
+
import_export(args.vault.resolve(), source, args.days)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def import_export(vault, source, days):
|
|
17
|
+
slack = vault / "08 Sources" / "Slack"
|
|
18
|
+
raw_dir = slack / "Raw"
|
|
19
|
+
chats_dir = slack / "ChatsByMonth"
|
|
20
|
+
state_dir = slack / ".sync-state"
|
|
21
|
+
for directory in (raw_dir, chats_dir, state_dir):
|
|
22
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
23
|
+
|
|
24
|
+
users = load_users(source / "users.json")
|
|
25
|
+
conversations = load_conversations(source)
|
|
26
|
+
seen_path = state_dir / "slack-seen-message-ids.json"
|
|
27
|
+
seen = load_seen(seen_path)
|
|
28
|
+
cutoff = datetime.now(timezone.utc).timestamp() - days * 24 * 60 * 60 if days else None
|
|
29
|
+
added = 0
|
|
30
|
+
|
|
31
|
+
for folder in sorted(path for path in source.iterdir() if path.is_dir()):
|
|
32
|
+
conversation = conversations.get(folder.name, {"name": folder.name, "is_group": True})
|
|
33
|
+
chat_name = conversation.get("name") or folder.name
|
|
34
|
+
is_group = conversation.get("is_group", True)
|
|
35
|
+
for file in sorted(folder.glob("*.json")):
|
|
36
|
+
for item in load_json(file, []):
|
|
37
|
+
if "ts" not in item or not item.get("text"):
|
|
38
|
+
continue
|
|
39
|
+
timestamp = float(item["ts"])
|
|
40
|
+
if cutoff and timestamp < cutoff:
|
|
41
|
+
continue
|
|
42
|
+
record = slack_record(item, users, chat_name, is_group, folder.name)
|
|
43
|
+
if record["id"] in seen:
|
|
44
|
+
continue
|
|
45
|
+
append_jsonl(raw_dir, record)
|
|
46
|
+
append_markdown(chats_dir, record)
|
|
47
|
+
seen.add(record["id"])
|
|
48
|
+
added += 1
|
|
49
|
+
|
|
50
|
+
save_seen(seen_path, seen)
|
|
51
|
+
print(f"Imported {added} Slack messages.")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def slack_record(item, users, chat_name, is_group, conversation_id):
|
|
55
|
+
user_id = item.get("user") or item.get("bot_id") or "unknown"
|
|
56
|
+
author = users.get(user_id, user_id)
|
|
57
|
+
timestamp = datetime.fromtimestamp(float(item["ts"]), tz=timezone.utc).isoformat()
|
|
58
|
+
return {
|
|
59
|
+
"id": f"slack-{conversation_id}-{item['ts']}",
|
|
60
|
+
"source": "Slack export",
|
|
61
|
+
"sourceSystem": "Slack",
|
|
62
|
+
"timestamp": timestamp,
|
|
63
|
+
"chatName": chat_name,
|
|
64
|
+
"chatId": conversation_id,
|
|
65
|
+
"isGroup": is_group,
|
|
66
|
+
"fromMe": False,
|
|
67
|
+
"author": author,
|
|
68
|
+
"authorId": user_id,
|
|
69
|
+
"body": item.get("text") or "",
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def load_users(path):
|
|
74
|
+
users = {}
|
|
75
|
+
for item in load_json(path, []):
|
|
76
|
+
user_id = item.get("id")
|
|
77
|
+
if not user_id:
|
|
78
|
+
continue
|
|
79
|
+
profile = item.get("profile") or {}
|
|
80
|
+
users[user_id] = profile.get("real_name") or profile.get("display_name") or item.get("name") or user_id
|
|
81
|
+
return users
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def load_conversations(source):
|
|
85
|
+
conversations = {}
|
|
86
|
+
for filename in ("channels.json", "groups.json", "dms.json", "mpims.json"):
|
|
87
|
+
for item in load_json(source / filename, []):
|
|
88
|
+
conversation_id = item.get("id") or item.get("name")
|
|
89
|
+
if not conversation_id:
|
|
90
|
+
continue
|
|
91
|
+
conversations[conversation_id] = {
|
|
92
|
+
"name": item.get("name") or item.get("name_normalized") or conversation_id,
|
|
93
|
+
"is_group": not filename == "dms.json",
|
|
94
|
+
}
|
|
95
|
+
return conversations
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def append_jsonl(raw_dir, record):
|
|
99
|
+
with (raw_dir / f"{record['timestamp'][:10]}.jsonl").open("a", encoding="utf-8") as f:
|
|
100
|
+
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def append_markdown(chats_dir, record):
|
|
104
|
+
directory = chats_dir / record["timestamp"][:7]
|
|
105
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
file_path = directory / f"{safe_filename(record['chatName'])}.md"
|
|
107
|
+
if not file_path.exists():
|
|
108
|
+
file_path.write_text(f"# {record['chatName']}\n\nSynced from Slack export.\n\n", encoding="utf-8")
|
|
109
|
+
body = " ".join((record.get("body") or "").split())
|
|
110
|
+
with file_path.open("a", encoding="utf-8") as f:
|
|
111
|
+
f.write(f"- {record['timestamp']} | {record['author']}: {body}\n")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def load_json(path, fallback):
|
|
115
|
+
if not path.exists():
|
|
116
|
+
return fallback
|
|
117
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def load_seen(path):
|
|
121
|
+
if not path.exists():
|
|
122
|
+
return set()
|
|
123
|
+
try:
|
|
124
|
+
return set(json.loads(path.read_text(encoding="utf-8")))
|
|
125
|
+
except Exception:
|
|
126
|
+
return set()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def save_seen(path, seen):
|
|
130
|
+
path.write_text(json.dumps(sorted(seen), indent=2), encoding="utf-8")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def safe_filename(value):
|
|
134
|
+
cleaned = "".join("-" if char in '/:\\?%*"<>|' else char for char in value)
|
|
135
|
+
return (" ".join(cleaned.split()).strip() or "Unknown Chat")[:120]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def unpack(input_path):
|
|
139
|
+
input_path = input_path.resolve()
|
|
140
|
+
if input_path.is_dir():
|
|
141
|
+
return NullContext(input_path)
|
|
142
|
+
temp = tempfile.TemporaryDirectory()
|
|
143
|
+
with zipfile.ZipFile(input_path) as archive:
|
|
144
|
+
archive.extractall(temp.name)
|
|
145
|
+
return TempContext(Path(temp.name), temp)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class NullContext:
|
|
149
|
+
def __init__(self, path):
|
|
150
|
+
self.path = path
|
|
151
|
+
|
|
152
|
+
def __enter__(self):
|
|
153
|
+
return self.path
|
|
154
|
+
|
|
155
|
+
def __exit__(self, *_):
|
|
156
|
+
return False
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class TempContext:
|
|
160
|
+
def __init__(self, path, temp):
|
|
161
|
+
self.path = path
|
|
162
|
+
self.temp = temp
|
|
163
|
+
|
|
164
|
+
def __enter__(self):
|
|
165
|
+
return self.path
|
|
166
|
+
|
|
167
|
+
def __exit__(self, *_):
|
|
168
|
+
self.temp.cleanup()
|
|
169
|
+
return False
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def parse_args():
|
|
173
|
+
parser = argparse.ArgumentParser()
|
|
174
|
+
parser.add_argument("--vault", type=Path, required=True)
|
|
175
|
+
parser.add_argument("--input", type=Path, required=True)
|
|
176
|
+
parser.add_argument("--days", type=int, default=365)
|
|
177
|
+
return parser.parse_args()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
if __name__ == "__main__":
|
|
181
|
+
main()
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
import argparse
|
|
3
|
+
import hashlib
|
|
3
4
|
import json
|
|
4
5
|
import sqlite3
|
|
5
6
|
import time
|
|
@@ -60,7 +61,7 @@ def sync_once(args, seen, raw_dir, chats_dir, whats_app):
|
|
|
60
61
|
|
|
61
62
|
added = 0
|
|
62
63
|
for row in rows:
|
|
63
|
-
record = row_to_record(row, args.self_name)
|
|
64
|
+
record = row_to_record(row, args.self_name, args.privacy_mode)
|
|
64
65
|
if args.no_groups and record["isGroup"]:
|
|
65
66
|
continue
|
|
66
67
|
if args.chat and args.chat.lower() not in record["chatName"].lower():
|
|
@@ -74,13 +75,16 @@ def sync_once(args, seen, raw_dir, chats_dir, whats_app):
|
|
|
74
75
|
return added
|
|
75
76
|
|
|
76
77
|
|
|
77
|
-
def row_to_record(row, self_name):
|
|
78
|
+
def row_to_record(row, self_name, privacy_mode):
|
|
78
79
|
timestamp = datetime.fromtimestamp(float(row["message_date"]) + CORE_DATA_EPOCH_OFFSET, tz=timezone.utc).isoformat()
|
|
79
80
|
chat_name = row["chat_name"] or row["chat_jid"] or "Unknown Chat"
|
|
80
81
|
from_me = bool(row["is_from_me"])
|
|
82
|
+
body = row["text"] or ""
|
|
83
|
+
record_id = compound_id(row, timestamp)
|
|
81
84
|
return {
|
|
82
|
-
"id":
|
|
85
|
+
"id": record_id,
|
|
83
86
|
"source": "WhatsApp Mac app ChatStorage.sqlite",
|
|
87
|
+
"sourceSystem": "WhatsApp",
|
|
84
88
|
"timestamp": timestamp,
|
|
85
89
|
"chatPk": row["chat_pk"],
|
|
86
90
|
"chatName": chat_name,
|
|
@@ -91,10 +95,23 @@ def row_to_record(row, self_name):
|
|
|
91
95
|
"fromJid": row["from_jid"],
|
|
92
96
|
"toJid": row["to_jid"],
|
|
93
97
|
"messageType": row["message_type"],
|
|
94
|
-
"body":
|
|
98
|
+
"body": "" if privacy_mode == "metadata-only" else body,
|
|
99
|
+
"bodyHash": hashlib.sha256(body.encode("utf-8")).hexdigest() if privacy_mode == "metadata-only" else "",
|
|
100
|
+
"bodyCharCount": len(body),
|
|
95
101
|
}
|
|
96
102
|
|
|
97
103
|
|
|
104
|
+
def compound_id(row, timestamp):
|
|
105
|
+
parts = [
|
|
106
|
+
"whatsapp",
|
|
107
|
+
str(row["chat_pk"] or row["chat_jid"] or "unknown-chat"),
|
|
108
|
+
str(row["stanza_id"] or "no-stanza"),
|
|
109
|
+
str(row["message_pk"] or "no-pk"),
|
|
110
|
+
timestamp,
|
|
111
|
+
]
|
|
112
|
+
return "::".join(parts)
|
|
113
|
+
|
|
114
|
+
|
|
98
115
|
def append_jsonl(raw_dir, record):
|
|
99
116
|
with (raw_dir / f"{record['timestamp'][:10]}.jsonl").open("a", encoding="utf-8") as f:
|
|
100
117
|
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
@@ -110,9 +127,9 @@ def append_markdown(chats_dir, whats_app, record, mode):
|
|
|
110
127
|
else:
|
|
111
128
|
file_path = chats_dir / f"{safe_filename(record['chatName'])}.md"
|
|
112
129
|
if not file_path.exists():
|
|
113
|
-
file_path
|
|
114
|
-
speaker = record["author"]
|
|
115
|
-
body = " ".join(record["body"].split())
|
|
130
|
+
write_text_atomic(file_path, f"# {escape_markdown(record['chatName'])}\n\nSynced from WhatsApp Mac app.\n\n")
|
|
131
|
+
speaker = escape_markdown(record["author"])
|
|
132
|
+
body = escape_markdown(" ".join(record["body"].split()))
|
|
116
133
|
with file_path.open("a", encoding="utf-8") as f:
|
|
117
134
|
f.write(f"- {record['timestamp']} | {speaker}: {body}\n")
|
|
118
135
|
|
|
@@ -127,7 +144,7 @@ def load_seen(path):
|
|
|
127
144
|
|
|
128
145
|
|
|
129
146
|
def save_seen(path, seen):
|
|
130
|
-
path
|
|
147
|
+
write_text_atomic(path, json.dumps(sorted(seen), indent=2))
|
|
131
148
|
|
|
132
149
|
|
|
133
150
|
def safe_filename(value):
|
|
@@ -135,6 +152,17 @@ def safe_filename(value):
|
|
|
135
152
|
return (" ".join(cleaned.split()).strip() or "Unknown Chat")[:120]
|
|
136
153
|
|
|
137
154
|
|
|
155
|
+
def escape_markdown(value):
|
|
156
|
+
text = str(value).replace("\n", " ").replace("\r", " ")
|
|
157
|
+
return text.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]").replace("|", "\\|")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def write_text_atomic(path, content):
|
|
161
|
+
temp = path.with_name(f"{path.name}.{time.time_ns()}.tmp")
|
|
162
|
+
temp.write_text(content, encoding="utf-8")
|
|
163
|
+
temp.replace(path)
|
|
164
|
+
|
|
165
|
+
|
|
138
166
|
def parse_args():
|
|
139
167
|
parser = argparse.ArgumentParser()
|
|
140
168
|
parser.add_argument("--vault", type=Path, required=True)
|
|
@@ -146,6 +174,7 @@ def parse_args():
|
|
|
146
174
|
parser.add_argument("--db", type=Path, default=DEFAULT_DB)
|
|
147
175
|
parser.add_argument("--self-name", default="Me")
|
|
148
176
|
parser.add_argument("--markdown-mode", choices=["chat", "month", "none"], default="chat")
|
|
177
|
+
parser.add_argument("--privacy-mode", choices=["standard", "metadata-only"], default="standard")
|
|
149
178
|
return parser.parse_args()
|
|
150
179
|
|
|
151
180
|
|
|
@@ -7,5 +7,5 @@ Rules:
|
|
|
7
7
|
- Prefer local evidence over assumptions.
|
|
8
8
|
- Be clear when a claim is inferred.
|
|
9
9
|
- Treat generated relationship notes as provisional.
|
|
10
|
+
- When replying to a person, check `06 AI Memory/Person Reply Context.md` first.
|
|
10
11
|
- Do not send messages without explicit permission.
|
|
11
|
-
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# Relationship Overrides
|
|
2
2
|
|
|
3
|
+
Keep human-edited relationship notes in this folder. Generated drafts are written to `06 AI Memory/Generated Relationship Drafts/` and may be overwritten.
|
|
4
|
+
|
|
3
5
|
Use `08 Sources/WhatsApp/relationship_overrides.json` to manually label important people.
|
|
4
6
|
|
|
5
7
|
Example:
|
|
@@ -13,4 +15,3 @@ Example:
|
|
|
13
15
|
}
|
|
14
16
|
}
|
|
15
17
|
```
|
|
16
|
-
|
|
@@ -5,13 +5,13 @@ Outbound sending is disabled unless you explicitly run the sender with `--yes`.
|
|
|
5
5
|
Draft:
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
digital-brain send-whatsapp --
|
|
8
|
+
digital-brain send-whatsapp --to "Name" --message "Text"
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
Send:
|
|
12
12
|
|
|
13
13
|
```bash
|
|
14
|
-
digital-brain send-whatsapp --
|
|
14
|
+
digital-brain send-whatsapp --to "Name" --message "Text" --yes
|
|
15
15
|
```
|
|
16
16
|
|
|
17
17
|
Disclosure rule:
|
|
@@ -8,6 +8,8 @@ Before answering personal-context questions, read:
|
|
|
8
8
|
- `00 Home/How AI Should Use This Vault.md`
|
|
9
9
|
- `01 Identity/Working Profile.md`
|
|
10
10
|
- `06 AI Memory/What AI Should Remember.md`
|
|
11
|
+
- `06 AI Memory/Person Context Index.md`
|
|
12
|
+
- `06 AI Memory/Person Reply Context.md`
|
|
11
13
|
- `06 AI Memory/Interpreted Relationship Memory.md`
|
|
12
14
|
|
|
13
15
|
Rules:
|
|
@@ -16,4 +18,6 @@ Rules:
|
|
|
16
18
|
- Separate facts from interpretation.
|
|
17
19
|
- Treat relationship labels as editable working notes.
|
|
18
20
|
- Do not expose private summaries unless asked.
|
|
19
|
-
|
|
21
|
+
- For reply help, prefer person-level context over source-specific notes when available.
|
|
22
|
+
- Do not read `08 Sources/` raw files unless the user explicitly asks you to inspect source data.
|
|
23
|
+
- Prefer `06 AI Memory/` summaries and `04 People/` human notes for normal context.
|
|
@@ -4,3 +4,6 @@ This is a Digital Brain vault. Use it as local personal context when relevant.
|
|
|
4
4
|
|
|
5
5
|
Start with `00 Home/Start Here.md`.
|
|
6
6
|
|
|
7
|
+
For reply help, prefer `06 AI Memory/Person Reply Context.md` when available.
|
|
8
|
+
|
|
9
|
+
Do not read `08 Sources/` raw files unless the user explicitly asks for source-level evidence.
|
|
@@ -3,3 +3,7 @@
|
|
|
3
3
|
This is a Digital Brain vault. Use it as local personal context when relevant.
|
|
4
4
|
|
|
5
5
|
Start with `00 Home/Start Here.md`.
|
|
6
|
+
|
|
7
|
+
For reply help, prefer `06 AI Memory/Person Reply Context.md` when available.
|
|
8
|
+
|
|
9
|
+
Do not read `08 Sources/` raw files unless the user explicitly asks for source-level evidence.
|
package/whatsapp-web/send.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
+
import crypto from "node:crypto";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import qrcode from "qrcode-terminal";
|
|
4
5
|
import pkg from "whatsapp-web.js";
|
|
@@ -12,6 +13,8 @@ const vault = path.resolve(args.vault);
|
|
|
12
13
|
const whatsAppDir = path.join(vault, "08 Sources", "WhatsApp");
|
|
13
14
|
const outboundDir = path.join(whatsAppDir, "Outbound");
|
|
14
15
|
const sessionDir = path.join(whatsAppDir, ".session");
|
|
16
|
+
const config = readConfig(vault);
|
|
17
|
+
const outboundLogMode = args["log-mode"] || config.outboundLogMode || "metadata";
|
|
15
18
|
fs.mkdirSync(outboundDir, { recursive: true });
|
|
16
19
|
|
|
17
20
|
if (!args.yes) {
|
|
@@ -36,7 +39,11 @@ client.on("ready", async () => {
|
|
|
36
39
|
try {
|
|
37
40
|
const chat = await resolveChat(args.to);
|
|
38
41
|
const disclosure = disclosureStatus(chat);
|
|
39
|
-
|
|
42
|
+
const bypassDisclosure = args["skip-disclosure-check"] && process.env.DIGITAL_BRAIN_ALLOW_DISCLOSURE_BYPASS === "1";
|
|
43
|
+
if (args["skip-disclosure-check"] && !bypassDisclosure) {
|
|
44
|
+
throw new Error("Disclosure bypass requires DIGITAL_BRAIN_ALLOW_DISCLOSURE_BYPASS=1.");
|
|
45
|
+
}
|
|
46
|
+
if (disclosure.required && !containsDisclosure(args.message) && !bypassDisclosure) {
|
|
40
47
|
throw new Error(
|
|
41
48
|
'AI disclosure required before sending. Add a clear disclosure like: "Just flagging this is my AI assistant helping draft/send this."',
|
|
42
49
|
);
|
|
@@ -46,13 +53,19 @@ client.on("ready", async () => {
|
|
|
46
53
|
timestamp: new Date().toISOString(),
|
|
47
54
|
to: args.to,
|
|
48
55
|
resolvedChatName: chatName(chat),
|
|
49
|
-
message: args.message,
|
|
56
|
+
message: outboundLogMode === "full" ? args.message : undefined,
|
|
57
|
+
messageHash: hash(args.message),
|
|
58
|
+
messageCharCount: args.message.length,
|
|
50
59
|
messageId: sent.id?._serialized || null,
|
|
51
60
|
aiAssisted: !args.human,
|
|
52
61
|
disclosureIncluded: containsDisclosure(args.message),
|
|
62
|
+
disclosureBypassed: bypassDisclosure,
|
|
53
63
|
};
|
|
54
|
-
|
|
55
|
-
|
|
64
|
+
if (outboundLogMode !== "off") {
|
|
65
|
+
fs.appendFileSync(path.join(outboundDir, "sent.jsonl"), `${JSON.stringify(record)}\n`);
|
|
66
|
+
const visible = outboundLogMode === "full" ? args.message : `[metadata only, ${record.messageCharCount} chars, ${record.messageHash.slice(0, 12)}]`;
|
|
67
|
+
fs.appendFileSync(path.join(outboundDir, "Sent.md"), `- ${record.timestamp} | ${record.resolvedChatName}: ${visible}\n`);
|
|
68
|
+
}
|
|
56
69
|
console.log(`Sent to ${record.resolvedChatName}`);
|
|
57
70
|
await client.destroy();
|
|
58
71
|
process.exit(0);
|
|
@@ -104,7 +117,7 @@ function parseArgs(argv) {
|
|
|
104
117
|
}
|
|
105
118
|
|
|
106
119
|
function usage() {
|
|
107
|
-
console.error('Usage: digital-brain send-whatsapp --vault <path> --to "Name" --message "Text" [--yes]');
|
|
120
|
+
console.error('Usage: digital-brain send-whatsapp --vault <path> --to "Name" --message "Text" [--yes] [--log-mode metadata|full|off]');
|
|
108
121
|
process.exit(1);
|
|
109
122
|
}
|
|
110
123
|
|
|
@@ -137,3 +150,17 @@ function disclosureStatus(chat) {
|
|
|
137
150
|
function containsDisclosure(message) {
|
|
138
151
|
return /\b(ai|assistant|automated|bot)\b/i.test(message);
|
|
139
152
|
}
|
|
153
|
+
|
|
154
|
+
function readConfig(vaultPath) {
|
|
155
|
+
const file = path.join(vaultPath, "digital-brain.config.json");
|
|
156
|
+
if (!fs.existsSync(file)) return {};
|
|
157
|
+
try {
|
|
158
|
+
return JSON.parse(fs.readFileSync(file, "utf8"));
|
|
159
|
+
} catch {
|
|
160
|
+
return {};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function hash(value) {
|
|
165
|
+
return crypto.createHash("sha256").update(value).digest("hex");
|
|
166
|
+
}
|