digital-brain 0.1.7 → 1.0.3

package/examples/sample-vault/08 Sources/WhatsApp/Analysis/person_identity_map.json

@@ -0,0 +1,78 @@
+[
+  {
+    "canonicalPersonKey": "person::mom",
+    "displayName": "Mom",
+    "aliases": [
+      "Mom"
+    ],
+    "sources": [
+      "WhatsApp"
+    ],
+    "sourceProfiles": [
+      {
+        "sourceSystem": "WhatsApp",
+        "chatName": "Mom",
+        "displayName": "Mom (WhatsApp)",
+        "messageCount": 2,
+        "firstSeen": "2026-01-01",
+        "lastSeen": "2026-01-01",
+        "relationshipGuess": "warm personal relationship",
+        "typingStyle": {
+          "sampleSize": 1,
+          "avgChars": 24.0,
+          "avgWords": 4.0,
+          "lowercaseShare": 0.0,
+          "uppercaseShare": 0.0,
+          "questionShare": 0.0,
+          "exclamationShare": 0.0,
+          "emojiShare": 0.0,
+          "slang": [],
+          "signature": "very short"
+        },
+        "identityConfidence": "medium",
+        "identityEvidence": "WhatsApp direct chat identity"
+      }
+    ],
+    "totalMessages": 2,
+    "firstSeen": "2026-01-01",
+    "lastSeen": "2026-01-01"
+  },
+  {
+    "canonicalPersonKey": "person::close friend",
+    "displayName": "Close Friend",
+    "aliases": [
+      "Close Friend"
+    ],
+    "sources": [
+      "WhatsApp"
+    ],
+    "sourceProfiles": [
+      {
+        "sourceSystem": "WhatsApp",
+        "chatName": "Close Friend",
+        "displayName": "Close Friend (WhatsApp)",
+        "messageCount": 2,
+        "firstSeen": "2026-01-01",
+        "lastSeen": "2026-01-01",
+        "relationshipGuess": "warm personal relationship",
+        "typingStyle": {
+          "sampleSize": 1,
+          "avgChars": 30.0,
+          "avgWords": 5.0,
+          "lowercaseShare": 0.0,
+          "uppercaseShare": 0.0,
+          "questionShare": 0.0,
+          "exclamationShare": 0.0,
+          "emojiShare": 0.0,
+          "slang": [],
+          "signature": "short"
+        },
+        "identityConfidence": "medium",
+        "identityEvidence": "WhatsApp direct chat identity"
+      }
+    ],
+    "totalMessages": 2,
+    "firstSeen": "2026-01-01",
+    "lastSeen": "2026-01-01"
+  }
+]

package/examples/sample-vault/08 Sources/WhatsApp/Analysis/relationship_profiles.json

@@ -1,6 +1,12 @@
 [
   {
     "chatName": "Mom",
+    "sourceSystem": "WhatsApp",
+    "displayName": "Mom (WhatsApp)",
+    "identityName": "Mom",
+    "canonicalPersonKey": "person::mom",
+    "identityConfidence": "medium",
+    "identityEvidence": "WhatsApp direct chat identity",
     "messageCount": 2,
     "inbound": 1,
     "outbound": 1,
@@ -35,6 +41,12 @@
   },
   {
     "chatName": "Project Team",
+    "sourceSystem": "WhatsApp",
+    "displayName": "Project Team (WhatsApp)",
+    "identityName": "Project Team",
+    "canonicalPersonKey": "group::whatsapp::project team",
+    "identityConfidence": "medium",
+    "identityEvidence": "group chat kept source-specific",
     "messageCount": 2,
     "inbound": 1,
     "outbound": 1,
@@ -69,6 +81,12 @@
   },
   {
     "chatName": "Close Friend",
+    "sourceSystem": "WhatsApp",
+    "displayName": "Close Friend (WhatsApp)",
+    "identityName": "Close Friend",
+    "canonicalPersonKey": "person::close friend",
+    "identityConfidence": "medium",
+    "identityEvidence": "WhatsApp direct chat identity",
     "messageCount": 2,
     "inbound": 1,
     "outbound": 1,

package/examples/sample-vault/08 Sources/WhatsApp/Raw/2026-01-01.jsonl

@@ -1,6 +1,6 @@
-{"id":"sample-1","source":"sample","timestamp":"2026-01-01T10:00:00+00:00","chatName":"Mom","isGroup":false,"fromMe":false,"author":"Mom","body":"Good morning, call me when you wake up"}
-{"id":"sample-2","source":"sample","timestamp":"2026-01-01T10:05:00+00:00","chatName":"Mom","isGroup":false,"fromMe":true,"author":"Me","body":"Morning, will call in 10"}
-{"id":"sample-3","source":"sample","timestamp":"2026-01-01T11:00:00+00:00","chatName":"Project Team","isGroup":true,"fromMe":false,"author":"Alex","body":"Can you send the deck before the client meeting?"}
-{"id":"sample-4","source":"sample","timestamp":"2026-01-01T11:03:00+00:00","chatName":"Project Team","isGroup":true,"fromMe":true,"author":"Me","body":"Yes, sending the updated deck now"}
-{"id":"sample-5","source":"sample","timestamp":"2026-01-01T20:00:00+00:00","chatName":"Close Friend","isGroup":false,"fromMe":false,"author":"Friend","body":"That was fun haha, let's plan another trip"}
-{"id":"sample-6","source":"sample","timestamp":"2026-01-01T20:02:00+00:00","chatName":"Close Friend","isGroup":false,"fromMe":true,"author":"Me","body":"Yes absolutely, that was great"}
+{"id":"sample-1","source":"sample","sourceSystem":"WhatsApp","timestamp":"2026-01-01T10:00:00+00:00","chatName":"Mom","isGroup":false,"fromMe":false,"author":"Mom","body":"Good morning, call me when you wake up"}
+{"id":"sample-2","source":"sample","sourceSystem":"WhatsApp","timestamp":"2026-01-01T10:05:00+00:00","chatName":"Mom","isGroup":false,"fromMe":true,"author":"Me","body":"Morning, will call in 10"}
+{"id":"sample-3","source":"sample","sourceSystem":"WhatsApp","timestamp":"2026-01-01T11:00:00+00:00","chatName":"Project Team","isGroup":true,"fromMe":false,"author":"Alex","body":"Can you send the deck before the client meeting?"}
+{"id":"sample-4","source":"sample","sourceSystem":"WhatsApp","timestamp":"2026-01-01T11:03:00+00:00","chatName":"Project Team","isGroup":true,"fromMe":true,"author":"Me","body":"Yes, sending the updated deck now"}
+{"id":"sample-5","source":"sample","sourceSystem":"WhatsApp","timestamp":"2026-01-01T20:00:00+00:00","chatName":"Close Friend","isGroup":false,"fromMe":false,"author":"Friend","body":"That was fun haha, let's plan another trip"}
+{"id":"sample-6","source":"sample","sourceSystem":"WhatsApp","timestamp":"2026-01-01T20:02:00+00:00","chatName":"Close Friend","isGroup":false,"fromMe":true,"author":"Me","body":"Yes absolutely, that was great"}

package/lib/fs.js

@@ -55,5 +55,11 @@ export function readDefaultVault() {
 export function writeDefaultVault(vault) {
   const file = globalConfigPath();
   ensureDir(path.dirname(file));
-  fs.writeFileSync(file, `${JSON.stringify({ defaultVault: vault }, null, 2)}\n`);
+  writeFileAtomic(file, `${JSON.stringify({ defaultVault: vault }, null, 2)}\n`);
+}
+
+function writeFileAtomic(file, content) {
+  const temp = `${file}.${process.pid}.tmp`;
+  fs.writeFileSync(temp, content);
+  fs.renameSync(temp, file);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "digital-brain",
-  "version": "0.1.7",
+  "version": "1.0.3",
   "description": "Your private digital imprint for AI assistants.",
   "type": "module",
   "bin": {
@@ -29,6 +29,7 @@
     "personal-ai",
     "obsidian",
     "whatsapp",
+    "imessage",
     "local-first",
     "context"
   ],

package/scripts/digital_brain_imessage_sync.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+import argparse
+import hashlib
+import json
+import sqlite3
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+CORE_DATA_EPOCH_OFFSET = 978_307_200
+DEFAULT_DB = Path.home() / "Library" / "Messages" / "chat.db"
+
+
+def main():
+    args = parse_args()
+    if not args.db.exists():
+        raise SystemExit(
+            f"Apple Messages database not found: {args.db}\n"
+            "Open Messages on macOS and grant Terminal Full Disk Access if macOS blocks access."
+        )
+
+    vault = args.vault.resolve()
+    source_dir = vault / "08 Sources" / "iMessage"
+    raw_dir = source_dir / "Raw"
+    chats_dir = source_dir / "ChatsByMonth"
+    state_dir = source_dir / ".sync-state"
+    for directory in (raw_dir, chats_dir, state_dir):
+        directory.mkdir(parents=True, exist_ok=True)
+
+    seen_path = state_dir / "imessage-seen-message-ids.json"
+    seen = load_seen(seen_path)
+    added = sync_once(args, seen, raw_dir, chats_dir)
+    save_seen(seen_path, seen)
+    print(f"Imported {added} iMessage messages.")
+
+
+def sync_once(args, seen, raw_dir, chats_dir):
+    cutoff = datetime.now(timezone.utc).timestamp() - args.days * 24 * 60 * 60 if args.days else None
+    conn = sqlite3.connect(f"file:{args.db}?mode=ro", uri=True)
+    conn.row_factory = sqlite3.Row
+    rows = conn.execute(
+        """
+        SELECT m.ROWID message_pk, m.guid guid, m.date message_date,
+               m.is_from_me is_from_me, m.text text, m.service service,
+               h.id handle_id,
+               c.ROWID chat_pk, c.display_name display_name, c.chat_identifier chat_identifier
+        FROM message m
+        LEFT JOIN handle h ON h.ROWID = m.handle_id
+        LEFT JOIN chat_message_join cmj ON cmj.message_id = m.ROWID
+        LEFT JOIN chat c ON c.ROWID = cmj.chat_id
+        WHERE m.text IS NOT NULL
+          AND length(m.text) > 0
+        ORDER BY m.date ASC, m.ROWID ASC
+        """
+    ).fetchall()
+    conn.close()
+
+    added = 0
+    for row in rows:
+        record = row_to_record(row, args.self_name, args.privacy_mode)
+        if cutoff and datetime.fromisoformat(record["timestamp"]).timestamp() < cutoff:
+            continue
+        if args.chat and args.chat.lower() not in record["chatName"].lower():
+            continue
+        if record["id"] in seen:
+            continue
+        append_jsonl(raw_dir, record)
+        append_markdown(chats_dir, record, args.markdown_mode)
+        seen.add(record["id"])
+        added += 1
+    return added
+
+
+def row_to_record(row, self_name, privacy_mode):
+    timestamp = apple_timestamp(row["message_date"])
+    body = row["text"] or ""
+    from_me = bool(row["is_from_me"])
+    chat_name = row["display_name"] or row["chat_identifier"] or row["handle_id"] or "iMessage"
+    return {
+        "id": compound_id(row, timestamp),
+        "source": "Apple Messages chat.db",
+        "sourceSystem": "iMessage",
+        "timestamp": timestamp,
+        "chatPk": row["chat_pk"],
+        "chatName": chat_name,
+        "chatIdentifier": row["chat_identifier"],
+        "isGroup": bool(row["display_name"]) or str(row["chat_identifier"] or "").startswith("chat"),
+        "fromMe": from_me,
+        "author": self_name if from_me else (row["handle_id"] or "Unknown"),
+        "handleId": row["handle_id"],
+        "service": row["service"],
+        "body": "" if privacy_mode == "metadata-only" else body,
+        "bodyHash": hashlib.sha256(body.encode("utf-8")).hexdigest() if privacy_mode == "metadata-only" else "",
+        "bodyCharCount": len(body),
+    }
+
+
+def apple_timestamp(value):
+    raw = float(value or 0)
+    seconds = raw / 1_000_000_000 if abs(raw) > 10_000_000_000 else raw
+    return datetime.fromtimestamp(seconds + CORE_DATA_EPOCH_OFFSET, tz=timezone.utc).isoformat()
+
+
+def compound_id(row, timestamp):
+    return "::".join([
+        "imessage",
+        str(row["chat_pk"] or row["chat_identifier"] or row["handle_id"] or "unknown-chat"),
+        str(row["guid"] or "no-guid"),
+        str(row["message_pk"] or "no-pk"),
+        timestamp,
+    ])
+
+
+def append_jsonl(raw_dir, record):
+    with (raw_dir / f"{record['timestamp'][:10]}.jsonl").open("a", encoding="utf-8") as f:
+        f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+def append_markdown(chats_dir, record, mode):
+    if mode == "none":
+        return
+    directory = chats_dir / record["timestamp"][:7]
+    directory.mkdir(parents=True, exist_ok=True)
+    file_path = directory / f"{safe_filename(record['chatName'])}.md"
+    if not file_path.exists():
+        write_text_atomic(file_path, f"# {escape_markdown(record['chatName'])}\n\nSynced from Apple Messages.\n\n")
+    speaker = escape_markdown(record["author"])
+    body = escape_markdown(" ".join(record["body"].split()))
+    with file_path.open("a", encoding="utf-8") as f:
+        f.write(f"- {record['timestamp']} | {speaker}: {body}\n")
+
+
+def load_seen(path):
+    if not path.exists():
+        return set()
+    try:
+        return set(json.loads(path.read_text(encoding="utf-8")))
+    except Exception:
+        return set()
+
+
+def save_seen(path, seen):
+    write_text_atomic(path, json.dumps(sorted(seen), indent=2))
+
+
+def safe_filename(value):
+    cleaned = "".join("-" if char in '/:\\?%*"<>|' else char for char in value)
+    return (" ".join(cleaned.split()).strip() or "iMessage")[:120]
+
+
+def escape_markdown(value):
+    text = str(value).replace("\n", " ").replace("\r", " ")
+    return text.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]").replace("|", "\\|")
+
+
+def write_text_atomic(path, content):
+    temp = path.with_name(f"{path.name}.{time.time_ns()}.tmp")
+    temp.write_text(content, encoding="utf-8")
+    temp.replace(path)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vault", type=Path, required=True)
+    parser.add_argument("--days", type=int, default=30)
+    parser.add_argument("--chat", default="")
+    parser.add_argument("--db", type=Path, default=DEFAULT_DB)
+    parser.add_argument("--self-name", default="Me")
+    parser.add_argument("--markdown-mode", choices=["chat", "month", "none"], default="none")
+    parser.add_argument("--privacy-mode", choices=["standard", "metadata-only"], default="standard")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main()

package/scripts/digital_brain_linkedin_export_import.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+import argparse
+import csv
+import json
+import tempfile
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def main():
+    args = parse_args()
+    with unpack(args.input) as source:
+        import_archive(args.vault.resolve(), source, args.days)
+
+
+def import_archive(vault, source, days):
+    linkedin = vault / "08 Sources" / "LinkedIn"
+    raw_dir = linkedin / "Raw"
+    state_dir = linkedin / ".sync-state"
+    people_dir = vault / "04 People"
+    for directory in (raw_dir, state_dir, people_dir):
+        directory.mkdir(parents=True, exist_ok=True)
+
+    cutoff = datetime.now(timezone.utc).timestamp() - days * 24 * 60 * 60 if days else None
+    seen_path = state_dir / "linkedin-seen-message-ids.json"
+    seen = load_seen(seen_path)
+    imported_messages = 0
+    imported_connections = write_connections(source, people_dir)
+
+    for file in find_csvs(source, "message"):
+        for row in read_csv(file):
+            record = message_record(row, file)
+            if not record:
+                continue
+            if cutoff and datetime.fromisoformat(record["timestamp"]).timestamp() < cutoff:
+                continue
+            if record["id"] in seen:
+                continue
+            append_jsonl(raw_dir, record)
+            seen.add(record["id"])
+            imported_messages += 1
+
+    save_seen(seen_path, seen)
+    print(f"Imported {imported_messages} LinkedIn messages.")
+    print(f"Indexed {imported_connections} LinkedIn connections.")
+
+
+def write_connections(source, people_dir):
+    files = find_csvs(source, "connection")
+    if not files:
+        return 0
+    rows = []
+    for file in files:
+        rows.extend(read_csv(file))
+    lines = ["# LinkedIn Connections", "", "Imported from LinkedIn data archive.", ""]
+    count = 0
+    for row in rows:
+        name = first_value(row, ["First Name", "FirstName", "First name"])
+        last = first_value(row, ["Last Name", "LastName", "Last name"])
+        full_name = " ".join(part for part in [name, last] if part).strip() or first_value(row, ["Name", "Full Name"])
+        company = first_value(row, ["Company", "Company Name"])
+        position = first_value(row, ["Position", "Title"])
+        connected_on = first_value(row, ["Connected On", "ConnectedOn"])
+        if not full_name:
+            continue
+        detail = ", ".join(part for part in [position, company, connected_on] if part)
+        lines.append(f"- {full_name}{f' - {detail}' if detail else ''}")
+        count += 1
+    if count:
+        (people_dir / "LinkedIn Connections.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return count
+
+
+def message_record(row, file):
+    body = first_value(row, ["Content", "Message", "Body", "Text"])
+    if not body:
+        return None
+    timestamp = parse_timestamp(first_value(row, ["Date", "Created At", "Timestamp", "Sent Date"]))
+    if not timestamp:
+        return None
+    sender = first_value(row, ["From", "Sender", "Sender Name", "From Name"]) or "Unknown"
+    recipients = first_value(row, ["To", "Recipient", "Recipients", "To Name"]) or "Unknown"
+    explicit_conversation = first_value(row, ["Conversation Title", "Subject", "Conversation"])
+    conversation = explicit_conversation or other_party(sender, recipients)
+    return {
+        "id": f"linkedin-{file.name}-{timestamp}-{sender}-{hash(body)}",
+        "source": "LinkedIn data archive",
+        "sourceSystem": "LinkedIn",
+        "timestamp": timestamp,
+        "chatName": conversation,
+        "isGroup": "," in recipients,
+        "fromMe": False,
+        "author": sender,
+        "to": recipients,
+        "body": body,
+    }
+
+
+def other_party(sender, recipients):
+    if sender and sender.lower() not in {"me", "you"}:
+        return sender
+    if recipients and recipients.lower() not in {"me", "you"}:
+        return recipients.split(",")[0].strip()
+    return sender or recipients or "Unknown"
+
+
+def parse_timestamp(value):
+    if not value:
+        return ""
+    value = value.strip()
+    formats = [
+        "%Y-%m-%d %H:%M:%S",
+        "%Y-%m-%dT%H:%M:%S.%fZ",
+        "%Y-%m-%dT%H:%M:%SZ",
+        "%m/%d/%Y, %I:%M %p",
+        "%m/%d/%Y",
+        "%Y-%m-%d",
+    ]
+    for fmt in formats:
+        try:
+            return datetime.strptime(value, fmt).replace(tzinfo=timezone.utc).isoformat()
+        except ValueError:
+            pass
+    try:
+        return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc).isoformat()
+    except ValueError:
+        return ""
+
+
+def append_jsonl(raw_dir, record):
+    with (raw_dir / f"{record['timestamp'][:10]}.jsonl").open("a", encoding="utf-8") as f:
+        f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+def find_csvs(source, needle):
+    return sorted(path for path in source.rglob("*.csv") if needle.lower() in path.name.lower())
+
+
+def read_csv(path):
+    with path.open("r", encoding="utf-8-sig", newline="") as f:
+        return list(csv.DictReader(f))
+
+
+def first_value(row, keys):
+    for key in keys:
+        value = row.get(key)
+        if value:
+            return value.strip()
+    lower = {key.lower(): value for key, value in row.items()}
+    for key in keys:
+        value = lower.get(key.lower())
+        if value:
+            return value.strip()
+    return ""
+
+
+def load_seen(path):
+    if not path.exists():
+        return set()
+    try:
+        return set(json.loads(path.read_text(encoding="utf-8")))
+    except Exception:
+        return set()
+
+
+def save_seen(path, seen):
+    path.write_text(json.dumps(sorted(seen), indent=2), encoding="utf-8")
+
+
+def unpack(input_path):
+    input_path = input_path.resolve()
+    if input_path.is_dir():
+        return NullContext(input_path)
+    temp = tempfile.TemporaryDirectory()
+    with zipfile.ZipFile(input_path) as archive:
+        archive.extractall(temp.name)
+    return TempContext(Path(temp.name), temp)
+
+
+class NullContext:
+    def __init__(self, path):
+        self.path = path
+
+    def __enter__(self):
+        return self.path
+
+    def __exit__(self, *_):
+        return False
+
+
+class TempContext:
+    def __init__(self, path, temp):
+        self.path = path
+        self.temp = temp
+
+    def __enter__(self):
+        return self.path
+
+    def __exit__(self, *_):
+        self.temp.cleanup()
+        return False
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vault", type=Path, required=True)
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--days", type=int, default=3650)
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main()