@amoree-code/ocean 0.0.0-stage → 1.2.0

package/dist/application/pull/pull.js

@@ -0,0 +1,202 @@
+import { execFile } from "node:child_process";
+import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { memorySettings } from "@ocean/brain/memory/memory-audit.js";
+import { harvestMemory, importClaudeExport, } from "@ocean/brain/memory/memory-harvest.js";
+import { approveCandidate } from "@ocean/brain/memory/memory-review.js";
+import { withFileLock } from "@ocean/core/shared/fs-utils.js";
+import { oceanPath, SYSTEM_DIR } from "@ocean/core/shared/paths.js";
+import { buildHistoryIndex } from "./history-index.js";
+import { loadProjectIndex, placeMemory } from "./project-match.js";
+import { captureServerHistory } from "./server-capture.js";
+// `ocean pull` (T-305): one command brings everything every AI client on this machine holds into
+// Ocean — each client's own memory and the instruction files the owner wrote (as candidates for
+// review, never straight into memory), a claude.ai export found in ~/Downloads (its projects, as
+// candidates too), and an index of every conversation (pointers only, transcripts stay put).
+// Everything pulled is sorted by the owner's projects and carries where it came from (client,
+// project, file, date). Nothing is pulled twice (content keys, also across spacing and case) and
+// nothing is lost: every item read is listed in the run's record with what happened to it.
+// A dry run until --yes. `--daily` returns at once when the last pull is under a day old, so the
+// session-start hook can run it every time.
+const DAY_MS = 24 * 60 * 60 * 1000;
+const stampFile = () => oceanPath(SYSTEM_DIR, "state", "pull", "last.json");
+const importsDir = () => oceanPath(SYSTEM_DIR, "state", "imports");
+export const runsDir = () => oceanPath(SYSTEM_DIR, "state", "pull", "runs");
+// With `pullAutoApprove: "on"` in bridge/config/memory.json, what this run proposed goes through
+// the same validated write path as an owner's approval, but only as a new record: no target, an
+// existing record to add to, a similar record or a refusal leaves the candidate for review.
+async function autoApprove(items) {
+    const approvals = new Map();
+    for (const item of items) {
+        if (item.status !== "proposed" || !item.candidate)
+            continue;
+        try {
+            const written = await approveCandidate({
+                id: path.basename(item.candidate, ".md"),
+                createOnly: true,
+                source: "pull-auto-approve",
+            });
+            approvals.set(item.candidate, { approved: written.path });
+        }
+        catch (error) {
+            approvals.set(item.candidate, {
+                review: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    return approvals;
+}
+function tally(items) {
+    const counts = {};
+    for (const item of items) {
+        const bySource = counts[item.source] ?? {};
+        bySource[item.status] = (bySource[item.status] ?? 0) + 1;
+        counts[item.source] = bySource;
+    }
+    return counts;
+}
+const exists = (target) => stat(target).then(() => true, () => false);
+// claude.ai exports in ~/Downloads: an unpacked folder holding projects.json, or the zip it came
+// in (`data-….zip`), which --yes unpacks once into the workspace's state folder.
+async function findExports(home) {
+    const downloads = path.join(home, "Downloads");
+    const entries = await readdir(downloads, { withFileTypes: true }).catch(() => []);
+    const found = [];
+    for (const entry of entries) {
+        const target = path.join(downloads, entry.name);
+        if (entry.isDirectory() &&
+            (await exists(path.join(target, "projects.json"))))
+            found.push({ path: target, kind: "folder" });
+        else if (entry.isFile() && /^data-.*\.zip$/i.test(entry.name))
+            found.push({ path: target, kind: "zip" });
+    }
+    return found;
+}
+async function unpack(zip) {
+    const target = path.join(importsDir(), path.basename(zip, ".zip"));
+    if (!(await exists(target))) {
+        await mkdir(target, { recursive: true, mode: 0o700 });
+        await promisify(execFile)("unzip", ["-o", "-q", zip, "-d", target]);
+    }
+    return target;
+}
+async function importExport(found, apply, classify) {
+    if (found.kind === "zip" && !apply)
+        return { found, items: [] };
+    const folder = found.kind === "zip" ? await unpack(found.path) : found.path;
+    const { items, problems } = await importClaudeExport(folder, {
+        apply,
+        classify,
+    });
+    const names = await readdir(folder).catch(() => []);
+    return {
+        items,
+        found: {
+            ...found,
+            items: items.length,
+            new: items.filter((item) => item.status === "new" || item.status === "proposed").length,
+            ...(problems.length ? { problems } : {}),
+            unread: names.filter((name) => name !== "projects.json" && !name.startsWith(".")),
+        },
+    };
+}
+// Clients starting at the same moment each start a pull: an applying run holds a lock, and the
+// daily check is made inside it, so a second run waits and then finds the pull done instead of
+// proposing the same items again.
+export async function pull(options = {}) {
+    if (!options.apply)
+        return pullOnce(options);
+    await mkdir(path.dirname(stampFile()), { recursive: true, mode: 0o700 });
+    return withFileLock(stampFile(), () => pullOnce(options));
+}
+async function pullOnce(options) {
+    const apply = Boolean(options.apply);
+    const home = options.home ?? os.homedir();
+    if (options.daily) {
+        const last = JSON.parse(await readFile(stampFile(), "utf8").catch(() => "{}"));
+        const age = Date.now() - Date.parse(last.at ?? "");
+        if (age < DAY_MS)
+            return { applied: false, skipped: `last pull ${last.at}` };
+    }
+    const projects = await loadProjectIndex(home);
+    const classify = (origin) => placeMemory(projects, origin);
+    const items = await harvestMemory({ apply, home, classify });
+    const exports = [];
+    for (const found of await findExports(home)) {
+        const imported = await importExport(found, apply, classify);
+        exports.push(imported.found);
+        items.push(...imported.items);
+    }
+    const server = await captureServerHistory({ home, apply, classify });
+    items.push(...server.items);
+    const conversations = await buildHistoryIndex({ home, write: apply });
+    const history = {};
+    const byProject = {};
+    const slot = (id) => {
+        byProject[id] ??= { memory: {}, history: {} };
+        return byProject[id];
+    };
+    for (const item of items) {
+        const counts = slot(item.project ?? "unassigned").memory;
+        counts[item.status] = (counts[item.status] ?? 0) + 1;
+    }
+    const loose = new Map();
+    for (const entry of conversations) {
+        history[entry.client] = (history[entry.client] ?? 0) + 1;
+        const counts = slot(entry.project).history;
+        counts[entry.client] = (counts[entry.client] ?? 0) + 1;
+        if (entry.project === "unassigned" && entry.cwd)
+            loose.set(entry.cwd, (loose.get(entry.cwd) ?? 0) + 1);
+    }
+    const result = {
+        applied: apply,
+        memory: tally(items),
+        exports,
+        server: server.snapshots,
+        history,
+        projects: byProject,
+        unassignedFolders: [...loose]
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 15)
+            .map(([folder, count]) => ({ folder, conversations: count })),
+    };
+    const approvals = apply && (await memorySettings()).pullAutoApprove
+        ? await autoApprove(items)
+        : undefined;
+    if (approvals) {
+        const approved = [...approvals.values()].filter((item) => item.approved);
+        result.autoApproved = {
+            approved: approved.length,
+            review: approvals.size - approved.length,
+        };
+    }
+    if (apply) {
+        // The run's record: every item read and what became of it.
+        const at = new Date().toISOString();
+        const record = path.join(runsDir(), `${at.replace(/[:.]/g, "-")}.json`);
+        await mkdir(runsDir(), { recursive: true, mode: 0o700 });
+        await writeFile(record, `${JSON.stringify({
+            at,
+            ...result,
+            items: items.map((item) => ({
+                ...item,
+                ...approvals?.get(item.candidate ?? ""),
+            })),
+        }, null, 2)}\n`, { mode: 0o600 });
+        result.record = path.relative(oceanPath(), record);
+        await mkdir(path.dirname(stampFile()), { recursive: true, mode: 0o700 });
+        await writeFile(stampFile(), `${JSON.stringify({ at: new Date().toISOString() })}\n`, { mode: 0o600 });
+    }
+    return result;
+}
+// The newest run's record, for `ocean pull report`.
+export async function lastPullRecord() {
+    const runs = (await readdir(runsDir()).catch(() => [])).filter((name) => name.endsWith(".json"));
+    const last = runs.sort().at(-1);
+    return last
+        ? JSON.parse(await readFile(path.join(runsDir(), last), "utf8"))
+        : null;
+}
+//# sourceMappingURL=pull.js.map

package/dist/application/pull/server-capture.js

@@ -0,0 +1,307 @@
+import { mkdir, open, readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import zlib from "node:zlib";
+import { proposeImported, } from "@ocean/brain/memory/memory-harvest.js";
+import { atomicWrite } from "@ocean/core/shared/fs-utils.js";
+import { claudeProjectsFile } from "@ocean/core/shared/project-resolution.js";
+import { serverHistoryDir } from "./history-index.js";
+// claude.ai and ChatGPT keep their conversations, projects and memory on their servers (T-305).
+// Claude Desktop and the browsers keep what they fetched of those pages in a Chromium HTTP cache;
+// `ocean pull` reads it, read-only, and keeps ids, titles, dates and project names in the server
+// snapshots the history index reads, and memory or instruction texts as review candidates. Only
+// the account pages below are opened; a URL naming a token, OAuth or session is never read.
+const WANTED = /https:\/\/(?:claude\.ai\/api\/organizations\/|chatgpt\.com\/backend-api\/(?:conversations|gizmos\/snorlax\/sidebar|user_system_messages|memories))[^\s]*/;
+const SECRET_URL = /oauth|token|session|auth/i;
+// The record that ends each stream of a Chromium "simple cache" entry file.
+const EOF_MAGIC = Buffer.from("d8410d97456ffaf4", "hex");
+const HEADER_BYTES = 24;
+const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
+// Claude Desktop's cache, then every profile of the Chromium browsers.
+async function cacheDirs(home) {
+    const dirs = [
+        path.join(home, "Library", "Application Support", "Claude", "Cache"),
+    ];
+    for (const browser of [
+        ["Google", "Chrome"],
+        ["Arc", "User Data"],
+        ["BraveSoftware", "Brave-Browser"],
+        ["Microsoft Edge"],
+    ]) {
+        const root = path.join(home, "Library", "Caches", ...browser);
+        for (const profile of await readdir(root).catch(() => []))
+            dirs.push(path.join(root, profile, "Cache"));
+    }
+    return dirs.map((dir) => path.join(dir, "Cache_Data"));
+}
+function decode(body, encoding) {
+    if (encoding === "br")
+        return zlib.brotliDecompressSync(body).toString();
+    if (encoding === "gzip")
+        return zlib.gunzipSync(body).toString();
+    if (encoding === "deflate")
+        return zlib.inflateSync(body).toString();
+    if (encoding === "zstd") {
+        const zstd = zlib
+            .zstdDecompressSync;
+        if (!zstd)
+            throw new Error("zstd needs Node 22.15");
+        return zstd(body).toString();
+    }
+    return body.toString();
+}
+// One entry file: the key (its URL) is read first; the body only when the URL is wanted.
+async function cachedJson(file) {
+    const handle = await open(file, "r").catch(() => undefined);
+    if (!handle)
+        return undefined;
+    try {
+        const head = Buffer.alloc(4096);
+        const { bytesRead } = await handle.read(head, 0, head.length, 0);
+        if (bytesRead < HEADER_BYTES)
+            return undefined;
+        const keyLength = head.readUInt32LE(12);
+        const key = head
+            .subarray(HEADER_BYTES, Math.min(HEADER_BYTES + keyLength, bytesRead))
+            .toString("latin1");
+        const url = WANTED.exec(key)?.[0];
+        if (!url || SECRET_URL.test(url))
+            return undefined;
+        const data = await readFile(file);
+        const start = HEADER_BYTES + keyLength;
+        const end = data.indexOf(EOF_MAGIC, start);
+        if (end < 0)
+            return undefined;
+        const headers = data.subarray(end).toString("latin1").toLowerCase();
+        const encoding = /content-encoding:\s*([a-z0-9-]+)/.exec(headers)?.[1];
+        const date = /[\0\n]date:\s*([^\0\r\n]+)/.exec(headers)?.[1];
+        const json = JSON.parse(decode(data.subarray(start, end), encoding));
+        return { url, at: Date.parse(date ?? "") || 0, json };
+    }
+    catch {
+        // a partial entry, another format or a body that is not JSON
+        return undefined;
+    }
+    finally {
+        await handle.close();
+    }
+}
+// A cache holds tens of thousands of entries; their keys are read a batch at a time.
+const BATCH = 128;
+async function readCaches(home) {
+    const files = [];
+    for (const dir of await cacheDirs(home))
+        for (const name of await readdir(dir).catch(() => []))
+            if (name.endsWith("_0"))
+                files.push(path.join(dir, name));
+    const found = [];
+    for (let at = 0; at < files.length; at += BATCH)
+        for (const entry of await Promise.all(files.slice(at, at + BATCH).map(cachedJson)))
+            if (entry)
+                found.push(entry);
+    return found;
+}
+const object = (value) => value && typeof value === "object" ? value : {};
+const list = (value) => {
+    const items = Array.isArray(value)
+        ? value
+        : (object(value).data ?? object(value).items);
+    return Array.isArray(items) ? items.map(object) : [];
+};
+const text = (value) => (typeof value === "string" ? value : "");
+const iso = (value) => {
+    const time = typeof value === "number"
+        ? value < 1e12
+            ? value * 1000
+            : value
+        : Date.parse(text(value));
+    return Number.isFinite(time) ? new Date(time).toISOString() : undefined;
+};
+// Project uuid → name, from the claude.ai project responses Claude Desktop and the browsers
+// cached.
+function projectNames(entries) {
+    const names = new Map();
+    for (const { url, json } of entries) {
+        if (/\/projects(_v2)?(\?|$)/.test(url))
+            for (const project of list(json))
+                if (text(project.uuid))
+                    names.set(text(project.uuid), text(project.name));
+        const one = new RegExp(`/projects/(${UUID})(\\?|$)`).exec(url)?.[1];
+        if (one && text(object(json).name))
+            names.set(one, text(object(json).name));
+    }
+    return names;
+}
+// The names of the owner's Claude projects that this machine has seen (T-307): what lets Ocean
+// tell a real Claude project from a topic a client calls a project. Read-only, from the caches.
+export async function claudeProjectNames(home) {
+    const entries = (await readCaches(home)).filter((entry) => entry.url.includes("claude.ai/"));
+    return [...new Set(projectNames(entries).values())].filter(Boolean);
+}
+// Writes those names (names only, owner-only file) where a tool call can read them at once.
+export async function refreshClaudeProjects(home) {
+    const names = await claudeProjectNames(home);
+    const file = claudeProjectsFile();
+    await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
+    await atomicWrite(file, `${JSON.stringify({ at: new Date().toISOString(), names }, null, 2)}\n`, 0o600);
+    return names.length;
+}
+function claudeAi(entries) {
+    const names = projectNames(entries);
+    const conversations = new Map();
+    const texts = new Map();
+    for (const { url, json, at } of entries) {
+        const inProject = new RegExp(`/projects/(${UUID})/conversations_v2`).exec(url)?.[1];
+        if (inProject || /\/chat_conversations(_v2)?(\?|$)/.test(url))
+            for (const row of list(json)) {
+                const id = text(row.uuid);
+                if (!id)
+                    continue;
+                const project = text(row.project_uuid) || text(object(row.project).uuid) || inProject;
+                conversations.set(id, {
+                    id,
+                    title: text(row.name),
+                    url: `https://claude.ai/chat/${id}`,
+                    updated: iso(row.updated_at),
+                    projectName: (project && names.get(project)) ||
+                        text(object(row.project).name) ||
+                        undefined,
+                });
+            }
+        const org = /organizations\/([0-9a-f-]{36})\/memory(\?|$)/.exec(url)?.[1];
+        const memory = text(object(json).memory).trim();
+        if (org && memory) {
+            const label = `claude.ai memory (org ${org.slice(0, 8)})`;
+            const updated = Date.parse(text(object(json).updated_at)) || at;
+            if (updated >= (texts.get(label)?.at ?? -1))
+                texts.set(label, {
+                    text: `# claude.ai memory\n\n${memory}`,
+                    at: updated,
+                });
+        }
+    }
+    return { conversations, texts };
+}
+function chatGpt(entries) {
+    const conversations = new Map();
+    const texts = new Map();
+    const keep = (label, body, at) => {
+        if (body.trim() && at >= (texts.get(label)?.at ?? -1))
+            texts.set(label, { text: body, at });
+    };
+    for (const { url, json, at } of entries) {
+        const add = (row, projectName) => {
+            const id = text(row.id);
+            if (id)
+                conversations.set(id, {
+                    id,
+                    title: text(row.title),
+                    url: `https://chatgpt.com/c/${id}`,
+                    updated: iso(row.update_time),
+                    projectName: projectName || undefined,
+                });
+        };
+        if (/\/backend-api\/conversations(\?|$)/.test(url))
+            for (const row of list(json))
+                add(row);
+        if (url.includes("/gizmos/snorlax/sidebar"))
+            for (const item of list(json)) {
+                const gizmo = object(object(item.gizmo).gizmo);
+                const name = text(object(gizmo.display).name);
+                for (const row of list(object(item).conversations))
+                    add(row, name);
+            }
+        if (url.includes("/user_system_messages")) {
+            const about = text(object(json).about_user_message);
+            const respond = text(object(json).about_model_message);
+            keep("ChatGPT custom instructions", `# ChatGPT custom instructions\n\n${about ? `## About me\n\n${about}\n\n` : ""}${respond ? `## How to respond\n\n${respond}\n` : ""}`, about || respond ? at : -1);
+        }
+        if (/\/backend-api\/memories(\?|$)/.test(url)) {
+            const lines = list(object(json).memories)
+                .map((memory) => text(memory.content).trim())
+                .filter(Boolean);
+            if (lines.length)
+                keep("ChatGPT memory", `# ChatGPT memory\n\n${lines.map((line) => `- ${line}`).join("\n")}\n`, at);
+        }
+    }
+    return { conversations, texts };
+}
+const newer = (a, b) => (Date.parse(a ?? "") || 0) >= (Date.parse(b ?? "") || 0) ? a : b;
+// A snapshot only grows: what the cache no longer holds stays, and a project set earlier (by hand
+// or by a capture) is kept; a newer title or date replaces the old one.
+async function mergeSnapshot(client, rows, write) {
+    const file = path.join(serverHistoryDir(), `${client}.jsonl`);
+    const known = new Map();
+    for (const line of (await readFile(file, "utf8").catch(() => "")).split("\n"))
+        try {
+            const row = JSON.parse(line);
+            if (row?.id)
+                known.set(row.id, row);
+        }
+        catch {
+            // an empty or foreign line
+        }
+    let added = 0;
+    for (const row of rows) {
+        const before = known.get(row.id);
+        if (!before)
+            added += 1;
+        known.set(row.id, before
+            ? {
+                ...before,
+                title: row.title || before.title,
+                updated: newer(row.updated, before.updated),
+            }
+            : row);
+    }
+    if (write && rows.length) {
+        await mkdir(serverHistoryDir(), { recursive: true, mode: 0o700 });
+        await atomicWrite(file, [...known.values()].map((row) => JSON.stringify(row)).join("\n") +
+            (known.size ? "\n" : ""), 0o600);
+    }
+    return { conversations: known.size, new: added };
+}
+export async function captureServerHistory(options) {
+    const entries = await readCaches(options.home);
+    const snapshots = {};
+    const imports = [];
+    for (const [client, tag, source, found] of [
+        [
+            "claude-ai",
+            "claude.ai",
+            "claude.ai",
+            claudeAi(entries.filter((entry) => entry.url.includes("claude.ai/"))),
+        ],
+        [
+            "chatgpt",
+            "chatgpt",
+            "chatgpt",
+            chatGpt(entries.filter((entry) => entry.url.includes("chatgpt.com/"))),
+        ],
+    ]) {
+        const rows = [...found.conversations.values()].map(({ projectName, ...row }) => {
+            // A conversation in a provider project goes where that project's name points.
+            const project = projectName
+                ? options.classify({
+                    client: source,
+                    file: row.url,
+                    text: "",
+                    name: projectName,
+                }).project
+                : undefined;
+            return {
+                ...row,
+                project: project ?? "unassigned",
+                tags: projectName ? [tag, projectName] : [tag],
+            };
+        });
+        snapshots[client] = await mergeSnapshot(client, rows, options.apply);
+        for (const [label, { text: body }] of found.texts)
+            imports.push({ label, text: body, source });
+    }
+    const items = await proposeImported(imports, {
+        apply: options.apply,
+        classify: options.classify,
+    });
+    return { snapshots, items };
+}
+//# sourceMappingURL=server-capture.js.map