brane 0.0.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +362 -0
- package/CHANGELOG.md +33 -0
- package/CONTRIBUTING.md +91 -0
- package/DESIGN.md +402 -0
- package/Dockerfile +61 -0
- package/LICENSE +656 -17
- package/README.md +174 -2
- package/SECURITY.md +205 -0
- package/dist/absorb/brane-id.js +11 -0
- package/dist/absorb/chatgpt.js +161 -0
- package/dist/absorb/claude-code.js +504 -0
- package/dist/absorb/index.js +543 -0
- package/dist/absorb/organelle.js +345 -0
- package/dist/absorb/run.js +520 -0
- package/dist/absorb/sources.js +168 -0
- package/dist/absorb/types.js +3 -0
- package/dist/agent-env.js +45 -0
- package/dist/agents.js +1168 -0
- package/dist/claude-live-session.js +124 -0
- package/dist/cli.js +50 -0
- package/dist/codex-tool-log.js +144 -0
- package/dist/commands/absorb.js +291 -0
- package/dist/commands/attach.js +31 -0
- package/dist/commands/certify.js +62 -0
- package/dist/commands/config.js +116 -0
- package/dist/commands/cost.js +126 -0
- package/dist/commands/daemon.js +350 -0
- package/dist/commands/doctor.js +262 -0
- package/dist/commands/fold.js +95 -0
- package/dist/commands/list.js +89 -0
- package/dist/commands/mcp.js +84 -0
- package/dist/commands/meld.js +332 -0
- package/dist/commands/memory.js +215 -0
- package/dist/commands/revive.js +74 -0
- package/dist/commands/run.js +85 -0
- package/dist/commands/spawn.js +1094 -0
- package/dist/commands/token.js +153 -0
- package/dist/copilot-tool-log.js +145 -0
- package/dist/cost/pricing.js +116 -0
- package/dist/cost/reprice.js +136 -0
- package/dist/crush-tool-log.js +293 -0
- package/dist/daemon/agent-scope.js +116 -0
- package/dist/daemon/awaiting.js +54 -0
- package/dist/daemon/cert-manager.js +349 -0
- package/dist/daemon/client.js +262 -0
- package/dist/daemon/gated-routes.js +112 -0
- package/dist/daemon/keep-awake.js +103 -0
- package/dist/daemon/local-socket.js +145 -0
- package/dist/daemon/presence-gate.js +206 -0
- package/dist/daemon/server.js +6349 -0
- package/dist/daemon/signal-source.js +35 -0
- package/dist/daemon/tailscale-rebind.js +58 -0
- package/dist/daemon/token-ring.js +263 -0
- package/dist/daemon/token-routes.js +190 -0
- package/dist/db/events.js +152 -0
- package/dist/db/mentions.js +179 -0
- package/dist/db/schema.js +633 -0
- package/dist/db/tokens.js +583 -0
- package/dist/db/tool-uses.js +647 -0
- package/dist/diagram/index.js +85 -0
- package/dist/diagram/label.js +455 -0
- package/dist/diagram/layout.js +476 -0
- package/dist/diagram/links.js +50 -0
- package/dist/diagram/metrics.js +152 -0
- package/dist/diagram/parse.js +487 -0
- package/dist/diagram/svg.js +389 -0
- package/dist/diagram/types.js +9 -0
- package/dist/export.js +629 -0
- package/dist/files/archive-types.js +2 -0
- package/dist/files/archive.js +182 -0
- package/dist/files/browse.js +327 -0
- package/dist/files/bytes.js +162 -0
- package/dist/files/codecs.js +33 -0
- package/dist/files/find.js +212 -0
- package/dist/files/tar.js +223 -0
- package/dist/files/zip.js +209 -0
- package/dist/folds.js +72 -0
- package/dist/grammars/ebnf.js +92 -0
- package/dist/highlight.js +250 -0
- package/dist/ids.js +24 -0
- package/dist/markdown-diagnostics.js +233 -0
- package/dist/math.js +168 -0
- package/dist/mcp/server.js +886 -0
- package/dist/memory/embedder.js +145 -0
- package/dist/memory/embedding-worker.js +71 -0
- package/dist/memory/embeddings.js +91 -0
- package/dist/memory/index.js +232 -0
- package/dist/memory/storage.js +587 -0
- package/dist/memory/wiki.js +688 -0
- package/dist/memory/wordpiece.js +247 -0
- package/dist/mentions-scan.js +40 -0
- package/dist/presence.js +259 -0
- package/dist/session-affinity.js +185 -0
- package/dist/shared/api-request.js +124 -0
- package/dist/shared/block-glyphs.js +114 -0
- package/dist/shared/diagram-css.js +17 -0
- package/dist/shared/export-selection.js +101 -0
- package/dist/shared/find-in-text.js +110 -0
- package/dist/shared/glow-hues.js +225 -0
- package/dist/shared/hex-dump.js +62 -0
- package/dist/shared/hex-pages.js +52 -0
- package/dist/shared/hit-drum.js +79 -0
- package/dist/shared/known-nids.js +58 -0
- package/dist/shared/list-scroll-keys.js +56 -0
- package/dist/shared/markdown-lists.js +72 -0
- package/dist/shared/markdown-math.js +485 -0
- package/dist/shared/markdown-mentions.js +79 -0
- package/dist/shared/markdown-nids.js +41 -0
- package/dist/shared/markdown-protocols.js +53 -0
- package/dist/shared/markdown-refs.js +123 -0
- package/dist/shared/membrane-invasion.js +342 -0
- package/dist/shared/membrane-motion.js +83 -0
- package/dist/shared/membrane-palette.js +253 -0
- package/dist/shared/membrane-terrain.js +212 -0
- package/dist/shared/message-selection.js +123 -0
- package/dist/shared/mo-glyph.js +57 -0
- package/dist/shared/narrative-format.js +237 -0
- package/dist/shared/narrative-menu.js +82 -0
- package/dist/shared/narrative-sort.js +47 -0
- package/dist/shared/narrative-status.js +52 -0
- package/dist/shared/no-pan.js +85 -0
- package/dist/shared/pane-layout.js +50 -0
- package/dist/shared/payload-inline.js +83 -0
- package/dist/shared/speaker-color.js +61 -0
- package/dist/shared/token-rotation.js +14 -0
- package/dist/shared/view-url.js +197 -0
- package/dist/shared/wiki-links.js +70 -0
- package/dist/shared/wiki-window.js +4 -0
- package/dist/tailscale.js +55 -0
- package/dist/test/absorb-claude.test.js +892 -0
- package/dist/test/agent-scope.test.js +193 -0
- package/dist/test/agents.test.js +55 -0
- package/dist/test/api-request.test.js +122 -0
- package/dist/test/awaiting-glow.test.js +54 -0
- package/dist/test/awaiting.test.js +122 -0
- package/dist/test/bind-token.test.js +88 -0
- package/dist/test/block-glyphs.test.js +68 -0
- package/dist/test/cert-manager.test.js +262 -0
- package/dist/test/claude-live-session.test.js +110 -0
- package/dist/test/codex-tool-log.test.js +568 -0
- package/dist/test/config.test.js +71 -0
- package/dist/test/copilot-tool-log.test.js +93 -0
- package/dist/test/cors-preflight.test.js +49 -0
- package/dist/test/cost-linger.test.js +31 -0
- package/dist/test/cost-pricing.test.js +72 -0
- package/dist/test/cost-reprice.test.js +139 -0
- package/dist/test/cost-sessions.test.js +301 -0
- package/dist/test/crush-tool-log.test.js +289 -0
- package/dist/test/daemon-client.test.js +109 -0
- package/dist/test/daemon-socket.test.js +293 -0
- package/dist/test/diagram-arrows.test.js +205 -0
- package/dist/test/diagram-export.test.js +210 -0
- package/dist/test/diagram-labels.test.js +313 -0
- package/dist/test/diagram-parse.test.js +191 -0
- package/dist/test/diagram-render.test.js +175 -0
- package/dist/test/diagram-review.test.js +288 -0
- package/dist/test/diagram-self-loop-labels.test.js +303 -0
- package/dist/test/embedder.test.js +84 -0
- package/dist/test/events.test.js +123 -0
- package/dist/test/export-selection.test.js +102 -0
- package/dist/test/export.test.js +505 -0
- package/dist/test/files-browse.test.js +393 -0
- package/dist/test/files-find.test.js +272 -0
- package/dist/test/find-in-text.test.js +86 -0
- package/dist/test/first-paint-viewport.test.js +59 -0
- package/dist/test/fixtures/embedding-golden.js +23 -0
- package/dist/test/fixtures/wordpiece-golden.js +167 -0
- package/dist/test/glow-hues.test.js +202 -0
- package/dist/test/harness-cost.test.js +423 -0
- package/dist/test/helpers.js +21 -0
- package/dist/test/hex-pages.test.js +68 -0
- package/dist/test/highlight.test.js +69 -0
- package/dist/test/hit-drum.test.js +71 -0
- package/dist/test/hook-config.test.js +262 -0
- package/dist/test/json-gzip.test.js +70 -0
- package/dist/test/key-routing.test.js +58 -0
- package/dist/test/known-nids.test.js +106 -0
- package/dist/test/link-preview.test.js +67 -0
- package/dist/test/list-scroll-keys.test.js +66 -0
- package/dist/test/listen-own-words.test.js +147 -0
- package/dist/test/listen-timeout.test.js +33 -0
- package/dist/test/markdown-diagnostics.test.js +121 -0
- package/dist/test/markdown-lists.test.js +182 -0
- package/dist/test/markdown-math.test.js +256 -0
- package/dist/test/markdown-nids.test.js +146 -0
- package/dist/test/markdown-protocols.test.js +139 -0
- package/dist/test/mcp-bridge-cancel.test.js +98 -0
- package/dist/test/mcp-bridge-reconnect.test.js +121 -0
- package/dist/test/mcp-narrative.test.js +193 -0
- package/dist/test/mcp.test.js +548 -0
- package/dist/test/meld.test.js +85 -0
- package/dist/test/membrane-invasion.test.js +482 -0
- package/dist/test/membrane-motion.test.js +101 -0
- package/dist/test/membrane-palette.test.js +166 -0
- package/dist/test/membrane-terrain.test.js +170 -0
- package/dist/test/memory-log.test.js +202 -0
- package/dist/test/mentions-e2e.test.js +261 -0
- package/dist/test/mentions-ledger.test.js +265 -0
- package/dist/test/mentions-listen.test.js +265 -0
- package/dist/test/mentions.test.js +238 -0
- package/dist/test/message-selection.test.js +114 -0
- package/dist/test/mo-glyph.test.js +22 -0
- package/dist/test/model-resolution.test.js +166 -0
- package/dist/test/narrate-validate.test.js +77 -0
- package/dist/test/narrate-wait.test.js +160 -0
- package/dist/test/narrative-format.test.js +219 -0
- package/dist/test/narrative-menu.test.js +137 -0
- package/dist/test/narrative-sort.test.js +75 -0
- package/dist/test/no-pan.test.js +60 -0
- package/dist/test/pane-layout.test.js +153 -0
- package/dist/test/paths.test.js +37 -0
- package/dist/test/payload-inline.test.js +91 -0
- package/dist/test/precache-fonts.test.js +72 -0
- package/dist/test/presence-gate.test.js +440 -0
- package/dist/test/presence.test.js +160 -0
- package/dist/test/read-events.test.js +220 -0
- package/dist/test/remote-meld.test.js +115 -0
- package/dist/test/revive-cli.test.js +93 -0
- package/dist/test/search-floor.test.js +206 -0
- package/dist/test/search-fusion.test.js +168 -0
- package/dist/test/search.test.js +205 -0
- package/dist/test/security.test.js +48 -0
- package/dist/test/selection-wiring.test.js +61 -0
- package/dist/test/session-affinity.test.js +295 -0
- package/dist/test/signal-source.test.js +32 -0
- package/dist/test/socket-account.test.js +268 -0
- package/dist/test/spawn-toml.test.js +72 -0
- package/dist/test/speaker-color.test.js +90 -0
- package/dist/test/tailscale-rebind.test.js +79 -0
- package/dist/test/terminal-cursor.test.js +61 -0
- package/dist/test/terminal-resync.test.js +23 -0
- package/dist/test/thin-cli.test.js +382 -0
- package/dist/test/token-bootstrap.test.js +272 -0
- package/dist/test/token-devices.test.js +587 -0
- package/dist/test/token-lapse.test.js +94 -0
- package/dist/test/token-ring.test.js +188 -0
- package/dist/test/token-rotate-cli.test.js +58 -0
- package/dist/test/token-routes.test.js +95 -0
- package/dist/test/tool-uses.test.js +1006 -0
- package/dist/test/usage-ledger.test.js +126 -0
- package/dist/test/view-url.test.js +581 -0
- package/dist/test/wiki-links.test.js +57 -0
- package/dist/test/wiki-sources.test.js +85 -0
- package/dist/test/wiki-window.test.js +31 -0
- package/dist/test/wiki.test.js +399 -0
- package/dist/test/wordpiece.test.js +68 -0
- package/dist/tls.js +148 -0
- package/dist/tmux.js +202 -0
- package/dist/user-agent.js +46 -0
- package/docker-entrypoint.sh +141 -0
- package/docs/README.md +3 -0
- package/docs/absorb.md +231 -0
- package/docs/cost-ledger.md +125 -0
- package/docs/daemon-api.md +189 -0
- package/docs/diagrams.md +163 -0
- package/docs/doctor.md +36 -0
- package/docs/export-scope.md +69 -0
- package/docs/files-view.md +133 -0
- package/docs/link-previews.md +38 -0
- package/docs/math.md +32 -0
- package/docs/mcp.md +69 -0
- package/docs/memory-search.md +75 -0
- package/docs/mentions.md +101 -0
- package/docs/multi-brane.md +14 -0
- package/docs/passkeys.md +165 -0
- package/docs/presence.md +202 -0
- package/docs/releasing.md +53 -0
- package/docs/rendering-warnings.md +22 -0
- package/docs/request-layer.md +72 -0
- package/docs/selection-store.md +50 -0
- package/docs/session-affinity.md +61 -0
- package/docs/terminal-mirroring.md +211 -0
- package/docs/token-rotation.md +413 -0
- package/docs/view-urls.md +114 -0
- package/docs/web-app.md +23 -0
- package/docs/wiki.md +43 -0
- package/expo/dist/_expo/static/js/web/entry-e56efc1291dca7258ea00b8108b033a4.js +1097 -0
- package/expo/dist/apple-touch-icon.png +0 -0
- package/expo/dist/favicon.svg +3 -0
- package/expo/dist/index.html +177 -0
- package/expo/dist/katex/fonts/KaTeX_AMS-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Caligraphic-Bold.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Caligraphic-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Fraktur-Bold.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Fraktur-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Main-Bold.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Main-BoldItalic.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Main-Italic.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Main-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Math-BoldItalic.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Math-Italic.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_SansSerif-Bold.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_SansSerif-Italic.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_SansSerif-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Script-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Size1-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Size2-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Size3-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Size4-Regular.woff2 +0 -0
- package/expo/dist/katex/fonts/KaTeX_Typewriter-Regular.woff2 +0 -0
- package/expo/dist/katex/katex.min.css +1 -0
- package/expo/dist/manifest.json +16 -0
- package/expo/dist/metadata.json +1 -0
- package/expo/dist/og/membrane.v1.png +0 -0
- package/expo/dist/pdf.min.mjs +28 -0
- package/expo/dist/pdf.worker.min.mjs +28 -0
- package/expo/dist/sw.js +154 -0
- package/package.json +81 -23
- package/protocols/LIVE_loop.md +407 -0
- package/protocols/bebrief.md +41 -0
- package/protocols/digest.md +231 -0
- package/protocols/fly.md +266 -0
- package/protocols/idle.md +81 -0
- package/protocols/keeper.md +829 -0
- package/protocols/markdown.md +80 -0
- package/protocols/memory.md +419 -0
- package/protocols/mentions.md +93 -0
- package/protocols/payloads.md +53 -0
- package/protocols/prelude.md +91 -0
- package/protocols/ramify.md +105 -0
- package/protocols/review.md +356 -0
- package/protocols/rrr.md +232 -0
- package/protocols/scribe.md +429 -0
- package/protocols/summarize.md +203 -0
- package/protocols/twin.md +174 -0
- package/protocols/wiki.md +438 -0
- package/protocols/worker.md +38 -0
- package/wiki/shared/append-only-event-log.md +148 -0
- package/wiki/shared/index.md +48 -0
- package/wiki/shared/owning-your-narratives.md +161 -0
- package/wiki/shared/premature-ontologies.md +435 -0
- package/wiki/shared/sedimentary-superintelligence.md +128 -0
- package/wiki/shared/sequestering-the-narrative.md +143 -0
- package/wiki/shared/why-wikis-work.md +216 -0
- package/.npmignore +0 -2
- package/.travis.yml +0 -5
- package/index.js +0 -8
- package/index.js~ +0 -12
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The absorb pipeline as the daemon runs it.
|
|
3
|
+
*
|
|
4
|
+
* `brane absorb` used to do all of this in the CLI process: parse the export,
|
|
5
|
+
* write narratives and events, copy assets into `~/.brane/payloads/`. Every
|
|
6
|
+
* one of those paths — the payload store, `~/.organelle/index/search.db`, the
|
|
7
|
+
* niche registry — resolves against HOME, and the daemon is the process whose
|
|
8
|
+
* HOME the rest of brane already agrees on (see the cost backfill). So the
|
|
9
|
+
* orchestration lives here and `POST /api/absorb` calls it; the CLI is a
|
|
10
|
+
* client that prints what this emits.
|
|
11
|
+
*
|
|
12
|
+
* Progress is reported through `emit`, which the route streams as NDJSON and
|
|
13
|
+
* the CLI prints as it always did. Everything that can be rejected up front
|
|
14
|
+
* (unknown provider, missing directory, no identity) throws an
|
|
15
|
+
* `AbsorbInputError` *before* the first emit, so the route can still answer
|
|
16
|
+
* with a plain 400 rather than a 200 that goes wrong halfway.
|
|
17
|
+
*/
|
|
18
|
+
import { existsSync, statSync } from "node:fs";
|
|
19
|
+
import { basename, join, resolve } from "node:path";
|
|
20
|
+
import { getBraneId, toPortablePath, fromPortablePath } from "../db/schema.js";
|
|
21
|
+
import { registerFold, foldPathRefusal } from "../folds.js";
|
|
22
|
+
import { getLocalSource, classifySessions } from "./sources.js";
|
|
23
|
+
import { deriveBraneId } from "./brane-id.js";
|
|
24
|
+
import { resolveClaudeRoot, claudeProjectsRoot } from "./claude-code.js";
|
|
25
|
+
import { getProvider, isKnownProvider, listProviders, absorb, createAbsorbContext, absorbOne, extractIdentityFromDirectory, findConversationsInDirectory, describeExportDirectory, buildFileIndex, } from "./index.js";
|
|
26
|
+
import { absorbOrganelle, describeOrganelleNiche, findSearchDb, readNichesRegistry, } from "./organelle.js";
|
|
27
|
+
/** A request the daemon can reject before doing any work — maps to 400. */
|
|
28
|
+
export class AbsorbInputError extends Error {
|
|
29
|
+
status = 400;
|
|
30
|
+
}
|
|
31
|
+
const DEFAULT_PATHS = {
|
|
32
|
+
organelle: () => join(process.env.HOME, ".organelle"),
|
|
33
|
+
claude: () => claudeProjectsRoot(process.env.HOME),
|
|
34
|
+
};
|
|
35
|
+
/** Resolve a fold the way every fold path is stored: `~/…` stays portable,
|
|
36
|
+
* anything else becomes portable from its absolute form. */
|
|
37
|
+
function portableFold(raw) {
|
|
38
|
+
return raw.startsWith("~") ? raw : toPortablePath(resolve(raw));
|
|
39
|
+
}
|
|
40
|
+
/** Refuse a fold the daemon would refuse to register — *before* the stream
|
|
41
|
+
* opens, so the operator gets a 400/403 rather than a full pre-flight report
|
|
42
|
+
* followed by an error line, and so a dry run catches what the real run
|
|
43
|
+
* would. `registerFold` re-checks on write; this is the early copy. */
|
|
44
|
+
function assertFoldsRegistrable(foldPaths, opts) {
|
|
45
|
+
for (const fp of foldPaths) {
|
|
46
|
+
const refusal = foldPathRefusal(resolve(fromPortablePath(fp)), {
|
|
47
|
+
confineToHome: opts.confineFoldsToHome,
|
|
48
|
+
});
|
|
49
|
+
if (refusal)
|
|
50
|
+
throw refusal;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
export { deriveBraneId } from "./brane-id.js";
|
|
54
|
+
/** Validate the request and resolve the input directory. Throws
|
|
55
|
+
* `AbsorbInputError` for anything the operator has to fix. */
|
|
56
|
+
export function resolveAbsorbInput(req) {
|
|
57
|
+
const provider = req.provider;
|
|
58
|
+
if (!provider || !isKnownProvider(provider)) {
|
|
59
|
+
throw new AbsorbInputError(`Unknown provider: ${provider}. Available: ${listProviders().join(", ")}`);
|
|
60
|
+
}
|
|
61
|
+
const rawPath = req.path ?? DEFAULT_PATHS[provider]?.();
|
|
62
|
+
if (!rawPath) {
|
|
63
|
+
throw new AbsorbInputError(`Path is required for provider "${provider}".\nUsage: brane absorb ${provider} <path>`);
|
|
64
|
+
}
|
|
65
|
+
const inputPath = resolve(rawPath.startsWith("~/") ? join(process.env.HOME, rawPath.slice(2)) : rawPath);
|
|
66
|
+
const stat = statSync(inputPath, { throwIfNoEntry: false });
|
|
67
|
+
if (!stat) {
|
|
68
|
+
throw new AbsorbInputError(getLocalSource(provider) && !req.path
|
|
69
|
+
? `No ${provider} session store at ${inputPath} — nothing to absorb on this machine.`
|
|
70
|
+
: `Not found: ${inputPath}`);
|
|
71
|
+
}
|
|
72
|
+
if (!stat.isDirectory()) {
|
|
73
|
+
throw new AbsorbInputError(`Expected a directory, got a file: ${inputPath}\n` +
|
|
74
|
+
`If this is a ZIP, extract it first, then pass the directory.`);
|
|
75
|
+
}
|
|
76
|
+
return { provider, inputPath };
|
|
77
|
+
}
|
|
78
|
+
/** Run one absorb end to end. Rejects with `AbsorbInputError` before the first
|
|
79
|
+
* `emit` for input problems; anything thrown after that is a run failure. */
|
|
80
|
+
export async function runAbsorb(db, req, emit, opts) {
|
|
81
|
+
const { provider, inputPath } = resolveAbsorbInput(req);
|
|
82
|
+
const dryRun = req.dry_run === true;
|
|
83
|
+
const local = getLocalSource(provider);
|
|
84
|
+
if (local)
|
|
85
|
+
return runLocalAbsorb(db, local, inputPath, req, dryRun, emit, opts);
|
|
86
|
+
if (req.rebuild) {
|
|
87
|
+
// Refused rather than ignored: a flag that does nothing teaches the
|
|
88
|
+
// operator it did something. Export providers can grow it — the export
|
|
89
|
+
// directory is on disk like a session file — but not silently.
|
|
90
|
+
throw new AbsorbInputError(`--rebuild is not supported for the ${provider} provider yet.`);
|
|
91
|
+
}
|
|
92
|
+
return provider === "organelle"
|
|
93
|
+
? runOrganelleAbsorb(db, inputPath, req, dryRun, emit, opts)
|
|
94
|
+
: runStreamAbsorb(db, provider, inputPath, req, dryRun, emit, opts);
|
|
95
|
+
}
|
|
96
|
+
/** `30m`, `12h`, `7d`, `2w`, or an ISO date → epoch ms cutoff. */
|
|
97
|
+
export function parseSince(spec, now = Date.now()) {
|
|
98
|
+
const m = /^(\d+)([mhdw])$/.exec(spec.trim());
|
|
99
|
+
if (m) {
|
|
100
|
+
const n = Number(m[1]);
|
|
101
|
+
const unit = { m: 60_000, h: 3_600_000, d: 86_400_000, w: 7 * 86_400_000 }[m[2]];
|
|
102
|
+
return now - n * unit;
|
|
103
|
+
}
|
|
104
|
+
const t = Date.parse(spec);
|
|
105
|
+
if (Number.isNaN(t)) {
|
|
106
|
+
throw new AbsorbInputError(`Cannot read --since "${spec}": use 30m, 12h, 7d, 2w, or an ISO date.`);
|
|
107
|
+
}
|
|
108
|
+
return t;
|
|
109
|
+
}
|
|
110
|
+
/** Narrow a store's sessions to what the operator asked for. Pure, so the
|
|
111
|
+
* three flags can be pinned without a store. */
|
|
112
|
+
export function selectSessions(candidates, req, now = Date.now()) {
|
|
113
|
+
let out = candidates;
|
|
114
|
+
if (req.fold) {
|
|
115
|
+
const want = resolve(req.fold.startsWith("~/") ? join(process.env.HOME, req.fold.slice(2)) : req.fold);
|
|
116
|
+
out = out.filter((c) => c.cwd !== null && resolve(c.cwd) === want);
|
|
117
|
+
}
|
|
118
|
+
if (req.since) {
|
|
119
|
+
const cutoff = parseSince(req.since, now);
|
|
120
|
+
out = out.filter((c) => c.updatedAt >= cutoff);
|
|
121
|
+
}
|
|
122
|
+
if (req.session) {
|
|
123
|
+
const want = req.session;
|
|
124
|
+
const exact = out.filter((c) => c.externalId === want);
|
|
125
|
+
if (exact.length > 0)
|
|
126
|
+
return exact;
|
|
127
|
+
const prefixed = out.filter((c) => c.externalId.startsWith(want));
|
|
128
|
+
if (prefixed.length > 1) {
|
|
129
|
+
throw new AbsorbInputError(`--session ${want} matches ${prefixed.length} sessions; give more of the id.`);
|
|
130
|
+
}
|
|
131
|
+
out = prefixed;
|
|
132
|
+
}
|
|
133
|
+
return out;
|
|
134
|
+
}
|
|
135
|
+
/** Events in an absorbed narrative that no import wrote: an operator's
|
|
136
|
+
* signal sent into it from the app, a foreign narration, anything else
|
|
137
|
+
* appended live. Every event an import writes is keyed in `absorb_messages`
|
|
138
|
+
* by the seq it landed at, so an event with no row there came from
|
|
139
|
+
* somewhere else — and is in no file a rebuild could re-derive it from. */
|
|
140
|
+
export function liveContributions(db, narrativeId, braneId, provider) {
|
|
141
|
+
const row = db
|
|
142
|
+
.prepare(`SELECT COUNT(*) AS n FROM events
|
|
143
|
+
WHERE narrative_id = ?
|
|
144
|
+
AND seq NOT IN (SELECT seq FROM absorb_messages WHERE brane_id = ? AND provider = ? AND narrative_id = ?)`)
|
|
145
|
+
.get(narrativeId, braneId, provider, narrativeId);
|
|
146
|
+
return row.n;
|
|
147
|
+
}
|
|
148
|
+
/** The one deleting operation in absorb, behind `--rebuild`: empty an
|
|
149
|
+
* absorbed narrative so the file it came from can be imported again.
|
|
150
|
+
*
|
|
151
|
+
* Why this is allowed under the golden rule: the flag is the operator's
|
|
152
|
+
* explicit instruction, and what is discarded is a *rendering* of a file
|
|
153
|
+
* that is still on disk — the caller only reaches here for a candidate the
|
|
154
|
+
* survey just found. That holds only while the narrative is nothing but
|
|
155
|
+
* that rendering. An absorbed narrative still takes live writes — the
|
|
156
|
+
* operator can signal into it from the app, an agent can narrate into it —
|
|
157
|
+
* and those are in no file, so a narrative holding any is refused
|
|
158
|
+
* (`liveContributions`): locked in, appended to but never re-derived. The
|
|
159
|
+
* narrative row itself is kept (id, brane,
|
|
160
|
+
* created_at), so a citation of the narrative from anywhere else still
|
|
161
|
+
* resolves; its seqs restart at 1, since the old ones were the rendering
|
|
162
|
+
* being replaced. The WHERE clauses are the guard: only an absorbed
|
|
163
|
+
* narrative, only in this brane, only the one bound to this session.
|
|
164
|
+
*
|
|
165
|
+
* Memory-index rows for the narrative go too, and its cursor, so the next
|
|
166
|
+
* indexing pass re-embeds the rebuilt events instead of believing it has
|
|
167
|
+
* already seen them. `memory_content_df` is left alone — it is a
|
|
168
|
+
* frequency heuristic that a full `brane memory index --rebuild` recomputes
|
|
169
|
+
* and that a stale count does not make wrong, only slightly off. */
|
|
170
|
+
export function resetAbsorbedNarrative(db, narrativeId, braneId, provider, externalId) {
|
|
171
|
+
db.transaction(() => {
|
|
172
|
+
const owned = db
|
|
173
|
+
.prepare(`SELECT 1 FROM narratives
|
|
174
|
+
WHERE id = ? AND brane_id = ? AND role = 'absorbed'
|
|
175
|
+
AND agent_type = ? AND agent_session_id = ?`)
|
|
176
|
+
.get(narrativeId, braneId, provider, externalId);
|
|
177
|
+
if (!owned)
|
|
178
|
+
throw new Error(`refusing to rebuild ${narrativeId}: not an absorbed ${provider} narrative of ${braneId}`);
|
|
179
|
+
const live = liveContributions(db, narrativeId, braneId, provider);
|
|
180
|
+
if (live > 0)
|
|
181
|
+
throw new Error(`refusing to rebuild ${narrativeId}: ${live} event(s) were not written by an import`);
|
|
182
|
+
db.prepare("DELETE FROM events WHERE narrative_id = ?").run(narrativeId);
|
|
183
|
+
db.prepare("DELETE FROM tool_uses WHERE narrative_id = ?").run(narrativeId);
|
|
184
|
+
db.prepare("DELETE FROM absorb_messages WHERE brane_id = ? AND provider = ? AND narrative_id = ?").run(braneId, provider, narrativeId);
|
|
185
|
+
db.prepare("DELETE FROM absorb_files WHERE brane_id = ? AND provider = ? AND external_id = ?").run(braneId, provider, externalId);
|
|
186
|
+
const tables = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map((t) => t.name));
|
|
187
|
+
if (tables.has("memory_chunks")) {
|
|
188
|
+
// memory_fts follows memory_chunks by trigger; memory_vec has no
|
|
189
|
+
// trigger and is keyed by chunk id.
|
|
190
|
+
if (tables.has("memory_vec")) {
|
|
191
|
+
db.prepare("DELETE FROM memory_vec WHERE chunk_id IN (SELECT id FROM memory_chunks WHERE narrative_id = ?)").run(narrativeId);
|
|
192
|
+
}
|
|
193
|
+
db.prepare("DELETE FROM memory_chunks WHERE narrative_id = ?").run(narrativeId);
|
|
194
|
+
}
|
|
195
|
+
if (tables.has("memory_cursors")) {
|
|
196
|
+
db.prepare("DELETE FROM memory_cursors WHERE narrative_id = ?").run(narrativeId);
|
|
197
|
+
}
|
|
198
|
+
})();
|
|
199
|
+
}
|
|
200
|
+
/** Local-source absorb: a session store on this machine, read in place.
|
|
201
|
+
*
|
|
202
|
+
* Differs from the export path in three deliberate ways. **No fold is
|
|
203
|
+
* registered.** Each session knows the directory it ran in, and that
|
|
204
|
+
* becomes the narrative's `fold_path`; but registering every such directory
|
|
205
|
+
* as a spawn point would bind existing folds (the operator's own `~/dev/x`)
|
|
206
|
+
* to the foreign brane, which would change what `brane spawn` does there.
|
|
207
|
+
* `--fold` is therefore a filter here — "sessions from this directory" —
|
|
208
|
+
* not a destination. **`--brane` is a destination, and destinations are
|
|
209
|
+
* independent**: the ledgers below are keyed by brane, so the same store
|
|
210
|
+
* imported into two branes is two complete copies (claude-xiy9e8wq:38).
|
|
211
|
+
* **Sessions brane already owns are skipped**, by
|
|
212
|
+
* `narratives.session_ids`, and counted as `managed` (claude-xiy9e8wq:13).
|
|
213
|
+
* **Unchanged files are not read**: `absorb_files` remembers each file's
|
|
214
|
+
* (size, mtime) at its last absorb, so a re-run over a store of hundreds of
|
|
215
|
+
* sessions costs a stat per file plus the parse of whatever grew. */
|
|
216
|
+
async function runLocalAbsorb(db, source, inputPath, req, dryRun, emit, _opts) {
|
|
217
|
+
const root = source.name === "claude" ? resolveClaudeRoot(inputPath) : inputPath;
|
|
218
|
+
if (!root)
|
|
219
|
+
throw new AbsorbInputError(`Not a ${source.name} session store: ${inputPath}`);
|
|
220
|
+
const home = process.env.HOME;
|
|
221
|
+
const identity = req.identity
|
|
222
|
+
? { id: req.identity, label: req.identity }
|
|
223
|
+
: source.identity(home);
|
|
224
|
+
const homeBraneId = getBraneId(db);
|
|
225
|
+
const braneId = req.brane === "self" ? homeBraneId : (req.brane ?? deriveBraneId(source.name, identity.id));
|
|
226
|
+
// Everything above and the survey below is validation-grade: nothing is
|
|
227
|
+
// parsed, nothing is written. Input errors still surface before any emit.
|
|
228
|
+
const all = source.survey(root);
|
|
229
|
+
const selected = selectSessions(all, req);
|
|
230
|
+
const status = classifySessions(db, source, selected, braneId);
|
|
231
|
+
const toRead = [];
|
|
232
|
+
const toRebuild = new Map(); // externalId → narrative id
|
|
233
|
+
let managed = 0;
|
|
234
|
+
let unchanged = 0;
|
|
235
|
+
let locked = 0;
|
|
236
|
+
let reparse = 0;
|
|
237
|
+
for (const c of selected) {
|
|
238
|
+
const st = status.get(c.externalId);
|
|
239
|
+
if (st.managedBy) {
|
|
240
|
+
managed++;
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
if (req.rebuild && st.absorbedBy) {
|
|
244
|
+
if (liveContributions(db, st.absorbedBy, braneId, source.name) > 0) {
|
|
245
|
+
// Written into since it was absorbed: a rebuild would delete what
|
|
246
|
+
// was written. Read like any other re-run, which only appends — and
|
|
247
|
+
// counted once, as locked, whether or not the file has grown.
|
|
248
|
+
locked++;
|
|
249
|
+
if (st.unchanged)
|
|
250
|
+
continue;
|
|
251
|
+
if (st.reparse)
|
|
252
|
+
reparse++;
|
|
253
|
+
toRead.push(c);
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
// A rebuild re-reads regardless of the stat; the candidate is here,
|
|
258
|
+
// so its file is on disk, which is the one precondition.
|
|
259
|
+
toRebuild.set(c.externalId, st.absorbedBy);
|
|
260
|
+
toRead.push(c);
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (st.unchanged) {
|
|
265
|
+
unchanged++;
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
if (st.reparse)
|
|
269
|
+
reparse++;
|
|
270
|
+
toRead.push(c);
|
|
271
|
+
}
|
|
272
|
+
// Oldest first, so a forked successor always finds its predecessor's
|
|
273
|
+
// messages already in absorb_messages and threads onto them rather than
|
|
274
|
+
// the other way round (the same ordering rule as cost backfill).
|
|
275
|
+
toRead.sort((a, b) => (a.startedAt ?? a.updatedAt) - (b.startedAt ?? b.updatedAt));
|
|
276
|
+
emit({ type: "info", message: `Store: ${toPortablePath(root)}` });
|
|
277
|
+
emit({
|
|
278
|
+
type: "info",
|
|
279
|
+
message: ` Sessions: ${all.length} found${selected.length !== all.length ? `, ${selected.length} selected` : ""}`,
|
|
280
|
+
});
|
|
281
|
+
if (managed > 0) {
|
|
282
|
+
emit({ type: "info", message: ` Managed by brane: ${managed} (live narratives own these; left alone)` });
|
|
283
|
+
}
|
|
284
|
+
if (unchanged > 0) {
|
|
285
|
+
emit({ type: "info", message: ` Unchanged since last absorb: ${unchanged} (not read)` });
|
|
286
|
+
}
|
|
287
|
+
if (toRebuild.size > 0) {
|
|
288
|
+
emit({
|
|
289
|
+
type: "info",
|
|
290
|
+
message: ` Rebuild: ${toRebuild.size} (events re-derived from source; narrative ids kept, seqs renumbered)`,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
if (locked > 0) {
|
|
294
|
+
emit({
|
|
295
|
+
type: "info",
|
|
296
|
+
message: ` Not rebuilt: ${locked} (hold signals or narrations no import wrote; read as an ordinary re-run)`,
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
if (reparse > 0) {
|
|
300
|
+
emit({ type: "info", message: ` Newer parser: ${reparse} (unchanged since last absorb; read again for what it can now find)` });
|
|
301
|
+
}
|
|
302
|
+
emit({ type: "info", message: ` To read: ${toRead.length}` });
|
|
303
|
+
emit({ type: "info", message: `Identity: ${identity.label}` });
|
|
304
|
+
emit({ type: "info", message: `Brane: ${braneId}${braneId === homeBraneId ? " (home)" : ""}` });
|
|
305
|
+
emit({ type: "info", message: `Folds: each session's own working directory (not registered as spawn points)` });
|
|
306
|
+
if (dryRun)
|
|
307
|
+
emit({ type: "info", message: `\n--- DRY RUN (no database changes) ---\n` });
|
|
308
|
+
const ctx = createAbsorbContext(db, source.name, braneId, toPortablePath(root), identity, dryRun);
|
|
309
|
+
ctx.summary.managed = managed;
|
|
310
|
+
ctx.summary.unchanged = unchanged;
|
|
311
|
+
if (locked > 0)
|
|
312
|
+
ctx.summary.locked = locked;
|
|
313
|
+
const rememberFile = db.prepare(`INSERT INTO absorb_files (brane_id, provider, external_id, path, bytes, mtime, absorbed_at, parser_version)
|
|
314
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
315
|
+
ON CONFLICT (brane_id, provider, external_id) DO UPDATE SET
|
|
316
|
+
path = excluded.path, bytes = excluded.bytes, mtime = excluded.mtime, absorbed_at = excluded.absorbed_at,
|
|
317
|
+
parser_version = excluded.parser_version`);
|
|
318
|
+
const foldPaths = new Set();
|
|
319
|
+
let done = 0;
|
|
320
|
+
for (const c of toRead) {
|
|
321
|
+
const rebuildNid = toRebuild.get(c.externalId);
|
|
322
|
+
if (rebuildNid) {
|
|
323
|
+
ctx.summary.rebuilt = (ctx.summary.rebuilt ?? 0) + 1;
|
|
324
|
+
if (!dryRun)
|
|
325
|
+
resetAbsorbedNarrative(db, rebuildNid, braneId, source.name, c.externalId);
|
|
326
|
+
}
|
|
327
|
+
const conv = source.extract(c);
|
|
328
|
+
if (conv) {
|
|
329
|
+
absorbOne(ctx, conv);
|
|
330
|
+
if (conv.cwd)
|
|
331
|
+
foldPaths.add(toPortablePath(conv.cwd));
|
|
332
|
+
}
|
|
333
|
+
else {
|
|
334
|
+
ctx.summary.skipped++;
|
|
335
|
+
}
|
|
336
|
+
if (!dryRun)
|
|
337
|
+
rememberFile.run(braneId, source.name, c.externalId, c.path, c.bytes, c.updatedAt, Date.now(), source.parserVersion);
|
|
338
|
+
done++;
|
|
339
|
+
emit({ type: "progress", done, title: conv?.title ?? null });
|
|
340
|
+
// Yield between sessions so the daemon keeps answering mid-run and the
|
|
341
|
+
// progress lines reach the client (the organelle importer's rule).
|
|
342
|
+
await new Promise((r) => setImmediate(r));
|
|
343
|
+
}
|
|
344
|
+
emit({
|
|
345
|
+
type: "summary",
|
|
346
|
+
dry_run: dryRun,
|
|
347
|
+
provider: source.name,
|
|
348
|
+
brane_id: braneId,
|
|
349
|
+
fold_paths: [...foldPaths],
|
|
350
|
+
summary: ctx.summary,
|
|
351
|
+
});
|
|
352
|
+
return ctx.summary;
|
|
353
|
+
}
|
|
354
|
+
/** Organelle absorb — reads from search.db + niche filesystem. */
|
|
355
|
+
async function runOrganelleAbsorb(db, inputPath, req, dryRun, emit, opts) {
|
|
356
|
+
const searchDbPath = findSearchDb();
|
|
357
|
+
if (!searchDbPath) {
|
|
358
|
+
throw new AbsorbInputError("Cannot find organelle search database at ~/.organelle/index/search.db");
|
|
359
|
+
}
|
|
360
|
+
// A single niche has .organelle/narratives/; the registry (~/.organelle)
|
|
361
|
+
// has niches.json and means "every niche in it".
|
|
362
|
+
const nichesJsonPath = resolve(inputPath, "niches.json");
|
|
363
|
+
const singleNicheDir = resolve(inputPath, ".organelle", "narratives");
|
|
364
|
+
let nichePaths;
|
|
365
|
+
if (existsSync(singleNicheDir)) {
|
|
366
|
+
nichePaths = [inputPath];
|
|
367
|
+
}
|
|
368
|
+
else if (existsSync(nichesJsonPath)) {
|
|
369
|
+
const registry = readNichesRegistry();
|
|
370
|
+
nichePaths = Object.values(registry)
|
|
371
|
+
.map((n) => n.path)
|
|
372
|
+
.filter((p) => existsSync(resolve(p, ".organelle", "narratives")));
|
|
373
|
+
if (nichePaths.length === 0) {
|
|
374
|
+
throw new AbsorbInputError("No valid niches found in registry.");
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
else {
|
|
378
|
+
throw new AbsorbInputError(`Not an organelle niche or registry: ${inputPath}\n` +
|
|
379
|
+
`Expected .organelle/narratives/ directory or niches.json file.`);
|
|
380
|
+
}
|
|
381
|
+
// Identity — default to the OS username (organelle is a local system).
|
|
382
|
+
const identity = req.identity
|
|
383
|
+
? { id: req.identity, label: req.identity }
|
|
384
|
+
: (() => {
|
|
385
|
+
const username = process.env.USER || process.env.USERNAME || "operator";
|
|
386
|
+
return { id: username, label: username };
|
|
387
|
+
})();
|
|
388
|
+
const homeBraneId = getBraneId(db);
|
|
389
|
+
const braneId = req.brane === "self" ? homeBraneId : (req.brane ?? deriveBraneId("organelle", identity.id));
|
|
390
|
+
// Each niche gets its own fold (from --fold only when there is exactly one).
|
|
391
|
+
const nichesToAbsorb = nichePaths.map((np) => ({
|
|
392
|
+
nichePath: np,
|
|
393
|
+
foldPath: portableFold(nichePaths.length === 1 ? (req.fold || np) : np),
|
|
394
|
+
}));
|
|
395
|
+
assertFoldsRegistrable(nichesToAbsorb.map((n) => n.foldPath), opts);
|
|
396
|
+
// Everything above is validation; the report starts here.
|
|
397
|
+
let totalNarratives = 0;
|
|
398
|
+
let totalWithEvents = 0;
|
|
399
|
+
let totalEvents = 0;
|
|
400
|
+
const allAgentTypes = {};
|
|
401
|
+
emit({ type: "info", message: `Search DB: ${searchDbPath}` });
|
|
402
|
+
emit({ type: "info", message: `Niches: ${nichePaths.length}` });
|
|
403
|
+
for (const np of nichePaths) {
|
|
404
|
+
const desc = describeOrganelleNiche(np);
|
|
405
|
+
totalNarratives += desc.narrativeCount;
|
|
406
|
+
totalWithEvents += desc.narrativesWithEvents;
|
|
407
|
+
totalEvents += desc.totalEvents;
|
|
408
|
+
for (const [t, c] of Object.entries(desc.agentTypes)) {
|
|
409
|
+
allAgentTypes[t] = (allAgentTypes[t] || 0) + c;
|
|
410
|
+
}
|
|
411
|
+
emit({
|
|
412
|
+
type: "info",
|
|
413
|
+
message: ` ${basename(np)}: ${desc.narrativesWithEvents} narratives, ${desc.totalEvents} events`,
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
emit({
|
|
417
|
+
type: "info",
|
|
418
|
+
message: `Total: ${totalWithEvents} narratives with events (${totalNarratives} total, skipping utilities)`,
|
|
419
|
+
});
|
|
420
|
+
emit({ type: "info", message: `Events: ${totalEvents} importable` });
|
|
421
|
+
const agentEntries = Object.entries(allAgentTypes).sort(([, a], [, b]) => b - a);
|
|
422
|
+
if (agentEntries.length > 0) {
|
|
423
|
+
emit({
|
|
424
|
+
type: "info",
|
|
425
|
+
message: `Agent types: ${agentEntries.map(([t, c]) => `${t} (${c})`).join(", ")}`,
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
emit({ type: "info", message: `Identity: ${identity.label}` });
|
|
429
|
+
emit({ type: "info", message: `Brane: ${braneId}${braneId === homeBraneId ? " (home)" : ""}` });
|
|
430
|
+
emit({
|
|
431
|
+
type: "info",
|
|
432
|
+
message: nichesToAbsorb.length === 1
|
|
433
|
+
? `Fold: ${nichesToAbsorb[0].foldPath}`
|
|
434
|
+
: `Folds: ${nichesToAbsorb.length} (one per niche)`,
|
|
435
|
+
});
|
|
436
|
+
if (dryRun) {
|
|
437
|
+
emit({ type: "info", message: `\n--- DRY RUN (no database changes) ---\n` });
|
|
438
|
+
}
|
|
439
|
+
else {
|
|
440
|
+
for (const { foldPath } of nichesToAbsorb) {
|
|
441
|
+
registerFold(db, foldPath, { braneId, confineToHome: opts.confineFoldsToHome });
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
const summary = await absorbOrganelle(db, nichesToAbsorb, braneId, identity, (done, title) => emit({ type: "progress", done, title }), dryRun);
|
|
445
|
+
emit({
|
|
446
|
+
type: "summary",
|
|
447
|
+
dry_run: dryRun,
|
|
448
|
+
provider: "organelle",
|
|
449
|
+
brane_id: braneId,
|
|
450
|
+
fold_paths: nichesToAbsorb.map((n) => n.foldPath),
|
|
451
|
+
summary,
|
|
452
|
+
});
|
|
453
|
+
return summary;
|
|
454
|
+
}
|
|
455
|
+
/** Stream-based absorb for chatgpt and similar JSON-export providers. */
|
|
456
|
+
async function runStreamAbsorb(db, providerName, inputPath, req, dryRun, emit, opts) {
|
|
457
|
+
const provider = getProvider(providerName);
|
|
458
|
+
if (!req.fold) {
|
|
459
|
+
throw new AbsorbInputError(`--fold is required for ${provider.name} provider.`);
|
|
460
|
+
}
|
|
461
|
+
const desc = describeExportDirectory(inputPath);
|
|
462
|
+
if (desc.conversationFiles.length === 0) {
|
|
463
|
+
throw new AbsorbInputError(`No conversation files found in ${inputPath}.\n` +
|
|
464
|
+
`Expected conversations.json or conversations-NNN.json files.`);
|
|
465
|
+
}
|
|
466
|
+
// --identity overrides whatever user.json says; without either, refuse.
|
|
467
|
+
let identity = extractIdentityFromDirectory(inputPath);
|
|
468
|
+
if (req.identity) {
|
|
469
|
+
identity = { id: req.identity, label: req.identity };
|
|
470
|
+
}
|
|
471
|
+
else if (!identity) {
|
|
472
|
+
throw new AbsorbInputError("No user.json found in directory. Pass --identity <email> to identify the account.");
|
|
473
|
+
}
|
|
474
|
+
const homeBraneId = getBraneId(db);
|
|
475
|
+
const foldPath = portableFold(req.fold);
|
|
476
|
+
const braneId = req.brane === "self" ? homeBraneId : (req.brane ?? deriveBraneId(provider.name, identity.id));
|
|
477
|
+
assertFoldsRegistrable([foldPath], opts);
|
|
478
|
+
emit({ type: "info", message: `Directory: ${inputPath}` });
|
|
479
|
+
emit({
|
|
480
|
+
type: "info",
|
|
481
|
+
message: ` Conversation files: ${desc.conversationFiles.length} (${desc.conversationFiles[0]}${desc.conversationFiles.length > 1 ? ` … ${desc.conversationFiles[desc.conversationFiles.length - 1]}` : ""})`,
|
|
482
|
+
});
|
|
483
|
+
if (desc.hasUserJson)
|
|
484
|
+
emit({ type: "info", message: ` user.json: found` });
|
|
485
|
+
if (desc.extraFiles.length > 0) {
|
|
486
|
+
emit({
|
|
487
|
+
type: "info",
|
|
488
|
+
message: ` Extra files: ${desc.extraFiles.length} (${desc.extraFiles.filter((f) => !f.startsWith(".")).slice(0, 3).join(", ")}${desc.extraFiles.length > 3 ? ", …" : ""})`,
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
if (desc.mediaDirs.length > 0) {
|
|
492
|
+
emit({
|
|
493
|
+
type: "info",
|
|
494
|
+
message: ` Media dirs: ${desc.mediaDirs.length} (${desc.mediaDirs.slice(0, 3).join(", ")}${desc.mediaDirs.length > 3 ? ", …" : ""})`,
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
const fileIndex = buildFileIndex(inputPath);
|
|
498
|
+
emit({ type: "info", message: ` Asset index: ${fileIndex.size} files` });
|
|
499
|
+
const sources = findConversationsInDirectory(inputPath);
|
|
500
|
+
emit({ type: "info", message: `Identity: ${identity.label}` });
|
|
501
|
+
emit({ type: "info", message: `Brane: ${braneId}${braneId === homeBraneId ? " (home)" : ""}` });
|
|
502
|
+
emit({ type: "info", message: `Fold: ${foldPath}` });
|
|
503
|
+
if (dryRun) {
|
|
504
|
+
emit({ type: "info", message: `\n--- DRY RUN (no database changes) ---\n` });
|
|
505
|
+
}
|
|
506
|
+
else {
|
|
507
|
+
registerFold(db, foldPath, { braneId, confineToHome: opts.confineFoldsToHome });
|
|
508
|
+
}
|
|
509
|
+
const summary = await absorb(db, sources, provider, braneId, foldPath, identity, (done, title) => emit({ type: "progress", done, title }), dryRun, fileIndex);
|
|
510
|
+
emit({
|
|
511
|
+
type: "summary",
|
|
512
|
+
dry_run: dryRun,
|
|
513
|
+
provider: provider.name,
|
|
514
|
+
brane_id: braneId,
|
|
515
|
+
fold_paths: [foldPath],
|
|
516
|
+
summary,
|
|
517
|
+
});
|
|
518
|
+
return summary;
|
|
519
|
+
}
|
|
520
|
+
//# sourceMappingURL=run.js.map
|