@anchrd/intel-api 0.6.6 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +44 -3
  2. package/dist/adapters/cloudflare/cloudflare.js +50 -16
  3. package/dist/adapters/cloudflare/cloudflare.types.d.ts +11 -0
  4. package/dist/adapters/content/content.d.ts +1 -1
  5. package/dist/adapters/db/db-flows.js +161 -20
  6. package/dist/adapters/db/db-grants.d.ts +13 -2
  7. package/dist/adapters/db/db-grants.js +25 -8
  8. package/dist/adapters/db/db-indexing.d.ts +2 -2
  9. package/dist/adapters/db/db-indexing.js +26 -19
  10. package/dist/adapters/db/db.d.ts +3 -3
  11. package/dist/adapters/db/db.js +448 -119
  12. package/dist/adapters/gate-applications/gate-applications.d.ts +23 -0
  13. package/dist/adapters/gate-applications/gate-applications.js +66 -0
  14. package/dist/adapters/index-queue/index-queue.d.ts +1 -1
  15. package/dist/adapters/index-queue/index-queue.js +2 -2
  16. package/dist/adapters/semantic-index/semantic-index.types.d.ts +2 -2
  17. package/dist/agent-runtime/agent-runtime.d.ts +16 -0
  18. package/dist/agent-runtime/agent-runtime.js +76 -0
  19. package/dist/agent-runtime/agent-runtime.types.d.ts +57 -0
  20. package/dist/bundle/bundle.d.ts +4 -0
  21. package/dist/bundle/bundle.js +1035 -0
  22. package/dist/bundle/bundle.types.d.ts +33 -0
  23. package/dist/bundle/bundle.types.js +1 -0
  24. package/dist/cli/cli.js +10 -1
  25. package/dist/flows/flows.d.ts +8 -8
  26. package/dist/flows/flows.js +158 -42
  27. package/dist/flows/flows.types.d.ts +40 -7
  28. package/dist/http/http.d.ts +1 -0
  29. package/dist/http/http.js +329 -63
  30. package/dist/http/http.types.d.ts +6 -2
  31. package/dist/indexing/indexing.js +14 -2
  32. package/dist/indexing/indexing.types.d.ts +2 -2
  33. package/dist/intel/intel.js +12 -3
  34. package/dist/intel/intel.types.d.ts +6 -2
  35. package/dist/mcp/mcp.js +483 -124
  36. package/dist/mcp/mcp.types.d.ts +11 -2
  37. package/dist/nodes/nodes.d.ts +2 -0
  38. package/dist/nodes/nodes.js +1337 -0
  39. package/dist/nodes/nodes.types.d.ts +314 -0
  40. package/dist/nodes/nodes.types.js +1 -0
  41. package/migrations/0011_one_name_for_the_tree.sql +53 -0
  42. package/migrations/0012_table_snapshots.sql +29 -0
  43. package/migrations/0013_agents_in_the_tree.sql +76 -0
  44. package/migrations/0014_agent_applications.sql +25 -0
  45. package/package.json +3 -2
  46. package/dist/knowledge/knowledge.d.ts +0 -2
  47. package/dist/knowledge/knowledge.js +0 -761
  48. package/dist/knowledge/knowledge.types.d.ts +0 -198
  49. /package/dist/{knowledge/knowledge.types.js → agent-runtime/agent-runtime.types.js} +0 -0
  50. /package/dist/{knowledge → nodes}/document-links/document-links.d.ts +0 -0
  51. /package/dist/{knowledge → nodes}/document-links/document-links.js +0 -0
@@ -0,0 +1,1035 @@
1
+ import { AgentDefinition, AgentMediaType, BlockNoteMediaType, BundleImportResult, BundleManifest, BundleManifestFilename, DocumentLinkInlineType, FlowGraph, TableMediaType, } from "@anchrd/intel-contract";
2
+ import { Unzip, UnzipInflate, Zip, ZipDeflate, ZipPassThrough } from "fflate";
3
+ import { documentLinkTargets } from "../nodes/document-links/document-links.js";
4
+ import { parseCsv } from "../shared/csv/csv.js";
5
+ import { IntelError } from "../shared/intel-error/intel-error.js";
6
+ // What every bundle leaves out, by decision rather than by accident (#136). The manifest says so,
7
+ // because a backup that is silent about what it does not hold will be trusted with exactly that.
8
+ const Excluded = ["version-history", "grants", "flow-runs", "archived-nodes"];
9
+ /**
10
+ * A title as a file name. Titles are free text and zip paths are not: a separator would move the
11
+ * entry, a control character corrupts the archive listing, and a trailing dot breaks extraction on
12
+ * Windows. What cannot be kept is replaced rather than refused — an export must not fail because of
13
+ * how something was named.
14
+ */
15
+ function sanitizeName(title) {
16
+ const cleaned = title
17
+ // biome-ignore lint/suspicious/noControlCharactersInRegex: control characters are exactly what has to go
18
+ .replace(/[\u0000-\u001f\u007f]/g, "")
19
+ .replace(/[/\\]/g, "-")
20
+ .trim()
21
+ .replace(/^\.+/, "")
22
+ .replace(/[. ]+$/, "")
23
+ .slice(0, 150);
24
+ return cleaned.length > 0 ? cleaned : "untitled";
25
+ }
26
+ // A name that is free in this directory. Case-insensitive, because the zip will be extracted onto
27
+ // filesystems where "Notes.md" and "notes.md" are the same file and the second would overwrite the
28
+ // first without a word.
29
+ function uniqueName(used, base, extension) {
30
+ const candidate = base.toLowerCase().endsWith(extension.toLowerCase())
31
+ ? base.slice(0, base.length - extension.length)
32
+ : base;
33
+ const first = `${candidate}${extension}`;
34
+ if (!used.has(first.toLowerCase())) {
35
+ used.add(first.toLowerCase());
36
+ return first;
37
+ }
38
+ for (let counter = 2;; counter++) {
39
+ const next = `${candidate} (${counter})${extension}`;
40
+ if (!used.has(next.toLowerCase())) {
41
+ used.add(next.toLowerCase());
42
+ return next;
43
+ }
44
+ }
45
+ }
46
+ // ── Import (#137) ───────────────────────────────────────────────────────────────────────────────
47
+ // The import limits, explicit rather than discovered (#137). The zip stream and the unpacked total
48
+ // are both bounded because the Worker holds an entry in memory while it lands: a bundle is parsed
49
+ // where it arrives, and an isolate has 128 MB for everything. The old 15 MB base64 door stays for
50
+ // single attachments; these carry a real bundle several times that size.
51
+ export const MaxImportZipBytes = 64 * 1024 * 1024;
52
+ export const MaxImportEntryBytes = 48 * 1024 * 1024;
53
+ const MaxImportTotalBytes = 96 * 1024 * 1024;
54
+ const MaxImportEntries = 1000;
55
+ // The same key shape `nodes.ts` writes: keys are stored, never derived back, so agreeing on the
56
+ // format costs nothing while a second format would split the bucket for no reason.
57
+ function contentKeyFor(nodeId, versionId) {
58
+ return `nodes/${nodeId}/versions/${versionId}`;
59
+ }
60
+ // Files archivers add that nobody filed: importing them would put macOS bookkeeping into the tree.
61
+ const JunkNames = new Set([".DS_Store", "Thumbs.db"]);
62
+ function isJunkPath(path) {
63
+ const segments = path.replace(/\/$/, "").split("/");
64
+ return segments[0] === "__MACOSX" || JunkNames.has(segments[segments.length - 1] ?? "");
65
+ }
66
+ // A zip entry name is attacker-controlled text, and "../" in one is how an archive writes outside
67
+ // its directory. Nothing here touches a filesystem, but a traversal segment would still corrupt
68
+ // the parent lookup, so the path is refused rather than interpreted.
69
+ function requireSafePath(path) {
70
+ if (path.includes("\\") || path.startsWith("/")) {
71
+ throw new IntelError(400, "import_invalid_path", `Bundle path is not usable: ${path}`);
72
+ }
73
+ for (const segment of path.replace(/\/$/, "").split("/")) {
74
+ if (segment === "" || segment === "." || segment === "..") {
75
+ throw new IntelError(400, "import_invalid_path", `Bundle path is not usable: ${path}`);
76
+ }
77
+ }
78
+ }
79
+ function concatChunks(chunks, size) {
80
+ const joined = new Uint8Array(size);
81
+ let offset = 0;
82
+ for (const chunk of chunks) {
83
+ joined.set(chunk, offset);
84
+ offset += chunk.length;
85
+ }
86
+ return joined;
87
+ }
88
+ /**
89
+ * The whole zip, read incrementally: the compressed stream is fed chunk by chunk and every entry
90
+ * is collected as it completes, so the peak cost is the unpacked entries — bounded above — and
91
+ * never compressed-plus-unpacked at once.
92
+ *
93
+ * ⚠️ A truncated archive does not fail loudly in fflate: entries simply never finish. The open
94
+ * counter turns that silence into the refusal it is — a bundle that ends mid-entry imports
95
+ * nothing (#137, Abbruch-Test).
96
+ */
97
+ async function readZip(source) {
98
+ const files = new Map();
99
+ const directories = new Set();
100
+ let failure = null;
101
+ let unpackedTotal = 0;
102
+ let entryCount = 0;
103
+ let openEntries = 0;
104
+ const invalid = () => new IntelError(400, "import_invalid_bundle", "The zip archive could not be read");
105
+ const unzip = new Unzip((file) => {
106
+ if (failure)
107
+ return;
108
+ const name = file.name;
109
+ if (name.endsWith("/")) {
110
+ directories.add(name);
111
+ return;
112
+ }
113
+ entryCount += 1;
114
+ if (entryCount > MaxImportEntries) {
115
+ failure = new IntelError(413, "import_too_many_entries", `The bundle exceeds ${MaxImportEntries} entries`);
116
+ return;
117
+ }
118
+ openEntries += 1;
119
+ const chunks = [];
120
+ let size = 0;
121
+ file.ondata = (error, chunk, final) => {
122
+ if (failure)
123
+ return;
124
+ if (error) {
125
+ failure = invalid();
126
+ return;
127
+ }
128
+ if (chunk) {
129
+ size += chunk.length;
130
+ unpackedTotal += chunk.length;
131
+ if (size > MaxImportEntryBytes) {
132
+ failure = new IntelError(413, "import_entry_too_large", `A bundle entry exceeds ${MaxImportEntryBytes} bytes unpacked: ${name}`);
133
+ return;
134
+ }
135
+ if (unpackedTotal > MaxImportTotalBytes) {
136
+ failure = new IntelError(413, "import_too_large", `The bundle exceeds ${MaxImportTotalBytes} bytes unpacked`);
137
+ return;
138
+ }
139
+ chunks.push(chunk);
140
+ }
141
+ if (final) {
142
+ openEntries -= 1;
143
+ files.set(name, concatChunks(chunks, size));
144
+ }
145
+ };
146
+ try {
147
+ file.start();
148
+ }
149
+ catch {
150
+ failure = invalid();
151
+ }
152
+ });
153
+ unzip.register(UnzipInflate);
154
+ const push = (chunk, final) => {
155
+ try {
156
+ unzip.push(chunk, final);
157
+ }
158
+ catch {
159
+ failure = failure ?? invalid();
160
+ }
161
+ };
162
+ if (source instanceof Uint8Array) {
163
+ if (source.length > MaxImportZipBytes) {
164
+ throw new IntelError(413, "import_too_large", `The zip exceeds ${MaxImportZipBytes} bytes`);
165
+ }
166
+ push(source, true);
167
+ }
168
+ else {
169
+ const reader = source.getReader();
170
+ let compressed = 0;
171
+ for (;;) {
172
+ const { done, value } = await reader.read();
173
+ if (done)
174
+ break;
175
+ compressed += value.length;
176
+ if (compressed > MaxImportZipBytes) {
177
+ throw new IntelError(413, "import_too_large", `The zip exceeds ${MaxImportZipBytes} bytes`);
178
+ }
179
+ push(value, false);
180
+ if (failure)
181
+ throw failure;
182
+ }
183
+ push(new Uint8Array(0), true);
184
+ }
185
+ if (failure)
186
+ throw failure;
187
+ if (openEntries > 0 || (files.size === 0 && directories.size === 0))
188
+ throw invalid();
189
+ return { files, directories };
190
+ }
191
+ const decoder = new TextDecoder("utf-8", { fatal: true });
192
+ function decodeText(body, path) {
193
+ try {
194
+ return decoder.decode(body);
195
+ }
196
+ catch {
197
+ throw new IntelError(400, "import_invalid_text", `Bundle entry is not UTF-8: ${path}`);
198
+ }
199
+ }
200
+ function depthOf(path) {
201
+ return path.replace(/\/$/, "").split("/").length;
202
+ }
203
+ function directoryOf(path) {
204
+ const trimmed = path.replace(/\/$/, "");
205
+ const cut = trimmed.lastIndexOf("/");
206
+ return cut === -1 ? "" : trimmed.slice(0, cut + 1);
207
+ }
208
+ // What a file extension says a naked upload is (#137): the three kinds a name can carry, and
209
+ // attachment for everything else. Only used without a manifest — a manifest already says.
210
+ function nakedKindOf(name) {
211
+ const lower = name.toLowerCase();
212
+ if (lower.endsWith(".md")) {
213
+ return { kind: "document", mediaType: "text/markdown", title: name.slice(0, -3) };
214
+ }
215
+ if (lower.endsWith(".csv")) {
216
+ return { kind: "table", mediaType: TableMediaType, title: name.slice(0, -4) };
217
+ }
218
+ return { kind: "attachment", mediaType: guessedMediaType(lower), title: name };
219
+ }
220
+ const KnownMediaTypes = {
221
+ ".pdf": "application/pdf",
222
+ ".png": "image/png",
223
+ ".jpg": "image/jpeg",
224
+ ".jpeg": "image/jpeg",
225
+ ".gif": "image/gif",
226
+ ".svg": "image/svg+xml",
227
+ ".txt": "text/plain",
228
+ ".json": "application/json",
229
+ ".zip": "application/zip",
230
+ };
231
+ function guessedMediaType(lowerName) {
232
+ const dot = lowerName.lastIndexOf(".");
233
+ return ((dot === -1 ? undefined : KnownMediaTypes[lowerName.slice(dot)]) ?? "application/octet-stream");
234
+ }
235
+ function isRecord(value) {
236
+ return typeof value === "object" && value !== null;
237
+ }
238
+ // The link rewrite of #137: every `documentLink` inline element whose target the bundle carried is
239
+ // pointed at the node the import just made; a link to a node outside the bundle stays as written.
240
+ // Tolerant like `documentLinkTargets`: what is not recognised is carried over unchanged.
241
+ function remapDocumentLinks(content, idMap) {
242
+ let parsed;
243
+ try {
244
+ parsed = JSON.parse(content);
245
+ }
246
+ catch {
247
+ return content;
248
+ }
249
+ function walk(value) {
250
+ if (Array.isArray(value)) {
251
+ for (const entry of value)
252
+ walk(entry);
253
+ return;
254
+ }
255
+ if (!isRecord(value))
256
+ return;
257
+ if (value.type === DocumentLinkInlineType && isRecord(value.props)) {
258
+ const target = value.props.nodeId;
259
+ if (typeof target === "string" && idMap.has(target)) {
260
+ value.props.nodeId = idMap.get(target);
261
+ }
262
+ }
263
+ // `blocks` is the document's own top level; `content` and `children` are where a block hides
264
+ // inline elements — the same three places `documentLinkTargets` reads.
265
+ walk(value.blocks);
266
+ walk(value.content);
267
+ walk(value.children);
268
+ }
269
+ walk(parsed);
270
+ return JSON.stringify(parsed);
271
+ }
272
+ /**
273
+ * The graph rewrite of #137: link nodes point at the imported copies of what they named, a subflow
274
+ * calls the imported copy of its callee. References to things outside the bundle stay untouched.
275
+ *
276
+ * ⚠️ A `pinned` sub-flow version cannot survive the remap of its callee: the pinned ID names a row
277
+ * of the OLD installation's history, which this import deliberately does not recreate. The call
278
+ * falls back to `latest` — the draft default — and publishing will pin it anew.
279
+ */
280
+ function remapFlowGraph(graph, idMap) {
281
+ return {
282
+ ...graph,
283
+ nodes: graph.nodes.map((node) => {
284
+ if ((node.kind === "folder" ||
285
+ node.kind === "document" ||
286
+ node.kind === "upload" ||
287
+ node.kind === "table") &&
288
+ idMap.has(node.configuration.resourceId)) {
289
+ return {
290
+ ...node,
291
+ configuration: {
292
+ ...node.configuration,
293
+ resourceId: idMap.get(node.configuration.resourceId),
294
+ },
295
+ };
296
+ }
297
+ if (node.kind === "subflow" && idMap.has(node.configuration.flowId)) {
298
+ return {
299
+ ...node,
300
+ configuration: {
301
+ flowId: idMap.get(node.configuration.flowId),
302
+ version: node.configuration.version.mode === "pinned"
303
+ ? { mode: "latest" }
304
+ : node.configuration.version,
305
+ },
306
+ };
307
+ }
308
+ return node;
309
+ }),
310
+ };
311
+ }
312
+ function flowGraphJson(version) {
313
+ // A flow that was never saved has no graph; its entry is an empty file rather than an invented
314
+ // graph, and the import creates the same empty flow back.
315
+ return version === undefined ? "" : JSON.stringify(version.graph, null, 2);
316
+ }
317
+ export function createBundle(deps) {
318
+ // Flow visibility is the flow repository's question and takes its actor shape. `canRun` gates
319
+ // starting runs, which no export does, so it is answered with the conservative constant.
320
+ function asFlowActor(actor) {
321
+ return {
322
+ id: actor.id,
323
+ email: actor.email,
324
+ canRun: false,
325
+ ...(actor.isAdmin === undefined ? {} : { isAdmin: actor.isAdmin }),
326
+ };
327
+ }
328
+ async function tableCsv(nodeId) {
329
+ const keys = await deps.repository.listVersionContentKeys(nodeId);
330
+ const segments = await Promise.all(keys.map(async (key) => await deps.content.get(key)));
331
+ if (segments.some((segment) => segment === null)) {
332
+ throw new IntelError(500, "content_missing", "Version content is missing");
333
+ }
334
+ return segments.join("");
335
+ }
336
+ function textLoader(contentKey) {
337
+ return async () => {
338
+ const body = await deps.content.get(contentKey);
339
+ if (body === null) {
340
+ throw new IntelError(500, "content_missing", "Version content is missing");
341
+ }
342
+ return body;
343
+ };
344
+ }
345
+ function plannedNode(row, directory, used) {
346
+ const { node, version } = row;
347
+ const base = sanitizeName(node.title);
348
+ if (node.kind === "folder") {
349
+ const name = uniqueName(used, base, "");
350
+ const path = `${directory}${name}/`;
351
+ return {
352
+ manifest: {
353
+ id: node.id,
354
+ kind: "folder",
355
+ title: node.title,
356
+ description: node.description,
357
+ mediaType: null,
358
+ path,
359
+ },
360
+ content: { type: "folder" },
361
+ };
362
+ }
363
+ if (node.kind === "table") {
364
+ const path = `${directory}${uniqueName(used, base, ".csv")}`;
365
+ return {
366
+ manifest: {
367
+ id: node.id,
368
+ kind: "table",
369
+ title: node.title,
370
+ description: node.description,
371
+ mediaType: TableMediaType,
372
+ path,
373
+ },
374
+ content: { type: "text", load: async () => await tableCsv(node.id) },
375
+ };
376
+ }
377
+ if (node.kind === "attachment") {
378
+ // Original bytes under the original name: an attachment's title IS its file name, so no
379
+ // extension is imposed on it.
380
+ const path = `${directory}${uniqueName(used, base, "")}`;
381
+ return {
382
+ manifest: {
383
+ id: node.id,
384
+ kind: "attachment",
385
+ title: node.title,
386
+ description: node.description,
387
+ mediaType: version?.mediaType ?? "application/octet-stream",
388
+ path,
389
+ },
390
+ content: {
391
+ type: "stream",
392
+ load: async () => version === null ? null : await deps.content.getStream(version.contentKey),
393
+ },
394
+ };
395
+ }
396
+ if (node.kind === "agent") {
397
+ const path = `${directory}${uniqueName(used, base, ".json")}`;
398
+ return {
399
+ manifest: {
400
+ id: node.id,
401
+ kind: "agent",
402
+ title: node.title,
403
+ description: node.description,
404
+ mediaType: version?.mediaType ?? "application/json",
405
+ path,
406
+ },
407
+ content: version === null
408
+ ? { type: "text", load: async () => "" }
409
+ : { type: "text", load: textLoader(version.contentKey) },
410
+ };
411
+ }
412
+ // A document: markdown stays `.md`, a BlockNote body is JSON and says so with its name.
413
+ const mediaType = version?.mediaType ?? "text/markdown";
414
+ const extension = mediaType === BlockNoteMediaType ? ".json" : ".md";
415
+ const path = `${directory}${uniqueName(used, base, extension)}`;
416
+ return {
417
+ manifest: {
418
+ id: node.id,
419
+ kind: "document",
420
+ title: node.title,
421
+ description: node.description,
422
+ mediaType,
423
+ path,
424
+ },
425
+ content: version === null
426
+ ? { type: "text", load: async () => "" }
427
+ : { type: "text", load: textLoader(version.contentKey) },
428
+ };
429
+ }
430
+ function plannedFlow(flow, version, directory, used) {
431
+ const path = `${directory}${uniqueName(used, sanitizeName(flow.title), ".json")}`;
432
+ return {
433
+ manifest: {
434
+ id: flow.id,
435
+ kind: "flow",
436
+ title: flow.title,
437
+ description: flow.description,
438
+ mediaType: "application/json",
439
+ path,
440
+ },
441
+ content: { type: "text", load: async () => flowGraphJson(version) },
442
+ };
443
+ }
444
+ /**
445
+ * The plan of one export: every entry with its path, parents before children.
446
+ *
447
+ * The tree is walked from the chosen root through what `listVisibleSubtree` answered — one
448
+ * statement carrying the same `allowed` predicate every other read uses, so a node without read
449
+ * is neither in the zip nor mentioned in the manifest (#136). Flows ride along by the folder they
450
+ * are filed in, only when the caller holds `flows/read` at all, and their graph is the published
451
+ * version when there is one — the frozen state — falling back to the current draft.
452
+ */
453
+ async function plan(actor, rootId) {
454
+ if (rootId !== null) {
455
+ const root = await deps.repository.getVisible(actor, rootId);
456
+ // An archived node is out of every bundle by decision, and "archived" must not be
457
+ // distinguishable from "absent" through this door either.
458
+ if (!root || root.archivedAt !== null) {
459
+ throw new IntelError(404, "node_not_found", "Node was not found");
460
+ }
461
+ }
462
+ const rows = await deps.repository.listVisibleSubtree(actor, rootId);
463
+ const byParent = new Map();
464
+ const included = new Set(rows.map((row) => row.node.id));
465
+ for (const row of rows) {
466
+ // The export root's own parent is outside the bundle; it anchors at the zip root.
467
+ const key = row.node.id === rootId || !row.node.parentId || !included.has(row.node.parentId)
468
+ ? null
469
+ : row.node.parentId;
470
+ const level = byParent.get(key) ?? [];
471
+ level.push(row);
472
+ byParent.set(key, level);
473
+ }
474
+ const flowsByParent = new Map();
475
+ const flowVersions = new Map();
476
+ if (actor.canReadFlows) {
477
+ const folderIds = new Set(rows.filter((row) => row.node.kind === "folder").map((row) => row.node.id));
478
+ const visible = (await deps.flows.listVisible(asFlowActor(actor))).filter((flow) => flow.parentId === null ? rootId === null : folderIds.has(flow.parentId));
479
+ const versionIds = visible
480
+ .map((flow) => flow.publishedVersionId ?? flow.currentVersionId)
481
+ .filter((id) => id !== null);
482
+ for (const version of await deps.flows.getVersions(versionIds)) {
483
+ flowVersions.set(version.id, version);
484
+ }
485
+ for (const flow of visible) {
486
+ const level = flowsByParent.get(flow.parentId) ?? [];
487
+ level.push(flow);
488
+ flowsByParent.set(flow.parentId, level);
489
+ }
490
+ }
491
+ const entries = [];
492
+ // Folders first, then titles, the order the tree itself draws — so the zip listing reads like
493
+ // the sidebar. Ties fall back to the id to stay deterministic.
494
+ function walk(parentKey, directory) {
495
+ const level = [
496
+ ...(byParent.get(parentKey) ?? []).map((row) => ({ row, flow: null })),
497
+ ...(flowsByParent.get(parentKey) ?? []).map((flow) => ({
498
+ row: null,
499
+ flow,
500
+ })),
501
+ ].sort((left, right) => {
502
+ const leftFolder = left.row?.node.kind === "folder" ? 0 : 1;
503
+ const rightFolder = right.row?.node.kind === "folder" ? 0 : 1;
504
+ const leftTitle = left.row?.node.title ?? left.flow?.title ?? "";
505
+ const rightTitle = right.row?.node.title ?? right.flow?.title ?? "";
506
+ const leftId = left.row?.node.id ?? left.flow?.id ?? "";
507
+ const rightId = right.row?.node.id ?? right.flow?.id ?? "";
508
+ return (leftFolder - rightFolder ||
509
+ leftTitle.toLowerCase().localeCompare(rightTitle.toLowerCase()) ||
510
+ leftId.localeCompare(rightId));
511
+ });
512
+ const used = new Set();
513
+ for (const item of level) {
514
+ if (item.row !== null) {
515
+ const entry = plannedNode(item.row, directory, used);
516
+ entries.push(entry);
517
+ if (item.row.node.kind === "folder") {
518
+ walk(item.row.node.id, entry.manifest.path);
519
+ }
520
+ continue;
521
+ }
522
+ if (item.flow !== null) {
523
+ const versionId = item.flow.publishedVersionId ?? item.flow.currentVersionId;
524
+ entries.push(plannedFlow(item.flow, versionId === null ? undefined : flowVersions.get(versionId), directory, used));
525
+ }
526
+ }
527
+ }
528
+ walk(null, "");
529
+ return entries;
530
+ }
531
+ function manifestOf(rootId, entries) {
532
+ return {
533
+ version: 1,
534
+ exportedAt: deps.now().toISOString(),
535
+ rootId,
536
+ entries: entries.map((entry) => entry.manifest),
537
+ excluded: Excluded,
538
+ };
539
+ }
540
+ /**
541
+ * The zip as a stream. fflate writes into a TransformStream whose readable side is the response
542
+ * body; the pump pushes one entry at a time and waits for `writer.ready` between chunks, so the
543
+ * memory in flight stays one chunk deep however large an attachment is — the whole point of
544
+ * streaming rather than assembling (#136).
545
+ */
546
+ function zipStream(manifest, entries) {
547
+ const { readable, writable } = new TransformStream();
548
+ const writer = writable.getWriter();
549
+ const zip = new Zip((error, chunk, final) => {
550
+ if (error) {
551
+ void writer.abort(error).catch(() => undefined);
552
+ return;
553
+ }
554
+ void writer.write(chunk).catch(() => undefined);
555
+ if (final)
556
+ void writer.close().catch(() => undefined);
557
+ });
558
+ const encoder = new TextEncoder();
559
+ async function pushText(name, text) {
560
+ // Text bodies deflate well and are bounded by the save limit; attachments go through the
561
+ // store path below instead, because recompressing archives and images buys nothing.
562
+ const file = new ZipDeflate(name, { level: 6 });
563
+ zip.add(file);
564
+ file.push(encoder.encode(text), true);
565
+ await writer.ready;
566
+ }
567
+ async function pump() {
568
+ // The manifest goes first so a streaming reader — the import — knows what it is walking into
569
+ // before the first content byte.
570
+ await pushText(BundleManifestFilename, JSON.stringify(manifest, null, 2));
571
+ for (const entry of entries) {
572
+ if (entry.content.type === "folder") {
573
+ const folder = new ZipPassThrough(entry.manifest.path);
574
+ zip.add(folder);
575
+ folder.push(new Uint8Array(0), true);
576
+ await writer.ready;
577
+ continue;
578
+ }
579
+ if (entry.content.type === "text") {
580
+ await pushText(entry.manifest.path, await entry.content.load());
581
+ continue;
582
+ }
583
+ const file = new ZipPassThrough(entry.manifest.path);
584
+ zip.add(file);
585
+ const stream = await entry.content.load();
586
+ if (stream === null) {
587
+ file.push(new Uint8Array(0), true);
588
+ await writer.ready;
589
+ continue;
590
+ }
591
+ const reader = stream.getReader();
592
+ for (;;) {
593
+ const { done, value } = await reader.read();
594
+ if (done)
595
+ break;
596
+ file.push(value);
597
+ await writer.ready;
598
+ }
599
+ file.push(new Uint8Array(0), true);
600
+ await writer.ready;
601
+ }
602
+ zip.end();
603
+ }
604
+ pump().catch((error) => {
605
+ void writer.abort(error).catch(() => undefined);
606
+ });
607
+ return readable;
608
+ }
609
+ // The manifest, when the zip carries one. A file that merely wears the name — a naked folder
610
+ // with somebody's own manifest.json in it — is left to the naked path; a file that clearly
611
+ // means to be a bundle manifest but does not parse is a broken bundle and refuses the import.
612
+ function readImportManifest(files) {
613
+ const raw = files.get(BundleManifestFilename);
614
+ if (raw === undefined)
615
+ return { manifest: null, text: null };
616
+ let json;
617
+ let text;
618
+ try {
619
+ text = decodeText(raw, BundleManifestFilename);
620
+ json = JSON.parse(text);
621
+ }
622
+ catch {
623
+ return { manifest: null, text: null };
624
+ }
625
+ if (!isRecord(json) || json.version === undefined || json.entries === undefined) {
626
+ return { manifest: null, text: null };
627
+ }
628
+ const parsed = BundleManifest.safeParse(json);
629
+ if (!parsed.success) {
630
+ throw new IntelError(400, "import_invalid_manifest", "manifest.json does not match the bundle manifest contract");
631
+ }
632
+ return { manifest: parsed.data, text };
633
+ }
634
+ // The plan a manifest dictates: kinds and titles from its entries, hierarchy from its paths.
635
+ // Sorted by path depth so a parent folder's new ID exists before its children ask for it — the
636
+ // same order the atomic insert needs (see `importTree`).
637
+ function importPlanFromManifest(manifest, files) {
638
+ const sorted = [...manifest.entries].sort((left, right) => depthOf(left.path) - depthOf(right.path));
639
+ const folderIdByPath = new Map();
640
+ const entries = [];
641
+ for (const entry of sorted) {
642
+ requireSafePath(entry.path);
643
+ const isFolder = entry.kind === "folder";
644
+ const parentPath = directoryOf(entry.path);
645
+ const parentNewId = parentPath === "" ? null : (folderIdByPath.get(parentPath) ?? null);
646
+ if (parentPath !== "" && parentNewId === null) {
647
+ throw new IntelError(400, "import_bundle_incomplete", `The manifest names no folder for: ${entry.path}`);
648
+ }
649
+ const newId = deps.id();
650
+ if (isFolder)
651
+ folderIdByPath.set(entry.path.endsWith("/") ? entry.path : `${entry.path}/`, newId);
652
+ let body = null;
653
+ if (!isFolder) {
654
+ const found = files.get(entry.path);
655
+ if (found === undefined) {
656
+ throw new IntelError(400, "import_bundle_incomplete", `The manifest names a file the zip does not carry: ${entry.path}`);
657
+ }
658
+ body = found;
659
+ }
660
+ entries.push({
661
+ kind: entry.kind,
662
+ oldId: entry.id,
663
+ newId,
664
+ parentNewId,
665
+ title: entry.title,
666
+ description: entry.description,
667
+ mediaType: entry.mediaType,
668
+ body,
669
+ path: entry.path,
670
+ });
671
+ }
672
+ return entries;
673
+ }
674
+ // The plan a naked folder allows: hierarchy from the paths, kinds from the extensions (#137),
675
+ // no IDs and therefore nothing to remap. Directories a zip only implies — a file three levels
676
+ // deep without its folders spelled out — are created all the same.
677
+ function importPlanNaked(contents) {
678
+ const folderPaths = new Set();
679
+ const filePaths = [];
680
+ for (const directory of contents.directories) {
681
+ if (isJunkPath(directory))
682
+ continue;
683
+ requireSafePath(directory);
684
+ folderPaths.add(directory.endsWith("/") ? directory : `${directory}/`);
685
+ }
686
+ for (const path of contents.files.keys()) {
687
+ if (isJunkPath(path))
688
+ continue;
689
+ requireSafePath(path);
690
+ filePaths.push(path);
691
+ }
692
+ for (const path of [...folderPaths, ...filePaths]) {
693
+ const segments = path.replace(/\/$/, "").split("/");
694
+ for (let depth = 1; depth < segments.length; depth++) {
695
+ folderPaths.add(`${segments.slice(0, depth).join("/")}/`);
696
+ }
697
+ }
698
+ const folderIdByPath = new Map();
699
+ const entries = [];
700
+ for (const path of [...folderPaths].sort((left, right) => depthOf(left) - depthOf(right))) {
701
+ const newId = deps.id();
702
+ folderIdByPath.set(path, newId);
703
+ const segments = path.replace(/\/$/, "").split("/");
704
+ entries.push({
705
+ kind: "folder",
706
+ oldId: null,
707
+ newId,
708
+ parentNewId: segments.length === 1
709
+ ? null
710
+ : folderIdByPath.get(`${segments.slice(0, -1).join("/")}/`),
711
+ title: segments[segments.length - 1],
712
+ description: null,
713
+ mediaType: null,
714
+ body: null,
715
+ path,
716
+ });
717
+ }
718
+ for (const path of filePaths.sort((left, right) => depthOf(left) - depthOf(right))) {
719
+ const segments = path.split("/");
720
+ const name = segments[segments.length - 1];
721
+ const derived = nakedKindOf(name);
722
+ entries.push({
723
+ kind: derived.kind,
724
+ oldId: null,
725
+ newId: deps.id(),
726
+ parentNewId: segments.length === 1
727
+ ? null
728
+ : folderIdByPath.get(`${segments.slice(0, -1).join("/")}/`),
729
+ title: derived.title.length > 0 ? derived.title.slice(0, 240) : name.slice(0, 240),
730
+ description: null,
731
+ mediaType: derived.mediaType,
732
+ body: contents.files.get(path) ?? null,
733
+ path,
734
+ });
735
+ }
736
+ return entries;
737
+ }
738
+ function replayResult(metadata) {
739
+ return BundleImportResult.parse({
740
+ nodes: typeof metadata.nodes === "number" ? metadata.nodes : 0,
741
+ flows: typeof metadata.flows === "number" ? metadata.flows : 0,
742
+ rootNodeIds: Array.isArray(metadata.rootNodeIds) ? metadata.rootNodeIds : [],
743
+ replayed: true,
744
+ });
745
+ }
746
+ return {
747
+ async exportSubtree(actor, rootId) {
748
+ const entries = await plan(actor, rootId);
749
+ const manifest = manifestOf(rootId, entries);
750
+ const rootEntry = rootId === null ? null : entries.find((entry) => entry.manifest.id === rootId);
751
+ const filename = rootEntry === null || rootEntry === undefined
752
+ ? "intel-export.zip"
753
+ : `${sanitizeName(rootEntry.manifest.title)}.zip`;
754
+ return { filename, stream: zipStream(manifest, entries) };
755
+ },
756
+ async exportFlow(actor, flowId) {
757
+ const flow = await deps.flows.getVisible(asFlowActor(actor), flowId);
758
+ if (!flow || flow.archivedAt !== null) {
759
+ throw new IntelError(404, "flow_not_found", "Flow was not found");
760
+ }
761
+ const versionId = flow.publishedVersionId ?? flow.currentVersionId;
762
+ const version = versionId === null ? undefined : (await deps.flows.getVersions([versionId]))[0];
763
+ const entries = [plannedFlow(flow, version, "", new Set())];
764
+ // A single flow's bundle is rooted at the flow itself: the manifest names it as the root the
765
+ // same way a single node export does.
766
+ const manifest = manifestOf(flow.id, entries);
767
+ return { filename: `${sanitizeName(flow.title)}.zip`, stream: zipStream(manifest, entries) };
768
+ },
769
+ async manifest(actor, rootId) {
770
+ return manifestOf(rootId, await plan(actor, rootId));
771
+ },
772
+ /**
773
+ * One bundle in, one new subtree out (#137, phase 1): always new nodes, never a merge, never
774
+ * an overwrite — a second import of the same bundle is a second subtree, and only the
775
+ * idempotency key makes a RETRY of the same request not be one.
776
+ *
777
+ * The order of operations is the whole safety story: authorize, parse and decide everything,
778
+ * write the R2 bodies, then land every row in ONE `db.batch`. The batch is a transaction, so a
779
+ * half-imported tree cannot exist; R2 objects written for a batch that refused are deleted
780
+ * again, exactly as `save` treats its single object.
781
+ */
782
+ async importBundle(actor, input) {
783
+ const earlier = await deps.repository.findImportReplay(actor.id, input.idempotencyKey);
784
+ if (earlier !== null)
785
+ return replayResult(earlier);
786
+ if (input.targetNodeId !== null) {
787
+ const target = await deps.repository.getVisible(actor, input.targetNodeId);
788
+ if (!target || target.archivedAt !== null) {
789
+ throw new IntelError(404, "node_not_found", "Node was not found");
790
+ }
791
+ if (target.kind !== "folder") {
792
+ throw new IntelError(409, "import_target_not_folder", "Imports land in a folder");
793
+ }
794
+ if (!(await deps.repository.can(actor, target.id, "write"))) {
795
+ throw new IntelError(403, "node_forbidden", "Target folder cannot be edited");
796
+ }
797
+ }
798
+ const contents = await readZip(input.zip);
799
+ const { manifest, text: manifestText } = readImportManifest(contents.files);
800
+ if (manifest !== null)
801
+ contents.files.delete(BundleManifestFilename);
802
+ const entries = manifest !== null
803
+ ? importPlanFromManifest(manifest, contents.files)
804
+ : importPlanNaked(contents);
805
+ if (entries.length === 0) {
806
+ throw new IntelError(400, "import_invalid_bundle", "The bundle holds nothing to import");
807
+ }
808
+ if (entries.some((entry) => entry.kind === "flow") && actor.canCreateFlows !== true) {
809
+ throw new IntelError(403, "permission_required", "Importing a bundle that carries flows needs the flows/create permission");
810
+ }
811
+ const idMap = new Map();
812
+ for (const entry of entries) {
813
+ if (entry.oldId !== null)
814
+ idMap.set(entry.oldId, entry.newId);
815
+ }
816
+ const occurredAt = deps.now().toISOString();
817
+ const nodes = [];
818
+ const versions = [];
819
+ const flows = [];
820
+ const flowVersions = [];
821
+ // Text bodies and media types per new node, kept for the link pass below.
822
+ const documentBodies = new Map();
823
+ const pendingWrites = [];
824
+ function versionRowFor(entry, content, mediaType, contentHash, segment) {
825
+ const versionId = deps.id();
826
+ const contentKey = contentKeyFor(entry.newId, versionId);
827
+ pendingWrites.push({ key: contentKey, mediaType, body: content });
828
+ return {
829
+ id: versionId,
830
+ nodeId: entry.newId,
831
+ sequence: 1,
832
+ contentKey,
833
+ mediaType,
834
+ contentHash,
835
+ size: typeof content === "string"
836
+ ? new TextEncoder().encode(content).byteLength
837
+ : content.byteLength,
838
+ segment,
839
+ createdBy: actor.id,
840
+ createdAt: occurredAt,
841
+ };
842
+ }
843
+ for (const entry of entries) {
844
+ const parentId = entry.parentNewId ?? input.targetNodeId;
845
+ if (entry.kind === "flow") {
846
+ const text = entry.body === null ? "" : decodeText(entry.body, entry.path);
847
+ let versionId = null;
848
+ if (text.trim().length > 0) {
849
+ let graph;
850
+ try {
851
+ graph = remapFlowGraph(FlowGraph.parse(JSON.parse(text)), idMap);
852
+ }
853
+ catch {
854
+ throw new IntelError(400, "import_invalid_flow", `Bundle entry is not a flow graph: ${entry.path}`);
855
+ }
856
+ versionId = deps.id();
857
+ flowVersions.push({
858
+ id: versionId,
859
+ flowId: entry.newId,
860
+ sequence: 1,
861
+ graph,
862
+ createdBy: actor.id,
863
+ createdAt: occurredAt,
864
+ });
865
+ }
866
+ flows.push({
867
+ id: entry.newId,
868
+ parentId,
869
+ title: entry.title,
870
+ description: entry.description,
871
+ ownerId: actor.id,
872
+ currentVersionId: versionId,
873
+ // Publishing froze a decision in the OLD installation; the import hands over a draft
874
+ // and whoever owns the new tree publishes it anew.
875
+ publishedVersionId: null,
876
+ createdAt: occurredAt,
877
+ updatedAt: occurredAt,
878
+ archivedAt: null,
879
+ });
880
+ continue;
881
+ }
882
+ let version = null;
883
+ if (entry.kind === "document") {
884
+ let text = entry.body === null ? "" : decodeText(entry.body, entry.path);
885
+ const mediaType = entry.mediaType ?? "text/markdown";
886
+ if (text.length > 0 && mediaType === BlockNoteMediaType) {
887
+ text = remapDocumentLinks(text, idMap);
888
+ }
889
+ if (text.length > 0) {
890
+ version = versionRowFor(entry, text, mediaType, await deps.hash(text), null);
891
+ documentBodies.set(entry.newId, { mediaType, content: text });
892
+ }
893
+ }
894
+ else if (entry.kind === "table") {
895
+ const text = entry.body === null ? "" : decodeText(entry.body, entry.path);
896
+ if (text.trim().length > 0) {
897
+ const header = parseCsv(text)[0] ?? [];
898
+ if (header.length === 0 || header.every((column) => column.trim().length === 0)) {
899
+ throw new IntelError(400, "import_invalid_table", `Bundle entry is not a table with a header row: ${entry.path}`);
900
+ }
901
+ // The whole file as one snapshot: the same shape a definition or a redefine writes
902
+ // (#135), so reading starts here and nothing older is expected to exist.
903
+ version = versionRowFor(entry, text, TableMediaType, await deps.hash(text), "snapshot");
904
+ }
905
+ }
906
+ else if (entry.kind === "agent") {
907
+ const text = entry.body === null ? "" : decodeText(entry.body, entry.path);
908
+ if (text.trim().length > 0) {
909
+ let definition;
910
+ try {
911
+ definition = AgentDefinition.parse(JSON.parse(text));
912
+ }
913
+ catch {
914
+ throw new IntelError(400, "import_invalid_agent", `Bundle entry is not an agent definition: ${entry.path}`);
915
+ }
916
+ const remapped = {
917
+ ...definition,
918
+ references: definition.references.map((reference) => idMap.has(reference.nodeId)
919
+ ? { ...reference, nodeId: idMap.get(reference.nodeId) }
920
+ : reference),
921
+ schedules: definition.schedules.map((schedule) => idMap.has(schedule.target.id)
922
+ ? {
923
+ ...schedule,
924
+ target: { ...schedule.target, id: idMap.get(schedule.target.id) },
925
+ }
926
+ : schedule),
927
+ };
928
+ const body = JSON.stringify(remapped);
929
+ version = versionRowFor(entry, body, AgentMediaType, await deps.hash(body), null);
930
+ }
931
+ }
932
+ else if (entry.kind === "attachment") {
933
+ const bytes = entry.body ?? new Uint8Array(0);
934
+ version = versionRowFor(entry, bytes, entry.mediaType ?? "application/octet-stream", await deps.hash(bytes), null);
935
+ }
936
+ if (version !== null)
937
+ versions.push(version);
938
+ nodes.push({
939
+ id: entry.newId,
940
+ parentId,
941
+ // Flows took the `continue` above; what reaches here is one of the five node kinds.
942
+ kind: entry.kind,
943
+ title: entry.title,
944
+ description: entry.description,
945
+ ownerId: actor.id,
946
+ currentVersionId: version?.id ?? null,
947
+ createdAt: occurredAt,
948
+ updatedAt: occurredAt,
949
+ archivedAt: null,
950
+ });
951
+ }
952
+ // The link pass: what the imported texts now point at. Targets inside the import are linked
953
+ // as written — they exist by the same transaction. Targets outside go through the same
954
+ // visibility question a save asks, so an import cannot mint a link into a part of the tree
955
+ // its author may not see.
956
+ const links = [];
957
+ const importedIds = new Set(nodes.map((node) => node.id));
958
+ const outsideTargets = new Set();
959
+ for (const [sourceId, body] of documentBodies) {
960
+ for (const target of documentLinkTargets(body.mediaType, body.content)) {
961
+ if (target === sourceId)
962
+ continue;
963
+ if (!importedIds.has(target))
964
+ outsideTargets.add(target);
965
+ }
966
+ }
967
+ const visibleOutside = new Set(outsideTargets.size === 0
968
+ ? []
969
+ : (await deps.repository.resolveVisibleTitles(actor, [...outsideTargets])).map((row) => row.nodeId));
970
+ for (const [sourceId, body] of documentBodies) {
971
+ for (const target of documentLinkTargets(body.mediaType, body.content)) {
972
+ if (target === sourceId)
973
+ continue;
974
+ if (importedIds.has(target) || visibleOutside.has(target)) {
975
+ links.push({ id: deps.id(), sourceNodeId: sourceId, targetNodeId: target });
976
+ }
977
+ }
978
+ }
979
+ const rootNodeIds = nodes
980
+ .filter((node) => node.parentId === input.targetNodeId)
981
+ .map((node) => node.id);
982
+ const metadata = {
983
+ manifestHash: await deps.hash(manifestText ?? [...contents.files.keys()].sort().join("\n")),
984
+ nodes: nodes.length,
985
+ flows: flows.length,
986
+ rootNodeIds,
987
+ };
988
+ // R2 first, rows second, cleanup on refusal — the same discipline every single-version
989
+ // write in `nodes.ts` follows, over many objects at once.
990
+ const written = [];
991
+ try {
992
+ for (const write of pendingWrites) {
993
+ if (typeof write.body === "string") {
994
+ await deps.content.put(write.key, write.body, write.mediaType);
995
+ }
996
+ else {
997
+ const copy = new Uint8Array(write.body.byteLength);
998
+ copy.set(write.body);
999
+ await deps.content.putBytes(write.key, copy.buffer, write.mediaType);
1000
+ }
1001
+ written.push(write.key);
1002
+ }
1003
+ const outcome = await deps.repository.importTree({
1004
+ nodes,
1005
+ versions,
1006
+ links,
1007
+ flows,
1008
+ flowVersions,
1009
+ actorId: actor.id,
1010
+ idempotencyKey: input.idempotencyKey,
1011
+ auditId: deps.id(),
1012
+ auditResourceId: input.targetNodeId ?? "root",
1013
+ metadata,
1014
+ occurredAt,
1015
+ });
1016
+ if (outcome === "replayed") {
1017
+ // A racing retry landed first; these objects belong to nobody's rows.
1018
+ await Promise.all(written.map(async (key) => await deps.content.delete(key).catch(() => undefined)));
1019
+ const stored = await deps.repository.findImportReplay(actor.id, input.idempotencyKey);
1020
+ return replayResult(stored ?? {});
1021
+ }
1022
+ }
1023
+ catch (error) {
1024
+ await Promise.all(written.map(async (key) => await deps.content.delete(key).catch(() => undefined)));
1025
+ throw error;
1026
+ }
1027
+ // Derived indexes only after the canonical write stands (#137): the queue reads D1 and R2,
1028
+ // and both now hold what it will find.
1029
+ for (const version of versions) {
1030
+ await deps.indexing.enqueue(version.id);
1031
+ }
1032
+ return { nodes: nodes.length, flows: flows.length, rootNodeIds, replayed: false };
1033
+ },
1034
+ };
1035
+ }