@tangle-network/agent-app 0.60.110 → 0.60.112

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -410,34 +410,54 @@ function paxPath(record) {
410
410
  }
411
411
  return void 0;
412
412
  }
413
- function createTarInspector(stats) {
413
+ function createTarInspector(stats, exclude = []) {
414
+ const excluded = new Set(exclude);
414
415
  const header = new Uint8Array(BLOCK);
415
416
  let headerFill = 0;
416
417
  let dataRemaining = 0;
417
418
  let paddingRemaining = 0;
419
+ let mode = "pass";
418
420
  let blanking = false;
419
421
  let capture = null;
420
422
  let pendingName;
423
+ let held = [];
421
424
  let ended = false;
422
- function onHeader() {
423
- if (header.every((b) => b === 0)) {
425
+ const isExcluded = (name) => excluded.size > 0 && name.split("/").some((segment) => excluded.has(segment));
426
+ function onHeader(emit) {
427
+ const block = header.slice();
428
+ if (block.every((b) => b === 0)) {
424
429
  ended = true;
430
+ for (const part of held) emit(part);
431
+ held = [];
432
+ emit(block);
425
433
  return;
426
434
  }
427
- const type = String.fromCharCode(header[156]);
428
- const size = readSize(header);
435
+ const type = String.fromCharCode(block[156]);
436
+ const size = readSize(block);
429
437
  dataRemaining = size;
430
438
  paddingRemaining = (BLOCK - size % BLOCK) % BLOCK;
431
439
  blanking = false;
432
440
  capture = null;
433
441
  if ((type === "L" || type === "x") && size <= MAX_NAME_RECORD) {
434
442
  capture = { kind: type, parts: [], length: 0 };
443
+ held.push(block);
444
+ mode = "hold";
435
445
  return;
436
446
  }
437
- const prefix = header[257] === 117 ? readString(header, 345, 155) : "";
438
- const base = readString(header, 0, 100);
447
+ const prefix = block[257] === 117 ? readString(block, 345, 155) : "";
448
+ const base = readString(block, 0, 100);
439
449
  const name = pendingName ?? (prefix ? `${prefix}/${base}` : base);
440
450
  pendingName = void 0;
451
+ if (isExcluded(name)) {
452
+ held = [];
453
+ mode = "drop";
454
+ if (type === "0" || type === "\0" || type === "7") stats.excluded = (stats.excluded ?? 0) + 1;
455
+ return;
456
+ }
457
+ for (const part of held) emit(part);
458
+ held = [];
459
+ emit(block);
460
+ mode = "pass";
441
461
  if (type === "0" || type === "\0" || type === "7") {
442
462
  stats.files += 1;
443
463
  stats.bytes += size;
@@ -449,16 +469,27 @@ function createTarInspector(stats) {
449
469
  }
450
470
  return new TransformStream({
451
471
  transform(chunk, controller) {
452
- const out = chunk.slice();
472
+ const out = [];
473
+ const emit = (bytes) => out.push(bytes);
474
+ const route = (bytes) => {
475
+ if (mode === "pass") emit(bytes);
476
+ else if (mode === "hold") held.push(bytes);
477
+ };
453
478
  let at = 0;
454
- while (at < out.length && !ended) {
479
+ while (at < chunk.length) {
480
+ if (ended) {
481
+ emit(chunk.slice(at));
482
+ break;
483
+ }
455
484
  if (dataRemaining > 0) {
456
- const take2 = Math.min(dataRemaining, out.length - at);
457
- if (blanking) out.fill(42, at, at + take2);
485
+ const take2 = Math.min(dataRemaining, chunk.length - at);
486
+ const data = chunk.slice(at, at + take2);
487
+ if (blanking) data.fill(42);
458
488
  if (capture) {
459
- capture.parts.push(out.slice(at, at + take2));
489
+ capture.parts.push(data);
460
490
  capture.length += take2;
461
491
  }
492
+ route(data);
462
493
  dataRemaining -= take2;
463
494
  at += take2;
464
495
  if (dataRemaining === 0 && capture) {
@@ -475,21 +506,36 @@ function createTarInspector(stats) {
475
506
  continue;
476
507
  }
477
508
  if (paddingRemaining > 0) {
478
- const take2 = Math.min(paddingRemaining, out.length - at);
509
+ const take2 = Math.min(paddingRemaining, chunk.length - at);
510
+ route(chunk.slice(at, at + take2));
479
511
  paddingRemaining -= take2;
480
512
  at += take2;
481
513
  continue;
482
514
  }
483
- const take = Math.min(BLOCK - headerFill, out.length - at);
484
- header.set(out.subarray(at, at + take), headerFill);
515
+ const take = Math.min(BLOCK - headerFill, chunk.length - at);
516
+ header.set(chunk.subarray(at, at + take), headerFill);
485
517
  headerFill += take;
486
518
  at += take;
487
519
  if (headerFill === BLOCK) {
488
520
  headerFill = 0;
489
- onHeader();
521
+ onHeader(emit);
490
522
  }
491
523
  }
492
- controller.enqueue(out);
524
+ if (out.length === 1) controller.enqueue(out[0]);
525
+ else if (out.length) {
526
+ const length = out.reduce((n, part) => n + part.length, 0);
527
+ const joined = new Uint8Array(length);
528
+ let offset = 0;
529
+ for (const part of out) {
530
+ joined.set(part, offset);
531
+ offset += part.length;
532
+ }
533
+ controller.enqueue(joined);
534
+ }
535
+ },
536
+ flush(controller) {
537
+ for (const part of held) controller.enqueue(part);
538
+ held = [];
493
539
  }
494
540
  });
495
541
  }
@@ -673,6 +719,19 @@ var FILES_IN_PARALLEL = 4;
673
719
  var SOURCE_NAME = /^[a-z0-9][a-z0-9_.-]{0,95}$/;
674
720
  var LeaseLost = class extends Error {
675
721
  };
722
+ var ExportRetryLater = class extends Error {
723
+ retryLater = true;
724
+ };
725
+ var MAX_DEFERRALS = 20;
726
+ function isRetryLater(err) {
727
+ return typeof err === "object" && err !== null && err.retryLater === true;
728
+ }
729
+ function csvCell(value) {
730
+ if (value === null || value === void 0) return "";
731
+ const text = typeof value === "string" ? value : typeof value === "object" ? JSON.stringify(value) : String(value);
732
+ const safe = /^[=+\-@\t\r]/.test(text) ? `'${text}` : text;
733
+ return /[",\r\n]/.test(safe) ? `"${safe.replace(/"/g, '""')}"` : safe;
734
+ }
676
735
  function iso(ms) {
677
736
  return new Date(ms).toISOString();
678
737
  }
@@ -836,21 +895,36 @@ function createExportEngine(options) {
836
895
  if (source.kind === "rows") {
837
896
  const omit = new Set(source.omitColumns ?? []);
838
897
  unit.expectedRows = source.expectedCount ? await source.expectedCount() : void 0;
898
+ const csv = source.format === "csv";
899
+ const columns = source.columns ?? [];
900
+ if (csv && !columns.length) throw new Error(`workspace-export: csv source "${source.name}" needs columns`);
839
901
  let rows = 0;
840
902
  const iterator = source.rows()[Symbol.asyncIterator]();
903
+ let first = await iterator.next();
904
+ while (!first.done && Array.isArray(first.value) && first.value.length === 0) first = await iterator.next();
905
+ if (first.done) {
906
+ unit.rows = 0;
907
+ unit.status = "empty";
908
+ return;
909
+ }
910
+ let header = csv ? `${columns.map(csvCell).join(",")}\r
911
+ ` : "";
841
912
  const body = new ReadableStream({
842
913
  async pull(controller) {
843
- const { done, value } = await iterator.next();
914
+ const { done, value } = first.done ? await iterator.next() : first;
915
+ first = { done: true, value: void 0 };
844
916
  if (done) {
845
917
  controller.close();
846
918
  return;
847
919
  }
848
920
  const page = Array.isArray(value) ? value : [value];
849
- let text = "";
921
+ let text = header;
922
+ header = "";
850
923
  for (const row of page) {
851
924
  const redacted = redactRow(row, omit, scanner);
852
925
  unit.redactions += redacted.count;
853
- text += `${JSON.stringify(redacted.row)}
926
+ text += csv ? `${columns.map((column) => csvCell(redacted.row[column])).join(",")}\r
927
+ ` : `${JSON.stringify(redacted.row)}
854
928
  `;
855
929
  rows += 1;
856
930
  }
@@ -860,7 +934,8 @@ function createExportEngine(options) {
860
934
  await iterator.return?.();
861
935
  }
862
936
  });
863
- unit.entries.push(await writeEntry(`${keyBase}/0`, `data/${source.name}.ndjson`, body, true));
937
+ const path = csv ? `spreadsheets/${source.name}.csv` : `data/${source.name}.ndjson`;
938
+ unit.entries.push(await writeEntry(`${keyBase}/0`, path, body, true));
864
939
  unit.rows = rows;
865
940
  unit.status = "done";
866
941
  return;
@@ -921,7 +996,7 @@ function createExportEngine(options) {
921
996
  return;
922
997
  }
923
998
  const stats = { files: 0, bytes: 0, blanked: [] };
924
- const tarGz = archive.pipeThrough(new DecompressionStream("gzip")).pipeThrough(createTarInspector(stats)).pipeThrough(scanner.maskBytes((n) => unit.redactions += n)).pipeThrough(new CompressionStream("gzip"));
999
+ const tarGz = archive.pipeThrough(new DecompressionStream("gzip")).pipeThrough(createTarInspector(stats, source.exclude)).pipeThrough(scanner.maskBytes((n) => unit.redactions += n)).pipeThrough(new CompressionStream("gzip"));
925
1000
  unit.entries.push(await writeEntry(`${keyBase}/0`, `sandbox/${source.name}.tar.gz`, tarGz, false));
926
1001
  unit.archive = stats;
927
1002
  unit.files = stats.files;
@@ -968,23 +1043,31 @@ function createExportEngine(options) {
968
1043
  };
969
1044
  }
970
1045
  function readme(job) {
971
- return [
972
- `${options.app} workspace export`,
973
- "",
974
- `Workspace: ${job.workspaceId}`,
975
- `Export: ${job.id}`,
976
- `Created: ${job.createdAt}`,
977
- "",
978
- "manifest.json what this archive contains: every file with its size and SHA-256, row counts per source, and the redaction policy",
979
- "data/*.ndjson one JSON object per line, one file per table or record type",
980
- "data/*.json single documents",
981
- "files/<source>/ uploaded and generated files, at their original paths",
982
- "sandbox/*.tar.gz the agent workspace filesystem; extract with `tar -xzf`",
1046
+ const entries = job.units.flatMap((u) => u.entries.map((e) => e.path));
1047
+ const count = (folder) => entries.filter((path) => path.startsWith(folder)).length;
1048
+ const folders = [
1049
+ ["spreadsheets/", "tables to open in a spreadsheet app, one row per record"],
1050
+ ["files/", "documents and uploads at their original names, one folder per source"],
1051
+ ["data/", "every database table that holds data, one JSON object per line (.ndjson) or one document (.json)"],
1052
+ ["sandbox/", "the agent's working files; extract with `tar -xzf`"]
1053
+ ];
1054
+ const lines = [`${options.app} workspace export`, "", `Workspace: ${job.workspaceId}`, `Export: ${job.id}`, `Created: ${job.createdAt}`, ""];
1055
+ for (const [folder, what] of folders) {
1056
+ const n = count(folder);
1057
+ if (n) lines.push(`${folder.padEnd(14)}${n} file${n === 1 ? "" : "s"}: ${what}`);
1058
+ }
1059
+ const fileSources = [...new Set(entries.filter((p) => p.startsWith("files/")).map((p) => p.split("/")[1]))];
1060
+ if (fileSources.length) lines.push("", `Files: ${fileSources.map((name) => `files/${name}/`).join(", ")}`);
1061
+ const empty = job.units.filter((u) => u.status === "empty").length;
1062
+ if (empty) lines.push("", `${empty} source${empty === 1 ? " was" : "s were"} empty and wrote no file; manifest.json lists every source.`);
1063
+ lines.push(
983
1064
  "",
1065
+ "manifest.json lists every file with its size and SHA-256, row counts per source, and the redaction policy.",
984
1066
  "Verify a file: `shasum -a 256 <path>` and compare with manifest.json.",
985
1067
  'Secrets are never exported. Credential fields read "[redacted]".',
986
1068
  ""
987
- ].join("\n");
1069
+ );
1070
+ return lines.join("\n");
988
1071
  }
989
1072
  async function finalize(job, plan, scanner) {
990
1073
  job.completedAt = iso(now());
@@ -1121,6 +1204,12 @@ function createExportEngine(options) {
1121
1204
  return false;
1122
1205
  } catch (err) {
1123
1206
  unit2.error = errorText(scanner, err);
1207
+ if (isRetryLater(err) && (unit2.deferrals ?? 0) < MAX_DEFERRALS) {
1208
+ unit2.deferrals = (unit2.deferrals ?? 0) + 1;
1209
+ unit2.attempts -= 1;
1210
+ if (unit2.cursor === void 0) unit2.entries = [];
1211
+ return "deferred";
1212
+ }
1124
1213
  if (unit2.cursor === void 0) unit2.entries = [];
1125
1214
  if (unit2.attempts >= MAX_ATTEMPTS) unit2.status = "failed";
1126
1215
  return true;
@@ -1136,6 +1225,10 @@ function createExportEngine(options) {
1136
1225
  await save();
1137
1226
  break;
1138
1227
  }
1228
+ if (failed.includes("deferred")) {
1229
+ await save();
1230
+ continue;
1231
+ }
1139
1232
  const { unit } = batch[0];
1140
1233
  if (unit.status === "pending") {
1141
1234
  unit.attempts = 0;
@@ -1180,7 +1273,8 @@ function createExportEngine(options) {
1180
1273
  completedAt: job.completedAt,
1181
1274
  error: job.error,
1182
1275
  unitsTotal: job.units.length,
1183
- unitsDone: job.units.filter((u) => u.status !== "pending").length,
1276
+ unitsDone: job.units.filter((u) => u.status === "done" || u.status === "empty").length,
1277
+ unitsFailed: job.units.filter((u) => u.status === "failed").length,
1184
1278
  current: job.status === "running" ? job.units.find((u) => u.status === "pending")?.name : void 0,
1185
1279
  bytes: job.units.reduce((n, u) => n + bytesOf(u), 0),
1186
1280
  units: job.units.map((u) => ({
@@ -1600,31 +1694,70 @@ function kvFiles(args) {
1600
1694
 
1601
1695
  // src/workspace-export/sandbox.ts
1602
1696
  var GONE = /* @__PURE__ */ new Set(["deleted", "expired", "failed"]);
1697
+ var DEFAULT_SANDBOX_EXCLUDE = ["node_modules", ".sidecar"];
1698
+ function isLifecycleBusy(err) {
1699
+ const e = err;
1700
+ return typeof e === "object" && e !== null && e.status === 409 && e.code === "SANDBOX_LIFECYCLE_IN_PROGRESS";
1701
+ }
1702
+ async function openRunning(getBox, timeoutMs) {
1703
+ const box = await getBox();
1704
+ if (!box || GONE.has(box.status)) return null;
1705
+ const wasStopped = box.status === "stopped";
1706
+ try {
1707
+ if (wasStopped) await box.resume();
1708
+ if (box.status !== "running" || wasStopped) await box.waitFor("running", { timeoutMs });
1709
+ } catch (err) {
1710
+ if (isLifecycleBusy(err)) throw new ExportRetryLater("the sandbox is busy with another operation; retrying shortly");
1711
+ throw err;
1712
+ }
1713
+ const restore = async () => {
1714
+ if (wasStopped && box.stop) await box.stop().catch(() => {
1715
+ });
1716
+ };
1717
+ return { box, restore };
1718
+ }
1603
1719
  function sandboxArchive(args) {
1604
1720
  return {
1605
1721
  kind: "archive",
1606
1722
  name: args.name,
1607
1723
  dataClass: args.dataClass ?? "sandbox",
1608
1724
  description: args.description,
1725
+ exclude: args.exclude ?? DEFAULT_SANDBOX_EXCLUDE,
1609
1726
  async open() {
1610
- const box = await args.getBox();
1611
- if (!box || GONE.has(box.status)) return null;
1612
- const wasStopped = box.status === "stopped";
1613
- if (wasStopped) await box.resume();
1614
- if (box.status !== "running" || wasStopped) await box.waitFor("running", { timeoutMs: args.resumeTimeoutMs ?? 12e4 });
1615
- const archive = await box.fs.openArchive(args.dir ?? ".");
1616
- if (!wasStopped || !box.stop) return archive;
1617
- const restore = () => box.stop().catch(() => {
1618
- });
1619
- return archive.pipeThrough(
1620
- new TransformStream({
1621
- flush: () => restore()
1622
- })
1623
- );
1727
+ const opened = await openRunning(args.getBox, args.resumeTimeoutMs ?? 12e4);
1728
+ if (!opened) return null;
1729
+ const archive = await opened.box.fs.openArchive(args.dir ?? ".");
1730
+ return archive.pipeThrough(new TransformStream({ flush: () => opened.restore() }));
1731
+ }
1732
+ };
1733
+ }
1734
+ function sandboxFiles(args) {
1735
+ return {
1736
+ kind: "files",
1737
+ name: args.name,
1738
+ dataClass: args.dataClass ?? "knowledge",
1739
+ description: args.description,
1740
+ async *files() {
1741
+ const listed = await args.list();
1742
+ if (!listed.length) return;
1743
+ const opened = await openRunning(args.getBox, args.resumeTimeoutMs ?? 12e4);
1744
+ if (!opened) return;
1745
+ const read = opened.box.fs.read;
1746
+ if (!read) throw new Error("workspace-export: this sandbox client cannot read files");
1747
+ try {
1748
+ for (const file of listed) {
1749
+ const text = await read.call(opened.box.fs, file.boxPath);
1750
+ yield { path: file.path, open: async () => text };
1751
+ }
1752
+ } finally {
1753
+ await opened.restore();
1754
+ }
1624
1755
  }
1625
1756
  };
1626
1757
  }
1627
1758
  export {
1759
+ DEFAULT_SANDBOX_EXCLUDE,
1760
+ ExportRetryLater,
1628
1761
  MANIFEST_SCHEMA,
1629
1762
  REDACTED,
1630
1763
  createExportEngine,
@@ -1637,6 +1770,7 @@ export {
1637
1770
  isSecretName,
1638
1771
  kvFiles,
1639
1772
  r2Files,
1640
- sandboxArchive
1773
+ sandboxArchive,
1774
+ sandboxFiles
1641
1775
  };
1642
1776
  //# sourceMappingURL=index.js.map