@tangle-network/agent-app 0.60.110 → 0.60.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/teams-react/index.js +8 -8
- package/dist/teams-react/index.js.map +1 -1
- package/dist/workspace-export/engine.d.ts +8 -0
- package/dist/workspace-export/index.d.ts +2 -2
- package/dist/workspace-export/index.js +185 -51
- package/dist/workspace-export/index.js.map +1 -1
- package/dist/workspace-export/react.js +4 -2
- package/dist/workspace-export/react.js.map +1 -1
- package/dist/workspace-export/sandbox.d.ts +37 -8
- package/dist/workspace-export/tar.d.ts +10 -6
- package/dist/workspace-export/types.d.ts +15 -2
- package/package.json +1 -1
|
@@ -410,34 +410,54 @@ function paxPath(record) {
|
|
|
410
410
|
}
|
|
411
411
|
return void 0;
|
|
412
412
|
}
|
|
413
|
-
function createTarInspector(stats) {
|
|
413
|
+
function createTarInspector(stats, exclude = []) {
|
|
414
|
+
const excluded = new Set(exclude);
|
|
414
415
|
const header = new Uint8Array(BLOCK);
|
|
415
416
|
let headerFill = 0;
|
|
416
417
|
let dataRemaining = 0;
|
|
417
418
|
let paddingRemaining = 0;
|
|
419
|
+
let mode = "pass";
|
|
418
420
|
let blanking = false;
|
|
419
421
|
let capture = null;
|
|
420
422
|
let pendingName;
|
|
423
|
+
let held = [];
|
|
421
424
|
let ended = false;
|
|
422
|
-
|
|
423
|
-
|
|
425
|
+
const isExcluded = (name) => excluded.size > 0 && name.split("/").some((segment) => excluded.has(segment));
|
|
426
|
+
function onHeader(emit) {
|
|
427
|
+
const block = header.slice();
|
|
428
|
+
if (block.every((b) => b === 0)) {
|
|
424
429
|
ended = true;
|
|
430
|
+
for (const part of held) emit(part);
|
|
431
|
+
held = [];
|
|
432
|
+
emit(block);
|
|
425
433
|
return;
|
|
426
434
|
}
|
|
427
|
-
const type = String.fromCharCode(
|
|
428
|
-
const size = readSize(
|
|
435
|
+
const type = String.fromCharCode(block[156]);
|
|
436
|
+
const size = readSize(block);
|
|
429
437
|
dataRemaining = size;
|
|
430
438
|
paddingRemaining = (BLOCK - size % BLOCK) % BLOCK;
|
|
431
439
|
blanking = false;
|
|
432
440
|
capture = null;
|
|
433
441
|
if ((type === "L" || type === "x") && size <= MAX_NAME_RECORD) {
|
|
434
442
|
capture = { kind: type, parts: [], length: 0 };
|
|
443
|
+
held.push(block);
|
|
444
|
+
mode = "hold";
|
|
435
445
|
return;
|
|
436
446
|
}
|
|
437
|
-
const prefix =
|
|
438
|
-
const base = readString(
|
|
447
|
+
const prefix = block[257] === 117 ? readString(block, 345, 155) : "";
|
|
448
|
+
const base = readString(block, 0, 100);
|
|
439
449
|
const name = pendingName ?? (prefix ? `${prefix}/${base}` : base);
|
|
440
450
|
pendingName = void 0;
|
|
451
|
+
if (isExcluded(name)) {
|
|
452
|
+
held = [];
|
|
453
|
+
mode = "drop";
|
|
454
|
+
if (type === "0" || type === "\0" || type === "7") stats.excluded = (stats.excluded ?? 0) + 1;
|
|
455
|
+
return;
|
|
456
|
+
}
|
|
457
|
+
for (const part of held) emit(part);
|
|
458
|
+
held = [];
|
|
459
|
+
emit(block);
|
|
460
|
+
mode = "pass";
|
|
441
461
|
if (type === "0" || type === "\0" || type === "7") {
|
|
442
462
|
stats.files += 1;
|
|
443
463
|
stats.bytes += size;
|
|
@@ -449,16 +469,27 @@ function createTarInspector(stats) {
|
|
|
449
469
|
}
|
|
450
470
|
return new TransformStream({
|
|
451
471
|
transform(chunk, controller) {
|
|
452
|
-
const out =
|
|
472
|
+
const out = [];
|
|
473
|
+
const emit = (bytes) => out.push(bytes);
|
|
474
|
+
const route = (bytes) => {
|
|
475
|
+
if (mode === "pass") emit(bytes);
|
|
476
|
+
else if (mode === "hold") held.push(bytes);
|
|
477
|
+
};
|
|
453
478
|
let at = 0;
|
|
454
|
-
while (at <
|
|
479
|
+
while (at < chunk.length) {
|
|
480
|
+
if (ended) {
|
|
481
|
+
emit(chunk.slice(at));
|
|
482
|
+
break;
|
|
483
|
+
}
|
|
455
484
|
if (dataRemaining > 0) {
|
|
456
|
-
const take2 = Math.min(dataRemaining,
|
|
457
|
-
|
|
485
|
+
const take2 = Math.min(dataRemaining, chunk.length - at);
|
|
486
|
+
const data = chunk.slice(at, at + take2);
|
|
487
|
+
if (blanking) data.fill(42);
|
|
458
488
|
if (capture) {
|
|
459
|
-
capture.parts.push(
|
|
489
|
+
capture.parts.push(data);
|
|
460
490
|
capture.length += take2;
|
|
461
491
|
}
|
|
492
|
+
route(data);
|
|
462
493
|
dataRemaining -= take2;
|
|
463
494
|
at += take2;
|
|
464
495
|
if (dataRemaining === 0 && capture) {
|
|
@@ -475,21 +506,36 @@ function createTarInspector(stats) {
|
|
|
475
506
|
continue;
|
|
476
507
|
}
|
|
477
508
|
if (paddingRemaining > 0) {
|
|
478
|
-
const take2 = Math.min(paddingRemaining,
|
|
509
|
+
const take2 = Math.min(paddingRemaining, chunk.length - at);
|
|
510
|
+
route(chunk.slice(at, at + take2));
|
|
479
511
|
paddingRemaining -= take2;
|
|
480
512
|
at += take2;
|
|
481
513
|
continue;
|
|
482
514
|
}
|
|
483
|
-
const take = Math.min(BLOCK - headerFill,
|
|
484
|
-
header.set(
|
|
515
|
+
const take = Math.min(BLOCK - headerFill, chunk.length - at);
|
|
516
|
+
header.set(chunk.subarray(at, at + take), headerFill);
|
|
485
517
|
headerFill += take;
|
|
486
518
|
at += take;
|
|
487
519
|
if (headerFill === BLOCK) {
|
|
488
520
|
headerFill = 0;
|
|
489
|
-
onHeader();
|
|
521
|
+
onHeader(emit);
|
|
490
522
|
}
|
|
491
523
|
}
|
|
492
|
-
controller.enqueue(out);
|
|
524
|
+
if (out.length === 1) controller.enqueue(out[0]);
|
|
525
|
+
else if (out.length) {
|
|
526
|
+
const length = out.reduce((n, part) => n + part.length, 0);
|
|
527
|
+
const joined = new Uint8Array(length);
|
|
528
|
+
let offset = 0;
|
|
529
|
+
for (const part of out) {
|
|
530
|
+
joined.set(part, offset);
|
|
531
|
+
offset += part.length;
|
|
532
|
+
}
|
|
533
|
+
controller.enqueue(joined);
|
|
534
|
+
}
|
|
535
|
+
},
|
|
536
|
+
flush(controller) {
|
|
537
|
+
for (const part of held) controller.enqueue(part);
|
|
538
|
+
held = [];
|
|
493
539
|
}
|
|
494
540
|
});
|
|
495
541
|
}
|
|
@@ -673,6 +719,19 @@ var FILES_IN_PARALLEL = 4;
|
|
|
673
719
|
var SOURCE_NAME = /^[a-z0-9][a-z0-9_.-]{0,95}$/;
|
|
674
720
|
var LeaseLost = class extends Error {
|
|
675
721
|
};
|
|
722
|
+
var ExportRetryLater = class extends Error {
|
|
723
|
+
retryLater = true;
|
|
724
|
+
};
|
|
725
|
+
var MAX_DEFERRALS = 20;
|
|
726
|
+
function isRetryLater(err) {
|
|
727
|
+
return typeof err === "object" && err !== null && err.retryLater === true;
|
|
728
|
+
}
|
|
729
|
+
function csvCell(value) {
|
|
730
|
+
if (value === null || value === void 0) return "";
|
|
731
|
+
const text = typeof value === "string" ? value : typeof value === "object" ? JSON.stringify(value) : String(value);
|
|
732
|
+
const safe = /^[=+\-@\t\r]/.test(text) ? `'${text}` : text;
|
|
733
|
+
return /[",\r\n]/.test(safe) ? `"${safe.replace(/"/g, '""')}"` : safe;
|
|
734
|
+
}
|
|
676
735
|
function iso(ms) {
|
|
677
736
|
return new Date(ms).toISOString();
|
|
678
737
|
}
|
|
@@ -836,21 +895,36 @@ function createExportEngine(options) {
|
|
|
836
895
|
if (source.kind === "rows") {
|
|
837
896
|
const omit = new Set(source.omitColumns ?? []);
|
|
838
897
|
unit.expectedRows = source.expectedCount ? await source.expectedCount() : void 0;
|
|
898
|
+
const csv = source.format === "csv";
|
|
899
|
+
const columns = source.columns ?? [];
|
|
900
|
+
if (csv && !columns.length) throw new Error(`workspace-export: csv source "${source.name}" needs columns`);
|
|
839
901
|
let rows = 0;
|
|
840
902
|
const iterator = source.rows()[Symbol.asyncIterator]();
|
|
903
|
+
let first = await iterator.next();
|
|
904
|
+
while (!first.done && Array.isArray(first.value) && first.value.length === 0) first = await iterator.next();
|
|
905
|
+
if (first.done) {
|
|
906
|
+
unit.rows = 0;
|
|
907
|
+
unit.status = "empty";
|
|
908
|
+
return;
|
|
909
|
+
}
|
|
910
|
+
let header = csv ? `${columns.map(csvCell).join(",")}\r
|
|
911
|
+
` : "";
|
|
841
912
|
const body = new ReadableStream({
|
|
842
913
|
async pull(controller) {
|
|
843
|
-
const { done, value } = await iterator.next();
|
|
914
|
+
const { done, value } = first.done ? await iterator.next() : first;
|
|
915
|
+
first = { done: true, value: void 0 };
|
|
844
916
|
if (done) {
|
|
845
917
|
controller.close();
|
|
846
918
|
return;
|
|
847
919
|
}
|
|
848
920
|
const page = Array.isArray(value) ? value : [value];
|
|
849
|
-
let text =
|
|
921
|
+
let text = header;
|
|
922
|
+
header = "";
|
|
850
923
|
for (const row of page) {
|
|
851
924
|
const redacted = redactRow(row, omit, scanner);
|
|
852
925
|
unit.redactions += redacted.count;
|
|
853
|
-
text += `${
|
|
926
|
+
text += csv ? `${columns.map((column) => csvCell(redacted.row[column])).join(",")}\r
|
|
927
|
+
` : `${JSON.stringify(redacted.row)}
|
|
854
928
|
`;
|
|
855
929
|
rows += 1;
|
|
856
930
|
}
|
|
@@ -860,7 +934,8 @@ function createExportEngine(options) {
|
|
|
860
934
|
await iterator.return?.();
|
|
861
935
|
}
|
|
862
936
|
});
|
|
863
|
-
|
|
937
|
+
const path = csv ? `spreadsheets/${source.name}.csv` : `data/${source.name}.ndjson`;
|
|
938
|
+
unit.entries.push(await writeEntry(`${keyBase}/0`, path, body, true));
|
|
864
939
|
unit.rows = rows;
|
|
865
940
|
unit.status = "done";
|
|
866
941
|
return;
|
|
@@ -921,7 +996,7 @@ function createExportEngine(options) {
|
|
|
921
996
|
return;
|
|
922
997
|
}
|
|
923
998
|
const stats = { files: 0, bytes: 0, blanked: [] };
|
|
924
|
-
const tarGz = archive.pipeThrough(new DecompressionStream("gzip")).pipeThrough(createTarInspector(stats)).pipeThrough(scanner.maskBytes((n) => unit.redactions += n)).pipeThrough(new CompressionStream("gzip"));
|
|
999
|
+
const tarGz = archive.pipeThrough(new DecompressionStream("gzip")).pipeThrough(createTarInspector(stats, source.exclude)).pipeThrough(scanner.maskBytes((n) => unit.redactions += n)).pipeThrough(new CompressionStream("gzip"));
|
|
925
1000
|
unit.entries.push(await writeEntry(`${keyBase}/0`, `sandbox/${source.name}.tar.gz`, tarGz, false));
|
|
926
1001
|
unit.archive = stats;
|
|
927
1002
|
unit.files = stats.files;
|
|
@@ -968,23 +1043,31 @@ function createExportEngine(options) {
|
|
|
968
1043
|
};
|
|
969
1044
|
}
|
|
970
1045
|
function readme(job) {
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
"",
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
1046
|
+
const entries = job.units.flatMap((u) => u.entries.map((e) => e.path));
|
|
1047
|
+
const count = (folder) => entries.filter((path) => path.startsWith(folder)).length;
|
|
1048
|
+
const folders = [
|
|
1049
|
+
["spreadsheets/", "tables to open in a spreadsheet app, one row per record"],
|
|
1050
|
+
["files/", "documents and uploads at their original names, one folder per source"],
|
|
1051
|
+
["data/", "every database table that holds data, one JSON object per line (.ndjson) or one document (.json)"],
|
|
1052
|
+
["sandbox/", "the agent's working files; extract with `tar -xzf`"]
|
|
1053
|
+
];
|
|
1054
|
+
const lines = [`${options.app} workspace export`, "", `Workspace: ${job.workspaceId}`, `Export: ${job.id}`, `Created: ${job.createdAt}`, ""];
|
|
1055
|
+
for (const [folder, what] of folders) {
|
|
1056
|
+
const n = count(folder);
|
|
1057
|
+
if (n) lines.push(`${folder.padEnd(14)}${n} file${n === 1 ? "" : "s"}: ${what}`);
|
|
1058
|
+
}
|
|
1059
|
+
const fileSources = [...new Set(entries.filter((p) => p.startsWith("files/")).map((p) => p.split("/")[1]))];
|
|
1060
|
+
if (fileSources.length) lines.push("", `Files: ${fileSources.map((name) => `files/${name}/`).join(", ")}`);
|
|
1061
|
+
const empty = job.units.filter((u) => u.status === "empty").length;
|
|
1062
|
+
if (empty) lines.push("", `${empty} source${empty === 1 ? " was" : "s were"} empty and wrote no file; manifest.json lists every source.`);
|
|
1063
|
+
lines.push(
|
|
983
1064
|
"",
|
|
1065
|
+
"manifest.json lists every file with its size and SHA-256, row counts per source, and the redaction policy.",
|
|
984
1066
|
"Verify a file: `shasum -a 256 <path>` and compare with manifest.json.",
|
|
985
1067
|
'Secrets are never exported. Credential fields read "[redacted]".',
|
|
986
1068
|
""
|
|
987
|
-
|
|
1069
|
+
);
|
|
1070
|
+
return lines.join("\n");
|
|
988
1071
|
}
|
|
989
1072
|
async function finalize(job, plan, scanner) {
|
|
990
1073
|
job.completedAt = iso(now());
|
|
@@ -1121,6 +1204,12 @@ function createExportEngine(options) {
|
|
|
1121
1204
|
return false;
|
|
1122
1205
|
} catch (err) {
|
|
1123
1206
|
unit2.error = errorText(scanner, err);
|
|
1207
|
+
if (isRetryLater(err) && (unit2.deferrals ?? 0) < MAX_DEFERRALS) {
|
|
1208
|
+
unit2.deferrals = (unit2.deferrals ?? 0) + 1;
|
|
1209
|
+
unit2.attempts -= 1;
|
|
1210
|
+
if (unit2.cursor === void 0) unit2.entries = [];
|
|
1211
|
+
return "deferred";
|
|
1212
|
+
}
|
|
1124
1213
|
if (unit2.cursor === void 0) unit2.entries = [];
|
|
1125
1214
|
if (unit2.attempts >= MAX_ATTEMPTS) unit2.status = "failed";
|
|
1126
1215
|
return true;
|
|
@@ -1136,6 +1225,10 @@ function createExportEngine(options) {
|
|
|
1136
1225
|
await save();
|
|
1137
1226
|
break;
|
|
1138
1227
|
}
|
|
1228
|
+
if (failed.includes("deferred")) {
|
|
1229
|
+
await save();
|
|
1230
|
+
continue;
|
|
1231
|
+
}
|
|
1139
1232
|
const { unit } = batch[0];
|
|
1140
1233
|
if (unit.status === "pending") {
|
|
1141
1234
|
unit.attempts = 0;
|
|
@@ -1180,7 +1273,8 @@ function createExportEngine(options) {
|
|
|
1180
1273
|
completedAt: job.completedAt,
|
|
1181
1274
|
error: job.error,
|
|
1182
1275
|
unitsTotal: job.units.length,
|
|
1183
|
-
unitsDone: job.units.filter((u) => u.status
|
|
1276
|
+
unitsDone: job.units.filter((u) => u.status === "done" || u.status === "empty").length,
|
|
1277
|
+
unitsFailed: job.units.filter((u) => u.status === "failed").length,
|
|
1184
1278
|
current: job.status === "running" ? job.units.find((u) => u.status === "pending")?.name : void 0,
|
|
1185
1279
|
bytes: job.units.reduce((n, u) => n + bytesOf(u), 0),
|
|
1186
1280
|
units: job.units.map((u) => ({
|
|
@@ -1600,31 +1694,70 @@ function kvFiles(args) {
|
|
|
1600
1694
|
|
|
1601
1695
|
// src/workspace-export/sandbox.ts
|
|
1602
1696
|
var GONE = /* @__PURE__ */ new Set(["deleted", "expired", "failed"]);
|
|
1697
|
+
var DEFAULT_SANDBOX_EXCLUDE = ["node_modules", ".sidecar"];
|
|
1698
|
+
function isLifecycleBusy(err) {
|
|
1699
|
+
const e = err;
|
|
1700
|
+
return typeof e === "object" && e !== null && e.status === 409 && e.code === "SANDBOX_LIFECYCLE_IN_PROGRESS";
|
|
1701
|
+
}
|
|
1702
|
+
async function openRunning(getBox, timeoutMs) {
|
|
1703
|
+
const box = await getBox();
|
|
1704
|
+
if (!box || GONE.has(box.status)) return null;
|
|
1705
|
+
const wasStopped = box.status === "stopped";
|
|
1706
|
+
try {
|
|
1707
|
+
if (wasStopped) await box.resume();
|
|
1708
|
+
if (box.status !== "running" || wasStopped) await box.waitFor("running", { timeoutMs });
|
|
1709
|
+
} catch (err) {
|
|
1710
|
+
if (isLifecycleBusy(err)) throw new ExportRetryLater("the sandbox is busy with another operation; retrying shortly");
|
|
1711
|
+
throw err;
|
|
1712
|
+
}
|
|
1713
|
+
const restore = async () => {
|
|
1714
|
+
if (wasStopped && box.stop) await box.stop().catch(() => {
|
|
1715
|
+
});
|
|
1716
|
+
};
|
|
1717
|
+
return { box, restore };
|
|
1718
|
+
}
|
|
1603
1719
|
function sandboxArchive(args) {
|
|
1604
1720
|
return {
|
|
1605
1721
|
kind: "archive",
|
|
1606
1722
|
name: args.name,
|
|
1607
1723
|
dataClass: args.dataClass ?? "sandbox",
|
|
1608
1724
|
description: args.description,
|
|
1725
|
+
exclude: args.exclude ?? DEFAULT_SANDBOX_EXCLUDE,
|
|
1609
1726
|
async open() {
|
|
1610
|
-
const
|
|
1611
|
-
if (!
|
|
1612
|
-
const
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1727
|
+
const opened = await openRunning(args.getBox, args.resumeTimeoutMs ?? 12e4);
|
|
1728
|
+
if (!opened) return null;
|
|
1729
|
+
const archive = await opened.box.fs.openArchive(args.dir ?? ".");
|
|
1730
|
+
return archive.pipeThrough(new TransformStream({ flush: () => opened.restore() }));
|
|
1731
|
+
}
|
|
1732
|
+
};
|
|
1733
|
+
}
|
|
1734
|
+
function sandboxFiles(args) {
|
|
1735
|
+
return {
|
|
1736
|
+
kind: "files",
|
|
1737
|
+
name: args.name,
|
|
1738
|
+
dataClass: args.dataClass ?? "knowledge",
|
|
1739
|
+
description: args.description,
|
|
1740
|
+
async *files() {
|
|
1741
|
+
const listed = await args.list();
|
|
1742
|
+
if (!listed.length) return;
|
|
1743
|
+
const opened = await openRunning(args.getBox, args.resumeTimeoutMs ?? 12e4);
|
|
1744
|
+
if (!opened) return;
|
|
1745
|
+
const read = opened.box.fs.read;
|
|
1746
|
+
if (!read) throw new Error("workspace-export: this sandbox client cannot read files");
|
|
1747
|
+
try {
|
|
1748
|
+
for (const file of listed) {
|
|
1749
|
+
const text = await read.call(opened.box.fs, file.boxPath);
|
|
1750
|
+
yield { path: file.path, open: async () => text };
|
|
1751
|
+
}
|
|
1752
|
+
} finally {
|
|
1753
|
+
await opened.restore();
|
|
1754
|
+
}
|
|
1624
1755
|
}
|
|
1625
1756
|
};
|
|
1626
1757
|
}
|
|
1627
1758
|
export {
|
|
1759
|
+
DEFAULT_SANDBOX_EXCLUDE,
|
|
1760
|
+
ExportRetryLater,
|
|
1628
1761
|
MANIFEST_SCHEMA,
|
|
1629
1762
|
REDACTED,
|
|
1630
1763
|
createExportEngine,
|
|
@@ -1637,6 +1770,7 @@ export {
|
|
|
1637
1770
|
isSecretName,
|
|
1638
1771
|
kvFiles,
|
|
1639
1772
|
r2Files,
|
|
1640
|
-
sandboxArchive
|
|
1773
|
+
sandboxArchive,
|
|
1774
|
+
sandboxFiles
|
|
1641
1775
|
};
|
|
1642
1776
|
//# sourceMappingURL=index.js.map
|