busa-cli 0.30.1 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -6,11 +6,11 @@ import { DEFAULT_BASE_URL, cloudContract, createBusabaseClient, getRecordByField
|
|
|
6
6
|
import { Command, CommanderError, InvalidArgumentError, Option } from "commander";
|
|
7
7
|
import { createInterface } from "node:readline/promises";
|
|
8
8
|
import { homedir, tmpdir } from "node:os";
|
|
9
|
+
import { access, mkdir, mkdtemp, open, readFile, readdir, rm, stat, unlink, writeFile } from "node:fs/promises";
|
|
9
10
|
import { PassThrough, Readable } from "node:stream";
|
|
10
11
|
import { pipeline } from "node:stream/promises";
|
|
11
12
|
import { ZstdCompress, ZstdDecompress } from "node:zlib";
|
|
12
13
|
import * as tar from "tar-stream";
|
|
13
|
-
import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
14
14
|
import { spawn } from "node:child_process";
|
|
15
15
|
import { createServer } from "node:http";
|
|
16
16
|
import { parse, stringify } from "yaml";
|
|
@@ -3003,19 +3003,55 @@ z.object({
|
|
|
3003
3003
|
textContentHash: z.string().nullable(),
|
|
3004
3004
|
byteCount: z.number().int().nonnegative()
|
|
3005
3005
|
});
|
|
3006
|
+
z.object({ nodeIds: z.array(z.string()).min(1).max(25) });
|
|
3006
3007
|
z.object({
|
|
3007
3008
|
/**
|
|
3008
|
-
*
|
|
3009
|
-
*
|
|
3010
|
-
*
|
|
3011
|
-
*
|
|
3012
|
-
*
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3009
|
+
* One entry per requested node that is a Doc in this space. A node id that
|
|
3010
|
+
* does not resolve is simply absent (not an error): the caller asked for a
|
|
3011
|
+
* batch, and one bad id must not cost it the other 24. A Doc that exists but
|
|
3012
|
+
* has no body object yet yields `markdown: ""`, matching what a read through
|
|
3013
|
+
* the Doc domain would return.
|
|
3014
|
+
*/
|
|
3015
|
+
bodies: z.array(z.object({
|
|
3016
|
+
nodeId: z.string(),
|
|
3017
|
+
markdown: z.string()
|
|
3018
|
+
})) });
|
|
3019
|
+
z.object({
|
|
3020
|
+
/**
|
|
3021
|
+
* The space id the archive was ORIGINALLY exported from (`manifest.spaceId`
|
|
3022
|
+
* in the `.bbdump`, already integrity-verified before this is called).
|
|
3023
|
+
* `importTableRows`'s "nodes" handling needs this to recognize the
|
|
3024
|
+
* archive's own root-node row deterministically (`rootNodeIdForSpace`) —
|
|
3025
|
+
* scanning each batch's rows for "the one with a null `parentId`" only
|
|
3026
|
+
* works when that row happens to land in the SAME batch as its children,
|
|
3027
|
+
* which cursor pagination (id-ordered, not tree-ordered) does not
|
|
3028
|
+
* guarantee once a space has more nodes than one page. See the matching
|
|
3029
|
+
* comment in `import-logic.ts`.
|
|
3030
|
+
*/
|
|
3031
|
+
sourceSpaceId: z.string(),
|
|
3032
|
+
/**
|
|
3033
|
+
* Continue a restore that was interrupted partway through, instead of
|
|
3034
|
+
* requiring an empty space.
|
|
3035
|
+
*
|
|
3036
|
+
* A restore that FAILS rolls itself back (`importAbort`), so the target is
|
|
3037
|
+
* left clean and a plain re-run works. What cannot roll itself back is a
|
|
3038
|
+
* restore whose process died — Ctrl-C, OOM, a dropped connection, the
|
|
3039
|
+
* machine rebooting. That leaves the space holding however many of the
|
|
3040
|
+
* archive's rows had landed, and every subsequent attempt is refused
|
|
3041
|
+
* ("requires an empty target space") with no way forward except wiping it.
|
|
3042
|
+
*
|
|
3043
|
+
* In this mode the empty-space guard is skipped and inserts become
|
|
3044
|
+
* `ON CONFLICT DO NOTHING`, so replaying the same archive re-lands only what
|
|
3045
|
+
* is missing. Blobs and doc bodies are content-addressed writes to object
|
|
3046
|
+
* storage and were already idempotent.
|
|
3047
|
+
*
|
|
3048
|
+
* DANGEROUS if pointed at the wrong space: rows that collide are silently
|
|
3049
|
+
* skipped rather than reported, so restoring archive A into a space holding
|
|
3050
|
+
* archive B's data would interleave the two instead of refusing. Only pass
|
|
3051
|
+
* it to continue the SAME archive into the SAME space.
|
|
3052
|
+
*/
|
|
3053
|
+
resume: z.boolean().optional().default(false)
|
|
3054
|
+
});
|
|
3019
3055
|
z.object({ sessionId: z.string() });
|
|
3020
3056
|
z.object({
|
|
3021
3057
|
sessionId: z.string(),
|
|
@@ -3034,6 +3070,85 @@ z.object({
|
|
|
3034
3070
|
warnings: z.array(z.string())
|
|
3035
3071
|
});
|
|
3036
3072
|
z.object({ ok: z.boolean() });
|
|
3073
|
+
//#endregion
|
|
3074
|
+
//#region src/retry.ts
|
|
3075
|
+
const BASE_DELAY_MS = 500;
|
|
3076
|
+
const MAX_DELAY_MS = 2e4;
|
|
3077
|
+
const READ_METHODS = /* @__PURE__ */ new Set([
|
|
3078
|
+
"GET",
|
|
3079
|
+
"HEAD",
|
|
3080
|
+
"OPTIONS"
|
|
3081
|
+
]);
|
|
3082
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
3083
|
+
const isNetworkError = (error) => error instanceof TypeError || /fetch failed|econnreset|econnrefused|etimedout|socket hang up/i.test(error instanceof Error ? error.message : String(error));
|
|
3084
|
+
/**
|
|
3085
|
+
* Whether an already-thrown error is worth another attempt.
|
|
3086
|
+
*
|
|
3087
|
+
* `shouldRetry` below decides from a `Response`; this decides from an `Error`,
|
|
3088
|
+
* for callers that retry ABOVE the fetch layer (see `backup`'s exporter, which
|
|
3089
|
+
* has to retry oRPC reads that `withRetry` refuses to repeat because they are
|
|
3090
|
+
* POSTs). The distinction that matters is transient-vs-deterministic: a dropped
|
|
3091
|
+
* socket or a 502 may answer differently next time, while a 404, a validation
|
|
3092
|
+
* refusal or a shape mismatch answers identically every time — retrying those
|
|
3093
|
+
* only burns the backoff, which at scale (one per attachment, on a space with
|
|
3094
|
+
* tens of thousands) is the difference between a slow backup and a stalled one.
|
|
3095
|
+
*/
|
|
3096
|
+
function isTransientError(error) {
|
|
3097
|
+
if (isNetworkError(error)) return true;
|
|
3098
|
+
const status = error?.status;
|
|
3099
|
+
if (typeof status === "number") return status === 429 || status >= 500;
|
|
3100
|
+
return /\bHTTP (429|5\d\d)\b/.test(error instanceof Error ? error.message : String(error));
|
|
3101
|
+
}
|
|
3102
|
+
/** `Retry-After` is either delay-seconds or an HTTP-date; both are legal. */
|
|
3103
|
+
function parseRetryAfter(header, now = Date.now()) {
|
|
3104
|
+
if (!header) return void 0;
|
|
3105
|
+
const seconds = Number(header.trim());
|
|
3106
|
+
if (Number.isFinite(seconds) && seconds >= 0) return Math.min(seconds * 1e3, MAX_DELAY_MS);
|
|
3107
|
+
const at = Date.parse(header);
|
|
3108
|
+
if (Number.isNaN(at)) return void 0;
|
|
3109
|
+
return Math.min(Math.max(at - now, 0), MAX_DELAY_MS);
|
|
3110
|
+
}
|
|
3111
|
+
/** Whether this outcome is worth another attempt for a request using `method`. */
|
|
3112
|
+
function shouldRetry(method, status) {
|
|
3113
|
+
if (status === 429) return true;
|
|
3114
|
+
if (!READ_METHODS.has(method.toUpperCase())) return false;
|
|
3115
|
+
return status === void 0 || status >= 500;
|
|
3116
|
+
}
|
|
3117
|
+
/**
|
|
3118
|
+
* Wrap a fetch so transient failures are retried with exponential backoff.
|
|
3119
|
+
* Returns `base` unchanged when `retries` is 0, so the no-retry path adds no
|
|
3120
|
+
* wrapper and no behaviour.
|
|
3121
|
+
*/
|
|
3122
|
+
function withRetry(base, options) {
|
|
3123
|
+
const { retries } = options;
|
|
3124
|
+
if (retries <= 0) return base;
|
|
3125
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
3126
|
+
const notify = options.notify ?? ((message) => console.error(message));
|
|
3127
|
+
const jitter = options.jitter ?? Math.random;
|
|
3128
|
+
return (async (input, init) => {
|
|
3129
|
+
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
|
|
3130
|
+
for (let attempt = 0;; attempt += 1) {
|
|
3131
|
+
let response;
|
|
3132
|
+
let failure;
|
|
3133
|
+
try {
|
|
3134
|
+
response = await base(input, init);
|
|
3135
|
+
} catch (error) {
|
|
3136
|
+
if (!isNetworkError(error)) throw error;
|
|
3137
|
+
failure = error;
|
|
3138
|
+
}
|
|
3139
|
+
const status = response?.status;
|
|
3140
|
+
const last = attempt >= retries;
|
|
3141
|
+
if (!shouldRetry(method, status) || last) {
|
|
3142
|
+
if (response) return response;
|
|
3143
|
+
throw failure;
|
|
3144
|
+
}
|
|
3145
|
+
const backoff = Math.min(BASE_DELAY_MS * 2 ** attempt, MAX_DELAY_MS);
|
|
3146
|
+
const delay = parseRetryAfter(response?.headers.get("retry-after") ?? null) ?? Math.round(backoff * (.5 + jitter() * .5));
|
|
3147
|
+
notify(`[busabase-cli] ${status ? `HTTP ${status}` : "connection failed"} — retrying in ${Math.round(delay / 1e3)}s (attempt ${attempt + 2} of ${retries + 1})`);
|
|
3148
|
+
await sleep(delay);
|
|
3149
|
+
}
|
|
3150
|
+
});
|
|
3151
|
+
}
|
|
3037
3152
|
/**
|
|
3038
3153
|
* `.bbdump` manifest — the archive's own table of contents + integrity
|
|
3039
3154
|
* record. Written last (after every entry's bytes/checksums are known) so it
|
|
@@ -3424,9 +3539,12 @@ async function streamNdjsonRows(body, send) {
|
|
|
3424
3539
|
* integrity-verified (`verifyArchive`) before this is called.
|
|
3425
3540
|
*/
|
|
3426
3541
|
async function importFull(options) {
|
|
3427
|
-
const { client, archivePath, sourceSpaceId, onProgress } = options;
|
|
3542
|
+
const { client, archivePath, sourceSpaceId, resume = false, onProgress } = options;
|
|
3428
3543
|
const log = onProgress ?? (() => {});
|
|
3429
|
-
const { sessionId } = await client.dump.importBegin({
|
|
3544
|
+
const { sessionId } = await client.dump.importBegin({
|
|
3545
|
+
sourceSpaceId,
|
|
3546
|
+
resume
|
|
3547
|
+
});
|
|
3430
3548
|
try {
|
|
3431
3549
|
const docBodyRows = [];
|
|
3432
3550
|
await streamArchive(archivePath, async (entry) => {
|
|
@@ -3483,7 +3601,7 @@ async function importFull(options) {
|
|
|
3483
3601
|
}
|
|
3484
3602
|
return await client.dump.importCommit({ sessionId });
|
|
3485
3603
|
} catch (error) {
|
|
3486
|
-
await client.dump.importAbort({ sessionId }).catch(() => void 0);
|
|
3604
|
+
if (!resume) await client.dump.importAbort({ sessionId }).catch(() => void 0);
|
|
3487
3605
|
throw error;
|
|
3488
3606
|
}
|
|
3489
3607
|
}
|
|
@@ -3500,33 +3618,285 @@ async function importFull(options) {
|
|
|
3500
3618
|
* the one page here, close to the failure, rather than widening the shared
|
|
3501
3619
|
* fetch-level wrapper (which would change retry behavior for every command).
|
|
3502
3620
|
*/
|
|
3503
|
-
const
|
|
3621
|
+
const DEFAULT_PAGE_FETCH_RETRIES = 3;
|
|
3504
3622
|
const PAGE_FETCH_RETRY_DELAY_MS = 1e3;
|
|
3505
|
-
const fetchPageWithRetry = async (fetchPage, onRetry) => {
|
|
3623
|
+
const fetchPageWithRetry = async (fetchPage, onRetry, shouldRetryError = () => true, retries = DEFAULT_PAGE_FETCH_RETRIES) => {
|
|
3506
3624
|
for (let attempt = 0;; attempt += 1) try {
|
|
3507
3625
|
return await fetchPage();
|
|
3508
3626
|
} catch (error) {
|
|
3509
|
-
if (
|
|
3627
|
+
if (!shouldRetryError(error)) throw error;
|
|
3628
|
+
if (attempt >= retries) throw error;
|
|
3510
3629
|
onRetry?.(attempt + 1, error);
|
|
3511
3630
|
await new Promise((resolve) => setTimeout(resolve, PAGE_FETCH_RETRY_DELAY_MS * (attempt + 1)));
|
|
3512
3631
|
}
|
|
3513
3632
|
};
|
|
3514
3633
|
/**
|
|
3634
|
+
* A per-item loop (one doc body, one attachment blob, one asset-text blob —
|
|
3635
|
+
* always exactly one request each, no server-side batching available) can run
|
|
3636
|
+
* for many minutes on a real space with no output between its start and its
|
|
3637
|
+
* own final summary line, which reads exactly like a hang. Call this once per
|
|
3638
|
+
* iteration; it logs `message()` only every `PROGRESS_INTERVAL_MS`, not on
|
|
3639
|
+
* every single item.
|
|
3640
|
+
*/
|
|
3641
|
+
const PROGRESS_INTERVAL_MS = 5e3;
|
|
3642
|
+
const throttledProgress = (log, message) => {
|
|
3643
|
+
let lastAt = Date.now();
|
|
3644
|
+
return () => {
|
|
3645
|
+
const now = Date.now();
|
|
3646
|
+
if (now - lastAt < PROGRESS_INTERVAL_MS) return;
|
|
3647
|
+
lastAt = now;
|
|
3648
|
+
log(message());
|
|
3649
|
+
};
|
|
3650
|
+
};
|
|
3651
|
+
/**
|
|
3652
|
+
* How many blob downloads may be in flight at once.
|
|
3653
|
+
*
|
|
3654
|
+
* The export used to be strictly one-request-at-a-time. On the real Vika Team
|
|
3655
|
+
* space that is 10,486 attachments x 2 sequential round trips each (resolve a
|
|
3656
|
+
* download URL, then fetch the bytes) — ~21k serialized requests, which is
|
|
3657
|
+
* where the bulk of a multi-HOUR backup went. Latency, not bandwidth, was the
|
|
3658
|
+
* limit: a single in-flight download never comes close to saturating the link.
|
|
3659
|
+
*
|
|
3660
|
+
* 8 is deliberately modest. These run against a live production server that is
|
|
3661
|
+
* also serving real users, and each in-flight download holds its whole body in
|
|
3662
|
+
* memory (see the buffer in the attachment loop), so the ceiling also bounds
|
|
3663
|
+
* peak heap at roughly 8 x the largest file rather than the whole set.
|
|
3664
|
+
*/
|
|
3665
|
+
const BLOB_DOWNLOAD_CONCURRENCY = 8;
|
|
3666
|
+
/**
|
|
3667
|
+
* Run `work` over `items` with at most `limit` in flight, handing each result
|
|
3668
|
+
* to `consume` strictly IN ORDER and one at a time.
|
|
3669
|
+
*
|
|
3670
|
+
* The ordering guarantee is what makes this safe to drop into the existing
|
|
3671
|
+
* loops: `consume` keeps doing exactly what the sequential body did (append to
|
|
3672
|
+
* a spool file, bump counters, dedupe against a Set), with no concurrent
|
|
3673
|
+
* access to any of it — only the network wait overlaps. Ordered output also
|
|
3674
|
+
* keeps an archive's bytes reproducible for a given space rather than varying
|
|
3675
|
+
* with which download happened to finish first.
|
|
3676
|
+
*
|
|
3677
|
+
* A rejected `work` is captured rather than thrown, so one bad item still
|
|
3678
|
+
* reaches `consume` as a failure and the run continues — the loops below
|
|
3679
|
+
* already treat a single unreachable blob as a warning, not a reason to lose
|
|
3680
|
+
* the other 10,485.
|
|
3681
|
+
*/
|
|
3682
|
+
const forEachConcurrent = async (items, limit, work, consume) => {
|
|
3683
|
+
const inFlight = /* @__PURE__ */ new Map();
|
|
3684
|
+
let started = 0;
|
|
3685
|
+
for (let emitted = 0; emitted < items.length; emitted += 1) {
|
|
3686
|
+
while (inFlight.size < limit && started < items.length) {
|
|
3687
|
+
const index = started;
|
|
3688
|
+
started += 1;
|
|
3689
|
+
inFlight.set(index, work(items[index]).then((value) => ({
|
|
3690
|
+
ok: true,
|
|
3691
|
+
value
|
|
3692
|
+
}), (error) => ({
|
|
3693
|
+
ok: false,
|
|
3694
|
+
error
|
|
3695
|
+
})));
|
|
3696
|
+
}
|
|
3697
|
+
const pending = inFlight.get(emitted);
|
|
3698
|
+
inFlight.delete(emitted);
|
|
3699
|
+
await consume(await pending, items[emitted]);
|
|
3700
|
+
}
|
|
3701
|
+
};
|
|
3702
|
+
/**
|
|
3515
3703
|
* Write an NDJSON archive entry from already-collected rows without ever
|
|
3516
3704
|
* concatenating them into a single JS string first. A real production table
|
|
3517
3705
|
* (e.g. `fieldValues` holding long CMS/article-body content) can produce an
|
|
3518
3706
|
* NDJSON blob well past V8's max string length — `"".join("\n")` on such a
|
|
3519
3707
|
* table throws `RangeError: Invalid string length` and aborts the whole
|
|
3520
|
-
* backup.
|
|
3521
|
-
*
|
|
3522
|
-
*
|
|
3708
|
+
* backup. Streaming line-sized buffers through `ArchiveWriter.addStream`
|
|
3709
|
+
* (already built for exactly this — see its own doc comment) avoids ever
|
|
3710
|
+
* holding one oversized string/buffer.
|
|
3711
|
+
*
|
|
3712
|
+
* `addStream` needs the entry's total byte size up front (a tar constraint),
|
|
3713
|
+
* which used to be met by materializing every line as a `Buffer` in an array
|
|
3714
|
+
* and summing it. On `fieldValues` that is 1.1M `Buffer`s alive at once while
|
|
3715
|
+
* the 1.1M row objects they came from are also still live — the same
|
|
3716
|
+
* peak-heap shape `SpooledNdjsonEntry` was written to avoid, on the one table
|
|
3717
|
+
* that dwarfs the rest. Measure the size in a first pass (keeping only a
|
|
3718
|
+
* running total), then re-serialize lazily from a generator so at most one
|
|
3719
|
+
* line exists at a time. The cost is `JSON.stringify` twice per row, which is
|
|
3720
|
+
* CPU against a backup whose real bottleneck is the network.
|
|
3523
3721
|
*/
|
|
3524
3722
|
const writeNdjsonEntry = async (writer, path, rows) => {
|
|
3525
|
-
|
|
3526
|
-
const totalSize
|
|
3527
|
-
|
|
3723
|
+
let totalSize = 0;
|
|
3724
|
+
for (const row of rows) totalSize += Buffer.byteLength(`${JSON.stringify(row)}\n`, "utf8");
|
|
3725
|
+
const lines = function* () {
|
|
3726
|
+
for (const row of rows) yield Buffer.from(`${JSON.stringify(row)}\n`, "utf8");
|
|
3727
|
+
};
|
|
3728
|
+
await writer.addStream(path, totalSize, Readable.from(lines()));
|
|
3729
|
+
};
|
|
3730
|
+
/**
|
|
3731
|
+
* Same NDJSON-entry shape as {@link writeNdjsonEntry}, but for a table built
|
|
3732
|
+
* one row at a time over a long fetch loop (attachment blobs, asset-text
|
|
3733
|
+
* blobs) instead of already sitting in memory. `ArchiveWriter.addStream`
|
|
3734
|
+
* requires the entry's total byte size up front (a tar constraint), which is
|
|
3735
|
+
* why `writeNdjsonEntry` collects every row into an array first — fine for
|
|
3736
|
+
* table rows, but a real space's attachment blobs (base64-encoded, one row
|
|
3737
|
+
* per file) can total many GB. Held as an array of Buffers so their combined
|
|
3738
|
+
* size could be summed, that blew Node's default heap outright — confirmed
|
|
3739
|
+
* live: "JavaScript heap out of memory" partway through a ~5.7GB attachment
|
|
3740
|
+
* set. Spooling each row to a temp file as it's produced, then streaming
|
|
3741
|
+
* that file into the archive once its final size is known from disk, keeps
|
|
3742
|
+
* peak memory bounded by one row at a time rather than the whole table.
|
|
3743
|
+
*/
|
|
3744
|
+
var SpooledNdjsonEntry = class {
|
|
3745
|
+
path;
|
|
3746
|
+
stream;
|
|
3747
|
+
/**
|
|
3748
|
+
* `spoolPath` is deterministic (derived from the archive path) rather than a
|
|
3749
|
+
* random temp name, so a killed backup leaves a file the next run can find
|
|
3750
|
+
* and continue from — see `recoverCompleteRows`. A random name in `tmpdir()`
|
|
3751
|
+
* left nothing but an orphan nobody could match back to its archive.
|
|
3752
|
+
*/
|
|
3753
|
+
constructor(spoolPath, append) {
|
|
3754
|
+
this.path = spoolPath;
|
|
3755
|
+
this.stream = createWriteStream(this.path, append ? { flags: "a" } : {});
|
|
3756
|
+
}
|
|
3757
|
+
async writeRow(row) {
|
|
3758
|
+
const buf = Buffer.from(`${JSON.stringify(row)}\n`, "utf8");
|
|
3759
|
+
if (!this.stream.write(buf)) await new Promise((resolve) => this.stream.once("drain", resolve));
|
|
3760
|
+
}
|
|
3761
|
+
async finish(writer, archivePath) {
|
|
3762
|
+
await new Promise((resolve, reject) => {
|
|
3763
|
+
this.stream.end((error) => error ? reject(error) : resolve());
|
|
3764
|
+
});
|
|
3765
|
+
const { size } = await stat(this.path);
|
|
3766
|
+
await writer.addStream(archivePath, size, createReadStream(this.path));
|
|
3767
|
+
await unlink(this.path);
|
|
3768
|
+
}
|
|
3769
|
+
};
|
|
3770
|
+
/**
|
|
3771
|
+
* What a previous, killed run already spooled: the identifying key of every
|
|
3772
|
+
* COMPLETE row, plus the totals those rows represent.
|
|
3773
|
+
*
|
|
3774
|
+
* "Complete" is load-bearing. A process killed mid-write leaves a final line
|
|
3775
|
+
* with no trailing newline — valid-looking JSON prefix, truncated content. The
|
|
3776
|
+
* file is therefore truncated back to its last newline before anything is
|
|
3777
|
+
* appended, so a resumed run can never splice new output onto half a row.
|
|
3778
|
+
*
|
|
3779
|
+
* Byte totals are recomputed from the base64 lengths WITHOUT decoding: a real
|
|
3780
|
+
* space's spool is ~5.7GB, and decoding it just to re-derive a counter would
|
|
3781
|
+
* cost more than re-downloading. base64 encodes 3 bytes per 4 chars, minus one
|
|
3782
|
+
* byte per `=` of padding.
|
|
3783
|
+
*/
|
|
3784
|
+
/**
|
|
3785
|
+
* Suffixes of the two spool files a backup writes next to its archive, in the
|
|
3786
|
+
* order `exportFull` derives them (attachment blobs, then asset-text blobs).
|
|
3787
|
+
* Exported so `commands.ts` can detect leftovers from an interrupted run
|
|
3788
|
+
* without duplicating the naming.
|
|
3789
|
+
*/
|
|
3790
|
+
const SPOOL_SUFFIXES = [".attachment-blobs.part", ".asset-text-blobs.part"];
|
|
3791
|
+
const decodedBase64Bytes = (base64) => {
|
|
3792
|
+
const padding = base64.endsWith("==") ? 2 : base64.endsWith("=") ? 1 : 0;
|
|
3793
|
+
return Math.max(0, base64.length * 3 / 4 - padding);
|
|
3794
|
+
};
|
|
3795
|
+
const recoverCompleteRows = async (spoolPath, keyField) => {
|
|
3796
|
+
const keys = /* @__PURE__ */ new Set();
|
|
3797
|
+
let count = 0;
|
|
3798
|
+
let bytes = 0;
|
|
3799
|
+
let completeBytes = 0;
|
|
3800
|
+
let handle;
|
|
3801
|
+
try {
|
|
3802
|
+
handle = await open(spoolPath, "r+");
|
|
3803
|
+
} catch {
|
|
3804
|
+
return {
|
|
3805
|
+
keys,
|
|
3806
|
+
count,
|
|
3807
|
+
bytes
|
|
3808
|
+
};
|
|
3809
|
+
}
|
|
3810
|
+
try {
|
|
3811
|
+
let carry = "";
|
|
3812
|
+
for await (const chunk of handle.createReadStream({
|
|
3813
|
+
encoding: "utf8",
|
|
3814
|
+
autoClose: false
|
|
3815
|
+
})) {
|
|
3816
|
+
carry += chunk;
|
|
3817
|
+
let newlineIndex;
|
|
3818
|
+
while ((newlineIndex = carry.indexOf("\n")) !== -1) {
|
|
3819
|
+
const line = carry.slice(0, newlineIndex);
|
|
3820
|
+
carry = carry.slice(newlineIndex + 1);
|
|
3821
|
+
completeBytes += Buffer.byteLength(line, "utf8") + 1;
|
|
3822
|
+
if (!line) continue;
|
|
3823
|
+
let row;
|
|
3824
|
+
try {
|
|
3825
|
+
row = JSON.parse(line);
|
|
3826
|
+
} catch {
|
|
3827
|
+
completeBytes -= Buffer.byteLength(line, "utf8") + 1;
|
|
3828
|
+
carry = "";
|
|
3829
|
+
break;
|
|
3830
|
+
}
|
|
3831
|
+
const key = row[keyField];
|
|
3832
|
+
if (typeof key !== "string") continue;
|
|
3833
|
+
keys.add(key);
|
|
3834
|
+
count += 1;
|
|
3835
|
+
bytes += decodedBase64Bytes(typeof row.base64 === "string" ? row.base64 : "");
|
|
3836
|
+
}
|
|
3837
|
+
}
|
|
3838
|
+
await handle.truncate(completeBytes);
|
|
3839
|
+
} finally {
|
|
3840
|
+
await handle.close();
|
|
3841
|
+
}
|
|
3842
|
+
return {
|
|
3843
|
+
keys,
|
|
3844
|
+
count,
|
|
3845
|
+
bytes
|
|
3846
|
+
};
|
|
3528
3847
|
};
|
|
3529
3848
|
const EXPORT_TABLES = [...DumpTableSchema.options];
|
|
3849
|
+
/**
|
|
3850
|
+
* Which dump tables reference which OTHER dump tables, column by column —
|
|
3851
|
+
* mirrors the real `.references()` declarations in busabase-core's schema
|
|
3852
|
+
* (`domains/base/schema.ts`, `domains/assets/schema/assets.ts`,
|
|
3853
|
+
* `db/schema.ts`). Only cross-table FKs matter here: a column with no
|
|
3854
|
+
* `.references()` (e.g. `fieldValues.fieldId`, a loose text ref by design, or
|
|
3855
|
+
* `assetUsages.recordId`, which defaults to `""`) cannot throw a violation and
|
|
3856
|
+
* must NOT be listed, or the drift check would drop rows Postgres would have
|
|
3857
|
+
* accepted.
|
|
3858
|
+
*
|
|
3859
|
+
* `nodes.parentId` is deliberately absent despite being a real self-FK: the
|
|
3860
|
+
* importer rewrites it (dropping the source root and remapping its children
|
|
3861
|
+
* onto the target's own root — see `import-logic.ts`), so a parentId that
|
|
3862
|
+
* looks dangling here is resolved server-side.
|
|
3863
|
+
*/
|
|
3864
|
+
const DUMP_FOREIGN_KEYS = {
|
|
3865
|
+
bases: { nodeId: "nodes" },
|
|
3866
|
+
baseFields: { baseId: "bases" },
|
|
3867
|
+
views: { baseId: "bases" },
|
|
3868
|
+
records: {
|
|
3869
|
+
baseId: "bases",
|
|
3870
|
+
headCommitId: "commits"
|
|
3871
|
+
},
|
|
3872
|
+
fieldValues: {
|
|
3873
|
+
baseId: "bases",
|
|
3874
|
+
recordId: "records",
|
|
3875
|
+
changeRequestId: "changeRequests",
|
|
3876
|
+
operationId: "operations",
|
|
3877
|
+
commitId: "commits"
|
|
3878
|
+
},
|
|
3879
|
+
recordLinks: {
|
|
3880
|
+
baseId: "bases",
|
|
3881
|
+
fieldId: "baseFields",
|
|
3882
|
+
sourceRecordId: "records",
|
|
3883
|
+
targetBaseId: "bases",
|
|
3884
|
+
targetRecordId: "records",
|
|
3885
|
+
commitId: "commits"
|
|
3886
|
+
},
|
|
3887
|
+
assetUsages: {
|
|
3888
|
+
assetId: "assets",
|
|
3889
|
+
nodeId: "nodes"
|
|
3890
|
+
},
|
|
3891
|
+
assetTexts: { assetId: "assets" },
|
|
3892
|
+
auditEvents: {
|
|
3893
|
+
baseId: "bases",
|
|
3894
|
+
recordId: "records",
|
|
3895
|
+
changeRequestId: "changeRequests",
|
|
3896
|
+
operationId: "operations",
|
|
3897
|
+
commitId: "commits"
|
|
3898
|
+
}
|
|
3899
|
+
};
|
|
3530
3900
|
const HISTORY_TABLES = [
|
|
3531
3901
|
"commits",
|
|
3532
3902
|
"changeRequests",
|
|
@@ -3536,6 +3906,84 @@ const HISTORY_TABLES = [
|
|
|
3536
3906
|
"auditEvents"
|
|
3537
3907
|
];
|
|
3538
3908
|
/**
|
|
3909
|
+
* A change request in one of these states is FINISHED — its content is history.
|
|
3910
|
+
* Anything else (`in_review`, `approved`, `conflict`, …) is work somebody has
|
|
3911
|
+
* not finished yet, and a backup that silently dropped it would lose a
|
|
3912
|
+
* colleague's in-flight work, not history. On the validated production space
|
|
3913
|
+
* that is 492 unfinished out of 22,828.
|
|
3914
|
+
*/
|
|
3915
|
+
const FINISHED_CR_STATUSES = /* @__PURE__ */ new Set([
|
|
3916
|
+
"merged",
|
|
3917
|
+
"closed",
|
|
3918
|
+
"rejected"
|
|
3919
|
+
]);
|
|
3920
|
+
/**
|
|
3921
|
+
* Reduce the history tables to what the CURRENT state (plus unfinished work)
|
|
3922
|
+
* genuinely needs, for `--no-history`.
|
|
3923
|
+
*
|
|
3924
|
+
* The flag used to just omit the history tables wholesale, which produced an
|
|
3925
|
+
* archive that could never be restored: `records.head_commit_id` is a NOT NULL
|
|
3926
|
+
* FK into `commits`, so every record dangled. Verified end-to-end — the
|
|
3927
|
+
* restore died on the first `records` batch. Dropping a table a live table
|
|
3928
|
+
* still points at is not "less history", it is a broken archive.
|
|
3929
|
+
*
|
|
3930
|
+
* What the shape of the data actually allows (measured on the real Vika Team
|
|
3931
|
+
* space, 1,130,898 fieldValues rows):
|
|
3932
|
+
* - Every (record, field) cell holds exactly ONE row — there are no
|
|
3933
|
+
* per-field historical versions to drop. 499,186 rows carry a `recordId`
|
|
3934
|
+
* and ARE the current values.
|
|
3935
|
+
* - The other 631,712 rows carry no `recordId` at all: they are values staged
|
|
3936
|
+
* inside change requests ("this CR proposes setting X to Y"). Those are the
|
|
3937
|
+
* history, and they are 56% of the table.
|
|
3938
|
+
* - 97.8% of change requests are already merged.
|
|
3939
|
+
*
|
|
3940
|
+
* So: keep every current value, keep unfinished change requests whole, and
|
|
3941
|
+
* drop the finished ones' contents. Commits are then kept transitively —
|
|
3942
|
+
* whatever the surviving rows still reference — rather than by any rule about
|
|
3943
|
+
* what a commit "is", so a NOT NULL `commitId` can never be left dangling.
|
|
3944
|
+
*/
|
|
3945
|
+
const pruneToCurrentState = (tableRows, tableCounts, log) => {
|
|
3946
|
+
const before = Object.fromEntries(HISTORY_TABLES.map((t) => [t, (tableRows[t] ?? []).length]));
|
|
3947
|
+
const beforeFieldValues = (tableRows.fieldValues ?? []).length;
|
|
3948
|
+
const keptCrIds = new Set((tableRows.changeRequests ?? []).filter((cr) => !FINISHED_CR_STATUSES.has(cr.status)).map((cr) => cr.id));
|
|
3949
|
+
const keptOperations = (tableRows.operations ?? []).filter((op) => keptCrIds.has(op.changeRequestId));
|
|
3950
|
+
const keptOpIds = new Set(keptOperations.map((op) => op.id));
|
|
3951
|
+
const keptFieldValues = (tableRows.fieldValues ?? []).filter((fv) => {
|
|
3952
|
+
if (fv.recordId) return true;
|
|
3953
|
+
if (fv.changeRequestId && keptCrIds.has(fv.changeRequestId)) return true;
|
|
3954
|
+
return Boolean(fv.operationId) && keptOpIds.has(fv.operationId);
|
|
3955
|
+
});
|
|
3956
|
+
for (const fv of keptFieldValues) {
|
|
3957
|
+
if (fv.changeRequestId && !keptCrIds.has(fv.changeRequestId)) fv.changeRequestId = null;
|
|
3958
|
+
if (fv.operationId && !keptOpIds.has(fv.operationId)) fv.operationId = null;
|
|
3959
|
+
}
|
|
3960
|
+
const keptReviews = (tableRows.reviews ?? []).filter((r) => keptCrIds.has(r.changeRequestId));
|
|
3961
|
+
const keptComments = (tableRows.comments ?? []).filter((c) => !c.changeRequestId || keptCrIds.has(c.changeRequestId));
|
|
3962
|
+
const neededCommitIds = /* @__PURE__ */ new Set();
|
|
3963
|
+
const need = (id) => {
|
|
3964
|
+
if (typeof id === "string" && id) neededCommitIds.add(id);
|
|
3965
|
+
};
|
|
3966
|
+
for (const r of tableRows.records ?? []) need(r.headCommitId);
|
|
3967
|
+
for (const rl of tableRows.recordLinks ?? []) need(rl.commitId);
|
|
3968
|
+
for (const fv of keptFieldValues) need(fv.commitId);
|
|
3969
|
+
for (const op of keptOperations) need(op.headCommitId);
|
|
3970
|
+
for (const c of keptComments) need(c.commitId);
|
|
3971
|
+
const keptCommits = (tableRows.commits ?? []).filter((c) => neededCommitIds.has(c.id));
|
|
3972
|
+
const assign = (table, rows) => {
|
|
3973
|
+
tableRows[table] = rows;
|
|
3974
|
+
tableCounts[table] = rows.length;
|
|
3975
|
+
};
|
|
3976
|
+
assign("fieldValues", keptFieldValues);
|
|
3977
|
+
assign("changeRequests", (tableRows.changeRequests ?? []).filter((cr) => keptCrIds.has(cr.id)));
|
|
3978
|
+
assign("operations", keptOperations);
|
|
3979
|
+
assign("reviews", keptReviews);
|
|
3980
|
+
assign("comments", keptComments);
|
|
3981
|
+
assign("commits", keptCommits);
|
|
3982
|
+
assign("auditEvents", []);
|
|
3983
|
+
const dropped = (table) => before[table] - (tableRows[table] ?? []).length;
|
|
3984
|
+
log(` --no-history: kept ${keptFieldValues.length}/${beforeFieldValues} fieldValues (dropped ${beforeFieldValues - keptFieldValues.length} staged in finished change requests), ${keptCrIds.size} unfinished change request(s); dropped ${dropped("commits")} commits, ${dropped("operations")} operations, ${dropped("reviews")} reviews, ${dropped("auditEvents")} audit events.`);
|
|
3985
|
+
};
|
|
3986
|
+
/**
|
|
3539
3987
|
* Full-fidelity export: pulls every dump-eligible table via the `dump.exportTables`
|
|
3540
3988
|
* endpoint (cursor-paginated, raw rows with original ids), doc bodies for
|
|
3541
3989
|
* every `doc` node via the public `docs.body` read, every asset's bytes via
|
|
@@ -3544,16 +3992,18 @@ const HISTORY_TABLES = [
|
|
|
3544
3992
|
* keeps the format uniform regardless of table size).
|
|
3545
3993
|
*/
|
|
3546
3994
|
async function exportFull(options) {
|
|
3547
|
-
const { client, outPath, spaceId, sourceHost, includeHistory, toolVersion, onProgress } = options;
|
|
3548
|
-
const
|
|
3995
|
+
const { client, outPath, spaceId, sourceHost, includeHistory, toolVersion, resume = false, retries = DEFAULT_PAGE_FETCH_RETRIES, onProgress } = options;
|
|
3996
|
+
const [attachmentSpoolPath, assetTextSpoolPath] = SPOOL_SUFFIXES.map((suffix) => `${outPath}${suffix}`);
|
|
3997
|
+
const pageLimit = options.pageLimit ?? 2e3;
|
|
3549
3998
|
const log = onProgress ?? (() => {});
|
|
3550
3999
|
const writer = ArchiveWriter.create(outPath);
|
|
3551
4000
|
const tableCounts = {};
|
|
3552
4001
|
const tableRows = {};
|
|
3553
|
-
const tables =
|
|
4002
|
+
const tables = EXPORT_TABLES;
|
|
3554
4003
|
for (const table of tables) {
|
|
3555
4004
|
const rows = [];
|
|
3556
4005
|
let cursor;
|
|
4006
|
+
const reportProgress = throttledProgress(log, () => ` ...${table}: ${rows.length} rows so far`);
|
|
3557
4007
|
do {
|
|
3558
4008
|
const page = await fetchPageWithRetry(() => client.dump.exportTables({
|
|
3559
4009
|
table,
|
|
@@ -3561,101 +4011,207 @@ async function exportFull(options) {
|
|
|
3561
4011
|
limit: pageLimit
|
|
3562
4012
|
}), (attempt, error) => {
|
|
3563
4013
|
const message = error instanceof Error ? error.message : String(error);
|
|
3564
|
-
log(` retrying ${table} page (attempt ${attempt}/${
|
|
3565
|
-
});
|
|
4014
|
+
log(` retrying ${table} page (attempt ${attempt}/${retries}): ${message}`);
|
|
4015
|
+
}, void 0, retries);
|
|
3566
4016
|
rows.push(...page.rows);
|
|
3567
4017
|
cursor = page.nextCursor ?? void 0;
|
|
4018
|
+
if (cursor) reportProgress();
|
|
3568
4019
|
} while (cursor);
|
|
3569
4020
|
tableRows[table] = rows;
|
|
3570
4021
|
tableCounts[table] = rows.length;
|
|
3571
4022
|
log(`exported ${table}: ${rows.length} rows`);
|
|
3572
4023
|
}
|
|
4024
|
+
if (!includeHistory) pruneToCurrentState(tableRows, tableCounts, log);
|
|
4025
|
+
{
|
|
4026
|
+
const exported = new Set(tables);
|
|
4027
|
+
const idsOf = (table) => exported.has(table) ? new Set((tableRows[table] ?? []).map((row) => row.id)) : null;
|
|
4028
|
+
const validIds = /* @__PURE__ */ new Map();
|
|
4029
|
+
const validIdsFor = (table) => {
|
|
4030
|
+
if (!validIds.has(table)) validIds.set(table, idsOf(table));
|
|
4031
|
+
return validIds.get(table) ?? null;
|
|
4032
|
+
};
|
|
4033
|
+
for (const table of IMPORT_ORDER) {
|
|
4034
|
+
const foreignKeys = DUMP_FOREIGN_KEYS[table];
|
|
4035
|
+
if (!foreignKeys || !exported.has(table)) continue;
|
|
4036
|
+
const rows = tableRows[table] ?? [];
|
|
4037
|
+
if (rows.length === 0) continue;
|
|
4038
|
+
const kept = rows.filter((row) => Object.entries(foreignKeys).every(([column, targetTable]) => {
|
|
4039
|
+
const id = row[column];
|
|
4040
|
+
if (id === null || id === void 0 || id === "") return true;
|
|
4041
|
+
const valid = validIdsFor(targetTable);
|
|
4042
|
+
return valid === null || valid.has(id);
|
|
4043
|
+
}));
|
|
4044
|
+
const dropped = rows.length - kept.length;
|
|
4045
|
+
if (dropped === 0) continue;
|
|
4046
|
+
log(` WARNING: dropped ${dropped} ${table} row(s) referencing a ${Object.values(foreignKeys).join("/")} that this archive does not carry — a long backup of an actively-used space can drift out of sync with itself. Re-run the backup for a tighter snapshot if these matter.`);
|
|
4047
|
+
tableRows[table] = kept;
|
|
4048
|
+
tableCounts[table] = kept.length;
|
|
4049
|
+
validIds.set(table, new Set(kept.map((row) => row.id)));
|
|
4050
|
+
}
|
|
4051
|
+
}
|
|
3573
4052
|
const docNodes = (tableRows.nodes ?? []).filter((n) => n.type === "doc");
|
|
3574
4053
|
const docBodyEntries = [];
|
|
3575
4054
|
const skippedDocBodies = [];
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
4055
|
+
const reportDocProgress = throttledProgress(log, () => ` ...doc bodies: ${docBodyEntries.length}/${docNodes.length} fetched`);
|
|
4056
|
+
const isMissingRoute = (error) => /not found|404|no procedure|unknown procedure/i.test(error instanceof Error ? error.message : String(error));
|
|
4057
|
+
const readOneDocBodyLegacy = async (nodeId) => await fetchPageWithRetry(async () => {
|
|
4058
|
+
const detail = await client.nodes.get({
|
|
4059
|
+
nodeId,
|
|
4060
|
+
type: "doc"
|
|
4061
|
+
});
|
|
4062
|
+
if (detail.type !== "doc") throw new Error(`Expected a doc node for ${nodeId}, got "${detail.type}"`);
|
|
4063
|
+
return detail.body ?? "";
|
|
4064
|
+
}, (attempt, error) => log(` retrying doc body ${nodeId} (attempt ${attempt}/${retries}): ${error instanceof Error ? error.message : String(error)}`), isTransientError, retries);
|
|
4065
|
+
const applyLegacyDocBody = (nodeId, body) => {
|
|
4066
|
+
docBodyEntries.push({
|
|
4067
|
+
nodeId,
|
|
4068
|
+
body
|
|
4069
|
+
});
|
|
4070
|
+
};
|
|
4071
|
+
let batchRouteAvailable = true;
|
|
4072
|
+
for (let i = 0; i < docNodes.length; i += 25) {
|
|
4073
|
+
reportDocProgress();
|
|
4074
|
+
const nodeIds = docNodes.slice(i, i + 25).map((node) => node.id);
|
|
4075
|
+
if (batchRouteAvailable) try {
|
|
4076
|
+
const { bodies } = await fetchPageWithRetry(() => client.dump.exportDocBodies({ nodeIds }), (attempt, error) => log(` retrying doc bodies batch (attempt ${attempt}): ${error instanceof Error ? error.message : String(error)}`), (error) => !isMissingRoute(error), retries);
|
|
4077
|
+
const byId = new Map(bodies.map((entry) => [entry.nodeId, entry.markdown]));
|
|
4078
|
+
for (const nodeId of nodeIds) {
|
|
4079
|
+
const markdown = byId.get(nodeId);
|
|
4080
|
+
if (markdown === void 0) {
|
|
4081
|
+
skippedDocBodies.push(nodeId);
|
|
4082
|
+
log(` WARNING: server returned no body for doc node ${nodeId}, skipping it`);
|
|
4083
|
+
continue;
|
|
4084
|
+
}
|
|
4085
|
+
docBodyEntries.push({
|
|
4086
|
+
nodeId,
|
|
4087
|
+
body: markdown
|
|
4088
|
+
});
|
|
4089
|
+
}
|
|
4090
|
+
continue;
|
|
3588
4091
|
} catch (error) {
|
|
4092
|
+
if (!isMissingRoute(error)) {
|
|
4093
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4094
|
+
skippedDocBodies.push(...nodeIds);
|
|
4095
|
+
log(` WARNING: could not read doc bodies for ${nodeIds.length} node(s), skipping them (${message})`);
|
|
4096
|
+
continue;
|
|
4097
|
+
}
|
|
4098
|
+
batchRouteAvailable = false;
|
|
4099
|
+
log(" NOTE: this server predates `dump.exportDocBodies`; falling back to one request per Doc. Archived Docs cannot be read that way, so their bodies will be missing from this archive — upgrade the server to capture them.");
|
|
4100
|
+
}
|
|
4101
|
+
await forEachConcurrent(nodeIds, BLOB_DOWNLOAD_CONCURRENCY, (nodeId) => readOneDocBodyLegacy(nodeId), (settled, nodeId) => {
|
|
4102
|
+
if (settled.ok) {
|
|
4103
|
+
applyLegacyDocBody(nodeId, settled.value);
|
|
4104
|
+
return;
|
|
4105
|
+
}
|
|
4106
|
+
const { error } = settled;
|
|
3589
4107
|
const message = error instanceof Error ? error.message : String(error);
|
|
3590
4108
|
skippedDocBodies.push(nodeId);
|
|
3591
4109
|
log(` WARNING: could not read doc body for node ${nodeId}, skipping it (${message})`);
|
|
3592
|
-
}
|
|
4110
|
+
});
|
|
3593
4111
|
}
|
|
3594
4112
|
log(`fetched ${docBodyEntries.length} doc bodies, skipped ${skippedDocBodies.length}`);
|
|
3595
4113
|
const attachmentsById = new Map((tableRows.attachments ?? []).map((a) => [a.id, a]));
|
|
3596
|
-
const
|
|
3597
|
-
|
|
3598
|
-
|
|
4114
|
+
const recoveredBlobs = resume ? await recoverCompleteRows(attachmentSpoolPath, "storageKey") : {
|
|
4115
|
+
keys: /* @__PURE__ */ new Set(),
|
|
4116
|
+
count: 0,
|
|
4117
|
+
bytes: 0
|
|
4118
|
+
};
|
|
4119
|
+
const attachmentBlobs = new SpooledNdjsonEntry(attachmentSpoolPath, resume);
|
|
4120
|
+
let blobCount = recoveredBlobs.count;
|
|
4121
|
+
let blobBytes = recoveredBlobs.bytes;
|
|
3599
4122
|
const skippedAssets = [];
|
|
3600
|
-
|
|
3601
|
-
|
|
4123
|
+
const assetRows = tableRows.assets ?? [];
|
|
4124
|
+
if (recoveredBlobs.count > 0) log(` resuming: ${recoveredBlobs.count} attachment blob(s) already downloaded (${recoveredBlobs.bytes} bytes), continuing with the rest`);
|
|
4125
|
+
const reportBlobProgress = throttledProgress(log, () => ` ...attachment blobs: ${blobCount}/${assetRows.length} fetched`);
|
|
4126
|
+
await forEachConcurrent(assetRows, BLOB_DOWNLOAD_CONCURRENCY, async (asset) => {
|
|
3602
4127
|
const attachment = attachmentsById.get(asset.attachmentId);
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
4128
|
+
if (!attachment) throw new Error("no matching attachments row (orphaned asset)");
|
|
4129
|
+
if (recoveredBlobs.keys.has(attachment.storageKey)) return null;
|
|
4130
|
+
return await fetchPageWithRetry(async () => {
|
|
4131
|
+
const download = await client.assets.download({ assetId: asset.id });
|
|
3606
4132
|
const resolvedUrl = new URL(download.downloadUrl, sourceHost).toString();
|
|
3607
4133
|
const res = await fetch(resolvedUrl);
|
|
3608
4134
|
if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`);
|
|
3609
|
-
|
|
3610
|
-
attachmentBlobRows.push({
|
|
4135
|
+
return {
|
|
3611
4136
|
storageKey: attachment.storageKey,
|
|
3612
4137
|
mimeType: download.mimeType,
|
|
3613
|
-
|
|
3614
|
-
}
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
4138
|
+
buf: Buffer.from(await res.arrayBuffer())
|
|
4139
|
+
};
|
|
4140
|
+
}, (attempt, error) => log(` retrying asset ${asset.id} (attempt ${attempt}/${retries}): ${error instanceof Error ? error.message : String(error)}`), isTransientError, retries);
|
|
4141
|
+
}, async (settled, asset) => {
|
|
4142
|
+
reportBlobProgress();
|
|
4143
|
+
const assetId = asset.id;
|
|
4144
|
+
if (!settled.ok) {
|
|
4145
|
+
const { error } = settled;
|
|
3618
4146
|
const message = error instanceof Error ? error.message : String(error);
|
|
3619
4147
|
skippedAssets.push(assetId);
|
|
3620
4148
|
log(` WARNING: could not download asset ${assetId}, skipping its blob (${message})`);
|
|
4149
|
+
return;
|
|
3621
4150
|
}
|
|
3622
|
-
|
|
4151
|
+
if (!settled.value) return;
|
|
4152
|
+
const { storageKey, mimeType, buf } = settled.value;
|
|
4153
|
+
await attachmentBlobs.writeRow({
|
|
4154
|
+
storageKey,
|
|
4155
|
+
mimeType,
|
|
4156
|
+
base64: buf.toString("base64")
|
|
4157
|
+
});
|
|
4158
|
+
blobCount += 1;
|
|
4159
|
+
blobBytes += buf.length;
|
|
4160
|
+
});
|
|
3623
4161
|
log(`fetched ${blobCount} attachment blobs (${blobBytes} bytes), skipped ${skippedAssets.length}`);
|
|
3624
|
-
const
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
4162
|
+
const recoveredTexts = resume ? await recoverCompleteRows(assetTextSpoolPath, "textStorageKey") : {
|
|
4163
|
+
keys: /* @__PURE__ */ new Set(),
|
|
4164
|
+
count: 0,
|
|
4165
|
+
bytes: 0
|
|
4166
|
+
};
|
|
4167
|
+
const textBlobs = new SpooledNdjsonEntry(assetTextSpoolPath, resume);
|
|
4168
|
+
const seenTextKeys = new Set(recoveredTexts.keys);
|
|
4169
|
+
let textBlobCount = recoveredTexts.count;
|
|
4170
|
+
let textBlobBytes = recoveredTexts.bytes;
|
|
3628
4171
|
const skippedAssetTexts = [];
|
|
3629
|
-
|
|
4172
|
+
const assetTextRows = tableRows.assetTexts ?? [];
|
|
4173
|
+
if (recoveredTexts.count > 0) log(` resuming: ${recoveredTexts.count} asset-text blob(s) already downloaded`);
|
|
4174
|
+
const reportTextProgress = throttledProgress(log, () => ` ...asset-text blobs: ${textBlobCount}/${assetTextRows.length} fetched`);
|
|
4175
|
+
await forEachConcurrent(assetTextRows, BLOB_DOWNLOAD_CONCURRENCY, async (assetText) => await fetchPageWithRetry(async () => {
|
|
4176
|
+
const info = await client.dump.exportAssetText({ assetId: assetText.assetId });
|
|
4177
|
+
if (!info.downloadUrl) return null;
|
|
4178
|
+
if (recoveredTexts.keys.has(info.textStorageKey)) return null;
|
|
4179
|
+
const resolvedUrl = new URL(info.downloadUrl, sourceHost).toString();
|
|
4180
|
+
const res = await fetch(resolvedUrl);
|
|
4181
|
+
if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`);
|
|
4182
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
4183
|
+
if (info.textContentHash) {
|
|
4184
|
+
const actual = `sha256:${createHash("sha256").update(buf).digest("hex")}`;
|
|
4185
|
+
if (actual !== info.textContentHash) throw new Error(`content hash mismatch (expected ${info.textContentHash}, downloaded ${actual})`);
|
|
4186
|
+
}
|
|
4187
|
+
return {
|
|
4188
|
+
textStorageKey: info.textStorageKey,
|
|
4189
|
+
buf
|
|
4190
|
+
};
|
|
4191
|
+
}, (attempt, error) => log(` retrying asset text ${assetText.assetId} (attempt ${attempt}/${retries}): ${error instanceof Error ? error.message : String(error)}`), isTransientError, retries), async (settled, assetText) => {
|
|
4192
|
+
reportTextProgress();
|
|
3630
4193
|
const assetId = assetText.assetId;
|
|
3631
|
-
|
|
3632
|
-
const
|
|
3633
|
-
if (!info.downloadUrl) continue;
|
|
3634
|
-
if (seenTextKeys.has(info.textStorageKey)) continue;
|
|
3635
|
-
const resolvedUrl = new URL(info.downloadUrl, sourceHost).toString();
|
|
3636
|
-
const res = await fetch(resolvedUrl);
|
|
3637
|
-
if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`);
|
|
3638
|
-
const buf = Buffer.from(await res.arrayBuffer());
|
|
3639
|
-
if (info.textContentHash) {
|
|
3640
|
-
const actual = `sha256:${createHash("sha256").update(buf).digest("hex")}`;
|
|
3641
|
-
if (actual !== info.textContentHash) throw new Error(`content hash mismatch (expected ${info.textContentHash}, downloaded ${actual})`);
|
|
3642
|
-
}
|
|
3643
|
-
seenTextKeys.add(info.textStorageKey);
|
|
3644
|
-
textBlobRows.push({
|
|
3645
|
-
textStorageKey: info.textStorageKey,
|
|
3646
|
-
base64: buf.toString("base64")
|
|
3647
|
-
});
|
|
3648
|
-
textBlobCount += 1;
|
|
3649
|
-
textBlobBytes += buf.length;
|
|
3650
|
-
} catch (error) {
|
|
4194
|
+
if (!settled.ok) {
|
|
4195
|
+
const { error } = settled;
|
|
3651
4196
|
const message = error instanceof Error ? error.message : String(error);
|
|
3652
4197
|
skippedAssetTexts.push(assetId);
|
|
3653
4198
|
log(` WARNING: could not download text for asset ${assetId}, skipping its blob (${message})`);
|
|
4199
|
+
return;
|
|
3654
4200
|
}
|
|
3655
|
-
|
|
4201
|
+
if (!settled.value) return;
|
|
4202
|
+
const { textStorageKey, buf } = settled.value;
|
|
4203
|
+
if (seenTextKeys.has(textStorageKey)) return;
|
|
4204
|
+
seenTextKeys.add(textStorageKey);
|
|
4205
|
+
await textBlobs.writeRow({
|
|
4206
|
+
textStorageKey,
|
|
4207
|
+
base64: buf.toString("base64")
|
|
4208
|
+
});
|
|
4209
|
+
textBlobCount += 1;
|
|
4210
|
+
textBlobBytes += buf.length;
|
|
4211
|
+
});
|
|
3656
4212
|
log(`fetched ${textBlobCount} asset-text blobs (${textBlobBytes} bytes), skipped ${skippedAssetTexts.length}`);
|
|
3657
|
-
await
|
|
3658
|
-
await
|
|
4213
|
+
await attachmentBlobs.finish(writer, "tree/attachmentBlobs.ndjson");
|
|
4214
|
+
await textBlobs.finish(writer, "tree/assetTextBlobs.ndjson");
|
|
3659
4215
|
for (const table of IMPORT_ORDER) {
|
|
3660
4216
|
if (!tables.includes(table)) continue;
|
|
3661
4217
|
await writeNdjsonEntry(writer, `tree/${table}.ndjson`, tableRows[table] ?? []);
|
|
@@ -3680,10 +4236,24 @@ async function exportFull(options) {
|
|
|
3680
4236
|
}
|
|
3681
4237
|
//#endregion
|
|
3682
4238
|
//#region src/backup/commands.ts
|
|
4239
|
+
/**
|
|
4240
|
+
* `backup` and `restore` — the terminal-facing halves of the full-fidelity
|
|
4241
|
+
* `.bbdump` archive format. Thin orchestration: drain every dump-eligible table
|
|
4242
|
+
* into an archive, or verify an archive and replay it into an empty space.
|
|
4243
|
+
*/
|
|
3683
4244
|
/** Progress is diagnostics, not data — it must never pollute `--output json` on stdout. */
|
|
3684
4245
|
const reportProgress$1 = (message) => {
|
|
3685
4246
|
console.error(message);
|
|
3686
4247
|
};
|
|
4248
|
+
/** Which of this archive's spool files a previous, interrupted run left behind. */
|
|
4249
|
+
const findSpoolFiles = async (outFile) => {
|
|
4250
|
+
const present = [];
|
|
4251
|
+
for (const suffix of SPOOL_SUFFIXES) try {
|
|
4252
|
+
await access(`${outFile}${suffix}`);
|
|
4253
|
+
present.push(`${outFile}${suffix}`);
|
|
4254
|
+
} catch {}
|
|
4255
|
+
return present;
|
|
4256
|
+
};
|
|
3687
4257
|
/**
|
|
3688
4258
|
* Deliberately not implemented: `export` already writes the portable current-state
|
|
3689
4259
|
* representation as a readable, diffable directory. The two formats stay separate
|
|
@@ -3718,7 +4288,11 @@ const runBackup = async (client, options) => {
|
|
|
3718
4288
|
if (options.stateOnly) throw new Error(STATE_ONLY_BACKUP_MESSAGE);
|
|
3719
4289
|
const { spaceId } = options;
|
|
3720
4290
|
if (!spaceId) throw new Error("No space configured. Pass --space-id <id>, export BUSABASE_SPACE_ID, or run `busabase-cli login`.");
|
|
3721
|
-
|
|
4291
|
+
if (!options.resume) {
|
|
4292
|
+
const orphaned = await findSpoolFiles(options.outFile);
|
|
4293
|
+
if (orphaned.length > 0) reportProgress$1(` NOTE: found ${orphaned.length} partial file(s) from an interrupted backup (${orphaned.join(", ")}). Re-run with --resume to continue from them; this run will start over and overwrite them.`);
|
|
4294
|
+
}
|
|
4295
|
+
reportProgress$1(`Backing up space ${spaceId} → ${options.outFile} (full fidelity, history=${options.includeHistory}${options.resume ? ", resuming" : ""})`);
|
|
3722
4296
|
const manifest = await exportFull({
|
|
3723
4297
|
client,
|
|
3724
4298
|
outPath: options.outFile,
|
|
@@ -3726,6 +4300,8 @@ const runBackup = async (client, options) => {
|
|
|
3726
4300
|
sourceHost: options.baseUrl,
|
|
3727
4301
|
includeHistory: options.includeHistory,
|
|
3728
4302
|
toolVersion: options.toolVersion,
|
|
4303
|
+
resume: Boolean(options.resume),
|
|
4304
|
+
retries: options.retries,
|
|
3729
4305
|
onProgress: (message) => reportProgress$1(` ${message}`)
|
|
3730
4306
|
});
|
|
3731
4307
|
const rows = Object.values(manifest.tables).reduce((total, count) => total + count, 0);
|
|
@@ -3744,7 +4320,7 @@ const runBackup = async (client, options) => {
|
|
|
3744
4320
|
].join("\n");
|
|
3745
4321
|
};
|
|
3746
4322
|
const runRestore = async (client, options) => {
|
|
3747
|
-
if (options.resume)
|
|
4323
|
+
if (options.resume) reportProgress$1("Resuming: rows already present are left alone, and this run will NOT roll back on failure. Only do this to continue THIS archive into THIS space.");
|
|
3748
4324
|
reportProgress$1(`Reading ${options.file} …`);
|
|
3749
4325
|
const manifest = await verifyArchive(options.file);
|
|
3750
4326
|
reportProgress$1(`Archive: fidelity=${manifest.fidelity} space=${manifest.spaceId} exportedAt=${manifest.exportedAt}`);
|
|
@@ -3754,6 +4330,7 @@ const runRestore = async (client, options) => {
|
|
|
3754
4330
|
client,
|
|
3755
4331
|
archivePath: options.file,
|
|
3756
4332
|
sourceSpaceId: manifest.spaceId,
|
|
4333
|
+
resume: Boolean(options.resume),
|
|
3757
4334
|
onProgress: (message) => reportProgress$1(` ${message}`)
|
|
3758
4335
|
});
|
|
3759
4336
|
if (options.json) return {
|
|
@@ -5448,6 +6025,7 @@ const nodeSchema = z.lazy(() => z.object({
|
|
|
5448
6025
|
name: z.string(),
|
|
5449
6026
|
description: z.string(),
|
|
5450
6027
|
metadata: z.object({ version: z.string().optional() }).catchall(z.unknown()).default({}),
|
|
6028
|
+
settings: nodeSettingsSchema.default({}),
|
|
5451
6029
|
explicitVisibility: z.enum([
|
|
5452
6030
|
"private",
|
|
5453
6031
|
"workspace",
|
|
@@ -5531,6 +6109,53 @@ z.object({
|
|
|
5531
6109
|
nodeId: z.string(),
|
|
5532
6110
|
metadata: z.record(z.string(), z.unknown())
|
|
5533
6111
|
});
|
|
6112
|
+
/**
|
|
6113
|
+
* System settings Busabase itself acts on, per node.
|
|
6114
|
+
*
|
|
6115
|
+
* A CLOSED object, unlike `metadata`'s `z.record(z.string(), z.unknown())`.
|
|
6116
|
+
* That difference is the point: `metadata` is the open bag callers put their
|
|
6117
|
+
* own keys in, and its endpoint merges whatever it is handed. Anything the
|
|
6118
|
+
* product *reads and acts on* must not live there — `metadata.visibility` had
|
|
6119
|
+
* to be pulled out into its own column precisely because the generic endpoint
|
|
6120
|
+
* handed a `write`-level actor a door into an ACL input.
|
|
6121
|
+
*
|
|
6122
|
+
* Adding a key here is a deliberate change to this schema, which is what keeps
|
|
6123
|
+
* `PATCH /nodes/{nodeId}/settings` from becoming a second free-form bag.
|
|
6124
|
+
*
|
|
6125
|
+
* `strictObject`, not `object`: zod's default is to STRIP an unknown key, which
|
|
6126
|
+
* would hand a client that misspelled `airappEngine` a 200 and no effect —
|
|
6127
|
+
* exactly the silent failure a closed schema exists to prevent. Verified
|
|
6128
|
+
* against a running server, where the permissive version returned 200.
|
|
6129
|
+
*/
|
|
6130
|
+
const nodeSettingsSchema = z.strictObject({
|
|
6131
|
+
/**
|
|
6132
|
+
* Which engine an AirApp runs on, when a human has chosen.
|
|
6133
|
+
*
|
|
6134
|
+
* `undefined` means "follow the app" — `airapp.json`'s `preferredEngine`
|
|
6135
|
+
* decides, or the default does. A value means somebody overrode it in the
|
|
6136
|
+
* node settings dialog, and it outranks the manifest from then on. So absence
|
|
6137
|
+
* must stay distinguishable from any particular value; `null` clears an
|
|
6138
|
+
* override and returns the node to following the app.
|
|
6139
|
+
*/
|
|
6140
|
+
airappEngine: z.enum([
|
|
6141
|
+
"browser",
|
|
6142
|
+
"local",
|
|
6143
|
+
"remote"
|
|
6144
|
+
]).nullish() });
|
|
6145
|
+
z.object({
|
|
6146
|
+
nodeId: z.string(),
|
|
6147
|
+
settings: nodeSettingsSchema
|
|
6148
|
+
});
|
|
6149
|
+
z.object({ nodeId: z.string() });
|
|
6150
|
+
z.object({
|
|
6151
|
+
nodeId: z.string(),
|
|
6152
|
+
agentPrompts: customAgentPromptsSchema.nullable()
|
|
6153
|
+
});
|
|
6154
|
+
z.object({
|
|
6155
|
+
nodeId: z.string(),
|
|
6156
|
+
/** Replaces the whole list — this is not a merge. Send `null` to clear. */
|
|
6157
|
+
agentPrompts: customAgentPromptsSchema.nullable()
|
|
6158
|
+
});
|
|
5534
6159
|
z.object({
|
|
5535
6160
|
query: z.string().min(1),
|
|
5536
6161
|
limit: z.coerce.number().int().min(1).max(50).optional().default(20)
|
|
@@ -5726,7 +6351,8 @@ z.object({
|
|
|
5726
6351
|
"change_request.reviewed",
|
|
5727
6352
|
"change_request.merged",
|
|
5728
6353
|
"change_request.pending_review",
|
|
5729
|
-
"node.metadata_updated"
|
|
6354
|
+
"node.metadata_updated",
|
|
6355
|
+
"node.settings_updated"
|
|
5730
6356
|
]),
|
|
5731
6357
|
spaceId: z.string(),
|
|
5732
6358
|
actorId: z.string(),
|
|
@@ -5757,6 +6383,8 @@ const auditActionSchema = z.enum([
|
|
|
5757
6383
|
"asset.text_written",
|
|
5758
6384
|
"asset.text_marked_none",
|
|
5759
6385
|
"node.metadata_updated",
|
|
6386
|
+
"node.settings_updated",
|
|
6387
|
+
"node.agent_prompts_updated",
|
|
5760
6388
|
"node.purged"
|
|
5761
6389
|
]);
|
|
5762
6390
|
z.object({
|
|
@@ -6605,6 +7233,38 @@ const decodeText = (bytes, filePath) => {
|
|
|
6605
7233
|
if (text.includes("�")) throw new Error(`${filePath} is not valid UTF-8 text.`);
|
|
6606
7234
|
return text;
|
|
6607
7235
|
};
|
|
7236
|
+
const AIRAPP_MANIFEST_FILENAME = "airapp.json";
|
|
7237
|
+
const PYTHON_RUNTIME_MARKERS = ["requirements.txt", "pyproject.toml"];
|
|
7238
|
+
/**
|
|
7239
|
+
* Resolve only the lifecycle facts the package format needs to validate.
|
|
7240
|
+
* Keep the same explicit -> inferred -> default order as Busabase's runtime
|
|
7241
|
+
* descriptor without making busabase-package depend on busabase-core (core
|
|
7242
|
+
* already depends on this package).
|
|
7243
|
+
*/
|
|
7244
|
+
const resolveAirAppPackageLifecycle = (files, slug) => {
|
|
7245
|
+
const paths = new Set(files.map((file) => file.path));
|
|
7246
|
+
const manifestFile = files.find((file) => file.path === AIRAPP_MANIFEST_FILENAME);
|
|
7247
|
+
let manifest = {};
|
|
7248
|
+
if (manifestFile) {
|
|
7249
|
+
let parsed;
|
|
7250
|
+
try {
|
|
7251
|
+
parsed = JSON.parse(decodeText(manifestFile.bytes, AIRAPP_MANIFEST_FILENAME));
|
|
7252
|
+
} catch (error) {
|
|
7253
|
+
throw new Error(`AirApp "${slug}" has an invalid ${AIRAPP_MANIFEST_FILENAME}: ${error.message}`);
|
|
7254
|
+
}
|
|
7255
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(`AirApp "${slug}" has an invalid ${AIRAPP_MANIFEST_FILENAME}: expected an object.`);
|
|
7256
|
+
manifest = parsed;
|
|
7257
|
+
}
|
|
7258
|
+
const declaredRuntime = manifest.runtime;
|
|
7259
|
+
if (declaredRuntime !== void 0 && declaredRuntime !== "node" && declaredRuntime !== "python") throw new Error(`AirApp "${slug}" declares unknown runtime ${JSON.stringify(declaredRuntime)} in ${AIRAPP_MANIFEST_FILENAME}.`);
|
|
7260
|
+
const runtime = declaredRuntime === "node" || declaredRuntime === "python" ? declaredRuntime : PYTHON_RUNTIME_MARKERS.some((path) => paths.has(path)) ? "python" : "node";
|
|
7261
|
+
const hasCustomCommand = (key) => typeof manifest[key] === "string" && manifest[key].trim().length > 0;
|
|
7262
|
+
return {
|
|
7263
|
+
runtime,
|
|
7264
|
+
usesDefaultInstall: !hasCustomCommand("install"),
|
|
7265
|
+
usesDefaultStart: !hasCustomCommand("start")
|
|
7266
|
+
};
|
|
7267
|
+
};
|
|
6608
7268
|
const parseJsonFile = (files, filePath) => {
|
|
6609
7269
|
const bytes = files.get(filePath);
|
|
6610
7270
|
if (!bytes) throw new Error(`Missing ${filePath}`);
|
|
@@ -6784,7 +7444,10 @@ const readDirNode = (files, dir, slug) => {
|
|
|
6784
7444
|
const ignore = readIgnoreFile(files, dir);
|
|
6785
7445
|
const entries = collectSubtree(files, dir).filter((entry) => entry.path !== "_node.json" && entry.path !== ".busabaseignore" && !ignore.ignores(entry.path));
|
|
6786
7446
|
assertNoReservedNames(entries.map((entry) => entry.path), `${meta.type} "${slug}"`);
|
|
6787
|
-
if (meta.type === "airapp"
|
|
7447
|
+
if (meta.type === "airapp") {
|
|
7448
|
+
const lifecycle = resolveAirAppPackageLifecycle(entries, slug);
|
|
7449
|
+
if (lifecycle.runtime === "node" && (lifecycle.usesDefaultInstall || lifecycle.usesDefaultStart) && !entries.some((entry) => entry.path === "package.json")) throw new Error(`AirApp "${slug}" has no package.json required by its default Node lifecycle${ignore.isEmpty ? "" : ` — check ${dir}${PACKAGE_AIRAPP_IGNORE}, which must never exclude it`}. Nothing was installed.`);
|
|
7450
|
+
}
|
|
6788
7451
|
return {
|
|
6789
7452
|
type: meta.type,
|
|
6790
7453
|
slug,
|
|
@@ -6981,6 +7644,8 @@ const validateTemplate = (tree) => {
|
|
|
6981
7644
|
else if (airapps.length > 0 || template?.airapp || template?.airapps?.length) errors.push(resolved.error);
|
|
6982
7645
|
for (const node of walkNodes(tree.nodes)) {
|
|
6983
7646
|
if (!isAirApp(node)) continue;
|
|
7647
|
+
const lifecycle = resolveAirAppPackageLifecycle(node.files, node.slug);
|
|
7648
|
+
if (lifecycle.runtime !== "node" || !lifecycle.usesDefaultStart) continue;
|
|
6984
7649
|
const manifestFile = node.files.find((file) => file.path === "package.json");
|
|
6985
7650
|
if (!manifestFile) continue;
|
|
6986
7651
|
let scripts;
|
|
@@ -6990,7 +7655,7 @@ const validateTemplate = (tree) => {
|
|
|
6990
7655
|
errors.push(`AirApp "${node.slug}": package.json is not valid JSON (${error.message}).`);
|
|
6991
7656
|
continue;
|
|
6992
7657
|
}
|
|
6993
|
-
if (typeof scripts?.dev !== "string") errors.push(`AirApp "${node.slug}": package.json has no \`dev\` script.
|
|
7658
|
+
if (typeof scripts?.dev !== "string") errors.push(`AirApp "${node.slug}": package.json has no \`dev\` script. This AirApp uses the default Node start command \`npm run dev\`, so without it the app installs and then never boots.`);
|
|
6994
7659
|
}
|
|
6995
7660
|
if (airapps.length === 0) warnings.push("No AirApp — this template installs data only. Users will see tables, not an app.");
|
|
6996
7661
|
if (template?.requires?.airapp && airapps.length === 0) errors.push("template.requires.airapp is true, but content/ contains no AirApp.");
|
|
@@ -7495,6 +8160,7 @@ const applyInstall = async (client, plan, options) => {
|
|
|
7495
8160
|
pendingChangeRequests: 0,
|
|
7496
8161
|
warnings: [...plan.warnings]
|
|
7497
8162
|
};
|
|
8163
|
+
const createdRootNodeIds = [];
|
|
7498
8164
|
const fieldIds = /* @__PURE__ */ new Map();
|
|
7499
8165
|
const bases = [];
|
|
7500
8166
|
const progress = options.onProgress ?? (() => {});
|
|
@@ -7514,6 +8180,7 @@ const applyInstall = async (client, plan, options) => {
|
|
|
7514
8180
|
metadata: app?.rootMetadata
|
|
7515
8181
|
});
|
|
7516
8182
|
state.created.folders++;
|
|
8183
|
+
createdRootNodeIds.push(state.targetFolderNodeId);
|
|
7517
8184
|
}
|
|
7518
8185
|
await createStructure(client, plan.tree.nodes, state.targetFolderNodeId, options, state, bases, fieldIds, docAssetIds, app);
|
|
7519
8186
|
if (app?.rootSkill) {
|
|
@@ -7605,8 +8272,16 @@ const applyInstall = async (client, plan, options) => {
|
|
|
7605
8272
|
const created = Object.entries(state.created).filter(([, count]) => count > 0).map(([kind, count]) => `${count} ${kind}`).join(", ");
|
|
7606
8273
|
const partial = created ? `Created before failure: ${created}. Target folder: ${plan.targetFolderSlug}.` : "No package resources were created.";
|
|
7607
8274
|
const pending = state.pendingChangeRequests ? ` ${state.pendingChangeRequests} ChangeRequest(s) may still be pending.` : "";
|
|
8275
|
+
const rollback = await rollbackPartialInstall(client, {
|
|
8276
|
+
createdRootNodeIds,
|
|
8277
|
+
enabled: options.rollbackOnFailure !== false,
|
|
8278
|
+
installedIntoExistingFolder: Boolean(plan.existingTargetFolderNodeId),
|
|
8279
|
+
anythingCreated: Object.values(state.created).some((count) => count > 0),
|
|
8280
|
+
submittedBy: options.submittedBy ?? "busabase-cli install",
|
|
8281
|
+
progress
|
|
8282
|
+
});
|
|
7608
8283
|
const reason = error instanceof Error ? error.message : String(error);
|
|
7609
|
-
const message = `Install stopped during ${phase}. ${partial}${pending} Cause: ${reason}`;
|
|
8284
|
+
const message = `Install stopped during ${phase}. ${partial}${pending}${rollback} Cause: ${reason}`;
|
|
7610
8285
|
const details = {
|
|
7611
8286
|
phase,
|
|
7612
8287
|
targetFolderSlug: plan.targetFolderSlug,
|
|
@@ -7621,6 +8296,47 @@ const applyInstall = async (client, plan, options) => {
|
|
|
7621
8296
|
}
|
|
7622
8297
|
};
|
|
7623
8298
|
/**
|
|
8299
|
+
* Undo a failed install by moving what it created to Trash.
|
|
8300
|
+
*
|
|
8301
|
+
* Deleting a node takes its whole subtree — Bases, their records, nested
|
|
8302
|
+
* folders — so undoing an install is a matter of deleting the roots it made,
|
|
8303
|
+
* not walking everything it touched. It moves them to Trash rather than
|
|
8304
|
+
* purging: purge refuses a subtree containing a Base (which every real package
|
|
8305
|
+
* has), and Trash is reversible, which matters for an automatic action taken
|
|
8306
|
+
* while something has already gone wrong.
|
|
8307
|
+
*
|
|
8308
|
+
* Only ever deletes a target folder THIS install created. Installing into a
|
|
8309
|
+
* folder the user already had is reported instead of guessed at: the new nodes
|
|
8310
|
+
* sit among the user's own, and there is no reliable way here to tell which
|
|
8311
|
+
* were ours. Deleting the wrong thing is far worse than leaving a mess.
|
|
8312
|
+
*
|
|
8313
|
+
* Best-effort: a rollback runs after a failure, so it can hit the same broken
|
|
8314
|
+
* condition. Whatever happens is reported, never thrown — the install error is
|
|
8315
|
+
* the one the caller needs.
|
|
8316
|
+
*/
|
|
8317
|
+
const rollbackPartialInstall = async (client, args) => {
|
|
8318
|
+
if (!args.anythingCreated) return "";
|
|
8319
|
+
if (!args.enabled) return " Left in place (--no-rollback).";
|
|
8320
|
+
if (args.createdRootNodeIds.length === 0) return args.installedIntoExistingFolder ? " Not rolled back: this installed into a folder that already existed, so its new items sit among yours and cannot be told apart safely — remove them by hand." : " Nothing to roll back.";
|
|
8321
|
+
args.progress("Install failed — moving what it created to Trash…");
|
|
8322
|
+
const failures = [];
|
|
8323
|
+
for (const nodeId of args.createdRootNodeIds) try {
|
|
8324
|
+
await client.nodes.createChangeRequest({
|
|
8325
|
+
operations: [{
|
|
8326
|
+
kind: "delete",
|
|
8327
|
+
nodeId
|
|
8328
|
+
}],
|
|
8329
|
+
message: "Roll back a failed install",
|
|
8330
|
+
submittedBy: args.submittedBy,
|
|
8331
|
+
autoMerge: true
|
|
8332
|
+
});
|
|
8333
|
+
} catch (rollbackError) {
|
|
8334
|
+
failures.push(`${nodeId} (${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)})`);
|
|
8335
|
+
}
|
|
8336
|
+
if (failures.length === 0) return " Rolled back: everything it created was moved to Trash.";
|
|
8337
|
+
return ` Rollback incomplete — could not remove ${failures.join("; ")}. Remove them by hand.`;
|
|
8338
|
+
};
|
|
8339
|
+
/**
|
|
7624
8340
|
* A property rather than a subclass: the whole point of the rethrow above is to
|
|
7625
8341
|
* keep the original error's prototype (an `ORPCError`'s `code`/`status`/`data`
|
|
7626
8342
|
* are what make a permission or quota failure actionable), so the diagnostics
|
|
@@ -8129,21 +8845,26 @@ const findSourceNode = (nodes, slugOrId) => {
|
|
|
8129
8845
|
const collectPackageTree = async (client, root, options) => {
|
|
8130
8846
|
const children = root.type === "folder" ? root.children ?? [] : [root];
|
|
8131
8847
|
const { skillSource, rest } = partitionTemplateSkill(children);
|
|
8132
|
-
const
|
|
8848
|
+
const collectOptions = {
|
|
8849
|
+
...options,
|
|
8850
|
+
baseResourceKeys: collectBaseResourceKeys(children)
|
|
8851
|
+
};
|
|
8852
|
+
const nodes = await collectNodes(client, rest, collectOptions);
|
|
8133
8853
|
const tree = {
|
|
8134
8854
|
manifest: {
|
|
8135
8855
|
...options.manifest,
|
|
8136
8856
|
format: PACKAGE_FORMAT
|
|
8137
8857
|
},
|
|
8138
|
-
nodes
|
|
8139
|
-
...skillSource ? { rootSkill: await collectRootSkill(client, skillSource,
|
|
8140
|
-
assets: await collectDocAssets(client, nodes,
|
|
8858
|
+
nodes,
|
|
8859
|
+
...skillSource ? { rootSkill: await collectRootSkill(client, skillSource, collectOptions) } : {},
|
|
8860
|
+
assets: await collectDocAssets(client, nodes, collectOptions)
|
|
8141
8861
|
};
|
|
8142
|
-
assertSelfContained(tree,
|
|
8862
|
+
assertSelfContained(tree, collectOptions);
|
|
8143
8863
|
return tree;
|
|
8144
8864
|
};
|
|
8145
8865
|
/**
|
|
8146
|
-
* Undo the
|
|
8866
|
+
* Undo the slug INSTALL chose for one node, restoring the one the package
|
|
8867
|
+
* declared.
|
|
8147
8868
|
*
|
|
8148
8869
|
* Install prefixes a template's Base slugs with the target folder (`contacts` →
|
|
8149
8870
|
* `kelly-crm-contacts`) and records the original as the node's `resourceKey`.
|
|
@@ -8153,53 +8874,48 @@ const collectPackageTree = async (client, root, options) => {
|
|
|
8153
8874
|
* which is the property that lets a user install a template, adjust it, and
|
|
8154
8875
|
* publish the result without the names drifting a little further each time.
|
|
8155
8876
|
*
|
|
8156
|
-
* The root
|
|
8157
|
-
*
|
|
8158
|
-
|
|
8159
|
-
|
|
8160
|
-
|
|
8161
|
-
|
|
8877
|
+
* The app-root stamp is skipped: `resourceKey` is the reserved `app-root`
|
|
8878
|
+
* marker there, not a slug anything should be named after.
|
|
8879
|
+
*
|
|
8880
|
+
* Applied to the ONE node `source` describes — never through a space-wide
|
|
8881
|
+
* slug→slug rename table. An earlier version built exactly that table (keyed
|
|
8882
|
+
* by each node's *current* slug) and applied it by re-matching slugs across
|
|
8883
|
+
* the whole exported tree; that broke the moment two unrelated nodes shared a
|
|
8884
|
+
* slug anywhere in the space — observed live: an AirApp installed into a
|
|
8885
|
+
* folder auto-named after it end up BOTH called "social-media" pre-restore
|
|
8886
|
+
* (the folder, and the app before its own stamp renamed it back to "airapp"),
|
|
8887
|
+
* so the app's rename bled onto its own parent folder on export.
|
|
8888
|
+
*/
|
|
8889
|
+
const restoredNodeSlug = (source) => {
|
|
8890
|
+
const key = source.metadata?.resourceKey;
|
|
8891
|
+
if (typeof key === "string" && key !== "app-root" && key !== source.slug && NODE_SLUG_PATTERN.test(key)) return key;
|
|
8892
|
+
return source.slug;
|
|
8893
|
+
};
|
|
8894
|
+
/**
|
|
8895
|
+
* The other half of slug restoration: a relation field's `targetBaseSlug` (set
|
|
8896
|
+
* from `context.baseSlugById` in {@link toPackageField}, itself built by
|
|
8897
|
+
* {@link buildIndexes} from each Base's raw current server slug) must resolve
|
|
8898
|
+
* to whatever THAT target Base's own node slug gets restored to, or the
|
|
8899
|
+
* package would ship a relation pointing at a name it never actually wrote.
|
|
8900
|
+
*
|
|
8901
|
+
* Keyed by Base id (globally unique, unlike slug — see {@link restoredNodeSlug}'s
|
|
8902
|
+
* doc) so `buildIndexes` can look a target base's restored slug up without
|
|
8903
|
+
* re-deriving it from a name that might collide with something else in the
|
|
8904
|
+
* space.
|
|
8905
|
+
*/
|
|
8906
|
+
const collectBaseResourceKeys = (sources) => {
|
|
8907
|
+
const byBaseId = /* @__PURE__ */ new Map();
|
|
8908
|
+
const walk = (list) => {
|
|
8162
8909
|
for (const source of list) {
|
|
8163
|
-
|
|
8164
|
-
|
|
8165
|
-
|
|
8166
|
-
}
|
|
8167
|
-
};
|
|
8168
|
-
index(sources);
|
|
8169
|
-
if (renames.size === 0) return [...nodes];
|
|
8170
|
-
const rewriteField = (field) => {
|
|
8171
|
-
const target = field.options.targetBaseSlug;
|
|
8172
|
-
if (!target) return field;
|
|
8173
|
-
const renamed = renames.get(target);
|
|
8174
|
-
return renamed ? {
|
|
8175
|
-
...field,
|
|
8176
|
-
options: {
|
|
8177
|
-
...field.options,
|
|
8178
|
-
targetBaseSlug: renamed
|
|
8910
|
+
if (source.type === "base") {
|
|
8911
|
+
const restored = restoredNodeSlug(source);
|
|
8912
|
+
if (restored !== source.slug) byBaseId.set(source.baseId ?? source.slug, restored);
|
|
8179
8913
|
}
|
|
8180
|
-
|
|
8181
|
-
|
|
8182
|
-
const rewrite = (node) => {
|
|
8183
|
-
const slug = renames.get(node.slug) ?? node.slug;
|
|
8184
|
-
if (node.type === "folder") return {
|
|
8185
|
-
...node,
|
|
8186
|
-
slug,
|
|
8187
|
-
children: node.children.map(rewrite)
|
|
8188
|
-
};
|
|
8189
|
-
if (node.type === "base") return {
|
|
8190
|
-
...node,
|
|
8191
|
-
slug,
|
|
8192
|
-
base: {
|
|
8193
|
-
...node.base,
|
|
8194
|
-
fields: node.base.fields.map(rewriteField)
|
|
8195
|
-
}
|
|
8196
|
-
};
|
|
8197
|
-
return {
|
|
8198
|
-
...node,
|
|
8199
|
-
slug
|
|
8200
|
-
};
|
|
8914
|
+
if (source.children) walk(source.children);
|
|
8915
|
+
}
|
|
8201
8916
|
};
|
|
8202
|
-
|
|
8917
|
+
walk(sources);
|
|
8918
|
+
return byBaseId;
|
|
8203
8919
|
};
|
|
8204
8920
|
/**
|
|
8205
8921
|
* Separate the app's manual from its resources.
|
|
@@ -8277,7 +8993,7 @@ const collectNodes = async (client, sources, options) => {
|
|
|
8277
8993
|
};
|
|
8278
8994
|
const collectNode = async (client, source, options) => {
|
|
8279
8995
|
const common = {
|
|
8280
|
-
slug: source
|
|
8996
|
+
slug: restoredNodeSlug(source),
|
|
8281
8997
|
name: source.name,
|
|
8282
8998
|
description: source.description ?? "",
|
|
8283
8999
|
position: source.position
|
|
@@ -8314,12 +9030,14 @@ const collectBase = async (client, source, common, options) => {
|
|
|
8314
9030
|
const baseId = source.baseId ?? source.slug;
|
|
8315
9031
|
const base = await client.bases.get({ baseId });
|
|
8316
9032
|
if (!base) throw new Error(`Base node "${source.slug}" has no Base behind it (looked up ${baseId}). The space may be mid-migration; re-run export.`);
|
|
8317
|
-
const
|
|
8318
|
-
const
|
|
9033
|
+
const allViews = await client.bases.listViews({ baseId: base.id });
|
|
9034
|
+
const views = allViews.filter((view) => view.slug.length > 0);
|
|
9035
|
+
if (views.length !== allViews.length) options.warn(`Base "${common.slug}" has ${allViews.length - views.length} view(s) with no slug — they were dropped (predates the slug requirement; rename it in the UI to keep it).`);
|
|
9036
|
+
const { baseSlugById, fieldSlugById } = await buildIndexes(client, options.baseResourceKeys ?? /* @__PURE__ */ new Map());
|
|
8319
9037
|
const fields = [...base.fields].sort((a, b) => a.position - b.position).map((field) => toPackageField(field, {
|
|
8320
9038
|
fieldSlugById,
|
|
8321
9039
|
baseSlugById,
|
|
8322
|
-
baseSlug:
|
|
9040
|
+
baseSlug: common.slug,
|
|
8323
9041
|
warn: options.warn
|
|
8324
9042
|
}));
|
|
8325
9043
|
const records = await collectRecords(client, base.id, base.fields, options);
|
|
@@ -8349,12 +9067,12 @@ const collectBase = async (client, source, common, options) => {
|
|
|
8349
9067
|
*
|
|
8350
9068
|
* `bases.list()` already hydrates each base's fields, so one call builds both.
|
|
8351
9069
|
*/
|
|
8352
|
-
const buildIndexes = async (client) => {
|
|
9070
|
+
const buildIndexes = async (client, baseResourceKeys) => {
|
|
8353
9071
|
const bases = await client.bases.list({});
|
|
8354
9072
|
const baseSlugById = /* @__PURE__ */ new Map();
|
|
8355
9073
|
const fieldSlugById = /* @__PURE__ */ new Map();
|
|
8356
9074
|
for (const base of bases) {
|
|
8357
|
-
baseSlugById.set(base.id, base.slug);
|
|
9075
|
+
baseSlugById.set(base.id, baseResourceKeys.get(base.id) ?? base.slug);
|
|
8358
9076
|
for (const field of base.fields ?? []) fieldSlugById.set(field.id, field.slug);
|
|
8359
9077
|
}
|
|
8360
9078
|
return {
|
|
@@ -8541,12 +9259,18 @@ const assertSelfContained = (tree, options) => {
|
|
|
8541
9259
|
const packagedRecordKeys = new Set(baseNodes.flatMap((node) => node.type === "base" ? node.records.map((record) => record.key) : []));
|
|
8542
9260
|
for (const node of baseNodes) {
|
|
8543
9261
|
if (node.type !== "base") continue;
|
|
9262
|
+
for (const field of node.base.fields) {
|
|
9263
|
+
if (field.type !== "attachment" || !field.required) continue;
|
|
9264
|
+
field.required = false;
|
|
9265
|
+
options.warn(`Base "${node.slug}": attachment field "${field.slug}" is required in the source, but ${PACKAGE_FORMAT} never carries attachment values — the field would be empty in every installed record and the install would fail. It is marked NOT required in this package.`);
|
|
9266
|
+
}
|
|
8544
9267
|
for (const field of node.base.fields) {
|
|
8545
9268
|
const targetBaseSlug = field.options.targetBaseSlug;
|
|
8546
9269
|
if (field.type !== "relation" || !targetBaseSlug) continue;
|
|
8547
9270
|
if (!packagedBaseSlugs.has(targetBaseSlug)) throw new Error(`Field "${node.slug}.${field.slug}" relates to Base "${targetBaseSlug}", which is outside the subtree being exported. A package must be self-contained — export a subtree that includes both Bases, or remove the field.`);
|
|
8548
9271
|
}
|
|
8549
9272
|
let dangling = 0;
|
|
9273
|
+
const emptiedRequired = /* @__PURE__ */ new Map();
|
|
8550
9274
|
for (const record of node.records) for (const field of node.base.fields) {
|
|
8551
9275
|
if (field.type !== "relation") continue;
|
|
8552
9276
|
const value = record.fields[field.slug];
|
|
@@ -8554,9 +9278,18 @@ const assertSelfContained = (tree, options) => {
|
|
|
8554
9278
|
if (keys.length === 0) continue;
|
|
8555
9279
|
const kept = keys.filter((key) => typeof key !== "string" || packagedRecordKeys.has(key));
|
|
8556
9280
|
dangling += keys.length - kept.length;
|
|
8557
|
-
if (kept.length !== keys.length)
|
|
9281
|
+
if (kept.length !== keys.length) {
|
|
9282
|
+
record.fields[field.slug] = kept;
|
|
9283
|
+
if (kept.length === 0 && field.required) emptiedRequired.set(field.slug, (emptiedRequired.get(field.slug) ?? 0) + 1);
|
|
9284
|
+
}
|
|
8558
9285
|
}
|
|
8559
9286
|
if (dangling > 0) options.warn(`Base "${node.slug}" has ${dangling} relation value(s) pointing at records outside the exported subtree — they were dropped.`);
|
|
9287
|
+
for (const [slug, count] of emptiedRequired) {
|
|
9288
|
+
const field = node.base.fields.find((candidate) => candidate.slug === slug);
|
|
9289
|
+
if (!field) continue;
|
|
9290
|
+
field.required = false;
|
|
9291
|
+
options.warn(`Base "${node.slug}": relation field "${slug}" is required in the source, but ${count} record(s) point only at records outside the export (typically archived ones, which are never exported). It is marked NOT required in this package so the package can be installed; those records install with that field empty.`);
|
|
9292
|
+
}
|
|
8560
9293
|
}
|
|
8561
9294
|
};
|
|
8562
9295
|
//#endregion
|
|
@@ -8895,15 +9628,59 @@ const renderNodeTree = (nodes, indent) => {
|
|
|
8895
9628
|
return lines;
|
|
8896
9629
|
};
|
|
8897
9630
|
/**
|
|
8898
|
-
*
|
|
8899
|
-
*
|
|
9631
|
+
* A required field the package cannot satisfy for at least one record. The
|
|
9632
|
+
* server rejects these — but only once it is already writing records, deep into
|
|
9633
|
+
* pass 4 of 5, having created everything before them.
|
|
9634
|
+
*
|
|
9635
|
+
* Real occurrence: a package whose `content-artifacts` Base declared a REQUIRED
|
|
9636
|
+
* attachment field (`busabase-package@1` never carries attachment bytes, so the
|
|
9637
|
+
* column is empty in every record) failed with `Invalid field value: 文件 is
|
|
9638
|
+
* required` **after creating 2,903 records**. `export` no longer produces such
|
|
9639
|
+
* packages, but one written by hand — or exported before that fix — still can,
|
|
9640
|
+
* and `install` should say so before touching the target.
|
|
8900
9641
|
*/
|
|
8901
|
-
const
|
|
9642
|
+
const findUnsatisfiableRequiredFields = (plan) => {
|
|
9643
|
+
const problems = [];
|
|
9644
|
+
for (const node of walkNodes(plan.tree.nodes)) {
|
|
9645
|
+
if (node.type !== "base") continue;
|
|
9646
|
+
for (const field of node.base.fields) {
|
|
9647
|
+
if (!field.required) continue;
|
|
9648
|
+
const missing = node.records.filter((record) => {
|
|
9649
|
+
const value = record.fields[field.slug];
|
|
9650
|
+
if (value === null || value === void 0 || value === "") return true;
|
|
9651
|
+
return Array.isArray(value) && value.length === 0;
|
|
9652
|
+
}).length;
|
|
9653
|
+
if (missing === 0) continue;
|
|
9654
|
+
problems.push(` • Base "${node.slug}", field "${field.slug}" (${field.type}) is required, but ${missing} of ${node.records.length} record(s) carry no value for it.`);
|
|
9655
|
+
}
|
|
9656
|
+
}
|
|
9657
|
+
return problems;
|
|
9658
|
+
};
|
|
9659
|
+
/**
|
|
9660
|
+
* Everything that would stop this plan from applying, as messages — so a dry
|
|
9661
|
+
* run can REPORT them and a real install can throw on them, from one source of
|
|
9662
|
+
* truth. Previously the checks lived only inside `assertPlanIsApplicable`,
|
|
9663
|
+
* which `--dry-run` returned before ever reaching: a dry run therefore printed
|
|
9664
|
+
* a clean-looking plan for a package that could not install at all.
|
|
9665
|
+
*/
|
|
9666
|
+
const findPlanBlockers = (plan, autoMerge) => {
|
|
9667
|
+
const blockers = [];
|
|
8902
9668
|
if (plan.collisions.some((collision) => !collision.renamedTo)) {
|
|
8903
9669
|
const list = plan.collisions.filter((collision) => !collision.renamedTo).map((collision) => ` • ${collision.kind} "${collision.slug}" (${collision.path})`).join("\n");
|
|
8904
|
-
|
|
9670
|
+
blockers.push(`Install would collide with content that already exists:\n${list}\n\nNothing was created. Re-run with --rename to install the colliding items under suffixed slugs (e.g. "-2"), or pick a different --into-folder.`);
|
|
8905
9671
|
}
|
|
8906
|
-
if (plan.requiresAutoMerge && !autoMerge)
|
|
9672
|
+
if (plan.requiresAutoMerge && !autoMerge) blockers.push("This package's records carry relation values, and a relation stores the ids of the records it points at — which only exist once the records are merged. Installing review-first would leave every relation empty.\n\nRe-run with --auto-merge to install it (this trusts the package author: records are created immediately instead of landing as change requests you review first).");
|
|
9673
|
+
const unsatisfiable = findUnsatisfiableRequiredFields(plan);
|
|
9674
|
+
if (unsatisfiable.length > 0) blockers.push(`This package cannot satisfy its own required fields:\n${unsatisfiable.join("\n")}\n\nThe server rejects these while writing records, partway through the install. Re-export the package (export relaxes fields it cannot fill), or edit the Base definitions in the package to make them optional.`);
|
|
9675
|
+
return blockers;
|
|
9676
|
+
};
|
|
9677
|
+
/**
|
|
9678
|
+
* §4 + §12: without `--auto-merge` an unmerged base has no field ids to patch and no
|
|
9679
|
+
* records to link, so passes 2-5 cannot run. Fail fast and say exactly why.
|
|
9680
|
+
*/
|
|
9681
|
+
const assertPlanIsApplicable = (plan, autoMerge) => {
|
|
9682
|
+
const blockers = findPlanBlockers(plan, autoMerge);
|
|
9683
|
+
if (blockers.length > 0) throw new Error(blockers.join("\n\n"));
|
|
8907
9684
|
};
|
|
8908
9685
|
//#endregion
|
|
8909
9686
|
//#region ../../packages/busabase-package/src/discover.ts
|
|
@@ -9541,9 +10318,12 @@ const runInstall = async (client, repoUrl, options) => {
|
|
|
9541
10318
|
installSampleRecords: !options.noSampleRecords
|
|
9542
10319
|
});
|
|
9543
10320
|
if (options.dryRun) {
|
|
9544
|
-
const
|
|
10321
|
+
const blockers = findPlanBlockers(plan, Boolean(options.autoMerge));
|
|
10322
|
+
const verdict = blockers.length === 0 ? "Dry run — nothing was created. No blockers found." : `Dry run — nothing was created.\n\nThis package would NOT install as-is:\n\n${blockers.join("\n\n")}`;
|
|
10323
|
+
const report = `${renderPlan(plan)}\n\n${verdict}`;
|
|
9545
10324
|
return options.json ? {
|
|
9546
10325
|
dryRun: true,
|
|
10326
|
+
blockers,
|
|
9547
10327
|
...toPlanSummary(plan)
|
|
9548
10328
|
} : report;
|
|
9549
10329
|
}
|
|
@@ -9554,6 +10334,7 @@ const runInstall = async (client, repoUrl, options) => {
|
|
|
9554
10334
|
onProgress: reportProgress,
|
|
9555
10335
|
serverUrl: options.serverUrl,
|
|
9556
10336
|
installSampleRecords: !options.noSampleRecords,
|
|
10337
|
+
rollbackOnFailure: !options.noRollback,
|
|
9557
10338
|
source: {
|
|
9558
10339
|
...github ? {
|
|
9559
10340
|
repo: `${github.owner}/${github.repo}`,
|
|
@@ -9865,67 +10646,6 @@ async function paginateAll(options) {
|
|
|
9865
10646
|
return written;
|
|
9866
10647
|
}
|
|
9867
10648
|
//#endregion
|
|
9868
|
-
//#region src/retry.ts
|
|
9869
|
-
const BASE_DELAY_MS = 500;
|
|
9870
|
-
const MAX_DELAY_MS = 2e4;
|
|
9871
|
-
const READ_METHODS = /* @__PURE__ */ new Set([
|
|
9872
|
-
"GET",
|
|
9873
|
-
"HEAD",
|
|
9874
|
-
"OPTIONS"
|
|
9875
|
-
]);
|
|
9876
|
-
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
9877
|
-
const isNetworkError = (error) => error instanceof TypeError || /fetch failed|econnreset|econnrefused|etimedout|socket hang up/i.test(error instanceof Error ? error.message : String(error));
|
|
9878
|
-
/** `Retry-After` is either delay-seconds or an HTTP-date; both are legal. */
|
|
9879
|
-
function parseRetryAfter(header, now = Date.now()) {
|
|
9880
|
-
if (!header) return void 0;
|
|
9881
|
-
const seconds = Number(header.trim());
|
|
9882
|
-
if (Number.isFinite(seconds) && seconds >= 0) return Math.min(seconds * 1e3, MAX_DELAY_MS);
|
|
9883
|
-
const at = Date.parse(header);
|
|
9884
|
-
if (Number.isNaN(at)) return void 0;
|
|
9885
|
-
return Math.min(Math.max(at - now, 0), MAX_DELAY_MS);
|
|
9886
|
-
}
|
|
9887
|
-
/** Whether this outcome is worth another attempt for a request using `method`. */
|
|
9888
|
-
function shouldRetry(method, status) {
|
|
9889
|
-
if (status === 429) return true;
|
|
9890
|
-
if (!READ_METHODS.has(method.toUpperCase())) return false;
|
|
9891
|
-
return status === void 0 || status >= 500;
|
|
9892
|
-
}
|
|
9893
|
-
/**
|
|
9894
|
-
* Wrap a fetch so transient failures are retried with exponential backoff.
|
|
9895
|
-
* Returns `base` unchanged when `retries` is 0, so the no-retry path adds no
|
|
9896
|
-
* wrapper and no behaviour.
|
|
9897
|
-
*/
|
|
9898
|
-
function withRetry(base, options) {
|
|
9899
|
-
const { retries } = options;
|
|
9900
|
-
if (retries <= 0) return base;
|
|
9901
|
-
const sleep = options.sleep ?? defaultSleep;
|
|
9902
|
-
const notify = options.notify ?? ((message) => console.error(message));
|
|
9903
|
-
const jitter = options.jitter ?? Math.random;
|
|
9904
|
-
return (async (input, init) => {
|
|
9905
|
-
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
|
|
9906
|
-
for (let attempt = 0;; attempt += 1) {
|
|
9907
|
-
let response;
|
|
9908
|
-
let failure;
|
|
9909
|
-
try {
|
|
9910
|
-
response = await base(input, init);
|
|
9911
|
-
} catch (error) {
|
|
9912
|
-
if (!isNetworkError(error)) throw error;
|
|
9913
|
-
failure = error;
|
|
9914
|
-
}
|
|
9915
|
-
const status = response?.status;
|
|
9916
|
-
const last = attempt >= retries;
|
|
9917
|
-
if (!shouldRetry(method, status) || last) {
|
|
9918
|
-
if (response) return response;
|
|
9919
|
-
throw failure;
|
|
9920
|
-
}
|
|
9921
|
-
const backoff = Math.min(BASE_DELAY_MS * 2 ** attempt, MAX_DELAY_MS);
|
|
9922
|
-
const delay = parseRetryAfter(response?.headers.get("retry-after") ?? null) ?? Math.round(backoff * (.5 + jitter() * .5));
|
|
9923
|
-
notify(`[busabase-cli] ${status ? `HTTP ${status}` : "connection failed"} — retrying in ${Math.round(delay / 1e3)}s (attempt ${attempt + 2} of ${retries + 1})`);
|
|
9924
|
-
await sleep(delay);
|
|
9925
|
-
}
|
|
9926
|
-
});
|
|
9927
|
-
}
|
|
9928
|
-
//#endregion
|
|
9929
10649
|
//#region src/task-command.ts
|
|
9930
10650
|
const collect = (value, previous = []) => [...previous, value];
|
|
9931
10651
|
/** Range-checks before the request goes out, so a cap surfaces as a usage error. */
|
|
@@ -10156,14 +10876,22 @@ function resolveConfig(opts) {
|
|
|
10156
10876
|
}));
|
|
10157
10877
|
const file = loadDotEnvFile();
|
|
10158
10878
|
const baseUrl = opts.baseUrl ?? process.env.BUSABASE_BASE_URL ?? file.BUSABASE_BASE_URL ?? DEFAULT_BASE_URL;
|
|
10879
|
+
const retries = opts.retries ?? DEFAULT_RETRIES;
|
|
10159
10880
|
return {
|
|
10160
10881
|
baseUrl,
|
|
10882
|
+
/**
|
|
10883
|
+
* The number itself, not just the `fetch` built from it. `backup` retries
|
|
10884
|
+
* at a level this wrapper cannot reach — see `FullExportOptions.retries` —
|
|
10885
|
+
* and has to honour the same flag, or `--retries` silently means nothing
|
|
10886
|
+
* for the one command whose failures cost hours.
|
|
10887
|
+
*/
|
|
10888
|
+
retries,
|
|
10161
10889
|
webUrl: process.env.BUSABASE_WEB_URL ?? file.BUSABASE_WEB_URL ?? baseUrl,
|
|
10162
10890
|
apiKey: opts.apiKey ?? process.env.BUSABASE_API_KEY ?? file.BUSABASE_API_KEY,
|
|
10163
10891
|
spaceId: opts.spaceId ?? process.env.BUSABASE_SPACE_ID ?? file.BUSABASE_SPACE_ID,
|
|
10164
10892
|
sourceChannel: "cli",
|
|
10165
10893
|
output: opts.output ?? configuredOutput() ?? "text",
|
|
10166
|
-
fetch: withRetry(fetch, { retries
|
|
10894
|
+
fetch: withRetry(fetch, { retries })
|
|
10167
10895
|
};
|
|
10168
10896
|
}
|
|
10169
10897
|
/** Enough to ride out a rate limiter or a proxy blip; not enough to hide a real outage. */
|
|
@@ -10414,6 +11142,24 @@ async function putTextCommand(client, opts) {
|
|
|
10414
11142
|
});
|
|
10415
11143
|
}
|
|
10416
11144
|
/** Wrap a leaf-command handler: resolve config, build the client, render the result. */
|
|
11145
|
+
/**
|
|
11146
|
+
* Is this a 404 — either "this server has no such route" or "no such node"?
|
|
11147
|
+
*
|
|
11148
|
+
* Duck-typed rather than an `instanceof ORPCError` check, because this package
|
|
11149
|
+
* deliberately depends on no `@orpc/*` package; the SDK builds the error and
|
|
11150
|
+
* only its `status` is needed here.
|
|
11151
|
+
*
|
|
11152
|
+
* Not distinguishing the two 404s is safe for the one thing this gates: a
|
|
11153
|
+
* version fallback whose other branch targets the SAME node. An old server
|
|
11154
|
+
* answers 404 because the route is missing and the fallback then succeeds; a
|
|
11155
|
+
* current server answers 404 because the node does not exist, the fallback
|
|
11156
|
+
* asks about the same missing node and 404s too, and the caller still sees a
|
|
11157
|
+
* "not found" error. The cost of guessing wrong is one extra request, not a
|
|
11158
|
+
* write landing somewhere unintended.
|
|
11159
|
+
*/
|
|
11160
|
+
function isNotFound(error) {
|
|
11161
|
+
return typeof error === "object" && error !== null && "status" in error && error.status === 404;
|
|
11162
|
+
}
|
|
10417
11163
|
function runAction(state, handler) {
|
|
10418
11164
|
return async (_opts, cmd) => {
|
|
10419
11165
|
const opts = cmd.optsWithGlobals();
|
|
@@ -10993,7 +11739,7 @@ locale) is reported entry-by-entry and nothing is written. \`--file\`'s
|
|
|
10993
11739
|
contents ARE the array to store; this command does not reshape it.
|
|
10994
11740
|
|
|
10995
11741
|
Examples:
|
|
10996
|
-
busabase-cli nodes set-agent-prompts --node-id nod_123 --file prompts.json`).action(runAction(state, (client, opts) => {
|
|
11742
|
+
busabase-cli nodes set-agent-prompts --node-id nod_123 --file prompts.json`).action(runAction(state, async (client, opts) => {
|
|
10997
11743
|
const filePath = opts.file;
|
|
10998
11744
|
let raw;
|
|
10999
11745
|
try {
|
|
@@ -11006,10 +11752,19 @@ Examples:
|
|
|
11006
11752
|
const detail = formatIssues(parsed.error.issues);
|
|
11007
11753
|
throw new CliOutcomeError("VALIDATION", `Invalid agent prompts in ${filePath}${detail ? ` — ${detail}` : ""}`);
|
|
11008
11754
|
}
|
|
11009
|
-
|
|
11010
|
-
|
|
11011
|
-
|
|
11012
|
-
|
|
11755
|
+
const nodeId = opts.nodeId;
|
|
11756
|
+
try {
|
|
11757
|
+
return await client.nodes.updateAgentPrompts({
|
|
11758
|
+
nodeId,
|
|
11759
|
+
agentPrompts: parsed.data
|
|
11760
|
+
});
|
|
11761
|
+
} catch (error) {
|
|
11762
|
+
if (!isNotFound(error)) throw error;
|
|
11763
|
+
return await client.nodes.updateMetadata({
|
|
11764
|
+
nodeId,
|
|
11765
|
+
metadata: { agentPrompts: parsed.data }
|
|
11766
|
+
});
|
|
11767
|
+
}
|
|
11013
11768
|
}));
|
|
11014
11769
|
const bases = program.command("bases").description("Bases (structured tables)");
|
|
11015
11770
|
addGlobalFlags(bases.command("list")).description("List Bases in the active space").addOption(new Option("--status <active|archived>", "live Bases (default) or archived ones").choices(["active", "archived"])).action(runAction(state, (client, opts) => client.bases.list({ status: opts.status ?? "active" })));
|
|
@@ -11134,7 +11889,7 @@ Examples:
|
|
|
11134
11889
|
startLine: opts.startLine,
|
|
11135
11890
|
endLine: opts.endLine
|
|
11136
11891
|
})));
|
|
11137
|
-
addGlobalFlags(program.command("install")).description("Install a Busabase package from a GitHub repo or a local directory into this space").argument("<source>", "GitHub repo URL (optionally /tree/<ref>[/<subdir>]), or a local directory / file:// URL written by `busabase-cli export`").option("--into-folder <name>", "target folder slug (default: the package's manifest name)").option("--dry-run", "print the plan (tree, record counts, collisions) and create nothing").option("--auto-merge", "merge the package's records and docs on the spot instead of leaving them as change requests to review — this TRUSTS THE PACKAGE AUTHOR, since skills and AirApps carry code your agents will run (default: review first)").option("--rename", "install colliding items under suffixed slugs (-2, -3, …) instead of failing").option("--skill <name>", "which package to install when the URL is a repository holding several (e.g. busabase/skills)").option("--no-sample-records", "propose a template's sample rows for review instead of merging them (default: merged, so the app is not empty on first open)").addHelpText("after", `
|
|
11892
|
+
addGlobalFlags(program.command("install")).description("Install a Busabase package from a GitHub repo or a local directory into this space").argument("<source>", "GitHub repo URL (optionally /tree/<ref>[/<subdir>]), or a local directory / file:// URL written by `busabase-cli export`").option("--into-folder <name>", "target folder slug (default: the package's manifest name)").option("--dry-run", "print the plan (tree, record counts, collisions) and create nothing").option("--auto-merge", "merge the package's records and docs on the spot instead of leaving them as change requests to review — this TRUSTS THE PACKAGE AUTHOR, since skills and AirApps carry code your agents will run (default: review first)").option("--rename", "install colliding items under suffixed slugs (-2, -3, …) instead of failing").option("--skill <name>", "which package to install when the URL is a repository holding several (e.g. busabase/skills)").option("--no-sample-records", "propose a template's sample rows for review instead of merging them (default: merged, so the app is not empty on first open)").option("--no-rollback", "keep whatever a failed install created (default: it is moved to Trash)").addHelpText("after", `
|
|
11138
11893
|
The URL's git ref is the version pin — a tag installs that tag's content forever,
|
|
11139
11894
|
even after the branch moves on. GITHUB_TOKEN is honored for private repos.
|
|
11140
11895
|
|
|
@@ -11174,6 +11929,7 @@ field with nothing linked yet does not trigger this.`).action(runArgAction(state
|
|
|
11174
11929
|
rename: Boolean(opts.rename),
|
|
11175
11930
|
skill: opts.skill,
|
|
11176
11931
|
noSampleRecords: opts.sampleRecords === false,
|
|
11932
|
+
noRollback: opts.rollback === false,
|
|
11177
11933
|
json: config.output === "json",
|
|
11178
11934
|
githubToken: process.env.GITHUB_TOKEN,
|
|
11179
11935
|
serverUrl: config.baseUrl
|
|
@@ -11236,7 +11992,7 @@ and as something \`busabase-cli install\` can install.`).action(runArgAction(sta
|
|
|
11236
11992
|
json: config.output === "json",
|
|
11237
11993
|
baseUrl: config.baseUrl
|
|
11238
11994
|
})));
|
|
11239
|
-
addGlobalFlags(program.command("backup")).description("Back up the active space to a full-fidelity .bbdump archive").option("-o, --out-file <file>", "output archive path", "space.bbdump").option("--no-history", "
|
|
11995
|
+
addGlobalFlags(program.command("backup")).description("Back up the active space to a full-fidelity .bbdump archive").option("-o, --out-file <file>", "output archive path", "space.bbdump").option("--no-history", "drop finished change requests, reviews, comments and audit events (unfinished work is kept)").option("--state-only", "superseded by `busabase-cli export` — see the error text").option("--resume", "continue an interrupted backup, reusing the attachment bytes it already downloaded").addHelpText("after", `
|
|
11240
11996
|
A backup is full fidelity: raw rows with their original ids, every doc body, and
|
|
11241
11997
|
every attachment's bytes, in one integrity-checked archive. Vault items and
|
|
11242
11998
|
webhook signing secrets are never written to it.
|
|
@@ -11244,19 +12000,27 @@ webhook signing secrets are never written to it.
|
|
|
11244
12000
|
Examples:
|
|
11245
12001
|
busabase-cli backup # → ./space.bbdump
|
|
11246
12002
|
busabase-cli backup -o ./backups/acme-2026-07-19.bbdump
|
|
11247
|
-
busabase-cli backup --no-history #
|
|
12003
|
+
busabase-cli backup --no-history # current state + unfinished work
|
|
12004
|
+
|
|
12005
|
+
\`--no-history\` still restores completely: every record, field value, file and
|
|
12006
|
+
permission is there. What it leaves out is the paper trail — change requests
|
|
12007
|
+
that are already finished, their reviews and comments, and the audit log of who
|
|
12008
|
+
did what when. Change requests still in review (or approved but unmerged) are
|
|
12009
|
+
KEPT, because those are unfinished work rather than history.
|
|
11248
12010
|
|
|
11249
12011
|
To hand content to someone else rather than back it up, use \`busabase-cli export\`,
|
|
11250
12012
|
which writes a readable, diffable package directory you can push to GitHub.`).action(runAction(state, (client, opts, config) => runBackup(client, {
|
|
11251
12013
|
outFile: opts.outFile,
|
|
11252
12014
|
includeHistory: opts.history !== false,
|
|
12015
|
+
resume: Boolean(opts.resume),
|
|
11253
12016
|
stateOnly: Boolean(opts.stateOnly),
|
|
11254
12017
|
json: config.output === "json",
|
|
11255
12018
|
baseUrl: config.baseUrl,
|
|
11256
12019
|
spaceId: config.spaceId,
|
|
12020
|
+
retries: config.retries,
|
|
11257
12021
|
toolVersion: pkgVersion()
|
|
11258
12022
|
})));
|
|
11259
|
-
addGlobalFlags(program.command("restore")).description("Restore a .bbdump archive into an EMPTY space, preserving original ids").argument("<file>", "path to the .bbdump archive").option("--resume", "
|
|
12023
|
+
addGlobalFlags(program.command("restore")).description("Restore a .bbdump archive into an EMPTY space, preserving original ids").argument("<file>", "path to the .bbdump archive").option("--resume", "continue a restore whose process was killed partway through (same archive, same space)").option("--into-folder <name>", "state-only archives only — see the error text").addHelpText("after", `
|
|
11260
12024
|
The archive's checksum is verified before a single row is written, and the target
|
|
11261
12025
|
space must be EMPTY — a restore replays original ids, so it cannot merge into a
|
|
11262
12026
|
space that already holds content. Those ids are also unique across the whole
|