@openparachute/vault 0.7.4 → 0.7.5-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.ts +113 -1
- package/src/mirror-import-jobs.ts +197 -0
- package/src/mirror-import.test.ts +102 -2
- package/src/mirror-import.ts +182 -22
- package/src/mirror-routes.test.ts +222 -3
- package/src/mirror-routes.ts +228 -86
- package/src/routing.ts +22 -6
- package/src/server.ts +62 -1
- package/src/transcription/install-whisper-cpp.test.ts +228 -0
- package/src/transcription/install-whisper-cpp.ts +219 -0
- package/src/transcription/install-whisper-exec.test.ts +244 -0
- package/src/transcription/install-whisper-exec.ts +237 -0
- package/src/transcription/models.test.ts +87 -0
- package/src/transcription/models.ts +191 -0
- package/src/transcription/providers/whisper-cpp.test.ts +218 -0
- package/src/transcription/providers/whisper-cpp.ts +241 -0
- package/src/transcription/resolve-binary.test.ts +131 -0
- package/src/transcription/resolve-binary.ts +111 -0
- package/src/transcription/select.ts +26 -3
- package/web/ui/dist/assets/index-CD4kPSY9.js +61 -0
- package/web/ui/dist/index.html +1 -1
- package/web/ui/dist/assets/index-NvwxfZcu.js +0 -61
package/src/mirror-import.ts
CHANGED
|
@@ -18,9 +18,10 @@
|
|
|
18
18
|
* - Creates a temp dir (`os.tmpdir() + /parachute-import-<rand>`).
|
|
19
19
|
* - Resolves the authed clone URL (stored credentials, supplied per-call
|
|
20
20
|
* PAT, or none).
|
|
21
|
-
* - Shells `git clone --depth 1 <authedUrl> <tempDir>` with
|
|
22
|
-
*
|
|
23
|
-
*
|
|
21
|
+
* - Shells `git clone --depth 1 --progress <authedUrl> <tempDir>` with
|
|
22
|
+
* `GIT_TERMINAL_PROMPT=0` so bad credentials fail fast rather than
|
|
23
|
+
* blocking on a stdin prompt. Bounded by a STALL timeout (no progress
|
|
24
|
+
* output for 10 min), not a wall-clock one — see `cloneTimeoutMs`.
|
|
24
25
|
* - Validates the clone looks like a vault export — `.parachute/vault.yaml`
|
|
25
26
|
* must be present. Refuses with a clear error otherwise.
|
|
26
27
|
* - On `mode: "replace"`: wipes notes + tags via `store.deleteNote()` /
|
|
@@ -128,8 +129,28 @@ export interface ImportOpts {
|
|
|
128
129
|
spawn?: GitSpawn;
|
|
129
130
|
/** Override the post-clone import path (test seam — assume cloned dir is a vault export). */
|
|
130
131
|
importer?: typeof importPortableVault;
|
|
131
|
-
/**
|
|
132
|
+
/**
|
|
133
|
+
* Absolute wall-clock cap on the clone. **Defaults to 0 — disabled.**
|
|
134
|
+
*
|
|
135
|
+
* This used to default to 60s, which is why importing any vault bigger than
|
|
136
|
+
* a demo failed with "git clone timed out after 60s" (vault#640). A clone's
|
|
137
|
+
* duration is a function of vault size and link speed; there is no honest
|
|
138
|
+
* wall-clock number that fits both a 200-note vault and a 40k-note one. The
|
|
139
|
+
* bound that actually distinguishes "big" from "broken" is
|
|
140
|
+
* `cloneStallTimeoutMs` below. Tests set this to force the timeout branch.
|
|
141
|
+
*/
|
|
132
142
|
cloneTimeoutMs?: number;
|
|
143
|
+
/**
|
|
144
|
+
* How long the clone may emit NO progress output before we call it wedged.
|
|
145
|
+
* Defaults to `DEFAULT_CLONE_STALL_TIMEOUT_MS` (10 min); `0` disables.
|
|
146
|
+
*/
|
|
147
|
+
cloneStallTimeoutMs?: number;
|
|
148
|
+
/**
|
|
149
|
+
* Progress sink. Called as the import moves between stages and as git
|
|
150
|
+
* reports clone progress. The job registry wires this to the record the
|
|
151
|
+
* status endpoint serves; direct callers can omit it.
|
|
152
|
+
*/
|
|
153
|
+
onProgress?: (update: ImportProgress) => void;
|
|
133
154
|
/**
|
|
134
155
|
* Override the git-presence probe (test seam — defaults to `Bun.which`).
|
|
135
156
|
* Inject a fn returning `null` to exercise the git-not-installed path.
|
|
@@ -137,6 +158,31 @@ export interface ImportOpts {
|
|
|
137
158
|
which?: (cmd: string) => string | null;
|
|
138
159
|
}
|
|
139
160
|
|
|
161
|
+
/**
|
|
162
|
+
* Default stall bound: ten minutes of total silence from `git clone`.
|
|
163
|
+
*
|
|
164
|
+
* Sized off what git actually does — with `--progress` it repaints the
|
|
165
|
+
* counter every few hundred milliseconds while it's moving, so ten minutes of
|
|
166
|
+
* nothing means the transfer is dead, not slow. Generous enough to cover a
|
|
167
|
+
* remote that's slow to enumerate objects on a very large repo before the
|
|
168
|
+
* first byte lands.
|
|
169
|
+
*/
|
|
170
|
+
export const DEFAULT_CLONE_STALL_TIMEOUT_MS = 10 * 60_000;
|
|
171
|
+
|
|
172
|
+
/** Coarse stage the import is in. Drives the SPA's progress copy. */
|
|
173
|
+
export type ImportStage = "cloning" | "importing" | "syncing";
|
|
174
|
+
|
|
175
|
+
/** A progress tick handed to `ImportOpts.onProgress`. */
|
|
176
|
+
export interface ImportProgress {
|
|
177
|
+
stage: ImportStage;
|
|
178
|
+
/**
|
|
179
|
+
* Human-readable detail for the current stage — for `cloning` this is the
|
|
180
|
+
* most recent `git --progress` line ("Receiving objects: 47% …").
|
|
181
|
+
* Absent when the stage has no finer detail to report.
|
|
182
|
+
*/
|
|
183
|
+
detail?: string;
|
|
184
|
+
}
|
|
185
|
+
|
|
140
186
|
/**
|
|
141
187
|
* Counts + warnings returned to the HTTP caller. `notes_imported` totals
|
|
142
188
|
* created+updated so the operator sees a single "imported N notes" number
|
|
@@ -185,13 +231,38 @@ export interface ImportResult {
|
|
|
185
231
|
*/
|
|
186
232
|
export type GitSpawn = (
|
|
187
233
|
argv: string[],
|
|
188
|
-
options:
|
|
234
|
+
options: GitSpawnOptions,
|
|
189
235
|
) => Promise<GitSpawnResult>;
|
|
190
236
|
|
|
237
|
+
export interface GitSpawnOptions {
|
|
238
|
+
cwd?: string;
|
|
239
|
+
/**
|
|
240
|
+
* Wall-clock cap on the whole command. `0` disables it — the default for
|
|
241
|
+
* clones (see `ImportOpts.cloneTimeoutMs`). Tests pass a small number to
|
|
242
|
+
* exercise the timeout branch.
|
|
243
|
+
*/
|
|
244
|
+
timeoutMs: number;
|
|
245
|
+
/**
|
|
246
|
+
* Cap on the gap BETWEEN progress lines. This — not `timeoutMs` — is what
|
|
247
|
+
* bounds a real clone: a 4 GB vault legitimately takes an hour, but a clone
|
|
248
|
+
* that has emitted nothing for 10 minutes is wedged (dead TCP connection,
|
|
249
|
+
* auth prompt we failed to suppress, remote hang). `0` disables it.
|
|
250
|
+
*/
|
|
251
|
+
stallTimeoutMs?: number;
|
|
252
|
+
/**
|
|
253
|
+
* Called for each line git writes to stderr, which with `--progress` is
|
|
254
|
+
* where "Receiving objects: 47% (…)" lands. Drives the job's live progress
|
|
255
|
+
* detail and resets the stall timer.
|
|
256
|
+
*/
|
|
257
|
+
onProgress?: (line: string) => void;
|
|
258
|
+
}
|
|
259
|
+
|
|
191
260
|
export interface GitSpawnResult {
|
|
192
261
|
exitCode: number;
|
|
193
262
|
stderr: string;
|
|
194
263
|
timedOut: boolean;
|
|
264
|
+
/** True when the kill came from the stall timer rather than `timeoutMs`. */
|
|
265
|
+
stalled?: boolean;
|
|
195
266
|
}
|
|
196
267
|
|
|
197
268
|
// ---------------------------------------------------------------------------
|
|
@@ -344,8 +415,27 @@ export function authedCloneUrl(
|
|
|
344
415
|
// ---------------------------------------------------------------------------
|
|
345
416
|
|
|
346
417
|
/**
|
|
347
|
-
* Run a git command with a
|
|
348
|
-
* exit code + stderr text +
|
|
418
|
+
* Run a git command with a non-interactive env, streaming stderr so the
|
|
419
|
+
* caller sees progress while it runs. Returns exit code + stderr text +
|
|
420
|
+
* timeout flags.
|
|
421
|
+
*
|
|
422
|
+
* **Why stderr is STREAMED, not buffered (vault#640).** The previous
|
|
423
|
+
* implementation awaited `proc.exited` and only then drained `proc.stderr`.
|
|
424
|
+
* That made progress structurally unobservable — the import was a black box
|
|
425
|
+
* until it finished — and it forced the caller to bound the clone with a
|
|
426
|
+
* wall-clock timeout, because a wedged clone and a slow one look identical
|
|
427
|
+
* when you can't see output. A 60s cap was the result, and it made vaults
|
|
428
|
+
* above a few thousand notes simply un-importable.
|
|
429
|
+
*
|
|
430
|
+
* Streaming lets us bound the RIGHT thing: the gap between progress lines
|
|
431
|
+
* (`stallTimeoutMs`). A big clone runs as long as it needs to as long as it's
|
|
432
|
+
* still moving; a wedged one dies in minutes. `timeoutMs` remains as an
|
|
433
|
+
* optional absolute ceiling (0 = disabled) mainly so tests can force the
|
|
434
|
+
* timeout branch deterministically.
|
|
435
|
+
*
|
|
436
|
+
* Draining stderr concurrently also fixes a latent deadlock: git blocks
|
|
437
|
+
* writing to a full stderr pipe, so a clone chatty enough to fill the pipe
|
|
438
|
+
* buffer would hang forever against the old await-then-read order.
|
|
349
439
|
*/
|
|
350
440
|
export const defaultGitSpawn: GitSpawn = async (argv, options) => {
|
|
351
441
|
let proc;
|
|
@@ -375,20 +465,74 @@ export const defaultGitSpawn: GitSpawn = async (argv, options) => {
|
|
|
375
465
|
throw err;
|
|
376
466
|
}
|
|
377
467
|
let timedOut = false;
|
|
378
|
-
|
|
379
|
-
|
|
468
|
+
let stalled = false;
|
|
469
|
+
const kill = () => {
|
|
380
470
|
try {
|
|
381
471
|
proc.kill();
|
|
382
472
|
} catch {
|
|
383
473
|
// already exited
|
|
384
474
|
}
|
|
385
|
-
}
|
|
475
|
+
};
|
|
476
|
+
|
|
477
|
+
const absoluteTimer =
|
|
478
|
+
options.timeoutMs > 0
|
|
479
|
+
? setTimeout(() => {
|
|
480
|
+
timedOut = true;
|
|
481
|
+
kill();
|
|
482
|
+
}, options.timeoutMs)
|
|
483
|
+
: null;
|
|
484
|
+
|
|
485
|
+
// Stall timer — rearmed on every line git emits. This is the real guard.
|
|
486
|
+
let stallTimer: ReturnType<typeof setTimeout> | null = null;
|
|
487
|
+
const stallMs = options.stallTimeoutMs ?? 0;
|
|
488
|
+
const armStall = () => {
|
|
489
|
+
if (stallMs <= 0) return;
|
|
490
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
491
|
+
stallTimer = setTimeout(() => {
|
|
492
|
+
stalled = true;
|
|
493
|
+
timedOut = true;
|
|
494
|
+
kill();
|
|
495
|
+
}, stallMs);
|
|
496
|
+
};
|
|
497
|
+
armStall();
|
|
498
|
+
|
|
499
|
+
// Drain stderr line-by-line. git writes progress with `\r` (carriage
|
|
500
|
+
// return, no newline) so it can repaint one line in a terminal — split on
|
|
501
|
+
// BOTH so "Receiving objects: 47%" surfaces as it happens rather than
|
|
502
|
+
// arriving as one giant line at the end.
|
|
503
|
+
const collected: string[] = [];
|
|
504
|
+
let pending = "";
|
|
505
|
+
const emit = (line: string) => {
|
|
506
|
+
const trimmed = line.trim();
|
|
507
|
+
if (trimmed.length === 0) return;
|
|
508
|
+
collected.push(trimmed);
|
|
509
|
+
armStall();
|
|
510
|
+
try {
|
|
511
|
+
options.onProgress?.(trimmed);
|
|
512
|
+
} catch {
|
|
513
|
+
// A throwing progress callback must never take down the clone.
|
|
514
|
+
}
|
|
515
|
+
};
|
|
516
|
+
const drain = (async () => {
|
|
517
|
+
const decoder = new TextDecoder();
|
|
518
|
+
for await (const chunk of proc.stderr) {
|
|
519
|
+
pending += decoder.decode(chunk, { stream: true });
|
|
520
|
+
const parts = pending.split(/[\r\n]+/);
|
|
521
|
+
pending = parts.pop() ?? "";
|
|
522
|
+
for (const part of parts) emit(part);
|
|
523
|
+
}
|
|
524
|
+
if (pending.length > 0) emit(pending);
|
|
525
|
+
})().catch(() => {
|
|
526
|
+
// Stream errors (killed process mid-read) aren't interesting — the exit
|
|
527
|
+
// code and the timeout flags already describe what happened.
|
|
528
|
+
});
|
|
529
|
+
|
|
386
530
|
const exitCode = await proc.exited;
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
return { exitCode, stderr, timedOut };
|
|
531
|
+
await drain;
|
|
532
|
+
if (absoluteTimer) clearTimeout(absoluteTimer);
|
|
533
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
534
|
+
|
|
535
|
+
return { exitCode, stderr: collected.join("\n").trim(), timedOut, stalled };
|
|
392
536
|
};
|
|
393
537
|
|
|
394
538
|
// ---------------------------------------------------------------------------
|
|
@@ -427,7 +571,12 @@ export async function cloneAndImport(opts: ImportOpts): Promise<ImportResult> {
|
|
|
427
571
|
const spawn = opts.spawn ?? defaultGitSpawn;
|
|
428
572
|
const importer = opts.importer ?? importPortableVault;
|
|
429
573
|
const workDirRoot = opts.workDirRoot ?? tmpdir();
|
|
430
|
-
|
|
574
|
+
// 0 = no absolute ceiling; the stall bound is the real guard. See the
|
|
575
|
+
// `cloneTimeoutMs` docstring on ImportOpts for why the old 60s default was
|
|
576
|
+
// wrong rather than merely too small.
|
|
577
|
+
const cloneTimeoutMs = opts.cloneTimeoutMs ?? 0;
|
|
578
|
+
const cloneStallTimeoutMs =
|
|
579
|
+
opts.cloneStallTimeoutMs ?? DEFAULT_CLONE_STALL_TIMEOUT_MS;
|
|
431
580
|
|
|
432
581
|
const authResult = authedCloneUrl(opts.remoteUrl, opts.auth, opts.vaultName);
|
|
433
582
|
if (!authResult) {
|
|
@@ -440,15 +589,25 @@ export async function cloneAndImport(opts: ImportOpts): Promise<ImportResult> {
|
|
|
440
589
|
|
|
441
590
|
const tempDir = mkdtempSync(join(workDirRoot, "parachute-import-"));
|
|
442
591
|
try {
|
|
592
|
+
opts.onProgress?.({ stage: "cloning" });
|
|
443
593
|
const cloneResult = await spawn(
|
|
444
|
-
|
|
445
|
-
|
|
594
|
+
// `--progress` forces the progress meter even though our stderr is a
|
|
595
|
+
// pipe, not a tty. Without it git stays silent, the stall timer has
|
|
596
|
+
// nothing to observe, and the operator watches a spinner with no
|
|
597
|
+
// information for the length of a multi-GB transfer.
|
|
598
|
+
["git", "clone", "--depth", "1", "--progress", authedUrl, tempDir],
|
|
599
|
+
{
|
|
600
|
+
timeoutMs: cloneTimeoutMs,
|
|
601
|
+
stallTimeoutMs: cloneStallTimeoutMs,
|
|
602
|
+
onProgress: (line) => opts.onProgress?.({ stage: "cloning", detail: line }),
|
|
603
|
+
},
|
|
446
604
|
);
|
|
447
605
|
if (cloneResult.timedOut) {
|
|
448
|
-
|
|
449
|
-
`git clone
|
|
450
|
-
`
|
|
451
|
-
|
|
606
|
+
const why = cloneResult.stalled
|
|
607
|
+
? `git clone stalled — no progress for ${Math.floor(cloneStallTimeoutMs / 60_000)} minutes. ` +
|
|
608
|
+
`The remote stopped responding, or the credential was rejected without an error.`
|
|
609
|
+
: `git clone exceeded its ${Math.floor(cloneTimeoutMs / 1000)}s limit.`;
|
|
610
|
+
throw new CloneFailedError(`${why} URL: ${redactRemoteUrl(opts.remoteUrl)}`);
|
|
452
611
|
}
|
|
453
612
|
if (cloneResult.exitCode !== 0) {
|
|
454
613
|
// Redact any leaked URLs in stderr — git error messages echo them.
|
|
@@ -469,6 +628,7 @@ export async function cloneAndImport(opts: ImportOpts): Promise<ImportResult> {
|
|
|
469
628
|
// Delegate to the importer. `blowAway: true` for replace mode triggers
|
|
470
629
|
// the wipe-then-import path (deletes notes via the public store API
|
|
471
630
|
// so hooks fire); `false` for merge does upsert-by-id.
|
|
631
|
+
opts.onProgress?.({ stage: "importing" });
|
|
472
632
|
const stats: ImportStats = await importer(opts.store, {
|
|
473
633
|
inDir: tempDir,
|
|
474
634
|
blowAway: opts.mode === "replace",
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
handleAuthPat,
|
|
35
35
|
handleMirrorGet,
|
|
36
36
|
handleMirrorImport,
|
|
37
|
+
handleMirrorImportStatus,
|
|
37
38
|
handleMirrorPushNow,
|
|
38
39
|
handleMirrorPut,
|
|
39
40
|
handleMirrorRunNow,
|
|
@@ -42,6 +43,7 @@ import {
|
|
|
42
43
|
_resetImportInFlightForTest,
|
|
43
44
|
type GitSpawn,
|
|
44
45
|
} from "./mirror-import.ts";
|
|
46
|
+
import { _resetImportJobsForTest, type ImportJob } from "./mirror-import-jobs.ts";
|
|
45
47
|
import { writeVaultConfig } from "./config.ts";
|
|
46
48
|
import { clearVaultStoreCache } from "./vault-store.ts";
|
|
47
49
|
import { exportVaultToDir } from "../core/src/portable-md.ts";
|
|
@@ -1984,6 +1986,7 @@ describe("handleMirrorImport", () => {
|
|
|
1984
1986
|
if (home) fs.rmSync(home, { recursive: true, force: true });
|
|
1985
1987
|
if (fixture) fs.rmSync(fixture, { recursive: true, force: true });
|
|
1986
1988
|
_resetImportInFlightForTest();
|
|
1989
|
+
_resetImportJobsForTest();
|
|
1987
1990
|
clearVaultStoreCache();
|
|
1988
1991
|
});
|
|
1989
1992
|
|
|
@@ -2034,6 +2037,7 @@ describe("handleMirrorImport", () => {
|
|
|
2034
2037
|
body: JSON.stringify({
|
|
2035
2038
|
remote_url: "https://github.com/a/b.git",
|
|
2036
2039
|
mode: "merge",
|
|
2040
|
+
wait: true,
|
|
2037
2041
|
credentials: { kind: "pat", token: "" },
|
|
2038
2042
|
}),
|
|
2039
2043
|
});
|
|
@@ -2051,6 +2055,7 @@ describe("handleMirrorImport", () => {
|
|
|
2051
2055
|
body: JSON.stringify({
|
|
2052
2056
|
remote_url: "https://github.com/a/b.git",
|
|
2053
2057
|
mode: "merge",
|
|
2058
|
+
wait: true,
|
|
2054
2059
|
credentials: { kind: "magic" },
|
|
2055
2060
|
}),
|
|
2056
2061
|
});
|
|
@@ -2069,6 +2074,7 @@ describe("handleMirrorImport", () => {
|
|
|
2069
2074
|
body: JSON.stringify({
|
|
2070
2075
|
remote_url: "https://github.com/a/b.git",
|
|
2071
2076
|
mode: "merge",
|
|
2077
|
+
wait: true,
|
|
2072
2078
|
credentials: { kind: "none" },
|
|
2073
2079
|
}),
|
|
2074
2080
|
});
|
|
@@ -2100,6 +2106,7 @@ describe("handleMirrorImport", () => {
|
|
|
2100
2106
|
body: JSON.stringify({
|
|
2101
2107
|
remote_url: "https://github.com/a/b.git",
|
|
2102
2108
|
mode: "replace",
|
|
2109
|
+
wait: true,
|
|
2103
2110
|
credentials: { kind: "none" },
|
|
2104
2111
|
}),
|
|
2105
2112
|
});
|
|
@@ -2121,6 +2128,7 @@ describe("handleMirrorImport", () => {
|
|
|
2121
2128
|
body: JSON.stringify({
|
|
2122
2129
|
remote_url: "https://github.com/a/b.git",
|
|
2123
2130
|
mode: "merge",
|
|
2131
|
+
wait: true,
|
|
2124
2132
|
credentials: { kind: "pat", token: "ghp_secret_xyz" },
|
|
2125
2133
|
}),
|
|
2126
2134
|
});
|
|
@@ -2143,6 +2151,7 @@ describe("handleMirrorImport", () => {
|
|
|
2143
2151
|
body: JSON.stringify({
|
|
2144
2152
|
remote_url: "https://github.com/a/b.git",
|
|
2145
2153
|
mode: "merge",
|
|
2154
|
+
wait: true,
|
|
2146
2155
|
credentials: { kind: "none" },
|
|
2147
2156
|
}),
|
|
2148
2157
|
});
|
|
@@ -2169,6 +2178,7 @@ describe("handleMirrorImport", () => {
|
|
|
2169
2178
|
body: JSON.stringify({
|
|
2170
2179
|
remote_url: "https://github.com/a/b.git",
|
|
2171
2180
|
mode: "merge",
|
|
2181
|
+
wait: true,
|
|
2172
2182
|
credentials: { kind: "none" },
|
|
2173
2183
|
}),
|
|
2174
2184
|
});
|
|
@@ -2210,6 +2220,7 @@ describe("handleMirrorImport", () => {
|
|
|
2210
2220
|
body: JSON.stringify({
|
|
2211
2221
|
remote_url: "https://github.com/a/b.git",
|
|
2212
2222
|
mode: "merge",
|
|
2223
|
+
wait: true,
|
|
2213
2224
|
credentials: null,
|
|
2214
2225
|
}),
|
|
2215
2226
|
});
|
|
@@ -2246,6 +2257,7 @@ describe("handleMirrorImport", () => {
|
|
|
2246
2257
|
body: JSON.stringify({
|
|
2247
2258
|
remote_url: "https://github.com/a/b.git",
|
|
2248
2259
|
mode: "merge",
|
|
2260
|
+
wait: true,
|
|
2249
2261
|
credentials: { kind: "pat", token: "ghp_per_call_only" },
|
|
2250
2262
|
}),
|
|
2251
2263
|
});
|
|
@@ -2272,6 +2284,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2272
2284
|
if (home) fs.rmSync(home, { recursive: true, force: true });
|
|
2273
2285
|
if (fixture) fs.rmSync(fixture, { recursive: true, force: true });
|
|
2274
2286
|
_resetImportInFlightForTest();
|
|
2287
|
+
_resetImportJobsForTest();
|
|
2275
2288
|
clearVaultStoreCache();
|
|
2276
2289
|
});
|
|
2277
2290
|
|
|
@@ -2286,6 +2299,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2286
2299
|
body: JSON.stringify({
|
|
2287
2300
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2288
2301
|
mode: "merge",
|
|
2302
|
+
wait: true,
|
|
2289
2303
|
credentials: { kind: "pat", token: "ghp_import_token_abc" },
|
|
2290
2304
|
enable_sync: true,
|
|
2291
2305
|
}),
|
|
@@ -2330,6 +2344,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2330
2344
|
body: JSON.stringify({
|
|
2331
2345
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2332
2346
|
mode: "merge",
|
|
2347
|
+
wait: true,
|
|
2333
2348
|
credentials: { kind: "pat", token: "ghp_import_token_abc" },
|
|
2334
2349
|
enable_sync: false,
|
|
2335
2350
|
}),
|
|
@@ -2365,6 +2380,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2365
2380
|
body: JSON.stringify({
|
|
2366
2381
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2367
2382
|
mode: "merge",
|
|
2383
|
+
wait: true,
|
|
2368
2384
|
credentials: { kind: "none" },
|
|
2369
2385
|
enable_sync: true,
|
|
2370
2386
|
}),
|
|
@@ -2392,7 +2408,12 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2392
2408
|
expect(readCredentials("default")).toBeNull();
|
|
2393
2409
|
});
|
|
2394
2410
|
|
|
2395
|
-
|
|
2411
|
+
// vault#641 — the default INVERTED. Import is a read; it must not arm a
|
|
2412
|
+
// write to the source repo unless the operator says so. Pinned as its own
|
|
2413
|
+
// test because the old default (ON, vault#416) was the single most dangerous
|
|
2414
|
+
// thing about this flow: pulling a vault onto a new box silently repointed
|
|
2415
|
+
// backup at the repo you pulled from.
|
|
2416
|
+
test("enable_sync defaults to FALSE when omitted — import never arms push-back on its own", async () => {
|
|
2396
2417
|
home = tmp("import-sync-default-");
|
|
2397
2418
|
await bootstrapVault(home);
|
|
2398
2419
|
fixture = await buildExportFixture();
|
|
@@ -2403,8 +2424,44 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2403
2424
|
body: JSON.stringify({
|
|
2404
2425
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2405
2426
|
mode: "merge",
|
|
2406
|
-
|
|
2407
|
-
|
|
2427
|
+
wait: true,
|
|
2428
|
+
credentials: { kind: "pat", token: "ghp_default_off_token" },
|
|
2429
|
+
// enable_sync omitted — must default OFF.
|
|
2430
|
+
}),
|
|
2431
|
+
});
|
|
2432
|
+
const res = await handleMirrorImport(
|
|
2433
|
+
req,
|
|
2434
|
+
"default",
|
|
2435
|
+
spawnCloneSuccess(fixture),
|
|
2436
|
+
undefined,
|
|
2437
|
+
manager,
|
|
2438
|
+
);
|
|
2439
|
+
expect(res.status).toBe(200);
|
|
2440
|
+
const body = (await res.json()) as {
|
|
2441
|
+
sync_enabled: boolean;
|
|
2442
|
+
sync_warning?: string;
|
|
2443
|
+
};
|
|
2444
|
+
expect(body.sync_enabled).toBe(false);
|
|
2445
|
+
// Not a failure — the operator didn't ask. No scary warning either.
|
|
2446
|
+
expect(body.sync_warning).toBeUndefined();
|
|
2447
|
+
// The decisive assertion: nothing was wired up.
|
|
2448
|
+
expect(readMirrorConfigForVault("default")?.auto_push ?? false).toBe(false);
|
|
2449
|
+
});
|
|
2450
|
+
|
|
2451
|
+
test("enable_sync: true still opts in explicitly", async () => {
|
|
2452
|
+
home = tmp("import-sync-optin-");
|
|
2453
|
+
await bootstrapVault(home);
|
|
2454
|
+
fixture = await buildExportFixture();
|
|
2455
|
+
manager = makeSyncManager(home);
|
|
2456
|
+
|
|
2457
|
+
const req = new Request("http://x/import", {
|
|
2458
|
+
method: "POST",
|
|
2459
|
+
body: JSON.stringify({
|
|
2460
|
+
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2461
|
+
mode: "merge",
|
|
2462
|
+
wait: true,
|
|
2463
|
+
enable_sync: true,
|
|
2464
|
+
credentials: { kind: "pat", token: "ghp_opt_in_token" },
|
|
2408
2465
|
}),
|
|
2409
2466
|
});
|
|
2410
2467
|
const res = await handleMirrorImport(
|
|
@@ -2448,6 +2505,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2448
2505
|
body: JSON.stringify({
|
|
2449
2506
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2450
2507
|
mode: "merge",
|
|
2508
|
+
wait: true,
|
|
2451
2509
|
credentials: { kind: "pat", token: "ghp_import_token_abc" },
|
|
2452
2510
|
enable_sync: true,
|
|
2453
2511
|
}),
|
|
@@ -2500,6 +2558,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2500
2558
|
body: JSON.stringify({
|
|
2501
2559
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2502
2560
|
mode: "merge",
|
|
2561
|
+
wait: true,
|
|
2503
2562
|
credentials: { kind: "pat", token: "ghp_import_token_abc" },
|
|
2504
2563
|
enable_sync: true,
|
|
2505
2564
|
}),
|
|
@@ -2555,6 +2614,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2555
2614
|
body: JSON.stringify({
|
|
2556
2615
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2557
2616
|
mode: "merge",
|
|
2617
|
+
wait: true,
|
|
2558
2618
|
credentials: { kind: "pat", token: "ghp_import_token_abc" },
|
|
2559
2619
|
enable_sync: true,
|
|
2560
2620
|
override: true,
|
|
@@ -2602,6 +2662,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2602
2662
|
body: JSON.stringify({
|
|
2603
2663
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2604
2664
|
mode: "merge",
|
|
2665
|
+
wait: true,
|
|
2605
2666
|
credentials: { kind: "pat", token: "ghp_same_token" },
|
|
2606
2667
|
enable_sync: true,
|
|
2607
2668
|
}),
|
|
@@ -2650,6 +2711,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2650
2711
|
body: JSON.stringify({
|
|
2651
2712
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2652
2713
|
mode: "merge",
|
|
2714
|
+
wait: true,
|
|
2653
2715
|
credentials: { kind: "pat", token: "ghp_import_token_abc" },
|
|
2654
2716
|
enable_sync: true,
|
|
2655
2717
|
}),
|
|
@@ -2693,6 +2755,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2693
2755
|
body: JSON.stringify({
|
|
2694
2756
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2695
2757
|
mode: "merge",
|
|
2758
|
+
wait: true,
|
|
2696
2759
|
credentials: { kind: "pat", token: "ghp_import_token_abc" },
|
|
2697
2760
|
enable_sync: true,
|
|
2698
2761
|
}),
|
|
@@ -2724,6 +2787,7 @@ describe("handleMirrorImport — auto-enable sync (vault#416)", () => {
|
|
|
2724
2787
|
body: JSON.stringify({
|
|
2725
2788
|
remote_url: "https://github.com/aaron/my-vault.git",
|
|
2726
2789
|
mode: "merge",
|
|
2790
|
+
wait: true,
|
|
2727
2791
|
credentials: { kind: "none" },
|
|
2728
2792
|
enable_sync: "yes",
|
|
2729
2793
|
}),
|
|
@@ -2769,6 +2833,161 @@ function seedSiblingVault(name: string, originUrl: string): void {
|
|
|
2769
2833
|
);
|
|
2770
2834
|
}
|
|
2771
2835
|
|
|
2836
|
+
// ---------------------------------------------------------------------------
|
|
2837
|
+
// vault#640 — the ASYNC import transport.
|
|
2838
|
+
//
|
|
2839
|
+
// The semantic tests above (what gets imported, how sync gets wired, which
|
|
2840
|
+
// credential is picked) run through `wait: true`, the back-compat synchronous
|
|
2841
|
+
// path. That's deliberate, not laziness: both paths call the SAME `runImport`
|
|
2842
|
+
// closure, so transport and semantics are genuinely orthogonal and testing
|
|
2843
|
+
// each import behaviour twice would buy nothing. This block covers the part
|
|
2844
|
+
// that IS different — the job lifecycle the SPA actually drives.
|
|
2845
|
+
// ---------------------------------------------------------------------------
|
|
2846
|
+
describe("handleMirrorImport — async job transport (vault#640)", () => {
|
|
2847
|
+
let home: string;
|
|
2848
|
+
let fixture: string;
|
|
2849
|
+
|
|
2850
|
+
afterEach(() => {
|
|
2851
|
+
if (home) fs.rmSync(home, { recursive: true, force: true });
|
|
2852
|
+
if (fixture) fs.rmSync(fixture, { recursive: true, force: true });
|
|
2853
|
+
_resetImportInFlightForTest();
|
|
2854
|
+
_resetImportJobsForTest();
|
|
2855
|
+
clearVaultStoreCache();
|
|
2856
|
+
});
|
|
2857
|
+
|
|
2858
|
+
/** POST an import, asserting the 202, and hand back the job record. */
|
|
2859
|
+
async function startImport(
|
|
2860
|
+
spawn: GitSpawn,
|
|
2861
|
+
body: Record<string, unknown> = {},
|
|
2862
|
+
): Promise<ImportJob> {
|
|
2863
|
+
const req = new Request("http://x/import", {
|
|
2864
|
+
method: "POST",
|
|
2865
|
+
body: JSON.stringify({
|
|
2866
|
+
remote_url: "https://github.com/a/b.git",
|
|
2867
|
+
mode: "merge",
|
|
2868
|
+
credentials: { kind: "none" },
|
|
2869
|
+
...body,
|
|
2870
|
+
}),
|
|
2871
|
+
});
|
|
2872
|
+
const res = await handleMirrorImport(req, "default", spawn);
|
|
2873
|
+
expect(res.status).toBe(202);
|
|
2874
|
+
return (await res.json()) as ImportJob;
|
|
2875
|
+
}
|
|
2876
|
+
|
|
2877
|
+
/** Poll the status route until the job leaves `running`. */
|
|
2878
|
+
async function pollUntilDone(jobId: string, vault = "default"): Promise<ImportJob> {
|
|
2879
|
+
for (let i = 0; i < 200; i++) {
|
|
2880
|
+
const res = handleMirrorImportStatus(vault, jobId);
|
|
2881
|
+
const job = (await res.json()) as ImportJob;
|
|
2882
|
+
if (job.status !== "running") return job;
|
|
2883
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
2884
|
+
}
|
|
2885
|
+
throw new Error("import job never reached a terminal state");
|
|
2886
|
+
}
|
|
2887
|
+
|
|
2888
|
+
test("POST returns 202 with a running job, not the result", async () => {
|
|
2889
|
+
home = tmp("import-async-202-");
|
|
2890
|
+
await bootstrapVault(home);
|
|
2891
|
+
fixture = await buildExportFixture();
|
|
2892
|
+
|
|
2893
|
+
const job = await startImport(spawnCloneSuccess(fixture));
|
|
2894
|
+
expect(job.job_id).toBeTruthy();
|
|
2895
|
+
expect(job.status).toBe("running");
|
|
2896
|
+
expect(job.stage).toBe("cloning");
|
|
2897
|
+
expect(job.vault_name).toBe("default");
|
|
2898
|
+
|
|
2899
|
+
await pollUntilDone(job.job_id);
|
|
2900
|
+
});
|
|
2901
|
+
|
|
2902
|
+
test("polling reaches succeeded and carries the import result", async () => {
|
|
2903
|
+
home = tmp("import-async-ok-");
|
|
2904
|
+
await bootstrapVault(home);
|
|
2905
|
+
fixture = await buildExportFixture();
|
|
2906
|
+
|
|
2907
|
+
const started = await startImport(spawnCloneSuccess(fixture));
|
|
2908
|
+
const done = await pollUntilDone(started.job_id);
|
|
2909
|
+
|
|
2910
|
+
expect(done.status).toBe("succeeded");
|
|
2911
|
+
expect(done.finished_at).toBeTruthy();
|
|
2912
|
+
expect(done.result?.notes_imported).toBe(2);
|
|
2913
|
+
expect(done.error).toBeUndefined();
|
|
2914
|
+
});
|
|
2915
|
+
|
|
2916
|
+
test("a clone failure lands on the job as clone_failed, with the token redacted", async () => {
|
|
2917
|
+
home = tmp("import-async-fail-");
|
|
2918
|
+
await bootstrapVault(home);
|
|
2919
|
+
|
|
2920
|
+
const started = await startImport(spawnCloneFail, {
|
|
2921
|
+
credentials: { kind: "pat", token: "ghp_secret_xyz" },
|
|
2922
|
+
});
|
|
2923
|
+
const done = await pollUntilDone(started.job_id);
|
|
2924
|
+
|
|
2925
|
+
expect(done.status).toBe("failed");
|
|
2926
|
+
expect(done.error?.error_type).toBe("clone_failed");
|
|
2927
|
+
expect(JSON.stringify(done)).not.toContain("ghp_secret_xyz");
|
|
2928
|
+
});
|
|
2929
|
+
|
|
2930
|
+
test("a second import while one is running → 409 carrying the in-flight job_id", async () => {
|
|
2931
|
+
home = tmp("import-async-conflict-");
|
|
2932
|
+
await bootstrapVault(home);
|
|
2933
|
+
fixture = await buildExportFixture();
|
|
2934
|
+
|
|
2935
|
+
const first = await startImport(spawnCloneSuccess(fixture));
|
|
2936
|
+
const req = new Request("http://x/import", {
|
|
2937
|
+
method: "POST",
|
|
2938
|
+
body: JSON.stringify({
|
|
2939
|
+
remote_url: "https://github.com/a/b.git",
|
|
2940
|
+
mode: "merge",
|
|
2941
|
+
credentials: { kind: "none" },
|
|
2942
|
+
}),
|
|
2943
|
+
});
|
|
2944
|
+
const res = await handleMirrorImport(req, "default", spawnCloneSuccess(fixture));
|
|
2945
|
+
expect(res.status).toBe(409);
|
|
2946
|
+
const body = (await res.json()) as { error_type: string; job_id?: string };
|
|
2947
|
+
expect(body.error_type).toBe("concurrent_import");
|
|
2948
|
+
// The second tab can attach to the running import instead of being stuck.
|
|
2949
|
+
expect(body.job_id).toBe(first.job_id);
|
|
2950
|
+
|
|
2951
|
+
await pollUntilDone(first.job_id);
|
|
2952
|
+
});
|
|
2953
|
+
|
|
2954
|
+
test("a finished job frees the vault for the next import", async () => {
|
|
2955
|
+
home = tmp("import-async-serial-");
|
|
2956
|
+
await bootstrapVault(home);
|
|
2957
|
+
fixture = await buildExportFixture();
|
|
2958
|
+
|
|
2959
|
+
const first = await startImport(spawnCloneSuccess(fixture));
|
|
2960
|
+
await pollUntilDone(first.job_id);
|
|
2961
|
+
|
|
2962
|
+
const second = await startImport(spawnCloneSuccess(fixture));
|
|
2963
|
+
expect(second.job_id).not.toBe(first.job_id);
|
|
2964
|
+
await pollUntilDone(second.job_id);
|
|
2965
|
+
});
|
|
2966
|
+
|
|
2967
|
+
test("status is scoped to the vault — another vault's admin can't read the job", async () => {
|
|
2968
|
+
home = tmp("import-async-scope-");
|
|
2969
|
+
await bootstrapVault(home);
|
|
2970
|
+
fixture = await buildExportFixture();
|
|
2971
|
+
|
|
2972
|
+
const started = await startImport(spawnCloneSuccess(fixture));
|
|
2973
|
+
const res = handleMirrorImportStatus("some-other-vault", started.job_id);
|
|
2974
|
+
expect(res.status).toBe(404);
|
|
2975
|
+
const body = (await res.json()) as { error_type: string };
|
|
2976
|
+
expect(body.error_type).toBe("job_not_found");
|
|
2977
|
+
|
|
2978
|
+
await pollUntilDone(started.job_id);
|
|
2979
|
+
});
|
|
2980
|
+
|
|
2981
|
+
test("unknown job id → 404 job_not_found", async () => {
|
|
2982
|
+
home = tmp("import-async-404-");
|
|
2983
|
+
await bootstrapVault(home);
|
|
2984
|
+
const res = handleMirrorImportStatus("default", "no-such-job");
|
|
2985
|
+
expect(res.status).toBe(404);
|
|
2986
|
+
const body = (await res.json()) as { error_type: string };
|
|
2987
|
+
expect(body.error_type).toBe("job_not_found");
|
|
2988
|
+
});
|
|
2989
|
+
});
|
|
2990
|
+
|
|
2772
2991
|
describe("cross-vault remote-clobber guard (vault#482)", () => {
|
|
2773
2992
|
let home: string;
|
|
2774
2993
|
afterEach(() => {
|