@openparachute/vault 0.6.4 → 0.6.5-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/__fixtures__/golden-vault.ts +125 -0
- package/core/src/__fixtures__/portable-export-golden.json +10 -0
- package/core/src/do-param-cap.test.ts +161 -0
- package/core/src/links.ts +8 -13
- package/core/src/mcp.ts +306 -314
- package/core/src/notes.ts +54 -62
- package/core/src/onboarding.ts +14 -296
- package/core/src/portable-md-batching.test.ts +67 -0
- package/core/src/portable-md-golden.test.ts +55 -0
- package/core/src/portable-md.ts +579 -435
- package/core/src/schema.ts +21 -43
- package/core/src/seed-packs.test.ts +191 -0
- package/core/src/seed-packs.ts +559 -0
- package/core/src/sql-in.test.ts +58 -0
- package/core/src/sql-in.ts +76 -0
- package/core/src/store.ts +14 -9
- package/core/src/transcription/provider.ts +141 -0
- package/core/src/txn.test.ts +229 -0
- package/core/src/txn.ts +105 -0
- package/core/src/types.ts +9 -0
- package/core/src/vault-projection.ts +1 -1
- package/core/src/wikilinks.ts +10 -4
- package/package.json +1 -1
- package/src/add-pack.test.ts +142 -0
- package/src/cli.ts +778 -7
- package/src/onboarding-seed.test.ts +126 -40
- package/src/onboarding-seed.ts +41 -46
- package/src/routes.ts +25 -7
- package/src/server.ts +108 -31
- package/src/transcription/build.test.ts +224 -0
- package/src/transcription/build.ts +252 -0
- package/src/transcription/capability.test.ts +118 -0
- package/src/transcription/capability.ts +96 -0
- package/src/transcription/install-python.test.ts +366 -0
- package/src/transcription/install-python.ts +471 -0
- package/src/transcription/install.test.ts +167 -0
- package/src/transcription/install.ts +296 -0
- package/src/transcription/providers/onnx-asr.test.ts +229 -0
- package/src/transcription/providers/onnx-asr.ts +239 -0
- package/src/transcription/providers/parakeet-mlx.test.ts +290 -0
- package/src/transcription/providers/parakeet-mlx.ts +242 -0
- package/src/transcription/providers/scribe-http.test.ts +195 -0
- package/src/transcription/providers/scribe-http.ts +144 -0
- package/src/transcription/providers/transcribe-cpp.test.ts +314 -0
- package/src/transcription/providers/transcribe-cpp.ts +293 -0
- package/src/transcription/select.test.ts +259 -0
- package/src/transcription/select.ts +334 -0
- package/src/transcription/tiers.test.ts +197 -0
- package/src/transcription/tiers.ts +184 -0
- package/src/transcription-worker.test.ts +44 -0
- package/src/transcription-worker.ts +57 -122
- package/src/vault-create.test.ts +38 -10
- package/src/vault.test.ts +48 -0
package/core/src/portable-md.ts
CHANGED
|
@@ -705,36 +705,168 @@ export class CaseCollisionError extends Error {
|
|
|
705
705
|
}
|
|
706
706
|
}
|
|
707
707
|
|
|
708
|
+
// ---------------------------------------------------------------------------
|
|
709
|
+
// Export sink — the destination seam (Phase-1 shared-core, cloud design §4)
|
|
710
|
+
// ---------------------------------------------------------------------------
|
|
711
|
+
|
|
712
|
+
/** Result of a sink write: `ok`, or a refusal the engine records as a skip
|
|
713
|
+
* (path-traversal, missing source) and surfaces in `ExportStats` without
|
|
714
|
+
* aborting the whole export. */
|
|
715
|
+
export type SinkWriteResult = { ok: true } | { ok: false; reason: string };
|
|
716
|
+
|
|
708
717
|
/**
|
|
709
|
-
*
|
|
710
|
-
*
|
|
711
|
-
*
|
|
712
|
-
*
|
|
713
|
-
*
|
|
714
|
-
*
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
+
* Where an export's bytes land. `exportVault` (the engine) owns every
|
|
719
|
+
* format, ordering, case-collision and since-filter decision and asks the
|
|
720
|
+
* sink only to persist bytes — so a Durable-Object/R2 backend plugs a
|
|
721
|
+
* key-value sink in behind the same engine (cloud design §4/§5). The
|
|
722
|
+
* `caseSensitive` flag lets a key-addressed sink opt out of the
|
|
723
|
+
* disambiguation path; `attachmentsEnabled` gates binary copies.
|
|
724
|
+
*/
|
|
725
|
+
export interface ExportSink {
|
|
726
|
+
/** Whether the destination namespace distinguishes filenames by case.
|
|
727
|
+
* The fs sink probes the real filesystem (macOS/Windows default to
|
|
728
|
+
* case-insensitive); a key-addressed sink (R2/DO) is always true. */
|
|
729
|
+
readonly caseSensitive: boolean;
|
|
730
|
+
/** Whether attachment binaries are copied. The fs sink enables this only
|
|
731
|
+
* when an `assetsDir` is wired; markdown-only exports still emit the
|
|
732
|
+
* frontmatter refs but leave the binaries in place. */
|
|
733
|
+
readonly attachmentsEnabled: boolean;
|
|
734
|
+
/** Persist a UTF-8 text document at export-root-relative `relPath`,
|
|
735
|
+
* creating any parent structure. */
|
|
736
|
+
writeText(relPath: string, content: string): SinkWriteResult;
|
|
737
|
+
/** Copy one attachment binary into the export. `srcRelPath` is relative
|
|
738
|
+
* to the attachment source (fs: `assetsDir`); `destRelPath` is export-
|
|
739
|
+
* root-relative. Only called when `attachmentsEnabled`. */
|
|
740
|
+
copyAttachment(srcRelPath: string, destRelPath: string): SinkWriteResult;
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
/**
|
|
744
|
+
* Filesystem `ExportSink` — the bun backend. Reproduces the pre-seam
|
|
745
|
+
* on-disk layout byte-for-byte (pinned by `portable-md-golden.test.ts`).
|
|
746
|
+
* Owns the case-sensitivity probe, the path-traversal guards (a note whose
|
|
747
|
+
* `path` escapes the export root is refused, never written off-tree), and
|
|
748
|
+
* attachment source resolution against `assetsDir`.
|
|
718
749
|
*
|
|
719
|
-
*
|
|
720
|
-
*
|
|
721
|
-
*
|
|
722
|
-
*
|
|
750
|
+
* Every guard resolves against the export root: keeping a write inside the
|
|
751
|
+
* export dir is the security property (the pre-seam per-subtree checks were
|
|
752
|
+
* belt-and-suspenders — a ULID sidecar id or a `basename()`-ed attachment
|
|
753
|
+
* name can't escape a subtree without also escaping the root).
|
|
723
754
|
*/
|
|
724
|
-
export
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
mkdirSync(
|
|
755
|
+
export class FsExportSink implements ExportSink {
|
|
756
|
+
readonly caseSensitive: boolean;
|
|
757
|
+
readonly attachmentsEnabled: boolean;
|
|
758
|
+
private readonly root: string;
|
|
759
|
+
private readonly rootResolved: string;
|
|
760
|
+
private readonly assetsDirResolved?: string;
|
|
761
|
+
|
|
762
|
+
constructor(opts: { outDir: string; assetsDir?: string; caseSensitiveOverride?: boolean }) {
|
|
763
|
+
this.root = opts.outDir;
|
|
764
|
+
mkdirSync(this.root, { recursive: true });
|
|
765
|
+
const sidecar = join(this.root, SIDECAR_DIR);
|
|
766
|
+
mkdirSync(sidecar, { recursive: true });
|
|
767
|
+
mkdirSync(join(sidecar, "schemas"), { recursive: true });
|
|
768
|
+
// attachments dir only when assetsDir is wired (caller opted in).
|
|
769
|
+
if (opts.assetsDir) {
|
|
770
|
+
mkdirSync(join(sidecar, "attachments"), { recursive: true });
|
|
771
|
+
this.assetsDirResolved = resolvePath(opts.assetsDir);
|
|
772
|
+
}
|
|
773
|
+
this.attachmentsEnabled = !!opts.assetsDir;
|
|
774
|
+
this.rootResolved = resolvePath(this.root);
|
|
775
|
+
// Probe AFTER outDir exists — probeCaseSensitive writes a tempfile into it.
|
|
776
|
+
this.caseSensitive = opts.caseSensitiveOverride ?? probeCaseSensitive(this.root);
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
writeText(relPath: string, content: string): SinkWriteResult {
|
|
780
|
+
const full = join(this.root, relPath);
|
|
781
|
+
const fullResolved = resolvePath(full);
|
|
782
|
+
if (!isWithinDir(fullResolved, this.rootResolved)) {
|
|
783
|
+
return {
|
|
784
|
+
ok: false,
|
|
785
|
+
reason: `path-traversal: resolved write target "${fullResolved}" escapes export root "${this.rootResolved}"`,
|
|
786
|
+
};
|
|
787
|
+
}
|
|
788
|
+
mkdirSync(dirname(full), { recursive: true });
|
|
789
|
+
writeFileSync(full, content);
|
|
790
|
+
return { ok: true };
|
|
736
791
|
}
|
|
737
792
|
|
|
793
|
+
copyAttachment(srcRelPath: string, destRelPath: string): SinkWriteResult {
|
|
794
|
+
// attachmentsEnabled guarantees assetsDirResolved is set.
|
|
795
|
+
const assetsDirResolved = this.assetsDirResolved!;
|
|
796
|
+
const srcResolved = resolvePath(join(assetsDirResolved, srcRelPath));
|
|
797
|
+
if (!isWithinDir(srcResolved, assetsDirResolved)) {
|
|
798
|
+
return {
|
|
799
|
+
ok: false,
|
|
800
|
+
reason: `path-traversal: source "${srcResolved}" escapes assetsDir "${assetsDirResolved}"`,
|
|
801
|
+
};
|
|
802
|
+
}
|
|
803
|
+
if (!existsSync(srcResolved)) {
|
|
804
|
+
return { ok: false, reason: `source file missing at "${srcResolved}"` };
|
|
805
|
+
}
|
|
806
|
+
const destFull = join(this.root, destRelPath);
|
|
807
|
+
const destResolved = resolvePath(destFull);
|
|
808
|
+
if (!isWithinDir(destResolved, this.rootResolved)) {
|
|
809
|
+
return {
|
|
810
|
+
ok: false,
|
|
811
|
+
reason: `path-traversal: dest "${destResolved}" escapes export root "${this.rootResolved}"`,
|
|
812
|
+
};
|
|
813
|
+
}
|
|
814
|
+
mkdirSync(dirname(destFull), { recursive: true });
|
|
815
|
+
copyFileSync(srcResolved, destResolved);
|
|
816
|
+
return { ok: true };
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/** Note-iteration batch size for the export walk — bounds the working set so
|
|
821
|
+
* a large vault never materializes its whole corpus at once (the pre-seam
|
|
822
|
+
* full `limit: 1_000_000` load was the flagged memory-spike hazard). */
|
|
823
|
+
const EXPORT_BATCH_SIZE = 500;
|
|
824
|
+
|
|
825
|
+
/**
|
|
826
|
+
* Iterate every note in `(created_at ASC, id ASC)` order — the exact order
|
|
827
|
+
* the pre-seam `queryNotes({ sort: "asc", limit: 1_000_000 })` produced — in
|
|
828
|
+
* batches of `EXPORT_BATCH_SIZE`, so the export streams instead of
|
|
829
|
+
* materializing the full corpus. Windowing uses `limit`/`offset`, not the
|
|
830
|
+
* opaque cursor: the cursor path forces `updated_at` ordering, which would
|
|
831
|
+
* change which of two case-colliding notes keeps the plain filename (that
|
|
832
|
+
* choice is order-pinned by tests). A `(created_at, id)` keyset predicate
|
|
833
|
+
* isn't on the Store API surface; if offset re-scan cost ever bites on a
|
|
834
|
+
* very large vault that's a follow-up — the memory hazard the refactor
|
|
835
|
+
* targets is already closed by the windowing.
|
|
836
|
+
*/
|
|
837
|
+
async function* iterateAllNotes(store: Store): AsyncGenerator<Note> {
|
|
838
|
+
let offset = 0;
|
|
839
|
+
for (;;) {
|
|
840
|
+
const batch = await store.queryNotes({ sort: "asc", limit: EXPORT_BATCH_SIZE, offset });
|
|
841
|
+
for (const note of batch) yield note;
|
|
842
|
+
if (batch.length < EXPORT_BATCH_SIZE) break;
|
|
843
|
+
offset += EXPORT_BATCH_SIZE;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
/** Engine-level export options — the subset of `ExportOptions` that isn't
|
|
848
|
+
* fs-sink-specific (`outDir`/`assetsDir`/`caseSensitiveOverride` are owned
|
|
849
|
+
* by `FsExportSink`). */
|
|
850
|
+
export interface ExportEngineOptions {
|
|
851
|
+
vaultName?: string;
|
|
852
|
+
vaultDescription?: string;
|
|
853
|
+
since?: string;
|
|
854
|
+
exportedAt?: string;
|
|
855
|
+
failOnCaseCollision?: boolean;
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/**
|
|
859
|
+
* Sink-agnostic export engine (Phase-1 shared-core, cloud design §4). Walks
|
|
860
|
+
* the vault in bounded batches and drives an `ExportSink`: `exportVaultToDir`
|
|
861
|
+
* wires the fs backend; a DO/R2 backend supplies its own sink. All format,
|
|
862
|
+
* ordering, case-collision and since-filter policy lives here — the sink only
|
|
863
|
+
* persists bytes, so the two runtimes stay byte-identical by construction.
|
|
864
|
+
*/
|
|
865
|
+
export async function exportVault(
|
|
866
|
+
store: Store,
|
|
867
|
+
sink: ExportSink,
|
|
868
|
+
opts: ExportEngineOptions = {},
|
|
869
|
+
): Promise<ExportStats> {
|
|
738
870
|
// 1. vault.yaml — vault meta + export format version. Trailing
|
|
739
871
|
// export-time timestamp is the one place where re-exports legitimately
|
|
740
872
|
// produce different bytes; callers wanting byte-equiv re-export pass
|
|
@@ -745,7 +877,7 @@ export async function exportVaultToDir(
|
|
|
745
877
|
...(opts.vaultName ? { name: opts.vaultName } : {}),
|
|
746
878
|
...(opts.vaultDescription ? { description: opts.vaultDescription } : {}),
|
|
747
879
|
};
|
|
748
|
-
|
|
880
|
+
sink.writeText(join(SIDECAR_DIR, "vault.yaml"), emitYamlDoc(vaultMeta as unknown as Record<string, unknown>));
|
|
749
881
|
|
|
750
882
|
// 2. Per-tag schemas. Only tags carrying at least one schema-shaped
|
|
751
883
|
// field (description, fields, relationships, parent_names) get a file;
|
|
@@ -762,25 +894,15 @@ export async function exportVaultToDir(
|
|
|
762
894
|
if (tag.parent_names !== undefined && tag.parent_names.length > 0) {
|
|
763
895
|
doc.parent_names = tag.parent_names;
|
|
764
896
|
}
|
|
765
|
-
|
|
897
|
+
sink.writeText(join(SIDECAR_DIR, "schemas", filename), emitYamlDoc(doc));
|
|
766
898
|
schemasWritten++;
|
|
767
899
|
}
|
|
768
900
|
|
|
769
|
-
// 3. Per-note files
|
|
770
|
-
// by updated_at >= since
|
|
771
|
-
//
|
|
772
|
-
// Note: in-memory bulk load. The 1M cap is a defensive ceiling — for
|
|
773
|
-
// very large vaults (>>100k notes) we should swap to a cursor /
|
|
774
|
-
// streaming query so the whole result set doesn't have to materialize
|
|
775
|
-
// at once. PR 2 follow-up if a real workload surfaces (vault#317 F5).
|
|
776
|
-
const allNotes = await store.queryNotes({ limit: 1_000_000, sort: "asc" });
|
|
901
|
+
// 3. Per-note files — streamed in `EXPORT_BATCH_SIZE` batches (no
|
|
902
|
+
// full-corpus load). If `since` is set, filter by updated_at >= since
|
|
903
|
+
// (incremental export).
|
|
777
904
|
const since = opts.since;
|
|
778
|
-
const
|
|
779
|
-
const assetsDirResolved = opts.assetsDir ? resolvePath(opts.assetsDir) : undefined;
|
|
780
|
-
const attachmentsRoot = join(sidecar, "attachments");
|
|
781
|
-
const attachmentsRootResolved = resolvePath(attachmentsRoot);
|
|
782
|
-
const notesMetaRoot = join(sidecar, NOTES_META_DIR);
|
|
783
|
-
const notesMetaRootResolved = resolvePath(notesMetaRoot);
|
|
905
|
+
const caseSensitive = sink.caseSensitive;
|
|
784
906
|
let notesWritten = 0;
|
|
785
907
|
let attachmentsWritten = 0;
|
|
786
908
|
let sidecarsWritten = 0;
|
|
@@ -788,48 +910,32 @@ export async function exportVaultToDir(
|
|
|
788
910
|
const skippedAttachments: { note_id: string; attachment_id: string; path: string; reason: string }[] = [];
|
|
789
911
|
const disambiguatedPaths: ExportStats["disambiguated_paths"] = [];
|
|
790
912
|
|
|
791
|
-
// Case-collision detection (vault#327). On case-insensitive
|
|
792
|
-
//
|
|
793
|
-
//
|
|
794
|
-
//
|
|
795
|
-
//
|
|
796
|
-
//
|
|
797
|
-
//
|
|
798
|
-
//
|
|
799
|
-
//
|
|
800
|
-
// only the on-disk filename is suffixed. Import recovers the
|
|
801
|
-
// canonical path from frontmatter/sidecar, not from the filename.
|
|
802
|
-
const caseSensitive = opts.caseSensitiveOverride ?? probeCaseSensitive(outDir);
|
|
803
|
-
// Lowercased `<path>|<ext>` → first-write note-id. Subsequent matches
|
|
804
|
-
// on the same key trigger disambiguation. Only populated on
|
|
805
|
-
// case-insensitive filesystems.
|
|
913
|
+
// Case-collision detection (vault#327). On case-insensitive filesystems
|
|
914
|
+
// (macOS APFS-default, Windows NTFS-default, FAT/exFAT), two notes whose
|
|
915
|
+
// paths differ only by case collapse into one file on write — silent
|
|
916
|
+
// data loss. The sink reports its namespace's case-sensitivity; on a
|
|
917
|
+
// case-insensitive sink we build a lowercase `(path, extension)` index
|
|
918
|
+
// during the walk and disambiguate colliding notes with an `__<id-short>`
|
|
919
|
+
// filename suffix. The note's stored `path` (in frontmatter + sidecar)
|
|
920
|
+
// stays canonical; only the on-disk filename is suffixed. Import recovers
|
|
921
|
+
// the canonical path from frontmatter/sidecar, not from the filename.
|
|
806
922
|
const seenLowerKeys = new Map<string, string>();
|
|
807
923
|
|
|
808
924
|
// Strict-mode pre-scan (vault#327 Phase 2). When the caller passes
|
|
809
|
-
// `failOnCaseCollision: true`, surface every collision group in one
|
|
810
|
-
//
|
|
811
|
-
//
|
|
812
|
-
//
|
|
813
|
-
//
|
|
814
|
-
//
|
|
815
|
-
//
|
|
816
|
-
//
|
|
817
|
-
//
|
|
818
|
-
//
|
|
819
|
-
//
|
|
820
|
-
// Perf: the pre-scan calls `noteToPortable` (3 DB queries per note:
|
|
821
|
-
// links, attachments, content). The main loop below also calls
|
|
822
|
-
// `noteToPortable` — without caching, every note that ALSO passes
|
|
823
|
-
// the since-filter would be serialized twice (~2x the DB round-
|
|
824
|
-
// trips on a large strict-mode export). Stash every pre-scan result
|
|
825
|
-
// in `prescanPortables` and reuse it below; cache-miss falls back
|
|
826
|
-
// to a fresh `noteToPortable` for safety. vault#350.
|
|
827
|
-
const prescanPortables = new Map<string, PortableNote>();
|
|
925
|
+
// `failOnCaseCollision: true`, surface every collision group in one typed
|
|
926
|
+
// error BEFORE any note write lands — partial-export-then-throw would
|
|
927
|
+
// leave the operator a half-mirrored dir to clean up. The pre-scan walks
|
|
928
|
+
// every note (NOT since-filtered: a collision can involve one old + one
|
|
929
|
+
// new path). It streams in the same batches as the main pass and keeps
|
|
930
|
+
// only lightweight `(id, path, extension)` entries — the pre-seam version
|
|
931
|
+
// cached every full PortableNote to avoid a second `noteToPortable`, but
|
|
932
|
+
// that reintroduced the whole-corpus memory hazard this refactor closes;
|
|
933
|
+
// strict mode is a one-shot CLI flow, so the second serialize pass is an
|
|
934
|
+
// acceptable trade for bounded memory.
|
|
828
935
|
if (opts.failOnCaseCollision && !caseSensitive) {
|
|
829
936
|
const groups = new Map<string, Array<{ note_id: string; path: string; extension: string }>>();
|
|
830
|
-
for (const note of
|
|
937
|
+
for await (const note of iterateAllNotes(store)) {
|
|
831
938
|
const portable = await noteToPortable(note, store);
|
|
832
|
-
prescanPortables.set(portable.id, portable);
|
|
833
939
|
if (!portable.path) continue; // _unpathed/<id>.<ext> is case-stable
|
|
834
940
|
const ext = portable.extension ?? "md";
|
|
835
941
|
const key = `${portable.path.toLowerCase()}|${ext.toLowerCase()}`;
|
|
@@ -847,28 +953,24 @@ export async function exportVaultToDir(
|
|
|
847
953
|
}
|
|
848
954
|
}
|
|
849
955
|
|
|
850
|
-
for (const note of
|
|
956
|
+
for await (const note of iterateAllNotes(store)) {
|
|
851
957
|
if (since && !shouldIncludeForSince(note, since)) continue;
|
|
852
|
-
|
|
853
|
-
// otherwise serialize fresh. Same PortableNote shape either way,
|
|
854
|
-
// so the rest of the loop is untouched. vault#350.
|
|
855
|
-
const portable = prescanPortables.get(note.id) ?? (await noteToPortable(note, store));
|
|
958
|
+
const portable = await noteToPortable(note, store);
|
|
856
959
|
let relPath = portableExportFilePath(portable);
|
|
857
960
|
|
|
858
|
-
// Decide whether this note's filename needs disambiguation
|
|
859
|
-
//
|
|
860
|
-
//
|
|
861
|
-
//
|
|
862
|
-
//
|
|
863
|
-
// is case-stable already.
|
|
961
|
+
// Decide whether this note's filename needs disambiguation (vault#327).
|
|
962
|
+
// Only meaningful when the sink is case-insensitive AND a prior note's
|
|
963
|
+
// (path, ext) tuple already claimed the same lowercased filename slot.
|
|
964
|
+
// Pathless notes (`_unpathed/<id>.<ext>`) are immune by construction —
|
|
965
|
+
// their filename embeds the id, which is case-stable already.
|
|
864
966
|
if (!caseSensitive && portable.path) {
|
|
865
967
|
const ext = portable.extension ?? "md";
|
|
866
968
|
const key = `${portable.path.toLowerCase()}|${ext.toLowerCase()}`;
|
|
867
969
|
const prior = seenLowerKeys.get(key);
|
|
868
970
|
if (prior !== undefined && prior !== portable.id) {
|
|
869
|
-
// Collision: emit the disambiguated form. The frontmatter /
|
|
870
|
-
//
|
|
871
|
-
//
|
|
971
|
+
// Collision: emit the disambiguated form. The frontmatter / sidecar
|
|
972
|
+
// `path:` still holds the canonical (original) path so import
|
|
973
|
+
// recovers the truth.
|
|
872
974
|
const disambig = disambiguateFilename(portable.path, ext, portable.id);
|
|
873
975
|
relPath = disambig;
|
|
874
976
|
disambiguatedPaths.push({
|
|
@@ -881,100 +983,61 @@ export async function exportVaultToDir(
|
|
|
881
983
|
}
|
|
882
984
|
}
|
|
883
985
|
|
|
884
|
-
|
|
885
|
-
//
|
|
886
|
-
//
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
// the note path source, e.g. ingest from external systems).
|
|
891
|
-
const fullPathResolved = resolvePath(fullPath);
|
|
892
|
-
if (!isWithinDir(fullPathResolved, outDirResolved)) {
|
|
893
|
-
skipped.push({
|
|
894
|
-
path: portable.path,
|
|
895
|
-
reason: `path-traversal: resolved write target "${fullPathResolved}" escapes export root "${outDirResolved}"`,
|
|
896
|
-
});
|
|
986
|
+
// vault#317 F3 — path-traversal guard now lives in the sink: a note with
|
|
987
|
+
// `path: "../../.ssh/authorized_keys"` is refused (not written off-tree)
|
|
988
|
+
// and surfaced in `skipped_notes` so the operator can fix the note.
|
|
989
|
+
const noteWrite = sink.writeText(relPath, toPortableMarkdown(portable));
|
|
990
|
+
if (!noteWrite.ok) {
|
|
991
|
+
skipped.push({ path: portable.path, reason: noteWrite.reason });
|
|
897
992
|
continue;
|
|
898
993
|
}
|
|
899
|
-
mkdirSync(dirname(fullPath), { recursive: true });
|
|
900
|
-
writeFileSync(fullPath, toPortableMarkdown(portable));
|
|
901
994
|
notesWritten++;
|
|
902
995
|
|
|
903
996
|
// Sidecar metadata write for non-frontmatter-compat extensions
|
|
904
|
-
// (vault#328). The content file holds raw bytes (no YAML); the
|
|
905
|
-
//
|
|
906
|
-
//
|
|
997
|
+
// (vault#328). The content file holds raw bytes (no YAML); the sidecar
|
|
998
|
+
// at .parachute/notes-meta/<id>.yaml carries id/path/tags/metadata/
|
|
999
|
+
// links/attachments/timestamps.
|
|
907
1000
|
const portableExt = portable.extension ?? "md";
|
|
908
1001
|
if (!supportsInlineFrontmatter(portableExt)) {
|
|
909
|
-
const
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
if (!
|
|
914
|
-
skipped.push({
|
|
915
|
-
path: portable.path,
|
|
916
|
-
reason: `path-traversal: sidecar write target "${sidecarResolved}" escapes notes-meta root "${notesMetaRootResolved}"`,
|
|
917
|
-
});
|
|
1002
|
+
const sidecarWrite = sink.writeText(
|
|
1003
|
+
join(SIDECAR_DIR, NOTES_META_DIR, `${portable.id}.yaml`),
|
|
1004
|
+
toSidecarYaml(portable),
|
|
1005
|
+
);
|
|
1006
|
+
if (!sidecarWrite.ok) {
|
|
1007
|
+
skipped.push({ path: portable.path, reason: sidecarWrite.reason });
|
|
918
1008
|
} else {
|
|
919
|
-
mkdirSync(notesMetaRoot, { recursive: true });
|
|
920
|
-
writeFileSync(sidecarResolved, toSidecarYaml(portable));
|
|
921
1009
|
sidecarsWritten++;
|
|
922
1010
|
}
|
|
923
1011
|
}
|
|
924
1012
|
|
|
925
|
-
// Copy attachment binaries when
|
|
926
|
-
// is path-traversal-guarded on both ends
|
|
927
|
-
//
|
|
928
|
-
//
|
|
929
|
-
|
|
930
|
-
if (assetsDirResolved && portable.attachments && portable.attachments.length > 0) {
|
|
1013
|
+
// Copy attachment binaries when the sink has them enabled. Each
|
|
1014
|
+
// attachment is path-traversal-guarded on both ends by the sink; missing
|
|
1015
|
+
// source files are skipped (warn) rather than aborting — the assets state
|
|
1016
|
+
// may legitimately lag the DB (e.g. file evicted while row persists).
|
|
1017
|
+
if (sink.attachmentsEnabled && portable.attachments && portable.attachments.length > 0) {
|
|
931
1018
|
for (const att of portable.attachments) {
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
});
|
|
941
|
-
continue;
|
|
942
|
-
}
|
|
943
|
-
if (!existsSync(srcResolved)) {
|
|
944
|
-
skippedAttachments.push({
|
|
945
|
-
note_id: portable.id,
|
|
946
|
-
attachment_id: att.id,
|
|
947
|
-
path: att.path,
|
|
948
|
-
reason: `source file missing at "${srcResolved}"`,
|
|
949
|
-
});
|
|
950
|
-
continue;
|
|
951
|
-
}
|
|
952
|
-
// Dest: .parachute/attachments/<att-id>/<basename(att.path)>.
|
|
953
|
-
// Using att.id as the directory name keeps multiple attachments
|
|
954
|
-
// with the same basename from colliding; basename keeps the
|
|
955
|
-
// filename human-readable.
|
|
956
|
-
const destDir = join(attachmentsRoot, att.id);
|
|
957
|
-
const destFile = join(destDir, basename(att.path));
|
|
958
|
-
const destResolved = resolvePath(destFile);
|
|
959
|
-
if (!isWithinDir(destResolved, attachmentsRootResolved)) {
|
|
1019
|
+
// Dest: .parachute/attachments/<att-id>/<basename(att.path)>. Using
|
|
1020
|
+
// att.id as the directory name keeps multiple attachments with the
|
|
1021
|
+
// same basename from colliding; basename keeps it human-readable.
|
|
1022
|
+
const destRelPath = join(SIDECAR_DIR, "attachments", att.id, basename(att.path));
|
|
1023
|
+
const copyResult = sink.copyAttachment(att.path, destRelPath);
|
|
1024
|
+
if (copyResult.ok) {
|
|
1025
|
+
attachmentsWritten++;
|
|
1026
|
+
} else {
|
|
960
1027
|
skippedAttachments.push({
|
|
961
1028
|
note_id: portable.id,
|
|
962
1029
|
attachment_id: att.id,
|
|
963
1030
|
path: att.path,
|
|
964
|
-
reason:
|
|
1031
|
+
reason: copyResult.reason,
|
|
965
1032
|
});
|
|
966
|
-
continue;
|
|
967
1033
|
}
|
|
968
|
-
mkdirSync(destDir, { recursive: true });
|
|
969
|
-
copyFileSync(srcResolved, destResolved);
|
|
970
|
-
attachmentsWritten++;
|
|
971
1034
|
}
|
|
972
1035
|
}
|
|
973
1036
|
}
|
|
974
1037
|
if (skipped.length > 0) {
|
|
975
|
-
// Surface to the caller without aborting — partial export is more
|
|
976
|
-
//
|
|
977
|
-
//
|
|
1038
|
+
// Surface to the caller without aborting — partial export is more useful
|
|
1039
|
+
// than no export. CLI prints the list; programmatic callers inspect the
|
|
1040
|
+
// return value.
|
|
978
1041
|
for (const s of skipped) {
|
|
979
1042
|
// eslint-disable-next-line no-console
|
|
980
1043
|
console.warn(`[export] skipped note (path="${s.path ?? "<unpathed>"}"): ${s.reason}`);
|
|
@@ -1001,6 +1064,41 @@ export async function exportVaultToDir(
|
|
|
1001
1064
|
};
|
|
1002
1065
|
}
|
|
1003
1066
|
|
|
1067
|
+
/**
|
|
1068
|
+
* Export a vault to a portable-markdown directory (the bun/fs backend —
|
|
1069
|
+
* thin wrapper over `exportVault` with an `FsExportSink`). Writes:
|
|
1070
|
+
* - `<outDir>/.parachute/vault.yaml`
|
|
1071
|
+
* - `<outDir>/.parachute/schemas/<tag>.yaml` for each tag that declares
|
|
1072
|
+
* description/fields/relationships/parent_names.
|
|
1073
|
+
* - `<outDir>/<note.path>.md` for each note (or `_unpathed/<id>.md`).
|
|
1074
|
+
* - `<outDir>/.parachute/attachments/<id>/<basename>` for each attachment
|
|
1075
|
+
* (only when `opts.assetsDir` is set; the sink's path-traversal guard
|
|
1076
|
+
* rejects attachments whose source escapes assetsDir or whose dest would
|
|
1077
|
+
* escape outDir).
|
|
1078
|
+
*
|
|
1079
|
+
* The frontmatter `attachments[].path` value preserves the original
|
|
1080
|
+
* vault-internal path (relative to `assetsDir`). Import restores the binary
|
|
1081
|
+
* to that path. The sidecar location is derived from `id` so it stays stable
|
|
1082
|
+
* across renames + different export runs.
|
|
1083
|
+
*/
|
|
1084
|
+
export async function exportVaultToDir(
|
|
1085
|
+
store: Store,
|
|
1086
|
+
opts: ExportOptions,
|
|
1087
|
+
): Promise<ExportStats> {
|
|
1088
|
+
const sink = new FsExportSink({
|
|
1089
|
+
outDir: opts.outDir,
|
|
1090
|
+
assetsDir: opts.assetsDir,
|
|
1091
|
+
caseSensitiveOverride: opts.caseSensitiveOverride,
|
|
1092
|
+
});
|
|
1093
|
+
return exportVault(store, sink, {
|
|
1094
|
+
vaultName: opts.vaultName,
|
|
1095
|
+
vaultDescription: opts.vaultDescription,
|
|
1096
|
+
since: opts.since,
|
|
1097
|
+
exportedAt: opts.exportedAt,
|
|
1098
|
+
failOnCaseCollision: opts.failOnCaseCollision,
|
|
1099
|
+
});
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1004
1102
|
// ---------------------------------------------------------------------------
|
|
1005
1103
|
// Orphan sweep — for git-mirror's delete propagation
|
|
1006
1104
|
// ---------------------------------------------------------------------------
|
|
@@ -1545,12 +1643,137 @@ export interface ImportStats {
|
|
|
1545
1643
|
indexes_declared: number;
|
|
1546
1644
|
}
|
|
1547
1645
|
|
|
1646
|
+
// ---------------------------------------------------------------------------
|
|
1647
|
+
// Import source — the read seam (Phase-1 shared-core, cloud design §4)
|
|
1648
|
+
// ---------------------------------------------------------------------------
|
|
1649
|
+
|
|
1548
1650
|
/**
|
|
1549
|
-
*
|
|
1550
|
-
*
|
|
1551
|
-
*
|
|
1552
|
-
*
|
|
1553
|
-
*
|
|
1651
|
+
* Where an import reads its bytes from. `importVault` (the engine) owns all
|
|
1652
|
+
* replay policy (schema-before-notes ordering, sidecar resolution, upsert
|
|
1653
|
+
* merge, forward-ref link replay) and asks the source only to enumerate +
|
|
1654
|
+
* read — so a DO/R2 import (from an uploaded tarball / bucket) plugs its own
|
|
1655
|
+
* source in behind the same engine. `FsImportSource` is the bun backend
|
|
1656
|
+
* (reads an `exportVaultToDir` output directory).
|
|
1657
|
+
*/
|
|
1658
|
+
export interface ImportSource {
|
|
1659
|
+
/** `*.yaml` schema docs under `.parachute/schemas/` (name + raw text). */
|
|
1660
|
+
readSchemaFiles(): Array<{ name: string; text: string }>;
|
|
1661
|
+
/** Raw text of each `*.yaml` sidecar under `.parachute/notes-meta/`. */
|
|
1662
|
+
readNotesMetaFiles(): string[];
|
|
1663
|
+
/** Content files as export-root-relative POSIX paths (sorted, dot-dirs
|
|
1664
|
+
* excluded, containment-guarded). */
|
|
1665
|
+
listContentFiles(): string[];
|
|
1666
|
+
/** Read one content file by its export-relative path. */
|
|
1667
|
+
readContentFile(relPath: string): string;
|
|
1668
|
+
/** Whether attachment binaries are restored (fs: `assetsDir` wired). */
|
|
1669
|
+
readonly attachmentsEnabled: boolean;
|
|
1670
|
+
/** Restore one attachment binary. `srcRelPath` is export-root-relative
|
|
1671
|
+
* (`.parachute/attachments/<att.id>/<basename>`); `destRelPath` is
|
|
1672
|
+
* assets-relative (the note's `attachments[].path`). Only meaningful
|
|
1673
|
+
* when `attachmentsEnabled`. */
|
|
1674
|
+
restoreAttachment(srcRelPath: string, destRelPath: string): SinkWriteResult;
|
|
1675
|
+
}
|
|
1676
|
+
|
|
1677
|
+
/**
|
|
1678
|
+
* Filesystem `ImportSource` — reads an `exportVaultToDir` output directory.
|
|
1679
|
+
* Owns the read-side path-traversal guards (refuse to follow a symlink out
|
|
1680
|
+
* of the sidecar, refuse an attachment whose source escapes the attachments
|
|
1681
|
+
* root or whose dest escapes assetsDir) so the engine consumes clean data.
|
|
1682
|
+
*/
|
|
1683
|
+
export class FsImportSource implements ImportSource {
|
|
1684
|
+
readonly attachmentsEnabled: boolean;
|
|
1685
|
+
private readonly inDir: string;
|
|
1686
|
+
private readonly inDirResolved: string;
|
|
1687
|
+
private readonly sidecar: string;
|
|
1688
|
+
private readonly assetsDirResolved?: string;
|
|
1689
|
+
|
|
1690
|
+
constructor(opts: { inDir: string; assetsDir?: string }) {
|
|
1691
|
+
this.inDir = opts.inDir;
|
|
1692
|
+
this.inDirResolved = resolvePath(opts.inDir);
|
|
1693
|
+
this.sidecar = join(opts.inDir, SIDECAR_DIR);
|
|
1694
|
+
if (opts.assetsDir) this.assetsDirResolved = resolvePath(opts.assetsDir);
|
|
1695
|
+
this.attachmentsEnabled = !!opts.assetsDir;
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
readSchemaFiles(): Array<{ name: string; text: string }> {
|
|
1699
|
+
const dir = join(this.sidecar, "schemas");
|
|
1700
|
+
if (!existsSync(dir)) return [];
|
|
1701
|
+
const dirResolved = resolvePath(dir);
|
|
1702
|
+
const out: Array<{ name: string; text: string }> = [];
|
|
1703
|
+
for (const entry of readdirSync(dir)) {
|
|
1704
|
+
if (!entry.endsWith(".yaml")) continue;
|
|
1705
|
+
const fullPath = join(dir, entry);
|
|
1706
|
+
// Path-traversal guard on the read side: refuse to follow a symlink
|
|
1707
|
+
// out of the sidecar (readdirSync only surfaces names — belt-and-
|
|
1708
|
+
// suspenders).
|
|
1709
|
+
if (!isWithinDir(resolvePath(fullPath), dirResolved)) continue;
|
|
1710
|
+
out.push({ name: entry, text: readFileSync(fullPath, "utf-8") });
|
|
1711
|
+
}
|
|
1712
|
+
return out;
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
readNotesMetaFiles(): string[] {
|
|
1716
|
+
const dir = join(this.sidecar, NOTES_META_DIR);
|
|
1717
|
+
if (!existsSync(dir)) return [];
|
|
1718
|
+
const dirResolved = resolvePath(dir);
|
|
1719
|
+
const out: string[] = [];
|
|
1720
|
+
for (const entry of readdirSync(dir)) {
|
|
1721
|
+
if (!entry.endsWith(".yaml")) continue;
|
|
1722
|
+
if (entry.startsWith(".")) continue;
|
|
1723
|
+
const fullPath = join(dir, entry);
|
|
1724
|
+
if (!isWithinDir(resolvePath(fullPath), dirResolved)) continue;
|
|
1725
|
+
out.push(readFileSync(fullPath, "utf-8"));
|
|
1726
|
+
}
|
|
1727
|
+
return out;
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
listContentFiles(): string[] {
|
|
1731
|
+
const out: string[] = [];
|
|
1732
|
+
for (const filePath of walkContentFiles(this.inDir)) {
|
|
1733
|
+
// Containment check — walkContentFiles should already be safe, but
|
|
1734
|
+
// verify the resolved path is inside inDir (symlinks).
|
|
1735
|
+
if (!isWithinDir(resolvePath(filePath), this.inDirResolved)) continue;
|
|
1736
|
+
out.push(relative(this.inDir, filePath).split(pathSep).join("/"));
|
|
1737
|
+
}
|
|
1738
|
+
return out;
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
readContentFile(relPath: string): string {
|
|
1742
|
+
return readFileSync(join(this.inDir, relPath), "utf-8");
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
restoreAttachment(srcRelPath: string, destRelPath: string): SinkWriteResult {
|
|
1746
|
+
const attachmentsRootResolved = resolvePath(join(this.sidecar, "attachments"));
|
|
1747
|
+
const srcResolved = resolvePath(join(this.inDir, srcRelPath));
|
|
1748
|
+
if (!isWithinDir(srcResolved, attachmentsRootResolved)) {
|
|
1749
|
+
return { ok: false, reason: `path-traversal on source: "${srcResolved}" escapes attachments root` };
|
|
1750
|
+
}
|
|
1751
|
+
if (!existsSync(srcResolved)) {
|
|
1752
|
+
return { ok: false, reason: `source attachment file missing at "${srcResolved}"` };
|
|
1753
|
+
}
|
|
1754
|
+
// attachmentsEnabled guarantees assetsDirResolved is set.
|
|
1755
|
+
const assetsDirResolved = this.assetsDirResolved!;
|
|
1756
|
+
const destResolved = resolvePath(join(assetsDirResolved, destRelPath));
|
|
1757
|
+
if (!isWithinDir(destResolved, assetsDirResolved)) {
|
|
1758
|
+
return { ok: false, reason: `path-traversal on dest: "${destResolved}" escapes assetsDir` };
|
|
1759
|
+
}
|
|
1760
|
+
mkdirSync(dirname(destResolved), { recursive: true });
|
|
1761
|
+
copyFileSync(srcResolved, destResolved);
|
|
1762
|
+
return { ok: true };
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
/** Engine-level import options — the subset of `ImportOptions` that isn't
|
|
1767
|
+
* fs-source-specific (`inDir`/`assetsDir` are owned by `FsImportSource`). */
|
|
1768
|
+
export interface ImportEngineOptions {
|
|
1769
|
+
blowAway?: boolean;
|
|
1770
|
+
dryRun?: boolean;
|
|
1771
|
+
}
|
|
1772
|
+
|
|
1773
|
+
/**
|
|
1774
|
+
* Sink-agnostic import engine (Phase-1 shared-core, cloud design §4). Reads
|
|
1775
|
+
* an `ImportSource` and replays it into `store`. `importPortableVault` wires
|
|
1776
|
+
* the fs backend; a DO/R2 backend supplies its own source.
|
|
1554
1777
|
*
|
|
1555
1778
|
* Restoration order (matters for forward refs):
|
|
1556
1779
|
* 1. Tag schemas (so notes with schema-bearing tags validate cleanly).
|
|
@@ -1561,25 +1784,15 @@ export interface ImportStats {
|
|
|
1561
1784
|
* pattern). Wikilinks rebuild themselves from `[[brackets]]` in
|
|
1562
1785
|
* content via the existing `syncAllWikilinks` pass.
|
|
1563
1786
|
* 4. Attachments — DB row first, then file copy from sidecar to
|
|
1564
|
-
* `<assetsDir>/<path>` when
|
|
1787
|
+
* `<assetsDir>/<path>` when the source has attachments enabled.
|
|
1565
1788
|
*
|
|
1566
1789
|
* See vault#308 PR 2.
|
|
1567
1790
|
*/
|
|
1568
|
-
export async function
|
|
1791
|
+
export async function importVault(
|
|
1569
1792
|
store: Store,
|
|
1570
|
-
|
|
1793
|
+
source: ImportSource,
|
|
1794
|
+
opts: ImportEngineOptions = {},
|
|
1571
1795
|
): Promise<ImportStats> {
|
|
1572
|
-
const inDir = opts.inDir;
|
|
1573
|
-
const inDirResolved = resolvePath(inDir);
|
|
1574
|
-
const sidecar = join(inDir, SIDECAR_DIR);
|
|
1575
|
-
if (!existsSync(join(sidecar, "vault.yaml"))) {
|
|
1576
|
-
throw new Error(
|
|
1577
|
-
`not a portable-md export: missing ${join(SIDECAR_DIR, "vault.yaml")} in "${inDir}". ` +
|
|
1578
|
-
`If this is a legacy Obsidian-shape directory, use the obsidian.ts \`parseObsidianVault\` ` +
|
|
1579
|
-
`path instead — vault#308 importer only handles the portable-md format.`,
|
|
1580
|
-
);
|
|
1581
|
-
}
|
|
1582
|
-
|
|
1583
1796
|
const stats: ImportStats = {
|
|
1584
1797
|
notes_created: 0,
|
|
1585
1798
|
notes_updated: 0,
|
|
@@ -1593,17 +1806,22 @@ export async function importPortableVault(
|
|
|
1593
1806
|
indexes_declared: 0,
|
|
1594
1807
|
};
|
|
1595
1808
|
|
|
1596
|
-
// 1. Optional wipe. Notes are deleted via the public Store API so
|
|
1597
|
-
//
|
|
1598
|
-
//
|
|
1809
|
+
// 1. Optional wipe. Notes are deleted via the public Store API so hooks
|
|
1810
|
+
// fire (callers depend on `attachment.deleted` hooks for assets-dir
|
|
1811
|
+
// cleanup; we don't bypass that on blow-away). Deleted in bounded batches
|
|
1812
|
+
// — always take the first N remaining rows, so deletion never has to
|
|
1813
|
+
// materialize the whole corpus (the pre-seam `limit: 1_000_000` load).
|
|
1599
1814
|
if (opts.blowAway && !opts.dryRun) {
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1815
|
+
for (;;) {
|
|
1816
|
+
const batch = await store.queryNotes({ sort: "asc", limit: EXPORT_BATCH_SIZE });
|
|
1817
|
+
if (batch.length === 0) break;
|
|
1818
|
+
for (const note of batch) {
|
|
1819
|
+
await store.deleteNote(note.id);
|
|
1820
|
+
stats.notes_wiped++;
|
|
1821
|
+
}
|
|
1603
1822
|
}
|
|
1604
|
-
|
|
1605
|
-
//
|
|
1606
|
-
// but leaves the `tags` table rows in place (orphaned schemas).
|
|
1823
|
+
// Clear tag rows too — `deleteNote` clears note_tags via FK cascade but
|
|
1824
|
+
// leaves the `tags` table rows in place (orphaned schemas).
|
|
1607
1825
|
const tagRecords = await store.listTagRecords();
|
|
1608
1826
|
for (const tag of tagRecords) {
|
|
1609
1827
|
await store.deleteTag(tag.tag);
|
|
@@ -1612,147 +1830,115 @@ export async function importPortableVault(
|
|
|
1612
1830
|
|
|
1613
1831
|
// 2. Tag schemas — restore before notes so any tag a note carries can
|
|
1614
1832
|
// validate against its schema on insert.
|
|
1615
|
-
const
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
if (!isWithinDir(resolved, resolvePath(schemasDir))) continue;
|
|
1625
|
-
const text = readFileSync(fullPath, "utf-8");
|
|
1626
|
-
// Reuse the frontmatter parser by wrapping the doc in `---`s.
|
|
1627
|
-
// The schema file is a YAML doc (no `---` markers); pad with them
|
|
1628
|
-
// so `parseFrontmatter` can chew on it via the same code path.
|
|
1629
|
-
const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
|
|
1630
|
-
const { frontmatter } = parseFrontmatter(wrapped);
|
|
1631
|
-
const tagName = typeof frontmatter.name === "string" ? frontmatter.name : null;
|
|
1632
|
-
if (!tagName) continue;
|
|
1633
|
-
if (opts.dryRun) {
|
|
1634
|
-
stats.schemas_restored++;
|
|
1635
|
-
continue;
|
|
1636
|
-
}
|
|
1637
|
-
await store.upsertTagRecord(tagName, {
|
|
1638
|
-
description: (frontmatter.description as string | null | undefined) ?? null,
|
|
1639
|
-
fields: (frontmatter.fields as Record<string, unknown> | null | undefined) as any ?? null,
|
|
1640
|
-
relationships: (frontmatter.relationships as Record<string, unknown> | null | undefined) as any ?? null,
|
|
1641
|
-
parent_names: (frontmatter.parent_names as string[] | null | undefined) ?? null,
|
|
1642
|
-
});
|
|
1833
|
+
for (const { text } of source.readSchemaFiles()) {
|
|
1834
|
+
// Reuse the frontmatter parser by wrapping the doc in `---`s. The schema
|
|
1835
|
+
// file is a YAML doc (no `---` markers); pad with them so
|
|
1836
|
+
// `parseFrontmatter` can chew on it via the same code path.
|
|
1837
|
+
const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
|
|
1838
|
+
const { frontmatter } = parseFrontmatter(wrapped);
|
|
1839
|
+
const tagName = typeof frontmatter.name === "string" ? frontmatter.name : null;
|
|
1840
|
+
if (!tagName) continue;
|
|
1841
|
+
if (opts.dryRun) {
|
|
1643
1842
|
stats.schemas_restored++;
|
|
1843
|
+
continue;
|
|
1644
1844
|
}
|
|
1845
|
+
await store.upsertTagRecord(tagName, {
|
|
1846
|
+
description: (frontmatter.description as string | null | undefined) ?? null,
|
|
1847
|
+
fields: (frontmatter.fields as Record<string, unknown> | null | undefined) as any ?? null,
|
|
1848
|
+
relationships: (frontmatter.relationships as Record<string, unknown> | null | undefined) as any ?? null,
|
|
1849
|
+
parent_names: (frontmatter.parent_names as string[] | null | undefined) ?? null,
|
|
1850
|
+
});
|
|
1851
|
+
stats.schemas_restored++;
|
|
1645
1852
|
}
|
|
1646
1853
|
|
|
1647
|
-
// 3. Notes. Walk every content file
|
|
1648
|
-
//
|
|
1649
|
-
//
|
|
1650
|
-
//
|
|
1854
|
+
// 3. Notes. Walk every content file (dot-dirs already excluded by the
|
|
1855
|
+
// source). For frontmatter-compatible extensions (md, mdx) parse inline
|
|
1856
|
+
// metadata. For sidecar-required extensions (csv, yaml, json, etc.) look
|
|
1857
|
+
// up metadata in `.parachute/notes-meta/<id>.yaml`.
|
|
1651
1858
|
//
|
|
1652
1859
|
// The sidecar's `path` + `extension` are the source of truth for the
|
|
1653
|
-
// user-visible filename — but we walk filenames first and INDEX
|
|
1654
|
-
//
|
|
1655
|
-
//
|
|
1656
|
-
//
|
|
1657
|
-
//
|
|
1658
|
-
//
|
|
1659
|
-
//
|
|
1660
|
-
//
|
|
1661
|
-
//
|
|
1662
|
-
//
|
|
1663
|
-
// case-preserved filename + the disambiguated-id-suffix heuristic.
|
|
1664
|
-
const notesMetaDir = join(sidecar, NOTES_META_DIR);
|
|
1860
|
+
// user-visible filename — but we walk filenames first and INDEX sidecars
|
|
1861
|
+
// by `(path, extension)` for O(1) lookup per content file. Sidecars with
|
|
1862
|
+
// no matching content file are warned about (and skipped) rather than
|
|
1863
|
+
// triggering a write — preserves the export-bytes-are-truth invariant.
|
|
1864
|
+
// Sidecar index keyed by lowercased `<path>|<ext>` so the walker's
|
|
1865
|
+
// filename-derived key (also lowered) matches. The bucket holds the full
|
|
1866
|
+
// sidecar(s) — multiple entries occur when two notes share a
|
|
1867
|
+
// case-insensitive path (vault#327 collisions). The lookup picks the right
|
|
1868
|
+
// sidecar from the bucket using the walker's case-preserved filename + the
|
|
1869
|
+
// disambiguated-id-suffix heuristic.
|
|
1665
1870
|
const sidecarByKey = new Map<string, Record<string, unknown>[]>();
|
|
1666
1871
|
const sidecarByIdLeftover = new Map<string, Record<string, unknown>>();
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
const
|
|
1681
|
-
const
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
// Index by (path, ext) tuple lowered so case-insensitive walks
|
|
1685
|
-
// match. Multi-value bucket lets two case-collided sidecars coexist
|
|
1686
|
-
// until the per-file lookup picks the right one.
|
|
1687
|
-
if (sidecarPath && sidecarExt) {
|
|
1688
|
-
const key = `${sidecarPath.toLowerCase()}|${sidecarExt.toLowerCase()}`;
|
|
1689
|
-
const bucket = sidecarByKey.get(key);
|
|
1690
|
-
if (bucket) bucket.push(frontmatter);
|
|
1691
|
-
else sidecarByKey.set(key, [frontmatter]);
|
|
1692
|
-
}
|
|
1693
|
-
sidecarByIdLeftover.set(sidecarId, frontmatter);
|
|
1872
|
+
for (const text of source.readNotesMetaFiles()) {
|
|
1873
|
+
// The sidecar is a bare YAML doc (no `---`); wrap to reuse the shared
|
|
1874
|
+
// frontmatter parser.
|
|
1875
|
+
const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
|
|
1876
|
+
const { frontmatter } = parseFrontmatter(wrapped);
|
|
1877
|
+
const sidecarId = typeof frontmatter.id === "string" ? frontmatter.id : null;
|
|
1878
|
+
const sidecarPath = typeof frontmatter.path === "string" ? frontmatter.path : null;
|
|
1879
|
+
const sidecarExt = typeof frontmatter.extension === "string" ? frontmatter.extension : null;
|
|
1880
|
+
if (!sidecarId) continue;
|
|
1881
|
+
// Index by (path, ext) tuple lowered so case-insensitive walks match.
|
|
1882
|
+
// Multi-value bucket lets two case-collided sidecars coexist until the
|
|
1883
|
+
// per-file lookup picks the right one.
|
|
1884
|
+
if (sidecarPath && sidecarExt) {
|
|
1885
|
+
const key = `${sidecarPath.toLowerCase()}|${sidecarExt.toLowerCase()}`;
|
|
1886
|
+
const bucket = sidecarByKey.get(key);
|
|
1887
|
+
if (bucket) bucket.push(frontmatter);
|
|
1888
|
+
else sidecarByKey.set(key, [frontmatter]);
|
|
1694
1889
|
}
|
|
1890
|
+
sidecarByIdLeftover.set(sidecarId, frontmatter);
|
|
1695
1891
|
}
|
|
1696
1892
|
|
|
1697
|
-
// Track per-import (id → portable) so we can replay typed links
|
|
1698
|
-
//
|
|
1893
|
+
// Track per-import (id → portable) so we can replay typed links after all
|
|
1894
|
+
// notes exist.
|
|
1699
1895
|
const seenNotes = new Map<string, PortableNote>();
|
|
1700
|
-
for (const
|
|
1701
|
-
// Containment check — readdirSync should already be safe, but
|
|
1702
|
-
// verify the resolved path is inside inDir (symlinks).
|
|
1703
|
-
const resolved = resolvePath(filePath);
|
|
1704
|
-
if (!isWithinDir(resolved, inDirResolved)) continue;
|
|
1705
|
-
|
|
1896
|
+
for (const relPath of source.listContentFiles()) {
|
|
1706
1897
|
// Derive the file's extension (lowercased, no leading dot) and the
|
|
1707
|
-
// user-visible note path (everything between
|
|
1708
|
-
// extension).
|
|
1709
|
-
const extWithDot = extname(
|
|
1898
|
+
// user-visible note path (everything between the export root and the
|
|
1899
|
+
// extension). The source returns POSIX-relative paths already.
|
|
1900
|
+
const extWithDot = extname(relPath); // e.g. ".csv"
|
|
1710
1901
|
const fileExt = extWithDot.slice(1).toLowerCase();
|
|
1711
1902
|
if (fileExt.length === 0) continue;
|
|
1712
|
-
const
|
|
1713
|
-
const relNoExt = relWithExt.slice(0, relWithExt.length - extWithDot.length);
|
|
1714
|
-
// Normalize path separators for cross-platform exports.
|
|
1715
|
-
const userPath = relNoExt.split(pathSep).join("/");
|
|
1903
|
+
const userPath = relPath.slice(0, relPath.length - extWithDot.length);
|
|
1716
1904
|
|
|
1717
1905
|
let frontmatter: Record<string, unknown>;
|
|
1718
1906
|
let content: string;
|
|
1719
1907
|
if (supportsInlineFrontmatter(fileExt)) {
|
|
1720
|
-
const raw =
|
|
1908
|
+
const raw = source.readContentFile(relPath);
|
|
1721
1909
|
const parsed = parseFrontmatter(raw);
|
|
1722
1910
|
frontmatter = parsed.frontmatter;
|
|
1723
1911
|
content = parsed.content;
|
|
1724
1912
|
} else {
|
|
1725
|
-
// Resolve which sidecar this content file belongs to. Two
|
|
1726
|
-
//
|
|
1727
|
-
// 1. vault#327 case-collisions on a case-insensitive FS —
|
|
1728
|
-
//
|
|
1729
|
-
// 2. Disambiguated filenames — `<base>__<id-prefix>.<ext>` —
|
|
1730
|
-
//
|
|
1731
|
-
// canonical path.
|
|
1913
|
+
// Resolve which sidecar this content file belongs to. Two sources of
|
|
1914
|
+
// ambiguity to handle:
|
|
1915
|
+
// 1. vault#327 case-collisions on a case-insensitive FS — multiple
|
|
1916
|
+
// sidecars share the same lowered (path, ext) key.
|
|
1917
|
+
// 2. Disambiguated filenames — `<base>__<id-prefix>.<ext>` — where
|
|
1918
|
+
// the walker's path doesn't match any sidecar's canonical path.
|
|
1732
1919
|
// Resolution order: exact-case match within the bucket → id-prefix
|
|
1733
|
-
// match against the disambiguation suffix → first remaining
|
|
1734
|
-
//
|
|
1735
|
-
// leftovers.
|
|
1920
|
+
// match against the disambiguation suffix → first remaining sidecar in
|
|
1921
|
+
// the bucket → fall through to id-prefix scan of all leftovers.
|
|
1736
1922
|
let found: Record<string, unknown> | undefined;
|
|
1737
1923
|
const key = `${userPath.toLowerCase()}|${fileExt}`;
|
|
1738
1924
|
const bucket = sidecarByKey.get(key);
|
|
1739
1925
|
if (bucket && bucket.length > 0) {
|
|
1740
1926
|
// Try exact-case match on canonical path first — distinguishes
|
|
1741
|
-
// case-collided notes when the FS preserves filename case but
|
|
1742
|
-
//
|
|
1927
|
+
// case-collided notes when the FS preserves filename case but not
|
|
1928
|
+
// equality.
|
|
1743
1929
|
found = bucket.find((s) => typeof s.path === "string" && s.path === userPath);
|
|
1744
1930
|
if (!found) {
|
|
1745
|
-
// No exact-case match. Pick a sidecar from the bucket whose
|
|
1746
|
-
//
|
|
1747
|
-
//
|
|
1748
|
-
//
|
|
1931
|
+
// No exact-case match. Pick a sidecar from the bucket whose id is
|
|
1932
|
+
// still in `leftover` (i.e. not yet consumed by a prior file in
|
|
1933
|
+
// the walk). This keeps two case-collided notes on a case-sensitive
|
|
1934
|
+
// replay from claiming the same sidecar twice.
|
|
1749
1935
|
found = bucket.find((s) => typeof s.id === "string" && sidecarByIdLeftover.has(s.id));
|
|
1750
1936
|
}
|
|
1751
1937
|
}
|
|
1752
1938
|
if (!found) {
|
|
1753
|
-
// Disambiguated filename fallback: `<base>__<id-prefix>.<ext>`.
|
|
1754
|
-
//
|
|
1755
|
-
//
|
|
1939
|
+
// Disambiguated filename fallback: `<base>__<id-prefix>.<ext>`. Strip
|
|
1940
|
+
// the suffix, then find a leftover sidecar whose id starts with that
|
|
1941
|
+
// prefix.
|
|
1756
1942
|
const disambigMatch = userPath.match(/^(.*)__([A-Za-z0-9-]{6,})$/);
|
|
1757
1943
|
if (disambigMatch) {
|
|
1758
1944
|
const idPrefix = disambigMatch[2]!;
|
|
@@ -1765,16 +1951,15 @@ export async function importPortableVault(
|
|
|
1765
1951
|
}
|
|
1766
1952
|
}
|
|
1767
1953
|
if (!found) {
|
|
1768
|
-
// No sidecar — log and skip. Importing the raw bytes with no
|
|
1769
|
-
//
|
|
1770
|
-
//
|
|
1771
|
-
// shape.
|
|
1954
|
+
// No sidecar — log and skip. Importing the raw bytes with no metadata
|
|
1955
|
+
// would orphan the row (no id, no path, no timestamps). Better to
|
|
1956
|
+
// surface the gap than silently lose shape.
|
|
1772
1957
|
// eslint-disable-next-line no-console
|
|
1773
|
-
console.warn(`[import] skipped "${
|
|
1958
|
+
console.warn(`[import] skipped "${relPath}": no matching sidecar at ${NOTES_META_DIR}/<id>.yaml (path="${userPath}", extension="${fileExt}")`);
|
|
1774
1959
|
continue;
|
|
1775
1960
|
}
|
|
1776
1961
|
frontmatter = found;
|
|
1777
|
-
content =
|
|
1962
|
+
content = source.readContentFile(relPath);
|
|
1778
1963
|
// Mark this sidecar as consumed so subsequent files (and the
|
|
1779
1964
|
// stale-sidecar pass) don't double-count.
|
|
1780
1965
|
const sidecarId = typeof found.id === "string" ? found.id : null;
|
|
@@ -1784,18 +1969,18 @@ export async function importPortableVault(
|
|
|
1784
1969
|
const id = typeof frontmatter.id === "string" ? frontmatter.id : null;
|
|
1785
1970
|
if (!id) {
|
|
1786
1971
|
// No `id` → legacy obsidian-style note. Skip with a warning; the
|
|
1787
|
-
// importer is for the portable-md format, the legacy path stays
|
|
1788
|
-
//
|
|
1972
|
+
// importer is for the portable-md format, the legacy path stays on the
|
|
1973
|
+
// obsidian.ts parseObsidianVault flow.
|
|
1789
1974
|
// eslint-disable-next-line no-console
|
|
1790
|
-
console.warn(`[import] skipped "${
|
|
1975
|
+
console.warn(`[import] skipped "${relPath}": no \`id\` in frontmatter (legacy obsidian format — use parseObsidianVault)`);
|
|
1791
1976
|
continue;
|
|
1792
1977
|
}
|
|
1793
1978
|
const created_at = typeof frontmatter.created_at === "string" ? frontmatter.created_at : new Date().toISOString();
|
|
1794
1979
|
const updated_at = typeof frontmatter.updated_at === "string" ? frontmatter.updated_at : created_at;
|
|
1795
1980
|
const path = typeof frontmatter.path === "string" ? frontmatter.path : undefined;
|
|
1796
1981
|
// Trust the frontmatter/sidecar extension first; fall back to the
|
|
1797
|
-
// filename extension. Notes without an explicit extension default
|
|
1798
|
-
//
|
|
1982
|
+
// filename extension. Notes without an explicit extension default to "md"
|
|
1983
|
+
// (back-compat with pre-vault#328 exports).
|
|
1799
1984
|
const extension = typeof frontmatter.extension === "string"
|
|
1800
1985
|
? frontmatter.extension
|
|
1801
1986
|
: fileExt || "md";
|
|
@@ -1829,35 +2014,27 @@ export async function importPortableVault(
|
|
|
1829
2014
|
// Upsert by id. createNote will throw on duplicate id; check first.
|
|
1830
2015
|
const existing = await store.getNote(id);
|
|
1831
2016
|
if (existing) {
|
|
1832
|
-
// **Upsert merge policy** (vault#319 F2 — pinned here so future
|
|
1833
|
-
//
|
|
2017
|
+
// **Upsert merge policy** (vault#319 F2 — pinned here so future edits
|
|
2018
|
+
// don't drift):
|
|
1834
2019
|
//
|
|
1835
|
-
// - `content`: ALWAYS replaced from the import. (Required —
|
|
1836
|
-
//
|
|
1837
|
-
//
|
|
1838
|
-
//
|
|
1839
|
-
// - `tags`: REPLACED WHOLESALE — existing tags removed,
|
|
1840
|
-
//
|
|
1841
|
-
//
|
|
1842
|
-
// - `path`: REPLACED if the frontmatter declares one;
|
|
1843
|
-
//
|
|
1844
|
-
//
|
|
1845
|
-
//
|
|
1846
|
-
//
|
|
1847
|
-
//
|
|
1848
|
-
// - `metadata`: REPLACED if the frontmatter declares one;
|
|
1849
|
-
// otherwise existing metadata is preserved. Same
|
|
1850
|
-
// upsert-by-field asymmetry as `path`.
|
|
2020
|
+
// - `content`: ALWAYS replaced from the import. (Required — the
|
|
2021
|
+
// import always has content, even if empty string,
|
|
2022
|
+
// and that's the unambiguous source of truth on a
|
|
2023
|
+
// non-blow-away upsert.)
|
|
2024
|
+
// - `tags`: REPLACED WHOLESALE — existing tags removed, imported
|
|
2025
|
+
// set applied. The export is the source of truth for
|
|
2026
|
+
// the current tag set.
|
|
2027
|
+
// - `path`: REPLACED if the frontmatter declares one; otherwise
|
|
2028
|
+
// the existing vault path is preserved. This is
|
|
2029
|
+
// upsert-by-field, NOT replace-by-id.
|
|
2030
|
+
// - `metadata`: REPLACED if the frontmatter declares one; otherwise
|
|
2031
|
+
// existing metadata is preserved. Same upsert-by-field
|
|
2032
|
+
// asymmetry as `path`.
|
|
1851
2033
|
//
|
|
1852
|
-
// For a strict replace-by-id
|
|
1853
|
-
//
|
|
1854
|
-
//
|
|
1855
|
-
//
|
|
1856
|
-
//
|
|
1857
|
-
// Store-level updateNote has no `if_updated_at` set → always
|
|
1858
|
-
// succeeds (precondition gate lives at the HTTP/MCP layer; the
|
|
1859
|
-
// Store accepts unconditional writes from importer/internal
|
|
1860
|
-
// callers).
|
|
2034
|
+
// For a strict replace-by-id, use `--blow-away`. Store-level updateNote
|
|
2035
|
+
// has no `if_updated_at` set → always succeeds (precondition gate lives
|
|
2036
|
+
// at the HTTP/MCP layer; the Store accepts unconditional writes from
|
|
2037
|
+
// importer/internal callers).
|
|
1861
2038
|
await store.updateNote(id, {
|
|
1862
2039
|
content,
|
|
1863
2040
|
...(path !== undefined ? { path } : {}),
|
|
@@ -1884,19 +2061,17 @@ export async function importPortableVault(
|
|
|
1884
2061
|
stats.notes_created++;
|
|
1885
2062
|
}
|
|
1886
2063
|
// Restore both timestamps explicitly. Two reasons:
|
|
1887
|
-
// 1. createNote sets updated_at = created_at; we want the
|
|
1888
|
-
//
|
|
1889
|
-
// 2. update path bumped updated_at to now(); we want to peg it
|
|
1890
|
-
//
|
|
2064
|
+
// 1. createNote sets updated_at = created_at; we want the exported
|
|
2065
|
+
// updated_at (may differ if the note was edited).
|
|
2066
|
+
// 2. update path bumped updated_at to now(); we want to peg it back to
|
|
2067
|
+
// the exported value.
|
|
1891
2068
|
await store.restoreNoteTimestamps(id, created_at, updated_at);
|
|
1892
2069
|
}
|
|
1893
2070
|
|
|
1894
2071
|
// 3b. Drain remaining sidecars (vault#330 S2). Any entry still in
|
|
1895
|
-
// `sidecarByIdLeftover` after the content-file walk is orphaned —
|
|
1896
|
-
//
|
|
2072
|
+
// `sidecarByIdLeftover` after the content-file walk is orphaned — its
|
|
2073
|
+
// expected content file wasn't on disk. Record the gap in
|
|
1897
2074
|
// `skipped_sidecars` so programmatic callers can surface or repair.
|
|
1898
|
-
// Common cause: an operator removed a content file by hand without
|
|
1899
|
-
// deleting the matching sidecar.
|
|
1900
2075
|
for (const [sidecarId, sidecar] of sidecarByIdLeftover) {
|
|
1901
2076
|
const expectedPath = typeof sidecar.path === "string" ? sidecar.path : null;
|
|
1902
2077
|
const expectedExt = typeof sidecar.extension === "string" ? sidecar.extension : null;
|
|
@@ -1912,15 +2087,15 @@ export async function importPortableVault(
|
|
|
1912
2087
|
console.warn(`[import] orphaned sidecar "${sidecarId}.yaml": ${stats.skipped_sidecars[stats.skipped_sidecars.length - 1]!.reason}`);
|
|
1913
2088
|
}
|
|
1914
2089
|
|
|
1915
|
-
// 4. Typed links — replay only now that all notes exist. Wikilinks
|
|
1916
|
-
//
|
|
1917
|
-
//
|
|
2090
|
+
// 4. Typed links — replay only now that all notes exist. Wikilinks (which
|
|
2091
|
+
// the exporter excludes from `links:`) rebuild from content brackets via
|
|
2092
|
+
// syncAllWikilinks (a callable Store method).
|
|
1918
2093
|
for (const [sourceId, portable] of seenNotes) {
|
|
1919
2094
|
if (!portable.links) continue;
|
|
1920
2095
|
for (const link of portable.links) {
|
|
1921
|
-
// Confirm target exists. Forward refs to notes the export
|
|
1922
|
-
//
|
|
1923
|
-
//
|
|
2096
|
+
// Confirm target exists. Forward refs to notes the export didn't
|
|
2097
|
+
// include (subset export) are skipped with a warning rather than
|
|
2098
|
+
// aborting.
|
|
1924
2099
|
const target = await store.getNote(link.target);
|
|
1925
2100
|
if (!target) {
|
|
1926
2101
|
stats.skipped_links.push({
|
|
@@ -1940,11 +2115,9 @@ export async function importPortableVault(
|
|
|
1940
2115
|
}
|
|
1941
2116
|
}
|
|
1942
2117
|
|
|
1943
|
-
// 5. Attachments — DB row then file copy (when
|
|
1944
|
-
// Skip the file-copy phase when
|
|
2118
|
+
// 5. Attachments — DB row then file copy (when the source has attachments
|
|
2119
|
+
// enabled). Skip the file-copy phase when it isn't; the DB row still
|
|
1945
2120
|
// restores so callers operating without an assetsDir keep parity.
|
|
1946
|
-
const assetsDirResolved = opts.assetsDir ? resolvePath(opts.assetsDir) : undefined;
|
|
1947
|
-
const attachmentsRootResolved = resolvePath(join(sidecar, "attachments"));
|
|
1948
2121
|
for (const [noteId, portable] of seenNotes) {
|
|
1949
2122
|
if (!portable.attachments) continue;
|
|
1950
2123
|
for (const att of portable.attachments) {
|
|
@@ -1952,28 +2125,11 @@ export async function importPortableVault(
|
|
|
1952
2125
|
stats.attachments_restored++;
|
|
1953
2126
|
continue;
|
|
1954
2127
|
}
|
|
1955
|
-
// DB row first so the path column matches the export. The
|
|
1956
|
-
//
|
|
1957
|
-
//
|
|
1958
|
-
//
|
|
1959
|
-
//
|
|
1960
|
-
// The public addAttachment generates a fresh id. To preserve the
|
|
1961
|
-
// exported id we need a low-level path; there isn't one in the
|
|
1962
|
-
// Store interface today. Workaround: addAttachment, then update
|
|
1963
|
-
// the row's id via the DB if the Store is a SqliteStore.
|
|
1964
|
-
// TODO: surface a `restoreAttachment(id, noteId, path, mimeType, metadata, createdAt)`
|
|
1965
|
-
// import-only method on the Store interface (parallel to
|
|
1966
|
-
// restoreNoteTimestamps). For now, attachment ids are
|
|
1967
|
-
// re-minted on import — this is a known PR-2-scope limitation
|
|
1968
|
-
// documented in CHANGELOG. Frontmatter refs still resolve by
|
|
1969
|
-
// (note_id, path) tuple on a round-trip; only the att.id values
|
|
1970
|
-
// change. Mark in skipped_attachments? No — the data is there.
|
|
1971
|
-
//
|
|
1972
|
-
// The exporter writes `attachments` keyed by exported att.id;
|
|
1973
|
-
// a round-trip where ids change will produce a byte-different
|
|
1974
|
-
// export (different att.id values). PR 2 round-trip test
|
|
1975
|
-
// therefore can't claim byte-equivalent attachment ids in the
|
|
1976
|
-
// first version — call this out in CHANGELOG.
|
|
2128
|
+
// DB row first so the path column matches the export. The public
|
|
2129
|
+
// addAttachment re-mints the attachment id (there's no id-preserving
|
|
2130
|
+
// Store method today), so a round-trip changes att.id values while
|
|
2131
|
+
// (note_id, path) stays stable — a known PR-2-scope limitation
|
|
2132
|
+
// documented in CHANGELOG.
|
|
1977
2133
|
const attachment = await store.addAttachment(
|
|
1978
2134
|
noteId,
|
|
1979
2135
|
att.path,
|
|
@@ -1981,87 +2137,51 @@ export async function importPortableVault(
|
|
|
1981
2137
|
att.metadata,
|
|
1982
2138
|
);
|
|
1983
2139
|
|
|
1984
|
-
// File copy: from sidecar to assetsDir.
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
const srcFile = join(sidecar, "attachments", att.id, basename(att.path));
|
|
1991
|
-
const srcResolved = resolvePath(srcFile);
|
|
1992
|
-
if (!isWithinDir(srcResolved, attachmentsRootResolved)) {
|
|
2140
|
+
// File copy: from sidecar to assetsDir. Source uses the EXPORTED
|
|
2141
|
+
// att.id (what the exporter wrote), not our freshly-minted row id.
|
|
2142
|
+
if (source.attachmentsEnabled) {
|
|
2143
|
+
const srcRelPath = join(SIDECAR_DIR, "attachments", att.id, basename(att.path));
|
|
2144
|
+
const result = source.restoreAttachment(srcRelPath, att.path);
|
|
2145
|
+
if (!result.ok) {
|
|
1993
2146
|
stats.skipped_attachments.push({
|
|
1994
2147
|
note_id: noteId,
|
|
1995
2148
|
attachment_id: attachment.id,
|
|
1996
|
-
reason:
|
|
2149
|
+
reason: result.reason,
|
|
1997
2150
|
});
|
|
1998
2151
|
continue;
|
|
1999
2152
|
}
|
|
2000
|
-
if (!existsSync(srcResolved)) {
|
|
2001
|
-
stats.skipped_attachments.push({
|
|
2002
|
-
note_id: noteId,
|
|
2003
|
-
attachment_id: attachment.id,
|
|
2004
|
-
reason: `source attachment file missing at "${srcResolved}"`,
|
|
2005
|
-
});
|
|
2006
|
-
continue;
|
|
2007
|
-
}
|
|
2008
|
-
const destFile = join(assetsDirResolved, att.path);
|
|
2009
|
-
const destResolved = resolvePath(destFile);
|
|
2010
|
-
if (!isWithinDir(destResolved, assetsDirResolved)) {
|
|
2011
|
-
stats.skipped_attachments.push({
|
|
2012
|
-
note_id: noteId,
|
|
2013
|
-
attachment_id: attachment.id,
|
|
2014
|
-
reason: `path-traversal on dest: "${destResolved}" escapes assetsDir`,
|
|
2015
|
-
});
|
|
2016
|
-
continue;
|
|
2017
|
-
}
|
|
2018
|
-
mkdirSync(dirname(destResolved), { recursive: true });
|
|
2019
|
-
copyFileSync(srcResolved, destResolved);
|
|
2020
2153
|
}
|
|
2021
2154
|
stats.attachments_restored++;
|
|
2022
2155
|
}
|
|
2023
2156
|
}
|
|
2024
2157
|
|
|
2025
|
-
// 6. Sync wikilinks across the imported set so `[[brackets]]` in
|
|
2026
|
-
//
|
|
2158
|
+
// 6. Sync wikilinks across the imported set so `[[brackets]]` in content
|
|
2159
|
+
// rebuild link rows for the imported notes.
|
|
2027
2160
|
if (!opts.dryRun) {
|
|
2028
2161
|
await store.syncAllWikilinks();
|
|
2029
2162
|
}
|
|
2030
2163
|
|
|
2031
2164
|
// 7. Re-declare indexed fields (belt-and-suspenders + authoritative count).
|
|
2032
|
-
// Step 2 restored tag schemas via `store.upsertTagRecord`, which
|
|
2033
|
-
// the indexed-field lifecycle is centralized in the store — already
|
|
2165
|
+
// Step 2 restored tag schemas via `store.upsertTagRecord`, which already
|
|
2034
2166
|
// materializes the backing generated columns + indexes as it persists each
|
|
2035
|
-
// schema. This explicit reconcile is therefore idempotent on the happy
|
|
2036
|
-
// it stays as a safety net
|
|
2037
|
-
//
|
|
2038
|
-
// count. Without it, a regression in step 2 would silently leave the
|
|
2039
|
-
// imported schemas advertising `indexed: true` while queries full-scan.
|
|
2167
|
+
// schema. This explicit reconcile is therefore idempotent on the happy
|
|
2168
|
+
// path; it stays as a safety net and gives the authoritative
|
|
2169
|
+
// `indexes_declared` count.
|
|
2040
2170
|
if (!opts.dryRun) {
|
|
2041
2171
|
stats.indexes_declared = await store.reconcileDeclaredIndexes();
|
|
2042
2172
|
} else {
|
|
2043
|
-
// Dry-run: count what WOULD be declared without touching the DB.
|
|
2044
|
-
//
|
|
2045
|
-
//
|
|
2046
|
-
//
|
|
2047
|
-
//
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
const resolved = resolvePath(fullPath);
|
|
2056
|
-
if (!isWithinDir(resolved, resolvePath(schemasDir2))) continue;
|
|
2057
|
-
const text = readFileSync(fullPath, "utf-8");
|
|
2058
|
-
const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
|
|
2059
|
-
const { frontmatter } = parseFrontmatter(wrapped);
|
|
2060
|
-
const fields = frontmatter.fields;
|
|
2061
|
-
if (fields && typeof fields === "object" && !Array.isArray(fields)) {
|
|
2062
|
-
for (const spec of Object.values(fields as Record<string, { indexed?: boolean }>)) {
|
|
2063
|
-
if (spec?.indexed === true) stats.indexes_declared++;
|
|
2064
|
-
}
|
|
2173
|
+
// Dry-run: count what WOULD be declared without touching the DB. Counts
|
|
2174
|
+
// every `indexed: true` field including unsupported types, whereas the
|
|
2175
|
+
// applied `reconcileDeclaredIndexes` skips un-indexable types — so the
|
|
2176
|
+
// dry-run can over-count. It's a "how much indexing work" signal, not a
|
|
2177
|
+
// row-exact promise.
|
|
2178
|
+
for (const { text } of source.readSchemaFiles()) {
|
|
2179
|
+
const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
|
|
2180
|
+
const { frontmatter } = parseFrontmatter(wrapped);
|
|
2181
|
+
const fields = frontmatter.fields;
|
|
2182
|
+
if (fields && typeof fields === "object" && !Array.isArray(fields)) {
|
|
2183
|
+
for (const spec of Object.values(fields as Record<string, { indexed?: boolean }>)) {
|
|
2184
|
+
if (spec?.indexed === true) stats.indexes_declared++;
|
|
2065
2185
|
}
|
|
2066
2186
|
}
|
|
2067
2187
|
}
|
|
@@ -2070,6 +2190,30 @@ export async function importPortableVault(
|
|
|
2070
2190
|
return stats;
|
|
2071
2191
|
}
|
|
2072
2192
|
|
|
2193
|
+
/**
|
|
2194
|
+
* Read a portable-md export directory back into a vault (the bun/fs backend
|
|
2195
|
+
* — thin wrapper over `importVault` with an `FsImportSource`). Lossless
|
|
2196
|
+
* counterpart to `exportVaultToDir`. With `blowAway: true`, replaces vault
|
|
2197
|
+
* state byte-equivalent to the export (the disaster-recovery path). Without
|
|
2198
|
+
* it, upserts by frontmatter `id` — existing notes updated in place, new
|
|
2199
|
+
* notes created. See `importVault` for the restoration order.
|
|
2200
|
+
*/
|
|
2201
|
+
export async function importPortableVault(
|
|
2202
|
+
store: Store,
|
|
2203
|
+
opts: ImportOptions,
|
|
2204
|
+
): Promise<ImportStats> {
|
|
2205
|
+
const sidecar = join(opts.inDir, SIDECAR_DIR);
|
|
2206
|
+
if (!existsSync(join(sidecar, "vault.yaml"))) {
|
|
2207
|
+
throw new Error(
|
|
2208
|
+
`not a portable-md export: missing ${join(SIDECAR_DIR, "vault.yaml")} in "${opts.inDir}". ` +
|
|
2209
|
+
`If this is a legacy Obsidian-shape directory, use the obsidian.ts \`parseObsidianVault\` ` +
|
|
2210
|
+
`path instead — vault#308 importer only handles the portable-md format.`,
|
|
2211
|
+
);
|
|
2212
|
+
}
|
|
2213
|
+
const source = new FsImportSource({ inDir: opts.inDir, assetsDir: opts.assetsDir });
|
|
2214
|
+
return importVault(store, source, { blowAway: opts.blowAway, dryRun: opts.dryRun });
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2073
2217
|
// ---------------------------------------------------------------------------
|
|
2074
2218
|
// Parser — shared with `obsidian.ts` (legacy back-compat) via re-export
|
|
2075
2219
|
// ---------------------------------------------------------------------------
|