daftari 1.29.0 → 1.31.0

package/dist/storage/sync.js

@@ -0,0 +1,308 @@
+// The sync engine (#6, spec Decision 3): pushes the canonical local working
+// copy — markdown tree, .git directory, and durable .daftari state — to a
+// storage backend, and can restore a vault from that backing into an empty
+// directory.
+//
+// Key namespace on the backend:
+//   tree/<vault-relative-path>   file bytes
+//   meta/manifest.json           what the backing currently holds
+//
+// The manifest (key → sha256) is the remote source of truth for diffing, so
+// sync is stateless locally and incremental: only changed/new keys upload,
+// removed keys delete. The manifest is written LAST — a crashed sync leaves
+// it understating what was pushed, so the next run re-uploads at worst, never
+// skips.
+//
+// What never syncs: the rebuildable SQLite databases (index.db, locks.db and
+// their journals), the process lock, and transient per-cycle staging files.
+// Durable-but-gitignored audit state (read-log, edges, provenance journals)
+// DOES sync — unlike the index it cannot be rebuilt from the markdown.
+//
+// Concurrency note: sync reads the live tree without taking the process
+// lock, so a write racing the walk can capture a torn snapshot (a ref
+// updated mid-copy). That snapshot is repaired by the next run — acceptable
+// for a backup channel, and the price of syncing while the server serves.
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { err, ok } from "../frontmatter/types.js";
+import { walkFiles } from "./fs-walk.js";
+export const MANIFEST_KEY = "meta/manifest.json";
+const TREE_PREFIX = "tree/";
+const CONCURRENCY = 8;
+// Rebuildable or transient files that must never reach the backing. Everything
+// else under the vault root — including .git and the durable .daftari
+// journals — is part of the backup.
+const SYNC_EXCLUDED = new Set([
+    ".daftari/process.lock",
+    ".daftari/index.db",
+    ".daftari/index.db-journal",
+    ".daftari/index.db-wal",
+    ".daftari/index.db-shm",
+    ".daftari/locks.db",
+    ".daftari/locks.db-journal",
+    ".daftari/locks.db-wal",
+    ".daftari/locks.db-shm",
+    ".daftari/backfill-plan.jsonl",
+    ".daftari/wake-queue.jsonl",
+    // Its own header calls it "ephemeral cross-session memory … rebuildable:
+    // absent OR corrupt ⇒ the empty default" — same class as the wake queue.
+    ".daftari/consolidate-state.json",
+]);
+function isExcluded(relPath) {
+    if (SYNC_EXCLUDED.has(relPath))
+        return true;
+    // Git EXECUTES what these declare — filter/fsmonitor command lines in
+    // config, hook scripts in hooks/ — and a `git clone` deliberately never
+    // transmits them from a remote for exactly that reason. Carrying them
+    // through the backing would turn backend write access into code execution
+    // on the operator's next auto-commit, so they are excluded in BOTH
+    // directions; the restore side re-enforces this against a poisoned
+    // manifest. After a restore, remotes and local config are re-added by the
+    // operator.
+    if (relPath === ".git/config" || relPath.startsWith(".git/hooks/"))
+        return true;
+    // Defensive: a vault should never contain one, but never ship it if it does.
+    return relPath.split("/").includes("node_modules");
+}
+// Streaming hash so the walk phase never buffers a whole file — .git is in
+// scope now, and packfiles can dwarf the markdown they version.
+function streamHash(abs) {
+    return new Promise((resolveHash, rejectHash) => {
+        const digest = createHash("sha256");
+        let size = 0;
+        const stream = createReadStream(abs);
+        stream.on("data", (chunk) => {
+            digest.update(chunk);
+            size += chunk.length;
+        });
+        stream.on("error", rejectHash);
+        stream.on("end", () => resolveHash({ hash: digest.digest("hex"), size }));
+    });
+}
+// (size, mtime) → hash memo so a periodic sync (`daftari serve`, every N
+// minutes) re-hashes only what actually changed instead of streaming the
+// whole tree — with .git in scope, full-history rehash every tick scales
+// with repo size, not the delta. In-memory on purpose: it lives exactly as
+// long as the serving process, needs no invalidation story across
+// processes, and a one-shot CLI run simply pays the honest full cost.
+const hashMemo = new Map();
+async function hashFile(abs) {
+    const s = await stat(abs);
+    const memo = hashMemo.get(abs);
+    if (memo && memo.size === s.size && memo.mtimeMs === s.mtimeMs) {
+        return { hash: memo.hash, size: memo.size };
+    }
+    const hashed = await streamHash(abs);
+    hashMemo.set(abs, { size: hashed.size, mtimeMs: s.mtimeMs, hash: hashed.hash });
+    return hashed;
+}
+// Walks the vault tree (dotfiles included — .git is the point), hashing every
+// non-excluded regular file. Symlinks are skipped and counted: an object
+// store has no symlink notion, and following one could escape the vault.
+async function collectLocalState(vaultRoot) {
+    const root = resolve(vaultRoot);
+    try {
+        const walked = await walkFiles(root);
+        const candidates = walked.files
+            .map((abs) => ({ abs, rel: relative(root, abs).split(sep).join("/") }))
+            .filter((c) => !isExcluded(c.rel));
+        // Evict memo entries for files that no longer exist under THIS root —
+        // without it the memo grows with total churn (git gc repacks, deletes),
+        // not live tree size. Scoped to the root so one process syncing several
+        // vaults never evicts another vault's entries.
+        const live = new Set(candidates.map((c) => c.abs));
+        const rootPrefix = root + sep;
+        for (const key of hashMemo.keys()) {
+            if (key.startsWith(rootPrefix) && !live.has(key))
+                hashMemo.delete(key);
+        }
+        // Hash with the same bounded pool the upload phase uses — the walk now
+        // covers .git, and serializing on disk I/O for packfiles wastes time.
+        const files = new Array(candidates.length);
+        const hashed = await runPool(candidates.map((c, i) => async () => {
+            try {
+                const { hash, size } = await hashFile(c.abs);
+                files[i] = { relPath: c.rel, absPath: c.abs, hash, size };
+                return ok(undefined);
+            }
+            catch (e) {
+                return err(new Error(`cannot hash ${c.rel}: ${e instanceof Error ? e.message : String(e)}`));
+            }
+        }));
+        if (!hashed.ok)
+            return hashed;
+        return ok({ files, skippedSymlinks: walked.symlinks });
+    }
+    catch (e) {
+        return err(new Error(`cannot walk vault: ${e instanceof Error ? e.message : String(e)}`));
+    }
+}
+async function readManifest(backend) {
+    const raw = await backend.get(MANIFEST_KEY);
+    if (!raw.ok)
+        return raw;
+    if (raw.value === null)
+        return ok({ version: 1, files: {} });
+    try {
+        const parsed = JSON.parse(raw.value.toString("utf-8"));
+        if (parsed.version !== 1 || typeof parsed.files !== "object" || parsed.files === null) {
+            return err(new Error(`backing manifest at ${MANIFEST_KEY} has an unknown shape`));
+        }
+        // Null prototype so a file literally named __proto__ (possible in a git
+        // object dir? no — but in the tree, yes) round-trips as an own key.
+        const files = Object.create(null);
+        for (const [k, v] of Object.entries(parsed.files)) {
+            if (typeof v === "string")
+                files[k] = v;
+        }
+        return ok({ version: 1, files });
+    }
+    catch (e) {
+        return err(new Error(`backing manifest is not valid JSON: ${e instanceof Error ? e.message : String(e)}`));
+    }
+}
+// Runs `tasks` with a bounded worker pool, stopping at the first error.
+async function runPool(tasks) {
+    let index = 0;
+    let failure = null;
+    async function worker() {
+        while (failure === null) {
+            const i = index++;
+            const task = tasks[i];
+            if (!task)
+                return;
+            const res = await task();
+            if (!res.ok && failure === null)
+                failure = res.error;
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(CONCURRENCY, tasks.length) }, worker));
+    return failure ? err(failure) : ok(undefined);
+}
+// Push the working copy to the backend. Incremental against the remote
+// manifest; writes the updated manifest last.
+export async function syncVault(vaultRoot, backend, options = {}) {
+    const local = await collectLocalState(vaultRoot);
+    if (!local.ok)
+        return local;
+    const manifest = await readManifest(backend);
+    if (!manifest.ok)
+        return manifest;
+    const remote = manifest.value.files;
+    const localByPath = new Map(local.value.files.map((f) => [f.relPath, f]));
+    const toUpload = local.value.files.filter((f) => remote[f.relPath] !== f.hash);
+    const toDelete = Object.keys(remote).filter((relPath) => !localByPath.has(relPath));
+    const unchanged = local.value.files.length - toUpload.length;
+    const summary = {
+        uploaded: toUpload.length,
+        deleted: toDelete.length,
+        unchanged,
+        skippedSymlinks: local.value.skippedSymlinks,
+        bytesUploaded: toUpload.reduce((n, f) => n + f.size, 0),
+    };
+    if (options.dryRun)
+        return ok(summary);
+    const uploads = toUpload.map((f) => async () => {
+        // Re-read at upload time: the hash was computed during the walk and the
+        // manifest must describe the bytes actually pushed, so hash what we send.
+        let bytes;
+        try {
+            bytes = await readFile(f.absPath);
+        }
+        catch (e) {
+            return err(new Error(`cannot read ${f.relPath}: ${e instanceof Error ? e.message : String(e)}`));
+        }
+        f.hash = createHash("sha256").update(bytes).digest("hex");
+        f.size = bytes.byteLength;
+        return backend.put(TREE_PREFIX + f.relPath, bytes);
+    });
+    const uploaded = await runPool(uploads);
+    if (!uploaded.ok)
+        return uploaded;
+    const deletes = toDelete.map((relPath) => () => backend.delete(TREE_PREFIX + relPath));
+    const deleted = await runPool(deletes);
+    if (!deleted.ok)
+        return deleted;
+    const nextManifest = { version: 1, files: {} };
+    const nextFiles = Object.create(null);
+    for (const f of local.value.files)
+        nextFiles[f.relPath] = f.hash;
+    nextManifest.files = nextFiles;
+    const wrote = await backend.put(MANIFEST_KEY, Buffer.from(JSON.stringify(nextManifest, null, 2), "utf-8"));
+    if (!wrote.ok)
+        return wrote;
+    return ok(summary);
+}
+// Restore a vault from the backing into `vaultRoot`, which must be empty or
+// absent — restore never merges into or overwrites live state. After a
+// restore the caller reindexes (the index was deliberately never synced).
+export async function restoreVault(vaultRoot, backend) {
+    const root = resolve(vaultRoot);
+    try {
+        const existing = await readdir(root);
+        if (existing.length > 0) {
+            return err(new Error(`refusing to restore into non-empty directory: ${root}`));
+        }
+    }
+    catch (e) {
+        if (e.code !== "ENOENT") {
+            return err(new Error(`cannot inspect ${root}: ${e instanceof Error ? e.message : String(e)}`));
+        }
+    }
+    const manifest = await readManifest(backend);
+    if (!manifest.ok)
+        return manifest;
+    // The manifest is data from the backing, not trusted input: re-apply the
+    // exclusion rules here so a poisoned manifest cannot smuggle in what push
+    // would never have uploaded (git config/hooks — code execution on the
+    // next auto-commit).
+    const allPaths = Object.keys(manifest.value.files);
+    const paths = allPaths.filter((p) => !isExcluded(p));
+    const skippedExcluded = allPaths.length - paths.length;
+    if (allPaths.length === 0) {
+        return err(new Error(`backing at ${backend.id} holds no manifest — nothing to restore`));
+    }
+    let bytesTotal = 0;
+    const tasks = paths.map((relPath) => async () => {
+        // Confine each manifest path to the restore root — the manifest is data
+        // from the backing, not trusted input.
+        const target = resolve(root, relPath);
+        const rel = relative(root, target);
+        if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) {
+            return err(new Error(`manifest path escapes restore root: ${relPath}`));
+        }
+        const got = await backend.get(TREE_PREFIX + relPath);
+        if (!got.ok)
+            return got;
+        if (got.value === null) {
+            return err(new Error(`backing is missing ${TREE_PREFIX}${relPath} listed in its manifest`));
+        }
+        // Verify the bytes against the manifest's hash — that is what the hash
+        // is FOR. This catches corruption, torn writes, and a push racing the
+        // restore; it is not a tamper defense (an attacker who can rewrite
+        // objects can rewrite the manifest too).
+        const gotHash = createHash("sha256").update(got.value).digest("hex");
+        const wantHash = manifest.value.files[relPath];
+        if (gotHash !== wantHash) {
+            return err(new Error(`backing object ${TREE_PREFIX}${relPath} does not match its manifest hash ` +
+                `(got ${gotHash.slice(0, 12)}…, manifest says ${wantHash?.slice(0, 12)}…) — ` +
+                "the backing is corrupt or a push is racing this restore"));
+        }
+        try {
+            await mkdir(dirname(target), { recursive: true });
+            await writeFile(target, got.value);
+        }
+        catch (e) {
+            return err(new Error(`cannot write ${relPath}: ${e instanceof Error ? e.message : String(e)}`));
+        }
+        bytesTotal += got.value.byteLength;
+        return ok(undefined);
+    });
+    const ran = await runPool(tasks);
+    if (!ran.ok)
+        return ran;
+    return ok({ restored: paths.length, bytes: bytesTotal, skippedExcluded });
+}
+//# sourceMappingURL=sync.js.map

package/dist/storage/sync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/storage/sync.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,0EAA0E;AAC1E,2EAA2E;AAC3E,aAAa;AACb,EAAE;AACF,gCAAgC;AAChC,4CAA4C;AAC5C,kEAAkE;AAClE,EAAE;AACF,4EAA4E;AAC5E,2EAA2E;AAC3E,4EAA4E;AAC5E,8EAA8E;AAC9E,SAAS;AACT,EAAE;AACF,6EAA6E;AAC7E,4EAA4E;AAC5E,4EAA4E;AAC5E,uEAAuE;AACvE,EAAE;AACF,wEAAwE;AACxE,sEAAsE;AACtE,4EAA4E;AAC5E,0EAA0E;AAE1E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACxE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,MAAM,CAAC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AACjD,MAAM,WAAW,GAAG,OAAO,CAAC;AAC5B,MAAM,WAAW,GAAG,CAAC,CAAC;AAEtB,+EAA+E;AAC/E,sEAAsE;AACtE,oCAAoC;AACpC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,uBAAuB;IACvB,mBAAmB;IACnB,2BAA2B;IAC3B,uBAAuB;IACvB,uBAAuB;IACvB,mBAAmB;IACnB,2BAA2B;IAC3B,uBAAuB;IACvB,uBAAuB;IACvB,8BAA8B;IAC9B,2BAA2B;IAC3B,yEAAyE;IACzE,yEAAyE;IACzE,iCAAiC;CAClC,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,OAAe;IACjC,IAAI,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,sEAAsE;IACtE,wEAAwE;IACxE,sEAAsE;IACtE,0EAA0E;IAC1E,mEAAmE;IACnE,mEAAmE;IACnE,0EAA0E;IAC1E,YAAY;IACZ,IAAI,OAAO,KAAK,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,IAAI,CAAC;IAChF,6EAA6E;IAC7E,OAAO,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;AACrD,CAAC;AAuBD,2EAA2E;AAC3E,gEAAgE;AAChE,SAAS,UAAU,CAAC,GAAW;IAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,UAAU,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC/B,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;AACL,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,yEAAyE;AACzE,2EAA2E;AAC3E,kEAAkE;AAClE,sEAAsE;AACtE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA2D,CAAC;AAEpF,KAAK,UAAU,QAAQ,CAAC,GAAW;IACjC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;QAC/D,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9C,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IACrC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAChF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,yEAAyE;AACzE,yEAAyE;AACzE,KAAK,UAAU,iBAAiB,CAC9B,SAAiB;IAEjB,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK;aAC5B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aACtE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,sEAAsE;QACtE,wEAAwE;QACxE,wEAAwE;QACxE,+CAA+C;QAC/C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,IAAI,GAAG,GAAG,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzE,CAAC;QACD,uEAAuE;QACvE,sEAAsE;QACtE,MAAM,KAAK,GAAgB,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,MAAM,OAAO,CAC1B,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,IAAkC,EAAE;YAChE,IAAI,CAAC;gBACH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC7C,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAC1D,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC;YACvB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,GAAG,CACR,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CACjF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,OAAO,MAAM,CAAC;QAC9B,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5F,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,OAAuB;IACjD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAC5C,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC;IACxB,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IAC7D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAiB,CAAC;QACvE,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACtF,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,uBAAuB,YAAY,uBAAuB,CAAC,CAAC,CAAC;QACpF,CAAC;QACD,wEAAwE;QACxE,oEAAoE;QACpE,MAAM,KAAK,GAA2B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,IAAI,OAAO,CAAC,KAAK,QAAQ;gBAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,GAAG,CACR,IAAI,KAAK,CACP,uCAAuC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACpF,CACF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,KAAK,UAAU,OAAO,CACpB,KAA6C;IAE7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAiB,IAAI,CAAC;IACjC,KAAK,UAAU,MAAM;QACnB,OAAO,OAAO,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,MAAM,GAAG,GAAG,MAAM,IAAI,EAAE,CAAC;YACzB,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,OAAO,KAAK,IAAI;gBAAE,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC;QACvD,CAAC;IACH,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IACvF,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;AAChD,CAAC;AAMD,uEAAuE;AACvE,8CAA8C;AAC9C,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,SAAiB,EACjB,OAAuB,EACvB,UAAuB,EAAE;IAEzB,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACjD,IAAI,CAAC,KAAK,CAAC,EAAE;QAAE,OAAO,KAAK,CAAC;IAC5B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,QAAQ,CAAC;IAElC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1E,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IACpF,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAE7D,MAAM,OAAO,GAAgB;QAC3B,QAAQ,EAAE,QAAQ,CAAC,MAAM;QACzB,OAAO,EAAE,QAAQ,CAAC,MAAM;QACxB,SAAS;QACT,eAAe,EAAE,KAAK,CAAC,KAAK,CAAC,eAAe;QAC5C,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;KACxD,CAAC;IACF,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC;IAEvC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,IAAkC,EAAE;QAC3E,wEAAwE;QACxE,0EAA0E;QAC1E,IAAI,KAAa,CAAC;QAClB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,CACR,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,OAAO,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CACrF,CAAC;QACJ,CAAC;QACD,CAAC,CAAC,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1D,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC;QAC1B,OAAO,OAAO,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,QAAQ,CAAC;IAElC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC;IACvF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACvC,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,OAAO,CAAC;IAEhC,MAAM,YAAY,GAAiB,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAC7D,MAAM,SAAS,GAA2B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9D,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK;QAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;IACjE,YAAY,CAAC,KAAK,GAAG,SAAS,CAAC;IAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,YAAY,EACZ,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAC5D,CAAC;IACF,IAAI,CAAC,KAAK,CAAC,EAAE;QAAE,OAAO,KAAK,CAAC;IAE5B,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC;AAWD,4EAA4E;AAC5E,uEAAuE;AACvE,0EAA0E;AAC1E,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAiB,EACjB,OAAuB;IAEvB,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,iDAAiD,IAAI,EAAE,CAAC,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAO,GAAG,CACR,IAAI,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CACnF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,QAAQ,CAAC;IAClC,yEAAyE;IACzE,0EAA0E;IAC1E,sEAAsE;IACtE,qBAAqB;IACrB,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACvD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,cAAc,OAAO,CAAC,EAAE,yCAAyC,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,IAAkC,EAAE;QAC5E,wEAAwE;QACxE,uCAAuC;QACvC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnC,IAAI,GAAG,KAAK,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1D,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,CAAC;QACrD,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,GAAG,CAAC;QACxB,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACvB,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,sBAAsB,WAAW,GAAG,OAAO,yBAAyB,CAAC,CAAC,CAAC;QAC9F,CAAC;QACD,uEAAuE;QACvE,sEAAsE;QACtE,mEAAmE;QACnE,yCAAyC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzB,OAAO,GAAG,CACR,IAAI,KAAK,CACP,kBAAkB,WAAW,GAAG,OAAO,oCAAoC;gBACzE,QAAQ,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO;gBAC7E,yDAAyD,CAC5D,CACF,CAAC;QACJ,CAAC;QACD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,CACR,IAAI,KAAK,CAAC,gBAAgB,OAAO,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CACpF,CAAC;QACJ,CAAC;QACD,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC;QACnC,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC;IAExB,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC,CAAC;AAC5E,CAAC"}

package/dist/sync/index.d.ts

@@ -0,0 +1,2 @@
+export declare function runSync(argv: string[]): Promise<number>;
+//# sourceMappingURL=index.d.ts.map

package/dist/sync/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sync/index.ts"],"names":[],"mappings":"AA8BA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAwD7D"}

package/dist/sync/index.js

@@ -0,0 +1,136 @@
+// `daftari sync` (#6): push the canonical local vault to its configured
+// storage backing, or restore a vault from the backing into an empty
+// directory. The backing is configured in .daftari/config.yaml (`storage`
+// block); credentials come from the SDKs' standard environment chains.
+import { parseFlag } from "../index.js";
+import { reindexVault, reindexWarnings } from "../search/reindex.js";
+import { createBackend } from "../storage/backend.js";
+import { restoreVault, syncVault } from "../storage/sync.js";
+import { loadConfig } from "../utils/config.js";
+const USAGE = `daftari sync — push a vault to its storage backing, or restore from it.
+
+Usage:
+  daftari sync --vault <path> [--dry-run]     Push the working copy to the backing
+  daftari sync --vault <path> --restore       Restore into an EMPTY directory, then reindex
+
+The backing is the \`storage\` block in .daftari/config.yaml (backends: fs,
+s3 — including MinIO/R2/GCS-interop endpoints — and azure). The local git
+working copy is canonical; the backing is durable backing, not a second
+source of truth. The SQLite index and lock files are never synced. A restore
+has no local config to read yet, so the backing is named with flags using
+the config block's names (--backend s3 --bucket team-vault [--prefix …]).
+
+Push reads the live tree without stopping a running server; a write racing
+the walk is repaired by the next sync.
+
+Exit codes: 2 config/usage error, 3 runtime error.
+`;
+export async function runSync(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        process.stdout.write(USAGE);
+        return 0;
+    }
+    const vault = parseFlag(argv, "vault");
+    if (!vault) {
+        process.stderr.write("daftari sync: --vault <path> is required\n\n");
+        process.stderr.write(USAGE);
+        return 2;
+    }
+    const restore = argv.includes("--restore");
+    const dryRun = argv.includes("--dry-run");
+    if (restore && dryRun) {
+        process.stderr.write("daftari sync: --restore and --dry-run cannot be combined\n");
+        return 2;
+    }
+    if (restore) {
+        return runRestore(vault, argv);
+    }
+    const config = loadConfig(vault);
+    if (!config.ok) {
+        process.stderr.write(`daftari sync: ${config.error.message}\n`);
+        return 2;
+    }
+    if (!config.value.storage) {
+        process.stderr.write("daftari sync: no storage backing configured — add a `storage` block " +
+            "to .daftari/config.yaml (see README: Storage backing)\n");
+        return 2;
+    }
+    const backend = await createBackend(config.value.storage);
+    if (!backend.ok) {
+        process.stderr.write(`daftari sync: ${backend.error.message}\n`);
+        return 2;
+    }
+    const result = await syncVault(vault, backend.value, { dryRun });
+    if (!result.ok) {
+        process.stderr.write(`daftari sync: ${result.error.message}\n`);
+        return 3;
+    }
+    const s = result.value;
+    process.stdout.write(`${dryRun ? "[dry-run] would push" : "pushed"} to ${backend.value.id}: ` +
+        `${s.uploaded} uploaded (${s.bytesUploaded} bytes), ${s.deleted} deleted, ` +
+        `${s.unchanged} unchanged` +
+        (s.skippedSymlinks > 0 ? `, ${s.skippedSymlinks} symlink(s) skipped` : "") +
+        "\n");
+    return 0;
+}
+// Restore cannot read the vault's config (the vault does not exist yet), so
+// the backing is described on the command line with the same names as the
+// config block.
+async function runRestore(vault, argv) {
+    const backendName = parseFlag(argv, "backend");
+    if (!backendName || !["fs", "s3", "azure"].includes(backendName)) {
+        process.stderr.write("daftari sync --restore: --backend <fs|s3|azure> is required " +
+            "(plus its target flag: --path, --bucket, or --container)\n");
+        return 2;
+    }
+    const storage = {
+        backend: backendName,
+        path: parseFlag(argv, "path") ?? undefined,
+        bucket: parseFlag(argv, "bucket") ?? undefined,
+        container: parseFlag(argv, "container") ?? undefined,
+        prefix: parseFlag(argv, "prefix") ?? undefined,
+        region: parseFlag(argv, "region") ?? undefined,
+        endpoint: parseFlag(argv, "endpoint") ?? undefined,
+        forcePathStyle: argv.includes("--force-path-style") ? true : undefined,
+    };
+    const required = { fs: "path", s3: "bucket", azure: "container" }[storage.backend];
+    if (!storage[required]) {
+        process.stderr.write(`daftari sync --restore: --${required} is required for ${storage.backend}\n`);
+        return 2;
+    }
+    const backend = await createBackend(storage);
+    if (!backend.ok) {
+        process.stderr.write(`daftari sync: ${backend.error.message}\n`);
+        return 2;
+    }
+    const restored = await restoreVault(vault, backend.value);
+    if (!restored.ok) {
+        process.stderr.write(`daftari sync: ${restored.error.message}\n`);
+        return 3;
+    }
+    process.stdout.write(`restored ${restored.value.restored} files (${restored.value.bytes} bytes) ` +
+        `from ${backend.value.id} into ${vault}\n`);
+    if (restored.value.skippedExcluded > 0) {
+        process.stderr.write(`daftari sync: warning: ${restored.value.skippedExcluded} manifest entr(ies) refused by ` +
+            "the exclusion rules (git config/hooks or rebuildable state) — the backing was " +
+            "written by something other than daftari sync\n");
+    }
+    process.stderr.write("daftari sync: note: git config and hooks are never synced or restored " +
+        "(git executes them; a backup channel must not deliver code) — re-add " +
+        "remotes and local git config by hand\n");
+    // The index was deliberately never synced — rebuild it so the restored
+    // vault is immediately servable.
+    const indexed = await reindexVault(vault);
+    if (indexed.ok) {
+        process.stdout.write(`reindexed ${indexed.value.documentCount} docs\n`);
+        for (const line of reindexWarnings(indexed.value)) {
+            process.stderr.write(`daftari sync: warning: ${line}\n`);
+        }
+    }
+    else {
+        process.stderr.write(`daftari sync: warning: reindex after restore failed: ${indexed.error.message}\n` +
+            "  run `daftari --vault <path> --reindex` before serving\n");
+    }
+    return 0;
+}
+//# sourceMappingURL=index.js.map

package/dist/sync/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sync/index.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,qEAAqE;AACrE,0EAA0E;AAC1E,uEAAuE;AAEvE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;CAiBb,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc;IAC1C,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACrE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC1C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;QACnF,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAChE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sEAAsE;YACpE,yDAAyD,CAC5D,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1D,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QACjE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAChE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;IACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,MAAM,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,OAAO,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI;QACtE,GAAG,CAAC,CAAC,QAAQ,cAAc,CAAC,CAAC,aAAa,YAAY,CAAC,CAAC,OAAO,YAAY;QAC3E,GAAG,CAAC,CAAC,SAAS,YAAY;QAC1B,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,eAAe,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,IAAI,CACP,CAAC;IACF,OAAO,CAAC,CAAC;AACX,CAAC;AAED,4EAA4E;AAC5E,0EAA0E;AAC1E,gBAAgB;AAChB,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,IAAc;IACrD,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC/C,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACjE,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,8DAA8D;YAC5D,4DAA4D,CAC/D,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,OAAO,GAAG;QACd,OAAO,EAAE,WAAoC;QAC7C,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,SAAS;QAC1C,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,SAAS;QAC9C,SAAS,EAAE,SAAS,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,SAAS;QACpD,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,SAAS;QAC9C,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,SAAS;QAC9C,QAAQ,EAAE,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,SAAS;QAClD,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;KACvE,CAAC;IACF,MAAM,QAAQ,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,CAGlE,CAAC;IAChB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6BAA6B,QAAQ,oBAAoB,OAAO,CAAC,OAAO,IAAI,CAC7E,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QACjE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,QAAQ,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAClE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,YAAY,QAAQ,CAAC,KAAK,CAAC,QAAQ,WAAW,QAAQ,CAAC,KAAK,CAAC,KAAK,UAAU;QAC1E,QAAQ,OAAO,CAAC,KAAK,CAAC,EAAE,SAAS,KAAK,IAAI,CAC7C,CAAC;IACF,IAAI,QAAQ,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,0BAA0B,QAAQ,CAAC,KAAK,CAAC,eAAe,iCAAiC;YACvF,gFAAgF;YAChF,gDAAgD,CACnD,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wEAAwE;QACtE,uEAAuE;QACvE,wCAAwC,CAC3C,CAAC;IAEF,uEAAuE;IACvE,iCAAiC;IACjC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,aAAa,SAAS,CAAC,CAAC;QACxE,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,IAAI,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wDAAwD,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI;YAC/E,2DAA2D,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/themes/clustering.d.ts

@@ -1,6 +1,5 @@
 export declare function seededRng(seed: number): () => number;
 export declare function l2Normalize(vec: Float32Array): Float32Array;
-export declare function meanPoolL2(vectors: Float32Array[]): Float32Array | null;
 export declare function kmeansPlusPlusInit(data: Float32Array[], k: number, rng: () => number): number[];
 export interface KMeansResult {
     assignments: number[];
@@ -16,15 +15,11 @@ export interface PickKResult {
     assignments: number[];
     centroids: Float32Array[];
 }
-export declare function pickK(data: Float32Array[], candidates: number[], rng: () => number, maxIter: number): PickKResult;
-export interface SecondaryMembership {
-    docIndex: number;
-    sim: number;
+export declare function strideSample(n: number, cap: number): number[];
+export declare function pickK(data: Float32Array[], candidates: number[], rng: () => number, maxIter: number, silhouetteCap?: number): PickKResult;
+export interface DocMembership {
+    cluster: number;
+    weight: number;
 }
-export interface SecondaryOptions {
-    delta: number;
-    minSimilarity: number;
-    maxPerDoc: number;
-}
-export declare function selectSecondaryMemberships(vectors: Float32Array[], assignments: number[], centroids: Float32Array[], options: SecondaryOptions): Map<number, SecondaryMembership[]>;
+export declare function membershipDistributions(chunkAssignments: number[], chunkDoc: number[], docCount: number, minFraction: number): DocMembership[][];
 //# sourceMappingURL=clustering.d.ts.map

package/dist/themes/clustering.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"clustering.d.ts","sourceRoot":"","sources":["../../src/themes/clustering.ts"],"names":[],"mappings":"AAwBA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CASpD;AAMD,wBAAgB,WAAW,CAAC,GAAG,EAAE,YAAY,GAAG,YAAY,CAa3D;AAKD,wBAAgB,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,GAAG,IAAI,CAsBvE;AAoCD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,YAAY,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,MAAM,GAAG,MAAM,EAAE,CAoE/F;AAID,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AAKD,wBAAgB,MAAM,CACpB,IAAI,EAAE,YAAY,EAAE,EACpB,CAAC,EAAE,MAAM,EACT,GAAG,EAAE,MAAM,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,YAAY,CAkFd;AAUD,wBAAgB,eAAe,CAAC,IAAI,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,CA6CnF;AAQD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CAahE;AAID,MAAM,WAAW,WAAW;IAC1B,CAAC,EAAE,MAAM,CAAC;IACV,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;CAC3B;AAOD,wBAAgB,KAAK,CACnB,IAAI,EAAE,YAAY,EAAE,EACpB,UAAU,EAAE,MAAM,EAAE,EACpB,GAAG,EAAE,MAAM,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,WAAW,CAuBb;AAQD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,gBAAgB;IAG/B,KAAK,EAAE,MAAM,CAAC;IAId,aAAa,EAAE,MAAM,CAAC;IAItB,SAAS,EAAE,MAAM,CAAC;CACnB;AAUD,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,YAAY,EAAE,EACvB,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,YAAY,EAAE,EACzB,OAAO,EAAE,gBAAgB,GACxB,GAAG,CAAC,MAAM,EAAE,mBAAmB,EAAE,CAAC,CA2CpC"}
+{"version":3,"file":"clustering.d.ts","sourceRoot":"","sources":["../../src/themes/clustering.ts"],"names":[],"mappings":"AAwBA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CASpD;AAMD,wBAAgB,WAAW,CAAC,GAAG,EAAE,YAAY,GAAG,YAAY,CAa3D;AAoCD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,YAAY,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,MAAM,GAAG,MAAM,EAAE,CAoE/F;AAID,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AAKD,wBAAgB,MAAM,CACpB,IAAI,EAAE,YAAY,EAAE,EACpB,CAAC,EAAE,MAAM,EACT,GAAG,EAAE,MAAM,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,YAAY,CAkFd;AAUD,wBAAgB,eAAe,CAAC,IAAI,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,CA6CnF;AAQD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CAahE;AAID,MAAM,WAAW,WAAW;IAC1B,CAAC,EAAE,MAAM,CAAC;IACV,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;CAC3B;AAMD,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAO7D;AAaD,wBAAgB,KAAK,CACnB,IAAI,EAAE,YAAY,EAAE,EACpB,UAAU,EAAE,MAAM,EAAE,EACpB,GAAG,EAAE,MAAM,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,aAAa,CAAC,EAAE,MAAM,GACrB,WAAW,CA8Bb;AAOD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAgBD,wBAAgB,uBAAuB,CACrC,gBAAgB,EAAE,MAAM,EAAE,EAC1B,QAAQ,EAAE,MAAM,EAAE,EAClB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,aAAa,EAAE,EAAE,CA2CnB"}