gnosys 5.11.4 → 5.12.0

package/dist/lib/syncIngestTimerCommand.js

@@ -0,0 +1,83 @@
+import { printStatus } from "./setup/ui/status.js";
+import { getSyncIngestTimerStatus, installSyncIngestTimer, uninstallSyncIngestTimer, } from "./syncIngestTimer.js";
+export async function runSyncIngestTimerCommand(opts) {
+    const intervalMinutes = Math.max(1, parseInt(opts.interval ?? "15", 10) || 15);
+    const timerStatus = getSyncIngestTimerStatus();
+    if (opts.json) {
+        if (opts.install) {
+            try {
+                const path = installSyncIngestTimer(intervalMinutes);
+                console.log(JSON.stringify({
+                    action: "install",
+                    installed: true,
+                    path,
+                    intervalMinutes,
+                    platform: timerStatus.platform,
+                }));
+            }
+            catch (err) {
+                process.exitCode = 1;
+                console.log(JSON.stringify({
+                    action: "install",
+                    installed: false,
+                    error: err instanceof Error ? err.message : String(err),
+                    platform: timerStatus.platform,
+                }));
+            }
+            return;
+        }
+        if (opts.uninstall) {
+            try {
+                const path = uninstallSyncIngestTimer();
+                console.log(JSON.stringify({ action: "uninstall", installed: false, path }));
+            }
+            catch (err) {
+                process.exitCode = 1;
+                console.log(JSON.stringify({
+                    action: "uninstall",
+                    error: err instanceof Error ? err.message : String(err),
+                }));
+            }
+            return;
+        }
+        console.log(JSON.stringify({ ...getSyncIngestTimerStatus(), intervalMinutes }));
+        return;
+    }
+    if (timerStatus.platform === "unsupported") {
+        console.log("OS-level ingest timer is not supported on this platform.");
+        console.log("Windows: use Task Scheduler manually or run `gnosys setup remote doctor --ingest`.");
+        if (opts.install || opts.uninstall)
+            process.exitCode = 1;
+        return;
+    }
+    if (opts.install) {
+        try {
+            const installedPath = installSyncIngestTimer(intervalMinutes);
+            printStatus("ok", `Ingest timer installed (${installedPath}, every ${intervalMinutes} min)`);
+        }
+        catch (err) {
+            printStatus("fail", err instanceof Error ? err.message : String(err));
+            process.exitCode = 1;
+        }
+        return;
+    }
+    if (opts.uninstall) {
+        try {
+            const removedPath = uninstallSyncIngestTimer();
+            printStatus("ok", `Ingest timer uninstalled (${removedPath})`);
+        }
+        catch (err) {
+            printStatus("fail", err instanceof Error ? err.message : String(err));
+            process.exitCode = 1;
+        }
+        return;
+    }
+    const current = getSyncIngestTimerStatus();
+    if (current.installed) {
+        console.log(`Ingest timer: installed (${current.path})`);
+    }
+    else {
+        console.log("Ingest timer: not installed");
+        console.log("Install with: gnosys setup remote timer --install");
+    }
+}

package/dist/lib/syncLock.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Sync advisory lock on the master's **local** ~/.gnosys (never on network share).
+ */
+/** Synchronous ingest lock for master sweep (same directory as lock file). */
+export declare function acquireWriteLockSync(lockPath: string, operation: string): () => void;
+export declare function releaseWriteLockSync(lockPath: string): void;

package/dist/lib/syncLock.js

@@ -0,0 +1,74 @@
+/**
+ * Sync advisory lock on the master's **local** ~/.gnosys (never on network share).
+ */
+import fs from "fs";
+import path from "path";
+const LOCK_STALE_MS = 120_000;
+function readLock(lockPath) {
+    try {
+        const raw = fs.readFileSync(lockPath, "utf-8");
+        const o = JSON.parse(raw);
+        if (typeof o.pid === "number" && typeof o.timestamp === "number")
+            return o;
+    }
+    catch {
+        // no lock
+    }
+    return null;
+}
+function isStale(lock) {
+    if (Date.now() - lock.timestamp > LOCK_STALE_MS)
+        return true;
+    try {
+        process.kill(lock.pid, 0);
+        return false;
+    }
+    catch {
+        return true;
+    }
+}
+/** Synchronous ingest lock for master sweep (same directory as lock file). */
+export function acquireWriteLockSync(lockPath, operation) {
+    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+    const start = Date.now();
+    while (true) {
+        const existing = readLock(lockPath);
+        if (existing && !isStale(existing)) {
+            if (Date.now() - start > 30_000) {
+                throw new Error(`ingest lock timeout (held by pid ${existing.pid})`);
+            }
+            // busy-wait briefly
+            const waitUntil = Date.now() + 50;
+            while (Date.now() < waitUntil) {
+                /* spin */
+            }
+            continue;
+        }
+        if (existing) {
+            try {
+                fs.unlinkSync(lockPath);
+            }
+            catch {
+                // race
+            }
+        }
+        try {
+            fs.writeFileSync(lockPath, JSON.stringify({ pid: process.pid, timestamp: Date.now(), operation }));
+            return () => releaseWriteLockSync(lockPath);
+        }
+        catch {
+            if (Date.now() - start > 30_000)
+                throw new Error("failed to acquire ingest lock");
+        }
+    }
+}
+export function releaseWriteLockSync(lockPath) {
+    try {
+        const existing = readLock(lockPath);
+        if (existing?.pid === process.pid)
+            fs.unlinkSync(lockPath);
+    }
+    catch {
+        // ignore
+    }
+}

package/dist/lib/syncSnapshot.d.ts

@@ -0,0 +1,30 @@
+/**
+ * v13 snapshot publication — immutable DB copies + manifest with epoch fencing.
+ */
+export interface SnapshotManifestFile {
+    epoch: number;
+    seq: number;
+    snapshotFile: string;
+    publishedAt: string;
+    checksum: string;
+    sizeBytes: number;
+}
+export declare function masterSnapshotsDir(masterPath: string): string;
+export declare function clientSnapshotStore(masterPath: string): string;
+/** Publish a new immutable snapshot on the master (re-validates lease epoch first). */
+export declare function publishMasterSnapshot(masterPath: string): Promise<SnapshotManifestFile | null>;
+export declare function compareSnapshotVersion(accepted: {
+    epoch: number;
+    seq: number;
+} | null, incoming: {
+    epoch: number;
+    seq: number;
+}): boolean;
+/** Copy master snapshot to local client store atomically. */
+export declare function acceptClientSnapshot(masterPath: string, manifest: SnapshotManifestFile): {
+    ok: boolean;
+    localPath?: string;
+    reason?: string;
+};
+export declare function getClientAcceptedManifest(masterPath: string): SnapshotManifestFile | null;
+export declare function formatSnapshotAge(publishedAt: string): string;

package/dist/lib/syncSnapshot.js

@@ -0,0 +1,184 @@
+/**
+ * v13 snapshot publication — immutable DB copies + manifest with epoch fencing.
+ */
+import { createHash } from "crypto";
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, } from "fs";
+import path from "path";
+import { GnosysDB } from "./db.js";
+import { getGnosysHome } from "./paths.js";
+import { atomicWriteFileSync } from "./atomicWrite.js";
+import { assertMasterLeaseHeld, readMasterMarker, touchMasterMarkerHeartbeat, validateLeaseEpochBeforeWrite, } from "./masterLease.js";
+import { ensureMachineConfig } from "./machineConfig.js";
+import { acquireWriteLockSync } from "./syncLock.js";
+const SNAPSHOT_RETENTION = 2;
+const MANIFEST_FILENAME = "snapshot-manifest.json";
+export function masterSnapshotsDir(masterPath) {
+    return path.join(masterPath, "snapshots");
+}
+export function clientSnapshotStore(masterPath) {
+    const key = createHash("sha256").update(path.resolve(masterPath)).digest("hex").slice(0, 16);
+    return path.join(getGnosysHome(), "client-snapshots", key);
+}
+function sha256File(filePath) {
+    const buf = readFileSync(filePath);
+    return createHash("sha256").update(buf).digest("hex");
+}
+function readManifestFile(masterPath) {
+    const p = path.join(masterSnapshotsDir(masterPath), MANIFEST_FILENAME);
+    if (!existsSync(p))
+        return null;
+    try {
+        return JSON.parse(readFileSync(p, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+function writeManifestFile(masterPath, manifest) {
+    const dir = masterSnapshotsDir(masterPath);
+    mkdirSync(dir, { recursive: true });
+    atomicWriteFileSync(path.join(dir, MANIFEST_FILENAME), JSON.stringify(manifest, null, 2) + "\n");
+}
+function gcOldSnapshots(masterPath, keepFiles) {
+    const dir = masterSnapshotsDir(masterPath);
+    if (!existsSync(dir))
+        return;
+    for (const name of readdirSync(dir)) {
+        if (!name.endsWith(".db"))
+            continue;
+        if (keepFiles.has(name))
+            continue;
+        try {
+            unlinkSync(path.join(dir, name));
+        }
+        catch {
+            // ignore
+        }
+    }
+}
+/** Publish a new immutable snapshot on the master (re-validates lease epoch first). */
+export async function publishMasterSnapshot(masterPath) {
+    const { config: mc } = ensureMachineConfig();
+    const lockPath = path.join(getGnosysHome(), "master-ingest.lock");
+    const release = acquireWriteLockSync(lockPath, "snapshot-publish");
+    try {
+        touchMasterMarkerHeartbeat(masterPath);
+        assertMasterLeaseHeld(masterPath, mc.machineId);
+        const marker = readMasterMarker(masterPath);
+        if (!marker)
+            return null;
+        validateLeaseEpochBeforeWrite(masterPath, marker.epoch, mc.machineId);
+        const masterDb = new GnosysDB(masterPath);
+        if (!masterDb.isAvailable()) {
+            masterDb.close();
+            return null;
+        }
+        const prevDb = masterDb.getSnapshotManifest();
+        const prevFile = readManifestFile(masterPath);
+        const prevSeq = Math.max(prevDb?.seq ?? 0, prevFile?.seq ?? 0);
+        const nextSeq = prevSeq + 1;
+        const snapDir = masterSnapshotsDir(masterPath);
+        mkdirSync(snapDir, { recursive: true });
+        const snapName = `snap-${marker.epoch}-${nextSeq}.db`;
+        const finalPath = path.join(snapDir, snapName);
+        const tmpPath = path.join(snapDir, `.${snapName}.tmp`);
+        await masterDb.backup(path.dirname(tmpPath));
+        const created = readdirSync(snapDir).find((n) => n.startsWith("gnosys-backup-") && n.endsWith(".db"));
+        if (created) {
+            renameSync(path.join(snapDir, created), tmpPath);
+        }
+        if (existsSync(tmpPath)) {
+            renameSync(tmpPath, finalPath);
+        }
+        const checksum = sha256File(finalPath);
+        const sizeBytes = statSync(finalPath).size;
+        const publishedAt = new Date().toISOString();
+        const fileManifest = {
+            epoch: marker.epoch,
+            seq: nextSeq,
+            snapshotFile: snapName,
+            publishedAt,
+            checksum,
+            sizeBytes,
+        };
+        writeManifestFile(masterPath, fileManifest);
+        masterDb.publishSnapshotManifest({
+            epoch: marker.epoch,
+            seq: nextSeq,
+            snapshotPath: finalPath,
+            publishedAt,
+            checksum,
+            sizeBytes,
+            heartbeatAt: publishedAt,
+        });
+        const keep = new Set(readdirSync(snapDir)
+            .filter((n) => n.endsWith(".db"))
+            .sort()
+            .slice(-SNAPSHOT_RETENTION));
+        gcOldSnapshots(masterPath, keep);
+        masterDb.close();
+        return fileManifest;
+    }
+    finally {
+        release();
+    }
+}
+export function compareSnapshotVersion(accepted, incoming) {
+    if (!accepted)
+        return true;
+    if (incoming.epoch > accepted.epoch)
+        return true;
+    if (incoming.epoch === accepted.epoch && incoming.seq > accepted.seq)
+        return true;
+    return false;
+}
+/** Copy master snapshot to local client store atomically. */
+export function acceptClientSnapshot(masterPath, manifest) {
+    const src = path.join(masterSnapshotsDir(masterPath), manifest.snapshotFile);
+    if (!existsSync(src)) {
+        return { ok: false, reason: "snapshot file missing on master" };
+    }
+    const localChecksum = sha256File(src);
+    if (localChecksum !== manifest.checksum) {
+        return { ok: false, reason: "snapshot checksum mismatch" };
+    }
+    const store = clientSnapshotStore(masterPath);
+    mkdirSync(store, { recursive: true });
+    const tmp = path.join(store, ".gnosys.db.tmp");
+    const current = path.join(store, "gnosys.db");
+    copyFileSync(src, tmp);
+    const roundTrip = sha256File(tmp);
+    if (roundTrip !== manifest.checksum) {
+        try {
+            unlinkSync(tmp);
+        }
+        catch {
+            // ignore
+        }
+        return { ok: false, reason: "local copy verification failed" };
+    }
+    renameSync(tmp, current);
+    const metaPath = path.join(store, "accepted-manifest.json");
+    atomicWriteFileSync(metaPath, JSON.stringify(manifest, null, 2) + "\n");
+    return { ok: true, localPath: current };
+}
+export function getClientAcceptedManifest(masterPath) {
+    const metaPath = path.join(clientSnapshotStore(masterPath), "accepted-manifest.json");
+    if (!existsSync(metaPath))
+        return null;
+    try {
+        return JSON.parse(readFileSync(metaPath, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+export function formatSnapshotAge(publishedAt) {
+    const ms = Date.now() - new Date(publishedAt).getTime();
+    const minutes = Math.max(0, Math.floor(ms / 60_000));
+    if (minutes < 1)
+        return "just now";
+    if (minutes === 1)
+        return "1 minute ago";
+    return `${minutes} minutes ago`;
+}

package/dist/lib/syncStaging.d.ts

@@ -0,0 +1,81 @@
+/**
+ * v13 hardened JSON staging for multi-machine sync.
+ *
+ * Clients write small JSON files under master-folder/.gnosys-staging/<machineId>/.
+ * Master ingest walks files ordered by ledger firstSeenAt (see GnosysDB.sync_staging_ledger).
+ */
+import type { GnosysDB } from "./db.js";
+/** Current staged-memory JSON schema version. */
+export declare const STAGING_SCHEMA_VERSION = 1;
+export declare const KNOWN_STAGING_SCHEMA_VERSIONS: Set<number>;
+export declare const UNKNOWN_SCHEMA_MESSAGE = "Update your master machine to a newer version of Gnosys.";
+export interface StagedMemoryPayload {
+    schemaVersion: number;
+    id: string;
+    title: string;
+    category: string;
+    content: string;
+    tags: string[];
+    project_id: string | null;
+    scope: string;
+    machineId: string;
+    writtenAt: string;
+    checksum: string;
+}
+export type StagedFileParseResult = {
+    ok: true;
+    payload: StagedMemoryPayload;
+    filePath: string;
+} | {
+    ok: false;
+    filePath: string;
+    reason: string;
+    quarantine: boolean;
+};
+export declare function stagingRoot(masterPath: string): string;
+export declare function machineStagingDir(masterPath: string, machineId: string): string;
+/** Alias used by remoteWizard and tests. */
+export declare const stagingDirForMachine: typeof machineStagingDir;
+export declare function clientPresencePath(masterPath: string, machineId: string): string;
+export declare function failedQuarantineDir(masterPath: string, machineId: string): string;
+export declare function buildStagingFileName(memoryUlid: string, unixMs?: number): string;
+/** SHA-256 of canonical JSON body (all fields except checksum). */
+export declare function stagingPayloadChecksum(body: Omit<StagedMemoryPayload, "checksum">): string;
+export declare function buildStagedMemoryPayload(input: {
+    id: string;
+    title: string;
+    category: string;
+    content: string;
+    machineId: string;
+    tags?: string[];
+    project_id?: string | null;
+    scope?: string;
+    writtenAt?: string;
+    schemaVersion?: number;
+}): StagedMemoryPayload;
+/**
+ * Atomically write a staged memory JSON file (`.tmp` in the same directory, then rename).
+ * Returns the final file path relative to the machine staging dir.
+ */
+export declare function writeStagedMemoryFile(masterPath: string, machineId: string, payload: StagedMemoryPayload): string;
+export declare function parseStagedFile(filePath: string): StagedFileParseResult;
+/** Move a bad staging file into machineId/failed/ (does not delete). */
+export declare function quarantineStagingFile(filePath: string, masterPath: string, machineId: string): string;
+/** Move stale `.tmp` files in a machine staging dir into `failed/`. */
+export declare function quarantineStaleTmpFiles(masterPath: string, machineId: string): number;
+export declare function countFailedStagingFiles(masterPath: string, machineId: string): number;
+export interface StagingQueueEntry {
+    filePath: string;
+    fileName: string;
+    memoryUlid: string;
+    sortKey: string;
+}
+/**
+ * List pending `.json` staging files for a machine (excludes failed/, tmp, presence).
+ * Sort by ledger firstSeenAt when db is provided, else by filename timestamp prefix.
+ */
+export declare function listPendingStagingQueue(masterPath: string, machineId: string, masterDb?: GnosysDB): StagingQueueEntry[];
+/** Master read-back: memory row exists after ingest before deleting staging file. */
+export declare function verifyMemoryExistsInDb(db: GnosysDB, memoryUlid: string): boolean;
+/** Record first observation in the master ledger (idempotent upsert without clobbering status). */
+export declare function observeStagingFile(masterDb: GnosysDB, machineId: string, fileName: string, memoryUlid: string | null): void;