@bitpub/cli 2.1.2 → 2.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bitpub/cli",
-  "version": "2.1.2",
+  "version": "2.1.3",
   "description": "BitPub CLI — local-first shared memory for AI agents. Six daily verbs (save/load/list/find/sync/delete), zero-config private namespace, encrypted client-side.",
   "bin": {
     "bitpub": "./bin/bitpub.js"

package/src/commands/browser.js

@@ -21,6 +21,7 @@ const { exec, spawn } = require('child_process');
 const { readConfig, BITPUB_DIR } = require('../config');
 const { getSyncedNamespaces, initCache } = require('../db/cache');
 const { isPrivateHcu, decrypt, isEncrypted } = require('../crypto');
+const { runOneShot: runSyncOneShot } = require('./sync');
 
 const Database = require('better-sqlite3');
 const DB_PATH = path.join(os.homedir(), '.bitpub', 'cache.db');
@@ -366,6 +367,128 @@ function handleBridgeRun(req, res, url) {
   });
 }
 
+/**
+ * Resolve the set of namespaces to sync for "sync everything visible
+ * to the current user." This is the same shape an attentive user would
+ * type by hand: their private root + every joined group.
+ *
+ * Returns an array of `{ pattern, label, config }` triples. The per-
+ * scope `config` is the user's primary config with `api_key` swapped
+ * out when a group entry pins its own key (multi-backend memberships).
+ */
+function resolveSyncTargets(baseConfig) {
+  const targets = [];
+  if (baseConfig && baseConfig.owner) {
+    targets.push({
+      pattern: `bitpub://private:${baseConfig.owner}/**`,
+      label: `private:${baseConfig.owner}`,
+      config: baseConfig,
+    });
+  }
+  const groups = Array.isArray(baseConfig && baseConfig.groups) ? baseConfig.groups : [];
+  for (const g of groups) {
+    if (!g || !g.slug) continue;
+    targets.push({
+      pattern: `bitpub://group:${g.slug}/**`,
+      label: `group:${g.slug}`,
+      // Per-group key/URL is only relevant on multi-backend setups; for
+      // the common single-backend case these fields equal the primary
+      // config and the fallback is a no-op.
+      config: {
+        ...baseConfig,
+        api_key: g.key || baseConfig.api_key,
+        api_url: g.api_url || baseConfig.api_url,
+      },
+    });
+  }
+  return targets;
+}
+
+/**
+ * Stream `bitpub sync` across every scope visible to the user as SSE.
+ *
+ * Wire format:
+ *   event: started   data: { targets: [{label, pattern}, ...] }
+ *   event: scope     data: { label, pattern, status: 'syncing' }
+ *   event: scope     data: { label, pattern, status: 'done',  count, hitLimit }
+ *   event: scope     data: { label, pattern, status: 'error', message }
+ *   event: done      data: { total, scopes }     ← terminal; connection closes
+ *
+ * This is browser-chrome plumbing, not an app function — it calls
+ * sync.js#runOneShot directly inside this process rather than spawning
+ * `bitpub sync` as a subprocess via the script.run primitive. Reasons:
+ *   • No shell-out cost, no quoting hazard, no env propagation issues.
+ *   • Per-scope progress is structured (event stream), not parsed stdout.
+ *   • Manifests are for user-published apps; the Sync button is part of
+ *     the browser shell and shouldn't go through that machinery.
+ */
+function handleBridgeSync(req, res) {
+  res.statusCode = 200;
+  res.setHeader('Content-Type', 'text/event-stream');
+  res.setHeader('Cache-Control', 'no-cache, no-transform');
+  res.setHeader('Connection', 'keep-alive');
+  res.setHeader('X-Accel-Buffering', 'no');
+
+  function send(event, payload) {
+    if (res.writableEnded) return;
+    res.write(`event: ${event}\n`);
+    res.write(`data: ${JSON.stringify(payload)}\n\n`);
+  }
+
+  const cfg = readConfig();
+  if (!cfg || !cfg.owner) {
+    send('error', { message: 'No bitpub identity configured. Run `bitpub setup` first.' });
+    send('done', { total: 0, scopes: 0 });
+    res.end();
+    return;
+  }
+
+  const targets = resolveSyncTargets(cfg);
+  send('started', { targets: targets.map(t => ({ label: t.label, pattern: t.pattern })) });
+
+  let cancelled = false;
+  req.on('close', () => { cancelled = true; });
+
+  // Sync scopes sequentially so the progress events tell a single story
+  // and we don't slam the same backend with N concurrent pulls.
+  (async () => {
+    let total = 0;
+    let okScopes = 0;
+    for (const t of targets) {
+      if (cancelled) return;
+      send('scope', { label: t.label, pattern: t.pattern, status: 'syncing' });
+      try {
+        const r = await runSyncOneShot(
+          { pattern: t.pattern, label: t.label, limit: 500, includeDeleted: false, quiet: true },
+          t.config
+        );
+        total += r.count;
+        okScopes += 1;
+        send('scope', {
+          label: t.label,
+          pattern: t.pattern,
+          status: 'done',
+          count: r.count,
+          hitLimit: !!r.hitLimit,
+        });
+      } catch (err) {
+        send('scope', {
+          label: t.label,
+          pattern: t.pattern,
+          status: 'error',
+          message: err && err.message ? err.message : String(err),
+        });
+      }
+    }
+    send('done', { total, scopes: okScopes });
+    res.end();
+  })().catch((err) => {
+    send('error', { message: err && err.message ? err.message : String(err) });
+    send('done', { total: 0, scopes: 0 });
+    res.end();
+  });
+}
+
 function openInBrowser(url) {
   const cmd = process.platform === 'darwin' ? 'open' :
     process.platform === 'win32' ? 'start' : 'xdg-open';
@@ -469,6 +592,15 @@ function startBrowserServer(opts = {}) {
         return;
       }
 
+      // Bridge: sync every namespace visible to this user (private +
+      // each joined group) and stream per-scope progress as SSE. This
+      // backs the sidebar "Sync" button — it's browser chrome, not an
+      // app function, and so doesn't go through the manifest system.
+      if (url.pathname === '/bridge/sync') {
+        handleBridgeSync(req, res);
+        return;
+      }
+
       // Bridge: deterministic namespace read. Apps call this to fetch
       // any slices that match an HCU pattern (exact or prefix-with-/*).
       // No LLM, no MCP, no auth in the loop — this is the read path

package/src/commands/sync.js

@@ -60,7 +60,7 @@ function resolveSyncPattern(input, config) {
   return { pattern: active.namespace + resolved, label: input };
 }
 
-async function runOneShot({ pattern, label, limit, includeDeleted }, config) {
+async function runOneShot({ pattern, label, limit, includeDeleted, quiet }, config) {
   const api = createApiClient(config);
   const limitNum = parseInt(limit, 10) || 500;
 
@@ -70,19 +70,24 @@ async function runOneShot({ pattern, label, limit, includeDeleted }, config) {
   for (const slice of slices) upsertSlice(slice);
   recordNamespaceSync(pattern, slices.length);
 
-  console.log(`Synced ${slices.length} slice(s) from ${label}`);
-
-  // Surface page-cap warning so the user knows to narrow their pattern.
-  if (slices.length >= Math.min(limitNum, 500)) {
-    console.log(
-      `\n  Hit the page limit (${slices.length} rows). Server caps single sync at 500.` +
-      ` If you expect more, sync sub-namespaces individually:`
-    );
-    const base = pattern.replace(/\/\*+$/, '');
-    console.log(`    bitpub sync '${base}/Projects/**'`);
-    console.log(`    bitpub sync '${base}/Memory/**'`);
-    console.log(`    bitpub sync '${base}/Sessions/**'`);
+  const hitLimit = slices.length >= Math.min(limitNum, 500);
+
+  if (!quiet) {
+    console.log(`Synced ${slices.length} slice(s) from ${label}`);
+    // Surface page-cap warning so the user knows to narrow their pattern.
+    if (hitLimit) {
+      console.log(
+        `\n  Hit the page limit (${slices.length} rows). Server caps single sync at 500.` +
+        ` If you expect more, sync sub-namespaces individually:`
+      );
+      const base = pattern.replace(/\/\*+$/, '');
+      console.log(`    bitpub sync '${base}/Projects/**'`);
+      console.log(`    bitpub sync '${base}/Memory/**'`);
+      console.log(`    bitpub sync '${base}/Sessions/**'`);
+    }
   }
+
+  return { count: slices.length, hitLimit, pattern, label };
 }
 
 function runWatch({ pattern, label }, config) {

package/static/console.html

@@ -217,9 +217,27 @@ svg { flex-shrink: 0; }
 #main:hover::-webkit-scrollbar-thumb { background: var(--border); background-clip: content-box; }
 
 /* ── Sidebar / Tree ─────────────────────────────────── */
-.sidebar-header { padding: 12px 14px 8px; font-size: 10.5px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; color: var(--text-subtle); display: flex; align-items: center; justify-content: space-between; }
+.sidebar-header { padding: 12px 14px 8px; font-size: 10.5px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; color: var(--text-subtle); display: flex; align-items: center; justify-content: space-between; gap: 8px; }
 .sidebar-header .clear-btn { font-size: 10.5px; color: var(--text-subtle); cursor: pointer; font-weight: 500; letter-spacing: 0; text-transform: none; }
 .sidebar-header .clear-btn:hover { color: var(--accent); }
+
+/* Sync button — icon-only, ghost style, lives in the sidebar header.
+   Spins while a /bridge/sync request is in flight; pulses green for a
+   moment after a successful sync so the user gets feedback even if no
+   new slices landed. */
+.sync-btn { display: inline-flex; align-items: center; gap: 4px; padding: 3px 6px; border: 0; background: transparent; color: var(--text-subtle); cursor: pointer; border-radius: var(--radius-sm); font: inherit; font-size: 10.5px; font-weight: 500; text-transform: none; letter-spacing: 0; }
+.sync-btn:hover:not(:disabled) { background: var(--bg-hover); color: var(--text); }
+.sync-btn:disabled { opacity: .6; cursor: default; }
+.sync-btn svg { width: 12px; height: 12px; transition: transform .2s ease-out; }
+.sync-btn.is-syncing svg { animation: bp-spin 1s linear infinite; }
+.sync-btn.is-done { color: #10b981; }
+@keyframes bp-spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } }
+.sync-status { padding: 4px 14px 8px; font-size: 11px; color: var(--text-subtle); line-height: 1.4; max-height: 120px; overflow-y: auto; }
+.sync-status.hidden { display: none; }
+.sync-status .row { display: flex; justify-content: space-between; gap: 8px; padding: 2px 0; }
+.sync-status .row .label { color: var(--text); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
+.sync-status .row .count { color: var(--text-subtle); flex-shrink: 0; font-variant-numeric: tabular-nums; }
+.sync-status .row.error .count { color: var(--scope-group); }
 .tree-wrap { flex: 1; overflow-y: auto; padding: 0 6px 12px; }
 .tree-wrap::-webkit-scrollbar { width: 8px; }
 .tree-wrap::-webkit-scrollbar-thumb { background: transparent; border-radius: 4px; }
@@ -972,8 +990,15 @@ svg { flex-shrink: 0; }
     <aside id="sidebar">
       <div class="sidebar-header">
         <span>Namespaces</span>
-        <span id="clear-filter-btn" class="clear-btn hidden" onclick="clearFilter()">Clear</span>
+        <span style="display:inline-flex; align-items:center; gap:6px;">
+          <span id="clear-filter-btn" class="clear-btn hidden" onclick="clearFilter()">Clear</span>
+          <button id="sync-btn" class="sync-btn" type="button" onclick="runFullSync()" title="Sync all namespaces from the cloud into your local cache">
+            <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"><path d="M8 3a5 5 0 0 1 4.546 2.914.5.5 0 0 0 .908-.418A6 6 0 1 0 2 8a.5.5 0 1 0 1 0 5 5 0 0 1 5-5Z"/><path d="M8 13a5 5 0 0 1-4.546-2.914.5.5 0 0 0-.908.418A6 6 0 1 0 14 8a.5.5 0 1 0-1 0 5 5 0 0 1-5 5Z"/><path d="M11.5 1.5a.5.5 0 0 1 .5.5v3a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1 0-1H11V2a.5.5 0 0 1 .5-.5ZM4.5 10.5a.5.5 0 0 1 .5.5v2.5h2.5a.5.5 0 0 1 0 1h-3a.5.5 0 0 1-.5-.5v-3a.5.5 0 0 1 .5-.5Z"/></svg>
+            <span class="sync-btn-label">Sync</span>
+          </button>
+        </span>
       </div>
+      <div id="sync-status" class="sync-status hidden"></div>
       <div id="tree" class="tree-wrap"></div>
     </aside>
     <main id="main"></main>
@@ -1456,6 +1481,135 @@ function clearFilter() {
   renderAll();
 }
 
+/* ── Sidebar Sync button ────────────────────────────────
+ *
+ * Streams /bridge/sync (SSE) and renders per-scope progress under the
+ * header. On `done`, force a fresh /api/data poll so the tree picks up
+ * any new arrivals immediately (without waiting for the 30s tick).
+ *
+ * The button is browser chrome — not an app function — so it talks
+ * directly to /bridge/sync rather than going through window.bitpub.run
+ * and the manifest dispatcher.
+ *
+ * Reentrancy: a second click while a sync is in flight is a no-op
+ * (we disable the button), so we don't need to track an EventSource
+ * handle for cancellation.
+ */
+async function runFullSync() {
+  const btn = $('sync-btn');
+  const status = $('sync-status');
+  if (!btn || btn.disabled) return;
+
+  btn.disabled = true;
+  btn.classList.add('is-syncing');
+  btn.classList.remove('is-done');
+  status.classList.remove('hidden');
+  status.innerHTML = '';
+
+  // Track rows so 'syncing' → 'done' overwrites in place rather than
+  // appending a second line per scope.
+  const rows = new Map();
+
+  function setRow(label, pattern, statusText, klass) {
+    let row = rows.get(label);
+    if (!row) {
+      row = document.createElement('div');
+      row.className = 'row';
+      const a = document.createElement('span'); a.className = 'label';
+      const b = document.createElement('span'); b.className = 'count';
+      row.append(a, b);
+      status.appendChild(row);
+      rows.set(label, row);
+    }
+    row.className = 'row' + (klass ? ' ' + klass : '');
+    row.title = pattern;
+    row.children[0].textContent = label;
+    row.children[1].textContent = statusText;
+  }
+
+  const base = location.pathname.endsWith('/') ? location.pathname : location.pathname + '/';
+  const es = new EventSource(base + 'bridge/sync');
+
+  function cleanup() {
+    es.close();
+    btn.disabled = false;
+    btn.classList.remove('is-syncing');
+  }
+
+  es.addEventListener('started', (e) => {
+    let payload = {};
+    try { payload = JSON.parse(e.data); } catch {}
+    for (const t of (payload.targets || [])) setRow(t.label, t.pattern, 'queued', '');
+  });
+
+  es.addEventListener('scope', (e) => {
+    let payload = {};
+    try { payload = JSON.parse(e.data); } catch {}
+    if (payload.status === 'syncing') {
+      setRow(payload.label, payload.pattern, 'syncing…', '');
+    } else if (payload.status === 'done') {
+      const cnt = (payload.count ?? 0) + ' slices';
+      const suffix = payload.hitLimit ? ' (capped)' : '';
+      setRow(payload.label, payload.pattern, cnt + suffix, '');
+    } else if (payload.status === 'error') {
+      setRow(payload.label, payload.pattern, 'error', 'error');
+      console.error('[sync]', payload.label, payload.message);
+    }
+  });
+
+  es.addEventListener('error', (e) => {
+    // Server-sent `event: error` (CLI-reported); EventSource also fires
+    // 'error' on connection drops with no payload. Treat both the same.
+    let payload = {};
+    if (e && e.data) { try { payload = JSON.parse(e.data); } catch {} }
+    if (payload.message) {
+      const row = document.createElement('div');
+      row.className = 'row error';
+      row.innerHTML = '<span class="label">sync failed</span><span class="count"></span>';
+      row.children[1].textContent = payload.message.slice(0, 60);
+      status.appendChild(row);
+    }
+    // Don't cleanup here — wait for `done`, or for the connection to
+    // close on its own. Real errors will be followed by `done`.
+  });
+
+  es.addEventListener('done', async (e) => {
+    let payload = {};
+    try { payload = JSON.parse(e.data); } catch {}
+    cleanup();
+    btn.classList.add('is-done');
+    setTimeout(() => btn.classList.remove('is-done'), 2000);
+
+    // Force-refresh the data so any newly-synced slices show up in
+    // the tree immediately instead of on the next 30s poll tick.
+    const data = await fetchData({ silent: true });
+    if (data) {
+      const arrivals = new Set();
+      const oldKey = new Set(S.slices.map(s => s.hcu + '|' + (s.last_synced || '')));
+      const nextSlices = data.slices.map(parseSlice);
+      for (const s of nextSlices) {
+        const k = s.hcu + '|' + (s.last_synced || '');
+        if (!oldKey.has(k)) arrivals.add(s.hcu);
+      }
+      S.slices = nextSlices;
+      S.tree = buildTree(S.slices);
+      S.stats = computeStats(S.slices);
+      S.lastUpdated = maxTs(S.slices);
+      S.newArrivals = arrivals;
+      renderAll();
+      setTimeout(() => { S.newArrivals = new Set(); }, 2500);
+    }
+
+    // Auto-hide the status panel after a beat if everything succeeded.
+    const anyErrors = Array.from(rows.values()).some(r => r.classList.contains('error'));
+    if (!anyErrors) {
+      setTimeout(() => {
+        if (!btn.classList.contains('is-syncing')) status.classList.add('hidden');
+      }, 4000);
+    }
+  });
+}
+
 /* ══════════════════════════════════════════════════════
    Top-level render
    ══════════════════════════════════════════════════════ */