@bitpub/cli 2.0.0

package/src/commands/save.js

@@ -0,0 +1,255 @@
+'use strict';
+
+/**
+ * `bitpub save <name> [content]` — write a slice.
+ *
+ * Accepts three input forms (same as load/find/delete):
+ *   - short name "notes"               → resolves through the active project
+ *   - alias ref  "@team/Eng/notes"     → expands against ~/.bitpub/aliases.json
+ *   - full URL   "bitpub://group:..."  → used verbatim
+ *
+ * Lazy + silent setup:
+ *   If the user has no identity yet AND the cwd isn't $HOME, save runs
+ *   `ensureIdentity` + `ensureWorkspace` automatically before the actual
+ *   save. Identity provisioning still prints a brief block (it's a
+ *   one-time event), but anchoring the folder is silent — the first
+ *   save in any folder simply produces a `✓ Saved → …` line plus the
+ *   suggestion block, exactly like every subsequent save.
+ *
+ * Default = private. The CLI never silently writes to a group address;
+ * the only way to land in `bitpub://group:...` is to ask for it
+ * explicitly with a fully-qualified URL or a `@team` alias.
+ *
+ * After a successful default-path save, suggest alternative places the
+ * slice could live (long-term memory, drafts, team scope) so the agent
+ * can offer the human a one-click "move it elsewhere" follow-up
+ * without inventing those addresses itself.
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { readConfig, isConfigured, authorIdFor, DEFAULT_CLOUD_URL } = require('../config');
+const { createApiClient } = require('../api');
+const { upsertSlice } = require('../db/cache');
+const { encrypt, decryptSlices, isPrivateHcu } = require('../crypto');
+const { resolveHcu } = require('../workspace');
+const { readAliases } = require('../aliases');
+
+function readStdin() {
+  return new Promise((resolve, reject) => {
+    if (process.stdin.isTTY) return resolve('');
+    let data = '';
+    process.stdin.setEncoding('utf-8');
+    process.stdin.on('data', chunk => { data += chunk; });
+    process.stdin.on('end', () => resolve(data));
+    process.stdin.on('error', reject);
+  });
+}
+
+/**
+ * Ensure we have an identity before any save touches the wire. Lazy:
+ * only kicks in when there's no config yet, and only when the user is
+ * inside a project folder (never in $HOME — that'd anchor the home
+ * directory, which is almost never what they want).
+ *
+ * Anchoring is silent: the first save in a new folder reads exactly
+ * the same as the hundredth (one save line, one suggestion block).
+ * Identity provisioning still prints (it's a one-time event that
+ * matters: the user just got a new cloud identity).
+ *
+ * Returns the resolved config or exits with a helpful error.
+ */
+async function ensureConfigOrSetup() {
+  let config = readConfig();
+  if (isConfigured(config)) {
+    // Even if identity is configured, make sure the cwd has an anchor
+    // so saves land in a stable project address. Silent.
+    if (config.owner) {
+      const cwd = process.cwd();
+      const home = path.resolve(os.homedir());
+      if (path.resolve(cwd) !== home) {
+        const { ensureWorkspace } = require('./setup');
+        ensureWorkspace({ owner: config.owner, quiet: true });
+      }
+    }
+    return config;
+  }
+
+  const cwd = process.cwd();
+  const home = path.resolve(os.homedir());
+  if (path.resolve(cwd) === home) {
+    console.error('No BitPub identity configured, and you\'re in $HOME.');
+    console.error('Run `bitpub setup` from a project folder to get started.');
+    process.exit(1);
+  }
+
+  // Defer the require until we actually need to provision — avoids
+  // pulling in axios on every save when the user is already configured.
+  const { ensureIdentity, ensureWorkspace } = require('./setup');
+  try {
+    config = await ensureIdentity({ url: DEFAULT_CLOUD_URL, quiet: false });
+    if (config?.owner) ensureWorkspace({ owner: config.owner, quiet: true });
+  } catch (err) {
+    console.error(`\n✗ Could not set up BitPub: ${err.message}`);
+    console.error('  Try running `bitpub setup --url <URL>` manually with a reachable backend.');
+    process.exit(1);
+  }
+  return config;
+}
+
+/**
+ * Format the suggestions block printed after a default-path save.
+ *
+ * Goal: surface a short list of *other private addresses this slice
+ * could plausibly live at*, without prescribing anything. The agent
+ * reads this output and can offer the user "want me to move it?" —
+ * the human sees concrete addresses, not abstract guidance.
+ *
+ * We suggest three canonical organizational namespaces in the user's
+ * private scope (Memory, Inbox, Drafts) plus any aliases they've
+ * defined (@memory, @inbox, @team, etc.). All suggestions are
+ * addresses the user can write to with a follow-up save.
+ *
+ * No invented addresses: if the user has no aliases defined and no
+ * private owner, we return an empty list rather than guessing.
+ */
+function suggestionLines(name, savedAddress, config) {
+  const lines = [];
+  const seen = new Set([savedAddress]);
+  const aliases = readAliases();
+  const tail = name.replace(/^\/+/, '');
+
+  // Canonical private organizational folders. These are predictable
+  // paths under the user's own scope — the slice would live next to
+  // whatever else they've saved to /Memory/, /Inbox/, /Drafts/.
+  if (config.owner) {
+    const privateBase = `bitpub://private:${config.owner}`;
+    const places = [
+      ['/Memory/',  'long-term memory'],
+      ['/Inbox/',   'quick capture / triage'],
+      ['/Drafts/',  'short-lived working drafts'],
+    ];
+    for (const [prefix, desc] of places) {
+      const addr = privateBase + prefix + tail;
+      if (seen.has(addr)) continue;
+      seen.add(addr);
+      // Display the path-only form (drops the noisy bitpub://private:<owner>
+      // prefix). The agent can re-qualify it when calling save.
+      lines.push(`    ${prefix}${tail}`.padEnd(38) + `# ${desc}`);
+    }
+  }
+
+  // User-defined aliases — these often point at custom places that
+  // matter more than the canonical ones above.
+  for (const [aliasName, aliasAddress] of Object.entries(aliases || {})) {
+    const addr = aliasAddress.replace(/\/?$/, '/') + tail;
+    if (seen.has(addr)) continue;
+    seen.add(addr);
+    const desc = aliasAddress.includes('group:') ? 'shared with your team' : 'alias';
+    lines.push(`    @${aliasName}/${tail}`.padEnd(38) + `# ${desc}`);
+  }
+
+  return lines;
+}
+
+module.exports = function registerSave(program) {
+  program
+    .command('save <name> [content]')
+    .description('Save a slice. Defaults to this project (private + encrypted); pass a full URL or @alias to write elsewhere.')
+    .option('--file <path>', 'Read content from a file')
+    .option('--stdin', 'Read content from stdin (default if no content/file given)')
+    .option('--tags <string>', 'Comma-separated tags')
+    .option('--append', 'Append to existing slice instead of overwriting')
+    .option('--expect-version <number>', 'Reject if current version differs (optimistic concurrency)', parseInt)
+    .option('--force', 'Overwrite a tombstoned slice (un-delete + new content in one step)')
+    .option('--format <string>', 'Content format', 'text/markdown')
+    .action(async (name, content, opts) => {
+      const config = await ensureConfigOrSetup();
+
+      let resolved;
+      try {
+        resolved = resolveHcu(name, config);
+      } catch (err) {
+        console.error(err.message);
+        process.exit(1);
+      }
+      const { hcu: address, source, workspace } = resolved;
+
+      let finalContent = content;
+      if (opts.file) {
+        finalContent = fs.readFileSync(opts.file, 'utf-8');
+      } else if (opts.stdin || !finalContent) {
+        finalContent = await readStdin();
+      }
+      if (!finalContent) {
+        console.error('No content provided. Pass it as an argument, --file <path>, or pipe via stdin.');
+        process.exit(1);
+      }
+
+      const tagList = opts.tags
+        ? opts.tags.split(',').map(t => t.trim()).filter(Boolean)
+        : [];
+
+      const api = createApiClient(config);
+      const payloadContent = isPrivateHcu(address)
+        ? encrypt(finalContent, config.api_key)
+        : finalContent;
+
+      const body = {
+        hcu: address,
+        metadata: {
+          author_id: authorIdFor(config),
+          timestamp: new Date().toISOString(),
+          tags: tagList,
+        },
+        payload: { format: opts.format, content: payloadContent },
+      };
+
+      try {
+        const result = await api.push(body, {
+          append: !!opts.append,
+          expectVersion: opts.expectVersion,
+          force: !!opts.force,
+        });
+        decryptSlices([result.slice], config.api_key);
+        upsertSlice(result.slice);
+        console.log(`✓ Saved → ${address}  (v${result.slice.metadata.version})`);
+
+        // Suggestions only fire on default-path saves: when the user
+        // gave us a short name resolved against the active project (or
+        // a session-default fallback). Explicit URLs and aliases are
+        // already deliberate — no second-guessing.
+        if (source === 'workspace' || source === 'session-default') {
+          const suggestions = suggestionLines(name, address, config);
+          if (suggestions.length > 0) {
+            console.log('');
+            console.log('  Other places this could live:');
+            for (const line of suggestions) console.log(line);
+            console.log('');
+            console.log(`  Move with: bitpub save <one-of-those> --file -   (or save again to that address)`);
+          }
+        }
+      } catch (err) {
+        if (err.response?.status === 409) {
+          const data = err.response.data;
+          if (data?.deleted_at) {
+            console.error(`Save blocked: "${name}" was deleted (${data.deleted_at}).`);
+            console.error('');
+            console.error('  Restore the prior content:  bitpub delete "' + name + '" --undo');
+            console.error('  Overwrite with new content: bitpub save "' + name + '" "..." --force');
+          } else {
+            console.error(
+              `Version conflict: expected v${data.expected_version} but server has v${data.actual_version}.`
+            );
+            console.error(`  Run: bitpub sync ${address}  to refresh, then retry.`);
+          }
+        } else {
+          console.error(`Save failed: ${err.message}`);
+        }
+        process.exit(1);
+      }
+    });
+};
+
+module.exports.suggestionLines = suggestionLines;

package/src/commands/seed.js

@@ -0,0 +1,152 @@
+'use strict';
+
+const { requireConfig } = require('../config');
+const { createApiClient } = require('../api');
+const { upsertSlice } = require('../db/cache');
+const axios = require('axios');
+const cheerio = require('cheerio');
+const TurndownService = require('turndown');
+const { URL } = require('url');
+
+module.exports = function registerSeed(program) {
+  program
+    .command('seed')
+    .description('Fetch top pages from a URL and push them as context slices (bootstraps the knowledge base)')
+    .requiredOption('--url <string>', 'Base URL to seed from (e.g. https://example.com)')
+    .requiredOption('--address <string>', 'Target namespace address (e.g. bitpub://group:domain.com/Public)')
+    .option('--pages <number>', 'Max pages to fetch', '5')
+    .action(async ({ url: baseUrl, address, pages }) => {
+      const config = requireConfig();
+      const api = createApiClient(config);
+      const maxPages = Math.min(parseInt(pages, 10) || 5, 20);
+      const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+
+      console.log(`Seeding from ${baseUrl} into ${address} (up to ${maxPages} pages)…\n`);
+
+      const urls = await discoverUrls(baseUrl, maxPages);
+
+      let seeded = 0;
+      for (const pageUrl of urls) {
+        try {
+          const { title, markdown } = await fetchAsMarkdown(pageUrl, td);
+          const slug = urlToSlug(pageUrl, baseUrl);
+          const targetAddress = `${address.replace(/\/$/, '')}/${slug}`;
+
+          const body = {
+            hcu: targetAddress,
+            metadata: {
+              author_id: 'seed_bot',
+              timestamp: new Date().toISOString(),
+              tags: ['seeded', 'web'],
+            },
+            payload: {
+              format: 'text/markdown',
+              content: `# ${title}\n\n> Source: ${pageUrl}\n\n${markdown}`,
+            },
+          };
+
+          const result = await api.push(body, false);
+          upsertSlice(result.slice);
+          console.log(`  ✓ ${targetAddress}`);
+          seeded++;
+        } catch (err) {
+          console.error(`  ✗ ${pageUrl}: ${err.message}`);
+        }
+      }
+
+      console.log(`\nSeeded ${seeded} / ${urls.length} pages into ${address}`);
+    });
+};
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+async function discoverUrls(baseUrl, maxPages) {
+  const base = new URL(baseUrl);
+  const seen = new Set([normalizeUrl(baseUrl)]);
+  const queue = [normalizeUrl(baseUrl)];
+  const result = [];
+
+  while (queue.length > 0 && result.length < maxPages) {
+    const current = queue.shift();
+    result.push(current);
+
+    try {
+      const res = await axios.get(current, { timeout: 10_000 });
+      const $ = cheerio.load(res.data);
+
+      $('a[href]').each((_, el) => {
+        try {
+          const href = $(el).attr('href');
+          const resolved = new URL(href, current);
+          const norm = normalizeUrl(resolved.href);
+
+          if (
+            resolved.hostname === base.hostname &&
+            !seen.has(norm) &&
+            !norm.match(/\.(pdf|jpg|jpeg|png|gif|svg|zip|css|js|xml|json)(\?.*)?$/i)
+          ) {
+            seen.add(norm);
+            queue.push(norm);
+          }
+        } catch {
+          // Invalid URL — skip
+        }
+      });
+    } catch {
+      // Failed to fetch page for link extraction — still include it in results
+    }
+  }
+
+  return result;
+}
+
+async function fetchAsMarkdown(url, td) {
+  const res = await axios.get(url, {
+    timeout: 15_000,
+    headers: { 'User-Agent': 'BitPub-Seed/1.0 (context indexer)' },
+  });
+
+  const $ = cheerio.load(res.data);
+
+  // Title
+  const title =
+    $('meta[property="og:title"]').attr('content') ||
+    $('title').text() ||
+    url;
+
+  // Remove nav/footer/script noise before converting
+  $('script, style, nav, footer, header, .nav, .footer, .header, [role="navigation"]').remove();
+
+  // Prefer main content area if available
+  const bodyEl =
+    $('main').length ? $('main') :
+    $('article').length ? $('article') :
+    $('[role="main"]').length ? $('[role="main"]') :
+    $('body');
+
+  const markdown = td.turndown(bodyEl.html() || '');
+
+  return { title: title.trim(), markdown };
+}
+
+function urlToSlug(pageUrl, baseUrl) {
+  const base = new URL(baseUrl);
+  const page = new URL(pageUrl);
+
+  let slug = page.pathname
+    .replace(/^\//, '')      // strip leading slash
+    .replace(/\/$/, '')      // strip trailing slash
+    .replace(/\//g, '_')     // slashes → underscores
+    .replace(/[^a-zA-Z0-9_-]/g, '_') // sanitize
+    .replace(/_+/g, '_')     // collapse runs
+    .replace(/^_|_$/g, '');  // trim
+
+  return slug || `${base.hostname.replace(/\./g, '_')}_home`;
+}
+
+function normalizeUrl(url) {
+  const u = new URL(url);
+  u.hash = '';       // ignore fragments
+  u.search = '';     // ignore query strings for deduplication
+  return u.href.replace(/\/$/, '');
+}