keychat-save 1.5.0 → 1.6.0

package/README.md

@@ -98,8 +98,24 @@ keychat-save keycloud get <txid> ~/Downloads
 
 # What you have uploaded from this machine
 keychat-save keycloud list
+
+# Publish identifiers for everything already on chain — see below
+keychat-save keycloud catalog
 ```
 
+### Listing without downloading
+
+Metadata travels in its own encrypted push, separate from the file. A listing
+therefore costs kilobytes: names, sizes and image thumbnails are read without
+moving a single file body, and the bodies are fetched only when you open one.
+
+Anything uploaded before that split has its metadata sealed inside its own body,
+so listing it the old way meant downloading it. `keycloud catalog` fixes that in
+one small transaction: it publishes the identifiers you already hold locally —
+names, sizes, hashes, and your RepoChain snapshots too — encrypted to your key.
+Any device with that key then lists your whole history for a few KB. Cataloguing
+51 items cost 2 Compute.
+
 Bytes are encrypted directly — never base64, which would add 33% to the size and therefore 33% to
 the cost. The file's sha256 travels with it and is checked on the way back, so a truncated fetch
 fails loudly instead of writing a corrupt file.

package/index.mjs

@@ -506,6 +506,9 @@ KEYCLOUD — files on chain, sealed to your own key:
   keycloud put <file>           Upload one file in a single transaction
   keycloud get <txid> <dest>    Fetch it back (sha256 verified before writing)
   keycloud list                 Files uploaded from this machine
+  keycloud catalog              Publish identifiers for everything already on
+                                chain, so any device can list it without
+                                downloading a single file
 
   Each snapshot is ONE transaction. A delta continues the chain automatically and
   carries the ordered txids of every ancestor, so a single txid restores everything.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keychat-save",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Save sessions, files and whole git repositories to the BSV blockchain via KeyChat",
   "type": "module",
   "bin": {

package/repo.mjs

@@ -907,11 +907,68 @@ async function restore (txid, dest) {
 // transaction no miner will accept.
 const CLOUD_PREFIX = '4b434c44' // "KCLD"
 const CLOUD_INDEX = 'keycloud.json'
+// v2 SPLITS METADATA OUT OF THE FILE'S CIPHERTEXT.
+//
+//   v1  [KCLD, 01, pub, encrypt(meta + bytes),            ts]
+//   v2  [KCLD, 02, pub, encrypt(meta), encrypt(bytes),    ts]
+//
+// v1 sealed the filename INSIDE the file's own ciphertext, so listing what you
+// own meant fetching and decrypting every byte you had ever uploaded — ten
+// 10 MB files is a 100 MB download to draw ten labels. A file browser is not
+// buildable on that.
+//
+// In v2 the meta push is a few hundred bytes plus an optional thumbnail, so the
+// bridge can index and serve it alone and the client renders a whole grid
+// without touching a single file body. The bytes are fetched only when someone
+// opens something.
+//
+// BOTH PUSHES STAY ENCRYPTED to the owner's key. Filenames never appear in
+// plaintext on chain — the point is that only the holder of the key learns
+// anything, and a filename is often the most revealing part of a file.
+//
+// v1 files stay readable: cloudGet accepts either shape.
+const CLOUD_VERSION_V1 = '01'
+const CLOUD_VERSION = '02'
 // Policy ceiling for a single transaction. The payload is the whole tx, so this
 // is deliberately conservative against the published 100 MB: envelope, ECIES
 // overhead and the funding input all sit inside the same limit.
 const CLOUD_MAX_SINGLE_TX = 95 * 1024 * 1024
 
+const MIME_BY_EXT = {
+  '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif',
+  '.webp': 'image/webp', '.heic': 'image/heic', '.tiff': 'image/tiff', '.bmp': 'image/bmp',
+  '.pdf': 'application/pdf', '.txt': 'text/plain', '.md': 'text/markdown',
+  '.json': 'application/json', '.csv': 'text/csv', '.zip': 'application/zip',
+  '.mp4': 'video/mp4', '.mov': 'video/quicktime', '.mp3': 'audio/mpeg', '.wav': 'audio/wav'
+}
+const mimeFor = name => MIME_BY_EXT[path.extname(name).toLowerCase()] || 'application/octet-stream'
+
+/**
+ * A 256px JPEG preview, base64, for the file grid to render without ever
+ * fetching the original — the same reason Finder feels instant.
+ *
+ * Uses `sips`, which ships with macOS, rather than adding an image dependency
+ * to a package whose only dep is @bsv/sdk. Every failure path returns null and
+ * the grid falls back to a type icon: a missing thumbnail must never be able to
+ * fail an upload.
+ */
+function makeThumb (abs, mime) {
+  if (!mime.startsWith('image/')) return null
+  const tmp = path.join(TMP, `kcthumb-${Math.random().toString(36).slice(2)}.jpg`)
+  try {
+    sh('sips', ['-Z', '256', '-s', 'format', 'jpeg', '-s', 'formatOptions', '60', abs, '--out', tmp],
+      { stdio: 'ignore' })
+    const b = fs.readFileSync(tmp)
+    // Sanity bound: a "thumbnail" bigger than this is not a thumbnail, and it
+    // would bloat the very push that exists to stay small.
+    return b.length > 64 * 1024 ? null : b.toString('base64')
+  } catch {
+    return null
+  } finally {
+    try { fs.unlinkSync(tmp) } catch { /* never existed */ }
+  }
+}
+
 function recordCloudFile (rec) {
   const idx = path.join(CONFIG_DIR, CLOUD_INDEX)
   let all = []
@@ -940,25 +997,39 @@ async function cloudPut (filePath, { dry = false } = {}) {
       `command yet — nothing was spent.`)
   }
   const sha = createHash('sha256').update(bytes).digest('hex')
+  const name = path.basename(abs)
+  const mime = mimeFor(name)
+  const thumb = makeThumb(abs, mime)
   const meta = {
-    v: 1,
+    v: 2,
     kind: 'file',
-    name: path.basename(abs),
+    name,
     size: bytes.length,
     sha256: sha,
-    at: Math.floor(Date.now() / 1000)
+    mime,
+    at: Math.floor(Date.now() / 1000),
+    ...(thumb ? { thumb } : {})
   }
-  console.log(`${meta.name}  ${(bytes.length / 1048576).toFixed(2)} MB`)
+  console.log(`${name}  ${(bytes.length / 1048576).toFixed(2)} MB  ${mime}` +
+    (thumb ? `  thumb ${(Buffer.byteLength(thumb, 'base64') / 1024).toFixed(1)} KB` : ''))
   console.log(`sha256   ${sha}`)
 
   process.stdout.write('encrypting ... ')
   const t = Date.now()
-  const cipher = EncryptedMessage.encrypt(
-    Array.from(framePayload(meta, bytes)), k, PublicKey.fromString(pubHex)
+  const pub = PublicKey.fromString(pubHex)
+  // TWO PUSHES. The meta push is what the bridge indexes and the file grid
+  // reads; the file push is fetched only when someone opens the file. Both are
+  // sealed to the same key — a filename is often the most revealing part of a
+  // file, so it never travels in plaintext.
+  const metaCipher = EncryptedMessage.encrypt(
+    Array.from(Buffer.from(JSON.stringify(meta), 'utf8')), k, pub
   )
-  console.log(`${((Date.now() - t) / 1000).toFixed(1)}s`)
+  const fileCipher = EncryptedMessage.encrypt(Array.from(bytes), k, pub)
+  console.log(`${((Date.now() - t) / 1000).toFixed(1)}s  ` +
+    `meta ${(metaCipher.length / 1024).toFixed(1)} KB, file ${(fileCipher.length / 1048576).toFixed(2)} MB`)
   const timestamp = Math.floor(Date.now() / 1000).toString(16).padStart(8, '0')
-  const scriptHex = buildOpReturn([CLOUD_PREFIX, VERSION, pubHex, hex(cipher), timestamp])
+  const scriptHex = buildOpReturn(
+    [CLOUD_PREFIX, CLOUD_VERSION, pubHex, hex(metaCipher), hex(fileCipher), timestamp])
 
   const res = await publishWithRetry(scriptHex, k, address, lockingScript, { dry })
   if (dry) {
@@ -993,8 +1064,18 @@ async function cloudGet (txid, dest) {
   }
   if (!parts || parts[0] !== CLOUD_PREFIX) throw new Error(`${txid} is not a KCLD keycloud transaction`)
   if (parts[2] !== pubHex) throw new Error(`${txid} is not encrypted to this identity`)
-  const plain = Buffer.from(EncryptedMessage.decrypt(Array.from(Buffer.from(parts[3], 'hex')), k))
-  const { meta, tar: bytes } = unframePayload(plain)
+
+  // BOTH SHAPES. v1 sealed meta and bytes in one ciphertext; v2 splits them so
+  // the meta can be indexed and listed without moving the file. Files uploaded
+  // before the split must keep opening — the version byte says which is which.
+  const dec = p => Buffer.from(EncryptedMessage.decrypt(Array.from(Buffer.from(p, 'hex')), k))
+  let meta, bytes
+  if (parts[1] === CLOUD_VERSION_V1) {
+    ({ meta, tar: bytes } = unframePayload(dec(parts[3])))
+  } else {
+    meta = JSON.parse(dec(parts[3]).toString('utf8'))
+    bytes = dec(parts[4])
+  }
 
   const got = createHash('sha256').update(bytes).digest('hex')
   if (meta.sha256 && got !== meta.sha256) {
@@ -1008,6 +1089,73 @@ async function cloudGet (txid, dest) {
   console.log(`wrote ${out}`)
 }
 
+/**
+ * Publish a CATALOG: one small transaction carrying identifiers for things
+ * already on chain.
+ *
+ * The backfill problem: everything uploaded before the v2 split sealed its
+ * metadata inside its own body, so listing a v1 item means downloading it —
+ * 34 MB to read a repo name. The bridge cannot help, because it cannot decrypt.
+ * Re-uploading to get v2 metadata would cost thousands of Compute.
+ *
+ * Arash's fix, and it is the right one: send the IDENTIFIERS as their own
+ * delta. We already hold every name, size and hash in the local indexes
+ * (repos.json, keycloud.json) — the expensive part was only ever getting them
+ * back OUT of chain. One catalog transaction publishes the lot, encrypted to
+ * the same key, and any device can then list a full history without moving a
+ * single body.
+ *
+ * Shipped AS a KCLD v2 envelope on purpose: the bridge already indexes those
+ * and serves their meta push, so this needs no new prefix, no new endpoint and
+ * no new indexer. `kind: 'catalog'` tells a reader to merge the entries rather
+ * than render a file.
+ */
+async function cloudCatalog ({ dry = false } = {}) {
+  const k = key()
+  const address = k.toPublicKey().toAddress()
+  const pubHex = k.toPublicKey().toString()
+  const lockingScript = new P2PKH().lock(address)
+
+  const read = f => { try { return JSON.parse(fs.readFileSync(path.join(CONFIG_DIR, f), 'utf8')) } catch { return [] } }
+  const entries = []
+  for (const r of read('repos.json')) {
+    if (!r.txid) continue
+    entries.push({ t: r.txid, k: 'repo', n: r.repo, h: r.head, s: r.tarSize ?? null, a: r.at ?? null, c: r.cost ?? null })
+  }
+  for (const f of read(CLOUD_INDEX)) {
+    if (!f.txid) continue
+    entries.push({ t: f.txid, k: 'file', n: f.name, m: f.mime ?? null, s: f.size ?? null, a: f.at ?? null, c: f.cost ?? null })
+  }
+  if (!entries.length) throw new Error('nothing to catalog — repos.json and keycloud.json are both empty')
+
+  const meta = { v: 2, kind: 'catalog', at: Math.floor(Date.now() / 1000), count: entries.length, entries }
+  const body = Buffer.from(JSON.stringify(meta), 'utf8')
+  console.log(`catalog  ${entries.length} entries  ${(body.length / 1024).toFixed(1)} KB`)
+  for (const kind of ['repo', 'file']) {
+    const n = entries.filter(e => e.k === kind).length
+    if (n) console.log(`  ${n} ${kind}${n === 1 ? '' : 's'}`)
+  }
+
+  process.stdout.write('encrypting ... ')
+  const pub = PublicKey.fromString(pubHex)
+  const metaCipher = EncryptedMessage.encrypt(Array.from(body), k, pub)
+  // The file push is deliberately empty-but-present: a catalog has no body, and
+  // keeping the six-push shape means every existing v2 reader parses it without
+  // a special case.
+  const fileCipher = EncryptedMessage.encrypt(Array.from(Buffer.alloc(0)), k, pub)
+  console.log(`meta ${(metaCipher.length / 1024).toFixed(1)} KB`)
+  const timestamp = Math.floor(Date.now() / 1000).toString(16).padStart(8, '0')
+  const scriptHex = buildOpReturn(
+    [CLOUD_PREFIX, CLOUD_VERSION, pubHex, hex(metaCipher), hex(fileCipher), timestamp])
+
+  const res = await publishWithRetry(scriptHex, k, address, lockingScript, { dry })
+  if (dry) {
+    console.log(`\n--dry: nothing broadcast\n  payload ${res.txid}`)
+    return
+  }
+  console.log(`\nCATALOG ${entries.length} entries -> ${res.txid}  (${res.cost} Compute)`)
+}
+
 function cloudList () {
   let all = []
   try { all = JSON.parse(fs.readFileSync(path.join(CONFIG_DIR, CLOUD_INDEX), 'utf8')) } catch {}
@@ -1065,9 +1213,11 @@ export async function runKeycloud (argv) {
   if (sub === 'put') return cloudPut(args[0], { dry })
   if (sub === 'get') return cloudGet(args[0], args[1])
   if (sub === 'list') return cloudList()
+  if (sub === 'catalog') return cloudCatalog({ dry })
   console.log('usage:')
   console.log('  keychat-save keycloud put  <file> [--dry]   one tx, sealed to your own key')
   console.log('  keychat-save keycloud get  <txid> <dest>')
   console.log('  keychat-save keycloud list')
+  console.log('  keychat-save keycloud catalog            publish identifiers for existing on-chain items')
   process.exitCode = 1
 }