@davait/aipictury 1.19.5 → 1.20.2

package/ext/cloud.js

@@ -8,10 +8,21 @@ const crypto = require('crypto')
 const zlib = require('zlib')
 const path = require('path')
 const { execFile } = require('child_process')
-const { Firestore } = require('./firestore')
+const { Firestore, fieldPath } = require('./firestore')
 const { Storage } = require('./storage')
 
 const CHUNK = 700_000                    // characters of base64 a chunk (the rules allow 900,000)
+// A conversation saved at every step is pushed at least this often, not only once it goes quiet:
+// a run killed in its tenth minute had pushed nothing since it started (1.20.0).
+const MAX_WAIT = 30_000
+// A conversation shared by remote control is listed while its window hosts it: the window writes
+// the listing again every REMOTE_BEAT, and a listing older than REMOTE_FRESH is no longer live —
+// its window crashed or was killed, and nothing deleted it (1.20.0: such a listing was offered on
+// aipictury.com for good).
+const REMOTE_BEAT = 60_000
+const REMOTE_FRESH = 150_000
+/** Whether a remote-control listing is still kept up by its window. */
+const remoteLive = (entry, now = Date.now()) => now - (Number(entry?.updated) || 0) <= REMOTE_FRESH
 const MAX_CHUNKS = 64                    // ~45 MB of base64: a conversation larger than that stays local
 // Preferences that follow you. Nothing else is sent: never env, headers, tokens or MCP config.
 const PREFERENCES = ['effort', 'approval', 'showThinking', 'focusView', 'autoMemory', 'notifications',
@@ -82,6 +93,16 @@ async function compactOn(server, assetId, onStage) {
   return stats
 }
 
+// As frontend/src/rules.js: a list that would not fit is refused with the reason, never cut short
+// without a word (slice dropped the newest rules, 1.20.0; firestore.rules refuses past 500).
+function rulesFit(rules) {
+  if (rules.length > 500) throw new Error(`your account keeps 500 rules at most and this would make ${rules.length}`)
+  if (Buffer.byteLength(JSON.stringify(rules)) > 900_000) throw new Error('your rules are as long as can be kept together: remove or shorten one first')
+}
+
+const KEEP_POLL = 3000                  // a render the server keeps: looked at this often
+const KEEP_WATCH = 30 * 60_000         // and this long, before the outbox takes it up later
+
 class Cloud {
   /**
    * `account`: account.js (who, and ID tokens). `device`: { id, name, app, platform, version }.
@@ -95,6 +116,10 @@ class Cloud {
     this.storage = bucket ? new Storage({ bucket, emulator: storageEmulator, getToken: () => account.idToken(), fetchImpl }) : null
     this.pushed = new Map()          // conversation path → hash last pushed (in this process)
     this.timers = new Map()
+    // Scheduled and not pushed yet, by conversation id: { folder, entry, owner, since } — kept
+    // here, so closing pushes them (flush) instead of dropping the timers' work (1.20.0).
+    this.pending = new Map()
+    this.scheduling = new Set()      // schedule() calls still finding out who is signed in
     // Signing out, or another account signing in, leaves nothing of the last session behind.
     this.subscription = account.onChange?.(() => this.reset())
   }
@@ -104,6 +129,7 @@ class Cloud {
   reset() {
     for (const t of this.timers.values()) clearTimeout(t)
     this.timers.clear()
+    this.pending.clear()
     this.pushed.clear()
   }
 
@@ -150,6 +176,15 @@ class Cloud {
     if (this.pushed.get(doc) === whole) return 'unchanged'
     const before = await this.db.get(doc)
     if (before?.hash === whole) { this.pushed.set(doc, whole); return 'unchanged' }
+    if (!before) {
+      // A new conversation: its project and a summary that says it is being saved go first, so
+      // its chunks always have a parent to be found by. A first save cut between the chunks and
+      // the summary left chunks under no conversation, which no delete ever listed (1.20.0).
+      // (No contentHash: a reader takes it for a conversation being saved, as it is.)
+      await this.touchProject(project, folder)
+      await this.db.set(doc, { ...summary, created: entry.created || Date.now(), updated: 0, chunks: 0,
+        origin: entry.origin || this.device.app, device: this.device.id })
+    }
     const stale = []
     if (before?.contentHash !== packed.hash) {
       // The chunks first, the summary last: a reader that sees the summary finds every chunk.
@@ -173,26 +208,55 @@ class Cloud {
       conversations: current?.conversations ?? 0 })
   }
 
-  /** Pushes a few seconds after the last save, so a busy conversation is sent once, not at every step. */
-  async schedule(folder, entry, delayMs = 5000) {
+  /** Pushes a few seconds after the last save, so a busy conversation is sent once, not at every
+   *  step — and at least every MAX_WAIT while it keeps being saved. */
+  schedule(folder, entry, delayMs = 5000) {
+    // Known at once, so a flush right after a save (closing) waits for it rather than missing it.
+    const scheduling = this.scheduleAfterWho(folder, entry, delayMs)
+    this.scheduling.add(scheduling)
+    return scheduling.finally(() => this.scheduling.delete(scheduling))
+  }
+
+  async scheduleAfterWho(folder, entry, delayMs) {
     // The push belongs to the account signed in now: if another one is by the time it runs, it is dropped.
     const owner = await this.uid()
     if (!owner) return
+    const was = this.pending.get(entry.id)
+    const since = was?.owner === owner ? was.since : Date.now()
+    this.pending.set(entry.id, { folder, entry, owner, since })
     clearTimeout(this.timers.get(entry.id))
-    const timer = setTimeout(async () => {
+    const timer = setTimeout(() => {
       this.timers.delete(entry.id)
-      if ((await this.uid()) !== owner) return
-      try { await this.pushConversation(folder, entry) } catch (error) { this.log(`cloud: ${entry.id} not pushed: ${error.message}`) }
-    }, delayMs)
+      this.pushPending(entry.id)
+    }, Math.max(0, Math.min(delayMs, since + MAX_WAIT - Date.now())))
     timer.unref?.()
     this.timers.set(entry.id, timer)
   }
 
-  /** Waits for what is scheduled (on closing). */
-  async flush(pending) {
+  /** The scheduled push of one conversation, now; never throws. */
+  async pushPending(id) {
+    const p = this.pending.get(id)
+    if (!p) return
+    this.pending.delete(id)
+    if ((await this.uid().catch(() => null)) !== p.owner) return
+    if ((p.entry.updated || 0) <= (this.clearedHere || 0)) return    // saved before "Delete everything"
+    try { await this.pushConversation(p.folder, p.entry) } catch (error) { this.log(`cloud: ${id} not pushed: ${error.message}`) }
+  }
+
+  /** A conversation deleted here: a push still scheduled for it would bring it back. */
+  cancel(id) {
+    clearTimeout(this.timers.get(id))
+    this.timers.delete(id)
+    this.pending.delete(id)
+  }
+
+  /** On closing: everything scheduled is pushed now, not dropped with its timer (1.20.0: VS Code
+   *  never called this, and the last few seconds of every conversation stayed off the account). */
+  async flush() {
+    await Promise.allSettled([...this.scheduling])
     for (const t of this.timers.values()) clearTimeout(t)
     this.timers.clear()
-    for (const [folder, entry] of pending || []) await this.pushConversation(folder, entry).catch(() => {})
+    for (const id of [...this.pending.keys()]) await this.pushPending(id)
   }
 
   /** A project's conversations, by its id (the web app's chats are the project "web"). */
@@ -206,6 +270,7 @@ class Cloud {
     const doc = `${await this.base()}/projects/${projectId}/conversations/${id}`
     const meta = await this.db.get(doc)
     if (!meta) throw new Error('that conversation is not in your history any more')
+    if (!meta.contentHash) throw new Error('it is being saved right now — try again in a moment')
     const parts = []
     for (let i = 0; i < meta.chunks; i += 1) {
       const c = await this.db.get(`${doc}/chunks/${i}`)
@@ -261,6 +326,7 @@ class Cloud {
     const doc = `${project.path}/conversations/${id}`
     const meta = await this.db.get(doc)
     if (!meta) throw new Error('that conversation is not in the cloud')
+    if (!meta.contentHash) throw new Error('the cloud copy is being written — try again in a moment')
     const parts = []
     for (let i = 0; i < meta.chunks; i += 1) {
       const c = await this.db.get(`${doc}/chunks/${i}`)
@@ -276,9 +342,10 @@ class Cloud {
   async deleteConversation(folder, id) {
     const project = await this.projectPath(folder)
     const doc = `${project.path}/conversations/${id}`
-    const meta = await this.db.get(doc)
     const writes = []
-    for (let i = 0; i < (meta?.chunks || 0); i += 1) writes.push({ delete: `${doc}/chunks/${i}` })
+    // Every chunk there is, listed, not as many as the summary says: a save cut short leaves
+    // chunks with no summary, or more than it counts (1.20.0).
+    for (const k of await this.db.list(`${doc}/chunks`)) writes.push({ delete: `${doc}/chunks/${k.id}` })
     writes.push({ delete: doc })
     await this.db.batch(writes)
     this.pushed.delete(doc)
@@ -293,7 +360,21 @@ class Cloud {
 
   /** Your rules: [{ id, text, created }]. */
   async getRules() { return (await this.db.get(`${await this.base()}/memory/rules`))?.rules || [] }
-  async setRules(rules) { await this.db.set(`${await this.base()}/memory/rules`, { rules: rules.slice(0, 500), updated: Date.now() }) }
+  async setRules(rules) { rulesFit(rules); await this.db.set(`${await this.base()}/memory/rules`, { rules, updated: Date.now() }) }
+  /** Rules added, and one taken out, on the server in one step each: a rule saved on another
+   *  machine (or the web) at the same moment stays. Reading the list, changing it and writing it
+   *  back lost one of the two (1.20.0). A rule is never edited, only added or removed, so taking
+   *  out the very rules that carry this id removes it whatever was added meanwhile. */
+  async addRules(rules) {
+    if (!rules.length) return
+    rulesFit([...await this.getRules(), ...rules])
+    await this.db.change(`${await this.base()}/memory/rules`, { set: { updated: Date.now() }, append: { rules } })
+  }
+
+  async removeRule(id) {
+    const gone = (await this.getRules()).filter((r) => String(r.id) === String(id))
+    await this.db.change(`${await this.base()}/memory/rules`, { set: { updated: Date.now() }, remove: { rules: gone } })
+  }
 
   async getPreferences() { return (await this.db.get(`${await this.base()}/settings/preferences`)) || null }
   /** This device's preferences, merged into the account's: what other apps keep there (the web
@@ -305,27 +386,57 @@ class Cloud {
     await this.db.set(path, { values: { ...current, ...allowed }, updated: Date.now(), device: this.device.id })
   }
 
-  /** Adds one turn to the day's totals. */
+  /** Adds one turn to the day's totals — added on the server, so two machines' turns at the same
+   *  moment both count (a read, a sum and a write back kept only one: 1.20.0). */
   async addUsage(day, turn) {
     const doc = `${await this.base()}/usage/${day}`
-    const now = (await this.db.get(doc)) || { turns: 0, prompt: 0, output: 0, genSeconds: 0, tools: {}, models: {} }
-    for (const t of turn.tools || []) now.tools[t] = (now.tools[t] || 0) + 1
-    for (const m of turn.models || []) now.models[m] = (now.models[m] || 0) + 1
-    await this.db.set(doc, { turns: now.turns + 1, prompt: now.prompt + (turn.context || 0), output: now.output + (turn.output || 0),
-      genSeconds: Math.round((now.genSeconds + (turn.genSeconds || 0)) * 10) / 10, tools: now.tools, models: now.models, updated: Date.now() })
+    const increment = { turns: 1, prompt: turn.context || 0, output: turn.output || 0, genSeconds: Math.round((turn.genSeconds || 0) * 10) / 10 }
+    for (const t of turn.tools || []) increment[fieldPath('tools', t)] = (increment[fieldPath('tools', t)] || 0) + 1
+    for (const m of turn.models || []) increment[fieldPath('models', m)] = (increment[fieldPath('models', m)] || 0) + 1
+    await this.db.change(doc, { set: { updated: Date.now() }, increment })
   }
 
-  /** Everything of yours in the cloud, deleted. The local files stay. */
   /**
    * A render the GPU server finished (its job), kept with the account as the web app keeps it:
    * each file and a clip's poster in Storage (media/), its entry in renders/. Once per job;
-   * false when it was kept already or there is nothing to keep.
+   * false when it was kept already or there is nothing to keep (the job gone from the server).
+   * Safe to run again after a cut: the same files are written again, the entry still last.
    */
+  /** Whether this GPU server keeps renders with accounts by itself (GET /api/health `keeps`). */
+  async serverKeeps(server) {
+    const r = await fetch(`${server}/api/health`, { signal: AbortSignal.timeout(10_000) }).catch(() => null)
+    return r?.ok ? !!(await r.json()).keeps : false
+  }
+
+  /** A render the server is keeping: how it goes, on its card. True once its entry is there; a
+   *  failure throws, so the outbox looks again later. The app can go at any moment: the server goes on. */
+  async followKeeping(server, jobId, path, onStage) {
+    const until = Date.now() + KEEP_WATCH
+    while (Date.now() < until) {
+      const r = await fetch(`${server}/api/keeping/${jobId}`, { signal: AbortSignal.timeout(20_000) })
+      if (!r.ok) throw new Error(`the server could not say how keeping it goes (${r.status})`)
+      const now = await r.json()
+      if (now.state === 'unknown') return false
+      if (now.state === 'done') {
+        const entry = await this.db.get(path)
+        const stats = (entry?.assets || []).find((a) => a.stats)?.stats
+        onStage(now.asset, stats ? { stage: 'kept', stats } : { stage: 'kept' })
+        return !!entry
+      }
+      if (now.state === 'failed' || now.state === 'retry') throw new Error(now.error || 'the server could not keep it')
+      if (now.stage) onStage(now.asset, { stage: now.stage, progress: now.progress })
+      await new Promise((resolve) => setTimeout(resolve, KEEP_POLL))
+    }
+    throw new Error('the server is still keeping it')
+  }
+
   async keepRender(server, jobId, onStage = () => {}) {
     if (!this.storage || !jobId) return false
     const path = `${await this.base()}/renders/${jobId}`
     if (await this.db.get(path)) return false
+    if (await this.serverKeeps(server)) return this.followKeeping(server, jobId, path, onStage)
     const r = await fetch(`${server}/api/jobs/${jobId}`, { signal: AbortSignal.timeout(20_000) })
+    if (r.status === 404) return false
     if (!r.ok) throw new Error(`the job could not be read (${r.status})`)
     const job = await r.json()
     if (job.status !== 'done') return false
@@ -365,13 +476,17 @@ class Cloud {
   }
 
   /** A conversation shared by remote control, listed with the account (aipictury.com offers to
-   *  control it): its link, title, machine; gone when sharing stops. */
-  async putRemote(id, { url, title, device, conversation = '' }) {
-    await this.db.set(`${await this.base()}/remote/${id}`, { url, title, device, conversation, started: Date.now(), updated: Date.now() })
+   *  control it): its link, title, machine; gone when sharing stops. Written again every
+   *  REMOTE_BEAT while it is hosted (`started` kept): `updated` says it is live. */
+  async putRemote(id, { url, title, device, conversation = '', started = Date.now() }) {
+    await this.db.set(`${await this.base()}/remote/${id}`, { url, title, device, conversation, started, updated: Date.now() })
   }
 
   async dropRemote(id) { await this.db.delete(`${await this.base()}/remote/${id}`) }
 
+  /** The shared conversations listed with the account: [{ id, url, title, device, conversation, … }]. */
+  async listRemote() { return (await this.db.list(`${await this.base()}/remote`)).map((d) => ({ ...d.data, id: d.id })) }
+
   /** One kind of what is kept, deleted from every machine's cloud copy (its switch turned off):
    *  'conversations' (with their chunks), 'memory' (your rules and each project's), 'settings'
    *  (preferences) or 'usage'. Returns how many records went. */
@@ -380,6 +495,8 @@ class Cloud {
     const writes = []
     let kept = null
     if (kind === 'conversations' || kind === 'memory') {
+      // A save writes the project and a summary before any chunk (pushConversation), so what a
+      // cut-short save left is listed here, with its chunks.
       for (const p of await this.db.list(`${base}/projects`)) {
         if (kind === 'memory') { writes.push({ delete: `${base}/projects/${p.id}/memory/current` }); continue }
         for (const c of await this.db.list(`${base}/projects/${p.id}/conversations`)) {
@@ -413,10 +530,29 @@ class Cloud {
     await this.db.set(`${await this.base()}/settings/preferences`, { values: { gpuServer }, updated: Date.now(), device: this.device.id })
   }
 
-  async deleteEverything() {
+  /**
+   * Everything kept with the account, deleted (the local files stay). The "cleared" marker goes
+   * FIRST, with the server choice, in one commit: it used to be written last, so a delete cut
+   * between its commits or in the Storage step left no marker, and another machine's bringUp
+   * carried the deleted conversations straight back (1.20.0). Each step is safe to run again.
+   */
+  async deleteEverything({ byServer = false } = {}) {
     const base = await this.base()
     const writes = []
     const server = await this.serverChoice()
+    const cleared = Date.now()
+    // What was saved here in the last few seconds and is still scheduled is from before the delete:
+    // its push came after it and put the conversation back — the marker only stops bringUp (1.20.0).
+    // Cancelled now, and a push already on its way stops at pushPending.
+    this.reset()
+    this.clearedHere = cleared
+    // The preferences are replaced by the server choice alone, never deleted then put back.
+    await this.db.batch([{ set: [base, { cleared }] }, server
+      ? { set: [`${base}/settings/preferences`, { values: { gpuServer: server }, updated: cleared, device: this.device.id }] }
+      : { delete: `${base}/settings/preferences` }])
+    // The server deletes the rest when it is asked, and carries on if this app goes (keeper.py): it
+    // also reaches what a signed-in app cannot list. The marker above already stops every device.
+    if (byServer) { this.pushed.clear(); return 0 }
     for (const p of await this.db.list(`${base}/projects`)) {
       for (const c of await this.db.list(`${base}/projects/${p.id}/conversations`)) {
         // Every chunk there is, not as many as the summary says: one left by a save cut short stays otherwise.
@@ -428,16 +564,13 @@ class Cloud {
       writes.push({ delete: `${base}/projects/${p.id}/memory/current` }, { delete: `${base}/projects/${p.id}` })
     }
     for (const col of ['usage', 'devices', 'renders', 'remote']) for (const d of await this.db.list(`${base}/${col}`)) writes.push({ delete: `${base}/${col}/${d.id}` })
-    writes.push({ delete: `${base}/memory/rules` }, { delete: `${base}/settings/preferences` }, { delete: base })
+    writes.push({ delete: `${base}/memory/rules` })
+    this.pushed.clear()
     await this.db.batch(writes)
     // The media kept with the account (pictures, clips, attachments) go too.
     if (this.storage) for (const o of await this.storage.list(`${base}/media/`)) { await this.storage.delete(o.name); writes.push(o) }
-    // The moment it was cleared: no app carries back up what it still holds from before.
-    await this.db.set(base, { cleared: Date.now() })
-    if (server) await this.keepServerChoice(server)
-    this.pushed.clear()
-    return writes.length
+    return writes.length + 2
   }
 }
 
-module.exports = { Cloud, projectOf, normalizeRemote, pack, unpack, PREFERENCES, CHUNK, MAX_CHUNKS }
+module.exports = { Cloud, projectOf, normalizeRemote, pack, unpack, remoteLive, PREFERENCES, CHUNK, MAX_CHUNKS, MAX_WAIT, REMOTE_BEAT, REMOTE_FRESH }

package/ext/dialogs.js

@@ -750,10 +750,10 @@ async function sendReport(server, body) {
     const response = await fetch(`${server}/api/feedback`, { method: 'POST', headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify(body), signal: controller.signal })
     const data = await response.json().catch(() => ({}))
-    if (!response.ok) throw new Error(data.detail || `the server answered ${response.status}`)
+    if (!response.ok) throw Object.assign(new Error(data.detail || `the server answered ${response.status}`), { status: response.status })
     return data.id
   } catch (error) {
-    throw new Error(error.name === 'AbortError' ? 'the server did not answer in 10 s' : error.message, { cause: error })
+    throw Object.assign(new Error(error.name === 'AbortError' ? 'the server did not answer in 10 s' : error.message, { cause: error }), { status: error.status })
   } finally { clearTimeout(timer) }
 }
 
@@ -767,12 +767,16 @@ function reportHandler({ server, version, send = sendReport }) {
       if (!description) throw new Error('say what happened')
       const kind = f.kind === 'feedback' ? 'feedback' : 'bug'
       saved = feedback.save({ kind, description, versions: { extension: version, vscode: vscode.version },
-        settings: settingsSnapshot(), log: log.tail() })
+        settings: settingsSnapshot(), log: log.tail(), send: f.send !== 'local' })
       log.info(`problem report saved: ${saved.file}`)
       let sent = 'Kept only on this machine.'
       if (f.send !== 'local') {
-        try { sentId = await send(server, saved.body); sent = `Sent to your AIPictury server as ${sentId}.` } catch (error) {
-          sent = `Not sent: ${error.message}. It is saved here — send it later from this folder.`
+        try {
+          sentId = await feedback.sendOne(server, saved.file, saved.body, send)   // never twice with the outbox's retry
+          sent = `Sent to your AIPictury server as ${sentId}.`
+        } catch (error) {
+          // Not lost: it waits in the reports folder and goes by itself once the server answers.
+          sent = `Not sent yet: ${error.message}. It is saved here, and sent by itself once your server answers.`
           log.warn('problem report not sent', error)
         }
       }