@lightcone-ai/daemon 0.15.77 → 0.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightcone-ai/daemon",
-  "version": "0.15.77",
+  "version": "0.16.0",
   "type": "module",
   "main": "src/index.js",
   "bin": {

package/src/chat-bridge.js

@@ -2,11 +2,14 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { z } from 'zod';
-import { createReadStream, existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { createReadStream, existsSync, mkdirSync, readFileSync, writeFileSync, statSync } from 'fs';
 import { createHash, randomUUID } from 'crypto';
 import path, { extname } from 'path';
+import os from 'os';
 import { recordUrlNarration } from './_vendor/video/recorder/index.js';
-import { writeLocalFileToWorkspace } from './workspace-file-upload.js';
+import { writeLocalFileToWorkspace, resolveWorkspaceFileUploadPlan } from './workspace-file-upload.js';
+import { UploadJobManager } from './upload-job-manager.js';
+import { createUploadServerApi } from './upload-server-api.js';
 import { runRecordUrlNarrationTool } from './record-url-narration-tool.js';
 import { runSubmitToLibraryTool } from './submit-to-library-tool.js';
 import { runRenderTextToImageTool } from './tools/render-text-to-image.js';
@@ -843,6 +846,23 @@ const IS_HOST_AGENT = await resolveHostAgentFlag();
 
 const server = new McpServer({ name: 'chat', version: '0.1.0' });
 
+// ── Upload job manager (async + multipart workspace-file uploads) ────────────
+// Per-agent jobDir so each chat-bridge sees only its own pending jobs. When the
+// chat-bridge process restarts (skill bind, manual restart) it picks up any
+// in-flight uploads from disk and resumes them; see docs/upload-pipeline-design.md.
+const _uploadJobDir = path.join(
+  os.homedir(),
+  '.lightcone',
+  'upload-jobs',
+  AGENT_ID || 'default'
+);
+const uploadJobManager = new UploadJobManager({
+  jobDir: _uploadJobDir,
+  serverApi: createUploadServerApi({ api: (...args) => api(...args) }),
+  log: (msg) => console.error(`[chat-bridge:UploadJobManager] ${msg}`),
+});
+uploadJobManager.start();
+
 // ── check_messages ────────────────────────────────────────────────────────────
 server.tool('check_messages', 'Check for new messages in your inbox', {}, async () => {
   const data = await api('GET', '/receive');
@@ -1099,38 +1119,67 @@ server.tool('write_workspace', 'Write a file to the shared workspace. Use this t
   return { content: [{ type: 'text', text: `Saved to workspace: ${path}` }] };
 });
 
-server.tool('write_workspace_file', 'Write a local file directly to the shared workspace. Prefer this over write_workspace for images/PDFs/binary files so large base64 content never enters the model context. The source file may be a relative path under the current agent workspace, or an absolute path inside the agent workspace/workspace shared artifacts/notes/tmp directories.', {
-  file_path: z.string().describe('Local file path. Relative paths resolve from the current agent workspace. Absolute paths must stay inside the agent/workspace.'),
-  path:      z.string().describe('Destination path relative to workspace root, e.g. "artifacts/cover.png"'),
-}, async ({ file_path, path }) => {
+server.tool('write_workspace_file',
+  'Write a local file directly to the shared workspace. Prefer this over write_workspace for images/PDFs/binary files so large base64 content never enters the model context.\n\n'
+  + 'Binary uploads are ASYNCHRONOUS: the tool queues an upload job and returns immediately with a jobId. The actual PUT runs in the background with retry + COS multipart (no codex 120s timeout, no single-PUT failures). When you next call submit_to_library on the same workspace path, the server waits up to ~90s for the upload to finish before indexing; if it returns "upload_still_in_progress" just retry submit_to_library a few seconds later — DO NOT re-record/re-compose the video, the source file is fine.\n\n'
+  + 'Text files (.md/.txt/.json/etc.) still upload synchronously since they are tiny.',
+  {
+    file_path: z.string().describe('Local file path. Relative paths resolve from the current agent workspace. Absolute paths must stay inside the agent/workspace.'),
+    path:      z.string().describe('Destination path relative to workspace root, e.g. "artifacts/cover.png"'),
+  },
+  async ({ file_path, path: workspacePath }) => {
   if (!currentWorkspaceId) return { content: [{ type: 'text', text: 'No workspace context.' }] };
   const localPath = resolveLocalWorkspaceFile(file_path);
-  const result = await writeLocalFileToWorkspace({
-    localPath,
-    workspacePath: path,
-    workspaceId: currentWorkspaceId,
-    readFileSyncFn: readFileSync,
-    uploadWorkspaceMemory: async ({ workspacePath, workspaceId, content }) => {
-      await api('PUT', `/workspace-memory?path=${encodeURIComponent(workspacePath)}&workspaceId=${encodeURIComponent(workspaceId)}`, { content });
-    },
-    presign: async ({ workspaceId, path: filePath, size, mime, sha256 }) => {
-      return api('POST', '/storage/presign', { workspaceId, path: filePath, size, mime, sha256 });
-    },
-    confirmUpload: async ({ workspaceId, path: filePath, objectKey }) => {
-      await api('POST', '/storage/confirm', { workspaceId, path: filePath, objectKey });
-    },
-  });
 
-  const finalMime = result?.mime ?? 'application/octet-stream';
-  const finalBytes = Number.isFinite(result?.bytes) ? result.bytes : 0;
-  const sizeText = finalBytes > 0 ? formatBytes(finalBytes) : 'unknown size';
-  const asyncNote = result?.async ? ' (uploading in background)' : '';
-  return {
-    content: [{
-      type: 'text',
-      text: `Saved local file to workspace: ${path} (${finalMime}, ${sizeText})${asyncNote}`,
-    }],
-  };
+  const plan = resolveWorkspaceFileUploadPlan({ localPath, workspacePath });
+
+  // ── Text path: tiny, sync, unchanged ────────────────────────────────────
+  if (plan.isText) {
+    const result = await writeLocalFileToWorkspace({
+      localPath,
+      workspacePath,
+      workspaceId: currentWorkspaceId,
+      readFileSyncFn: readFileSync,
+      uploadWorkspaceMemory: async ({ workspacePath: wp, workspaceId, content }) => {
+        await api('PUT', `/workspace-memory?path=${encodeURIComponent(wp)}&workspaceId=${encodeURIComponent(workspaceId)}`, { content });
+      },
+      // Binary path below replaces these; keep these stubs in case the plan
+      // misclassifies, to avoid TypeError mid-tool-call.
+      presign: async () => { throw new Error('binary upload should not reach legacy presign'); },
+      confirmUpload: async () => { throw new Error('binary upload should not reach legacy confirmUpload'); },
+    });
+    const bytes = Number.isFinite(result?.bytes) ? result.bytes : 0;
+    return {
+      content: [{
+        type: 'text',
+        text: `Saved local text file to workspace: ${workspacePath} (${plan.mime}, ${formatBytes(bytes)})`,
+      }],
+    };
+  }
+
+  // ── Binary path: async enqueue ──────────────────────────────────────────
+  try {
+    const enq = await uploadJobManager.enqueue({
+      workspaceId: currentWorkspaceId,
+      agentId: AGENT_ID || null,
+      localPath,
+      workspacePath,
+      mime: plan.mime,
+    });
+    const lines = [
+      `Queued upload to workspace: ${workspacePath} (${plan.mime}, ${formatBytes(enq.size)})`,
+      `mode=${enq.mode}${enq.mode === 'multipart' ? ` chunks=${enq.totalChunks}` : ''}`,
+      `job_id=${enq.jobId}`,
+      `status=pending`,
+      `note=Upload runs in background. Call submit_to_library when ready — it waits up to ~90s for the upload to finish. If it returns "upload_still_in_progress", just retry submit_to_library a few seconds later. DO NOT re-record or re-compose.`,
+    ];
+    return { content: [{ type: 'text', text: lines.join('\n') }] };
+  } catch (err) {
+    return {
+      isError: true,
+      content: [{ type: 'text', text: `write_workspace_file failed: ${err?.message ?? err}` }],
+    };
+  }
 });
 
 // ── skill_list ───────────────────────────────────────────────────────────────

package/src/submit-to-library-tool.js

@@ -30,17 +30,18 @@ export async function runSubmitToLibraryTool({
       '/content-library/submit',
       buildSubmitToLibraryBody(args, currentWorkspaceId)
     );
-    // Server returns 2xx + body.error for transient "still processing" cases
-    // (e.g. HTTP 202 + {error: "video is still uploading, retry shortly"} when
-    // the workspace file's object_status is still 'uploading'). The HTTP client
-    // treats any 2xx as success and returns the body verbatim, so we must
-    // surface body.error here — otherwise the agent reads `data.itemId` as
-    // undefined and (because the literal "undefined" looks like proof the
-    // submit silently failed) tends to redo the entire video instead of just
-    // retrying submit_to_library. Fail loudly with the server's message so the
-    // agent retries the right step.
+    // Server returns 2xx + body.error for transient "still processing" cases:
+    //   HTTP 202 + {error: 'upload_still_in_progress', message: '...'} when the
+    //   async upload pipeline (see docs/upload-pipeline-design.md) hasn't
+    //   finished. The HTTP client treats any 2xx as success and returns the
+    //   body verbatim, so we must surface body.error here — otherwise the agent
+    //   reads `data.itemId` as undefined and (because the literal "undefined"
+    //   looks like proof the submit silently failed) tends to redo the entire
+    //   video instead of just retrying submit_to_library. Prefer body.message
+    //   when present (rich human-readable retry hint); fall back to body.error.
     if (data && typeof data === 'object' && typeof data.error === 'string' && data.error.trim()) {
-      return toolError(`submit_to_library not ready: ${data.error}. Retry submit_to_library with the same video_path in a few seconds — do NOT re-record or re-compose the video.`);
+      const detail = (typeof data.message === 'string' && data.message.trim()) ? data.message : data.error;
+      return toolError(`submit_to_library not ready: ${detail}. Retry submit_to_library with the same video_path in a few seconds — do NOT re-record or re-compose the video.`);
     }
     if (!data || typeof data.itemId !== 'string' || !data.itemId) {
       return toolError(`submit_to_library returned no itemId. Server response: ${JSON.stringify(data).slice(0, 300)}. Retry submit_to_library with the same video_path — do NOT re-record or re-compose.`);

package/src/upload-job-manager.js

@@ -0,0 +1,542 @@
+// Daemon-owned upload job manager.
+//
+// Decouples the agent-facing `write_workspace_file` tool from the actual
+// PUT-to-COS round-trip so:
+//   - the tool returns in <50ms (just enqueues a job) rather than holding the
+//     codex/claude tool slot for minutes
+//   - PUT failures are retried with exponential backoff
+//   - large files use COS multipart so a single dropped chunk doesn't restart
+//     the whole upload
+//   - daemon restarts pick up in-flight jobs from disk and resume
+//
+// Out of scope for this module: the actual COS API calls. Those live behind
+// the injected `serverApi` interface (M3 implements the server endpoints, M4
+// wires this manager to it).
+//
+// Schema and state machine: see docs/upload-pipeline-design.md.
+
+import {
+  readFileSync, writeFileSync, readdirSync, statSync,
+  promises as fsPromises,
+  createReadStream,
+  mkdirSync, renameSync,
+} from 'node:fs';
+import { open as fsOpen } from 'node:fs/promises';
+import path from 'node:path';
+import crypto, { randomUUID } from 'node:crypto';
+import os from 'node:os';
+
+export const SCHEMA_VERSION = 1;
+export const DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024;   // 8 MB
+export const MULTIPART_THRESHOLD = 5 * 1024 * 1024;  // COS minimum part size
+export const MAX_JOB_ATTEMPTS = 5;
+export const DEFAULT_TICK_INTERVAL_MS = 2_000;
+export const PART_RETRY_ATTEMPTS = 3;
+export const PART_RETRY_BASE_MS = 1_000;             // 1s, 3s, 9s
+export const TERMINAL_JOB_TTL_MS = 7 * 24 * 3600 * 1000;     // sweep done/dead_letter after 7 days
+export const HOUSEKEEPING_INTERVAL_MS = 6 * 3600 * 1000;     // run housekeeping every 6h
+
+function nowIso() { return new Date().toISOString(); }
+
+async function sha256OfFile(localPath) {
+  return new Promise((resolve, reject) => {
+    const h = crypto.createHash('sha256');
+    const stream = createReadStream(localPath);
+    stream.on('data', chunk => h.update(chunk));
+    stream.on('end', () => resolve(h.digest('hex')));
+    stream.on('error', reject);
+  });
+}
+
+// Job-level backoff: attempts²×30s.
+// 1st retry: +30s, 2nd: +120s, 3rd: +270s, 4th: +480s.
+export function jobBackoffMs(attempts) {
+  return attempts * attempts * 30_000;
+}
+
+// Chunk-level backoff: 1s, 3s, 9s.
+function partBackoffMs(attempt) {
+  return PART_RETRY_BASE_MS * (3 ** (attempt - 1));
+}
+
+function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+
+/**
+ * UploadJobManager — singleton in daemon process.
+ *
+ *   const mgr = new UploadJobManager({ serverApi });
+ *   mgr.start();
+ *   const { jobId } = await mgr.enqueue({ workspaceId, agentId, localPath, workspacePath, mime });
+ *   // job runs in background; later:
+ *   mgr.getStatus(jobId);
+ *   // or shutting down:
+ *   mgr.stop();
+ *
+ * serverApi shape (M3 implements):
+ *   presignSingle({ workspaceId, path, size, mime, sha256 }) → { objectKey, uploadUrl, method?, headers? }
+ *   confirmSingle({ workspaceId, path, objectKey })          → { ok: true }
+ *   presignMultipart({ workspaceId, path, size, mime, sha256 }) → { objectKey, cosUploadId }
+ *   presignPart({ workspaceId, objectKey, cosUploadId, partNumber }) → { url, method?, headers? }
+ *   completeMultipart({ workspaceId, path, objectKey, cosUploadId, parts, size, sha256 }) → { ok: true }
+ *   abortMultipart({ workspaceId, path, objectKey, cosUploadId })  → { ok: true }
+ */
+export class UploadJobManager {
+  constructor({
+    jobDir = path.join(os.homedir(), '.lightcone', 'upload-jobs'),
+    serverApi,
+    fetchFn = globalThis.fetch,
+    nowFn = () => Date.now(),
+    log = (msg) => console.error(`[UploadJobManager] ${msg}`),
+    tickIntervalMs = DEFAULT_TICK_INTERVAL_MS,
+    chunkSize = DEFAULT_CHUNK_SIZE,
+    multipartThreshold = MULTIPART_THRESHOLD,
+  } = {}) {
+    if (!serverApi) throw new Error('UploadJobManager: serverApi is required');
+    this.jobDir = jobDir;
+    this.serverApi = serverApi;
+    this.fetchFn = fetchFn;
+    this.nowFn = nowFn;
+    this.log = log;
+    this.tickIntervalMs = tickIntervalMs;
+    this.chunkSize = chunkSize;
+    this.multipartThreshold = multipartThreshold;
+
+    this._tickInterval = null;
+    this._housekeepingInterval = null;
+    this._stopping = false;
+    this._activeJobs = new Set();    // jobIds currently advancing
+    this._pathLocks = new Map();     // `${workspaceId}|${workspacePath}` → jobId
+
+    mkdirSync(this.jobDir, { recursive: true });
+  }
+
+  // ─── public API ──────────────────────────────────────────────────────────
+
+  /**
+   * Stat + sha256 the file, choose mode (single vs multipart), persist a
+   * pending job, and schedule a tick. Throws synchronously on path-lock
+   * conflict or missing/unreadable file.
+   */
+  async enqueue({ workspaceId, agentId = null, localPath, workspacePath, mime = 'application/octet-stream' }) {
+    if (!workspaceId) throw new Error('enqueue: workspaceId required');
+    if (!localPath) throw new Error('enqueue: localPath required');
+    if (!workspacePath) throw new Error('enqueue: workspacePath required');
+
+    const pathKey = `${workspaceId}|${workspacePath}`;
+    const existing = this._pathLocks.get(pathKey);
+    if (existing) {
+      throw new Error(`upload_path_locked: another upload to ${workspacePath} is in progress (jobId=${existing})`);
+    }
+
+    let st;
+    try { st = statSync(localPath); }
+    catch (err) { throw new Error(`localPath not readable: ${err.message}`); }
+    const size = st.size;
+    if (!Number.isFinite(size) || size <= 0) {
+      throw new Error(`localPath has invalid size: ${size}`);
+    }
+    const mtimeMs = st.mtimeMs;
+    const sha256 = await sha256OfFile(localPath);
+
+    const uploadMode = size >= this.multipartThreshold ? 'multipart' : 'single';
+    const totalChunks = uploadMode === 'multipart' ? Math.ceil(size / this.chunkSize) : 1;
+
+    const job = {
+      schemaVersion: SCHEMA_VERSION,
+      jobId: randomUUID(),
+      createdAt: nowIso(),
+      updatedAt: nowIso(),
+      agentId,
+      workspaceId,
+      localPath,
+      workspacePath,
+      mime,
+      objectKey: null,
+      size,
+      mtimeMs,
+      sha256,
+      uploadMode,
+      chunkSize: this.chunkSize,
+      totalChunks,
+      cosUploadId: null,
+      doneParts: [],
+      status: 'pending',
+      attempts: 0,
+      lastError: null,
+      lastErrorAt: null,
+      nextAttemptAt: this.nowFn(),
+    };
+
+    this._persist(job);
+    this._pathLocks.set(pathKey, job.jobId);
+
+    // Best-effort immediate tick so the very first job doesn't wait for the interval.
+    setImmediate(() => this._tick().catch(err => this.log(`tick (post-enqueue) failed: ${err.message}`)));
+
+    return {
+      jobId: job.jobId,
+      mode: uploadMode,
+      totalChunks,
+      status: 'pending',
+      size,
+    };
+  }
+
+  getStatus(jobId) {
+    const job = this._loadById(jobId);
+    return job ? this._publicState(job) : null;
+  }
+
+  start() {
+    if (this._tickInterval) return;
+    this._stopping = false;
+    // Rebuild path locks from on-disk jobs so concurrent enqueue still respects them.
+    for (const job of this._listJobs()) {
+      if (job.status === 'pending' || job.status === 'uploading') {
+        const pathKey = `${job.workspaceId}|${job.workspacePath}`;
+        this._pathLocks.set(pathKey, job.jobId);
+      }
+    }
+    this._tickInterval = setInterval(() => {
+      this._tick().catch(err => this.log(`tick failed: ${err.message}`));
+    }, this.tickIntervalMs);
+    // Run housekeeping immediately + on a slow schedule.
+    this._housekeep();
+    this._housekeepingInterval = setInterval(() => {
+      this._housekeep();
+    }, HOUSEKEEPING_INTERVAL_MS);
+    this.log(`started (jobDir=${this.jobDir}, tick=${this.tickIntervalMs}ms)`);
+  }
+
+  /**
+   * Signal stop. In-flight `_advance` calls notice via `_stopping` flag and
+   * exit between chunks; partial state is persisted so the next start() can
+   * resume. Does NOT abort the underlying fetch in progress — a single chunk
+   * may still complete after stop returns.
+   */
+  stop() {
+    this._stopping = true;
+    if (this._tickInterval) clearInterval(this._tickInterval);
+    this._tickInterval = null;
+    if (this._housekeepingInterval) clearInterval(this._housekeepingInterval);
+    this._housekeepingInterval = null;
+  }
+
+  /**
+   * Drop terminal (done / dead_letter) job files older than TERMINAL_JOB_TTL_MS.
+   * Keeps recent ones around for observability / debugging.
+   */
+  _housekeep() {
+    try {
+      const now = this.nowFn();
+      let pruned = 0;
+      for (const job of this._listJobs()) {
+        if (job.status !== 'done' && job.status !== 'dead_letter') continue;
+        const updatedAt = job.updatedAt ? Date.parse(job.updatedAt) : 0;
+        if (!Number.isFinite(updatedAt)) continue;
+        if (now - updatedAt < TERMINAL_JOB_TTL_MS) continue;
+        try {
+          // unlink via fsPromises is fine to do sync-ish since housekeeping is rare
+          // and the file is tiny.
+          fsPromises.unlink(this._jobPath(job.jobId)).catch(() => {});
+          pruned += 1;
+        } catch { /* ignore */ }
+      }
+      if (pruned > 0) this.log(`housekeeping: pruned ${pruned} terminal job(s) older than ${TERMINAL_JOB_TTL_MS / 86_400_000}d`);
+    } catch (err) {
+      this.log(`housekeeping failed: ${err?.message ?? err}`);
+    }
+  }
+
+  /**
+   * Test helper: wait until no jobs are pending/uploading or until timeout.
+   */
+  async waitAllSettled({ timeoutMs = 30_000 } = {}) {
+    const start = this.nowFn();
+    while (this.nowFn() - start < timeoutMs) {
+      const jobs = this._listJobs();
+      const inFlight = jobs.some(j => j.status === 'pending' || j.status === 'uploading');
+      if (!inFlight && this._activeJobs.size === 0) return;
+      await sleep(50);
+    }
+    throw new Error('waitAllSettled: timeout');
+  }
+
+  // ─── internal ────────────────────────────────────────────────────────────
+
+  async _tick() {
+    if (this._stopping) return;
+    const jobs = this._listJobs();
+    const now = this.nowFn();
+    for (const job of jobs) {
+      if (this._stopping) break;
+      if (job.status === 'done' || job.status === 'dead_letter') continue;
+      if (this._activeJobs.has(job.jobId)) continue;
+      if (job.nextAttemptAt && job.nextAttemptAt > now) continue;
+      this._activeJobs.add(job.jobId);
+      this._advance(job)
+        .catch(err => this.log(`advance(${job.jobId}) crashed: ${err.message}`))
+        .finally(() => this._activeJobs.delete(job.jobId));
+    }
+  }
+
+  async _advance(job) {
+    try {
+      // Local-file integrity check before each attempt.
+      let st;
+      try { st = statSync(job.localPath); }
+      catch (err) {
+        return this._markDeadLetter(job, `local_file_gone:${err.code ?? err.message}`);
+      }
+      if (st.size !== job.size) {
+        return this._markDeadLetter(job, `local_file_changed: declared=${job.size}, now=${st.size}`);
+      }
+
+      if (job.status !== 'uploading') {
+        job.status = 'uploading';
+        this._persist(job);
+      }
+
+      if (job.uploadMode === 'single') {
+        await this._advanceSingle(job);
+      } else {
+        await this._advanceMultipart(job);
+      }
+    } catch (err) {
+      this._recordFailure(job, err);
+    }
+  }
+
+  async _advanceSingle(job) {
+    const presign = await this.serverApi.presignSingle({
+      workspaceId: job.workspaceId,
+      path: job.workspacePath,
+      size: job.size,
+      mime: job.mime,
+      sha256: job.sha256,
+    });
+    job.objectKey = presign.objectKey;
+    this._persist(job);
+
+    const fileBuf = await fsPromises.readFile(job.localPath);
+    const resp = await this.fetchFn(presign.uploadUrl, {
+      method: presign.method ?? 'PUT',
+      headers: {
+        'Content-Type': job.mime,
+        'Content-Length': String(job.size),
+        ...(presign.headers ?? {}),
+      },
+      body: fileBuf,
+    });
+    if (!resp.ok) {
+      const text = await resp.text().catch(() => '');
+      throw new Error(`single PUT failed: HTTP ${resp.status} ${text.slice(0, 200)}`);
+    }
+
+    await this.serverApi.confirmSingle({
+      workspaceId: job.workspaceId,
+      path: job.workspacePath,
+      objectKey: job.objectKey,
+    });
+
+    this._markDone(job);
+  }
+
+  async _advanceMultipart(job) {
+    if (!job.cosUploadId) {
+      const init = await this.serverApi.presignMultipart({
+        workspaceId: job.workspaceId,
+        path: job.workspacePath,
+        size: job.size,
+        mime: job.mime,
+        sha256: job.sha256,
+      });
+      job.objectKey = init.objectKey;
+      job.cosUploadId = init.cosUploadId;
+      this._persist(job);
+    }
+
+    const doneSet = new Set(job.doneParts.map(p => p.partNumber));
+    const fh = await fsOpen(job.localPath, 'r');
+    try {
+      for (let i = 1; i <= job.totalChunks; i++) {
+        if (this._stopping) return;          // graceful shutdown mid-job
+        if (doneSet.has(i)) continue;
+
+        const offset = (i - 1) * job.chunkSize;
+        const remaining = job.size - offset;
+        const partLen = Math.min(job.chunkSize, remaining);
+        const buf = Buffer.alloc(partLen);
+        await fh.read(buf, 0, partLen, offset);
+
+        const etag = await this._uploadPartWithRetry(job, i, buf);
+        job.doneParts.push({ partNumber: i, etag });
+        this._persist(job);
+      }
+    } finally {
+      await fh.close().catch(() => {});
+    }
+
+    if (this._stopping) return;
+
+    await this.serverApi.completeMultipart({
+      workspaceId: job.workspaceId,
+      path: job.workspacePath,
+      objectKey: job.objectKey,
+      cosUploadId: job.cosUploadId,
+      parts: job.doneParts,
+      size: job.size,
+      sha256: job.sha256,
+    });
+
+    this._markDone(job);
+  }
+
+  async _uploadPartWithRetry(job, partNumber, buf) {
+    let lastErr;
+    for (let attempt = 1; attempt <= PART_RETRY_ATTEMPTS; attempt++) {
+      if (this._stopping) throw new Error('stopping');
+      try {
+        const presign = await this.serverApi.presignPart({
+          workspaceId: job.workspaceId,
+          objectKey: job.objectKey,
+          cosUploadId: job.cosUploadId,
+          partNumber,
+        });
+        const resp = await this.fetchFn(presign.url, {
+          method: presign.method ?? 'PUT',
+          headers: {
+            'Content-Length': String(buf.length),
+            ...(presign.headers ?? {}),
+          },
+          body: buf,
+        });
+        if (!resp.ok) {
+          const text = await resp.text().catch(() => '');
+          throw new Error(`HTTP ${resp.status} ${text.slice(0, 200)}`);
+        }
+        const etag = resp.headers.get?.('etag') ?? resp.headers.get?.('ETag') ?? presign.etag ?? '';
+        if (!etag) throw new Error(`PUT part ${partNumber} missing etag`);
+        return etag;
+      } catch (err) {
+        lastErr = err;
+        this.log(`part ${partNumber} attempt ${attempt}/${PART_RETRY_ATTEMPTS} failed: ${err.message}`);
+        if (attempt < PART_RETRY_ATTEMPTS) {
+          await sleep(partBackoffMs(attempt));
+        }
+      }
+    }
+    throw new Error(`part ${partNumber} exhausted ${PART_RETRY_ATTEMPTS} retries: ${lastErr?.message ?? lastErr}`);
+  }
+
+  _markDone(job) {
+    job.status = 'done';
+    job.updatedAt = nowIso();
+    job.lastError = null;
+    job.lastErrorAt = null;
+    job.nextAttemptAt = null;
+    this._persist(job);
+    this._releaseLock(job);
+    this.log(`job ${job.jobId} done (${job.workspacePath}, ${job.size} bytes, ${job.uploadMode})`);
+  }
+
+  _markDeadLetter(job, reason) {
+    // Best-effort COS cleanup so we don't leak storage cost on aborted multipart.
+    if (job.uploadMode === 'multipart' && job.cosUploadId && job.objectKey) {
+      Promise.resolve(this.serverApi.abortMultipart({
+        workspaceId: job.workspaceId,
+        path: job.workspacePath,
+        objectKey: job.objectKey,
+        cosUploadId: job.cosUploadId,
+      })).catch(err => this.log(`abort_multipart for ${job.jobId} failed: ${err.message}`));
+    }
+    job.status = 'dead_letter';
+    job.lastError = String(reason);
+    job.lastErrorAt = nowIso();
+    job.updatedAt = nowIso();
+    job.nextAttemptAt = null;
+    this._persist(job);
+    this._releaseLock(job);
+    this.log(`job ${job.jobId} dead_letter: ${reason}`);
+  }
+
+  _recordFailure(job, err) {
+    job.attempts = (job.attempts ?? 0) + 1;
+    job.lastError = String(err?.message ?? err);
+    job.lastErrorAt = nowIso();
+    job.updatedAt = nowIso();
+    if (job.attempts >= MAX_JOB_ATTEMPTS) {
+      this._markDeadLetter(job, `max_attempts_exhausted: ${job.lastError}`);
+      return;
+    }
+    job.status = 'pending';
+    const backoff = jobBackoffMs(job.attempts);
+    job.nextAttemptAt = this.nowFn() + backoff;
+    this._persist(job);
+    this.log(`job ${job.jobId} attempt ${job.attempts} failed: ${job.lastError}; next retry in ${Math.round(backoff / 1000)}s`);
+  }
+
+  _releaseLock(job) {
+    const pathKey = `${job.workspaceId}|${job.workspacePath}`;
+    if (this._pathLocks.get(pathKey) === job.jobId) {
+      this._pathLocks.delete(pathKey);
+    }
+  }
+
+  // ─── persistence ─────────────────────────────────────────────────────────
+
+  _jobPath(jobId) {
+    return path.join(this.jobDir, `${jobId}.json`);
+  }
+
+  _persist(job) {
+    job.updatedAt = nowIso();
+    const dest = this._jobPath(job.jobId);
+    const tmp = `${dest}.tmp`;
+    writeFileSync(tmp, JSON.stringify(job, null, 2));
+    renameSync(tmp, dest);                // atomic on POSIX
+  }
+
+  _loadById(jobId) {
+    try {
+      const text = readFileSync(this._jobPath(jobId), 'utf8');
+      const job = JSON.parse(text);
+      if (job.schemaVersion !== SCHEMA_VERSION) {
+        this.log(`job ${jobId}: unsupported schemaVersion ${job.schemaVersion}, ignored`);
+        return null;
+      }
+      return job;
+    } catch { return null; }
+  }
+
+  _listJobs() {
+    let names;
+    try { names = readdirSync(this.jobDir); }
+    catch { return []; }
+    const out = [];
+    for (const name of names) {
+      if (!name.endsWith('.json') || name.endsWith('.tmp.json')) continue;
+      const jobId = name.slice(0, -5);
+      const job = this._loadById(jobId);
+      if (job) out.push(job);
+    }
+    return out;
+  }
+
+  _publicState(job) {
+    return {
+      jobId: job.jobId,
+      status: job.status,
+      mode: job.uploadMode,
+      size: job.size,
+      progress: job.uploadMode === 'multipart'
+        ? { donePartCount: job.doneParts.length, totalChunks: job.totalChunks }
+        : { donePartCount: job.status === 'done' ? 1 : 0, totalChunks: 1 },
+      attempts: job.attempts,
+      lastError: job.lastError,
+      lastErrorAt: job.lastErrorAt,
+      nextAttemptAt: job.nextAttemptAt,
+      objectKey: job.objectKey,
+    };
+  }
+}

package/src/upload-server-api.js

@@ -0,0 +1,80 @@
+// Thin daemon-side wrapper around the server's /storage/* endpoints.
+//
+// Implements the `serverApi` interface consumed by UploadJobManager. Single
+// place that translates the abstract { presignSingle / confirmSingle /
+// presignMultipart / presignPart / completeMultipart / abortMultipart } calls
+// into HTTP requests against the lightcone server's internal/agent/.../storage/*
+// endpoints. The actual HTTP plumbing — auth headers, governance, retries —
+// is delegated to the `api` function passed in (typically chat-bridge's `api`).
+//
+// Why split this out: keeps UploadJobManager pure (no HTTP knowledge) and lets
+// us write the manager's tests entirely against an injected mock.
+
+/**
+ * @param {Object} deps
+ * @param {(method: string, path: string, body?: unknown) => Promise<unknown>} deps.api
+ *   HTTP helper that POSTs to `/internal/agent/<agentId>/<path>` and returns
+ *   the JSON body. Throws on non-2xx.
+ */
+export function createUploadServerApi({ api }) {
+  if (typeof api !== 'function') {
+    throw new Error('createUploadServerApi: api function is required');
+  }
+
+  return {
+    async presignSingle({ workspaceId, path, size, mime, sha256 }) {
+      // Existing endpoint, reused as-is. Returns { objectKey, uploadUrl, method?, headers? }.
+      const data = await api('POST', '/storage/presign', {
+        workspaceId, path, size, mime, sha256,
+      });
+      // /storage/presign returns { uploadUrl, method, headers, objectKey, alreadyExists }
+      // for the actual presigned PUT URL flow; surface them in the shape the
+      // manager expects.
+      return {
+        objectKey: data.objectKey,
+        uploadUrl: data.uploadUrl,
+        method: data.method ?? 'PUT',
+        headers: data.headers ?? {},
+        alreadyExists: !!data.alreadyExists,
+      };
+    },
+
+    async confirmSingle({ workspaceId, path, objectKey }) {
+      return api('POST', '/storage/confirm', { workspaceId, path, objectKey });
+    },
+
+    async presignMultipart({ workspaceId, path, size, mime, sha256 }) {
+      const data = await api('POST', '/storage/presign-multipart', {
+        workspaceId, path, size, mime, sha256,
+      });
+      return {
+        objectKey: data.objectKey,
+        cosUploadId: data.cosUploadId,
+        alreadyExists: !!data.alreadyExists,
+      };
+    },
+
+    async presignPart({ workspaceId, objectKey, cosUploadId, partNumber }) {
+      const data = await api('POST', '/storage/presign-part', {
+        workspaceId, objectKey, cosUploadId, partNumber,
+      });
+      return {
+        url: data.url,
+        method: data.method ?? 'PUT',
+        headers: data.headers ?? {},
+      };
+    },
+
+    async completeMultipart({ workspaceId, path, objectKey, cosUploadId, parts, size, sha256 }) {
+      return api('POST', '/storage/complete-multipart', {
+        workspaceId, path, objectKey, cosUploadId, parts, size, sha256,
+      });
+    },
+
+    async abortMultipart({ workspaceId, path, objectKey, cosUploadId }) {
+      return api('POST', '/storage/abort-multipart', {
+        workspaceId, path, objectKey, cosUploadId,
+      });
+    },
+  };
+}