@lightcone-ai/daemon 0.15.49 → 0.15.51

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightcone-ai/daemon",
-  "version": "0.15.49",
+  "version": "0.15.51",
   "type": "module",
   "main": "src/index.js",
   "bin": {

package/src/chat-bridge.js

@@ -6,11 +6,7 @@ import { createReadStream, existsSync, mkdirSync, readFileSync, writeFileSync }
 import { createHash, randomUUID } from 'crypto';
 import path, { extname } from 'path';
 import { recordUrlNarration } from './_vendor/video/recorder/index.js';
-import {
-  VIDEO_EXT,
-  VIDEO_MIME,
-  writeLocalFileToWorkspace,
-} from './workspace-file-upload.js';
+import { writeLocalFileToWorkspace } from './workspace-file-upload.js';
 import { runRecordUrlNarrationTool } from './record-url-narration-tool.js';
 import { runSubmitToLibraryTool } from './submit-to-library-tool.js';
 import { runRenderTextToImageTool } from './tools/render-text-to-image.js';
@@ -491,7 +487,7 @@ async function directApi(method, apiPath, body) {
 async function directApiVideoUpload(apiPath, {
   localPath,
   filename,
-  contentType = VIDEO_MIME,
+  contentType = 'video/mp4',
 }) {
   const url = `${SERVER_URL}/internal/agent/${AGENT_ID}${apiPath}`;
   const headers = {
@@ -1090,44 +1086,22 @@ server.tool('write_workspace_file', 'Write a local file directly to the shared w
     uploadWorkspaceMemory: async ({ workspacePath, workspaceId, content }) => {
       await api('PUT', `/workspace-memory?path=${encodeURIComponent(workspacePath)}&workspaceId=${encodeURIComponent(workspaceId)}`, { content });
     },
-    uploadVideo: async ({
-      localPath: videoLocalPath,
-      workspacePath,
-      workspaceId,
-      filename,
-      mime,
-    }) => {
-      const uploadResult = await runMandatoryLocalTool({
-        toolName: 'write_workspace_file',
-        toolInput: {
-          file_path,
-          path: workspacePath,
-          upload_mode: 'upload_video',
-        },
-        executor: async () => directApiVideoUpload(
-          `/upload-video?workspaceId=${encodeURIComponent(workspaceId)}&filename=${encodeURIComponent(filename)}&path=${encodeURIComponent(workspacePath)}`,
-          {
-            localPath: videoLocalPath,
-            filename,
-            contentType: mime,
-          }
-        ),
-      });
-      return {
-        mode: 'upload-video',
-        mime: VIDEO_MIME,
-        bytes: Number(uploadResult?.sizeBytes) || 0,
-      };
+    presign: async ({ workspaceId, path: filePath, size, mime, sha256 }) => {
+      return api('POST', '/storage/presign', { workspaceId, path: filePath, size, mime, sha256 });
+    },
+    confirmUpload: async ({ workspaceId, path: filePath, objectKey }) => {
+      await api('POST', '/storage/confirm', { workspaceId, path: filePath, objectKey });
     },
   });
 
-  const finalMime = result?.mime ?? (extname(path || localPath).toLowerCase() === VIDEO_EXT ? VIDEO_MIME : 'application/octet-stream');
+  const finalMime = result?.mime ?? 'application/octet-stream';
   const finalBytes = Number.isFinite(result?.bytes) ? result.bytes : 0;
   const sizeText = finalBytes > 0 ? formatBytes(finalBytes) : 'unknown size';
+  const asyncNote = result?.async ? ' (uploading in background)' : '';
   return {
     content: [{
       type: 'text',
-      text: `Saved local file to workspace: ${path} (${finalMime}, ${sizeText})`,
+      text: `Saved local file to workspace: ${path} (${finalMime}, ${sizeText})${asyncNote}`,
     }],
   };
 });

package/src/publish-job-runner.js

@@ -229,37 +229,6 @@ export async function fetchPublishJobWorkspaceFile({ serverUrl, machineApiKey, j
   return res.json();
 }
 
-export async function streamPublishJobVideo({
-  serverUrl,
-  machineApiKey,
-  jobId,
-  videoId,
-  localPath,
-  writeBinaryFile = writeFileAtomically,
-}) {
-  const normalizedJobId = String(jobId ?? '').trim();
-  if (!normalizedJobId) {
-    throw new Error('publish job id is required to fetch video files');
-  }
-  const normalizedVideoId = String(videoId ?? '').trim();
-  if (!normalizedVideoId) {
-    throw new Error('video id is required to fetch video files');
-  }
-  const url = `${String(serverUrl).replace(/\/$/, '')}/internal/agent/publish-jobs/${encodeURIComponent(normalizedJobId)}/video-files/${encodeURIComponent(normalizedVideoId)}`;
-  const res = await fetch(url, {
-    headers: {
-      Authorization: `Bearer ${machineApiKey}`,
-    },
-  });
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`publish-jobs video-files GET failed (${res.status}): ${text}`);
-  }
-  const data = Buffer.from(await res.arrayBuffer());
-  writeBinaryFile(localPath, data);
-  return localPath;
-}
-
 export function workspacePathFromMediaPath(filePath, workspaceId) {
   if (!filePath) return null;
 
@@ -294,7 +263,6 @@ export async function materializeWorkspaceMedia({
   machineApiKey,
   jobId,
   fetchWorkspaceFile = fetchPublishJobWorkspaceFile,
-  streamVideoFile = streamPublishJobVideo,
   writeBinaryFile = writeFileAtomically,
 }) {
   if (!filePath) return filePath;
@@ -317,20 +285,33 @@ export async function materializeWorkspaceMedia({
     jobId,
     relPath: workspacePath.relPath,
   });
-  if (isWorkspaceCacheValid(localPath, data)) return localPath;
 
-  if (data?.videoId) {
-    await streamVideoFile({
-      serverUrl,
-      machineApiKey,
-      jobId,
-      videoId: data.videoId,
-      localPath,
-    });
-  } else {
+  // New: signed URL from object storage
+  if (data?.signedUrl) {
+    // Use sha256-based cache key in tmp/cache to avoid duplicate downloads
+    const cacheDir = path.join(workspaceRootDir, 'tmp', 'cache');
+    const ext = path.extname(workspacePath.relPath) || '';
+    const cacheKey = data.object_hash ?? data.size_bytes?.toString() ?? 'nokey';
+    const cachePath = path.join(cacheDir, `${cacheKey}${ext}`);
+
+    if (existsSync(cachePath)) return cachePath;
+
+    mkdirSync(cacheDir, { recursive: true });
+    const res = await fetch(data.signedUrl);
+    if (!res.ok) throw new Error(`Failed to download from signed URL: ${res.status}`);
+    const buf = Buffer.from(await res.arrayBuffer());
+    writeFileAtomically(cachePath, buf);
+    return cachePath;
+  }
+
+  // Legacy: text content in DB
+  if (data?.content != null) {
+    if (isWorkspaceCacheValid(localPath, data)) return localPath;
     writeBinaryFile(localPath, decodeWorkspaceContent(data.content));
+    return localPath;
   }
-  return localPath;
+
+  throw new Error(`No content or signedUrl in workspace file response for: ${filePath}`);
 }
 
 async function materializeMedia({

package/src/workspace-file-upload.js

@@ -1,36 +1,89 @@
 import path, { extname } from 'node:path';
-import { readFileSync } from 'node:fs';
+import { readFileSync, createReadStream } from 'node:fs';
+import { createHash } from 'node:crypto';
 
 export const WORKSPACE_BINARY_MIME = {
-  '.png': 'image/png',
-  '.jpg': 'image/jpeg',
+  '.mp4':  'video/mp4',
+  '.png':  'image/png',
+  '.jpg':  'image/jpeg',
   '.jpeg': 'image/jpeg',
   '.webp': 'image/webp',
-  '.gif': 'image/gif',
-  '.pdf': 'application/pdf',
+  '.gif':  'image/gif',
+  '.pdf':  'application/pdf',
+  '.mp3':  'audio/mpeg',
+  '.wav':  'audio/wav',
+  '.m4a':  'audio/mp4',
 };
 
-export const VIDEO_EXT = '.mp4';
-export const VIDEO_MIME = 'video/mp4';
+const TEXT_EXTS = new Set([
+  '.md', '.txt', '.json', '.js', '.ts', '.html', '.css',
+  '.csv', '.xml', '.yaml', '.yml', '.sh',
+]);
+
+const LARGE_FILE_THRESHOLD = 5 * 1024 * 1024; // 5MB
 
 export function resolveWorkspaceFileUploadPlan({ localPath, workspacePath }) {
   const target = String(workspacePath ?? '').trim() || String(localPath ?? '').trim();
   const ext = extname(target).toLowerCase();
   const filename = path.basename(target || localPath);
-  if (ext === VIDEO_EXT) {
-    return {
-      mode: 'upload-video',
-      ext,
-      filename,
-      mime: VIDEO_MIME,
-    };
+  const mime = WORKSPACE_BINARY_MIME[ext] ?? 'application/octet-stream';
+  const isText = TEXT_EXTS.has(ext);
+  return { ext, filename, mime, isText };
+}
+
+function sha256ofBuffer(buf) {
+  return createHash('sha256').update(buf).digest('hex');
+}
+
+/**
+ * Upload a local file to the workspace via the new presign → PUT → confirm flow.
+ * For large files (>= 5MB), presign returns immediately (DB status = uploading)
+ * and the PUT + confirm happen in the background.
+ */
+async function uploadBinaryFile({
+  localPath,
+  workspacePath,
+  workspaceId,
+  mime,
+  presign,   // async fn({ workspaceId, path, size, mime, sha256 }) → { uploadUrl, objectKey, alreadyExists }
+  confirmUpload, // async fn({ workspaceId, path, objectKey }) → void
+}) {
+  const buf = readFileSync(localPath);
+  const size = buf.length;
+  const sha256 = sha256ofBuffer(buf);
+
+  const { uploadUrl, objectKey, alreadyExists } = await presign({
+    workspaceId,
+    path: workspacePath,
+    size,
+    mime,
+    sha256,
+  });
+
+  if (alreadyExists) {
+    return { mode: 'object-storage', mime, bytes: size, objectKey, deduped: true };
   }
-  return {
-    mode: 'workspace-memory',
-    ext,
-    filename,
-    mime: WORKSPACE_BINARY_MIME[ext] ?? 'application/octet-stream',
+
+  const isLarge = size >= LARGE_FILE_THRESHOLD;
+
+  const doPut = async () => {
+    const resp = await fetch(uploadUrl, {
+      method: 'PUT',
+      body: buf,
+      headers: { 'Content-Type': mime, 'Content-Length': String(size) },
+    });
+    if (!resp.ok) throw new Error(`COS PUT failed: ${resp.status} ${await resp.text()}`);
+    await confirmUpload({ workspaceId, path: workspacePath, objectKey });
   };
+
+  if (isLarge) {
+    // Fire-and-forget for large files; server already wrote `uploading` status
+    doPut().catch(err => console.error('[workspace-upload] large file background upload failed:', err?.message));
+    return { mode: 'object-storage', mime, bytes: size, objectKey, async: true };
+  }
+
+  await doPut();
+  return { mode: 'object-storage', mime, bytes: size, objectKey, async: false };
 }
 
 export async function writeLocalFileToWorkspace({
@@ -38,34 +91,33 @@ export async function writeLocalFileToWorkspace({
   workspacePath,
   workspaceId,
   readFileSyncFn = readFileSync,
-  uploadWorkspaceMemory,
-  uploadVideo,
+  uploadWorkspaceMemory, // legacy text path
+  presign,
+  confirmUpload,
 }) {
   const plan = resolveWorkspaceFileUploadPlan({ localPath, workspacePath });
-  if (plan.mode === 'upload-video') {
-    return uploadVideo({
+
+  if (!plan.isText) {
+    return uploadBinaryFile({
       localPath,
       workspacePath,
       workspaceId,
-      filename: plan.filename,
       mime: plan.mime,
-      mode: plan.mode,
+      presign,
+      confirmUpload,
     });
   }
 
+  // Text files: existing path via workspace-memory
   const buf = readFileSyncFn(localPath);
-  const content = `data:${plan.mime};base64,${buf.toString('base64')}`;
+  const content = buf.toString('utf8');
   await uploadWorkspaceMemory({
     workspacePath,
     workspaceId,
     content,
     mime: plan.mime,
     bytes: buf.length,
-    mode: plan.mode,
+    mode: 'workspace-memory',
   });
-  return {
-    mode: plan.mode,
-    mime: plan.mime,
-    bytes: buf.length,
-  };
+  return { mode: 'workspace-memory', mime: plan.mime, bytes: buf.length };
 }