@palettelab/cli 0.3.43 → 0.3.45

package/README.md

@@ -328,6 +328,8 @@ async def sync_invoices(ctx: PluginContext = Depends(get_plugin_context)):
     return {"room": room, "folder": folder, "bytes": len(content or b"")}
 ```
 
+App storage is different from Data Room uploads. Use `ctx.storage` or `palette.storage` for app-owned files written directly to the OS-configured storage backend, currently GCS in hosted environments. Use `ctx.data_rooms` / `palette.dataRooms` only when the file should be managed as a Data Room document.
+
 Python backend app-storage example:
 
 ```python
@@ -340,6 +342,18 @@ async def save_report(ctx: PluginContext = Depends(get_plugin_context)):
         key="reports/summary.json",
     )
     return saved
+
+@router.post("/exports", dependencies=[require_permission("reports:write")])
+async def save_large_export(ctx: PluginContext = Depends(get_plugin_context)):
+    progress = []
+    saved = await ctx.storage.upload_file_path(
+        "/tmp/export.zip",
+        "application/zip",
+        key="exports/export.zip",
+        chunk_size=8 * 1024 * 1024,
+        on_progress=lambda event: progress.append(event),
+    )
+    return {"saved": saved, "progress": progress[-1:]}
 ```
 
 Frontend apps can use `createPaletteClient(platform).storage.upload(file, {

package/backend-sdk/palette_sdk/storage.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
+import inspect
 import mimetypes
+import os
 import re
 import uuid
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, BinaryIO, Callable
 
 
 def _slug_segment(value: str | None, fallback: str) -> str:
@@ -31,6 +33,48 @@ def _validate_relative_key(key: str) -> str:
     return "/".join(_sanitize_filename(part) for part in parts)
 
 
+MIN_CHUNK_SIZE = 256 * 1024
+DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024
+
+ProgressCallback = Callable[[dict[str, Any]], Any]
+
+def _normalize_chunk_size(value: int | None) -> int:
+    raw = max(value or DEFAULT_CHUNK_SIZE, MIN_CHUNK_SIZE)
+    return ((raw + MIN_CHUNK_SIZE - 1) // MIN_CHUNK_SIZE) * MIN_CHUNK_SIZE
+
+def _infer_stream_size(stream: BinaryIO) -> int | None:
+    try:
+        current = stream.tell()
+        stream.seek(0, os.SEEK_END)
+        end = stream.tell()
+        stream.seek(current, os.SEEK_SET)
+        return max(0, end - current)
+    except Exception:
+        return None
+
+async def _maybe_report_progress(
+    callback: ProgressCallback | None,
+    *,
+    loaded: int,
+    total: int,
+    chunk_index: int,
+    chunk_count: int,
+    state: str,
+) -> None:
+    if callback is None:
+        return
+    payload = {
+        "loaded": loaded,
+        "total": total,
+        "percentage": min(100, round((loaded / total) * 10000) / 100) if total > 0 else 100,
+        "chunk_index": chunk_index,
+        "chunk_count": chunk_count,
+        "state": state,
+    }
+    result = callback(payload)
+    if inspect.isawaitable(result):
+        await result
+
 def _content_type(filename: str | None, content_type: str | None) -> str:
     browser_ct = content_type or "application/octet-stream"
     if browser_ct == "application/octet-stream":
@@ -86,6 +130,101 @@ class LocalStorageService:
             "size": len(content),
         }
 
+    async def upload_file_stream(
+        self,
+        filename: str,
+        stream: BinaryIO,
+        content_type: str | None = None,
+        *,
+        size: int | None = None,
+        key: str | None = None,
+        chunk_size: int | None = None,
+        on_progress: ProgressCallback | None = None,
+    ) -> dict[str, Any]:
+        total = size if size is not None else _infer_stream_size(stream)
+        if total is None:
+            raise ValueError("size is required for non-seekable storage streams")
+        chunk_bytes = _normalize_chunk_size(chunk_size)
+        chunk_count = max(1, (total + chunk_bytes - 1) // chunk_bytes)
+        object_path = self.object_path(filename, key=key)
+        target = self._target(object_path)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        loaded = 0
+        await _maybe_report_progress(
+            on_progress,
+            loaded=0,
+            total=total,
+            chunk_index=0,
+            chunk_count=chunk_count,
+            state="starting",
+        )
+        if total == 0:
+            target.write_bytes(b"")
+            await _maybe_report_progress(
+                on_progress,
+                loaded=0,
+                total=0,
+                chunk_index=0,
+                chunk_count=chunk_count,
+                state="complete",
+            )
+            return {
+                "bucket": "local",
+                "object_path": object_path,
+                "file_url": target.as_uri(),
+                "content_type": _content_type(filename, content_type),
+                "size": 0,
+            }
+        with target.open("wb") as out:
+            chunk_index = 0
+            while True:
+                chunk = stream.read(chunk_bytes)
+                if not chunk:
+                    break
+                chunk_index += 1
+                out.write(chunk)
+                loaded += len(chunk)
+                await _maybe_report_progress(
+                    on_progress,
+                    loaded=loaded,
+                    total=total,
+                    chunk_index=chunk_index,
+                    chunk_count=chunk_count,
+                    state="complete" if loaded >= total else "uploading",
+                )
+        if loaded != total:
+            raise RuntimeError(f"storage upload incomplete: uploaded {loaded} of {total} bytes")
+        return {
+            "bucket": "local",
+            "object_path": object_path,
+            "file_url": target.as_uri(),
+            "content_type": _content_type(filename, content_type),
+            "size": loaded,
+        }
+
+    async def upload_file_path(
+        self,
+        path: str | Path,
+        content_type: str | None = None,
+        *,
+        filename: str | None = None,
+        key: str | None = None,
+        chunk_size: int | None = None,
+        on_progress: ProgressCallback | None = None,
+    ) -> dict[str, Any]:
+        source = Path(path)
+        resolved_filename = filename or source.name
+        with source.open("rb") as stream:
+            return await self.upload_file_stream(
+                resolved_filename,
+                stream,
+                content_type,
+                size=source.stat().st_size,
+                key=key,
+                chunk_size=chunk_size,
+                on_progress=on_progress,
+            )
+
     async def create_resumable_upload(
         self,
         filename: str,

package/docs/python-backend-sdk.md

@@ -740,6 +740,8 @@ Local mocks live in `.palette/app-services.local.json`:
 }
 ```
 
+App storage is separate from Data Rooms. Use `ctx.storage` and `palette.storage` for app-owned files that go directly to the OS-configured storage backend, currently GCS in hosted environments. Use `ctx.data_rooms` or `palette.dataRooms` only when the file should be visible and governed as a Data Room document.
+
 Palette scopes storage the same way. Files written through `ctx.storage` or the
 frontend storage client live under:
 
@@ -757,12 +759,28 @@ saved = await ctx.storage.upload_file(
     key="reports/summary.json",
 )
 
-session = await ctx.storage.create_resumable_upload(
-    "video.mp4",
+# Large backend-generated files should stream from disk. This avoids loading
+# GB-sized files into memory and reports chunk progress.
+progress = []
+saved = await ctx.storage.upload_file_path(
+    "/tmp/video.mp4",
     "video/mp4",
-    size=video_size,
     key="videos/video.mp4",
+    chunk_size=8 * 1024 * 1024,
+    on_progress=lambda event: progress.append(event),
 )
+
+# If you already have an open binary stream, use upload_file_stream(...).
+# Non-seekable streams must pass size=.
+with open("/tmp/video.mp4", "rb") as stream:
+    saved = await ctx.storage.upload_file_stream(
+        "video.mp4",
+        stream,
+        "video/mp4",
+        size=video_size,
+        key="videos/video.mp4",
+        on_progress=lambda event: progress.append(event),
+    )
 ```
 
 Frontend apps can use the browser SDK for chunked/resumable upload progress:

package/lib/commands/dev.js

@@ -83,7 +83,7 @@ async function run(args, { cwd }) {
   loadLocalEnv(cwd)
   if (cloud) {
     const json = args.includes("--json")
-    const publishArgs = []
+    const publishArgs = ["--publish-type", "preview"]
     if (flags.env) publishArgs.push("--env", flags.env)
     if (flags.yes) publishArgs.push("--yes")
     if (args.includes("--json")) publishArgs.push("--json")

package/lib/commands/publish.js

@@ -165,6 +165,20 @@ function runPreflight(cwd, json) {
   process.exit(res.status || 1)
 }
 
+
+function parsePublishType(argv) {
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i]
+    if (arg === "--preview") return "preview"
+    if (arg === "--release") return "release"
+    if (arg === "--publish-type") return argv[i + 1] === "preview" ? "preview" : "release"
+    if (arg.startsWith("--publish-type=")) {
+      return arg.slice("--publish-type=".length) === "preview" ? "preview" : "release"
+    }
+  }
+  return "release"
+}
+
 function makeApi(env) {
   return async function api(pathname, { method = "GET", body, headers = {} } = {}) {
     const url = `${env.url}${pathname}`
@@ -266,6 +280,7 @@ async function run(argv, { cwd }) {
   }
 
   const manifest = loadManifest(cwd)
+  const publishType = parsePublishType(argv)
   loadLocalEnv(cwd)
   const errors = validateManifest(manifest)
   if (errors.length) {
@@ -310,6 +325,7 @@ async function run(argv, { cwd }) {
       plugin_id: manifest.id,
       version: manifest.version,
       bundle_sha256: backendSha,
+      publish_type: publishType,
     },
   })
 
@@ -334,6 +350,7 @@ async function run(argv, { cwd }) {
     bundle_path: signed.bundle_path,
     bundle_sha256: backendSha,
     manifest,
+    publish_type: publishType,
     environment: env.name,
   }
   if (Object.keys(pluginSecrets).length) publishBody.plugin_secrets = pluginSecrets
@@ -357,6 +374,7 @@ async function run(argv, { cwd }) {
           version: record.version,
           env: env.name,
           url: env.url,
+          publish_type: record.publish_type || publishType,
           preview_url: record.preview_url,
           preview_expires_at: record.preview_expires_at,
           published_at: new Date().toISOString(),
@@ -375,12 +393,16 @@ async function run(argv, { cwd }) {
   }
 
   console.log(
-    `[pltt] published ${record.plugin_id}@${record.version} (status=${record.status})`,
+    `[pltt] ${publishType === "preview" ? "previewed" : "published"} ${record.plugin_id}@${record.version} (status=${record.status})`,
   )
   if (record.status === "active") {
     console.log(`[pltt] active on ${env.url}`)
+  } else if (record.status === "preview_ready") {
+    console.log(`[pltt] preview ready on ${env.url}`)
+  } else if (record.status === "preview_pending") {
+    console.log(`[pltt] awaiting superadmin preview approval on ${env.url}`)
   } else {
-    console.log(`[pltt] awaiting superadmin review on ${env.url}`)
+    console.log(`[pltt] awaiting superadmin publish approval on ${env.url}`)
   }
   if (record.review_url && record.status !== "active") {
     console.log(`[pltt] review queue: ${record.review_url}`)
@@ -393,7 +415,7 @@ async function run(argv, { cwd }) {
   }
   if (record.status === "active") {
     console.log(`[pltt] live at ${env.url}${record.catalog_url}`)
-  } else {
+  } else if (publishType === "release") {
     console.log(`[pltt] once approved, live at ${env.url}${record.catalog_url}`)
   }
   return record

package/lib/commands/status.js

@@ -63,6 +63,7 @@ async function run(argv, { cwd }) {
   }
 
   console.log(`[pltt] ${data.plugin_id}@${data.version}`)
+  if (data.publish_type) console.log(`  type:      ${data.publish_type}`)
   console.log(`  status:    ${data.status}`)
   if (data.review_decision) console.log(`  decision:  ${data.review_decision}`)
   if (data.review_reason) console.log(`  reason:    ${data.review_reason}`)

package/lib/dev-simulator.js

@@ -145,7 +145,7 @@ function ensurePythonEnv(cwd, devDir, manifest) {
   return venvPython
 }
 
-function writeBackendRunner(cwd, devDir, manifest, backendEntry) {
+function writeBackendRunner(cwd, devDir, manifest, backendEntry, backendPort) {
   const runner = path.join(devDir, "backend_runner.py")
   const sdkPath = localBackendSdkPath()
   const databasePath = path.join(devDir, `${manifest.id}.sqlite3`)
@@ -159,10 +159,12 @@ import importlib.util
 import json
 import os
 import pathlib
+import re
 import sys
+import uuid
 from types import SimpleNamespace
 
-from fastapi import FastAPI, Request
+from fastapi import FastAPI, HTTPException, Request, Response
 from fastapi.middleware.cors import CORSMiddleware
 from starlette.middleware.base import BaseHTTPMiddleware
 
@@ -175,6 +177,7 @@ DATABASE_URL = os.environ.get("PALETTE_DEV_DATABASE_URL", "sqlite+aiosqlite:///$
 DEV_SECRETS = json.loads(${JSON.stringify(JSON.stringify(devSecrets))})
 DEV_CONNECTIONS = json.loads(${JSON.stringify(JSON.stringify(devConnections))})
 DEV_APP_MOCKS = json.loads(${JSON.stringify(JSON.stringify(devAppMocks))})
+BACKEND_BASE = "http://127.0.0.1:${backendPort}"
 
 def _service_enabled(name: str) -> bool:
     services = MANIFEST.get("platform_services") or []
@@ -209,6 +212,34 @@ if _service_enabled("storage"):
         organization_name="Palette Dev",
     )
 
+LOCAL_UPLOADS = {}
+CONTENT_RANGE_RE = re.compile(r"^bytes (?P<start>\\d+)-(?P<end>\\d+)/(?P<total>\\d+)$")
+
+def _local_storage_enabled():
+    return _service_enabled("storage") and DEV_STORAGE is not None
+
+def _parse_content_range(value: str | None):
+    if not value:
+        raise HTTPException(status_code=411, detail="Content-Range header is required")
+    match = CONTENT_RANGE_RE.match(value)
+    if not match:
+        raise HTTPException(status_code=400, detail="Invalid Content-Range header")
+    start = int(match.group("start"))
+    end = int(match.group("end"))
+    total = int(match.group("total"))
+    if end < start or total < 0 or end >= total:
+        raise HTTPException(status_code=400, detail="Invalid Content-Range byte range")
+    return start, end, total
+
+def _require_local_storage(plugin_id: str):
+    if plugin_id != MANIFEST.get("id", ""):
+        raise HTTPException(status_code=404, detail="App not found")
+    if not _local_storage_enabled():
+        raise HTTPException(status_code=403, detail='App must declare platform_services: ["storage"]')
+
+def _api_url(path: str):
+    return f"{BACKEND_BASE}/api/v1{path}"
+
 class LocalAppInteropService:
     def service(self, service_id: str):
         return LocalAppServiceClient(service_id)
@@ -310,6 +341,84 @@ app.add_middleware(
 app.add_middleware(DevPluginContextMiddleware)
 app.include_router(router, prefix=f"/api/v1/plugins/{MANIFEST['id']}")
 
+@app.post("/api/v1/app-storage/{plugin_id}/uploads")
+async def create_local_app_storage_upload(plugin_id: str, request: Request):
+    _require_local_storage(plugin_id)
+    body = await request.json()
+    filename = body.get("filename") or "upload"
+    content_type = body.get("content_type") or "application/octet-stream"
+    size = int(body.get("size") or 0)
+    key = body.get("key")
+    chunk_size = int(body.get("chunk_size") or 8 * 1024 * 1024)
+    upload_id = uuid.uuid4().hex
+    object_path = DEV_STORAGE.object_path(filename, key=key)
+    temp_path = DEV_STORAGE._target(f".tmp/app-storage/{upload_id}.part")
+    temp_path.parent.mkdir(parents=True, exist_ok=True)
+    temp_path.write_bytes(b"")
+    LOCAL_UPLOADS[upload_id] = {
+        "plugin_id": plugin_id,
+        "object_path": object_path,
+        "content_type": content_type,
+        "size": size,
+        "uploaded_bytes": 0,
+        "complete": False,
+    }
+    return {
+        "upload_id": upload_id,
+        "mode": "local_resumable",
+        "bucket": "local",
+        "object_path": object_path,
+        "file_url": DEV_STORAGE._target(object_path).as_uri(),
+        "upload_url": _api_url(f"/app-storage/{plugin_id}/uploads/{upload_id}/chunks"),
+        "status_url": _api_url(f"/app-storage/{plugin_id}/uploads/{upload_id}/status"),
+        "content_type": content_type,
+        "size": size,
+        "chunk_size": chunk_size,
+    }
+
+@app.get("/api/v1/app-storage/{plugin_id}/uploads/{upload_id}/status")
+async def local_app_storage_upload_status(plugin_id: str, upload_id: str):
+    _require_local_storage(plugin_id)
+    session = LOCAL_UPLOADS.get(upload_id)
+    if not session or session["plugin_id"] != plugin_id:
+        raise HTTPException(status_code=404, detail="Upload session not found")
+    return {
+        "upload_id": upload_id,
+        "object_path": session["object_path"],
+        "uploaded_bytes": session["uploaded_bytes"],
+        "size": session["size"],
+        "complete": session["complete"],
+        "file_url": DEV_STORAGE._target(session["object_path"]).as_uri(),
+    }
+
+@app.put("/api/v1/app-storage/{plugin_id}/uploads/{upload_id}/chunks")
+async def upload_local_app_storage_chunk(plugin_id: str, upload_id: str, request: Request):
+    _require_local_storage(plugin_id)
+    session = LOCAL_UPLOADS.get(upload_id)
+    if not session or session["plugin_id"] != plugin_id:
+        raise HTTPException(status_code=404, detail="Upload session not found")
+    if session["complete"]:
+        return Response(status_code=204)
+    start, end, total = _parse_content_range(request.headers.get("content-range"))
+    if total != session["size"]:
+        raise HTTPException(status_code=400, detail="Chunk total does not match upload size")
+    if start != session["uploaded_bytes"]:
+        raise HTTPException(status_code=409, detail=f"Expected chunk to start at byte {session['uploaded_bytes']}")
+    payload = await request.body()
+    if len(payload) != end - start + 1:
+        raise HTTPException(status_code=400, detail="Chunk size does not match Content-Range")
+    temp_path = DEV_STORAGE._target(f".tmp/app-storage/{upload_id}.part")
+    with temp_path.open("ab") as fh:
+        fh.write(payload)
+    session["uploaded_bytes"] = end + 1
+    if session["uploaded_bytes"] >= session["size"]:
+        final_path = DEV_STORAGE._target(session["object_path"])
+        final_path.parent.mkdir(parents=True, exist_ok=True)
+        temp_path.replace(final_path)
+        session["complete"] = True
+        return Response(status_code=201)
+    return Response(status_code=308, headers={"Range": f"bytes=0-{session['uploaded_bytes'] - 1}"})
+
 @app.on_event("startup")
 async def create_local_database_tables():
     if engine is None:
@@ -328,7 +437,7 @@ function startBackend(cwd, devDir, manifest, backendPort) {
   if (!fs.existsSync(absEntry)) throw new Error(`backend entry not found: ${backendEntry}`)
 
   const python = ensurePythonEnv(cwd, devDir, manifest)
-  const runner = writeBackendRunner(cwd, devDir, manifest, backendEntry)
+  const runner = writeBackendRunner(cwd, devDir, manifest, backendEntry, backendPort)
   const sdkPath = localBackendSdkPath()
   const env = { ...process.env }
   if (sdkPath) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@palettelab/cli",
-  "version": "0.3.43",
+  "version": "0.3.45",
   "description": "Developer CLI for building Palette platform plugins — no platform source access required.",
   "bin": {
     "pltt": "bin/pltt.js"