@springbrand/space 0.2.0-alpha.15 → 0.2.0-alpha.17

package/src/space/fs-backend.ts

@@ -2,7 +2,6 @@ import {
   createWorkspaceStateBackend,
   FileSystemStateBackend,
   InMemoryFs,
-  Workspace,
   WorkspaceFileSystem,
   type FileSystem,
 } from "@cloudflare/shell"
@@ -12,33 +11,16 @@ import { ArtifactsFileSystem } from "./artifacts-fs"
 import { ArtifactsSync, type ArtifactsRemoteStore } from "./artifacts-sync"
 import { CheckpointStore } from "./checkpoint"
 import { createArtifactsBaseSource } from "./git-objects"
+import { inferMimeType } from "./fileinfo"
+import type { FileContentCacheOptions } from "./file-content-cache"
+import { SpaceFiles } from "./space-files"
 
-export const ARTIFACTS_REMOTE_URL_KEY = "artifacts:remoteUrl"
-export const SPACE_FS_BACKEND_KEY = "space:fsBackend"
+const ARTIFACTS_REMOTE_URL_KEY = "artifacts:remoteUrl"
+const SPACE_FS_BACKEND_KEY = "space:fsBackend"
 const ARTIFACTS_BASE_BRANCH = "main"
 const ARTIFACTS_INIT_TIMEOUT_MS = 10_000
 const CHECKPOINT_DEBOUNCE_MS = 2_000
 
-export type SpaceFsBackendMode = "artifacts" | "sql"
-
-export async function resolveSpaceFsBackendMode(
-  storage: Pick<DurableObjectStorage, "get" | "put" | "sql">,
-  env: Pick<Env, "ENABLE_ARTIFACTS" | "ARTIFACTS">,
-): Promise<SpaceFsBackendMode> {
-  let mode = await storage.get<SpaceFsBackendMode>(SPACE_FS_BACKEND_KEY)
-  if (mode) return mode
-
-  const priorArtifacts = (await storage.get<string>(ARTIFACTS_REMOTE_URL_KEY)) !== undefined
-  let hasCheckpoint = false
-  try {
-    hasCheckpoint = storage.sql.exec("SELECT 1 FROM space_checkpoint LIMIT 1").toArray().length > 0
-  } catch {}
-  const artifactsEnabled = env.ENABLE_ARTIFACTS === "true" && !!env.ARTIFACTS
-  mode = priorArtifacts || hasCheckpoint || artifactsEnabled ? "artifacts" : "sql"
-  await storage.put(SPACE_FS_BACKEND_KEY, mode)
-  return mode
-}
-
 export interface SpaceFsBackend {
   readonly fs: FileSystem
   readonly overlay: FileSystem
@@ -47,7 +29,13 @@ export interface SpaceFsBackend {
   ready(): Promise<void>
   hydrate(path: string): Promise<void>
   materializeAll(): Promise<void>
-  writeFileStream(path: string, content: ReadableStream<Uint8Array>, mediaType: string): Promise<void>
+  writeFileStream(
+    path: string,
+    content: ReadableStream<Uint8Array>,
+    mediaType: string,
+    origin?: import("./space-files").SpaceFileOrigin,
+    contentLength?: number,
+  ): Promise<void>
   flushCheckpoint(): Promise<void>
   push(branch: string): Promise<boolean>
   fetch(branch: string): Promise<void>
@@ -135,6 +123,10 @@ export class ArtifactsBackend implements SpaceFsBackend {
     await (this.fs as ArtifactsFileSystem).whenFullyMaterialized()
   }
 
+  async materializeForMigration(): Promise<void> {
+    await (this.fs as ArtifactsFileSystem).materializeForMigration()
+  }
+
   async writeFileStream(): Promise<void> {
     throw new Error("Streaming writes require the SQL Space backend")
   }
@@ -204,63 +196,12 @@ export function spaceR2Prefix(spaceId: string): string {
   return `spaces/v1/${spaceId}`
 }
 
-const R2_MULTIPART_CHUNK_BYTES = 5 * 1024 * 1024
-
-async function putStream(
-  bucket: R2Bucket,
-  key: string,
-  content: ReadableStream<Uint8Array>,
-  mediaType: string,
-): Promise<R2Object> {
-  const upload = await bucket.createMultipartUpload(key, {
-    httpMetadata: { contentType: mediaType },
-  })
-  const uploaded: R2UploadedPart[] = []
-  const reader = content.getReader()
-  let buffer = new Uint8Array(R2_MULTIPART_CHUNK_BYTES)
-  let buffered = 0
-  let partNumber = 1
-  const flush = async () => {
-    if (buffered === 0) return
-    uploaded.push(await upload.uploadPart(partNumber++, buffer.slice(0, buffered)))
-    buffer = new Uint8Array(R2_MULTIPART_CHUNK_BYTES)
-    buffered = 0
-  }
-  try {
-    while (true) {
-      const next = await reader.read()
-      if (next.done) break
-      let offset = 0
-      while (offset < next.value.byteLength) {
-        const copied = Math.min(buffer.byteLength - buffered, next.value.byteLength - offset)
-        buffer.set(next.value.subarray(offset, offset + copied), buffered)
-        buffered += copied
-        offset += copied
-        if (buffered === buffer.byteLength) await flush()
-      }
-    }
-    await flush()
-    if (uploaded.length === 0) {
-      await upload.abort()
-      return (await bucket.put(key, new Uint8Array(), {
-        httpMetadata: { contentType: mediaType },
-      }))
-    }
-    return upload.complete(uploaded)
-  } catch (error) {
-    await upload.abort().catch(() => undefined)
-    throw error
-  } finally {
-    reader.releaseLock()
-  }
-}
-
 export class SqlBackend implements SpaceFsBackend {
   readonly overlay: FileSystem
   readonly fs: FileSystem
   readonly git: Git
   readonly stateBackend: FileSystemStateBackend
-  readonly workspace: Workspace
+  readonly workspace: SpaceFiles
 
   /**
    * Large files spill to R2 rather than to the DO's own SQLite. A Space holds
@@ -272,12 +213,11 @@ export class SqlBackend implements SpaceFsBackend {
     private readonly ctx: DurableObjectState,
     private readonly repoName: string,
     private readonly r2?: R2Bucket,
+    publicOrigin = "https://workspace.invalid",
+    fileCacheOptions?: FileContentCacheOptions,
   ) {
-    const workspace = new Workspace({
-      sql: ctx.storage.sql,
-      name: () => repoName,
-      ...(r2 ? { r2, r2Prefix: spaceR2Prefix(repoName) } : {}),
-    })
+    if (!r2) throw new Error("Space files require WORKSPACE_R2")
+    const workspace = new SpaceFiles(ctx, r2, repoName, publicOrigin, fileCacheOptions)
     this.workspace = workspace
     const fs = new WorkspaceFileSystem(workspace)
     this.overlay = fs
@@ -286,67 +226,58 @@ export class SqlBackend implements SpaceFsBackend {
     this.stateBackend = createWorkspaceStateBackend(workspace)
   }
 
-  async ready(): Promise<void> {}
+  async ready(): Promise<void> { await this.workspace.ready() }
   async hydrate(_path: string): Promise<void> {}
   async materializeAll(): Promise<void> {}
   async writeFileStream(
     path: string,
     content: ReadableStream<Uint8Array>,
     mediaType: string,
+    origin: import("./space-files").SpaceFileOrigin = "agent",
+    contentLength?: number,
   ): Promise<void> {
-    if (!this.r2) throw new Error("Streaming writes require an R2 binding")
-    await this.workspace.exists("/")
-    const existing = await this.workspace.stat(path)
-    if (existing?.type === "directory") throw new Error(`EISDIR: ${path} is a directory`)
-    const parent = path.slice(0, path.lastIndexOf("/")) || "/"
-    await this.workspace.mkdir(parent, { recursive: true })
-
-    // @cloudflare/shell@0.4.3 owns this table. Its public writeFileStream buffers;
-    // this adapter keeps the same row contract while R2 consumes the body stream.
-    const table = "cf_workspace_default"
-    const previous = this.ctx.storage.sql.exec<{ r2Key: string | null }>(
-      `SELECT r2_key AS r2Key FROM ${table} WHERE path = ?`,
-      path,
-    ).toArray()[0]
-    const key = `${spaceR2Prefix(this.repoName)}/streams/${crypto.randomUUID()}`
-    try {
-      const object = await putStream(this.r2, key, content, mediaType)
-      const name = path.split("/").at(-1)!
-      const now = Math.floor(Date.now() / 1_000)
-      this.ctx.storage.sql.exec(
-        `INSERT INTO ${table}
-           (path, parent_path, name, type, mime_type, size, storage_backend,
-            r2_key, content_encoding, content, created_at, modified_at)
-         VALUES (?, ?, ?, 'file', ?, ?, 'r2', ?, 'base64', NULL, ?, ?)
-         ON CONFLICT(path) DO UPDATE SET
-           parent_path = excluded.parent_path,
-           name = excluded.name,
-           type = 'file',
-           mime_type = excluded.mime_type,
-           size = excluded.size,
-           storage_backend = 'r2',
-           r2_key = excluded.r2_key,
-           content_encoding = 'base64',
-           content = NULL,
-           modified_at = excluded.modified_at`,
-        path,
-        parent,
-        name,
-        mediaType,
-        object.size,
-        key,
-        now,
-        now,
-      )
-    } catch (error) {
-      await this.r2.delete(key).catch(() => undefined)
-      throw error
-    }
-    if (previous?.r2Key && previous.r2Key !== key) {
-      await this.r2.delete(previous.r2Key).catch(() => undefined)
-    }
+    await this.workspace.writeFileStream(path, content, mediaType, origin, contentLength)
   }
   async flushCheckpoint(): Promise<void> {}
   async push(_branch: string): Promise<boolean> { return false }
   async fetch(_branch: string): Promise<void> {}
 }
+
+export async function migrateArtifactsFiles(
+  source: FileSystem,
+  target: SpaceFiles,
+): Promise<void> {
+  const paths = (await source.glob("/**")).sort((left, right) =>
+    left.length - right.length || left.localeCompare(right))
+  for (const path of paths) {
+    if (path === "/") continue
+    const stat = await source.lstat(path)
+    if (stat.type === "directory") {
+      await target.mkdir(path, { recursive: true })
+    } else if (stat.type === "symlink") {
+      await target.symlink(await source.readlink(path), path)
+    } else {
+      await target.writeFileBytes(path, await source.readFileBytes(path), inferMimeType(path))
+    }
+  }
+}
+
+export async function migrateArtifactsBackend(
+  ctx: DurableObjectState,
+  env: Env,
+  repoName: string,
+  target: SpaceFiles,
+): Promise<void> {
+  if (await ctx.storage.get(SPACE_FS_BACKEND_KEY) === "sql") return
+  const hasRemote = await ctx.storage.get(ARTIFACTS_REMOTE_URL_KEY) !== undefined
+  let hasCheckpoint = false
+  try {
+    hasCheckpoint = ctx.storage.sql.exec("SELECT 1 FROM space_checkpoint LIMIT 1").toArray().length > 0
+  } catch {}
+  if (hasRemote || hasCheckpoint) {
+    const artifacts = new ArtifactsBackend(ctx, env, repoName)
+    await artifacts.materializeForMigration()
+    await migrateArtifactsFiles(artifacts.fs, target)
+  }
+  await ctx.storage.put(SPACE_FS_BACKEND_KEY, "sql")
+}

package/src/space/git-objects.ts

@@ -272,27 +272,6 @@ export function createArtifactsBaseSource(opts: {
   }
 }
 
-/**
- * Drop the git index so the next status walk re-reads file contents.
- *
- * Git decides a file is unchanged when its size and modification time still
- * match the index, and only hashes it otherwise. On a real filesystem that is
- * a safe shortcut; here it is not. A Space's files live in memory or SQLite,
- * where an agent can rewrite a file to different content of the same length
- * within the same millisecond — and git would then see nothing to commit. The
- * work would be in the Space but absent from every version of it.
- *
- * The index is pure cache: deleting it costs one re-hash of the tree and
- * cannot lose anything.
- */
-async function discardIndexStatCache(fs: FileSystem): Promise<void> {
-  try {
-    await fs.rm("/.git/index", { force: true })
-  } catch {
-    // No index yet, or the backend refused; the status walk still works.
-  }
-}
-
 /**
  * Stage the whole working tree (like the shell's `git add .`, which also
  * stages deletions via `statusMatrix`) while skipping reserved paths — the
@@ -302,14 +281,22 @@ async function discardIndexStatCache(fs: FileSystem): Promise<void> {
 export async function stageWorkdir(
   fs: FileSystem,
   skip: (absPath: string) => boolean,
-): Promise<void> {
+  prefix = "/",
+): Promise<boolean> {
   const gitFs = createGitFs(fs)
-  await discardIndexStatCache(fs)
-  const matrix = await git.statusMatrix({ fs: gitFs, dir: "/" })
+  const relativePrefix = prefix.replace(/^\/+|\/+$/gu, "")
+  const matrix = await git.statusMatrix({
+    fs: gitFs,
+    dir: "/",
+    ...(relativePrefix ? { filepaths: [relativePrefix] } : {}),
+  })
+  let changed = false
   for (const [filepath, head, workdir, stage] of matrix) {
     if (skip(`/${filepath}`)) continue
+    if (head !== workdir) changed = true
     if (`${head}${workdir}${stage}` === "111") continue
     if (workdir === 0) await git.remove({ fs: gitFs, dir: "/", filepath })
     else await git.add({ fs: gitFs, dir: "/", filepath })
   }
+  return changed
 }