buckyos 0.7.118 → 0.7.120

package/cli/dist/cli.mjs

@@ -506,6 +506,9 @@ function buildDistributionPolicy(options) {
   if (options.distribution === "system")
     readPaths.add(buckyosRoot);
   const parsed = collectArgumentPaths(argv);
+  if (options.distribution === "system" && parsed.module === "app" && ["fetch", "install"].includes(parsed.verb ?? "")) {
+    writePaths.add(path.join(buckyosRoot, "cache", "control_panel", "pikg_staging", "incoming"));
+  }
   for (const candidate of parsed.read)
     readPaths.add(resolveInputPath(candidate, cwd, path));
   for (const candidate of parsed.write)
@@ -4848,7 +4851,7 @@ function invalid(stage, message, entry) {
 function stableJsonDigest(value) {
   return sha256Bytes(new TextEncoder().encode(ndn.toCanonicalJsonString(value)));
 }
-const PACKAGE_VERSION = "0.7.118";
+const PACKAGE_VERSION = "0.7.120";
 const TOOL_VERSION = PACKAGE_VERSION;
 const SDK_VERSION = PACKAGE_VERSION;
 const PROTOCOL_VERSION = "1";
@@ -6709,7 +6712,7 @@ async function installApp(ctx, input, dependencies) {
       ...sourceRpcParams(source),
       ...planScopeAndOptions(inspection.plan),
       options: installOptions(input),
-      plan: submittedPlan ?? null,
+      plan: submittedPlan ?? inspection.plan,
       approved_plan_fingerprint: expectString$1(inspection.plan, "plan_fingerprint"),
       idempotency_key: idempotencyKey(ctx)
     };
@@ -6781,6 +6784,7 @@ async function upgradeApps(ctx, input, dependencies) {
         ...sourceRpcParams(source),
         ...planScopeAndOptions(inspection.plan),
         options: installOptions(input),
+        plan: inspection.plan,
         approved_plan_fingerprint: expectString$1(inspection.plan, "plan_fingerprint"),
         idempotency_key: idempotencyKey(ctx)
       }),
@@ -6852,36 +6856,79 @@ async function prepareSource(ctx, input, purpose, dependencies) {
   const classified = await classifySource(ctx, input);
   if (classified.kind === "catalog")
     return classified;
-  const snapshot = classified.kind === "url" ? await downloadPikg(classified.url, dependencies.download, ctx.signal) : await readPikg(classified.path, classified.kind);
-  const stage = dependencies.stagePikg ?? stagePikg;
-  const metadata = await stage(ctx, snapshot, purpose);
-  if (metadata.schema_version !== PLAN_SCHEMA_VERSION) {
-    throw new ToolError("UNSUPPORTED_SCHEMA_VERSION", "staging returned a non-v4 schema");
+  const host2 = getHost();
+  const localDirectory = localPikgDirectory(ctx);
+  let directory;
+  let localFileId;
+  if (localDirectory) {
+    try {
+      await host2.mkdir(localDirectory, { recursive: true, mode: 448 });
+      directory = localDirectory;
+      localFileId = crypto.randomUUID().replaceAll("-", "");
+    } catch (error) {
+      if (!isHostError(error, "PermissionDenied") && !isHostError(error, "NotFound"))
+        throw error;
+    }
   }
-  if (metadata.pikg_digest !== snapshot.digest || metadata.size !== snapshot.size) {
-    throw new ToolError(
-      "PIKG_DIGEST_MISMATCH",
-      "the staged PIKG does not match the client byte snapshot",
-      EXIT_OPERATION,
-      false,
-      {
-        expected_digest: snapshot.digest,
-        staged_digest: metadata.pikg_digest,
-        expected_size: snapshot.size,
-        staged_size: metadata.size
-      }
-    );
+  if (!localFileId) {
+    await host2.mkdir(ctx.configStore.root, { recursive: true, mode: 448 });
+    directory = await host2.makeTempDir({ dir: ctx.configStore.root, prefix: ".pikg-" });
   }
-  if (metadata.purpose !== purpose) {
-    throw new ToolError("INVALID_SERVICE_RESPONSE", "staging purpose does not match the request", 9);
+  let path = host2.path.join(directory, `${localFileId ?? "source"}.pikg`);
+  const snapshotAt = (destination) => classified.kind === "url" ? downloadPikg(classified.url, destination, dependencies.download, ctx.signal) : readPikg(classified.path, destination, ctx.signal);
+  try {
+    let snapshot;
+    try {
+      snapshot = await snapshotAt(path);
+    } catch (error) {
+      if (!localFileId || !isHostError(error, "PermissionDenied"))
+        throw error;
+      await host2.remove(path).catch(() => {
+      });
+      localFileId = void 0;
+      await host2.mkdir(ctx.configStore.root, { recursive: true, mode: 448 });
+      directory = await host2.makeTempDir({ dir: ctx.configStore.root, prefix: ".pikg-" });
+      path = host2.path.join(directory, "source.pikg");
+      snapshot = await snapshotAt(path);
+    }
+    snapshot.localFileId = localFileId;
+    const stage = dependencies.stagePikg ?? stagePikg;
+    const metadata = await stage(ctx, snapshot, purpose);
+    if (metadata.schema_version !== PLAN_SCHEMA_VERSION) {
+      throw new ToolError("UNSUPPORTED_SCHEMA_VERSION", "staging returned a non-v4 schema");
+    }
+    if (metadata.pikg_digest !== snapshot.digest || metadata.size !== snapshot.size) {
+      throw new ToolError(
+        "PIKG_DIGEST_MISMATCH",
+        "the staged PIKG does not match the client byte snapshot",
+        EXIT_OPERATION,
+        false,
+        {
+          expected_digest: snapshot.digest,
+          staged_digest: metadata.pikg_digest,
+          expected_size: snapshot.size,
+          staged_size: metadata.size
+        }
+      );
+    }
+    if (metadata.purpose !== purpose) {
+      throw new ToolError(
+        "INVALID_SERVICE_RESPONSE",
+        "staging purpose does not match the request",
+        9
+      );
+    }
+    return {
+      kind: snapshot.kind,
+      display: snapshot.display,
+      snapshot,
+      staging: metadata,
+      serviceSource: { kind: "local_pikg", staging_handle: metadata.handle }
+    };
+  } finally {
+    await host2.remove(localFileId ? path : directory, { recursive: !localFileId }).catch(() => {
+    });
   }
-  return {
-    kind: snapshot.kind,
-    display: snapshot.display,
-    snapshot,
-    staging: metadata,
-    serviceSource: { kind: "local_pikg", staging_handle: metadata.handle }
-  };
 }
 async function classifySource(ctx, input) {
   const positional = optionalString(input.source);
@@ -6929,69 +6976,106 @@ async function classifySource(ctx, input) {
     serviceSource: { kind: "identifier", identifier }
   };
 }
-async function readPikg(path, kind) {
+function localPikgDirectory(ctx) {
+  const root = getHost().policy.buckyosRoot;
+  const hostname = new URL(ctx.connection.endpoint).hostname;
+  if (!root || !["localhost", "127.0.0.1", "[::1]"].includes(hostname))
+    return void 0;
+  return getHost().path.join(root, "cache", "control_panel", "pikg_staging", "incoming");
+}
+async function* fileChunks(file, size) {
+  const buffer = new Uint8Array(size);
+  while (true) {
+    let used = 0;
+    while (used < buffer.length) {
+      const read = await file.read(buffer.subarray(used));
+      if (read === null)
+        break;
+      if (read === 0)
+        throw new UsageError("PIKG_READ_FAILED", "PIKG read made no progress");
+      used += read;
+    }
+    if (used === 0)
+      break;
+    yield buffer.subarray(0, used);
+  }
+}
+async function writePikgSnapshot(path, chunks, signal) {
+  const output = await getHost().open(path, { write: true, createNew: true, mode: 384 });
+  const hash = getHost().createHash("sha256");
+  const magic = new Uint8Array(ZIP_LOCAL_MAGIC.length);
+  let size = 0;
+  try {
+    for await (const chunk of chunks) {
+      signal.throwIfAborted();
+      if (size < magic.length)
+        magic.set(chunk.subarray(0, magic.length - size), size);
+      hash.update(chunk);
+      size += chunk.length;
+      if (!Number.isSafeInteger(size))
+        throw new UsageError("PIKG_TOO_LARGE", "PIKG is too large");
+      let offset = 0;
+      while (offset < chunk.length) {
+        const written = await output.write(chunk.subarray(offset));
+        if (written === 0)
+          throw new UsageError("PIKG_WRITE_FAILED", "PIKG write made no progress");
+        offset += written;
+      }
+    }
+    validatePikgSnapshot(magic.subarray(0, Math.min(size, magic.length)));
+    await output.sync();
+    return { path, size, digest: hash.digestHex() };
+  } finally {
+    await output.close();
+  }
+}
+async function readPikg(source, destination, signal) {
   let file;
   try {
-    file = await getHost().open(path, { read: true });
+    file = await getHost().open(source, { read: true });
   } catch (error) {
     throw new UsageError("PIKG_READ_FAILED", `failed to open PIKG: ${errorMessage(error)}`);
   }
   try {
-    const stat2 = await file.stat();
-    if (!stat2.isFile)
+    const before = await file.stat();
+    if (!before.isFile)
       throw new UsageError("INVALID_PIKG_SOURCE", "PIKG source is not a file");
-    if (stat2.size > Number.MAX_SAFE_INTEGER) {
-      throw new UsageError("PIKG_TOO_LARGE", "PIKG is too large for this client");
-    }
-    const bytes = new Uint8Array(stat2.size);
-    let offset = 0;
-    while (offset < bytes.length) {
-      const read = await file.read(bytes.subarray(offset));
-      if (read === null)
-        break;
-      offset += read;
-    }
-    if (offset !== bytes.length) {
-      throw new UsageError("PIKG_READ_FAILED", "PIKG changed size while being read");
+    const snapshot = await writePikgSnapshot(destination, fileChunks(file, 256 * 1024), signal);
+    const after = await file.stat();
+    if (snapshot.size !== before.size || before.size !== after.size || before.mtime?.getTime() !== after.mtime?.getTime()) {
+      throw new UsageError("PIKG_READ_FAILED", "PIKG changed while being read");
     }
-    validatePikgSnapshot(bytes);
-    return {
-      kind,
-      display: path,
-      bytes,
-      digest: await sha256Id(bytes),
-      size: bytes.length
-    };
+    return { kind: "pikg", display: source, ...snapshot };
   } finally {
     await file.close();
   }
 }
-async function downloadPikg(url, downloader, signal) {
-  let bytes;
+async function downloadPikg(url, destination, downloader, signal) {
+  let chunks;
   try {
-    bytes = downloader ? await downloader(url, signal) : await defaultDownload(url, signal);
+    chunks = downloader ? [await downloader(url, signal)] : await defaultDownload(url, signal);
+    const snapshot = await writePikgSnapshot(destination, chunks, signal);
+    return { kind: "url", display: safeUrl(url), ...snapshot };
   } catch (error) {
-    if (error instanceof ToolError)
+    if (error instanceof ToolError || isHostError(error, "PermissionDenied"))
       throw error;
+    signal.throwIfAborted();
     throw new ToolError(
       "PIKG_DOWNLOAD_FAILED",
       `failed to download PIKG: ${errorMessage(error)}`,
       5,
       true
     );
+  } finally {
+    if (chunks instanceof ReadableStream && !chunks.locked)
+      await chunks.cancel().catch(() => {
+      });
   }
-  validatePikgSnapshot(bytes);
-  return {
-    kind: "url",
-    display: safeUrl(url),
-    bytes,
-    digest: await sha256Id(bytes),
-    size: bytes.length
-  };
 }
 async function defaultDownload(url, signal) {
   const response = await fetch(url, { method: "GET", redirect: "follow", signal });
-  if (!response.ok) {
+  if (!response.ok || !response.body) {
+    await response.body?.cancel();
     throw new ToolError(
       "PIKG_DOWNLOAD_FAILED",
       `PIKG download returned HTTP ${response.status}`,
@@ -6999,46 +7083,84 @@ async function defaultDownload(url, signal) {
       response.status >= 500
     );
   }
-  return new Uint8Array(await response.arrayBuffer());
+  return response.body;
 }
 async function stagePikg(ctx, snapshot, purpose) {
+  const identity = { pikg_digest: snapshot.digest, size: snapshot.size, purpose };
+  if (snapshot.localFileId) {
+    try {
+      const value2 = expectObject$1(
+        await callControl(ctx, "apps.staging.finalize", {
+          ...identity,
+          local_file_id: snapshot.localFileId
+        }),
+        "apps.staging.finalize response"
+      );
+      return parseStagingMetadata(value2);
+    } catch (error) {
+      if (!errorMessage(error).includes("LOCAL_PIKG_UNAVAILABLE"))
+        throw error;
+    }
+  }
   const session = ctx.session;
   if (!session)
     throw new ToolError("AUTH_REQUIRED", "authenticated session is required", 3);
   const token = (await session.ensureValid()).token;
-  const hash = ndn.sha256Bytes(snapshot.bytes);
-  const chunkId = ndn.ChunkId.fromMix256Result(snapshot.size, hash).toString();
   const proxy = ndm_proxy.createNdmProxyClient({
     endpoint: gatewayOrigin(ctx.connection.endpoint),
     sessionToken: token,
     fetcher: (request, init) => fetch(normalizeNdmRequestUrl(request), { ...init, signal: ctx.signal })
   });
-  let uploadError;
-  try {
-    await proxy.putChunk(chunkId, snapshot.bytes);
-  } catch (error) {
-    uploadError = error;
-  }
-  let value;
+  let fileObjectId;
   try {
-    value = expectObject$1(
-      await callControl(ctx, "apps.staging.finalize", {
-        source_obj_id: chunkId,
-        purpose
-      }),
-      "apps.staging.finalize response"
-    );
+    const file = await getHost().open(snapshot.path, { read: true });
+    const chunks = new ndn.SimpleChunkList();
+    const hash = getHost().createHash("sha256");
+    try {
+      for await (const bytes of fileChunks(file, 32 * 1024 * 1024)) {
+        ctx.signal.throwIfAborted();
+        hash.update(bytes);
+        const chunk = ndn.ChunkId.fromMix256Result(bytes.length, ndn.sha256Bytes(bytes));
+        await proxy.putChunk(chunk.toString(), bytes);
+        chunks.appendChunk(chunk);
+      }
+    } finally {
+      await file.close();
+    }
+    if (chunks.total_size !== snapshot.size || hash.digestHex() !== snapshot.digest) {
+      throw new ToolError("PIKG_DIGEST_MISMATCH", "PIKG snapshot changed during upload");
+    }
+    let content = chunks.body[0].toString();
+    if (chunks.body.length > 1) {
+      const [id2, body2] = chunks.genObjId();
+      await proxy.putObject({ obj_id: id2.toString(), obj_data: body2 });
+      content = id2.toString();
+    }
+    const fileObject = new ndn.FileObject(`${snapshot.digest}.pikg`, snapshot.size, content);
+    const [id, body] = fileObject.genObjId();
+    await proxy.putObject({ obj_id: id.toString(), obj_data: body });
+    fileObjectId = id.toString();
   } catch (error) {
-    if (uploadError === void 0)
+    if (error instanceof ToolError)
       throw error;
+    ctx.signal.throwIfAborted();
     throw new ToolError(
       "PIKG_UPLOAD_FAILED",
-      `failed to upload PIKG snapshot: ${errorMessage(uploadError)}`,
+      `failed to upload PIKG: ${errorMessage(error)}`,
       5,
-      true,
-      { finalize_error: errorMessage(error) }
+      true
     );
   }
+  const value = expectObject$1(
+    await callControl(ctx, "apps.staging.finalize", {
+      ...identity,
+      source_obj_id: fileObjectId
+    }),
+    "apps.staging.finalize response"
+  );
+  return parseStagingMetadata(value);
+}
+function parseStagingMetadata(value) {
   return {
     schema_version: expectNumber(value, "schema_version"),
     handle: expectString$1(value, "handle"),
@@ -7522,11 +7644,6 @@ function validatePikgSnapshot(bytes) {
     throw new UsageError("INVALID_PIKG", "PIKG magic mismatch: expected a ZIP container");
   }
 }
-async function sha256Id(bytes) {
-  const digestInput = Uint8Array.from(bytes);
-  const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", digestInput.buffer));
-  return [...digest].map((byte) => byte.toString(16).padStart(2, "0")).join("");
-}
 function gatewayOrigin(endpoint) {
   const url = new URL(endpoint);
   const marker = url.pathname.indexOf("/kapi");