@procedures/s3 0.47.2 → 0.48.1

package/src/module.ts

@@ -21,7 +21,7 @@ import {
   s3MultipartAbort,
 } from "./procedures/s3/index.js";
 import {
-  S3UploadInputSchema,
+  S3UploadCallSchema,
   S3DownloadInputSchema,
   S3ListInputSchema,
   S3ListAllInputSchema,
@@ -30,7 +30,7 @@ import {
   S3MultipartUploadInputSchema,
   S3MultipartCompleteInputSchema,
   S3MultipartAbortInputSchema,
-  type S3UploadInput,
+  type S3UploadCall,
   type S3DownloadInput,
   type S3ListInput,
   type S3ListAllInput,
@@ -76,30 +76,35 @@ import {
   type S3PresignOutput,
 } from "./types.js";
 
+// =============================================================================
+// The rules under a root of the files (roadmap R33, tracker item O36)
+// =============================================================================
+
+/** `s3.upload`: under a root, the file of `sourcePath` is in the root. A `body` names no file. */
+const UPLOAD_IN_A_ROOT: FsRootRules = { paths: ["sourcePath"] };
+
 // =============================================================================
 // Procedures
 // =============================================================================
 
 const s3UploadProcedure = createProcedure()
   .path(["s3", "upload"])
-  .input(S3UploadInputSchema)
+  .input(S3UploadCallSchema)
   .output(S3UploadOutputSchema)
   .meta({
     description: "Upload content to S3 bucket",
-    // A put replaces the object of an existing key
-    effects: { reads: ["s3", "network"], writes: ["s3", "network"] },
+    // A put replaces the object of an existing key. With sourcePath, it reads that file.
+    effects: { reads: ["s3", "network", "fs"], writes: ["s3", "network"] },
     shorts: { bucket: "b", key: "k" },
     output: "json",
+    fsRoot: UPLOAD_IN_A_ROOT,
   })
-  .handler(async (input: S3UploadInput): Promise<S3UploadOutput> => {
-    return s3Upload(input);
+  .handler(async (input): Promise<S3UploadOutput> => {
+    // The two refinements of the schema give this type: a body or a sourcePath, and not the two
+    return s3Upload(input as S3UploadCall);
   })
   .build();
 
-// =============================================================================
-// The rules under a root of the files (roadmap R33, tracker item O36)
-// =============================================================================
-
 /** `s3.download`: under a root, the file of `destPath` is in the root. Without it, the object stays in memory. */
 const DOWNLOAD_IN_A_ROOT: FsRootRules = { paths: ["destPath"] };
 

package/src/procedures/s3/upload.ts

@@ -1,12 +1,126 @@
 /**
  * s3.upload procedure
  *
- * Upload content to S3 bucket.
+ * Upload content to S3 bucket, from a string in memory or from a file.
  */
 
-import { PutObjectCommand } from "@aws-sdk/client-s3";
+import { open, type FileHandle } from "node:fs/promises";
+import type { Stats } from "node:fs";
+import {
+  AbortMultipartUploadCommand,
+  CompleteMultipartUploadCommand,
+  CreateMultipartUploadCommand,
+  PutObjectCommand,
+  UploadPartCommand,
+  type CompletedPart,
+} from "@aws-sdk/client-s3";
 import { getS3Client } from "../../s3-client.js";
-import type { S3UploadInput, S3UploadOutput } from "../../types.js";
+import type { S3UploadCall, S3UploadInput, S3UploadOutput } from "../../types.js";
+
+/** The fields of an upload that do not depend on where its content comes from. */
+type UploadTarget = Pick<S3UploadInput, "bucket" | "key" | "contentType" | "metadata">;
+
+/** The size of one part of a file, in bytes. A file of this size or less goes in one put. */
+export const PART_BYTES: number = 8 * 1024 * 1024;
+
+/** The largest number of parts that S3 takes in one multipart upload. */
+export const MOST_PARTS = 10_000;
+
+/**
+ * This function gives the size of the parts of a file. A very large file has larger parts, because
+ * S3 takes at most `MOST_PARTS` parts. The peak memory of an upload is one part of this size.
+ */
+export function partBytesOf(size: number): number {
+  return Math.max(PART_BYTES, Math.ceil(size / MOST_PARTS));
+}
+
+/** This function gives the error of a file that changed after the upload started. */
+function changed(sourcePath: string): Error {
+  return new Error(`The file ${sourcePath} changed during the upload, so the object is not complete. Upload the file again.`);
+}
+
+/** This function fills the start of `buffer` from the file. A short file fails, because a short part is not the file. */
+async function readPart(file: FileHandle, buffer: Buffer, length: number, position: number, sourcePath: string): Promise<void> {
+  let done = 0;
+  while (done < length) {
+    const { bytesRead } = await file.read(buffer, done, length - done, position + done);
+    if (bytesRead === 0) throw changed(sourcePath);
+    done += bytesRead;
+  }
+}
+
+/** This function fails when the size or the time of the file is not the same as at the start. */
+async function assertSame(file: FileHandle, start: Stats, sourcePath: string): Promise<void> {
+  const now = await file.stat();
+  if (now.size !== start.size || now.mtimeMs !== start.mtimeMs) throw changed(sourcePath);
+}
+
+/**
+ * This function sends the file part by part, one part at a time, with one buffer. On a failure it
+ * aborts the upload, so S3 keeps no object and no parts. The first error stays the error of the
+ * call, also when the abort fails.
+ */
+async function uploadParts(file: FileHandle, start: Stats, partBytes: number, input: UploadTarget, sourcePath: string): Promise<S3UploadOutput> {
+  const client = getS3Client();
+  const target = { Bucket: input.bucket, Key: input.key };
+  const created = await client.send(
+    new CreateMultipartUploadCommand({ ...target, ContentType: input.contentType, Metadata: input.metadata }),
+  );
+  const UploadId = created.UploadId;
+  if (UploadId === undefined) throw new Error(`S3 gave no upload id for ${input.key}`);
+  try {
+    const buffer = Buffer.allocUnsafeSlow(partBytes);
+    const parts: CompletedPart[] = [];
+    for (let position = 0; position < start.size; position += partBytes) {
+      const length = Math.min(partBytes, start.size - position);
+      await readPart(file, buffer, length, position, sourcePath);
+      const PartNumber = parts.length + 1;
+      const sent = await client.send(new UploadPartCommand({ ...target, UploadId, PartNumber, Body: buffer.subarray(0, length) }));
+      parts.push({ ETag: sent.ETag, PartNumber });
+    }
+    await assertSame(file, start, sourcePath);
+    const done = await client.send(new CompleteMultipartUploadCommand({ ...target, UploadId, MultipartUpload: { Parts: parts } }));
+    return { etag: done.ETag?.replace(/"/g, "") || "", versionId: done.VersionId, location: `s3://${input.bucket}/${input.key}` };
+  } catch (error) {
+    await client.send(new AbortMultipartUploadCommand({ ...target, UploadId })).catch(() => undefined);
+    throw error;
+  }
+}
+
+/** This function uploads a file: in one put up to the size of one part, and in parts above it. */
+async function uploadFile(input: UploadTarget, sourcePath: string): Promise<S3UploadOutput> {
+  const file = await open(sourcePath, "r");
+  try {
+    const start = await file.stat();
+    if (!start.isFile()) throw new Error(`${sourcePath} is not a file`);
+    const partBytes = partBytesOf(start.size);
+    if (start.size > partBytes) return await uploadParts(file, start, partBytes, input, sourcePath);
+    const body = Buffer.allocUnsafeSlow(start.size);
+    await readPart(file, body, start.size, 0, sourcePath);
+    await assertSame(file, start, sourcePath);
+    return await put(input, body);
+  } finally {
+    await file.close();
+  }
+}
+
+/** This function sends one put. */
+async function put(input: UploadTarget, body: Buffer | string): Promise<S3UploadOutput> {
+  const response = await getS3Client().send(
+    new PutObjectCommand({
+      Bucket: input.bucket,
+      Key: input.key,
+      Body: body,
+      ContentType: input.contentType,
+      Metadata: input.metadata,
+    }),
+  );
+  return {
+    etag: response.ETag?.replace(/"/g, "") || "",
+    versionId: response.VersionId,
+    location: `s3://${input.bucket}/${input.key}`,
+  };
+}
 
 /**
  * Upload content to S3
@@ -29,28 +143,20 @@ import type { S3UploadInput, S3UploadOutput } from "../../types.js";
  *   base64: true,
  *   contentType: "image/png",
  * });
+ *
+ * @example
+ * // Upload a large file in bounded memory: the object is whole or absent
+ * await client.call(["s3", "upload"], {
+ *   bucket: "my-bucket",
+ *   key: "backups/archive.tar.gz",
+ *   sourcePath: "/tmp/archive.tar.gz",
+ * });
  */
-export async function s3Upload(input: S3UploadInput): Promise<S3UploadOutput> {
-  const client = getS3Client();
-
-  // Convert body to Buffer if base64
-  const body = input.base64
-    ? Buffer.from(input.body, "base64")
-    : input.body;
-
-  const command = new PutObjectCommand({
-    Bucket: input.bucket,
-    Key: input.key,
-    Body: body,
-    ContentType: input.contentType,
-    Metadata: input.metadata,
-  });
-
-  const response = await client.send(command);
-
-  return {
-    etag: response.ETag?.replace(/"/g, "") || "",
-    versionId: response.VersionId,
-    location: `s3://${input.bucket}/${input.key}`,
-  };
+export async function s3Upload(input: S3UploadCall): Promise<S3UploadOutput> {
+  if (input.sourcePath !== undefined) {
+    if (input.body !== undefined) throw new Error("Give body or sourcePath, and not the two");
+    return uploadFile(input, input.sourcePath);
+  }
+  if (input.body === undefined) throw new Error("Give body or sourcePath, and not the two");
+  return put(input, input.base64 ? Buffer.from(input.body, "base64") : input.body);
 }

package/src/types.ts

@@ -59,6 +59,58 @@ export const S3UploadInputSchema: z.ZodObject<{
 
 export type S3UploadInput = z.infer<typeof S3UploadInputSchema>;
 
+/** The input of an upload from a file. It refuses `body` and `base64`. */
+export const S3UploadFileInputSchema: z.ZodObject<{
+  bucket: z.ZodString;
+  key: z.ZodString;
+  sourcePath: z.ZodString;
+  contentType: z.ZodOptional<z.ZodString>;
+  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+  body: z.ZodOptional<z.ZodUndefined>;
+  base64: z.ZodOptional<z.ZodUndefined>;
+}> = z.object({
+  bucket: S3UploadInputSchema.shape.bucket,
+  key: S3UploadInputSchema.shape.key,
+  /** The path of a file to upload. The upload reads one part at a time, and the object is whole or absent. */
+  sourcePath: z.string().min(1).describe("The path of a file to upload. The upload reads one part at a time, and the object is whole or absent."),
+  contentType: S3UploadInputSchema.shape.contentType,
+  metadata: S3UploadInputSchema.shape.metadata,
+  body: z.undefined().optional(),
+  base64: z.undefined().optional(),
+});
+
+export type S3UploadFileInput = z.infer<typeof S3UploadFileInputSchema>;
+
+/** A call of `s3.upload`: a call with both `body` and `sourcePath`, or with neither, fails at the type. */
+export type S3UploadCall = (S3UploadInput & { sourcePath?: undefined }) | S3UploadFileInput;
+
+/**
+ * The schema of the input of the procedure `s3.upload`: one object with two refinements.
+ *
+ * This schema is not a `z.union`, because a schema of an object removes unknown keys. A plain union
+ * reads a call with both inputs as a call with `body`, and that call sends the wrong content without an error.
+ */
+export const S3UploadCallSchema: z.ZodObject<{
+  bucket: z.ZodString;
+  key: z.ZodString;
+  body: z.ZodOptional<z.ZodString>;
+  sourcePath: z.ZodOptional<z.ZodString>;
+  contentType: z.ZodOptional<z.ZodString>;
+  base64: z.ZodOptional<z.ZodBoolean>;
+  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+}> = z
+  .object({
+    bucket: S3UploadInputSchema.shape.bucket,
+    key: S3UploadInputSchema.shape.key,
+    body: z.string().optional().describe("Content to upload - string or base64 encoded binary. Give body or sourcePath, and not the two."),
+    sourcePath: z.string().min(1).optional().describe("The path of a file to upload. The upload reads one part at a time, and the object is whole or absent."),
+    contentType: S3UploadInputSchema.shape.contentType,
+    base64: S3UploadInputSchema.shape.base64,
+    metadata: S3UploadInputSchema.shape.metadata,
+  })
+  .refine((input) => (input.body !== undefined) !== (input.sourcePath !== undefined), { message: "Give body or sourcePath, and not the two" })
+  .refine((input) => input.base64 !== true || input.body !== undefined, { message: "base64 applies only to body" });
+
 export interface S3UploadOutput {
   /** ETag of uploaded object */
   etag: string;