@cirrobio/sdk 0.2.4 → 0.2.5

package/dist/file/util/s3-client.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createS3Client = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+/**
+ * Creates an S3 client using the provided credentials.
+ */
+function createS3Client(credentials) {
+    return new client_s3_1.S3Client({
+        credentials: {
+            accessKeyId: credentials.accessKeyId,
+            secretAccessKey: credentials.secretAccessKey,
+            sessionToken: credentials.sessionToken,
+        },
+        region: credentials.region,
+        useDualstackEndpoint: true,
+    });
+}
+exports.createS3Client = createS3Client;

package/dist/file/util/s3-utils.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Converts an S3 URI to a bucket and key
+ */
+export declare function s3UriToParams(uri: string): {
+    Bucket: string;
+    Key: string;
+};

package/dist/file/util/s3-utils.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.s3UriToParams = void 0;
+/**
+ * Converts an S3 URI to a bucket and key
+ */
+function s3UriToParams(uri) {
+    const matches = /^s3:\/\/([^/]+)\/(.+)$/.exec(uri);
+    if (!matches) {
+        throw new Error(`Received invalid uri: '${uri}'`);
+    }
+    return {
+        Bucket: matches[1],
+        Key: matches[2],
+    };
+}
+exports.s3UriToParams = s3UriToParams;

package/dist/file.d.ts

@@ -0,0 +1,11 @@
+export * from './file/extensions.fn';
+export { FileService } from './file/file.service';
+export { deleteFile, DeleteFileParams } from './file/actions/delete.fn';
+export { getSignedUrl, GetSignedUrlOptions } from './file/actions/sign-url.fn';
+export { uploadFile, UploadFileParams } from './file/actions/upload.fn';
+export { ProjectFileAccessContext } from './file/project-access-context';
+export { FileSystemObject, FileSystemObjectType } from './file/file-object.model';
+export { s3UriToParams } from './file/util/s3-utils';
+export { createS3Client } from './file/util/s3-client';
+export { getProjectS3Bucket } from './file/shared';
+export { Progress, Upload } from '@aws-sdk/lib-storage';

package/dist/file.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Upload = exports.getProjectS3Bucket = exports.createS3Client = exports.s3UriToParams = exports.FileSystemObjectType = exports.ProjectFileAccessContext = exports.uploadFile = exports.getSignedUrl = exports.deleteFile = exports.FileService = void 0;
+__exportStar(require("./file/extensions.fn"), exports);
+var file_service_1 = require("./file/file.service");
+Object.defineProperty(exports, "FileService", { enumerable: true, get: function () { return file_service_1.FileService; } });
+var delete_fn_1 = require("./file/actions/delete.fn");
+Object.defineProperty(exports, "deleteFile", { enumerable: true, get: function () { return delete_fn_1.deleteFile; } });
+var sign_url_fn_1 = require("./file/actions/sign-url.fn");
+Object.defineProperty(exports, "getSignedUrl", { enumerable: true, get: function () { return sign_url_fn_1.getSignedUrl; } });
+var upload_fn_1 = require("./file/actions/upload.fn");
+Object.defineProperty(exports, "uploadFile", { enumerable: true, get: function () { return upload_fn_1.uploadFile; } });
+var project_access_context_1 = require("./file/project-access-context");
+Object.defineProperty(exports, "ProjectFileAccessContext", { enumerable: true, get: function () { return project_access_context_1.ProjectFileAccessContext; } });
+var file_object_model_1 = require("./file/file-object.model");
+Object.defineProperty(exports, "FileSystemObjectType", { enumerable: true, get: function () { return file_object_model_1.FileSystemObjectType; } });
+var s3_utils_1 = require("./file/util/s3-utils");
+Object.defineProperty(exports, "s3UriToParams", { enumerable: true, get: function () { return s3_utils_1.s3UriToParams; } });
+var s3_client_1 = require("./file/util/s3-client");
+Object.defineProperty(exports, "createS3Client", { enumerable: true, get: function () { return s3_client_1.createS3Client; } });
+var shared_1 = require("./file/shared");
+Object.defineProperty(exports, "getProjectS3Bucket", { enumerable: true, get: function () { return shared_1.getProjectS3Bucket; } });
+var lib_storage_1 = require("@aws-sdk/lib-storage");
+Object.defineProperty(exports, "Upload", { enumerable: true, get: function () { return lib_storage_1.Upload; } });

package/dist/index.d.ts

@@ -1,2 +1,2 @@
+export * from './file';
 export * from './api';
-export * from './extensions.fn';

package/dist/index.js

@@ -14,5 +14,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./file"), exports);
 __exportStar(require("./api"), exports);
-__exportStar(require("./extensions.fn"), exports);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cirrobio/sdk",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "description": "SDK for Cirro",
   "author": "CirroBio",
   "repository": {
@@ -16,10 +16,23 @@
     "build": "tsc",
     "prepare": "npm run build"
   },
+  "jest": {
+    "coverageReporters": [
+      "lcov"
+    ],
+    "preset": "ts-jest"
+  },
   "devDependencies": {
-    "typescript": "^4.0"
+    "typescript": "^4.0",
+    "@types/jest": "^29.5.5",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/node": "^22.13.1"
   },
   "dependencies": {
-    "@cirrobio/api-client": "^0.1"
+    "@cirrobio/api-client": "^0.2.4",
+    "@aws-sdk/client-s3": "^3.34.0",
+    "@aws-sdk/lib-storage": "^3.34.0",
+    "@aws-sdk/s3-request-presigner": "^3.34.0"
   }
 }

package/src/api/error-handler.ts

@@ -0,0 +1,36 @@
+import { Middleware, ResponseContext } from "@cirrobio/api-client";
+
+export class ApiError extends Error {
+  errors: string[];
+
+  constructor(message: string, errors: string[]) {
+    super(message);
+    this.errors = errors;
+  }
+}
+
+export class PortalErrorHandler implements Middleware {
+  async post(context: ResponseContext): Promise<Response | void> {
+    const { response } = context;
+    if (response && (response.status >= 200 && response.status < 300)) {
+      return response;
+    }
+
+    // Handle Error
+    let errorMessage: string;
+    const errors = [];
+    try {
+      const err = await response.json();
+      console.warn(err);
+      if ('errorDetail' in err) {
+        errorMessage = err.errorDetail;
+        errors.push(err.errors.map((e: any) => e.message));
+      } else {
+        errorMessage = err.message;
+      }
+    } catch (ignore) {
+      errorMessage = "Unknown Error";
+    }
+    throw new ApiError(errorMessage, errors);
+  }
+}

package/src/api.ts

@@ -1,36 +1 @@
-import { Middleware, ResponseContext } from "@cirrobio/api-client";
-
-export class ApiError extends Error {
-  errors: string[];
-
-  constructor(message: string, errors: string[]) {
-    super(message);
-    this.errors = errors;
-  }
-}
-
-export class PortalErrorHandler implements Middleware {
-  async post(context: ResponseContext): Promise<Response | void> {
-    const { response } = context;
-    if (response && (response.status >= 200 && response.status < 300)) {
-      return response;
-    }
-
-    // Handle Error
-    let errorMessage: string;
-    const errors = [];
-    try {
-      const err = await response.json();
-      console.warn(err);
-      if ('errorDetail' in err) {
-        errorMessage = err.errorDetail;
-        errors.push(err.errors.map((e: any) => e.message));
-      } else {
-        errorMessage = err.message;
-      }
-    } catch (ignore) {
-      errorMessage = "Unknown Error";
-    }
-    throw new ApiError(errorMessage, errors);
-  }
-}
+export { PortalErrorHandler, ApiError } from './api/error-handler';

package/src/file/__test__/s3-utils.test.ts

@@ -0,0 +1,17 @@
+import { s3UriToParams } from "../../file";
+
+
+describe("s3-utils", () => {
+  describe("s3UriToParams", () => {
+    it("should parse a valid s3 URI", () => {
+      const uri = "s3://my-bucket/my-key/nested/folder";
+      const result = s3UriToParams(uri);
+      expect(result).toEqual({ Bucket: "my-bucket", Key: "my-key/nested/folder" });
+    });
+
+    it("should throw an error for an invalid URI", () => {
+      const uri = "invalid-uri";
+      expect(() => s3UriToParams(uri)).toThrowError(`Received invalid uri: '${uri}'`);
+    });
+  });
+})

package/src/file/actions/delete.fn.ts

@@ -0,0 +1,18 @@
+import { AWSCredentials } from "@cirrobio/api-client";
+import { DeleteObjectCommand } from "@aws-sdk/client-s3";
+import { s3UriToParams } from "../util/s3-utils";
+import { createS3Client } from "../util/s3-client";
+
+export interface DeleteFileParams {
+  url: string;
+  credentials: AWSCredentials;
+}
+
+/**
+ * Delete a file from S3 given its URL and credentials.
+ */
+export async function deleteFile({ url, credentials }: DeleteFileParams): Promise<void> {
+  const { Bucket, Key } = s3UriToParams(url);
+  const cmd = new DeleteObjectCommand({ Bucket, Key });
+  await createS3Client(credentials).send(cmd);
+}

package/src/file/actions/sign-url.fn.ts

@@ -0,0 +1,35 @@
+import { AWSCredentials } from "@cirrobio/api-client";
+import { GetObjectCommand, GetObjectCommandInput } from "@aws-sdk/client-s3";
+import { getSignedUrl as getSignedUrlInternal } from "@aws-sdk/s3-request-presigner";
+import { createS3Client } from "../util/s3-client";
+import { s3UriToParams } from "../util/s3-utils";
+
+export interface GetFileUrlParams extends GetSignedUrlOptions {
+  url: string;
+  credentials?: AWSCredentials;
+}
+
+export interface GetSignedUrlOptions {
+  download?: boolean;
+  gzip?: boolean;
+  timeout?: number;
+  filename?: string;
+}
+
+/**
+ * Get a signed URL for a file in S3 given its S3 URI.
+ */
+export function getSignedUrl({ url, credentials, ...params }: GetFileUrlParams): Promise<string> {
+  const client = createS3Client(credentials);
+  const { Bucket, Key } = s3UriToParams(url);
+  const fileName = params.filename ?? Key.split('/').pop();
+  const args: GetObjectCommandInput = { Bucket, Key };
+  if (params?.download) {
+    args.ResponseContentDisposition = `attachment; filename="${fileName}"`
+  }
+  if (params?.gzip) {
+    args.ResponseContentEncoding = 'gzip'
+  }
+  const command = new GetObjectCommand(args);
+  return getSignedUrlInternal(client, command, { expiresIn: 60 * (params?.timeout ?? 5) });
+}

package/src/file/actions/upload.fn.ts

@@ -0,0 +1,30 @@
+import { Upload } from "@aws-sdk/lib-storage";
+import { AWSCredentials } from '@cirrobio/api-client';
+import { createS3Client } from "../util/s3-client";
+
+export interface UploadFileParams {
+  bucket: string;
+  path: string;
+  file: File;
+  credentials: AWSCredentials;
+  metadata?: Record<string, string>;
+}
+
+/**
+ * Upload a file to S3
+ */
+export function uploadFile({ bucket, path, file, credentials, metadata }: UploadFileParams): Upload {
+  const params = {
+    Bucket: bucket,
+    Key: path,
+    Body: file,
+    ContentType: file.type,
+    Metadata: metadata,
+  };
+  return new Upload({
+    client: createS3Client(credentials),
+    queueSize: 4,
+    params,
+  });
+}
+

package/src/{extensions.fn.ts → file/extensions.fn.ts}

@@ -2,7 +2,6 @@
 /**
  * An array of file extensions that can be rendered in a genome viewer
  */
-
 export const FILE_TRACK_ANNOTATION = ['bed', 'bed.gz', 'gtf', 'gtf.gz'];
 export const FILE_TRACK_ALIGNMENTS = ['cram', 'cram.gz'];  // TODO: put back bam
 export const FILE_TRACK_VARIANT = ['vcf', 'vcf.gz'];
@@ -86,4 +85,4 @@ export function matchesExtension(filePath: string, extensions: string[]): boolea
 
     // Check if the fileExtension is in the list of extensions
     return extensions.includes(fileExtension);
-}
+}

package/src/file/file-object.model.ts

@@ -0,0 +1,57 @@
+import { ProjectFileAccessContext } from "./project-access-context";
+
+export enum FileSystemObjectType {
+  FILE = 'file',
+  FOLDER = 'folder'
+}
+
+/**
+ * Represents a file that can be downloaded
+ */
+export interface DownloadableFile {
+  url: string;
+  name?: string;
+  fileAccessContext: ProjectFileAccessContext;
+}
+
+/**
+ * Represents a file in Cirro
+ */
+export interface FileSystemObject extends DownloadableFile {
+  /**
+   * Unique Id For Object
+   */
+  id: string;
+  /**
+   * Object S3 Url
+   */
+  url: string;
+  /**
+   * Object Path
+   */
+  path: string;
+  /**
+   * Object Name
+   */
+  name: string;
+  /**
+   * Object Size
+   */
+  size: number;
+  /**
+   * Object Kind (PNG, TXT)
+   */
+  kind: string;
+  /**
+   * Object Type (File or Folder)
+   */
+  type: FileSystemObjectType;
+  /**
+   * Metadata
+   */
+  metadata?: Record<string, any>;
+  /**
+   * Access context
+   */
+  fileAccessContext: ProjectFileAccessContext;
+}

package/src/file/file.service.ts

@@ -0,0 +1,79 @@
+import { AccessType, AWSCredentials, FileApi } from "@cirrobio/api-client";
+import { ProjectFileAccessContext } from "./project-access-context";
+import { DownloadableFile } from "./file-object.model";
+import { credentialsCache, credentialsMutex } from "./util/credentials-mutex.so";
+import { GetSignedUrlOptions, GetFileUrlParams, getSignedUrl } from "./actions/sign-url.fn";
+import { getProjectS3Bucket } from "./shared";
+
+/**
+ * Service for viewing files in Cirro
+ *  currently this only operates on files within a project
+ */
+export class FileService {
+  constructor(
+    private readonly fileApi: FileApi
+  ) {}
+
+  /**
+   * Get contents of a file
+   */
+  async getProjectFile(file: DownloadableFile): Promise<Response> {
+    const url = await this.getSignedUrlFromProjectFile(file);
+    return fetch(url);
+  }
+
+  /**
+   * Get a signed URL for a file
+   */
+  async getSignedUrlFromProjectFile(file: DownloadableFile, params?: GetSignedUrlOptions): Promise<string> {
+    const credentials = await this.getProjectAccessCredentials(file.fileAccessContext);
+    const _params: GetFileUrlParams = {
+      ...params,
+      filename: file.name,
+      url: file.url,
+      credentials,
+    };
+    return getSignedUrl(_params);
+  }
+
+  /**
+   * Get a signed URL for a file given a path
+   */
+  async getSignedUrlFromProjectPath(fileAccessContext: ProjectFileAccessContext, path: string, params?: GetFileUrlParams): Promise<string> {
+    const credentials = await this.getProjectAccessCredentials(fileAccessContext);
+    const _params: GetFileUrlParams = {
+      ...params,
+      filename: path.split('/').pop(),
+      url: `s3://${getProjectS3Bucket(fileAccessContext.project.id)}/${path}`,
+      credentials,
+    };
+    return getSignedUrl(_params);
+  }
+
+  /**
+   * Get credentials for accessing a project file
+   */
+  async getProjectAccessCredentials(fileAccessContext: ProjectFileAccessContext): Promise<AWSCredentials> {
+    // Special case for project download, since we can cache the credentials
+    if (fileAccessContext.fileAccessRequest.accessType === AccessType.ProjectDownload) {
+      return this.getProjectReadCredentials(fileAccessContext);
+    }
+    return this.fileApi.generateProjectFileAccessToken({ projectId: fileAccessContext.project.id, fileAccessRequest: fileAccessContext.fileAccessRequest });
+  }
+
+  private async getProjectReadCredentials(fileAccessContext: ProjectFileAccessContext): Promise<AWSCredentials> {
+    const projectId = fileAccessContext.project.id;
+    return credentialsMutex.dispatch(async () => {
+      const cachedCredentials = credentialsCache.get(projectId);
+      const expirationTime = cachedCredentials ? cachedCredentials?.expiration : null;
+      const fetchNewCredentials = !expirationTime || expirationTime < new Date();
+      if (fetchNewCredentials) {
+        const fileAccessRequest = fileAccessContext.fileAccessRequest;
+        const credentials = await this.fileApi.generateProjectFileAccessToken({ projectId, fileAccessRequest });
+        credentialsCache.set(projectId, credentials);
+        return credentials;
+      }
+      return cachedCredentials;
+    });
+  }
+}

package/src/file/project-access-context.ts

@@ -0,0 +1,26 @@
+import { AccessType, DatasetDetail, FileAccessRequest, Project } from "@cirrobio/api-client";
+
+type ProjectIdentifiable = Pick<Project, 'id'>;
+
+/**
+ * Helper class to encapsulate the file access for a project.
+ */
+export class ProjectFileAccessContext {
+  constructor(
+    readonly project: ProjectIdentifiable,
+    readonly dataset: DatasetDetail,
+    readonly fileAccessRequest: FileAccessRequest
+  ) {
+  }
+
+  static projectDownload(project: ProjectIdentifiable): ProjectFileAccessContext {
+    const request: FileAccessRequest = { accessType: AccessType.ProjectDownload };
+    return new ProjectFileAccessContext(project, null, request);
+  }
+
+  static datasetDownload(project: ProjectIdentifiable, dataset: DatasetDetail): ProjectFileAccessContext {
+    const accessType = dataset.share ? AccessType.SharedDatasetDownload : AccessType.ProjectDownload;
+    const request: FileAccessRequest = { accessType, datasetId: dataset.id };
+    return new ProjectFileAccessContext(project, dataset, request);
+  }
+}

package/src/file/shared.ts

@@ -0,0 +1,7 @@
+/**
+ * Get the S3 bucket name for a project.
+ * Will be deprecated in the future.
+ */
+export function getProjectS3Bucket(projectId: string): string {
+  return `project-${projectId}`;
+}

package/src/file/util/credentials-mutex.so.ts

@@ -0,0 +1,33 @@
+import { AWSCredentials } from '@cirrobio/api-client';
+
+class Mutex<T> {
+  private mutex = Promise.resolve();
+
+  lock(): PromiseLike<() => void> {
+    let begin: (unlock: () => void) => void = () => { /* Do nothing */ };
+
+    this.mutex = this.mutex.then(() => new Promise(begin));
+
+    return new Promise((res) => {
+      begin = res;
+    });
+  }
+
+  async dispatch(fn: (() => T) | (() => PromiseLike<T>)): Promise<T> {
+    const unlock = await this.lock();
+    try {
+      return await Promise.resolve(fn());
+    } finally {
+      unlock();
+    }
+  }
+}
+
+/**
+ * A mutex to ensure that only one request for credentials is made at a time.
+ */
+export const credentialsMutex = new Mutex<AWSCredentials>();
+/**
+ * A cache of credentials to avoid making multiple requests for the same credentials.
+ */
+export const credentialsCache: Map<string, AWSCredentials> = new Map<string, AWSCredentials>();

package/src/file/util/s3-client.ts

@@ -0,0 +1,17 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import { AWSCredentials } from "@cirrobio/api-client";
+
+/**
+ * Creates an S3 client using the provided credentials.
+ */
+export function createS3Client(credentials: AWSCredentials): S3Client {
+  return new S3Client({
+    credentials: {
+      accessKeyId: credentials.accessKeyId,
+      secretAccessKey: credentials.secretAccessKey,
+      sessionToken: credentials.sessionToken,
+    },
+    region: credentials.region,
+    useDualstackEndpoint: true,
+  });
+}

package/src/file/util/s3-utils.ts

@@ -0,0 +1,14 @@
+/**
+ * Converts an S3 URI to a bucket and key
+ */
+export function s3UriToParams(uri: string): { Bucket: string, Key: string } {
+  const matches = /^s3:\/\/([^/]+)\/(.+)$/.exec(uri);
+  if (!matches) {
+    throw new Error(`Received invalid uri: '${uri}'`);
+  }
+  return {
+    Bucket: matches[1],
+    Key: matches[2],
+  };
+}
+

package/src/file.ts

@@ -0,0 +1,11 @@
+export * from './file/extensions.fn'
+export { FileService } from './file/file.service'
+export { deleteFile, DeleteFileParams } from './file/actions/delete.fn'
+export { getSignedUrl, GetSignedUrlOptions } from './file/actions/sign-url.fn'
+export { uploadFile, UploadFileParams } from './file/actions/upload.fn'
+export { ProjectFileAccessContext } from './file/project-access-context'
+export { FileSystemObject, FileSystemObjectType } from './file/file-object.model'
+export { s3UriToParams } from './file/util/s3-utils'
+export { createS3Client } from './file/util/s3-client'
+export { getProjectS3Bucket } from './file/shared'
+export { Progress, Upload } from '@aws-sdk/lib-storage';

package/src/index.ts

@@ -1,2 +1,2 @@
-export * from './api'
-export * from './extensions.fn'
+export * from './file';
+export * from './api';

package/tsconfig.json

@@ -1,12 +1,12 @@
 {
   "compilerOptions": {
     "declaration": true,
-    "target": "es5",
+    "target": "es2018",
     "module": "commonjs",
     "moduleResolution": "node",
     "outDir": "dist",
     "lib": [
-      "es2016",
+      "es2018",
       "dom"
     ],
     "typeRoots": [
@@ -17,4 +17,4 @@
     "dist",
     "node_modules"
   ]
-}
+}