@cirrobio/sdk 0.12.11 → 0.12.12

package/src/file/file.service.ts

@@ -0,0 +1,90 @@
+import { AWSCredentials, GenerateProjectFileAccessTokenRequest, ProjectAccessType } from "@cirrobio/api-client";
+import { ProjectFileAccessContext } from "./project-access-context";
+import { credentialsCache, credentialsMutex } from "./util/credentials-mutex.so";
+import { GetFileUrlParams, getSignedUrl, GetSignedUrlOptions } from "./actions/sign-url.fn";
+import { getProjectS3Bucket } from "./shared";
+import { DownloadableFile } from "./models/file-object.model";
+
+export interface IFileCredentialsApi {
+  generateProjectFileAccessToken: (params: GenerateProjectFileAccessTokenRequest) => Promise<AWSCredentials>;
+}
+
+/**
+ * Service for viewing files in Cirro
+ *  currently this only operates on files within a project
+ */
+export class FileService {
+  constructor(
+    private readonly fileCredsApi: IFileCredentialsApi
+  ) {}
+
+  /**
+   * Get contents of a file
+   */
+  async getProjectFile(file: DownloadableFile, params?: GetSignedUrlOptions): Promise<Response> {
+    const url = await this.getSignedUrlFromProjectFile(file, params);
+    return fetch(url);
+  }
+
+  /**
+   * Get a signed URL for a file
+   */
+  async getSignedUrlFromProjectFile(file: DownloadableFile, params?: GetSignedUrlOptions): Promise<string> {
+    const credentials = await this.getProjectAccessCredentials(file.fileAccessContext);
+    const _params: GetFileUrlParams = {
+      ...params,
+      filename: file.name,
+      url: file.url,
+      credentials,
+    };
+    return getSignedUrl(_params);
+  }
+
+  /**
+   * Get a signed URL for a file given a path
+   */
+  async getSignedUrlFromProjectPath(fileAccessContext: ProjectFileAccessContext, path: string, params?: GetSignedUrlOptions): Promise<string> {
+    const credentials = await this.getProjectAccessCredentials(fileAccessContext);
+    const _params: GetFileUrlParams = {
+      ...params,
+      filename: path.split('/').pop(),
+      url: `s3://${getProjectS3Bucket(fileAccessContext.project.id)}/${path}`,
+      credentials,
+    };
+    return getSignedUrl(_params);
+  }
+
+  /**
+   * Get credentials for accessing a project file
+   */
+  async getProjectAccessCredentials(fileAccessContext: ProjectFileAccessContext): Promise<AWSCredentials> {
+    const accessType = fileAccessContext.fileAccessRequest.accessType;
+    // Special case for project download, since we can cache the credentials
+    if (accessType === ProjectAccessType.ProjectDownload || accessType === ProjectAccessType.SharedDatasetDownload) {
+      return this.getProjectReadCredentials(fileAccessContext);
+    }
+    return this.fileCredsApi.generateProjectFileAccessToken({ projectId: fileAccessContext.project.id, projectFileAccessRequest: fileAccessContext.fileAccessRequest });
+  }
+
+  private async getProjectReadCredentials(fileAccessContext: ProjectFileAccessContext): Promise<AWSCredentials> {
+    const projectId = fileAccessContext.project.id;
+    // Append datasetId to cache key for shared dataset downloads since we need to generate a new token for each dataset
+    let cacheKey = projectId;
+    if (fileAccessContext.fileAccessRequest.accessType === ProjectAccessType.SharedDatasetDownload) {
+      cacheKey = `${projectId}-${fileAccessContext.fileAccessRequest.datasetId}`;
+    }
+
+    return credentialsMutex.dispatch(async () => {
+      const cachedCredentials = credentialsCache.get(cacheKey);
+      const expirationTime = cachedCredentials ? cachedCredentials?.expiration : null;
+      const fetchNewCredentials = !expirationTime || expirationTime < new Date();
+      if (fetchNewCredentials) {
+        const projectFileAccessRequest = fileAccessContext.fileAccessRequest;
+        const credentials = await this.fileCredsApi.generateProjectFileAccessToken({ projectId, projectFileAccessRequest });
+        credentialsCache.set(cacheKey, credentials);
+        return credentials;
+      }
+      return cachedCredentials;
+    });
+  }
+}

package/src/file/manifest-parser.ts

@@ -0,0 +1,63 @@
+import { DatasetAssetsManifest, FileEntry } from '@cirrobio/api-client';
+import { Assets } from "./models/assets";
+import { Folder } from "./models/folder";
+import { File } from "./models/file";
+import { ProjectFileAccessContext } from "./project-access-context";
+import { removeEndingSlash } from "../formatters";
+
+
+export class ManifestParser {
+  private readonly manifest: DatasetAssetsManifest;
+  private readonly accessContext: ProjectFileAccessContext;
+
+  constructor(manifest: DatasetAssetsManifest, accessContext?: ProjectFileAccessContext) {
+    this.manifest = manifest;
+    this.accessContext = accessContext;
+  }
+
+  private get files(): Array<FileEntry> {
+    return this.manifest?.files ?? [];
+  }
+
+  /**
+   * Generates asset objects from the manifest, optionally including folder entries.
+   */
+  public generateAssets(generateFolders = false): Assets {
+    const domain = removeEndingSlash(this.manifest?.domain);
+    const assets = new Assets();
+
+    for (const file of this.files) {
+      assets.push(new File(domain, file.path, file.size, file.metadata, this.accessContext));
+    }
+
+    // Optionally add folder entries
+    if (generateFolders) {
+      assets.push(...this.generateFolderAssets(assets, domain));
+    }
+
+    return assets;
+  }
+
+  // Create folder entries, useful for visualizing folders as items in a file browser
+  private generateFolderAssets(assets: Assets, domain: string): Folder[] {
+    const folderSet = new Set<string>();
+
+    // Collect all unique folder paths
+    for (const asset of assets) {
+      const parts = asset.path.split('/').filter(p => p.length > 0);
+      let currentPath = '';
+      for (const part of parts) {
+        currentPath = currentPath ? `${currentPath}/${part}` : part;
+        folderSet.add(currentPath);
+      }
+    }
+
+    // Create Folder objects from unique paths
+    return Array.from(folderSet)
+      .map(folderPath => {
+        const name = folderPath.substring(folderPath.lastIndexOf('/') + 1);
+        const parentPath = folderPath.includes('/') ? folderPath.substring(0, folderPath.lastIndexOf('/')) : '';
+        return new Folder(folderPath, name, parentPath, domain);
+      });
+  }
+}

package/src/file/models/assets.ts

@@ -0,0 +1,27 @@
+import { calculateTotalSize } from "../calculate-size";
+import { bytesToString } from "../../formatters";
+import { FileSystemObject } from "./file-object.model";
+
+export class Assets extends Array<FileSystemObject> {
+  private _totalSize: string;
+  private _totalSizeBytes: number;
+
+  // Total size of all files in human-readable format
+  get totalSize(): string {
+    if (!this._totalSize) {
+      this.calculateSize();
+    }
+    return this._totalSize;
+  }
+
+  get totalSizeBytes(): number {
+    if (!this._totalSizeBytes) {
+      this.calculateSize();
+    }
+    return this._totalSizeBytes;
+  }
+  private calculateSize(): void {
+    this._totalSizeBytes = calculateTotalSize(this);
+    this._totalSize = bytesToString(this._totalSizeBytes)
+  }
+}

package/src/file/models/file-object.model.ts

@@ -0,0 +1,61 @@
+import { ProjectFileAccessContext } from "../project-access-context";
+
+export enum FileSystemObjectType {
+  FILE = 'file',
+  FOLDER = 'folder'
+}
+
+/**
+ * Represents a file that can be downloaded
+ */
+export interface DownloadableFile {
+  url: string;
+  name?: string;
+  fileAccessContext: ProjectFileAccessContext;
+}
+
+/**
+ * Represents a file in Cirro
+ */
+export interface FileSystemObject extends DownloadableFile {
+  /**
+   * Unique Id For Object
+   */
+  id: string;
+  /**
+   * Object S3 Url
+   */
+  url: string;
+  /**
+   * Object Path
+   */
+  path: string;
+  /**
+   * Object Display Path
+   */
+  displayPath: string;
+  /**
+   * Object Name
+   */
+  name: string;
+  /**
+   * Object Size
+   */
+  size: number;
+  /**
+   * Object Kind (PNG, TXT)
+   */
+  kind: string;
+  /**
+   * Object Type (File or Folder)
+   */
+  type: FileSystemObjectType;
+  /**
+   * Metadata
+   */
+  metadata?: Record<string, any>;
+  /**
+   * Access context
+   */
+  fileAccessContext: ProjectFileAccessContext;
+}

package/src/file/models/file.ts

@@ -0,0 +1,43 @@
+import { FileSystemObject, FileSystemObjectType } from "./file-object.model";
+import { ProjectFileAccessContext } from "../project-access-context";
+import { v4 as uuidv4 } from 'uuid';
+import { getDisplayPath } from "../util/get-display.fn";
+
+export class File implements FileSystemObject {
+  readonly id: string;
+  readonly url: string
+  readonly size: number;
+  readonly path: string;
+  readonly name: string;
+  readonly metadata: Record<string, any>;
+  readonly fileAccessContext: ProjectFileAccessContext;
+
+  get type() {
+    return FileSystemObjectType.FILE;
+  }
+
+  get kind() {
+    return this.name.slice(this.name.lastIndexOf('.') + 1);
+  }
+
+  get displayPath() {
+    return getDisplayPath(this);
+  }
+
+  constructor(domain: string, file: string, size: number, metadata: Record<string, any>, accessContext: ProjectFileAccessContext) {
+    this.id = uuidv4();
+    this.url = `${domain}/${file}`;
+    this.size = size;
+    this.metadata = metadata;
+    this.fileAccessContext = accessContext;
+
+    const fileFolderIdx = file.lastIndexOf('/');
+    if (fileFolderIdx === -1) {
+      this.path = '';
+      this.name = file;
+    } else {
+      this.path = file.substring(0, fileFolderIdx);
+      this.name = file.substring(fileFolderIdx + 1);
+    }
+  }
+}

package/src/file/models/folder.ts

@@ -0,0 +1,27 @@
+import { v4 as uuidv4 } from 'uuid';
+import { FileSystemObject, FileSystemObjectType } from "./file-object.model";
+import { getDisplayPath } from "../util/get-display.fn";
+
+export class Folder implements FileSystemObject {
+  readonly id: string;
+  readonly url: string;
+  readonly path: string;
+  readonly name: string;
+  readonly domain: string;
+  readonly kind = 'Folder';
+  readonly fileAccessContext = null;
+  readonly size = 0;
+  readonly type = FileSystemObjectType.FOLDER;
+
+  get displayPath() {
+    return getDisplayPath(this);
+  }
+
+  constructor(url: string, name: string, path: string, domain: string) {
+    this.id = uuidv4();
+    this.url = url;
+    this.name = name;
+    this.path = path;
+    this.domain = domain;
+  }
+}

package/src/file/project-access-context.ts

@@ -0,0 +1,26 @@
+import { ProjectAccessType, DatasetDetail, ProjectFileAccessRequest, Project } from "@cirrobio/api-client";
+
+type ProjectIdentifiable = Pick<Project, 'id'>;
+
+/**
+ * Helper class to encapsulate the file access for a project.
+ */
+export class ProjectFileAccessContext {
+  constructor(
+    readonly project: ProjectIdentifiable,
+    readonly dataset: DatasetDetail,
+    readonly fileAccessRequest: ProjectFileAccessRequest
+  ) {
+  }
+
+  static projectDownload(project: ProjectIdentifiable): ProjectFileAccessContext {
+    const request: ProjectFileAccessRequest = { accessType: ProjectAccessType.ProjectDownload };
+    return new ProjectFileAccessContext(project, null, request);
+  }
+
+  static datasetDownload(project: ProjectIdentifiable, dataset: DatasetDetail): ProjectFileAccessContext {
+    const accessType = dataset.share ? ProjectAccessType.SharedDatasetDownload : ProjectAccessType.ProjectDownload;
+    const request: ProjectFileAccessRequest = { accessType, datasetId: dataset.id };
+    return new ProjectFileAccessContext(project, dataset, request);
+  }
+}

package/src/file/shared.ts

@@ -0,0 +1,9 @@
+import { getResourceName } from "../util/get-resource-name";
+
+/**
+ * Get the S3 bucket name for a project.
+ * Will be deprecated in the future, use domain from manifest instead.
+ */
+export function getProjectS3Bucket(projectId: string): string {
+  return getResourceName(projectId)
+}

package/src/file/util/credentials-mutex.so.ts

@@ -0,0 +1,33 @@
+import { AWSCredentials } from '@cirrobio/api-client';
+
+export class Mutex<T> {
+  private mutex = Promise.resolve();
+
+  lock(): PromiseLike<() => void> {
+    let begin: (unlock: () => void) => void = () => { /* Do nothing */ };
+
+    this.mutex = this.mutex.then(() => new Promise(begin));
+
+    return new Promise((res) => {
+      begin = res;
+    });
+  }
+
+  async dispatch(fn: (() => T) | (() => PromiseLike<T>)): Promise<T> {
+    const unlock = await this.lock();
+    try {
+      return await Promise.resolve(fn());
+    } finally {
+      unlock();
+    }
+  }
+}
+
+/**
+ * A mutex to ensure that only one request for credentials is made at a time.
+ */
+export const credentialsMutex = new Mutex<AWSCredentials>();
+/**
+ * A cache of credentials to avoid making multiple requests for the same credentials.
+ */
+export const credentialsCache: Map<string, AWSCredentials> = new Map<string, AWSCredentials>();

package/src/file/util/get-display.fn.ts

@@ -0,0 +1,6 @@
+import { removeStartingSlash } from "../../formatters/slash";
+import { FileSystemObject } from "../models/file-object.model";
+
+export function getDisplayPath(file: FileSystemObject, dataPrefix: string = "data/") {
+  return removeStartingSlash(`${file.path}/${file.name}`.replace(dataPrefix, ''));
+}

package/src/file/util/get-parent.fn.ts

@@ -0,0 +1,7 @@
+
+/**
+ * Returns the parent directory of a given file path.
+ */
+export function getParentPath(path: string): string {
+  return path.split("/").slice(0, -1).join("/") ?? ""
+}

package/src/file/util/s3-client.ts

@@ -0,0 +1,43 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import { AWSCredentials } from "@cirrobio/api-client";
+import { S3ClientConfigType } from "@aws-sdk/client-s3/dist-types/S3Client";
+
+const clientCache = new Map<string, S3Client>();
+
+const clientDefaults: S3ClientConfigType = {
+  // Support ipv6 endpoints
+  useDualstackEndpoint: true,
+  // Protection against not having the correct region (does not work for signed url fn)
+  followRegionRedirects: true,
+}
+
+function createS3ClientInternal(credentials: AWSCredentials, clientOverrides: S3ClientConfigType): S3Client {
+  return new S3Client({
+    credentials: {
+      accessKeyId: credentials.accessKeyId,
+      secretAccessKey: credentials.secretAccessKey,
+      sessionToken: credentials.sessionToken,
+    },
+    region: credentials.region,
+    ...clientDefaults,
+    ...clientOverrides
+  });
+}
+
+/**
+ * Creates an S3 client using the provided credentials.
+ */
+export function createS3Client(credentials: AWSCredentials, clientOverrides: S3ClientConfigType = {}): S3Client {
+  // Bypass cache when creating s3 client with overrides, since these values may be changed.
+  if (clientOverrides && Object.keys(clientOverrides).length > 0) {
+    return createS3ClientInternal(credentials, clientOverrides);
+  }
+  const cacheKey = `${credentials.accessKeyId}-${credentials.region}`;
+  const cachedClient = clientCache.get(cacheKey);
+  if (cachedClient) {
+    return cachedClient;
+  }
+  const newClient = createS3ClientInternal(credentials, clientOverrides)
+  clientCache.set(cacheKey, newClient);
+  return newClient;
+}

package/src/file/util/s3-utils.ts

@@ -0,0 +1,14 @@
+/**
+ * Converts an S3 URI to a bucket and key
+ */
+export function s3UriToParams(uri: string): { Bucket: string, Key: string } {
+  const matches = /^s3:\/\/([^/]+)\/(.+)$/.exec(uri);
+  if (!matches) {
+    throw new Error(`Received invalid uri: '${uri}'`);
+  }
+  return {
+    Bucket: matches[1],
+    Key: matches[2],
+  };
+}
+

package/src/file.ts

@@ -0,0 +1,16 @@
+export * from './file/extensions.fn'
+export { FileService, IFileCredentialsApi } from './file/file.service'
+export { deleteFile, DeleteFileParams } from './file/actions/delete.fn'
+export { getSignedUrl, GetSignedUrlOptions } from './file/actions/sign-url.fn'
+export { uploadFile, UploadFileParams } from './file/actions/upload.fn'
+export { ProjectFileAccessContext } from './file/project-access-context'
+export { FileSystemObject, FileSystemObjectType, DownloadableFile } from './file/models/file-object.model'
+export { s3UriToParams } from './file/util/s3-utils'
+export { createS3Client } from './file/util/s3-client'
+export { credentialsCache, credentialsMutex } from './file/util/credentials-mutex.so';
+export { getProjectS3Bucket } from './file/shared'
+export { calculateTotalSize } from './file/calculate-size'
+export { Progress, Upload } from '@aws-sdk/lib-storage';
+export { Assets } from './file/models/assets';
+export { ManifestParser } from './file/manifest-parser';
+export { getParentPath } from './file/util/get-parent.fn';

package/src/formatters/__tests__/formatters.spec.ts

@@ -0,0 +1,101 @@
+import { bytesToString } from "../bytes-to-string";
+import { normalizeDate } from "../normalize-date";
+import { removeEndingSlash, removeStartingSlash } from "../slash";
+import { toFriendlyName } from "../to-friendly-name";
+import { toMoney } from "../to-money";
+import { toTitleCase } from "../to-title-case";
+import { normalizeString } from "../normalize-string";
+import { toDateFormat } from "../to-date-format";
+import { toPascalCase } from "../to-pascal-case";
+
+describe('bytesToString', () => {
+  it('should convert IEC bytes to string', () => {
+    expect(bytesToString(12521, false)).toBe('12.2 KiB');
+  });
+
+  it('should convert SI bytes to string', () => {
+    expect(bytesToString(12521, true, 2)).toBe('12.52 kB');
+  });
+});
+
+describe('normalizeDate', () => {
+  it('should work on a string input', () => {
+    const dateString = '2022-12-01'
+    const output = normalizeDate(dateString).toISOString().split('T')[0]
+    expect(output).toEqual(dateString);
+  });
+
+  it('should work on a Date object input', () => {
+    const dateString = '2022-12-01'
+    const output = normalizeDate(new Date(dateString)).toISOString().split('T')[0]
+    expect(output).toEqual(dateString);
+  });
+});
+
+describe('normalizeString', () => {
+  it('should trim extra spaces', () => {
+    expect(normalizeString(' test  ')).toEqual('test');
+    expect(normalizeString(' test')).toEqual('test');
+  });
+  it('should convert blank to null', () => {
+    expect(normalizeString('   ')).toEqual(null);
+    expect(normalizeString('')).toEqual(null);
+  })
+})
+
+describe('removeStartingSlash', () => {
+  it('should remove the starting slash from a string', () => {
+    expect(removeStartingSlash('/example/path/')).toBe('example/path/');
+    expect(removeStartingSlash('./example/path/')).toBe('example/path/');
+  });
+  it('should not do anything with no starting slash', () => {
+    expect(removeStartingSlash('path/')).toBe('path/');
+  })
+});
+
+describe('removeEndingSlash', () => {
+  it('should remove the ending slash from a string', () => {
+    expect(removeEndingSlash('/example/path/')).toBe('/example/path');
+  });
+});
+
+describe('toDateFormat', () => {
+  it('should display proper date format', () => {
+    expect(toDateFormat("2023-10-01 05:00:00 PST")).toBe('10/01/23');
+  });
+  it('should display proper date format', () => {
+    expect(toDateFormat(new Date(Date.parse("2023-10-01 05:00:00 PST")))).toBe('10/01/23');
+  });
+  it('should work with null', () => {
+    expect(toDateFormat(null)).toBe(null);
+  });
+})
+
+describe('toFriendlyName', () => {
+  it('should add correct spaces to a string', () => {
+    expect(toFriendlyName('ExampleUnfriendly|Name_here')).toBe('Example unfriendly name here');
+
+  });
+});
+
+describe('toMoney', () => {
+  it('convert an integer value to a money-formatted string', () => {
+    expect(toMoney(25)).toBe('$25.00');
+  });
+
+  it('convert a float value to a money-formatted string', () => {
+    expect(toMoney(117.15)).toBe('$117.15');
+  });
+});
+
+describe('toTitleCase', () => {
+  it('should convert a string to title case', () => {
+    expect(toTitleCase('convert To Title case')).toBe('Convert to title case');
+  });
+});
+
+describe('toPascalCase', () => {
+  it('should convert a string to pascal case', () => {
+    expect(toPascalCase('convert to pascal case')).toBe('ConvertToPascalCase');
+  });
+});

package/src/formatters/bytes-to-string.ts

@@ -0,0 +1,32 @@
+/**
+ * Format bytes as human-readable text.
+ * @param bytes Number of bytes.
+ * @param si True to use metric(SI) units, aka powers of 1000. False to use
+ * binary(IEC), aka powers of 1024.
+   * @param dp Number of decimal places to display.
+ *
+ * @return Formatted string.
+ */
+export function bytesToString(bytes: number, si = false, dp = 1): string {
+  const thresh = si ? 1000 : 1024;
+
+  if (Math.abs(bytes) < thresh) {
+    return `${bytes} B`;
+  }
+
+  const units = si
+    ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+    : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
+  let u = -1;
+  const r = 10 ** dp;
+
+  do {
+    bytes /= thresh;
+    ++u;
+  } while (
+    Math.round(Math.abs(bytes) * r) / r >= thresh
+    && u < units.length - 1
+    );
+
+  return `${bytes.toFixed(dp)} ${units[u]}`;
+}

package/src/formatters/json-pretty-print.ts

@@ -0,0 +1,8 @@
+/**
+ * Pretty prints a JSON object with the specified indentation level.
+ * @param obj JSON object to be pretty printed.
+ * @param indentLevel Number of spaces to use for indentation.
+ */
+export function jsonPrettyPrint(obj: object, indentLevel=2): string {
+  return JSON.stringify(obj, null, indentLevel);
+}

package/src/formatters/normalize-date.ts

@@ -0,0 +1,10 @@
+/**
+ * Normalizes a date to UTC by adjusting for the timezone offset.
+ * Useful when you are working with dates that may not include time information.
+ * @param date
+ */
+export function normalizeDate(date: Date | string | undefined | null): Date {
+  if (!date) throw new Error("Attempt to normalize undefined");
+  if (!(date instanceof Date)) date = new Date(date);
+  return new Date(date.getTime() - date.getTimezoneOffset() * -60000);
+}

package/src/formatters/normalize-string.ts

@@ -0,0 +1,8 @@
+/**
+ * Normalize a string by trimming it and returning null if it is empty.
+ * @param input The string to normalize.
+ */
+export function normalizeString(input: string): string {
+  if (!input?.trim()) return null;
+  return input.trim();
+}

package/src/formatters/slash.ts

@@ -0,0 +1,18 @@
+/**
+ * Removes the starting slash from a path.
+ * @param path Input path
+ */
+export function removeStartingSlash(path: string): string {
+  if (!path) return path;
+  if (path.startsWith('./')) return path.substring(2);
+  if (path.startsWith('/')) return path.substring(1);
+  return path;
+}
+
+/**
+ * Removes the ending slash from a path.
+ * @param path Input path
+ */
+export function removeEndingSlash(path: string): string {
+  return path?.endsWith('/') ? path.slice(0, -1) : path;
+}

package/src/formatters/to-date-format.ts

@@ -0,0 +1,12 @@
+const dateFormat = new Intl.DateTimeFormat('en-US', { month: '2-digit', day: '2-digit', year: '2-digit' });
+
+/**
+ * Converts a date string or Date object to MM/DD/YY date string.
+ * Ex., "2023-10-01" -> "10/01/23"
+ * @param date Input date string or Date object.
+ */
+export function toDateFormat(date: string | Date): string {
+  if (!date) return null;
+  const d: Date = (typeof date === 'string') ? new Date(date) : date
+  return dateFormat.format(d);
+}