@lossless.org/client 0.1.0

package/ts/objectstorage/classes.connection.ts

@@ -0,0 +1,36 @@
+import { SmartBucket } from './classes.smartbucket.js';
+import type { ICapabilities, IOperationOptions, IReadiness } from '../core/interfaces.js';
+
+export interface IObjectStorageConnectionOptions {
+  backend: 'objectstorage' | 's3';
+  endpoint: string;
+  region?: string;
+  credentials: { accessKeyId: string; secretAccessKey: string };
+  forcePathStyle?: boolean;
+  /** Existing bucket used by the read-only readiness probe; no bucket is created automatically. */
+  readinessBucket?: string;
+}
+
+export class ObjectStorageConnection extends SmartBucket {
+  public readonly backend: 'objectstorage' | 's3';
+  public readonly capabilities: ICapabilities;
+  constructor(public readonly options: IObjectStorageConnectionOptions) {
+    const endpoint = new URL(options.endpoint);
+    if (!['http:', 'https:'].includes(endpoint.protocol) || endpoint.username || endpoint.password ||
+        endpoint.pathname !== '/' || endpoint.search || endpoint.hash) {
+      throw new TypeError('Object storage requires an HTTP(S) origin with separate credentials.');
+    }
+    super({ endpoint: endpoint.hostname, port: Number(endpoint.port || (endpoint.protocol === 'https:' ? 443 : 80)),
+      useSsl: endpoint.protocol === 'https:', accessKey: options.credentials.accessKeyId,
+      accessSecret: options.credentials.secretAccessKey, region: options.region ?? 'us-east-1' },
+      { forcePathStyle: options.forcePathStyle });
+    this.backend = options.backend;
+    this.capabilities = Object.freeze({ backend: this.backend, transactions: 'unsupported', changeStreams: 'unsupported',
+      objectMetrics: 'unsupported', diskResidentScans: 'unknown' });
+  }
+  public async ready(options: IOperationOptions = {}): Promise<IReadiness> {
+    if (!this.options.readinessBucket) return { ready: false, reason: 'readiness_bucket_required' };
+    const result = await this.checkReadiness(this.options.readinessBucket, options);
+    return result.ready ? { ready: true } : { ready: false, reason: result.reason };
+  }
+}

package/ts/objectstorage/classes.directory.ts

@@ -0,0 +1,425 @@
+// classes.directory.ts
+
+import * as plugins from './plugins.js';
+import { Bucket } from './classes.bucket.js';
+import { File } from './classes.file.js';
+import * as helpers from './helpers.js';
+import type { IDirectoryFastPutStreamOptions } from './interfaces.js';
+
+export class Directory {
+  public bucketRef: Bucket;
+  public parentDirectoryRef: Directory;
+  public name: string;
+
+  public tree!: string[];
+  public files!: string[];
+  public folders!: string[];
+
+  constructor(bucketRefArg: Bucket, parentDirectory: Directory, name: string) {
+    this.bucketRef = bucketRefArg;
+    this.parentDirectoryRef = parentDirectory;
+    this.name = name;
+  }
+
+  /**
+   * returns an array of parent directories
+   */
+  public getParentDirectories(): Directory[] {
+    let parentDirectories: Directory[] = [];
+    if (this.parentDirectoryRef) {
+      parentDirectories.push(this.parentDirectoryRef);
+      parentDirectories = parentDirectories.concat(this.parentDirectoryRef.getParentDirectories());
+    }
+    return parentDirectories;
+  }
+
+  /**
+   * returns the directory level
+   */
+  public getDirectoryLevel(): number {
+    return this.getParentDirectories().length;
+  }
+
+  /**
+   * updates the base path
+   */
+  public getBasePath(): string {
+    const parentDirectories = this.getParentDirectories();
+    let basePath = '';
+    for (const parentDir of parentDirectories) {
+      if (!parentDir.name && !basePath) {
+        basePath = this.name + '/';
+        continue;
+      }
+      if (parentDir.name && !basePath) {
+        basePath = parentDir.name + '/' + this.name + '/';
+        continue;
+      }
+      if (parentDir.name && basePath) {
+        basePath = parentDir.name + '/' + basePath;
+        continue;
+      }
+    }
+    return basePath;
+  }
+
+  /**
+   * gets a file by name
+   */
+  public async getFile(optionsArg: {
+    path: string;
+    createWithContents?: string | Buffer;
+    getFromTrash?: boolean;
+  }): Promise<File> {
+    const pathDescriptor = {
+      directory: this,
+      path: optionsArg.path,
+    };
+    const exists = await this.bucketRef.fastExists({
+      path: await helpers.reducePathDescriptorToPath(pathDescriptor),
+    });
+    if (!exists && optionsArg.getFromTrash) {
+      const trash = await this.bucketRef.getTrash();
+      const trashedFile = await trash.getTrashedFileByOriginalName(pathDescriptor);
+      return trashedFile;
+    }
+    if (!exists && !optionsArg.createWithContents) {
+      throw new Error(`File not found at path '${optionsArg.path}'`);
+    }
+    if (!exists && optionsArg.createWithContents) {
+      await File.create({
+        directory: this,
+        name: optionsArg.path,
+        contents: optionsArg.createWithContents,
+      });
+    }
+    return new File({
+      directoryRefArg: this,
+      fileName: optionsArg.path,
+    });
+  }
+
+
+  /**
+   * Check if a file exists in this directory
+   */
+  public async fileExists(optionsArg: { path: string }): Promise<boolean> {
+    const pathDescriptor = {
+      directory: this,
+      path: optionsArg.path,
+    };
+    return this.bucketRef.fastExists({
+      path: await helpers.reducePathDescriptorToPath(pathDescriptor),
+    });
+  }
+
+  /**
+   * Check if a subdirectory exists
+   */
+  public async directoryExists(dirNameArg: string): Promise<boolean> {
+    const directories = await this.listDirectories();
+    return directories.some(dir => dir.name === dirNameArg);
+  }
+
+  /**
+   * Collects all ListObjectsV2 pages for a prefix.
+   */
+  private async listObjectsV2AllPages(prefix: string, delimiter?: string) {
+    const allContents: plugins.s3._Object[] = [];
+    const allCommonPrefixes: plugins.s3.CommonPrefix[] = [];
+    let continuationToken: string | undefined;
+
+    do {
+      const command = new plugins.s3.ListObjectsV2Command({
+        Bucket: this.bucketRef.name,
+        Prefix: prefix,
+        Delimiter: delimiter,
+        ContinuationToken: continuationToken,
+      });
+      const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
+
+      if (response.Contents) {
+        allContents.push(...response.Contents);
+      }
+      if (response.CommonPrefixes) {
+        allCommonPrefixes.push(...response.CommonPrefixes);
+      }
+
+      continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
+    } while (continuationToken);
+
+    return { contents: allContents, commonPrefixes: allCommonPrefixes };
+  }
+
+  /**
+   * lists all files
+   */
+  public async listFiles(): Promise<File[]> {
+    const { contents } = await this.listObjectsV2AllPages(this.getBasePath(), '/');
+    const fileArray: File[] = [];
+
+    contents.forEach((item) => {
+      if (item.Key && !item.Key.endsWith('/')) {
+        const subtractedPath = item.Key.replace(this.getBasePath(), '');
+        if (!subtractedPath.includes('/')) {
+          fileArray.push(
+            new File({
+              directoryRefArg: this,
+              fileName: subtractedPath,
+            })
+          );
+        }
+      }
+    });
+
+    return fileArray;
+  }
+
+  /**
+   * lists all folders
+   */
+  public async listDirectories(): Promise<Directory[]> {
+    try {
+      const { commonPrefixes } = await this.listObjectsV2AllPages(this.getBasePath(), '/');
+      const directoryArray: Directory[] = [];
+  
+      if (commonPrefixes) {
+        commonPrefixes.forEach((item) => {
+          if (item.Prefix) {
+            const subtractedPath = item.Prefix.replace(this.getBasePath(), '');
+            if (subtractedPath.endsWith('/')) {
+              const dirName = subtractedPath.slice(0, -1);
+              // Ensure the directory name is not empty (which would indicate the base directory itself)
+              if (dirName) {
+                directoryArray.push(new Directory(this.bucketRef, this, dirName));
+              }
+            }
+          }
+        });
+      }
+  
+      return directoryArray;
+    } catch (error) {
+      console.error('Error listing directories:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * gets an array that has all objects with a certain prefix
+   */
+  public async getTreeArray() {
+    const command = new plugins.s3.ListObjectsV2Command({
+      Bucket: this.bucketRef.name,
+      Prefix: this.getBasePath(),
+      Delimiter: '/',
+    });
+    const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
+    return response.Contents;
+  }
+
+  /**
+   * gets a sub directory by name
+   */
+  public async getSubDirectoryByName(dirNameArg: string, optionsArg: {
+    /**
+     * in object storage a directory does not exist if it is empty
+     * this option returns a directory even if it is empty
+     */
+    getEmptyDirectory?: boolean;
+    /**
+     * in object storage a directory does not exist if it is empty
+     * this option creates a directory even if it is empty using a initializer file
+     */
+    createWithInitializerFile?: boolean;
+    /**
+     * if the path is a file path, it will be treated as a file and the parent directory will be returned
+     */
+    couldBeFilePath?: boolean;
+  } = {}): Promise<Directory> {
+
+    const dirNameArray = dirNameArg.split('/').filter(str => str.trim() !== "");
+
+    optionsArg = {
+      getEmptyDirectory: false,
+      createWithInitializerFile: false,
+      ...optionsArg,
+    }
+
+
+    const getDirectory = async (directoryArg: Directory, dirNameToSearch: string, isFinalDirectory: boolean) => {
+      const directories = await directoryArg.listDirectories();
+      let returnDirectory = directories.find((directory) => {
+        return directory.name === dirNameToSearch;
+      });
+      if (returnDirectory) {
+        return returnDirectory;
+      }
+      if (optionsArg.getEmptyDirectory || optionsArg.createWithInitializerFile) {
+        returnDirectory = new Directory(this.bucketRef, directoryArg, dirNameToSearch);
+      }
+      if (isFinalDirectory && optionsArg.createWithInitializerFile) {
+        returnDirectory?.createEmptyFile('00init.txt');
+      }
+      return returnDirectory || null;
+    };
+
+    if (optionsArg.couldBeFilePath) {
+      const baseDirectory = await this.bucketRef.getBaseDirectory();
+      const existingFile = await baseDirectory.getFile({
+        path: dirNameArg,
+      });
+      if (existingFile) {
+        const adjustedPath = dirNameArg.substring(0, dirNameArg.lastIndexOf('/'));
+        return this.getSubDirectoryByName(adjustedPath);
+      }
+    }
+
+    let wantedDirectory: Directory | null = null;
+    let counter = 0;
+    for (const dirNameToSearch of dirNameArray) {
+      counter++;
+      const directoryToSearchIn = wantedDirectory ? wantedDirectory : this;
+      wantedDirectory = await getDirectory(directoryToSearchIn, dirNameToSearch, counter === dirNameArray.length);
+    }
+
+    if (!wantedDirectory) {
+      throw new Error(`Directory not found at path '${dirNameArg}'`);
+    }
+    return wantedDirectory;
+  }
+
+
+  /**
+   * moves the directory
+   */
+  public async move() {
+    // TODO
+    throw new Error('Moving a directory is not yet implemented');
+  }
+
+  /**
+   * creates an empty file within this directory
+   * @param relativePathArg
+   */
+  public async createEmptyFile(relativePathArg: string) {
+    const emptyFile = await File.create({
+      directory: this,
+      name: relativePathArg,
+      contents: '',
+    });
+    return emptyFile;
+  }
+
+  // file operations
+  public async fastPut(optionsArg: { path: string; contents: string | Buffer }) {
+    const path = plugins.path.join(this.getBasePath(), optionsArg.path);
+    await this.bucketRef.fastPut({
+      path,
+      contents: optionsArg.contents,
+    });
+  }
+
+  public async fastGet(optionsArg: { path: string }) {
+    const path = plugins.path.join(this.getBasePath(), optionsArg.path);
+    const result = await this.bucketRef.fastGet({
+      path,
+    });
+    return result;
+  }
+
+  public fastGetStream(
+    optionsArg: {
+      path: string;
+      signal?: AbortSignal;
+    },
+    typeArg: 'webstream'
+  ): Promise<ReadableStream>;
+  public async fastGetStream(
+    optionsArg: {
+      path: string;
+      signal?: AbortSignal;
+    },
+    typeArg: 'nodestream'
+  ): Promise<plugins.stream.Readable>;
+
+  /**
+   * fastGetStream
+   * @param optionsArg
+   * @returns
+   */
+  public async fastGetStream(
+    optionsArg: { path: string; signal?: AbortSignal },
+    typeArg: 'webstream' | 'nodestream'
+  ): Promise<ReadableStream | plugins.stream.Readable> {
+    const path = plugins.path.join(this.getBasePath(), optionsArg.path);
+    const result = await this.bucketRef.fastGetStream(
+      {
+        path,
+        signal: optionsArg.signal,
+      },
+      typeArg as any
+    );
+    return result;
+  }
+
+  /**
+   * fast put stream
+   */
+  public async fastPutStream(
+    optionsArg: IDirectoryFastPutStreamOptions,
+  ): Promise<void> {
+    const path = plugins.path.join(this.getBasePath(), optionsArg.path);
+    await this.bucketRef.fastPutStream({
+      path,
+      readableStream: optionsArg.stream,
+      contentLength: optionsArg.contentLength,
+      signal: optionsArg.signal,
+    });
+  }
+
+  /**
+   * removes a file within the directory
+   * uses file class to make sure effects for metadata etc. are handled correctly
+   * @param optionsArg
+   */
+  public async fastRemove(optionsArg: {
+    path: string
+    /**
+     * wether the file should be placed into trash. Default is false.
+     */
+    mode?: 'permanent' | 'trash';
+  }) {
+    const file = await this.getFile({
+      path: optionsArg.path,
+    });
+    await file.delete({
+      mode: optionsArg.mode ? optionsArg.mode : 'permanent',
+    });
+  }
+
+  /**
+   * deletes the directory with all its contents
+   */
+  public async delete(optionsArg: {
+    mode?: 'permanent' | 'trash';
+  }) {
+    const deleteDirectory = async (directoryArg: Directory) => {
+      const childDirectories = await directoryArg.listDirectories();
+      if (childDirectories.length === 0) {
+        console.log('Directory empty! Path complete!');
+      } else {
+        for (const childDir of childDirectories) {
+          await deleteDirectory(childDir);
+        }
+      }
+      const files = await directoryArg.listFiles();
+      for (const file of files) {
+        await file.delete({
+          mode: optionsArg.mode ? optionsArg.mode : 'permanent',
+        })
+      }
+    };
+    await deleteDirectory(this);
+  }
+}

package/ts/objectstorage/classes.exactpathpurgeerror.ts

@@ -0,0 +1,24 @@
+import type {
+  IExactPathPurgeFailureState,
+  TExactPathPurgeFailurePhase,
+} from './interfaces.js';
+
+const phaseMessages: Record<TExactPathPurgeFailurePhase, string> = {
+  capability: 'The exact-path purge capability is invalid or stale.',
+  inventory: 'The exact path could not be inventoried safely.',
+  consistency: 'The exact path changed before purge mutation.',
+  delete: 'The exact path could not be deleted safely.',
+  verification: 'The exact-path purge could not be verified.',
+};
+
+export class ExactPathPurgeError extends Error {
+  public readonly state: IExactPathPurgeFailureState;
+
+  constructor(stateArg: IExactPathPurgeFailureState, causeArg: unknown) {
+    super(phaseMessages[stateArg.phase], {
+      cause: causeArg,
+    });
+    this.name = 'ExactPathPurgeError';
+    this.state = stateArg;
+  }
+}

package/ts/objectstorage/classes.exactuploaderror.ts

@@ -0,0 +1,25 @@
+import type {
+  IExactUploadFailureState,
+  TExactUploadFailurePhase,
+} from './interfaces.js';
+
+const phaseMessages: Record<TExactUploadFailurePhase, string> = {
+  capability: 'The exact upload capability is invalid or stale.',
+  source: 'The exact upload source could not be consumed.',
+  'put-zero': 'The conditional zero-byte object upload failed.',
+  'create-multipart': 'The multipart upload could not be created.',
+  'upload-part': 'A multipart upload part failed.',
+  'complete-multipart': 'The conditional multipart completion failed.',
+};
+
+export class ExactUploadError extends Error {
+  public readonly state: IExactUploadFailureState;
+
+  constructor(stateArg: IExactUploadFailureState, causeArg: unknown) {
+    super(phaseMessages[stateArg.phase], {
+      cause: causeArg,
+    });
+    this.name = 'ExactUploadError';
+    this.state = stateArg;
+  }
+}