@memberjunction/storage 2.43.0 → 2.44.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memberjunction/storage",
-  "version": "2.43.0",
+  "version": "2.44.0",
   "description": "This library provides a set of objects that handle the interface between the server-side API and various cloud storage providers.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -25,9 +25,9 @@
     "@azure/identity": "^4.0.1",
     "@azure/storage-blob": "^12.17.0",
     "@google-cloud/storage": "^7.9.0",
-    "@memberjunction/core": "2.43.0",
-    "@memberjunction/core-entities": "2.43.0",
-    "@memberjunction/global": "2.43.0",
+    "@memberjunction/core": "2.44.0",
+    "@memberjunction/core-entities": "2.44.0",
+    "@memberjunction/global": "2.44.0",
     "@microsoft/mgt-element": "^3.1.3",
     "@microsoft/mgt-msal2-provider": "^3.1.3",
     "@microsoft/mgt-sharepoint-provider": "^3.1.3",

package/readme.md

@@ -4,10 +4,18 @@ The `@memberjunction/storage` library provides a unified interface for interacti
 
 [![MemberJunction Logo](https://memberjunction.com/images/MJ_Dark_Logo_Transparent_tm.png)](https://memberjunction.com)
 
+## Overview
+
+This library is a key component of the MemberJunction platform, providing seamless file storage operations across multiple cloud providers. It offers a provider-agnostic approach to file management, allowing applications to switch between storage providers without code changes.
+
 ## Features
 
-- **Unified API**: Consistent methods across all storage providers
+- **Unified API**: Consistent methods across all storage providers via the `FileStorageBase` abstract class
+- **Type-Safe**: Full TypeScript support with comprehensive type definitions
 - **Flexible Provider Selection**: Use any number of storage providers simultaneously based on your application needs
+- **Pre-authenticated URLs**: Secure upload and download operations using time-limited URLs
+- **Metadata Support**: Store and retrieve custom metadata with your files
+- **Error Handling**: Provider-specific errors are normalized with clear error messages
 - **Commonly Supported Storage Providers**: 
   - [AWS S3](https://aws.amazon.com/s3/)
   - [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs)
@@ -21,11 +29,12 @@ The `@memberjunction/storage` library provides a unified interface for interacti
   - Download files (via pre-authenticated URLs)
   - Copy and move files
   - Delete files and directories
-  - List files and directories
+  - List files and directories with metadata
   - Create and manage directories
-  - Get file metadata
+  - Get detailed file metadata
   - Check file/directory existence
-- **Extensible**: Easy to add new storage providers
+  - Direct upload/download via Buffer
+- **Extensible**: Easy to add new storage providers by extending `FileStorageBase`
 
 ## Installation
 
@@ -33,6 +42,14 @@ The `@memberjunction/storage` library provides a unified interface for interacti
 npm install @memberjunction/storage
 ```
 
+## Dependencies
+
+This package depends on:
+- `@memberjunction/core` - Core MemberJunction functionality
+- `@memberjunction/core-entities` - Entity definitions including `FileStorageProviderEntity`
+- `@memberjunction/global` - Global utilities and class registration
+- Provider-specific SDKs (installed as dependencies)
+
 ## Usage
 
 ### Basic Setup
@@ -48,83 +65,225 @@ STORAGE_AZURE_ACCOUNT_NAME=your-account-name
 STORAGE_AZURE_ACCOUNT_KEY=your-account-key
 ```
 
-### Code Example
+### Using Utility Functions (Recommended)
+
+The library provides high-level utility functions that work with MemberJunction's entity system:
 
 ```typescript
-import { AzureFileStorage, createUploadUrl, createDownloadUrl, deleteObject } from '@memberjunction/storage';
+import { createUploadUrl, createDownloadUrl, deleteObject, moveObject } from '@memberjunction/storage';
 import { FileStorageProviderEntity } from '@memberjunction/core-entities';
+import { Metadata } from '@memberjunction/core';
 
-// Assuming you have a FileStorageProviderEntity loaded from your database
-async function fileOperationsExample(provider: FileStorageProviderEntity) {
+// Load a FileStorageProviderEntity from the database
+async function fileOperationsExample() {
+  const md = new Metadata();
+  const provider = await md.GetEntityObject<FileStorageProviderEntity>('File Storage Providers');
+  await provider.Load('your-provider-id');
+  
   // Create pre-authenticated upload URL
   const { updatedInput, UploadUrl } = await createUploadUrl(
     provider, 
     { 
       ID: '123', 
-      Name: 'document.pdf', 
+      Name: 'documents/report.pdf', 
       ProviderID: provider.ID 
     }
   );
 
   // The client can use the UploadUrl directly to upload the file
   console.log(`Upload URL: ${UploadUrl}`);
+  console.log(`File status: ${updatedInput.Status}`); // 'Uploading'
+  console.log(`Content type: ${updatedInput.ContentType}`); // 'application/pdf'
+  
+  // If a ProviderKey was returned, use it for future operations
+  const fileIdentifier = updatedInput.ProviderKey || updatedInput.Name;
   
   // Later, create pre-authenticated download URL
-  const downloadUrl = await createDownloadUrl(provider, updatedInput.Name);
+  const downloadUrl = await createDownloadUrl(provider, fileIdentifier);
   console.log(`Download URL: ${downloadUrl}`);
   
+  // Move the file to a new location
+  const moved = await moveObject(
+    provider,
+    fileIdentifier,
+    'documents/archived/report_2024.pdf'
+  );
+  console.log(`File moved: ${moved}`);
+  
   // Delete the file when no longer needed
-  const deleted = await deleteObject(provider, updatedInput.Name);
+  const deleted = await deleteObject(provider, 'documents/archived/report_2024.pdf');
   console.log(`File deleted: ${deleted}`);
 }
 ```
 
 ### Direct Provider Usage
 
-You can also work directly with a storage provider:
+You can also work directly with a storage provider by instantiating it using the MemberJunction class factory:
 
 ```typescript
-import { AzureFileStorage } from '@memberjunction/storage';
+import { FileStorageBase } from '@memberjunction/storage';
+import { MJGlobal } from '@memberjunction/global';
 
 async function directProviderExample() {
-  const storage = new AzureFileStorage();
+  // Create storage instance using the class factory (recommended)
+  const storage = MJGlobal.Instance.ClassFactory.CreateInstance<FileStorageBase>(
+    FileStorageBase, 
+    'Azure Blob Storage'
+  );
+  
+  // Initialize the storage provider if needed
+  await storage.initialize();
   
   // List all files in a directory
   const result = await storage.ListObjects('documents/');
   console.log('Files:', result.objects);
   console.log('Directories:', result.prefixes);
   
+  // Display detailed metadata for each file
+  for (const file of result.objects) {
+    console.log(`\nFile: ${file.name}`);
+    console.log(`  Path: ${file.path}`);
+    console.log(`  Full Path: ${file.fullPath}`);
+    console.log(`  Size: ${file.size} bytes`);
+    console.log(`  Type: ${file.contentType}`);
+    console.log(`  Modified: ${file.lastModified}`);
+    console.log(`  Is Directory: ${file.isDirectory}`);
+  }
+  
   // Create a directory
-  await storage.CreateDirectory('documents/reports/');
+  const dirCreated = await storage.CreateDirectory('documents/reports/');
+  console.log(`Directory created: ${dirCreated}`);
   
-  // Upload a file directly
+  // Upload a file directly with metadata
   const content = Buffer.from('Hello, World!');
-  await storage.PutObject('documents/reports/hello.txt', content, 'text/plain');
+  const uploaded = await storage.PutObject(
+    'documents/reports/hello.txt', 
+    content, 
+    'text/plain',
+    { 
+      author: 'John Doe',
+      department: 'Engineering',
+      version: '1.0' 
+    }
+  );
+  console.log(`File uploaded: ${uploaded}`);
+  
+  // Get file metadata without downloading content
+  const metadata = await storage.GetObjectMetadata('documents/reports/hello.txt');
+  console.log('File metadata:', metadata);
+  
+  // Download file content
+  const fileContent = await storage.GetObject('documents/reports/hello.txt');
+  console.log('File content:', fileContent.toString('utf8'));
   
   // Copy a file
-  await storage.CopyObject('documents/reports/hello.txt', 'documents/archive/hello-copy.txt');
+  const copied = await storage.CopyObject(
+    'documents/reports/hello.txt', 
+    'documents/archive/hello-backup.txt'
+  );
+  console.log(`File copied: ${copied}`);
   
   // Check if a file exists
   const exists = await storage.ObjectExists('documents/reports/hello.txt');
+  console.log(`File exists: ${exists}`);
+  
+  // Check if a directory exists
+  const dirExists = await storage.DirectoryExists('documents/reports/');
+  console.log(`Directory exists: ${dirExists}`);
+  
+  // Delete a directory and all its contents
+  const dirDeleted = await storage.DeleteDirectory('documents/reports/', true);
+  console.log(`Directory deleted: ${dirDeleted}`);
 }
 ```
 
+## API Reference
+
+### Core Types
+
+#### `CreatePreAuthUploadUrlPayload`
+```typescript
+type CreatePreAuthUploadUrlPayload = {
+  UploadUrl: string;      // Pre-authenticated URL for upload
+  ProviderKey?: string;   // Optional provider-specific key
+};
+```
+
+#### `StorageObjectMetadata`
+```typescript
+type StorageObjectMetadata = {
+  name: string;           // Object name (filename)
+  path: string;           // Directory path
+  fullPath: string;       // Complete path including name
+  size: number;           // Size in bytes
+  contentType: string;    // MIME type
+  lastModified: Date;     // Last modification date
+  isDirectory: boolean;   // Whether this is a directory
+  etag?: string;          // Entity tag for caching
+  cacheControl?: string;  // Cache control directives
+  customMetadata?: Record<string, string>; // Custom metadata
+};
+```
+
+#### `StorageListResult`
+```typescript
+type StorageListResult = {
+  objects: StorageObjectMetadata[];  // Files found
+  prefixes: string[];                // Directories found
+};
+```
+
+### FileStorageBase Methods
+
+All storage providers implement these methods:
+
+- `CreatePreAuthUploadUrl(objectName: string): Promise<CreatePreAuthUploadUrlPayload>`
+- `CreatePreAuthDownloadUrl(objectName: string): Promise<string>`
+- `MoveObject(oldObjectName: string, newObjectName: string): Promise<boolean>`
+- `DeleteObject(objectName: string): Promise<boolean>`
+- `ListObjects(prefix: string, delimiter?: string): Promise<StorageListResult>`
+- `CreateDirectory(directoryPath: string): Promise<boolean>`
+- `DeleteDirectory(directoryPath: string, recursive?: boolean): Promise<boolean>`
+- `GetObjectMetadata(objectName: string): Promise<StorageObjectMetadata>`
+- `GetObject(objectName: string): Promise<Buffer>`
+- `PutObject(objectName: string, data: Buffer, contentType?: string, metadata?: Record<string, string>): Promise<boolean>`
+- `CopyObject(sourceObjectName: string, destinationObjectName: string): Promise<boolean>`
+- `ObjectExists(objectName: string): Promise<boolean>`
+- `DirectoryExists(directoryPath: string): Promise<boolean>`
+- `initialize(): Promise<void>` (optional, for async initialization)
+
+### Utility Functions
+
+- `createUploadUrl<T>(provider: FileStorageProviderEntity, input: T): Promise<{ updatedInput: T & { Status: string; ContentType: string }, UploadUrl: string }>`
+- `createDownloadUrl(provider: FileStorageProviderEntity, providerKeyOrName: string): Promise<string>`
+- `moveObject(provider: FileStorageProviderEntity, oldProviderKeyOrName: string, newProviderKeyOrName: string): Promise<boolean>`
+- `deleteObject(provider: FileStorageProviderEntity, providerKeyOrName: string): Promise<boolean>`
+
 ## Architecture
 
-The library uses a class hierarchy with `FileStorageBase` as the abstract base class that defines the common interface. Each storage provider implements this interface in its own way:
+The library uses a class hierarchy with `FileStorageBase` as the abstract base class that defines the common interface. Each storage provider implements this interface:
 
 ```
 FileStorageBase (Abstract Base Class)
-├── AWSFileStorage
-├── AzureFileStorage
-├── GoogleFileStorage
-├── GoogleDriveFileStorage
-├── SharePointFileStorage
-├── DropboxFileStorage
-└── BoxFileStorage
+├── AWSFileStorage (@RegisterClass: 'AWS S3')
+├── AzureFileStorage (@RegisterClass: 'Azure Blob Storage')
+├── GoogleFileStorage (@RegisterClass: 'Google Cloud Storage')
+├── GoogleDriveFileStorage (@RegisterClass: 'Google Drive')
+├── SharePointFileStorage (@RegisterClass: 'SharePoint')
+├── DropboxFileStorage (@RegisterClass: 'Dropbox')
+└── BoxFileStorage (@RegisterClass: 'Box')
 ```
 
-Utility functions provide a simplified interface for common operations.
+Classes are registered with the MemberJunction global class factory using the `@RegisterClass` decorator, enabling dynamic instantiation based on provider keys.
+
+### Integration with MemberJunction
+
+This library integrates seamlessly with the MemberJunction platform:
+
+1. **Entity System**: Works with `FileStorageProviderEntity` from `@memberjunction/core-entities`
+2. **Class Factory**: Uses `@memberjunction/global` for dynamic provider instantiation
+3. **Configuration**: Provider settings are stored in the MemberJunction database
+4. **Type Safety**: Fully typed interfaces ensure compile-time safety
 
 ## Storage Provider Configuration
 
@@ -187,22 +346,142 @@ For more information, see [Box Platform Documentation](https://developer.box.com
 
 ## Implementing Additional Providers
 
-The library is designed to be extensible. You can implement new storage providers by:
+The library is designed to be extensible. To add a new storage provider:
+
+### 1. Create a New Provider Class
+
+```typescript
+import { FileStorageBase, StorageObjectMetadata, StorageListResult } from '@memberjunction/storage';
+import { RegisterClass } from '@memberjunction/global';
+
+@RegisterClass(FileStorageBase, 'My Custom Storage')
+export class MyCustomStorage extends FileStorageBase {
+  protected readonly providerName = 'My Custom Storage';
+  
+  constructor() {
+    super();
+    // Initialize your storage client here
+  }
+  
+  public async initialize(): Promise<void> {
+    // Optional: Perform async initialization
+    // e.g., authenticate, verify permissions
+  }
+  
+  public async CreatePreAuthUploadUrl(objectName: string): Promise<CreatePreAuthUploadUrlPayload> {
+    // Implement upload URL generation
+    // Return { UploadUrl: string, ProviderKey?: string }
+  }
+  
+  public async CreatePreAuthDownloadUrl(objectName: string): Promise<string> {
+    // Implement download URL generation
+  }
+  
+  // Implement all other abstract methods...
+}
+```
+
+### 2. Handle Unsupported Operations
+
+If your provider doesn't support certain operations:
+
+```typescript
+public async CreateDirectory(directoryPath: string): Promise<boolean> {
+  // If directories aren't supported
+  this.throwUnsupportedOperationError('CreateDirectory');
+}
+```
+
+### 3. Register Environment Variables
+
+Document required environment variables:
+
+```typescript
+import * as env from 'env-var';
+
+constructor() {
+  super();
+  const apiKey = env.get('STORAGE_MYCUSTOM_API_KEY').required().asString();
+  const endpoint = env.get('STORAGE_MYCUSTOM_ENDPOINT').required().asString();
+  // Use these to initialize your client
+}
+```
+
+### 4. Export from Index
+
+Add to `src/index.ts`:
+
+```typescript
+export * from './drivers/MyCustomStorage';
+```
+
+### 5. Add to Documentation
+
+Update this README with configuration requirements and any provider-specific notes.
+
+## Error Handling
+
+The library provides consistent error handling across all providers:
+
+### UnsupportedOperationError
+
+Thrown when a provider doesn't support a specific operation:
+
+```typescript
+try {
+  await storage.CreateDirectory('/some/path/');
+} catch (error) {
+  if (error instanceof UnsupportedOperationError) {
+    console.log(`Provider doesn't support directories: ${error.message}`);
+  }
+}
+```
+
+### Provider-Specific Errors
+
+Each provider may throw errors specific to its underlying SDK. These are not wrapped, allowing you to handle provider-specific error conditions:
+
+```typescript
+try {
+  await storage.GetObject('non-existent-file.txt');
+} catch (error) {
+  // Handle provider-specific errors
+  if (error.code === 'NoSuchKey') { // AWS S3
+    console.log('File not found');
+  } else if (error.code === 'BlobNotFound') { // Azure
+    console.log('Blob not found');
+  }
+}
+```
+
+## Best Practices
+
+1. **Use ProviderKey**: Always check for and use `ProviderKey` if returned by `CreatePreAuthUploadUrl`
+2. **Error Handling**: Implement proper error handling for both generic and provider-specific errors
+3. **Environment Variables**: Store sensitive credentials securely and never commit them to version control
+4. **Content Types**: Always specify content types for better browser handling and security
+5. **Metadata**: Use custom metadata to store additional information without modifying file content
+6. **Directory Paths**: Always use trailing slashes for directory paths (e.g., `documents/` not `documents`)
+7. **Initialize Providers**: Call `initialize()` on providers that require async setup
 
-1. Creating a new class that extends `FileStorageBase`
-2. Implementing all required abstract methods
-3. Registering the class using the `@RegisterClass` decorator from `@memberjunction/global`
+## Performance Considerations
 
-Refer to the existing provider implementations for guidance on how to implement a new provider.
+- **Pre-authenticated URLs**: Use these for client uploads/downloads to reduce server load
+- **Buffering**: The `GetObject` and `PutObject` methods load entire files into memory; for large files, consider streaming approaches
+- **List Operations**: Use appropriate prefixes and delimiters to limit results
+- **Caching**: Utilize ETags and cache control headers when available
 
 ## Contributing
 
 Contributions are welcome! To add a new storage provider:
 
-1. Create a new class in the `src/drivers` directory
-2. Extend the `FileStorageBase` class
-3. Implement all required methods
-4. Add exports to `src/index.ts`
+1. Fork the repository
+2. Create a feature branch (`git checkout -b feature/new-provider`)
+3. Create your provider class in `src/drivers/`
+4. Implement all required methods from `FileStorageBase`
+5. Add comprehensive tests
+6. Update documentation
+7. Submit a pull request
 
 ## License