@lossless.org/client 0.1.0

package/ts/objectstorage/classes.watcher.ts

@@ -0,0 +1,289 @@
+// classes.watcher.ts
+
+import * as plugins from './plugins.js';
+import * as interfaces from './interfaces.js';
+import type { Bucket } from './classes.bucket.js';
+import { EventEmitter } from 'node:events';
+
+/**
+ * BucketWatcher monitors a storage bucket for changes (add/modify/delete)
+ * using a polling-based approach. Designed to follow the SmartdataDbWatcher pattern.
+ *
+ * @example
+ * ```typescript
+ * const watcher = bucket.createWatcher({ prefix: 'uploads/', pollIntervalMs: 3000 });
+ *
+ * // RxJS Observable pattern
+ * watcher.changeSubject.subscribe((change) => {
+ *   console.log('Change:', change);
+ * });
+ *
+ * // EventEmitter pattern
+ * watcher.on('change', (change) => console.log(change));
+ * watcher.on('error', (err) => console.error(err));
+ *
+ * await watcher.start();
+ * await watcher.readyDeferred.promise; // Wait for initial state
+ *
+ * // Later...
+ * await watcher.stop();
+ * ```
+ */
+export class BucketWatcher extends EventEmitter {
+  /** Deferred that resolves when initial state is built and watcher is ready */
+  public readyDeferred = plugins.smartpromise.defer();
+
+  /** Observable for receiving change events (supports RxJS operators) */
+  public changeSubject: plugins.smartrx.rxjs.Observable<interfaces.IStorageChangeEvent | interfaces.IStorageChangeEvent[]>;
+
+  // Internal subjects and state
+  private rawSubject: plugins.smartrx.rxjs.Subject<interfaces.IStorageChangeEvent>;
+  private previousState: Map<string, interfaces.IStorageObjectState>;
+  private pollIntervalId: ReturnType<typeof setInterval> | null = null;
+  private isPolling = false;
+  private isStopped = false;
+
+  // Configuration
+  private readonly bucketRef: Bucket;
+  private readonly prefix: string;
+  private readonly pollIntervalMs: number;
+  private readonly bufferTimeMs?: number;
+  private readonly includeInitial: boolean;
+  private readonly pageSize: number;
+
+  constructor(bucketRef: Bucket, options: interfaces.IBucketWatcherOptions = {}) {
+    super();
+
+    this.bucketRef = bucketRef;
+    this.prefix = options.prefix ?? '';
+    this.pollIntervalMs = options.pollIntervalMs ?? 5000;
+    this.bufferTimeMs = options.bufferTimeMs;
+    this.includeInitial = options.includeInitial ?? false;
+    this.pageSize = options.pageSize ?? 1000;
+
+    // Initialize state tracking
+    this.previousState = new Map();
+
+    // Initialize raw subject for emitting changes
+    this.rawSubject = new plugins.smartrx.rxjs.Subject<interfaces.IStorageChangeEvent>();
+
+    // Configure the public observable with optional buffering
+    if (this.bufferTimeMs && this.bufferTimeMs > 0) {
+      this.changeSubject = this.rawSubject.pipe(
+        plugins.smartrx.rxjs.ops.bufferTime(this.bufferTimeMs),
+        plugins.smartrx.rxjs.ops.filter((events: interfaces.IStorageChangeEvent[]) => events.length > 0)
+      );
+    } else {
+      this.changeSubject = this.rawSubject.asObservable();
+    }
+  }
+
+  /**
+   * Start watching the bucket for changes
+   */
+  public async start(): Promise<void> {
+    if (this.pollIntervalId !== null) {
+      console.log('BucketWatcher is already running');
+      return;
+    }
+
+    this.isStopped = false;
+
+    // Build initial state
+    await this.buildInitialState();
+
+    // Emit initial state as 'add' events if configured
+    if (this.includeInitial) {
+      for (const state of this.previousState.values()) {
+        this.emitChange({
+          type: 'add',
+          key: state.key,
+          size: state.size,
+          etag: state.etag,
+          lastModified: state.lastModified,
+          bucket: this.bucketRef.name,
+        });
+      }
+    }
+
+    // Mark as ready
+    this.readyDeferred.resolve();
+
+    // Start polling loop
+    this.pollIntervalId = setInterval(() => {
+      this.poll().catch((err) => {
+        this.emit('error', err);
+      });
+    }, this.pollIntervalMs);
+  }
+
+  /**
+   * Stop watching the bucket
+   */
+  public async stop(): Promise<void> {
+    this.isStopped = true;
+
+    if (this.pollIntervalId !== null) {
+      clearInterval(this.pollIntervalId);
+      this.pollIntervalId = null;
+    }
+
+    // Wait for any in-progress poll to complete
+    while (this.isPolling) {
+      await new Promise<void>((resolve) => setTimeout(resolve, 50));
+    }
+
+    this.rawSubject.complete();
+  }
+
+  /**
+   * Alias for stop() - for consistency with other APIs
+   */
+  public async close(): Promise<void> {
+    return this.stop();
+  }
+
+  /**
+   * Build the initial state by listing all objects with metadata
+   */
+  private async buildInitialState(): Promise<void> {
+    this.previousState.clear();
+
+    for await (const obj of this.listObjectsWithMetadata()) {
+      if (obj.Key) {
+        this.previousState.set(obj.Key, {
+          key: obj.Key,
+          etag: obj.ETag ?? '',
+          size: obj.Size ?? 0,
+          lastModified: obj.LastModified ?? new Date(0),
+        });
+      }
+    }
+  }
+
+  /**
+   * Poll for changes by comparing current state against previous state
+   */
+  private async poll(): Promise<void> {
+    // Guard against overlapping polls
+    if (this.isPolling || this.isStopped) {
+      return;
+    }
+
+    this.isPolling = true;
+
+    try {
+      // Build current state
+      const currentState = new Map<string, interfaces.IStorageObjectState>();
+
+      for await (const obj of this.listObjectsWithMetadata()) {
+        if (this.isStopped) {
+          break;
+        }
+
+        if (obj.Key) {
+          currentState.set(obj.Key, {
+            key: obj.Key,
+            etag: obj.ETag ?? '',
+            size: obj.Size ?? 0,
+            lastModified: obj.LastModified ?? new Date(0),
+          });
+        }
+      }
+
+      if (!this.isStopped) {
+        this.detectChanges(currentState);
+        this.previousState = currentState;
+      }
+    } catch (err) {
+      this.emit('error', err);
+    } finally {
+      this.isPolling = false;
+    }
+  }
+
+  /**
+   * Detect changes between current and previous state
+   */
+  private detectChanges(currentState: Map<string, interfaces.IStorageObjectState>): void {
+    // Detect added and modified objects
+    for (const [key, current] of currentState) {
+      const previous = this.previousState.get(key);
+
+      if (!previous) {
+        // New object - emit 'add' event
+        this.emitChange({
+          type: 'add',
+          key: current.key,
+          size: current.size,
+          etag: current.etag,
+          lastModified: current.lastModified,
+          bucket: this.bucketRef.name,
+        });
+      } else if (
+        previous.etag !== current.etag ||
+        previous.size !== current.size ||
+        previous.lastModified.getTime() !== current.lastModified.getTime()
+      ) {
+        // Object modified - emit 'modify' event
+        this.emitChange({
+          type: 'modify',
+          key: current.key,
+          size: current.size,
+          etag: current.etag,
+          lastModified: current.lastModified,
+          bucket: this.bucketRef.name,
+        });
+      }
+    }
+
+    // Detect deleted objects
+    for (const [key, previous] of this.previousState) {
+      if (!currentState.has(key)) {
+        // Object deleted - emit 'delete' event
+        this.emitChange({
+          type: 'delete',
+          key: previous.key,
+          bucket: this.bucketRef.name,
+        });
+      }
+    }
+  }
+
+  /**
+   * Emit a change event via both RxJS Subject and EventEmitter
+   */
+  private emitChange(event: interfaces.IStorageChangeEvent): void {
+    this.rawSubject.next(event);
+    this.emit('change', event);
+  }
+
+  /**
+   * List objects with full metadata (ETag, Size, LastModified)
+   * This is a private method that yields full _Object data, not just keys
+   */
+  private async *listObjectsWithMetadata(): AsyncIterableIterator<plugins.s3._Object> {
+    let continuationToken: string | undefined;
+
+    do {
+      if (this.isStopped) {
+        return;
+      }
+
+      const command = new plugins.s3.ListObjectsV2Command({
+        Bucket: this.bucketRef.name,
+        Prefix: this.prefix,
+        MaxKeys: this.pageSize,
+        ContinuationToken: continuationToken,
+      });
+
+      const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
+
+      for (const obj of response.Contents || []) {
+        yield obj;
+      }
+
+      continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+  }
+}

package/ts/objectstorage/helpers.ts

@@ -0,0 +1,254 @@
+import * as plugins from './plugins.js';
+import * as interfaces from './interfaces.js';
+
+export const reducePathDescriptorToPath = async (pathDescriptorArg: interfaces.IPathDecriptor): Promise<string> => {
+  let returnPath = ``
+  if (pathDescriptorArg.directory) {
+    if (pathDescriptorArg.path && plugins.path.isAbsolute(pathDescriptorArg.path)) {
+      console.warn('Directory is being ignored when path is absolute.');
+      returnPath = pathDescriptorArg.path;
+    } else if (pathDescriptorArg.path) {
+      returnPath = plugins.path.join(pathDescriptorArg.directory.getBasePath(), pathDescriptorArg.path);
+    }
+  } else if (pathDescriptorArg.path) {
+    returnPath = pathDescriptorArg.path;
+  } else {
+    throw new Error('You must specify either a path or a directory.');
+  }
+  if (returnPath.startsWith('/')) {
+    returnPath = returnPath.substring(1);
+  }
+  return returnPath;
+}
+
+// Storage Descriptor Normalization
+export interface IStorageWarning {
+  code: string;
+  message: string;
+}
+
+export interface INormalizedStorageConfig {
+  endpointUrl: string;
+  host: string;
+  protocol: 'http' | 'https';
+  port?: number;
+  region: string;
+  credentials: {
+    accessKeyId: string;
+    secretAccessKey: string;
+  };
+  forcePathStyle: boolean;
+}
+
+function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warning?: IStorageWarning } {
+  if (typeof value === 'boolean') return { value };
+  if (typeof value === 'string') {
+    const v = value.trim().toLowerCase();
+    if (v === 'true' || v === '1') {
+      return { 
+        value: true, 
+        warning: { 
+          code: 'SBK_S3_COERCED_USESSL', 
+          message: `Coerced useSsl='${value}' (string) to boolean true.` 
+        } 
+      };
+    }
+    if (v === 'false' || v === '0') {
+      return { 
+        value: false, 
+        warning: { 
+          code: 'SBK_S3_COERCED_USESSL', 
+          message: `Coerced useSsl='${value}' (string) to boolean false.` 
+        } 
+      };
+    }
+  }
+  return { value: undefined };
+}
+
+function coercePortMaybe(port: unknown): { value: number | undefined; warning?: IStorageWarning } {
+  if (port === undefined || port === null || port === '') return { value: undefined };
+  const n = typeof port === 'number' ? port : Number(String(port).trim());
+  if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0 || n > 65535) {
+    return { 
+      value: undefined, 
+      warning: { 
+        code: 'SBK_S3_INVALID_PORT', 
+        message: `Invalid port '${String(port)}' - expected integer in [1..65535].` 
+      } 
+    };
+  }
+  return { value: n };
+}
+
+function sanitizeEndpointString(raw: unknown): { value: string; warnings: IStorageWarning[] } {
+  const warnings: IStorageWarning[] = [];
+  let s = String(raw ?? '').trim();
+  if (s !== String(raw ?? '')) {
+    warnings.push({ 
+      code: 'SBK_S3_TRIMMED_ENDPOINT', 
+      message: 'Trimmed surrounding whitespace from endpoint.' 
+    });
+  }
+  return { value: s, warnings };
+}
+
+function parseEndpointHostPort(
+  endpoint: string, 
+  provisionalProtocol: 'http' | 'https'
+): { 
+  hadScheme: boolean; 
+  host: string; 
+  port?: number; 
+  extras: { 
+    droppedPath?: boolean; 
+    droppedQuery?: boolean; 
+    droppedCreds?: boolean 
+  } 
+} {
+  let url: URL | undefined;
+  const extras: { droppedPath?: boolean; droppedQuery?: boolean; droppedCreds?: boolean } = {};
+  
+  // Check if endpoint already has a scheme
+  const hasScheme = /^https?:\/\//i.test(endpoint);
+  
+  // Try parsing as full URL first
+  try {
+    if (hasScheme) {
+      url = new URL(endpoint);
+    } else {
+      // Not a full URL; try host[:port] by attaching provisional scheme
+      // Remove anything after first '/' for safety
+      const cleanEndpoint = endpoint.replace(/\/.*/, '');
+      url = new URL(`${provisionalProtocol}://${cleanEndpoint}`);
+    }
+  } catch (e) {
+    throw new Error(`Unable to parse endpoint '${endpoint}'.`);
+  }
+  
+  // Check for dropped components
+  if (url.username || url.password) extras.droppedCreds = true;
+  if (url.pathname && url.pathname !== '/') extras.droppedPath = true;
+  if (url.search) extras.droppedQuery = true;
+
+  const hadScheme = hasScheme;
+  const host = url.hostname; // hostnames lowercased by URL; IPs preserved
+  const port = url.port ? Number(url.port) : undefined;
+
+  return { hadScheme, host, port, extras };
+}
+
+export function normalizeStorageDescriptor(
+  input: plugins.tsclass.storage.IStorageDescriptor,
+  logger?: { warn: (msg: string) => void }
+): { normalized: INormalizedStorageConfig; warnings: IStorageWarning[] } {
+  const warnings: IStorageWarning[] = [];
+  const logWarn = (w: IStorageWarning) => { 
+    warnings.push(w); 
+    if (logger) {
+      logger.warn(`[SmartBucket] ${w.code}: ${w.message}`);
+    } else {
+      console.warn(`[SmartBucket] ${w.code}: ${w.message}`);
+    }
+  };
+
+  // Coerce and sanitize inputs
+  const { value: coercedUseSsl, warning: useSslWarn } = coerceBooleanMaybe((input as any).useSsl);
+  if (useSslWarn) logWarn(useSslWarn);
+  
+  const { value: coercedPort, warning: portWarn } = coercePortMaybe((input as any).port);
+  if (portWarn) logWarn(portWarn);
+
+  const { value: endpointStr, warnings: endpointSanWarnings } = sanitizeEndpointString((input as any).endpoint);
+  endpointSanWarnings.forEach(logWarn);
+
+  if (!endpointStr) {
+    throw new Error('Storage endpoint is required (got empty string). Provide hostname or URL.');
+  }
+
+  // Provisional protocol selection for parsing host:port forms
+  const provisionalProtocol: 'http' | 'https' = coercedUseSsl === false ? 'http' : 'https';
+
+  const { hadScheme, host, port: epPort, extras } = parseEndpointHostPort(endpointStr, provisionalProtocol);
+
+  if (extras.droppedCreds) {
+    logWarn({ 
+      code: 'SBK_S3_DROPPED_CREDENTIALS', 
+      message: 'Ignored credentials in endpoint URL.' 
+    });
+  }
+  if (extras.droppedPath) {
+    logWarn({ 
+      code: 'SBK_S3_DROPPED_PATH', 
+      message: 'Removed path segment from endpoint URL; S3 endpoint should be host[:port] only.' 
+    });
+  }
+  if (extras.droppedQuery) {
+    logWarn({ 
+      code: 'SBK_S3_DROPPED_QUERY', 
+      message: 'Removed query string from endpoint URL; S3 endpoint should be host[:port] only.' 
+    });
+  }
+
+  // Final protocol decision
+  let finalProtocol: 'http' | 'https';
+  if (hadScheme) {
+    // Scheme from endpoint wins
+    const schemeFromEndpoint = endpointStr.trim().toLowerCase().startsWith('http://') ? 'http' : 'https';
+    finalProtocol = schemeFromEndpoint;
+    if (typeof coercedUseSsl === 'boolean') {
+      const expected = coercedUseSsl ? 'https' : 'http';
+      if (expected !== finalProtocol) {
+        logWarn({ 
+          code: 'SBK_S3_SCHEME_CONFLICT', 
+          message: `useSsl=${String(coercedUseSsl)} conflicts with endpoint scheme '${finalProtocol}'; using endpoint scheme.` 
+        });
+      }
+    }
+  } else {
+    if (typeof coercedUseSsl === 'boolean') {
+      finalProtocol = coercedUseSsl ? 'https' : 'http';
+    } else {
+      finalProtocol = 'https';
+      logWarn({ 
+        code: 'SBK_S3_GUESSED_PROTOCOL', 
+        message: "No scheme in endpoint and useSsl not provided; defaulting to 'https'." 
+      });
+    }
+  }
+
+  // Final port decision
+  let finalPort: number | undefined = undefined;
+  if (coercedPort !== undefined && epPort !== undefined && coercedPort !== epPort) {
+    logWarn({ 
+      code: 'SBK_S3_PORT_CONFLICT', 
+      message: `Port in config (${coercedPort}) conflicts with endpoint port (${epPort}); using config port.` 
+    });
+    finalPort = coercedPort;
+  } else {
+    finalPort = (coercedPort !== undefined) ? coercedPort : epPort;
+  }
+
+  // Build canonical endpoint URL (origin only, no trailing slash)
+  const url = new URL(`${finalProtocol}://${host}`);
+  if (finalPort !== undefined) url.port = String(finalPort);
+  const endpointUrl = url.origin;
+
+  const region = input.region || 'us-east-1';
+
+  return {
+    normalized: {
+      endpointUrl,
+      host,
+      protocol: finalProtocol,
+      port: finalPort,
+      region,
+      credentials: {
+        accessKeyId: input.accessKey,
+        secretAccessKey: input.accessSecret,
+      },
+      forcePathStyle: true,
+    },
+    warnings,
+  };
+}

package/ts/objectstorage/index.ts

@@ -0,0 +1,13 @@
+export * from './classes.smartbucket.js';
+export * from './classes.bucket.js';
+export * from './classes.exactuploaderror.js';
+export * from './classes.exactpathpurgeerror.js';
+export * from './classes.directory.js';
+export * from './classes.file.js';
+export * from './classes.listcursor.js';
+export * from './classes.metadata.js';
+export * from './classes.trash.js';
+export * from './classes.watcher.js';
+export * from './interfaces.js';
+export { ObjectStorageConnection } from './classes.connection.js';
+export type { IObjectStorageConnectionOptions } from './classes.connection.js';