@push.rocks/smartregistry 2.6.0 → 2.8.0

package/ts/core/classes.registrystorage.ts

@@ -34,8 +34,8 @@ import type {
  * ```
  */
 export class RegistryStorage implements IStorageBackend {
-  private smartBucket: plugins.smartbucket.SmartBucket;
-  private bucket: plugins.smartbucket.Bucket;
+  private smartBucket!: plugins.smartbucket.SmartBucket;
+  private bucket!: plugins.smartbucket.Bucket;
   private bucketName: string;
   private hooks?: IStorageHooks;
 
@@ -1266,4 +1266,135 @@ export class RegistryStorage implements IStorageBackend {
   private getRubyGemsMetadataPath(gemName: string): string {
     return `rubygems/metadata/${gemName}/metadata.json`;
   }
+
+  // ========================================================================
+  // STREAMING METHODS (Web Streams API)
+  // ========================================================================
+
+  /**
+   * Get an object as a ReadableStream. Returns null if not found.
+   */
+  public async getObjectStream(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
+    try {
+      const stat = await this.bucket.fastStat({ path: key });
+      const size = stat.ContentLength ?? 0;
+      const stream = await this.bucket.fastGetStream({ path: key }, 'webstream');
+
+      // Call afterGet hook (non-blocking)
+      if (this.hooks?.afterGet) {
+        const context = this.currentContext;
+        if (context) {
+          this.hooks.afterGet({
+            operation: 'get',
+            key,
+            protocol: context.protocol,
+            actor: context.actor,
+            metadata: context.metadata,
+            timestamp: new Date(),
+          }).catch(() => {});
+        }
+      }
+
+      return { stream: stream as ReadableStream<Uint8Array>, size };
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Store an object from a ReadableStream.
+   */
+  public async putObjectStream(key: string, stream: ReadableStream<Uint8Array>): Promise<void> {
+    if (this.hooks?.beforePut) {
+      const context = this.currentContext;
+      if (context) {
+        const hookContext: IStorageHookContext = {
+          operation: 'put',
+          key,
+          protocol: context.protocol,
+          actor: context.actor,
+          metadata: context.metadata,
+          timestamp: new Date(),
+        };
+        const result = await this.hooks.beforePut(hookContext);
+        if (!result.allowed) {
+          throw new Error(result.reason || 'Storage operation denied by hook');
+        }
+      }
+    }
+
+    // Convert WebStream to Node Readable at the S3 SDK boundary
+    // AWS SDK v3 PutObjectCommand requires a Node.js Readable (not WebStream)
+    const { Readable } = await import('stream');
+    const nodeStream = Readable.fromWeb(stream as any);
+    await this.bucket.fastPutStream({
+      path: key,
+      readableStream: nodeStream,
+      overwrite: true,
+    });
+
+    if (this.hooks?.afterPut) {
+      const context = this.currentContext;
+      if (context) {
+        this.hooks.afterPut({
+          operation: 'put',
+          key,
+          protocol: context.protocol,
+          actor: context.actor,
+          metadata: context.metadata,
+          timestamp: new Date(),
+        }).catch(() => {});
+      }
+    }
+  }
+
+  /**
+   * Get object size without reading data (S3 HEAD request).
+   */
+  public async getObjectSize(key: string): Promise<number | null> {
+    try {
+      const stat = await this.bucket.fastStat({ path: key });
+      return stat.ContentLength ?? null;
+    } catch {
+      return null;
+    }
+  }
+
+  // ---- Protocol-specific streaming wrappers ----
+
+  public async getOciBlobStream(digest: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
+    return this.getObjectStream(this.getOciBlobPath(digest));
+  }
+
+  public async putOciBlobStream(digest: string, stream: ReadableStream<Uint8Array>): Promise<void> {
+    return this.putObjectStream(this.getOciBlobPath(digest), stream);
+  }
+
+  public async getOciBlobSize(digest: string): Promise<number | null> {
+    return this.getObjectSize(this.getOciBlobPath(digest));
+  }
+
+  public async getNpmTarballStream(packageName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
+    return this.getObjectStream(this.getNpmTarballPath(packageName, version));
+  }
+
+  public async getMavenArtifactStream(groupId: string, artifactId: string, version: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
+    return this.getObjectStream(this.getMavenArtifactPath(groupId, artifactId, version, filename));
+  }
+
+  public async getCargoCrateStream(crateName: string, version: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
+    return this.getObjectStream(this.getCargoCratePath(crateName, version));
+  }
+
+  public async getComposerPackageZipStream(vendorPackage: string, reference: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
+    return this.getObjectStream(this.getComposerZipPath(vendorPackage, reference));
+  }
+
+  public async getPypiPackageFileStream(packageName: string, filename: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
+    return this.getObjectStream(this.getPypiPackageFilePath(packageName, filename));
+  }
+
+  public async getRubyGemsGemStream(gemName: string, version: string, platform?: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null> {
+    return this.getObjectStream(this.getRubyGemsGemPath(gemName, version, platform));
+  }
 }

package/ts/core/helpers.stream.ts

@@ -0,0 +1,63 @@
+import * as crypto from 'crypto';
+
+/**
+ * Convert Buffer, Uint8Array, string, or JSON object to a ReadableStream<Uint8Array>.
+ */
+export function toReadableStream(data: Buffer | Uint8Array | string | object): ReadableStream<Uint8Array> {
+  const buf = Buffer.isBuffer(data)
+    ? data
+    : data instanceof Uint8Array
+      ? Buffer.from(data)
+      : typeof data === 'string'
+        ? Buffer.from(data, 'utf-8')
+        : Buffer.from(JSON.stringify(data), 'utf-8');
+  return new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(new Uint8Array(buf));
+      controller.close();
+    },
+  });
+}
+
+/**
+ * Consume a ReadableStream into a Buffer.
+ */
+export async function streamToBuffer(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (value) chunks.push(value);
+  }
+  return Buffer.concat(chunks);
+}
+
+/**
+ * Consume a ReadableStream into a parsed JSON object.
+ */
+export async function streamToJson<T = any>(stream: ReadableStream<Uint8Array>): Promise<T> {
+  const buf = await streamToBuffer(stream);
+  return JSON.parse(buf.toString('utf-8'));
+}
+
+/**
+ * Create a TransformStream that incrementally hashes data passing through.
+ * Data flows through unchanged; the digest is available after the stream completes.
+ */
+export function createHashTransform(algorithm: string = 'sha256'): {
+  transform: TransformStream<Uint8Array, Uint8Array>;
+  getDigest: () => string;
+} {
+  const hash = crypto.createHash(algorithm);
+  const transform = new TransformStream<Uint8Array, Uint8Array>({
+    transform(chunk, controller) {
+      hash.update(chunk);
+      controller.enqueue(chunk);
+    },
+  });
+  return {
+    transform,
+    getDigest: () => hash.digest('hex'),
+  };
+}

package/ts/core/index.ts

@@ -12,6 +12,9 @@ export { DefaultAuthProvider } from './classes.defaultauthprovider.js';
 // Storage interfaces and hooks
 export * from './interfaces.storage.js';
 
+// Stream helpers
+export { toReadableStream, streamToBuffer, streamToJson, createHashTransform } from './helpers.stream.js';
+
 // Classes
 export { BaseRegistry } from './classes.baseregistry.js';
 export { RegistryStorage } from './classes.registrystorage.js';

package/ts/core/interfaces.core.ts

@@ -3,7 +3,7 @@
  */
 
 import type * as plugins from '../plugins.js';
-import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
+import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
 import type { IAuthProvider } from './interfaces.auth.js';
 import type { IStorageHooks } from './interfaces.storage.js';
 
@@ -88,9 +88,8 @@ export interface IAuthConfig {
 export interface IProtocolConfig {
   enabled: boolean;
   basePath: string;
+  registryUrl?: string;
   features?: Record<string, boolean>;
-  /** Upstream registry configuration for proxying/caching */
-  upstream?: IProtocolUpstreamConfig;
 }
 
 /**
@@ -113,6 +112,13 @@ export interface IRegistryConfig {
    */
   storageHooks?: IStorageHooks;
 
+  /**
+   * Dynamic upstream configuration provider.
+   * Called per-request to resolve which upstream registries to use.
+   * Use StaticUpstreamProvider for simple static configurations.
+   */
+  upstreamProvider?: IUpstreamProvider;
+
   oci?: IProtocolConfig;
   npm?: IProtocolConfig;
   maven?: IProtocolConfig;
@@ -155,6 +161,21 @@ export interface IStorageBackend {
    * Get object metadata
    */
   getMetadata(key: string): Promise<Record<string, string> | null>;
+
+  /**
+   * Get an object as a ReadableStream. Returns null if not found.
+   */
+  getObjectStream?(key: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number } | null>;
+
+  /**
+   * Store an object from a ReadableStream.
+   */
+  putObjectStream?(key: string, stream: ReadableStream<Uint8Array>): Promise<void>;
+
+  /**
+   * Get object size without reading data (S3 HEAD request).
+   */
+  getObjectSize?(key: string): Promise<number | null>;
 }
 
 /**
@@ -210,10 +231,13 @@ export interface IRequestContext {
 }
 
 /**
- * Base response structure
+ * Base response structure.
+ * `body` is always a `ReadableStream<Uint8Array>` at the public API boundary.
+ * Internal handlers may return Buffer/string/object — the SmartRegistry orchestrator
+ * auto-wraps them via `toReadableStream()` before returning to the caller.
  */
 export interface IResponse {
   status: number;
   headers: Record<string, string>;
-  body?: any;
+  body?: ReadableStream<Uint8Array> | any;
 }

package/ts/maven/classes.mavenregistry.ts

@@ -6,8 +6,8 @@
 import { BaseRegistry } from '../core/classes.baseregistry.js';
 import type { RegistryStorage } from '../core/classes.registrystorage.js';
 import type { AuthManager } from '../core/classes.authmanager.js';
-import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
-import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
+import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
+import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
 import { toBuffer } from '../core/helpers.buffer.js';
 import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js';
 import {
@@ -33,34 +33,64 @@ export class MavenRegistry extends BaseRegistry {
   private authManager: AuthManager;
   private basePath: string = '/maven';
   private registryUrl: string;
-  private upstream: MavenUpstream | null = null;
+  private upstreamProvider: IUpstreamProvider | null = null;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string,
     registryUrl: string,
-    upstreamConfig?: IProtocolUpstreamConfig
+    upstreamProvider?: IUpstreamProvider
   ) {
     super();
     this.storage = storage;
     this.authManager = authManager;
     this.basePath = basePath;
     this.registryUrl = registryUrl;
+    this.upstreamProvider = upstreamProvider || null;
+  }
 
-    // Initialize upstream if configured
-    if (upstreamConfig?.enabled) {
-      this.upstream = new MavenUpstream(upstreamConfig);
-    }
+  /**
+   * Extract scope from Maven coordinates.
+   * For Maven, the groupId is the scope.
+   * @example "com.example" from "com.example:my-lib"
+   */
+  private extractScope(groupId: string): string | null {
+    return groupId || null;
+  }
+
+  /**
+   * Get upstream for a specific request.
+   * Calls the provider to resolve upstream config dynamically.
+   */
+  private async getUpstreamForRequest(
+    resource: string,
+    resourceType: string,
+    method: string,
+    actor?: IRequestActor
+  ): Promise<MavenUpstream | null> {
+    if (!this.upstreamProvider) return null;
+
+    // For Maven, resource is "groupId:artifactId"
+    const [groupId] = resource.split(':');
+    const config = await this.upstreamProvider.resolveUpstreamConfig({
+      protocol: 'maven',
+      resource,
+      scope: this.extractScope(groupId),
+      actor,
+      method,
+      resourceType,
+    });
+
+    if (!config?.enabled) return null;
+    return new MavenUpstream(config);
   }
 
   /**
    * Clean up resources (timers, connections, etc.)
    */
   public destroy(): void {
-    if (this.upstream) {
-      this.upstream.stop();
-    }
+    // No persistent upstream to clean up with dynamic provider
   }
 
   public async init(): Promise<void> {
@@ -85,13 +115,21 @@ export class MavenRegistry extends BaseRegistry {
       token = await this.authManager.validateToken(tokenString, 'maven');
     }
 
+    // Build actor from context and validated token
+    const actor: IRequestActor = {
+      ...context.actor,
+      userId: token?.userId,
+      ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
+      userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
+    };
+
     // Parse path to determine request type
     const coordinate = pathToGAV(path);
 
     if (!coordinate) {
       // Not a valid artifact path, could be metadata or root
       if (path.endsWith('/maven-metadata.xml')) {
-        return this.handleMetadataRequest(context.method, path, token);
+        return this.handleMetadataRequest(context.method, path, token, actor);
       }
 
       return {
@@ -108,7 +146,7 @@ export class MavenRegistry extends BaseRegistry {
     }
 
     // Handle artifact requests (JAR, POM, WAR, etc.)
-    return this.handleArtifactRequest(context.method, coordinate, token, context.body);
+    return this.handleArtifactRequest(context.method, coordinate, token, context.body, actor);
   }
 
   protected async checkPermission(
@@ -128,7 +166,8 @@ export class MavenRegistry extends BaseRegistry {
     method: string,
     coordinate: IMavenCoordinate,
     token: IAuthToken | null,
-    body?: Buffer | any
+    body?: Buffer | any,
+    actor?: IRequestActor
   ): Promise<IResponse> {
     const { groupId, artifactId, version } = coordinate;
     const filename = buildFilename(coordinate);
@@ -139,7 +178,7 @@ export class MavenRegistry extends BaseRegistry {
       case 'HEAD':
         // Maven repositories typically allow anonymous reads
         return method === 'GET'
-          ? this.getArtifact(groupId, artifactId, version, filename)
+          ? this.getArtifact(groupId, artifactId, version, filename, actor)
           : this.headArtifact(groupId, artifactId, version, filename);
 
       case 'PUT':
@@ -211,7 +250,8 @@ export class MavenRegistry extends BaseRegistry {
   private async handleMetadataRequest(
     method: string,
     path: string,
-    token: IAuthToken | null
+    token: IAuthToken | null,
+    actor?: IRequestActor
   ): Promise<IResponse> {
     // Parse path to extract groupId and artifactId
     // Path format: /com/example/my-lib/maven-metadata.xml
@@ -232,7 +272,7 @@ export class MavenRegistry extends BaseRegistry {
     if (method === 'GET') {
       // Metadata is usually public (read permission optional)
       // Some registries allow anonymous metadata access
-      return this.getMetadata(groupId, artifactId);
+      return this.getMetadata(groupId, artifactId, actor);
     }
 
     return {
@@ -250,16 +290,33 @@ export class MavenRegistry extends BaseRegistry {
     groupId: string,
     artifactId: string,
     version: string,
-    filename: string
+    filename: string,
+    actor?: IRequestActor
   ): Promise<IResponse> {
-    let data = await this.storage.getMavenArtifact(groupId, artifactId, version, filename);
+    // Try local storage first (streaming)
+    const streamResult = await this.storage.getMavenArtifactStream(groupId, artifactId, version, filename);
+    if (streamResult) {
+      const ext = filename.split('.').pop() || '';
+      const contentType = this.getContentType(ext);
+      return {
+        status: 200,
+        headers: {
+          'Content-Type': contentType,
+          'Content-Length': streamResult.size.toString(),
+        },
+        body: streamResult.stream,
+      };
+    }
 
     // Try upstream if not found locally
-    if (!data && this.upstream) {
+    let data: Buffer | null = null;
+    const resource = `${groupId}:${artifactId}`;
+    const upstream = await this.getUpstreamForRequest(resource, 'artifact', 'GET', actor);
+    if (upstream) {
       // Parse the filename to extract extension and classifier
       const { extension, classifier } = this.parseFilename(filename, artifactId, version);
       if (extension) {
-        data = await this.upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
+        data = await upstream.fetchArtifact(groupId, artifactId, version, extension, classifier);
         if (data) {
           // Cache the artifact locally
           await this.storage.putMavenArtifact(groupId, artifactId, version, filename, data);
@@ -495,16 +552,20 @@ export class MavenRegistry extends BaseRegistry {
   // METADATA OPERATIONS
   // ========================================================================
 
-  private async getMetadata(groupId: string, artifactId: string): Promise<IResponse> {
+  private async getMetadata(groupId: string, artifactId: string, actor?: IRequestActor): Promise<IResponse> {
     let metadataBuffer = await this.storage.getMavenMetadata(groupId, artifactId);
 
     // Try upstream if not found locally
-    if (!metadataBuffer && this.upstream) {
-      const upstreamMetadata = await this.upstream.fetchMetadata(groupId, artifactId);
-      if (upstreamMetadata) {
-        metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8');
-        // Cache the metadata locally
-        await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer);
+    if (!metadataBuffer) {
+      const resource = `${groupId}:${artifactId}`;
+      const upstream = await this.getUpstreamForRequest(resource, 'metadata', 'GET', actor);
+      if (upstream) {
+        const upstreamMetadata = await upstream.fetchMetadata(groupId, artifactId);
+        if (upstreamMetadata) {
+          metadataBuffer = Buffer.from(upstreamMetadata, 'utf-8');
+          // Cache the metadata locally
+          await this.storage.putMavenMetadata(groupId, artifactId, metadataBuffer);
+        }
       }
     }