@push.rocks/smartregistry 2.6.0 → 2.8.0

package/ts/pypi/classes.pypiregistry.ts

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
 import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
-import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
-import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
+import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
+import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
 import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
 import type {
   IPypiPackageMetadata,
@@ -24,20 +24,21 @@ export class PypiRegistry extends BaseRegistry {
   private basePath: string = '/pypi';
   private registryUrl: string;
   private logger: Smartlog;
-  private upstream: PypiUpstream | null = null;
+  private upstreamProvider: IUpstreamProvider | null = null;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string = '/pypi',
     registryUrl: string = 'http://localhost:5000',
-    upstreamConfig?: IProtocolUpstreamConfig
+    upstreamProvider?: IUpstreamProvider
   ) {
     super();
     this.storage = storage;
     this.authManager = authManager;
     this.basePath = basePath;
     this.registryUrl = registryUrl;
+    this.upstreamProvider = upstreamProvider || null;
 
     // Initialize logger
     this.logger = new Smartlog({
@@ -51,20 +52,38 @@ export class PypiRegistry extends BaseRegistry {
       }
     });
     this.logger.enableConsole();
+  }
 
-    // Initialize upstream if configured
-    if (upstreamConfig?.enabled) {
-      this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger);
-    }
+  /**
+   * Get upstream for a specific request.
+   * Calls the provider to resolve upstream config dynamically.
+   */
+  private async getUpstreamForRequest(
+    resource: string,
+    resourceType: string,
+    method: string,
+    actor?: IRequestActor
+  ): Promise<PypiUpstream | null> {
+    if (!this.upstreamProvider) return null;
+
+    const config = await this.upstreamProvider.resolveUpstreamConfig({
+      protocol: 'pypi',
+      resource,
+      scope: resource, // For PyPI, package name is the scope
+      actor,
+      method,
+      resourceType,
+    });
+
+    if (!config?.enabled) return null;
+    return new PypiUpstream(config, this.registryUrl, this.logger);
   }
 
   /**
    * Clean up resources (timers, connections, etc.)
    */
   public destroy(): void {
-    if (this.upstream) {
-      this.upstream.stop();
-    }
+    // No persistent upstream to clean up with dynamic provider
   }
 
   public async init(): Promise<void> {
@@ -84,15 +103,23 @@ export class PypiRegistry extends BaseRegistry {
   public async handleRequest(context: IRequestContext): Promise<IResponse> {
     let path = context.path.replace(this.basePath, '');
 
+    // Extract token (Basic Auth or Bearer)
+    const token = await this.extractToken(context);
+
+    // Build actor from context and validated token
+    const actor: IRequestActor = {
+      ...context.actor,
+      userId: token?.userId,
+      ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
+      userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
+    };
+
     // Also handle /simple path prefix
     if (path.startsWith('/simple')) {
       path = path.replace('/simple', '');
-      return this.handleSimpleRequest(path, context);
+      return this.handleSimpleRequest(path, context, actor);
     }
 
-    // Extract token (Basic Auth or Bearer)
-    const token = await this.extractToken(context);
-
     this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
       method: context.method,
       path,
@@ -119,7 +146,7 @@ export class PypiRegistry extends BaseRegistry {
     // Package file download: GET /packages/{package}/{filename}
     const downloadMatch = path.match(/^\/packages\/([^\/]+)\/(.+)$/);
     if (downloadMatch && context.method === 'GET') {
-      return this.handleDownload(downloadMatch[1], downloadMatch[2]);
+      return this.handleDownload(downloadMatch[1], downloadMatch[2], actor);
     }
 
     // Delete package: DELETE /packages/{package}
@@ -156,7 +183,7 @@ export class PypiRegistry extends BaseRegistry {
   /**
    * Handle Simple API requests (PEP 503 HTML or PEP 691 JSON)
    */
-  private async handleSimpleRequest(path: string, context: IRequestContext): Promise<IResponse> {
+  private async handleSimpleRequest(path: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
     // Ensure path ends with / (PEP 503 requirement)
     if (!path.endsWith('/') && !path.includes('.')) {
       return {
@@ -174,7 +201,7 @@ export class PypiRegistry extends BaseRegistry {
     // Package index: /simple/{package}/
     const packageMatch = path.match(/^\/([^\/]+)\/$/);
     if (packageMatch) {
-      return this.handleSimplePackage(packageMatch[1], context);
+      return this.handleSimplePackage(packageMatch[1], context, actor);
     }
 
     return {
@@ -228,46 +255,49 @@ export class PypiRegistry extends BaseRegistry {
    * Handle Simple API package index
    * Returns HTML (PEP 503) or JSON (PEP 691) based on Accept header
    */
-  private async handleSimplePackage(packageName: string, context: IRequestContext): Promise<IResponse> {
+  private async handleSimplePackage(packageName: string, context: IRequestContext, actor?: IRequestActor): Promise<IResponse> {
     const normalized = helpers.normalizePypiPackageName(packageName);
 
     // Get package metadata
     let metadata = await this.storage.getPypiPackageMetadata(normalized);
 
     // Try upstream if not found locally
-    if (!metadata && this.upstream) {
-      const upstreamHtml = await this.upstream.fetchSimplePackage(normalized);
-      if (upstreamHtml) {
-        // Parse the HTML to extract file information and cache it
-        // For now, just return the upstream HTML directly (caching can be improved later)
-        const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
-        const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
-                           acceptHeader.includes('json');
-
-        if (preferJson) {
-          // Try to get JSON format from upstream
-          const upstreamJson = await this.upstream.fetchPackageJson(normalized);
-          if (upstreamJson) {
-            return {
-              status: 200,
-              headers: {
-                'Content-Type': 'application/vnd.pypi.simple.v1+json',
-                'Cache-Control': 'public, max-age=300'
-              },
-              body: upstreamJson,
-            };
+    if (!metadata) {
+      const upstream = await this.getUpstreamForRequest(normalized, 'simple', 'GET', actor);
+      if (upstream) {
+        const upstreamHtml = await upstream.fetchSimplePackage(normalized);
+        if (upstreamHtml) {
+          // Parse the HTML to extract file information and cache it
+          // For now, just return the upstream HTML directly (caching can be improved later)
+          const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
+          const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
+                             acceptHeader.includes('json');
+
+          if (preferJson) {
+            // Try to get JSON format from upstream
+            const upstreamJson = await upstream.fetchPackageJson(normalized);
+            if (upstreamJson) {
+              return {
+                status: 200,
+                headers: {
+                  'Content-Type': 'application/vnd.pypi.simple.v1+json',
+                  'Cache-Control': 'public, max-age=300'
+                },
+                body: upstreamJson,
+              };
+            }
           }
-        }
 
-        // Return HTML format
-        return {
-          status: 200,
-          headers: {
-            'Content-Type': 'text/html; charset=utf-8',
-            'Cache-Control': 'public, max-age=300'
-          },
-          body: upstreamHtml,
-        };
+          // Return HTML format
+          return {
+            status: 200,
+            headers: {
+              'Content-Type': 'text/html; charset=utf-8',
+              'Cache-Control': 'public, max-age=300'
+            },
+            body: upstreamHtml,
+          };
+        }
       }
     }
 
@@ -503,13 +533,29 @@ export class PypiRegistry extends BaseRegistry {
   /**
    * Handle package download
    */
-  private async handleDownload(packageName: string, filename: string): Promise<IResponse> {
+  private async handleDownload(packageName: string, filename: string, actor?: IRequestActor): Promise<IResponse> {
     const normalized = helpers.normalizePypiPackageName(packageName);
-    let fileData = await this.storage.getPypiPackageFile(normalized, filename);
+
+    // Try streaming from local storage first
+    const streamResult = await this.storage.getPypiPackageFileStream(normalized, filename);
+
+    if (streamResult) {
+      return {
+        status: 200,
+        headers: {
+          'Content-Type': 'application/octet-stream',
+          'Content-Disposition': `attachment; filename="${filename}"`,
+          'Content-Length': streamResult.size.toString()
+        },
+        body: streamResult.stream,
+      };
+    }
 
     // Try upstream if not found locally
-    if (!fileData && this.upstream) {
-      fileData = await this.upstream.fetchPackageFile(normalized, filename);
+    let fileData: Buffer | null = null;
+    const upstream = await this.getUpstreamForRequest(normalized, 'file', 'GET', actor);
+    if (upstream) {
+      fileData = await upstream.fetchPackageFile(normalized, filename);
       if (fileData) {
         // Cache locally
         await this.storage.putPypiPackageFile(normalized, filename, fileData);

package/ts/rubygems/classes.rubygemsregistry.ts

@@ -2,8 +2,8 @@ import { Smartlog } from '@push.rocks/smartlog';
 import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
-import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
-import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
+import type { IRequestContext, IResponse, IAuthToken, IRequestActor } from '../core/interfaces.core.js';
+import type { IUpstreamProvider } from '../upstream/interfaces.upstream.js';
 import type {
   IRubyGemsMetadata,
   IRubyGemsVersionMetadata,
@@ -25,20 +25,21 @@ export class RubyGemsRegistry extends BaseRegistry {
   private basePath: string = '/rubygems';
   private registryUrl: string;
   private logger: Smartlog;
-  private upstream: RubygemsUpstream | null = null;
+  private upstreamProvider: IUpstreamProvider | null = null;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string = '/rubygems',
     registryUrl: string = 'http://localhost:5000/rubygems',
-    upstreamConfig?: IProtocolUpstreamConfig
+    upstreamProvider?: IUpstreamProvider
   ) {
     super();
     this.storage = storage;
     this.authManager = authManager;
     this.basePath = basePath;
     this.registryUrl = registryUrl;
+    this.upstreamProvider = upstreamProvider || null;
 
     // Initialize logger
     this.logger = new Smartlog({
@@ -52,20 +53,38 @@ export class RubyGemsRegistry extends BaseRegistry {
       }
     });
     this.logger.enableConsole();
+  }
 
-    // Initialize upstream if configured
-    if (upstreamConfig?.enabled) {
-      this.upstream = new RubygemsUpstream(upstreamConfig, this.logger);
-    }
+  /**
+   * Get upstream for a specific request.
+   * Calls the provider to resolve upstream config dynamically.
+   */
+  private async getUpstreamForRequest(
+    resource: string,
+    resourceType: string,
+    method: string,
+    actor?: IRequestActor
+  ): Promise<RubygemsUpstream | null> {
+    if (!this.upstreamProvider) return null;
+
+    const config = await this.upstreamProvider.resolveUpstreamConfig({
+      protocol: 'rubygems',
+      resource,
+      scope: resource, // gem name is the scope
+      actor,
+      method,
+      resourceType,
+    });
+
+    if (!config?.enabled) return null;
+    return new RubygemsUpstream(config, this.logger);
   }
 
   /**
    * Clean up resources (timers, connections, etc.)
    */
   public destroy(): void {
-    if (this.upstream) {
-      this.upstream.stop();
-    }
+    // No persistent upstream to clean up with dynamic provider
   }
 
   public async init(): Promise<void> {
@@ -95,6 +114,14 @@ export class RubyGemsRegistry extends BaseRegistry {
     // Extract token (Authorization header)
     const token = await this.extractToken(context);
 
+    // Build actor from context and validated token
+    const actor: IRequestActor = {
+      ...context.actor,
+      userId: token?.userId,
+      ip: context.headers['x-forwarded-for'] || context.headers['X-Forwarded-For'],
+      userAgent: context.headers['user-agent'] || context.headers['User-Agent'],
+    };
+
     this.logger.log('debug', `handleRequest: ${context.method} ${path}`, {
       method: context.method,
       path,
@@ -113,13 +140,13 @@ export class RubyGemsRegistry extends BaseRegistry {
     // Info file: GET /info/{gem}
     const infoMatch = path.match(/^\/info\/([^\/]+)$/);
     if (infoMatch && context.method === 'GET') {
-      return this.handleInfoFile(infoMatch[1]);
+      return this.handleInfoFile(infoMatch[1], actor);
     }
 
     // Gem download: GET /gems/{gem}-{version}[-{platform}].gem
     const downloadMatch = path.match(/^\/gems\/(.+\.gem)$/);
     if (downloadMatch && context.method === 'GET') {
-      return this.handleDownload(downloadMatch[1]);
+      return this.handleDownload(downloadMatch[1], actor);
     }
 
     // Legacy specs endpoints (Marshal format)
@@ -232,16 +259,19 @@ export class RubyGemsRegistry extends BaseRegistry {
   /**
    * Handle /info/{gem} endpoint (Compact Index)
    */
-  private async handleInfoFile(gemName: string): Promise<IResponse> {
+  private async handleInfoFile(gemName: string, actor?: IRequestActor): Promise<IResponse> {
     let content = await this.storage.getRubyGemsInfo(gemName);
 
     // Try upstream if not found locally
-    if (!content && this.upstream) {
-      const upstreamInfo = await this.upstream.fetchInfo(gemName);
-      if (upstreamInfo) {
-        // Cache locally
-        await this.storage.putRubyGemsInfo(gemName, upstreamInfo);
-        content = upstreamInfo;
+    if (!content) {
+      const upstream = await this.getUpstreamForRequest(gemName, 'info', 'GET', actor);
+      if (upstream) {
+        const upstreamInfo = await upstream.fetchInfo(gemName);
+        if (upstreamInfo) {
+          // Cache locally
+          await this.storage.putRubyGemsInfo(gemName, upstreamInfo);
+          content = upstreamInfo;
+        }
       }
     }
 
@@ -267,21 +297,36 @@ export class RubyGemsRegistry extends BaseRegistry {
   /**
    * Handle gem file download
    */
-  private async handleDownload(filename: string): Promise<IResponse> {
+  private async handleDownload(filename: string, actor?: IRequestActor): Promise<IResponse> {
     const parsed = helpers.parseGemFilename(filename);
     if (!parsed) {
       return this.errorResponse(400, 'Invalid gem filename');
     }
 
-    let gemData = await this.storage.getRubyGemsGem(
+    // Try streaming from local storage first
+    const streamResult = await this.storage.getRubyGemsGemStream(
       parsed.name,
       parsed.version,
       parsed.platform
     );
 
+    if (streamResult) {
+      return {
+        status: 200,
+        headers: {
+          'Content-Type': 'application/octet-stream',
+          'Content-Disposition': `attachment; filename="${filename}"`,
+          'Content-Length': streamResult.size.toString()
+        },
+        body: streamResult.stream,
+      };
+    }
+
     // Try upstream if not found locally
-    if (!gemData && this.upstream) {
-      gemData = await this.upstream.fetchGem(parsed.name, parsed.version);
+    let gemData: Buffer | null = null;
+    const upstream = await this.getUpstreamForRequest(parsed.name, 'gem', 'GET', actor);
+    if (upstream) {
+      gemData = await upstream.fetchGem(parsed.name, parsed.version);
       if (gemData) {
         // Cache locally
         await this.storage.putRubyGemsGem(parsed.name, parsed.version, gemData, parsed.platform);

package/ts/rubygems/helpers.rubygems.ts

@@ -427,7 +427,7 @@ export async function extractGemMetadata(gemData: Buffer): Promise<{
     // Step 2: Decompress the gzipped metadata
     const gzipTools = new plugins.smartarchive.GzipTools();
     const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
-    const yamlContent = metadataYaml.toString('utf-8');
+    const yamlContent = Buffer.from(metadataYaml).toString('utf-8');
 
     // Step 3: Parse the YAML to extract name, version, platform
     // Look for name: field in YAML
@@ -503,7 +503,7 @@ export async function generateSpecsGz(specs: Array<[string, string, string]>): P
   }
 
   const uncompressed = Buffer.concat(parts);
-  return gzipTools.compress(uncompressed);
+  return Buffer.from(await gzipTools.compress(uncompressed));
 }
 
 /**

package/ts/upstream/classes.upstreamcache.ts

@@ -105,7 +105,7 @@ export class UpstreamCache {
 
     // If not in memory and we have storage, check S3
     if (!entry && this.storage) {
-      entry = await this.loadFromStorage(key);
+      entry = (await this.loadFromStorage(key)) ?? undefined;
       if (entry) {
         // Promote to memory cache
         this.memoryCache.set(key, entry);

package/ts/upstream/interfaces.upstream.ts

@@ -1,4 +1,4 @@
-import type { TRegistryProtocol } from '../core/interfaces.core.js';
+import type { TRegistryProtocol, IRequestActor } from '../core/interfaces.core.js';
 
 /**
  * Scope rule for routing requests to specific upstreams.
@@ -86,6 +86,8 @@ export interface IUpstreamRegistryConfig {
   cache?: Partial<IUpstreamCacheConfig>;
   /** Resilience configuration overrides */
   resilience?: Partial<IUpstreamResilienceConfig>;
+  /** API path prefix for OCI registries (default: /v2). Useful for registries behind reverse proxies. */
+  apiPrefix?: string;
 }
 
 /**
@@ -146,6 +148,8 @@ export interface IUpstreamFetchContext {
   headers: Record<string, string>;
   /** Query parameters */
   query: Record<string, string>;
+  /** Actor performing the request (for cache key isolation) */
+  actor?: IRequestActor;
 }
 
 /**
@@ -193,3 +197,80 @@ export const DEFAULT_RESILIENCE_CONFIG: IUpstreamResilienceConfig = {
   circuitBreakerThreshold: 5,
   circuitBreakerResetMs: 30000,
 };
+
+// ============================================================================
+// Upstream Provider Interfaces
+// ============================================================================
+
+/**
+ * Context for resolving upstream configuration.
+ * Passed to IUpstreamProvider per-request to enable dynamic upstream routing.
+ */
+export interface IUpstreamResolutionContext {
+  /** Protocol being accessed */
+  protocol: TRegistryProtocol;
+  /** Resource identifier (package name, repository, coordinates, etc.) */
+  resource: string;
+  /** Extracted scope (e.g., "company" from "@company/pkg", "myorg" from "myorg/image") */
+  scope: string | null;
+  /** Actor performing the request */
+  actor?: IRequestActor;
+  /** HTTP method */
+  method: string;
+  /** Resource type (packument, tarball, manifest, blob, etc.) */
+  resourceType: string;
+}
+
+/**
+ * Dynamic upstream configuration provider.
+ * Implement this interface to provide per-request upstream routing
+ * based on actor context (user, organization, etc.)
+ *
+ * @example
+ * ```typescript
+ * class OrgUpstreamProvider implements IUpstreamProvider {
+ *   constructor(private db: Database) {}
+ *
+ *   async resolveUpstreamConfig(ctx: IUpstreamResolutionContext) {
+ *     if (ctx.actor?.orgId) {
+ *       const orgConfig = await this.db.getOrgUpstream(ctx.actor.orgId, ctx.protocol);
+ *       if (orgConfig) return orgConfig;
+ *     }
+ *     return this.db.getDefaultUpstream(ctx.protocol);
+ *   }
+ * }
+ * ```
+ */
+export interface IUpstreamProvider {
+  /** Optional initialization */
+  init?(): Promise<void>;
+
+  /**
+   * Resolve upstream configuration for a request.
+   * @param context - Information about the current request
+   * @returns Upstream config to use, or null to skip upstream lookup
+   */
+  resolveUpstreamConfig(context: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null>;
+}
+
+/**
+ * Static upstream provider for simple configurations.
+ * Use this when you have fixed upstream registries that don't change per-request.
+ *
+ * @example
+ * ```typescript
+ * const provider = new StaticUpstreamProvider({
+ *   npm: {
+ *     enabled: true,
+ *     upstreams: [{ id: 'npmjs', url: 'https://registry.npmjs.org', priority: 1, enabled: true, auth: { type: 'none' } }],
+ *   },
+ * });
+ * ```
+ */
+export class StaticUpstreamProvider implements IUpstreamProvider {
+  constructor(private configs: Partial<Record<TRegistryProtocol, IProtocolUpstreamConfig>>) {}
+
+  async resolveUpstreamConfig(ctx: IUpstreamResolutionContext): Promise<IProtocolUpstreamConfig | null> {
+    return this.configs[ctx.protocol] ?? null;
+  }
+}

package/npmextra.json

@@ -1,18 +0,0 @@
-{
-  "gitzone": {
-    "projectType": "npm",
-    "module": {
-      "githost": "code.foss.global",
-      "gitscope": "push.rocks",
-      "gitrepo": "smartregistry",
-      "description": "a registry for npm modules and oci images",
-      "npmPackagename": "@push.rocks/smartregistry",
-      "license": "MIT",
-      "projectDomain": "push.rocks"
-    }
-  },
-  "npmci": {
-    "npmGlobalTools": [],
-    "npmAccessLevel": "public"
-  }
-}