@push.rocks/smartregistry 2.2.3 → 2.4.0

package/ts/npm/classes.npmregistry.ts

@@ -3,6 +3,8 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
 import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
+import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
+import { NpmUpstream } from './classes.npmupstream.js';
 import type {
   IPackument,
   INpmVersion,
@@ -25,12 +27,14 @@ export class NpmRegistry extends BaseRegistry {
   private basePath: string = '/npm';
   private registryUrl: string;
   private logger: Smartlog;
+  private upstream: NpmUpstream | null = null;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string = '/npm',
-    registryUrl: string = 'http://localhost:5000/npm'
+    registryUrl: string = 'http://localhost:5000/npm',
+    upstreamConfig?: IProtocolUpstreamConfig
   ) {
     super();
     this.storage = storage;
@@ -50,6 +54,14 @@ export class NpmRegistry extends BaseRegistry {
       }
     });
     this.logger.enableConsole();
+
+    // Initialize upstream if configured
+    if (upstreamConfig?.enabled) {
+      this.upstream = new NpmUpstream(upstreamConfig, registryUrl, this.logger);
+      this.logger.log('info', 'NPM upstream initialized', {
+        upstreams: upstreamConfig.upstreams.map(u => u.name),
+      });
+    }
   }
 
   public async init(): Promise<void> {
@@ -209,13 +221,28 @@ export class NpmRegistry extends BaseRegistry {
     token: IAuthToken | null,
     query: Record<string, string>
   ): Promise<IResponse> {
-    const packument = await this.storage.getNpmPackument(packageName);
+    let packument = await this.storage.getNpmPackument(packageName);
     this.logger.log('debug', `getPackument: ${packageName}`, {
       packageName,
       found: !!packument,
       versions: packument ? Object.keys(packument.versions).length : 0
     });
 
+    // If not found locally, try upstream
+    if (!packument && this.upstream) {
+      this.logger.log('debug', `getPackument: fetching from upstream`, { packageName });
+      const upstreamPackument = await this.upstream.fetchPackument(packageName);
+      if (upstreamPackument) {
+        this.logger.log('debug', `getPackument: found in upstream`, {
+          packageName,
+          versions: Object.keys(upstreamPackument.versions || {}).length
+        });
+        packument = upstreamPackument;
+        // Optionally cache the packument locally (without tarballs)
+        // We don't store tarballs here - they'll be fetched on demand
+      }
+    }
+
     if (!packument) {
       return {
         status: 404,
@@ -255,11 +282,21 @@ export class NpmRegistry extends BaseRegistry {
     token: IAuthToken | null
   ): Promise<IResponse> {
     this.logger.log('debug', 'handlePackageVersion', { packageName, version });
-    const packument = await this.storage.getNpmPackument(packageName);
+    let packument = await this.storage.getNpmPackument(packageName);
     this.logger.log('debug', 'handlePackageVersion packument', { found: !!packument });
     if (packument) {
       this.logger.log('debug', 'handlePackageVersion versions', { versions: Object.keys(packument.versions || {}) });
     }
+
+    // If not found locally, try upstream
+    if (!packument && this.upstream) {
+      this.logger.log('debug', 'handlePackageVersion: fetching from upstream', { packageName });
+      const upstreamPackument = await this.upstream.fetchPackument(packageName);
+      if (upstreamPackument) {
+        packument = upstreamPackument;
+      }
+    }
+
     if (!packument) {
       return {
         status: 404,
@@ -529,7 +566,7 @@ export class NpmRegistry extends BaseRegistry {
     token: IAuthToken | null
   ): Promise<IResponse> {
     // Extract version from filename: package-name-1.0.0.tgz
-    const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/);
+    const versionMatch = filename.match(/-([\d.]+(?:-[a-z0-9.]+)?)\.tgz$/i);
     if (!versionMatch) {
       return {
         status: 400,
@@ -539,7 +576,26 @@ export class NpmRegistry extends BaseRegistry {
     }
 
     const version = versionMatch[1];
-    const tarball = await this.storage.getNpmTarball(packageName, version);
+    let tarball = await this.storage.getNpmTarball(packageName, version);
+
+    // If not found locally, try upstream
+    if (!tarball && this.upstream) {
+      this.logger.log('debug', 'handleTarballDownload: fetching from upstream', {
+        packageName,
+        version,
+      });
+      const upstreamTarball = await this.upstream.fetchTarball(packageName, version);
+      if (upstreamTarball) {
+        tarball = upstreamTarball;
+        // Cache the tarball locally for future requests
+        await this.storage.putNpmTarball(packageName, version, tarball);
+        this.logger.log('debug', 'handleTarballDownload: cached tarball locally', {
+          packageName,
+          version,
+          size: tarball.length,
+        });
+      }
+    }
 
     if (!tarball) {
       return {

package/ts/npm/classes.npmupstream.ts

@@ -0,0 +1,260 @@
+import * as plugins from '../plugins.js';
+import { BaseUpstream } from '../upstream/classes.baseupstream.js';
+import type {
+  IProtocolUpstreamConfig,
+  IUpstreamFetchContext,
+  IUpstreamResult,
+  IUpstreamRegistryConfig,
+} from '../upstream/interfaces.upstream.js';
+import type { IPackument, INpmVersion } from './interfaces.npm.js';
+
+/**
+ * NPM-specific upstream implementation.
+ *
+ * Handles:
+ * - Package metadata (packument) fetching
+ * - Tarball proxying
+ * - Scoped package routing (@scope/* patterns)
+ * - NPM-specific URL rewriting
+ */
+export class NpmUpstream extends BaseUpstream {
+  protected readonly protocolName = 'npm';
+
+  /** Local registry URL for rewriting tarball URLs */
+  private readonly localRegistryUrl: string;
+
+  constructor(
+    config: IProtocolUpstreamConfig,
+    localRegistryUrl: string,
+    logger?: plugins.smartlog.Smartlog,
+  ) {
+    super(config, logger);
+    this.localRegistryUrl = localRegistryUrl;
+  }
+
+  /**
+   * Fetch a packument from upstream registries.
+   */
+  public async fetchPackument(packageName: string): Promise<IPackument | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'npm',
+      resource: packageName,
+      resourceType: 'packument',
+      path: `/${encodeURIComponent(packageName).replace('%40', '@')}`,
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    // Parse and process packument
+    let packument: IPackument;
+    if (Buffer.isBuffer(result.body)) {
+      packument = JSON.parse(result.body.toString('utf8'));
+    } else {
+      packument = result.body;
+    }
+
+    // Rewrite tarball URLs to point to local registry
+    packument = this.rewriteTarballUrls(packument);
+
+    return packument;
+  }
+
+  /**
+   * Fetch a specific version from upstream registries.
+   */
+  public async fetchVersion(packageName: string, version: string): Promise<INpmVersion | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'npm',
+      resource: packageName,
+      resourceType: 'version',
+      path: `/${encodeURIComponent(packageName).replace('%40', '@')}/${version}`,
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    let versionData: INpmVersion;
+    if (Buffer.isBuffer(result.body)) {
+      versionData = JSON.parse(result.body.toString('utf8'));
+    } else {
+      versionData = result.body;
+    }
+
+    // Rewrite tarball URL
+    if (versionData.dist?.tarball) {
+      versionData.dist.tarball = this.rewriteSingleTarballUrl(
+        packageName,
+        versionData.version,
+        versionData.dist.tarball,
+      );
+    }
+
+    return versionData;
+  }
+
+  /**
+   * Fetch a tarball from upstream registries.
+   */
+  public async fetchTarball(packageName: string, version: string): Promise<Buffer | null> {
+    // First, try to get the tarball URL from packument
+    const packument = await this.fetchPackument(packageName);
+    let tarballPath: string;
+
+    if (packument?.versions?.[version]?.dist?.tarball) {
+      // Extract path from original (upstream) tarball URL
+      const tarballUrl = packument.versions[version].dist.tarball;
+      try {
+        const url = new URL(tarballUrl);
+        tarballPath = url.pathname;
+      } catch {
+        // Fallback to standard NPM tarball path
+        tarballPath = this.buildTarballPath(packageName, version);
+      }
+    } else {
+      tarballPath = this.buildTarballPath(packageName, version);
+    }
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'npm',
+      resource: packageName,
+      resourceType: 'tarball',
+      path: tarballPath,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
+  }
+
+  /**
+   * Search packages in upstream registries.
+   */
+  public async search(text: string, size: number = 20, from: number = 0): Promise<any | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'npm',
+      resource: '*',
+      resourceType: 'search',
+      path: '/-/v1/search',
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {
+        text,
+        size: size.toString(),
+        from: from.toString(),
+      },
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return JSON.parse(result.body.toString('utf8'));
+    }
+
+    return result.body;
+  }
+
+  /**
+   * Build the standard NPM tarball path.
+   */
+  private buildTarballPath(packageName: string, version: string): string {
+    // NPM uses: /{package}/-/{package-name}-{version}.tgz
+    // For scoped packages: /@scope/name/-/name-version.tgz
+    if (packageName.startsWith('@')) {
+      const [scope, name] = packageName.split('/');
+      return `/${scope}/${name}/-/${name}-${version}.tgz`;
+    } else {
+      return `/${packageName}/-/${packageName}-${version}.tgz`;
+    }
+  }
+
+  /**
+   * Rewrite all tarball URLs in a packument to point to local registry.
+   */
+  private rewriteTarballUrls(packument: IPackument): IPackument {
+    if (!packument.versions) {
+      return packument;
+    }
+
+    const rewritten = { ...packument };
+    rewritten.versions = {};
+
+    for (const [version, versionData] of Object.entries(packument.versions)) {
+      const newVersionData = { ...versionData };
+      if (newVersionData.dist?.tarball) {
+        newVersionData.dist = {
+          ...newVersionData.dist,
+          tarball: this.rewriteSingleTarballUrl(
+            packument.name,
+            version,
+            newVersionData.dist.tarball,
+          ),
+        };
+      }
+      rewritten.versions[version] = newVersionData;
+    }
+
+    return rewritten;
+  }
+
+  /**
+   * Rewrite a single tarball URL to point to local registry.
+   */
+  private rewriteSingleTarballUrl(
+    packageName: string,
+    version: string,
+    _originalUrl: string,
+  ): string {
+    // Generate local tarball URL
+    // Format: {localRegistryUrl}/{package}/-/{package-name}-{version}.tgz
+    const safeName = packageName.replace('@', '').replace('/', '-');
+    return `${this.localRegistryUrl}/${packageName}/-/${safeName}-${version}.tgz`;
+  }
+
+  /**
+   * Override URL building for NPM-specific handling.
+   */
+  protected buildUpstreamUrl(
+    upstream: IUpstreamRegistryConfig,
+    context: IUpstreamFetchContext,
+  ): string {
+    // NPM registries often don't have trailing slashes
+    let baseUrl = upstream.url;
+    if (baseUrl.endsWith('/')) {
+      baseUrl = baseUrl.slice(0, -1);
+    }
+
+    return `${baseUrl}${context.path}`;
+  }
+}

package/ts/npm/index.ts

@@ -3,4 +3,5 @@
  */
 
 export { NpmRegistry } from './classes.npmregistry.js';
+export { NpmUpstream } from './classes.npmupstream.js';
 export * from './interfaces.npm.js';

package/ts/oci/classes.ociregistry.ts

@@ -1,7 +1,10 @@
+import { Smartlog } from '@push.rocks/smartlog';
 import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
 import type { IRequestContext, IResponse, IAuthToken, IRegistryError } from '../core/interfaces.core.js';
+import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
+import { OciUpstream } from './classes.ociupstream.js';
 import type {
   IUploadSession,
   IOciManifest,
@@ -21,18 +24,42 @@ export class OciRegistry extends BaseRegistry {
   private basePath: string = '/oci';
   private cleanupInterval?: NodeJS.Timeout;
   private ociTokens?: { realm: string; service: string };
+  private upstream: OciUpstream | null = null;
+  private logger: Smartlog;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string = '/oci',
-    ociTokens?: { realm: string; service: string }
+    ociTokens?: { realm: string; service: string },
+    upstreamConfig?: IProtocolUpstreamConfig
   ) {
     super();
     this.storage = storage;
     this.authManager = authManager;
     this.basePath = basePath;
     this.ociTokens = ociTokens;
+
+    // Initialize logger
+    this.logger = new Smartlog({
+      logContext: {
+        company: 'push.rocks',
+        companyunit: 'smartregistry',
+        containerName: 'oci-registry',
+        environment: (process.env.NODE_ENV as any) || 'development',
+        runtime: 'node',
+        zone: 'oci'
+      }
+    });
+    this.logger.enableConsole();
+
+    // Initialize upstream if configured
+    if (upstreamConfig?.enabled) {
+      this.upstream = new OciUpstream(upstreamConfig, basePath, this.logger);
+      this.logger.log('info', 'OCI upstream initialized', {
+        upstreams: upstreamConfig.upstreams.map(u => u.name),
+      });
+    }
   }
 
   public async init(): Promise<void> {
@@ -302,16 +329,50 @@ export class OciRegistry extends BaseRegistry {
     if (!reference.startsWith('sha256:')) {
       const tags = await this.getTagsData(repository);
       digest = tags[reference];
-      if (!digest) {
-        return {
-          status: 404,
-          headers: {},
-          body: this.createError('MANIFEST_UNKNOWN', 'Manifest not found'),
-        };
+    }
+
+    // Try local storage first (if we have a digest)
+    let manifestData: Buffer | null = null;
+    let contentType: string | null = null;
+
+    if (digest) {
+      manifestData = await this.storage.getOciManifest(repository, digest);
+      if (manifestData) {
+        contentType = await this.storage.getOciManifestContentType(repository, digest);
+        if (!contentType) {
+          contentType = this.detectManifestContentType(manifestData);
+        }
+      }
+    }
+
+    // If not found locally, try upstream
+    if (!manifestData && this.upstream) {
+      this.logger.log('debug', 'getManifest: fetching from upstream', { repository, reference });
+      const upstreamResult = await this.upstream.fetchManifest(repository, reference);
+      if (upstreamResult) {
+        manifestData = Buffer.from(JSON.stringify(upstreamResult.manifest), 'utf8');
+        contentType = upstreamResult.contentType;
+        digest = upstreamResult.digest;
+
+        // Cache the manifest locally
+        await this.storage.putOciManifest(repository, digest, manifestData, contentType);
+
+        // If reference is a tag, update tags mapping
+        if (!reference.startsWith('sha256:')) {
+          const tags = await this.getTagsData(repository);
+          tags[reference] = digest;
+          const tagsPath = `oci/tags/${repository}/tags.json`;
+          await this.storage.putObject(tagsPath, Buffer.from(JSON.stringify(tags), 'utf-8'));
+        }
+
+        this.logger.log('debug', 'getManifest: cached manifest locally', {
+          repository,
+          reference,
+          digest,
+        });
       }
     }
 
-    const manifestData = await this.storage.getOciManifest(repository, digest);
     if (!manifestData) {
       return {
         status: 404,
@@ -320,17 +381,10 @@ export class OciRegistry extends BaseRegistry {
       };
     }
 
-    // Get stored content type, falling back to detecting from manifest content
-    let contentType = await this.storage.getOciManifestContentType(repository, digest);
-    if (!contentType) {
-      // Fallback: detect content type from manifest content
-      contentType = this.detectManifestContentType(manifestData);
-    }
-
     return {
       status: 200,
       headers: {
-        'Content-Type': contentType,
+        'Content-Type': contentType || 'application/vnd.oci.image.manifest.v1+json',
         'Docker-Content-Digest': digest,
       },
       body: manifestData,
@@ -466,7 +520,25 @@ export class OciRegistry extends BaseRegistry {
       return this.createUnauthorizedResponse(repository, 'pull');
     }
 
-    const data = await this.storage.getOciBlob(digest);
+    // Try local storage first
+    let data = await this.storage.getOciBlob(digest);
+
+    // If not found locally, try upstream
+    if (!data && this.upstream) {
+      this.logger.log('debug', 'getBlob: fetching from upstream', { repository, digest });
+      const upstreamBlob = await this.upstream.fetchBlob(repository, digest);
+      if (upstreamBlob) {
+        data = upstreamBlob;
+        // Cache the blob locally (blobs are content-addressable and immutable)
+        await this.storage.putOciBlob(digest, data);
+        this.logger.log('debug', 'getBlob: cached blob locally', {
+          repository,
+          digest,
+          size: data.length,
+        });
+      }
+    }
+
     if (!data) {
       return {
         status: 404,