@push.rocks/smartregistry 2.2.3 → 2.3.0

package/ts/oci/classes.ociupstream.ts

@@ -0,0 +1,263 @@
+import * as plugins from '../plugins.js';
+import { BaseUpstream } from '../upstream/classes.baseupstream.js';
+import type {
+  IProtocolUpstreamConfig,
+  IUpstreamFetchContext,
+  IUpstreamResult,
+  IUpstreamRegistryConfig,
+} from '../upstream/interfaces.upstream.js';
+import type { IOciManifest, IOciImageIndex, ITagList } from './interfaces.oci.js';
+
+/**
+ * OCI-specific upstream implementation.
+ *
+ * Handles:
+ * - Manifest fetching (image manifests and index manifests)
+ * - Blob proxying (layers, configs)
+ * - Tag list fetching
+ * - Content-addressable caching (blobs are immutable)
+ * - Docker Hub authentication flow
+ */
+export class OciUpstream extends BaseUpstream {
+  protected readonly protocolName = 'oci';
+
+  /** Local registry base path for URL building */
+  private readonly localBasePath: string;
+
+  constructor(
+    config: IProtocolUpstreamConfig,
+    localBasePath: string = '/oci',
+    logger?: plugins.smartlog.Smartlog,
+  ) {
+    super(config, logger);
+    this.localBasePath = localBasePath;
+  }
+
+  /**
+   * Fetch a manifest from upstream registries.
+   */
+  public async fetchManifest(
+    repository: string,
+    reference: string,
+  ): Promise<{ manifest: IOciManifest | IOciImageIndex; contentType: string; digest: string } | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'oci',
+      resource: repository,
+      resourceType: 'manifest',
+      path: `/v2/${repository}/manifests/${reference}`,
+      method: 'GET',
+      headers: {
+        'accept': [
+          'application/vnd.oci.image.manifest.v1+json',
+          'application/vnd.oci.image.index.v1+json',
+          'application/vnd.docker.distribution.manifest.v2+json',
+          'application/vnd.docker.distribution.manifest.list.v2+json',
+          'application/vnd.docker.distribution.manifest.v1+json',
+        ].join(', '),
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    let manifest: IOciManifest | IOciImageIndex;
+    if (Buffer.isBuffer(result.body)) {
+      manifest = JSON.parse(result.body.toString('utf8'));
+    } else {
+      manifest = result.body;
+    }
+
+    const contentType = result.headers['content-type'] || 'application/vnd.oci.image.manifest.v1+json';
+    const digest = result.headers['docker-content-digest'] || '';
+
+    return { manifest, contentType, digest };
+  }
+
+  /**
+   * Check if a manifest exists in upstream (HEAD request).
+   */
+  public async headManifest(
+    repository: string,
+    reference: string,
+  ): Promise<{ exists: boolean; contentType?: string; digest?: string; size?: number } | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'oci',
+      resource: repository,
+      resourceType: 'manifest',
+      path: `/v2/${repository}/manifests/${reference}`,
+      method: 'HEAD',
+      headers: {
+        'accept': [
+          'application/vnd.oci.image.manifest.v1+json',
+          'application/vnd.oci.image.index.v1+json',
+          'application/vnd.docker.distribution.manifest.v2+json',
+          'application/vnd.docker.distribution.manifest.list.v2+json',
+        ].join(', '),
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result) {
+      return null;
+    }
+
+    if (!result.success) {
+      return { exists: false };
+    }
+
+    return {
+      exists: true,
+      contentType: result.headers['content-type'],
+      digest: result.headers['docker-content-digest'],
+      size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
+    };
+  }
+
+  /**
+   * Fetch a blob from upstream registries.
+   */
+  public async fetchBlob(repository: string, digest: string): Promise<Buffer | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'oci',
+      resource: repository,
+      resourceType: 'blob',
+      path: `/v2/${repository}/blobs/${digest}`,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
+  }
+
+  /**
+   * Check if a blob exists in upstream (HEAD request).
+   */
+  public async headBlob(
+    repository: string,
+    digest: string,
+  ): Promise<{ exists: boolean; size?: number } | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'oci',
+      resource: repository,
+      resourceType: 'blob',
+      path: `/v2/${repository}/blobs/${digest}`,
+      method: 'HEAD',
+      headers: {},
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result) {
+      return null;
+    }
+
+    if (!result.success) {
+      return { exists: false };
+    }
+
+    return {
+      exists: true,
+      size: result.headers['content-length'] ? parseInt(result.headers['content-length'], 10) : undefined,
+    };
+  }
+
+  /**
+   * Fetch the tag list for a repository.
+   */
+  public async fetchTags(repository: string, n?: number, last?: string): Promise<ITagList | null> {
+    const query: Record<string, string> = {};
+    if (n) query.n = n.toString();
+    if (last) query.last = last;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'oci',
+      resource: repository,
+      resourceType: 'tags',
+      path: `/v2/${repository}/tags/list`,
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query,
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    let tagList: ITagList;
+    if (Buffer.isBuffer(result.body)) {
+      tagList = JSON.parse(result.body.toString('utf8'));
+    } else {
+      tagList = result.body;
+    }
+
+    return tagList;
+  }
+
+  /**
+   * Override URL building for OCI-specific handling.
+   * OCI registries use /v2/ prefix and may require special handling for Docker Hub.
+   */
+  protected buildUpstreamUrl(
+    upstream: IUpstreamRegistryConfig,
+    context: IUpstreamFetchContext,
+  ): string {
+    let baseUrl = upstream.url;
+
+    // Remove trailing slash
+    if (baseUrl.endsWith('/')) {
+      baseUrl = baseUrl.slice(0, -1);
+    }
+
+    // Handle Docker Hub special case
+    // Docker Hub uses registry-1.docker.io but library images need special handling
+    if (baseUrl.includes('docker.io') || baseUrl.includes('registry-1.docker.io')) {
+      // For library images (e.g., "nginx" -> "library/nginx")
+      const pathParts = context.path.match(/^\/v2\/([^\/]+)\/(.+)$/);
+      if (pathParts) {
+        const [, repository, rest] = pathParts;
+        // If repository doesn't contain a slash, it's a library image
+        if (!repository.includes('/')) {
+          return `${baseUrl}/v2/library/${repository}/${rest}`;
+        }
+      }
+    }
+
+    return `${baseUrl}${context.path}`;
+  }
+
+  /**
+   * Override header building for OCI-specific authentication.
+   * OCI registries may require token-based auth obtained from a separate endpoint.
+   */
+  protected buildHeaders(
+    upstream: IUpstreamRegistryConfig,
+    context: IUpstreamFetchContext,
+  ): Record<string, string> {
+    const headers = super.buildHeaders(upstream, context);
+
+    // OCI registries typically use Docker-Distribution-API-Version header
+    headers['docker-distribution-api-version'] = 'registry/2.0';
+
+    return headers;
+  }
+}

package/ts/oci/index.ts

@@ -3,4 +3,5 @@
  */
 
 export { OciRegistry } from './classes.ociregistry.js';
+export { OciUpstream } from './classes.ociupstream.js';
 export * from './interfaces.oci.js';

package/ts/plugins.ts

@@ -8,10 +8,16 @@ import * as smartarchive from '@push.rocks/smartarchive';
 import * as smartbucket from '@push.rocks/smartbucket';
 import * as smartlog from '@push.rocks/smartlog';
 import * as smartpath from '@push.rocks/smartpath';
+import * as smartrequest from '@push.rocks/smartrequest';
 
-export { smartarchive, smartbucket, smartlog, smartpath };
+export { smartarchive, smartbucket, smartlog, smartpath, smartrequest };
 
 // @tsclass scope
 import * as tsclass from '@tsclass/tsclass';
 
 export { tsclass };
+
+// third party
+import { minimatch } from 'minimatch';
+
+export { minimatch };

package/ts/pypi/classes.pypiregistry.ts

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
 import { RegistryStorage } from '../core/classes.registrystorage.js';
 import { AuthManager } from '../core/classes.authmanager.js';
 import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
+import type { IProtocolUpstreamConfig } from '../upstream/interfaces.upstream.js';
 import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
 import type {
   IPypiPackageMetadata,
@@ -11,6 +12,7 @@ import type {
   IPypiUploadResponse,
 } from './interfaces.pypi.js';
 import * as helpers from './helpers.pypi.js';
+import { PypiUpstream } from './classes.pypiupstream.js';
 
 /**
  * PyPI registry implementation
@@ -22,12 +24,14 @@ export class PypiRegistry extends BaseRegistry {
   private basePath: string = '/pypi';
   private registryUrl: string;
   private logger: Smartlog;
+  private upstream: PypiUpstream | null = null;
 
   constructor(
     storage: RegistryStorage,
     authManager: AuthManager,
     basePath: string = '/pypi',
-    registryUrl: string = 'http://localhost:5000'
+    registryUrl: string = 'http://localhost:5000',
+    upstreamConfig?: IProtocolUpstreamConfig
   ) {
     super();
     this.storage = storage;
@@ -47,6 +51,20 @@ export class PypiRegistry extends BaseRegistry {
       }
     });
     this.logger.enableConsole();
+
+    // Initialize upstream if configured
+    if (upstreamConfig?.enabled) {
+      this.upstream = new PypiUpstream(upstreamConfig, registryUrl, this.logger);
+    }
+  }
+
+  /**
+   * Clean up resources (timers, connections, etc.)
+   */
+  public destroy(): void {
+    if (this.upstream) {
+      this.upstream.stop();
+    }
   }
 
   public async init(): Promise<void> {
@@ -214,7 +232,45 @@ export class PypiRegistry extends BaseRegistry {
     const normalized = helpers.normalizePypiPackageName(packageName);
 
     // Get package metadata
-    const metadata = await this.storage.getPypiPackageMetadata(normalized);
+    let metadata = await this.storage.getPypiPackageMetadata(normalized);
+
+    // Try upstream if not found locally
+    if (!metadata && this.upstream) {
+      const upstreamHtml = await this.upstream.fetchSimplePackage(normalized);
+      if (upstreamHtml) {
+        // Parse the HTML to extract file information and cache it
+        // For now, just return the upstream HTML directly (caching can be improved later)
+        const acceptHeader = context.headers['accept'] || context.headers['Accept'] || '';
+        const preferJson = acceptHeader.includes('application/vnd.pypi.simple') &&
+                           acceptHeader.includes('json');
+
+        if (preferJson) {
+          // Try to get JSON format from upstream
+          const upstreamJson = await this.upstream.fetchPackageJson(normalized);
+          if (upstreamJson) {
+            return {
+              status: 200,
+              headers: {
+                'Content-Type': 'application/vnd.pypi.simple.v1+json',
+                'Cache-Control': 'public, max-age=300'
+              },
+              body: upstreamJson,
+            };
+          }
+        }
+
+        // Return HTML format
+        return {
+          status: 200,
+          headers: {
+            'Content-Type': 'text/html; charset=utf-8',
+            'Cache-Control': 'public, max-age=300'
+          },
+          body: upstreamHtml,
+        };
+      }
+    }
+
     if (!metadata) {
       return this.errorResponse(404, 'Package not found');
     }
@@ -449,7 +505,16 @@ export class PypiRegistry extends BaseRegistry {
    */
   private async handleDownload(packageName: string, filename: string): Promise<IResponse> {
     const normalized = helpers.normalizePypiPackageName(packageName);
-    const fileData = await this.storage.getPypiPackageFile(normalized, filename);
+    let fileData = await this.storage.getPypiPackageFile(normalized, filename);
+
+    // Try upstream if not found locally
+    if (!fileData && this.upstream) {
+      fileData = await this.upstream.fetchPackageFile(normalized, filename);
+      if (fileData) {
+        // Cache locally
+        await this.storage.putPypiPackageFile(normalized, filename, fileData);
+      }
+    }
 
     if (!fileData) {
       return {

package/ts/pypi/classes.pypiupstream.ts

@@ -0,0 +1,211 @@
+import * as plugins from '../plugins.js';
+import { BaseUpstream } from '../upstream/classes.baseupstream.js';
+import type {
+  IProtocolUpstreamConfig,
+  IUpstreamFetchContext,
+  IUpstreamRegistryConfig,
+} from '../upstream/interfaces.upstream.js';
+
+/**
+ * PyPI-specific upstream implementation.
+ *
+ * Handles:
+ * - Simple API (HTML) - PEP 503
+ * - JSON API - PEP 691
+ * - Package file downloads (wheels, sdists)
+ * - Package name normalization
+ */
+export class PypiUpstream extends BaseUpstream {
+  protected readonly protocolName = 'pypi';
+
+  /** Local registry URL for rewriting download URLs */
+  private readonly localRegistryUrl: string;
+
+  constructor(
+    config: IProtocolUpstreamConfig,
+    localRegistryUrl: string,
+    logger?: plugins.smartlog.Smartlog,
+  ) {
+    super(config, logger);
+    this.localRegistryUrl = localRegistryUrl;
+  }
+
+  /**
+   * Fetch Simple API index (list of all packages) in HTML format.
+   */
+  public async fetchSimpleIndex(): Promise<string | null> {
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: '*',
+      resourceType: 'index',
+      path: '/simple/',
+      method: 'GET',
+      headers: {
+        'accept': 'text/html',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return result.body.toString('utf8');
+    }
+
+    return typeof result.body === 'string' ? result.body : null;
+  }
+
+  /**
+   * Fetch Simple API package page (list of files) in HTML format.
+   */
+  public async fetchSimplePackage(packageName: string): Promise<string | null> {
+    const normalizedName = this.normalizePackageName(packageName);
+    const path = `/simple/${normalizedName}/`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: packageName,
+      resourceType: 'simple',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'text/html',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return result.body.toString('utf8');
+    }
+
+    return typeof result.body === 'string' ? result.body : null;
+  }
+
+  /**
+   * Fetch package metadata using JSON API (PEP 691).
+   */
+  public async fetchPackageJson(packageName: string): Promise<any | null> {
+    const normalizedName = this.normalizePackageName(packageName);
+    const path = `/simple/${normalizedName}/`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: packageName,
+      resourceType: 'metadata',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/vnd.pypi.simple.v1+json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return JSON.parse(result.body.toString('utf8'));
+    }
+
+    return result.body;
+  }
+
+  /**
+   * Fetch full package info from PyPI JSON API (/pypi/{package}/json).
+   */
+  public async fetchPypiJson(packageName: string): Promise<any | null> {
+    const normalizedName = this.normalizePackageName(packageName);
+    const path = `/pypi/${normalizedName}/json`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: packageName,
+      resourceType: 'pypi-json',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/json',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    if (Buffer.isBuffer(result.body)) {
+      return JSON.parse(result.body.toString('utf8'));
+    }
+
+    return result.body;
+  }
+
+  /**
+   * Fetch a package file (wheel or sdist) from upstream.
+   */
+  public async fetchPackageFile(packageName: string, filename: string): Promise<Buffer | null> {
+    const normalizedName = this.normalizePackageName(packageName);
+    const path = `/packages/${normalizedName}/${filename}`;
+
+    const context: IUpstreamFetchContext = {
+      protocol: 'pypi',
+      resource: packageName,
+      resourceType: 'package',
+      path,
+      method: 'GET',
+      headers: {
+        'accept': 'application/octet-stream',
+      },
+      query: {},
+    };
+
+    const result = await this.fetch(context);
+
+    if (!result || !result.success) {
+      return null;
+    }
+
+    return Buffer.isBuffer(result.body) ? result.body : Buffer.from(result.body);
+  }
+
+  /**
+   * Normalize a PyPI package name according to PEP 503.
+   * - Lowercase all characters
+   * - Replace runs of ., -, _ with single -
+   */
+  private normalizePackageName(name: string): string {
+    return name.toLowerCase().replace(/[-_.]+/g, '-');
+  }
+
+  /**
+   * Override URL building for PyPI-specific handling.
+   */
+  protected buildUpstreamUrl(
+    upstream: IUpstreamRegistryConfig,
+    context: IUpstreamFetchContext,
+  ): string {
+    let baseUrl = upstream.url;
+
+    // Remove trailing slash
+    if (baseUrl.endsWith('/')) {
+      baseUrl = baseUrl.slice(0, -1);
+    }
+
+    return `${baseUrl}${context.path}`;
+  }
+}

package/ts/pypi/index.ts

@@ -5,4 +5,5 @@
 
 export * from './interfaces.pypi.js';
 export * from './classes.pypiregistry.js';
+export { PypiUpstream } from './classes.pypiupstream.js';
 export * as pypiHelpers from './helpers.pypi.js';