@push.rocks/smartregistry 1.4.1 → 1.6.0

package/readme.md

@@ -1,6 +1,10 @@
 # @push.rocks/smartregistry
 
-> 🚀 A composable TypeScript library implementing **OCI Distribution Specification v1.1**, **NPM Registry API**, **Maven Repository**, **Cargo/crates.io Registry**, and **Composer/Packagist** for building unified container and package registries.
+> 🚀 A composable TypeScript library implementing **OCI Distribution Specification v1.1**, **NPM Registry API**, **Maven Repository**, **Cargo/crates.io Registry**, **Composer/Packagist**, **PyPI (Python Package Index)**, and **RubyGems Registry** for building unified container and package registries.
+
+## Issue Reporting and Security
+
+For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who want to sign a contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
 
 ## ✨ Features
 
@@ -10,12 +14,14 @@
 - **Maven Repository**: Java/JVM artifact management with POM support
 - **Cargo/crates.io Registry**: Rust crate registry with sparse HTTP protocol
 - **Composer/Packagist**: PHP package registry with Composer v2 protocol
+- **PyPI (Python Package Index)**: Python package registry with PEP 503/691 support
+- **RubyGems Registry**: Ruby gem registry with compact index protocol
 
 ### 🏗️ Unified Architecture
 - **Composable Design**: Core infrastructure with protocol plugins
 - **Shared Storage**: Cloud-agnostic S3-compatible backend ([@push.rocks/smartbucket](https://www.npmjs.com/package/@push.rocks/smartbucket))
 - **Unified Authentication**: Scope-based permissions across all protocols
-- **Path-based Routing**: `/oci/*` for containers, `/npm/*` for packages, `/maven/*` for Java artifacts, `/cargo/*` for Rust crates, `/composer/*` for PHP packages
+- **Path-based Routing**: `/oci/*` for containers, `/npm/*` for packages, `/maven/*` for Java artifacts, `/cargo/*` for Rust crates, `/composer/*` for PHP packages, `/pypi/*` for Python packages, `/rubygems/*` for Ruby gems
 
 ### 🔐 Authentication & Authorization
 - NPM UUID tokens for package operations
@@ -59,6 +65,23 @@
 - ✅ Dependency resolution
 - ✅ PSR-4/PSR-0 autoloading support
 
+**PyPI Features:**
+- ✅ PEP 503 Simple Repository API (HTML)
+- ✅ PEP 691 JSON-based Simple API
+- ✅ Package upload (wheel and sdist)
+- ✅ Package name normalization
+- ✅ Hash verification (SHA256, MD5, Blake2b)
+- ✅ Content negotiation (JSON/HTML)
+- ✅ Metadata API (JSON endpoints)
+
+**RubyGems Features:**
+- ✅ Compact Index protocol (modern Bundler)
+- ✅ Gem publish/download (.gem files)
+- ✅ Version yank/unyank
+- ✅ Platform-specific gems
+- ✅ Dependency resolution
+- ✅ Legacy API compatibility
+
 ## 📥 Installation
 
 ```bash
@@ -114,6 +137,14 @@ const config: IRegistryConfig = {
     enabled: true,
     basePath: '/composer',
   },
+  pypi: {
+    enabled: true,
+    basePath: '/pypi',
+  },
+  rubygems: {
+    enabled: true,
+    basePath: '/rubygems',
+  },
 };
 
 const registry = new SmartRegistry(config);
@@ -145,6 +176,11 @@ ts/
 ├── npm/                     # NPM implementation
 │   ├── classes.npmregistry.ts
 │   └── interfaces.npm.ts
+├── maven/                   # Maven implementation
+├── cargo/                   # Cargo implementation
+├── composer/                # Composer implementation
+├── pypi/                    # PyPI implementation
+├── rubygems/                # RubyGems implementation
 └── classes.smartregistry.ts # Main orchestrator
 ```
 
@@ -157,7 +193,12 @@ SmartRegistry (orchestrator)
     ↓
 Path-based routing
     ├─→ /oci/* → OciRegistry
-    └─→ /npm/* → NpmRegistry
+    ├─→ /npm/* → NpmRegistry
+    ├─→ /maven/* → MavenRegistry
+    ├─→ /cargo/* → CargoRegistry
+    ├─→ /composer/* → ComposerRegistry
+    ├─→ /pypi/* → PypiRegistry
+    └─→ /rubygems/* → RubyGemsRegistry
            ↓
     Shared Storage & Auth
            ↓
@@ -409,6 +450,171 @@ composer require vendor/package
 composer update
 ```
 
+### 🐍 PyPI Registry (Python Packages)
+
+```typescript
+// Get package index (PEP 503 HTML format)
+const htmlIndex = await registry.handleRequest({
+  method: 'GET',
+  path: '/simple/requests/',
+  headers: { 'Accept': 'text/html' },
+  query: {},
+});
+
+// Get package index (PEP 691 JSON format)
+const jsonIndex = await registry.handleRequest({
+  method: 'GET',
+  path: '/simple/requests/',
+  headers: { 'Accept': 'application/vnd.pypi.simple.v1+json' },
+  query: {},
+});
+
+// Upload a Python package (wheel or sdist)
+const formData = new FormData();
+formData.append(':action', 'file_upload');
+formData.append('protocol_version', '1');
+formData.append('name', 'my-package');
+formData.append('version', '1.0.0');
+formData.append('filetype', 'bdist_wheel');
+formData.append('pyversion', 'py3');
+formData.append('metadata_version', '2.1');
+formData.append('sha256_digest', 'abc123...');
+formData.append('content', packageFile, { filename: 'my_package-1.0.0-py3-none-any.whl' });
+
+const upload = await registry.handleRequest({
+  method: 'POST',
+  path: '/pypi/legacy/',
+  headers: {
+    'Authorization': `Bearer <pypi-token>`,
+    'Content-Type': 'multipart/form-data',
+  },
+  query: {},
+  body: formData,
+});
+
+// Get package metadata (PyPI JSON API)
+const metadata = await registry.handleRequest({
+  method: 'GET',
+  path: '/pypi/my-package/json',
+  headers: {},
+  query: {},
+});
+
+// Download a specific version
+const download = await registry.handleRequest({
+  method: 'GET',
+  path: '/packages/my-package/my_package-1.0.0-py3-none-any.whl',
+  headers: {},
+  query: {},
+});
+```
+
+**Using with pip:**
+
+```bash
+# Install from custom registry
+pip install --index-url https://registry.example.com/simple/ my-package
+
+# Upload to custom registry
+python -m twine upload --repository-url https://registry.example.com/pypi/legacy/ dist/*
+
+# Configure in pip.conf or pip.ini
+[global]
+index-url = https://registry.example.com/simple/
+```
+
+### 💎 RubyGems Registry (Ruby Gems)
+
+```typescript
+// Get versions file (compact index)
+const versions = await registry.handleRequest({
+  method: 'GET',
+  path: '/rubygems/versions',
+  headers: {},
+  query: {},
+});
+
+// Get gem-specific info
+const gemInfo = await registry.handleRequest({
+  method: 'GET',
+  path: '/rubygems/info/rails',
+  headers: {},
+  query: {},
+});
+
+// Get list of all gem names
+const names = await registry.handleRequest({
+  method: 'GET',
+  path: '/rubygems/names',
+  headers: {},
+  query: {},
+});
+
+// Upload a gem file
+const gemBuffer = await readFile('my-gem-1.0.0.gem');
+const uploadGem = await registry.handleRequest({
+  method: 'POST',
+  path: '/rubygems/api/v1/gems',
+  headers: { 'Authorization': '<rubygems-api-key>' },
+  query: {},
+  body: gemBuffer,
+});
+
+// Yank a version (make unavailable for install)
+const yank = await registry.handleRequest({
+  method: 'DELETE',
+  path: '/rubygems/api/v1/gems/yank',
+  headers: { 'Authorization': '<rubygems-api-key>' },
+  query: { gem_name: 'my-gem', version: '1.0.0' },
+});
+
+// Unyank a version
+const unyank = await registry.handleRequest({
+  method: 'PUT',
+  path: '/rubygems/api/v1/gems/unyank',
+  headers: { 'Authorization': '<rubygems-api-key>' },
+  query: { gem_name: 'my-gem', version: '1.0.0' },
+});
+
+// Get gem version metadata
+const versionMeta = await registry.handleRequest({
+  method: 'GET',
+  path: '/rubygems/api/v1/versions/rails.json',
+  headers: {},
+  query: {},
+});
+
+// Download gem file
+const gemDownload = await registry.handleRequest({
+  method: 'GET',
+  path: '/rubygems/gems/rails-7.0.0.gem',
+  headers: {},
+  query: {},
+});
+```
+
+**Using with Bundler:**
+
+```ruby
+# Gemfile
+source 'https://registry.example.com/rubygems' do
+  gem 'my-gem'
+  gem 'rails'
+end
+```
+
+```bash
+# Install gems
+bundle install
+
+# Push gem to custom registry
+gem push my-gem-1.0.0.gem --host https://registry.example.com/rubygems
+
+# Configure gem source
+gem sources --add https://registry.example.com/rubygems/
+gem sources --remove https://rubygems.org/
+```
+
 ### 🔐 Authentication
 
 ```typescript
@@ -530,6 +736,20 @@ Unified storage abstraction for both OCI and NPM content.
 - `getNpmTarball(name, version)` - Get tarball
 - `putNpmTarball(name, version, data)` - Store tarball
 
+**PyPI Methods:**
+- `getPypiPackageMetadata(name)` - Get package metadata
+- `putPypiPackageMetadata(name, data)` - Store package metadata
+- `getPypiPackageFile(name, filename)` - Get package file
+- `putPypiPackageFile(name, filename, data)` - Store package file
+
+**RubyGems Methods:**
+- `getRubyGemsVersions()` - Get versions index
+- `putRubyGemsVersions(data)` - Store versions index
+- `getRubyGemsInfo(gemName)` - Get gem info
+- `putRubyGemsInfo(gemName, data)` - Store gem info
+- `getRubyGem(gemName, version)` - Get .gem file
+- `putRubyGem(gemName, version, data)` - Store .gem file
+
 #### AuthManager
 
 Unified authentication manager supporting both NPM and OCI authentication schemes.
@@ -607,11 +827,45 @@ Composer v2 repository API compliant implementation.
 - `DELETE /packages/{vendor}/{package}` - Delete entire package
 - `DELETE /packages/{vendor}/{package}/{version}` - Delete specific version
 
-**Package Format:**
-- ZIP archives with composer.json in root
-- SHA-1 checksums for verification
-- Version normalization (1.0.0 → 1.0.0.0)
-- PSR-4/PSR-0 autoloading configuration
+#### PypiRegistry
+
+PyPI (Python Package Index) registry implementing PEP 503 and PEP 691.
+
+**Endpoints:**
+- `GET /simple/` - List all packages (HTML or JSON)
+- `GET /simple/{package}/` - List package files (HTML or JSON)
+- `POST /legacy/` - Upload package (multipart/form-data)
+- `GET /pypi/{package}/json` - Package metadata API
+- `GET /pypi/{package}/{version}/json` - Version-specific metadata
+- `GET /packages/{package}/{filename}` - Download package file
+
+**Features:**
+- PEP 503 Simple Repository API (HTML)
+- PEP 691 JSON-based Simple API
+- Content negotiation via Accept header
+- Package name normalization
+- Hash verification (SHA256, MD5, Blake2b)
+
+#### RubyGemsRegistry
+
+RubyGems registry with compact index protocol for modern Bundler.
+
+**Endpoints:**
+- `GET /versions` - Master versions file (all gems)
+- `GET /info/{gem}` - Gem-specific info file
+- `GET /names` - List of all gem names
+- `POST /api/v1/gems` - Upload gem file
+- `DELETE /api/v1/gems/yank` - Yank (deprecate) version
+- `PUT /api/v1/gems/unyank` - Unyank version
+- `GET /api/v1/versions/{gem}.json` - Version metadata
+- `GET /gems/{gem}-{version}.gem` - Download gem file
+
+**Features:**
+- Compact Index format (append-only text files)
+- Platform-specific gems support
+- Yank/unyank functionality
+- Checksum calculations (MD5 for index, SHA256 for gems)
+- Legacy Marshal API compatibility
 
 ## 🗄️ Storage Structure
 
@@ -651,11 +905,24 @@ bucket/
 │   │   └── {p1}/{p2}/{name}       # 4+ char (e.g., "se/rd/serde")
 │   └── crates/
 │       └── {name}/{name}-{version}.crate  # Gzipped tar archives
-└── composer/
-    └── packages/
-        └── {vendor}/{package}/
-            ├── metadata.json       # All versions metadata
-            └── {reference}.zip     # Package ZIP files
+├── composer/
+│   └── packages/
+│       └── {vendor}/{package}/
+│           ├── metadata.json       # All versions metadata
+│           └── {reference}.zip     # Package ZIP files
+├── pypi/
+│   ├── simple/                     # PEP 503 HTML files
+│   │   ├── index.html              # All packages list
+│   │   └── {package}/index.html   # Package versions list
+│   ├── packages/
+│   │   └── {package}/{filename}   # .whl and .tar.gz files
+│   └── metadata/
+│       └── {package}/metadata.json # Package metadata
+└── rubygems/
+    ├── versions                    # Master versions file
+    ├── info/{gemname}              # Per-gem info files
+    ├── names                       # All gem names
+    └── gems/{gemname}-{version}.gem # .gem files
 ```
 
 ## 🎯 Scope Format
@@ -685,6 +952,14 @@ Examples:
   composer:package:vendor/package:read   # Read Composer package
   composer:package:*:write          # Write any package
   composer:*:*:*                    # Full Composer access
+
+  pypi:package:my-package:read      # Read PyPI package
+  pypi:package:*:write              # Write any package
+  pypi:*:*:*                        # Full PyPI access
+
+  rubygems:gem:rails:read           # Read RubyGems gem
+  rubygems:gem:*:write              # Write any gem
+  rubygems:*:*:*                    # Full RubyGems access
 ```
 
 ## 🔌 Integration Examples

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartregistry',
-  version: '1.4.1',
-  description: 'a registry for npm modules and oci images'
+  version: '1.6.0',
+  description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
 }

package/ts/classes.smartregistry.ts

@@ -7,10 +7,12 @@ import { NpmRegistry } from './npm/classes.npmregistry.js';
 import { MavenRegistry } from './maven/classes.mavenregistry.js';
 import { CargoRegistry } from './cargo/classes.cargoregistry.js';
 import { ComposerRegistry } from './composer/classes.composerregistry.js';
+import { PypiRegistry } from './pypi/classes.pypiregistry.js';
+import { RubyGemsRegistry } from './rubygems/classes.rubygemsregistry.js';
 
 /**
  * Main registry orchestrator
- * Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, or Composer)
+ * Routes requests to appropriate protocol handlers (OCI, NPM, Maven, Cargo, Composer, PyPI, or RubyGems)
  */
 export class SmartRegistry {
   private storage: RegistryStorage;
@@ -81,6 +83,24 @@ export class SmartRegistry {
       this.registries.set('composer', composerRegistry);
     }
 
+    // Initialize PyPI registry if enabled
+    if (this.config.pypi?.enabled) {
+      const pypiBasePath = this.config.pypi.basePath || '/pypi';
+      const registryUrl = `http://localhost:5000`; // TODO: Make configurable
+      const pypiRegistry = new PypiRegistry(this.storage, this.authManager, pypiBasePath, registryUrl);
+      await pypiRegistry.init();
+      this.registries.set('pypi', pypiRegistry);
+    }
+
+    // Initialize RubyGems registry if enabled
+    if (this.config.rubygems?.enabled) {
+      const rubygemsBasePath = this.config.rubygems.basePath || '/rubygems';
+      const registryUrl = `http://localhost:5000${rubygemsBasePath}`; // TODO: Make configurable
+      const rubygemsRegistry = new RubyGemsRegistry(this.storage, this.authManager, rubygemsBasePath, registryUrl);
+      await rubygemsRegistry.init();
+      this.registries.set('rubygems', rubygemsRegistry);
+    }
+
     this.initialized = true;
   }
 
@@ -131,6 +151,25 @@ export class SmartRegistry {
       }
     }
 
+    // Route to PyPI registry (also handles /simple prefix)
+    if (this.config.pypi?.enabled) {
+      const pypiBasePath = this.config.pypi.basePath || '/pypi';
+      if (path.startsWith(pypiBasePath) || path.startsWith('/simple')) {
+        const pypiRegistry = this.registries.get('pypi');
+        if (pypiRegistry) {
+          return pypiRegistry.handleRequest(context);
+        }
+      }
+    }
+
+    // Route to RubyGems registry
+    if (this.config.rubygems?.enabled && path.startsWith(this.config.rubygems.basePath)) {
+      const rubygemsRegistry = this.registries.get('rubygems');
+      if (rubygemsRegistry) {
+        return rubygemsRegistry.handleRequest(context);
+      }
+    }
+
     // No matching registry
     return {
       status: 404,
@@ -159,7 +198,7 @@ export class SmartRegistry {
   /**
    * Get a specific registry handler
    */
-  public getRegistry(protocol: 'oci' | 'npm' | 'maven' | 'cargo' | 'composer'): BaseRegistry | undefined {
+  public getRegistry(protocol: 'oci' | 'npm' | 'maven' | 'cargo' | 'composer' | 'pypi' | 'rubygems'): BaseRegistry | undefined {
     return this.registries.get(protocol);
   }
 

package/ts/core/classes.authmanager.ts

@@ -317,12 +317,153 @@ export class AuthManager {
     this.tokenStore.delete(token);
   }
 
+  // ========================================================================
+  // CARGO TOKEN MANAGEMENT
+  // ========================================================================
+
+  /**
+   * Create a Cargo token
+   * @param userId - User ID
+   * @param readonly - Whether the token is readonly
+   * @returns Cargo UUID token
+   */
+  public async createCargoToken(userId: string, readonly: boolean = false): Promise<string> {
+    const scopes = readonly ? ['cargo:*:*:read'] : ['cargo:*:*:*'];
+    return this.createUuidToken(userId, 'cargo', scopes, readonly);
+  }
+
+  /**
+   * Validate a Cargo token
+   * @param token - Cargo UUID token
+   * @returns Auth token object or null
+   */
+  public async validateCargoToken(token: string): Promise<IAuthToken | null> {
+    if (!this.isValidUuid(token)) {
+      return null;
+    }
+
+    const authToken = this.tokenStore.get(token);
+    if (!authToken || authToken.type !== 'cargo') {
+      return null;
+    }
+
+    // Check expiration if set
+    if (authToken.expiresAt && authToken.expiresAt < new Date()) {
+      this.tokenStore.delete(token);
+      return null;
+    }
+
+    return authToken;
+  }
+
+  /**
+   * Revoke a Cargo token
+   * @param token - Cargo UUID token
+   */
+  public async revokeCargoToken(token: string): Promise<void> {
+    this.tokenStore.delete(token);
+  }
+
+  // ========================================================================
+  // PYPI AUTHENTICATION
+  // ========================================================================
+
+  /**
+   * Create a PyPI token
+   * @param userId - User ID
+   * @param readonly - Whether the token is readonly
+   * @returns PyPI UUID token
+   */
+  public async createPypiToken(userId: string, readonly: boolean = false): Promise<string> {
+    const scopes = readonly ? ['pypi:*:*:read'] : ['pypi:*:*:*'];
+    return this.createUuidToken(userId, 'pypi', scopes, readonly);
+  }
+
+  /**
+   * Validate a PyPI token
+   * @param token - PyPI UUID token
+   * @returns Auth token object or null
+   */
+  public async validatePypiToken(token: string): Promise<IAuthToken | null> {
+    if (!this.isValidUuid(token)) {
+      return null;
+    }
+
+    const authToken = this.tokenStore.get(token);
+    if (!authToken || authToken.type !== 'pypi') {
+      return null;
+    }
+
+    // Check expiration if set
+    if (authToken.expiresAt && authToken.expiresAt < new Date()) {
+      this.tokenStore.delete(token);
+      return null;
+    }
+
+    return authToken;
+  }
+
+  /**
+   * Revoke a PyPI token
+   * @param token - PyPI UUID token
+   */
+  public async revokePypiToken(token: string): Promise<void> {
+    this.tokenStore.delete(token);
+  }
+
+  // ========================================================================
+  // RUBYGEMS AUTHENTICATION
+  // ========================================================================
+
+  /**
+   * Create a RubyGems token
+   * @param userId - User ID
+   * @param readonly - Whether the token is readonly
+   * @returns RubyGems UUID token
+   */
+  public async createRubyGemsToken(userId: string, readonly: boolean = false): Promise<string> {
+    const scopes = readonly ? ['rubygems:*:*:read'] : ['rubygems:*:*:*'];
+    return this.createUuidToken(userId, 'rubygems', scopes, readonly);
+  }
+
+  /**
+   * Validate a RubyGems token
+   * @param token - RubyGems UUID token
+   * @returns Auth token object or null
+   */
+  public async validateRubyGemsToken(token: string): Promise<IAuthToken | null> {
+    if (!this.isValidUuid(token)) {
+      return null;
+    }
+
+    const authToken = this.tokenStore.get(token);
+    if (!authToken || authToken.type !== 'rubygems') {
+      return null;
+    }
+
+    // Check expiration if set
+    if (authToken.expiresAt && authToken.expiresAt < new Date()) {
+      this.tokenStore.delete(token);
+      return null;
+    }
+
+    return authToken;
+  }
+
+  /**
+   * Revoke a RubyGems token
+   * @param token - RubyGems UUID token
+   */
+  public async revokeRubyGemsToken(token: string): Promise<void> {
+    this.tokenStore.delete(token);
+  }
+
   // ========================================================================
   // UNIFIED AUTHENTICATION
   // ========================================================================
 
   /**
-   * Validate any token (NPM, Maven, or OCI)
+   * Validate any token (NPM, Maven, OCI, PyPI, RubyGems, Composer, Cargo)
    * @param tokenString - Token string (UUID or JWT)
    * @param protocol - Expected protocol type
    * @returns Auth token object or null
@@ -331,7 +472,7 @@ export class AuthManager {
     tokenString: string,
     protocol?: TRegistryProtocol
   ): Promise<IAuthToken | null> {
-    // Try UUID-based tokens (NPM, Maven, Composer)
+    // Try UUID-based tokens (NPM, Maven, Composer, Cargo, PyPI, RubyGems)
     if (this.isValidUuid(tokenString)) {
       // Try NPM token
       const npmToken = await this.validateNpmToken(tokenString);
@@ -350,6 +491,24 @@ export class AuthManager {
       if (composerToken && (!protocol || protocol === 'composer')) {
         return composerToken;
       }
+
+      // Try Cargo token
+      const cargoToken = await this.validateCargoToken(tokenString);
+      if (cargoToken && (!protocol || protocol === 'cargo')) {
+        return cargoToken;
+      }
+
+      // Try PyPI token
+      const pypiToken = await this.validatePypiToken(tokenString);
+      if (pypiToken && (!protocol || protocol === 'pypi')) {
+        return pypiToken;
+      }
+
+      // Try RubyGems token
+      const rubygemsToken = await this.validateRubyGemsToken(tokenString);
+      if (rubygemsToken && (!protocol || protocol === 'rubygems')) {
+        return rubygemsToken;
+      }
     }
 
     // Try OCI JWT