go-duck-cli 1.0.9 → 1.1.12

package/generators/storage.js

@@ -0,0 +1,589 @@
+import fs from 'fs-extra';
+import path from 'path';
+import chalk from 'chalk';
+
+export const generateStorageCode = async (config, outputDir, configSourceDir) => {
+    const storageDir = path.join(outputDir, 'internal/storage');
+    await fs.ensureDir(storageDir);
+
+    // 1. Manage GCS Credentials File (Local CLI Flow - for backward compatibility)
+    if (config.storage?.gcs?.enabled && config.storage.gcs['credentials-file']) {
+        const credsFile = config.storage.gcs['credentials-file'];
+        const sourcePath = path.join(configSourceDir, credsFile);
+        const destPath = path.join(outputDir, 'config', credsFile);
+        
+        if (await fs.pathExists(sourcePath)) {
+            await fs.ensureDir(path.dirname(destPath));
+            await fs.copy(sourcePath, destPath);
+            console.log(chalk.gray(`  Copied GCS credentials: ${credsFile}`));
+        } else {
+            console.warn(chalk.yellow(`  ⚠️ GCS credentials file not found at ${sourcePath}`));
+        }
+    }
+
+    const storageInterfaceGo = `package storage
+
+import (
+	"context"
+	"time"
+)
+
+// StorageProvider defines the universal interface for object storage across multiple providers
+type StorageProvider interface {
+	Upload(ctx context.Context, key string, data []byte) (string, error)
+	Download(ctx context.Context, key string) ([]byte, error)
+	Delete(ctx context.Context, key string) error
+	GetSignedURL(ctx context.Context, key string, expires time.Duration) (string, error)
+}
+
+var Providers = make(map[string]StorageProvider)
+`;
+
+    const bootstrapperGo = `package storage
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"{{app_name}}/config"
+)
+
+// BootstrapCredentials pulls secret files (id_rsa, service.json) from GitHub before app start
+func BootstrapCredentials(cfg *config.Config) error {
+	if !cfg.GoDuck.Storage.Bootstrap.Enabled {
+		return nil
+	}
+
+	log.Println("🚀 Starting Remote GitHub Credential Bootstrapping...")
+	
+	client := &http.Client{}
+	b := cfg.GoDuck.Storage.Bootstrap
+
+	for _, filename := range b.Files {
+		// GitHub Raw Content URL
+		url := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", b.Owner, b.Repo, b.Branch, filename)
+		req, _ := http.NewRequest("GET", url, nil)
+		
+		if b.Token != "" {
+			req.Header.Set("Authorization", "token "+b.Token)
+		}
+
+		resp, err := client.Do(req)
+		if err != nil {
+			return fmt.Errorf("failed to fetch %s: %v", filename, err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != 200 {
+			return fmt.Errorf("GitHub returned %d for %s", resp.StatusCode, filename)
+		}
+
+		body, _ := ioutil.ReadAll(resp.Body)
+		
+		// Save to local config folder (K8s ephemeral storage)
+		dest := filepath.Join("config", filename)
+		os.MkdirAll("config", 0755)
+		
+		if err := ioutil.WriteFile(dest, body, 0600); err != nil {
+			return fmt.Errorf("failed to write bootstrap file %s: %v", filename, err)
+		}
+		
+		log.Printf("✅ Successfully bootstrapped: %s", filename)
+	}
+	
+	return nil
+}
+`;
+
+    const s3ProviderGo = `package storage
+
+import (
+	"context"
+	"time"
+	"{{app_name}}/config"
+	s3vg2 "github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type S3Provider struct {
+	client *s3vg2.Client
+	bucket string
+}
+
+func NewS3Provider(cfg config.Config) *S3Provider {
+	s3Cfg := cfg.GoDuck.Storage.S3
+	return &S3Provider{bucket: s3Cfg.Bucket}
+}
+
+func (p *S3Provider) Upload(ctx context.Context, key string, data []byte) (string, error) {
+	return "s3://" + p.bucket + "/" + key, nil
+}
+
+func (p *S3Provider) Download(ctx context.Context, key string) ([]byte, error) {
+	return nil, nil
+}
+
+func (p *S3Provider) Delete(ctx context.Context, key string) error {
+	return nil
+}
+
+func (p *S3Provider) GetSignedURL(ctx context.Context, key string, expires time.Duration) (string, error) {
+	return "", nil
+}
+`;
+
+    const gcsProviderGo = `package storage
+
+import (
+	"context"
+	"time"
+	"{{app_name}}/config"
+)
+
+type GCSProvider struct {
+	bucket string
+}
+
+func NewGCSProvider(cfg config.Config) *GCSProvider {
+	return &GCSProvider{bucket: cfg.GoDuck.Storage.GCS.Bucket}
+}
+
+func (p *GCSProvider) Upload(ctx context.Context, key string, data []byte) (string, error) {
+	return "gs://" + p.bucket + "/" + key, nil
+}
+
+func (p *GCSProvider) Download(ctx context.Context, key string) ([]byte, error) {
+	return nil, nil
+}
+
+func (p *GCSProvider) Delete(ctx context.Context, key string) error {
+	return nil
+}
+
+func (p *GCSProvider) GetSignedURL(ctx context.Context, key string, expires time.Duration) (string, error) {
+	return "", nil
+}
+`;
+
+    const providerFactoryGo = `package storage
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+	"github.com/pkg/sftp"
+	"{{app_name}}/config"
+)
+
+func InitStorage(cfg *config.Config) error {
+    // 1. Fetch Remote Credentials if enabled
+    if err := BootstrapCredentials(cfg); err != nil {
+        return err
+    }
+
+	if cfg.GoDuck.Storage.S3.Enabled {
+		Providers["s3"] = NewS3Provider(*cfg)
+	}
+	if cfg.GoDuck.Storage.GCS.Enabled {
+		Providers["gcs"] = NewGCSProvider(*cfg)
+	}
+	if cfg.GoDuck.Storage.MinIO.Enabled {
+		Providers["minio"] = NewS3Provider(*cfg)
+	}
+	if cfg.GoDuck.Storage.R2.Enabled {
+		Providers["r2"] = NewS3Provider(*cfg)
+	}
+	if cfg.GoDuck.Storage.Generic.Enabled {
+		Providers["generic"] = NewS3Provider(*cfg)
+	}
+	if cfg.GoDuck.Storage.SFTP.Enabled {
+		Providers["sftp"] = NewSFTPProvider(*cfg)
+	}
+	if cfg.GoDuck.Storage.GitHub.Enabled {
+		Providers["github"] = NewGitHubProvider(*cfg)
+	}
+
+    return nil
+}
+
+type SFTPProvider struct {
+	host       string
+	remotePath string
+	client     *sftp.Client
+	sshClient  *ssh.Client
+}
+
+func NewSFTPProvider(cfg config.Config) *SFTPProvider {
+	sftpCfg := cfg.GoDuck.Storage.SFTP
+	
+	var authMethod ssh.AuthMethod
+	if sftpCfg.Password == "" && sftpCfg.KeyFile != "" {
+		keyPath := filepath.Join("config", sftpCfg.KeyFile) // Loaded from github bootstrap
+		if key, err := os.ReadFile(keyPath); err == nil {
+			if signer, parseErr := ssh.ParsePrivateKey(key); parseErr == nil {
+				authMethod = ssh.PublicKeys(signer)
+			}
+		}
+	} else if sftpCfg.Password != "" {
+		authMethod = ssh.Password(sftpCfg.Password)
+	}
+
+	if authMethod == nil {
+		authMethod = ssh.Password("fallback")
+	}
+
+	clientConfig := &ssh.ClientConfig{
+		User:            sftpCfg.Username,
+		Auth:            []ssh.AuthMethod{authMethod},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		Timeout:         5 * time.Second,
+	}
+
+	addr := fmt.Sprintf("%s:%d", sftpCfg.Host, sftpCfg.Port)
+	sshConn, err := ssh.Dial("tcp", addr, clientConfig)
+	var sftpClient *sftp.Client
+	if err == nil {
+		sftpClient, _ = sftp.NewClient(sshConn)
+	}
+
+	return &SFTPProvider{
+		host:       sftpCfg.Host,
+		remotePath: sftpCfg.RemotePath,
+		client:     sftpClient,
+		sshClient:  sshConn,
+	}
+}
+
+func (p *SFTPProvider) Upload(ctx context.Context, key string, data []byte) (string, error) {
+	if p.client == nil {
+		return "", fmt.Errorf("sftp client is not connected")
+	}
+
+	// Calculate absolute remote path
+	remotePath := key
+	if p.remotePath != "" {
+		remotePath = filepath.Join(p.remotePath, key)
+	}
+
+	// Ensure structural integrity
+	dir := filepath.Dir(remotePath)
+	p.client.MkdirAll(dir) // Automatically handles parent folders
+
+	f, err := p.client.Create(remotePath)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	if _, err := f.Write(data); err != nil {
+		return "", err
+	}
+	return "sftp://" + p.host + "/" + key, nil
+}
+
+func (p *SFTPProvider) Download(ctx context.Context, key string) ([]byte, error) {
+	if p.client == nil {
+		return nil, fmt.Errorf("sftp client is not connected")
+	}
+	
+	f, err := p.client.Open(key)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	return io.ReadAll(f)
+}
+
+func (p *SFTPProvider) Delete(ctx context.Context, key string) error {
+	return nil
+}
+
+func (p *SFTPProvider) GetSignedURL(ctx context.Context, key string, expires time.Duration) (string, error) {
+	return "", nil
+}
+
+type GitHubProvider struct {
+	owner  string
+	repo   string
+	branch string
+	token  string
+}
+
+func NewGitHubProvider(cfg config.Config) *GitHubProvider {
+	ghCfg := cfg.GoDuck.Storage.GitHub
+	branch := ghCfg.Branch
+	if branch == "" {
+		branch = "main" // Default safeguard
+	}
+	return &GitHubProvider{
+		owner:  ghCfg.Owner,
+		repo:   ghCfg.Repo,
+		branch: branch,
+		token:  ghCfg.Token,
+	}
+}
+
+type ghCommitPayload struct {
+	Message string \`json:"message"\`
+	Content string \`json:"content"\`
+	Branch  string \`json:"branch"\`
+}
+
+func (p *GitHubProvider) Upload(ctx context.Context, key string, data []byte) (string, error) {
+	if p.owner == "" || p.repo == "" {
+		return "", fmt.Errorf("GitHub provider missing owner or repo in config")
+	}
+
+	url := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s", p.owner, p.repo, key)
+	
+	payload := ghCommitPayload{
+		Message: "Automated upload via GO-DUCK GitHub Provider",
+		Content: base64.StdEncoding.EncodeToString(data),
+		Branch:  p.branch,
+	}
+	bodyBytes, err := json.Marshal(payload)
+	if err != nil {
+		return "", err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "PUT", url, bytes.NewBuffer(bodyBytes))
+	if err != nil {
+		return "", err
+	}
+
+	if p.token != "" {
+		req.Header.Set("Authorization", "token "+p.token)
+	}
+
+	client := &http.Client{Timeout: 15 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("GitHub failed to upload %s (Status: %d)", url, resp.StatusCode)
+	}
+
+	return "gh://" + p.repo + "/" + key, nil
+}
+
+func (p *GitHubProvider) Download(ctx context.Context, key string) ([]byte, error) {
+	if p.owner == "" || p.repo == "" {
+		return nil, fmt.Errorf("GitHub provider missing owner or repo in config")
+	}
+
+	url := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", p.owner, p.repo, p.branch, key)
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if p.token != "" {
+		req.Header.Set("Authorization", "token "+p.token)
+	}
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GitHub failed to fetch %s (Status: %d)", url, resp.StatusCode)
+	}
+
+	return io.ReadAll(resp.Body)
+}
+
+func (p *GitHubProvider) Delete(ctx context.Context, key string) error {
+	return nil
+}
+
+func (p *GitHubProvider) GetSignedURL(ctx context.Context, key string, expires time.Duration) (string, error) {
+	return "", nil
+}
+`;
+
+    const storageControllerGo = `package controllers
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"{{app_name}}/internal/storage"
+)
+
+type StorageController struct{}
+
+func (ctrl *StorageController) Upload(c *gin.Context) {
+	if len(storage.Providers) == 0 {
+		c.JSON(http.StatusNotImplemented, gin.H{"error": "No storage providers enabled"})
+		return
+	}
+
+	providerName := c.Query("provider")
+	if providerName == "" {
+		for k := range storage.Providers {
+			providerName = k
+			break
+		}
+	}
+
+	provider, exists := storage.Providers[providerName]
+	if !exists {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Requested storage provider is not enabled: " + providerName})
+		return
+	}
+
+	file, header, err := c.Request.FormFile("file")
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "No file uploaded (form-data: 'file')"})
+		return
+	}
+	defer file.Close()
+
+	// Optional precise structured folder path mapping
+	folder := c.PostForm("folder")
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read file"})
+		return
+	}
+
+	timestamp := time.Now().UnixNano()
+	var key string
+	if folder != "" {
+		folder = strings.Trim(folder, "/")
+		key = fmt.Sprintf("%s/%d_%s", folder, timestamp, header.Filename)
+	} else {
+		key = fmt.Sprintf("%d_%s", timestamp, header.Filename)
+	}
+
+	_, err = provider.Upload(c.Request.Context(), key, data)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"message":  "Upload successful",
+		"key":      key,
+		"provider": providerName,
+		"url":      fmt.Sprintf("/api/storage/download/%s?provider=%s", key, providerName),
+	})
+}
+
+func (ctrl *StorageController) Download(c *gin.Context) {
+	if len(storage.Providers) == 0 {
+		c.JSON(http.StatusNotImplemented, gin.H{"error": "No storage providers enabled"})
+		return
+	}
+
+	key := c.Param("key")
+	key = strings.TrimPrefix(key, "/")
+
+	prefixes := []string{"sftp:/", "s3:/", "gs:/", "gh:/"}
+	for _, p := range prefixes {
+		if strings.HasPrefix(key, p) {
+			parts := strings.Split(key, "/")
+			if len(parts) > 0 {
+				key = parts[len(parts)-1]
+			}
+			break
+		}
+	}
+
+	providerName := c.Query("provider")
+	if providerName == "" {
+		for k := range storage.Providers {
+			providerName = k
+			break
+		}
+	}
+
+	provider, exists := storage.Providers[providerName]
+	if !exists {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Requested storage provider is not enabled: " + providerName})
+		return
+	}
+
+	data, err := provider.Download(c.Request.Context(), key)
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
+		return
+	}
+
+	contentType := http.DetectContentType(data)
+	c.Data(http.StatusOK, contentType, data)
+}
+
+func (ctrl *StorageController) CrossScanDownload(c *gin.Context) {
+	if len(storage.Providers) == 0 {
+		c.JSON(http.StatusNotImplemented, gin.H{"error": "No storage providers enabled"})
+		return
+	}
+
+	key := c.Param("key")
+	key = strings.TrimPrefix(key, "/")
+
+	prefixes := []string{"sftp:/", "s3:/", "gs:/", "gh:/"}
+	for _, p := range prefixes {
+		if strings.HasPrefix(key, p) {
+			parts := strings.Split(key, "/")
+			if len(parts) > 0 {
+				key = parts[len(parts)-1]
+			}
+			break
+		}
+	}
+
+	// Distributed Cross-Scan: Iterate over every single active data lake node until the file is found
+	for pName, provider := range storage.Providers {
+		data, err := provider.Download(c.Request.Context(), key)
+		if err == nil && len(data) > 0 {
+			// File successfully located in this provider!
+			contentType := http.DetectContentType(data)
+			
+			// Expose which node successfully returned the payload
+			c.Header("X-Storage-Provider-Found", pName)
+			c.Data(http.StatusOK, contentType, data)
+			return
+		}
+	}
+
+	c.JSON(http.StatusNotFound, gin.H{"error": "Cross-Scan Failed: File not found across any active storage providers"})
+}
+`
+
+    const appName = config.name;
+    await fs.writeFile(path.join(storageDir, 'storage.go'), storageInterfaceGo);
+    await fs.writeFile(path.join(storageDir, 'bootstrap.go'), bootstrapperGo.replace(/{{app_name}}/g, appName));
+    await fs.writeFile(path.join(storageDir, 's3.go'), s3ProviderGo.replace(/{{app_name}}/g, appName));
+    await fs.writeFile(path.join(storageDir, 'gcs.go'), gcsProviderGo.replace(/{{app_name}}/g, appName));
+    await fs.writeFile(path.join(storageDir, 'provider.go'), providerFactoryGo.replace(/{{app_name}}/g, appName));
+
+    const controllersDir = path.join(outputDir, 'controllers');
+    await fs.ensureDir(controllersDir);
+    await fs.writeFile(path.join(controllersDir, 'storage_controller.go'), storageControllerGo.replace(/{{app_name}}/g, appName));
+
+    console.log(chalk.gray('  Generated Native Object Storage Engine (Includes HTTP StorageController endpoints)'));
+};