@paakd/config 0.0.2

package/src/pkl/tmpl/SharedTmpl.pkl

@@ -0,0 +1,40 @@
+@go.Package { name = "paakd.com/packages/config/gen/go/shared" }
+module paakd.com.packages.config.Shared
+
+import "package://pkg.pkl-lang.org/pkl-go/pkl.golang@0.12.1#/go.pkl"
+
+class Build {
+  enterpriseEnv: String
+  envPath: String
+  hidden envFileContent:Mapping = read*(envPath)
+  hidden envData = envFileContent[envPath].text.split("\n").map((it) -> it.split("=")).toMap((it) -> it[0].trim(), (it) -> if (it.length == 2) it[1].trim() else  "")
+  const binUser: String = "owe"
+  const binUserGroup: String = "kiosk"
+  const binUserId: UInt16  = 10001
+  certPath: String
+  const serviceDomainName: String = "shofluent.com"
+  vaultUrl: String = "https://vault.shofluent.xyz"
+  goEnv: String
+  nodeEnv: String
+  shopId: String = envData.getOrNull("SHOP_ID")
+  hidden vaultToken: String = envData.getOrNull("VAULT_TOKEN")
+  tenant: String = "\(shopId).\(serviceDomainName)"
+  envFilePath: String
+  shopBucketName: String = "shop-bucket-\(shopId)"
+  const productTaxonomyIndexBucket: String = "taxonomy-index"
+  fixed apiVersion: String = "v0.1"
+
+  envFile: String
+  version: String
+  storefrontHost: String
+  storefrontPort: UInt16
+  secureCookiePassword: String
+
+  function getAssetsDomain(): String = "assets.\(shopId).\(serviceDomainName)"
+  function getAssetsURL():String = "https://\(getAssetsDomain())"
+  function shopVaultEnvPath(val: String): String = "vault:\(vaultToken)~shops/\(shopId)/env~\(val)"
+  function shopVaultThirdpartyPath(val: String): String = "vault:\(vaultToken)~services/third_party~\(val)"
+  function shopVaultInternalEnvPath(val: String): String = "vault:\(vaultToken)~shops/\(shopId)/internal/env~\(val)"
+}
+
+build: Build

package/src/pkl/tmpl/WorkerTmpl.pkl

@@ -0,0 +1,27 @@
+@go.Package { name = "paakd.com/packages/config/gen/go/worker" }
+module paakd.com.packages.config.Worker
+
+import "package://pkg.pkl-lang.org/pkl-go/pkl.golang@0.12.1#/go.pkl"
+
+
+class WorkerConnect {
+    endpoint: String
+    hidden user: String
+    hidden password: String
+    local baseValue: String = "\(user):\(password)".base64
+
+    function getBasicAuth(): String  =  "Basic \(baseValue)"
+}
+
+breeze:WorkerConnect
+orchestrator:WorkerConnect
+mesh:WorkerConnect
+tfState:WorkerConnect
+tfStateShopBackend:WorkerConnect
+tfStateServiceBackend:WorkerConnect
+tfStateHostsBackend:WorkerConnect
+
+orchestratorAuth: String
+shopBackendAuth: String
+serviceBackendAuth: String
+hostBackendAuth: String

package/src/pkl-reader/main.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/apple/pkl-go/pkl"
+	json_encoding "github.com/segmentio/encoding/json"
+	"go.uber.org/zap"
+)
+
+var (
+	VAULT_ADDR    = os.Getenv("VAULT_ADDR")
+	VAULT_TOKEN   = os.Getenv("VAULT_TOKEN")
+	CACHE_TTL_MIN = 15 * time.Minute
+)
+
+type (
+	vaultReader struct {
+		logger *zap.Logger
+	}
+
+	ResStore struct {
+		Data      map[string]map[string]interface{}
+		CreatedAt time.Time
+		sync.Mutex
+	}
+
+	VaultRes struct {
+		Data map[string]interface{} `json:"data"`
+	}
+)
+
+var (
+	_ pkl.ResourceReader = &vaultReader{logger: zap.NewNop()}
+
+	responseCache = ResStore{
+		Data: make(map[string]map[string]interface{}),
+	}
+)
+
+func main() {
+	if VAULT_ADDR == "" {
+		VAULT_ADDR = "https://vault.shofluent.xyz"
+	}
+
+	logger, _ := zap.NewDevelopment()
+	client, err := pkl.NewExternalReaderClient(pkl.WithExternalClientResourceReader(vaultReader{
+		logger: logger,
+	}))
+	if err != nil {
+		logger.Fatal("create pkl client", zap.Error(err))
+	}
+	if err := client.Run(); err != nil {
+		logger.Fatal("run pkl client", zap.Error(err))
+	}
+}
+
+func (r vaultReader) Scheme() string {
+	return "vault"
+}
+
+func (r vaultReader) HasHierarchicalUris() bool {
+	return false
+}
+
+func (r vaultReader) IsGlobbable() bool {
+	return false
+}
+
+func (r vaultReader) ListElements(baseURI url.URL) ([]pkl.PathElement, error) {
+	r.logger.Debug("vaultReader.ListElements", zap.String("baseURI", baseURI.String()))
+	return nil, nil
+}
+
+func (r vaultReader) makeAPICallToVault(vaultToken, path string) (map[string]interface{}, error) {
+	if VAULT_TOKEN == "" {
+		VAULT_TOKEN = vaultToken
+	}
+
+	var (
+		header = map[string]string{
+			"X-Vault-Token": VAULT_TOKEN,
+		}
+		url = VAULT_ADDR + "/v1/secret/data/" + path
+	)
+
+	responseCache.Lock()
+	defer responseCache.Unlock()
+	if data, found := responseCache.Data[url]; found && time.Since(responseCache.CreatedAt) < CACHE_TTL_MIN {
+		return data, nil
+	}
+
+	r.logger.Debug("vaultReader.makeAPICallToVault", zap.String("url", url))
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		r.logger.Error("create request", zap.Error(err))
+		return nil, err
+	}
+	for key, value := range header {
+		req.Header.Add(key, value)
+	}
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		r.logger.Error("make request", zap.Error(err))
+		return nil, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		// ready body string
+		bodyBytes, _ := io.ReadAll(resp.Body)
+		r.logger.Error("get data from vault", zap.String("body", string(bodyBytes)))
+		return nil, fmt.Errorf("get data from vault %w", err)
+	}
+	var res VaultRes
+	if err := json_encoding.NewDecoder(resp.Body).Decode(&res); err != nil {
+		r.logger.Error("decode response", zap.Error(err))
+		return nil, err
+	}
+	if res.Data == nil {
+		r.logger.Error("no data found in vault")
+		return nil, errors.New("no data found in vault")
+	}
+	if data, ok := res.Data["data"].(map[string]interface{}); ok {
+		responseCache.Data[url] = data
+		responseCache.CreatedAt = time.Now()
+		return data, nil
+	} else {
+		return nil, errors.New("invalid data format")
+	}
+}
+
+func (r vaultReader) Read(uri url.URL) ([]byte, error) {
+	path := strings.Split(uri.Opaque, "~")
+	if len(path) != 3 {
+		r.logger.Error("invalid vault path", zap.String("path", uri.Opaque))
+		return nil, errors.New("invalid vault path")
+	}
+
+	if os.Getenv("ENTERPRISE_ENV") == "docker" || os.Getenv("CI") != "" {
+		return []byte(path[2]), nil
+	}
+	data, err := r.makeAPICallToVault(path[0], path[1])
+	if err != nil {
+		r.logger.Error("read vault secret", zap.Error(err))
+		return nil, err
+	}
+
+	switch data[path[2]].(type) {
+	case string:
+		return []byte(data[path[2]].(string)), nil
+	case []byte:
+		return data[path[2]].([]byte), nil
+	case nil:
+		return nil, nil
+	default:
+		return nil, errors.New("invalid data type")
+	}
+}

package/src/shared.ts

@@ -0,0 +1,65 @@
+import * as pklTypescript from '@pkl-community/pkl-typescript'
+import { load as devLoad } from '../gen/js/development/basket_eco_packages_config_shared.pkl'
+import {
+  load as prodLoad,
+  type Shared,
+} from '../gen/js/production/basket_eco_packages_config_shared.pkl'
+import { load as stgLoad } from '../gen/js/staging/basket_eco_packages_config_shared.pkl'
+import { vaultReader } from './vault'
+
+export const getSharedConfig: () => Promise<Shared> = async () => {
+  const evaluator = await pklTypescript.newEvaluator({
+    allowedResources: [
+      'http:',
+      'https:',
+      'file:',
+      'env:',
+      'prop:',
+      'modulepath:',
+      'package:',
+      'projectpackage:',
+      'vault:',
+    ],
+    allowedModules: [
+      'pkl:',
+      'repl:',
+      'file:',
+      'http:',
+      'https:',
+      'modulepath:',
+      'package:',
+      'projectpackage:',
+    ],
+    projectDir: '../../packages/config/src/pkl/production',
+    declaredProjectDependencies: {
+      localDependencies: {},
+      remoteDependencies: {},
+    },
+    resourceReaders: [
+      {
+        scheme: 'vault',
+        isGlobbable: false,
+        hasHierarchicalUris: false,
+        listElements: (uri: URL): any => {
+          return []
+        },
+        read: vaultReader,
+      },
+    ],
+  })
+  try {
+    let load = devLoad
+    let sourcePath = '../../packages/config/src/pkl/development/Shared.pkl'
+    if (process.env.BUILD_ENV === 'staging') {
+      load = stgLoad
+      sourcePath = '../../packages/config/src/pkl/staging/Shared.pkl'
+    } else if (process.env.BUILD_ENV === 'production') {
+      load = prodLoad
+      sourcePath = '../../packages/config/src/pkl/production/Shared.pkl'
+    }
+    const result = await load(evaluator, pklTypescript.FileSource(sourcePath))
+    return result
+  } finally {
+    evaluator.close()
+  }
+}

package/src/vault.spec.ts

@@ -0,0 +1,62 @@
+import { vaultReader } from './vault'
+
+describe('vaultReader', () => {
+  const OLD_ENV = process.env
+
+  beforeEach(() => {
+    process.env = { ...OLD_ENV }
+  })
+
+  afterEach(() => {
+    process.env = OLD_ENV
+  })
+
+  it('returns CI_ENVIRONMENT buffer in CI', async () => {
+    process.env.CI = '1'
+    const olVal = process.env.VITEST
+    delete process.env.VITEST // Ensure VITEST is not set
+    const uri = new URL('vault://dummy/vaultkey~datapath~field')
+    const result = await vaultReader(uri)
+    expect(Buffer.from(result).toString()).toBe('CI_ENVIRONMENT')
+    process.env.VITEST = olVal // Restore VITEST
+  })
+
+  it('returns empty Uint8Array for invalid path', async () => {
+    const uri = new URL('vault://dummy/vaultkey~datapath')
+    const result = await vaultReader(uri)
+    expect(result).toBeInstanceOf(Uint8Array)
+    expect(result.length).toBe(0)
+  })
+
+  it('returns empty Uint8Array for non-ok fetch', async () => {
+    global.fetch = vi.fn().mockResolvedValue({ ok: false, statusText: 'fail' })
+    const uri = new URL('vault://dummy/vaultkey~datapath~field')
+    delete process.env.CI
+    const result = await vaultReader(uri)
+    expect(result).toBeInstanceOf(Uint8Array)
+    expect(result.length).toBe(0)
+  })
+
+  it('returns empty Uint8Array for missing field', async () => {
+    global.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({ data: { data: { other: 'value' } } }),
+    })
+    const uri = new URL('vault://dummy/vaultkey~datapath~field')
+    const result = await vaultReader(uri)
+    expect(result).toBeInstanceOf(Uint8Array)
+    expect(result.length).toBe(0)
+  })
+
+  it('returns decoded buffer for valid field', async () => {
+    global.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({
+        data: { data: { field2: Buffer.from('hello').toString() } },
+      }),
+    })
+    const uri = new URL('vault://dummy/vaultkey2~datapath2~field2')
+    const result = await vaultReader(uri)
+    expect(Buffer.from(result).toString()).toBe('hello')
+  })
+})

package/src/vault.ts

@@ -0,0 +1,68 @@
+// Simple in-memory cache using a Map
+export const vaultPathCache = new Map()
+// const vaultPathCache = new Map<string, Uint8Array>();
+export const vaultReader = async (uri: URL): Promise<Uint8Array> => {
+  // Remove leading slash if present
+  const cleanPath = uri.pathname.startsWith('/')
+    ? uri.pathname.slice(1)
+    : uri.pathname
+  const path = cleanPath.split('~')
+  if (path.length !== 3) {
+    console.error(`Invalid vault path: ${uri}`)
+    return new Uint8Array()
+  }
+
+  if (process.env.CI && !process.env.VITEST) {
+    // In CI, we don't want to access the vault, so we return an empty Uint8Array
+    // This is useful for testing purposes where we don't want to hit the actual vault
+    console.warn('Skipping vault access in CI environment')
+    return new Uint8Array(Buffer.from('CI_ENVIRONMENT', 'utf-8'))
+  }
+
+  const [vaultKey, dataPath, fieldName] = path
+  let resData = vaultPathCache.get(`${vaultKey}~${dataPath}`)
+  if (!resData || process.env.CI) {
+    const vaultAddr = process.env.VAULT_PATH || 'https://vault.shofluent.xyz'
+    const url = `${vaultAddr}/v1/secret/data/${dataPath}`
+    const headers = {
+      'X-Vault-Token': vaultKey,
+      'Content-Type': 'application/json',
+      Accept: 'application/json',
+    }
+    const res = await fetch(url, {
+      method: 'GET',
+      headers,
+    })
+    if (!res.ok) {
+      console.error(`fetch from vault: ${res.statusText}`)
+      return new Uint8Array()
+    }
+    const { data } = await res.json()
+    resData = data?.data
+    // Store value with timestamp for manual expiration
+    vaultPathCache.set(`${vaultKey}~${dataPath}`, {
+      value: resData,
+      expires: Date.now() + 15 * 60 * 1000, // 15 minutes
+    })
+  } else if (resData.expires && resData.expires < Date.now()) {
+    // Expired, remove and fetch again
+    vaultPathCache.delete(`${vaultKey}~${dataPath}`)
+    return vaultReader(uri) // Recursively re-fetch
+  } else {
+    resData = resData.value
+  }
+  if (!resData?.[fieldName]) {
+    console.error(
+      `Invalid data from vault: ${JSON.stringify(resData)}, ${Object.keys(resData)}`
+    )
+    return new Uint8Array()
+  }
+
+  const fieldData = resData[fieldName]
+  if (typeof fieldData !== 'string') {
+    console.error(`Invalid field data from vault: ${JSON.stringify(fieldData)}`)
+    return new Uint8Array()
+  }
+  const decodedData = Buffer.from(fieldData)
+  return decodedData as Uint8Array
+}