@ant.sh/colony 0.1.0

package/src/resolver.js

@@ -0,0 +1,286 @@
+import { applyInterpolationDeep } from "./strings.js";
+import { getDeep, setDeep, deepMerge, isPlainObject } from "./util.js";
+
+export function resolveRules({ rules, dims, ctx, vars, allowedEnvVars = null, allowedVars = null, warnings = [] }) {
+  const indexed = [];
+  for (const r of rules) {
+    const scope = r.keySegments.slice(0, dims.length);
+    const keyPath = r.keySegments.slice(dims.length);
+
+    if (scope.length !== dims.length || keyPath.length === 0) {
+      throw new Error(
+        `${r.filePath}:${r.line}: Key must have ${dims.length} scope segments + at least one key segment: ${r.keyRaw}`
+      );
+    }
+
+    indexed.push({
+      ...r,
+      scope,
+      keyPath,
+      keyPathStr: keyPath.join("."),
+    });
+  }
+
+  const ctxScope = dims.map((d) => String(ctx[d] ?? ""));
+
+  const candidatesByKey = new Map();
+  const postOps = [];
+
+  for (const r of indexed) {
+    if (!matches(r.scope, ctxScope)) continue;
+
+    if (r.op === "+=" || r.op === "-=") postOps.push(r);
+    else {
+      if (!candidatesByKey.has(r.keyPathStr)) candidatesByKey.set(r.keyPathStr, []);
+      candidatesByKey.get(r.keyPathStr).push(r);
+    }
+  }
+
+  const out = {};
+  const trace = new Map();
+
+  for (const [key, cand] of candidatesByKey.entries()) {
+    let winner = cand[0];
+    let best = specificity(winner.scope);
+    for (let i = 1; i < cand.length; i++) {
+      const s = specificity(cand[i].scope);
+      if (s > best) {
+        best = s;
+        winner = cand[i];
+      } else if (s === best) {
+        winner = cand[i];
+      }
+    }
+
+    const existing = getDeep(out, winner.keyPath);
+
+    if (winner.op === ":=") {
+      if (existing === undefined) {
+        setDeep(out, winner.keyPath, clone(winner.value));
+        trace.set(key, packTrace(winner, best));
+      }
+      continue;
+    }
+
+    if (winner.op === "|=") {
+      if (existing === undefined) {
+        setDeep(out, winner.keyPath, clone(winner.value));
+      } else if (isPlainObject(existing) && isPlainObject(winner.value)) {
+        setDeep(out, winner.keyPath, deepMerge(existing, winner.value));
+      } else {
+        setDeep(out, winner.keyPath, clone(winner.value));
+      }
+      trace.set(key, packTrace(winner, best));
+      continue;
+    }
+
+    setDeep(out, winner.keyPath, clone(winner.value));
+    trace.set(key, packTrace(winner, best));
+  }
+
+  postOps.sort((a, b) => specificity(a.scope) - specificity(b.scope));
+
+  for (const r of postOps) {
+    const key = r.keyPathStr;
+    const best = specificity(r.scope);
+
+    const existing = getDeep(out, r.keyPath);
+    const val = clone(r.value);
+
+    if (r.op === "+=") {
+      const add = Array.isArray(val) ? val : [val];
+      if (existing === undefined) setDeep(out, r.keyPath, add);
+      else if (Array.isArray(existing)) setDeep(out, r.keyPath, existing.concat(add));
+      else setDeep(out, r.keyPath, [existing].concat(add));
+      trace.set(key, packTrace(r, best));
+      continue;
+    }
+
+    if (r.op === "-=") {
+      const remove = new Set(Array.isArray(val) ? val : [val]);
+      if (Array.isArray(existing)) {
+        setDeep(out, r.keyPath, existing.filter((x) => !remove.has(x)));
+        trace.set(key, packTrace(r, best));
+      }
+      continue;
+    }
+  }
+
+  const finalCfg = applyInterpolationDeep(out, { ctx, vars, allowedEnvVars, allowedVars, warnings });
+
+  Object.defineProperties(finalCfg, {
+    // Core methods
+    get: { enumerable: false, value: (p) => getByPath(finalCfg, p) },
+    explain: { enumerable: false, value: (p) => explainByPath(trace, p) },
+
+    // Serialization - returns a plain object copy without non-enumerable methods
+    toJSON: {
+      enumerable: false,
+      value: () => {
+        const plain = {};
+        for (const [k, v] of Object.entries(finalCfg)) {
+          plain[k] = clone(v);
+        }
+        return plain;
+      },
+    },
+
+    // List all keys (dot-notation paths)
+    keys: {
+      enumerable: false,
+      value: () => collectKeys(finalCfg),
+    },
+
+    // Diff against another config
+    diff: {
+      enumerable: false,
+      value: (other) => diffConfigs(finalCfg, other),
+    },
+
+    // Internal trace data
+    _trace: { enumerable: false, value: trace },
+  });
+
+  return finalCfg;
+}
+
+/**
+ * Collect all leaf keys in dot notation
+ * @param {object} obj
+ * @param {string} prefix
+ * @returns {string[]}
+ */
+function collectKeys(obj, prefix = "") {
+  const keys = [];
+
+  for (const [k, v] of Object.entries(obj)) {
+    const path = prefix ? `${prefix}.${k}` : k;
+
+    if (isPlainObject(v)) {
+      keys.push(...collectKeys(v, path));
+    } else {
+      keys.push(path);
+    }
+  }
+
+  return keys.sort();
+}
+
+/**
+ * Diff two configs, returning added, removed, and changed keys
+ * @param {object} a - First config
+ * @param {object} b - Second config
+ * @returns {{ added: string[], removed: string[], changed: Array<{key: string, from: any, to: any}> }}
+ */
+function diffConfigs(a, b) {
+  const aKeys = new Set(collectKeys(a));
+  const bKeys = new Set(collectKeys(b));
+
+  const added = [];
+  const removed = [];
+  const changed = [];
+
+  // Keys in b but not in a
+  for (const key of bKeys) {
+    if (!aKeys.has(key)) {
+      added.push(key);
+    }
+  }
+
+  // Keys in a but not in b
+  for (const key of aKeys) {
+    if (!bKeys.has(key)) {
+      removed.push(key);
+    }
+  }
+
+  // Keys in both - check for changes
+  for (const key of aKeys) {
+    if (bKeys.has(key)) {
+      const aVal = getByPath(a, key);
+      const bVal = getByPath(b, key);
+
+      if (!deepEqual(aVal, bVal)) {
+        changed.push({ key, from: aVal, to: bVal });
+      }
+    }
+  }
+
+  return {
+    added: added.sort(),
+    removed: removed.sort(),
+    changed: changed.sort((x, y) => x.key.localeCompare(y.key)),
+  };
+}
+
+/**
+ * Deep equality check for config values.
+ * Note: Does not handle circular references (will stack overflow).
+ * Config values should never be circular in practice.
+ */
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (a === null || b === null) return a === b;
+
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    return a.every((v, i) => deepEqual(v, b[i]));
+  }
+
+  if (typeof a === "object" && typeof b === "object") {
+    const aKeys = Object.keys(a);
+    const bKeys = Object.keys(b);
+    if (aKeys.length !== bKeys.length) return false;
+    return aKeys.every((k) => deepEqual(a[k], b[k]));
+  }
+
+  return false;
+}
+
+function getByPath(obj, p) {
+  const segs = String(p).split(".").filter(Boolean);
+  return getDeep(obj, segs);
+}
+
+function explainByPath(trace, p) {
+  const key = String(p);
+  return trace.get(key) ?? null;
+}
+
+function matches(ruleScope, ctxScope) {
+  for (let i = 0; i < ruleScope.length; i++) {
+    const r = String(ruleScope[i]);
+    const c = String(ctxScope[i]);
+    if (r === "*") continue;
+    if (r !== c) return false;
+  }
+  return true;
+}
+
+function specificity(ruleScope) {
+  let s = 0;
+  for (const seg of ruleScope) if (seg !== "*") s++;
+  return s;
+}
+
+function packTrace(rule, spec) {
+  return {
+    op: rule.op,
+    scope: rule.scope.map(String),
+    specificity: spec,
+    filePath: rule.filePath,
+    line: rule.line,
+    col: rule.col ?? 0,
+    keyRaw: rule.keyRaw,
+    // Source map style location
+    source: `${rule.filePath}:${rule.line}:${rule.col ?? 0}`,
+  };
+}
+
+function clone(v) {
+  if (v === null || v === undefined) return v;
+  if (Array.isArray(v)) return v.map(clone);
+  if (typeof v === "object") return structuredClone(v);
+  return v;
+}

package/src/secrets.js

@@ -0,0 +1,313 @@
+/**
+ * Secrets provider system for colony
+ */
+
+import { isPlainObject } from "./util.js";
+
+// Global provider registry
+const globalRegistry = new Map();
+
+/**
+ * Register a secret provider globally
+ * @param {object} provider - Provider with prefix and fetch()
+ */
+export function registerSecretProvider(provider) {
+  if (!provider.prefix || typeof provider.fetch !== "function") {
+    throw new Error("Invalid provider: must have prefix and fetch()");
+  }
+  globalRegistry.set(provider.prefix.toUpperCase(), provider);
+}
+
+/**
+ * Unregister a provider by prefix
+ * @param {string} prefix - Provider prefix to remove
+ * @returns {boolean} True if provider was removed
+ */
+export function unregisterSecretProvider(prefix) {
+  return globalRegistry.delete(prefix.toUpperCase());
+}
+
+/**
+ * Clear all registered providers
+ */
+export function clearSecretProviders() {
+  globalRegistry.clear();
+}
+
+/**
+ * Check if any global providers are registered
+ * @returns {boolean}
+ */
+export function hasGlobalProviders() {
+  return globalRegistry.size > 0;
+}
+
+/**
+ * Simple LRU cache for secrets
+ */
+export class SecretCache {
+  constructor(maxSize = 100) {
+    this.maxSize = maxSize;
+    this.cache = new Map();
+  }
+
+  get(key) {
+    const entry = this.cache.get(key);
+    if (!entry) return undefined;
+    if (Date.now() > entry.expires) {
+      this.cache.delete(key);
+      return undefined;
+    }
+    // Move to end for LRU behavior
+    this.cache.delete(key);
+    this.cache.set(key, entry);
+    return entry.value;
+  }
+
+  set(key, value, ttl) {
+    if (this.cache.size >= this.maxSize) {
+      // Remove oldest (first) entry
+      const firstKey = this.cache.keys().next().value;
+      this.cache.delete(firstKey);
+    }
+    this.cache.set(key, { value, expires: Date.now() + ttl });
+  }
+
+  invalidate(pattern) {
+    if (!pattern) {
+      this.cache.clear();
+      return;
+    }
+    const regex = new RegExp("^" + globToRegex(pattern) + "$");
+    for (const key of this.cache.keys()) {
+      if (regex.test(key)) this.cache.delete(key);
+    }
+  }
+}
+
+/**
+ * Convert a glob pattern to regex, escaping special chars except *
+ * @param {string} pattern - Glob pattern (e.g., "myapp/*")
+ * @returns {string} Regex pattern string
+ */
+function globToRegex(pattern) {
+  return pattern
+    .replace(/[.+?^${}()|[\]\\]/g, "\\$&") // escape regex special chars
+    .replace(/\*/g, ".*"); // convert glob * to regex .*
+}
+
+// Regex to match secret interpolations: ${PROVIDER:key}
+// Provider must start with uppercase letter, followed by uppercase letters, digits, or underscores
+const RX_SECRET = /\$\{([A-Z][A-Z0-9_]*):([^}]+)\}/g;
+
+// Reserved prefixes that are not secrets
+const RESERVED = new Set(["ENV", "VAR"]);
+
+/**
+ * Collect all secret references from a value tree
+ * @param {any} value - Value to scan
+ * @param {Map} refs - Map to collect refs into
+ * @returns {Map} Map of fullKey -> { provider, key }
+ */
+export function collectSecretRefs(value, refs = new Map()) {
+  if (typeof value === "string") {
+    let match;
+    RX_SECRET.lastIndex = 0;
+    while ((match = RX_SECRET.exec(value)) !== null) {
+      const [, provider, key] = match;
+      if (RESERVED.has(provider)) continue;
+
+      const fullKey = `${provider}:${key.trim()}`;
+      if (!refs.has(fullKey)) {
+        refs.set(fullKey, { provider, key: key.trim() });
+      }
+    }
+    return refs;
+  }
+
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      collectSecretRefs(item, refs);
+    }
+    return refs;
+  }
+
+  if (isPlainObject(value)) {
+    for (const v of Object.values(value)) {
+      collectSecretRefs(v, refs);
+    }
+  }
+
+  return refs;
+}
+
+/**
+ * Check if a secret key matches any allowed pattern
+ * @param {string} fullKey - Full key like "AWS:myapp/db"
+ * @param {string[]|null} allowedSecrets - Allowed patterns
+ * @returns {boolean}
+ */
+function isAllowed(fullKey, allowedSecrets) {
+  if (allowedSecrets === null || allowedSecrets === undefined) return true;
+
+  for (const pattern of allowedSecrets) {
+    // Exact match
+    if (pattern === fullKey) return true;
+
+    // Glob pattern match on full key
+    const regex = new RegExp("^" + globToRegex(pattern) + "$");
+    if (regex.test(fullKey)) return true;
+
+    // Pattern without provider matches any provider
+    if (!pattern.includes(":")) {
+      const keyOnly = fullKey.split(":")[1];
+      const keyRegex = new RegExp("^" + globToRegex(pattern) + "$");
+      if (keyRegex.test(keyOnly)) return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Fetch all secrets and apply to value tree
+ * @param {any} value - Config value tree
+ * @param {object} options - Options
+ * @returns {Promise<any>} Value tree with secrets replaced
+ */
+export async function applySecretsDeep(value, options = {}) {
+  const {
+    providers = [],
+    allowedSecrets = null,
+    cache = null,
+    cacheTtl = 300000,
+    onNotFound = "warn",
+    warnings = [],
+  } = options;
+
+  // Merge local providers with global registry
+  const registry = new Map(globalRegistry);
+  for (const p of providers) {
+    registry.set(p.prefix.toUpperCase(), p);
+  }
+
+  // Collect all secret references
+  const refs = collectSecretRefs(value);
+  if (refs.size === 0) return value;
+
+  // Fetch all secrets in parallel
+  const resolved = new Map();
+  const fetchPromises = [];
+
+  for (const [fullKey, { provider, key }] of refs) {
+    // Check allowlist
+    if (!isAllowed(fullKey, allowedSecrets)) {
+      warnings.push({
+        type: "blocked_secret",
+        provider,
+        key,
+        message: `Access to secret "${fullKey}" blocked by allowedSecrets`,
+      });
+      resolved.set(fullKey, "");
+      continue;
+    }
+
+    // Check cache
+    if (cache) {
+      const cached = cache.get(fullKey);
+      if (cached !== undefined) {
+        resolved.set(fullKey, cached);
+        continue;
+      }
+    }
+
+    // Check provider exists
+    const providerInstance = registry.get(provider);
+    if (!providerInstance) {
+      warnings.push({
+        type: "unknown_provider",
+        provider,
+        key,
+        message: `No provider registered for "${provider}"`,
+      });
+      resolved.set(fullKey, "");
+      continue;
+    }
+
+    // Queue fetch
+    fetchPromises.push(
+      providerInstance
+        .fetch(key)
+        .then((val) => {
+          const strVal = val ?? "";
+          if (cache) cache.set(fullKey, strVal, cacheTtl);
+          resolved.set(fullKey, strVal);
+        })
+        .catch((err) => {
+          const isNotFound =
+            err.code === "NOT_FOUND" ||
+            err.name === "ResourceNotFoundException" ||
+            err.message?.includes("not found");
+
+          if (isNotFound) {
+            if (onNotFound === "error") {
+              throw new Error(`COLONY: Secret not found: ${fullKey}`);
+            }
+            warnings.push({
+              type: "secret_not_found",
+              provider,
+              key,
+              message: `Secret "${fullKey}" not found`,
+            });
+            resolved.set(fullKey, "");
+          } else {
+            if (onNotFound === "error") {
+              throw err;
+            }
+            warnings.push({
+              type: "secret_fetch_error",
+              provider,
+              key,
+              message: `Failed to fetch "${fullKey}": ${err.message}`,
+            });
+            resolved.set(fullKey, "");
+          }
+        })
+    );
+  }
+
+  await Promise.all(fetchPromises);
+
+  // Apply resolved secrets to value tree
+  return replaceSecrets(value, resolved);
+}
+
+/**
+ * Replace secret placeholders with resolved values
+ * @param {any} value - Value to process
+ * @param {Map} resolved - Map of fullKey -> resolved value
+ * @returns {any} Value with secrets replaced
+ */
+function replaceSecrets(value, resolved) {
+  if (typeof value === "string") {
+    return value.replace(RX_SECRET, (match, provider, key) => {
+      if (RESERVED.has(provider)) return match;
+      const fullKey = `${provider}:${key.trim()}`;
+      return resolved.get(fullKey) ?? "";
+    });
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((v) => replaceSecrets(v, resolved));
+  }
+
+  if (isPlainObject(value)) {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+      out[k] = replaceSecrets(v, resolved);
+    }
+    return out;
+  }
+
+  return value;
+}

package/src/strings.js

@@ -0,0 +1,84 @@
+import { isPlainObject } from "./util.js";
+
+// Regex to detect secret provider patterns: ${PROVIDER:key}
+// Provider must start with uppercase letter, followed by uppercase letters, digits, or underscores
+const RX_SECRET_PROVIDER = /^[A-Z][A-Z0-9_]*:/;
+
+export function applyInterpolationDeep(value, { ctx, vars, allowedEnvVars = null, allowedVars = null, warnings = [] }) {
+  if (typeof value === "string") return interpolate(value, { ctx, vars, allowedEnvVars, allowedVars, warnings });
+  if (Array.isArray(value)) return value.map((v) => applyInterpolationDeep(v, { ctx, vars, allowedEnvVars, allowedVars, warnings }));
+  if (isPlainObject(value)) {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+      out[k] = applyInterpolationDeep(v, { ctx, vars, allowedEnvVars, allowedVars, warnings });
+    }
+    return out;
+  }
+  return value;
+}
+
+export function interpolate(s, { ctx, vars, allowedEnvVars = null, allowedVars = null, warnings = [] }) {
+  return s.replace(/\$\{([^}]+)\}/g, (match, exprRaw) => {
+    const expr = exprRaw.trim();
+
+    if (expr.startsWith("ENV:")) {
+      const k = expr.slice(4).trim();
+      // Security: check if env var is allowed
+      if (allowedEnvVars !== null && !allowedEnvVars.includes(k)) {
+        warnings.push({
+          type: "blocked_env_var",
+          var: k,
+          message: `Access to environment variable "${k}" blocked by allowedEnvVars whitelist`,
+        });
+        return "";
+      }
+      return process.env[k] ?? "";
+    }
+
+    if (expr.startsWith("VAR:")) {
+      const k = expr.slice(4).trim();
+      // Security: check if custom var is allowed
+      if (allowedVars !== null && !allowedVars.includes(k)) {
+        warnings.push({
+          type: "blocked_var",
+          var: k,
+          message: `Access to custom variable "${k}" blocked by allowedVars whitelist`,
+        });
+        return "";
+      }
+      if (vars?.[k] === undefined) {
+        warnings.push({
+          type: "unknown_var",
+          var: k,
+          message: `Unknown VAR "${k}" in interpolation ${match}`,
+        });
+      }
+      return String(vars?.[k] ?? "");
+    }
+
+    if (expr.startsWith("ctx.")) {
+      const k = expr.slice(4).trim();
+      if (ctx?.[k] === undefined) {
+        warnings.push({
+          type: "unknown_ctx",
+          var: k,
+          message: `Unknown ctx dimension "${k}" in interpolation ${match}`,
+        });
+      }
+      return String(ctx?.[k] ?? "");
+    }
+
+    // Secret provider patterns (e.g., ${AWS:...}, ${OPENBAO:...}) - leave for secrets.js
+    if (RX_SECRET_PROVIDER.test(expr)) {
+      return match; // Keep the pattern intact for later secret processing
+    }
+
+    // Unknown interpolation pattern
+    warnings.push({
+      type: "unknown_interpolation",
+      pattern: match,
+      message: `Unknown interpolation pattern: ${match}`,
+    });
+    return "";
+  });
+}