@kustodian/sources 1.1.0

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@kustodian/sources",
+  "version": "1.1.0",
+  "description": "Template source fetching and caching for remote Git, HTTP, and OCI sources",
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "import": "./src/index.ts"
+    }
+  },
+  "files": [
+    "src"
+  ],
+  "scripts": {
+    "test": "bun test",
+    "test:watch": "bun test --watch",
+    "typecheck": "bun run tsc --noEmit"
+  },
+  "keywords": [
+    "kustodian",
+    "sources",
+    "git",
+    "oci",
+    "templates"
+  ],
+  "author": "Luca Silverentand <luca@onezero.company>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/lucasilverentand/kustodian.git",
+    "directory": "packages/sources"
+  },
+  "publishConfig": {
+    "registry": "https://npm.pkg.github.com"
+  },
+  "dependencies": {
+    "@kustodian/core": "workspace:*",
+    "@kustodian/loader": "workspace:*",
+    "@kustodian/schema": "workspace:*",
+    "zod": "3.25.30"
+  }
+}

package/src/cache/index.ts

@@ -0,0 +1,339 @@
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import {
+  Errors,
+  type KustodianErrorType,
+  type ResultType,
+  failure,
+  success,
+} from '@kustodian/core';
+import type { CacheEntryType, CacheManagerType } from '../types.js';
+import {
+  type CacheMetaType,
+  META_FILENAME,
+  TEMPLATES_DIRNAME,
+  cache_meta_schema,
+} from './metadata.js';
+import { calculate_expiry, is_expired } from './ttl.js';
+
+export { parse_ttl, calculate_expiry, is_expired, DEFAULT_TTL } from './ttl.js';
+export {
+  META_FILENAME,
+  TEMPLATES_DIRNAME,
+  cache_meta_schema,
+  type CacheMetaType,
+} from './metadata.js';
+export type { CacheManagerType } from '../types.js';
+
+/**
+ * Creates a safe directory name from a source name.
+ */
+function sanitize_name(name: string): string {
+  return name.replace(/[^a-zA-Z0-9-_]/g, '_');
+}
+
+/**
+ * Creates a safe directory name from a version.
+ */
+function sanitize_version(version: string): string {
+  return version.replace(/[^a-zA-Z0-9-_.]/g, '_');
+}
+
+/**
+ * Creates a cache manager instance.
+ */
+export function create_cache_manager(cache_dir: string): CacheManagerType {
+  return new CacheManager(cache_dir);
+}
+
+class CacheManager implements CacheManagerType {
+  readonly cache_dir: string;
+
+  constructor(cache_dir: string) {
+    this.cache_dir = cache_dir;
+  }
+
+  private get_entry_path(source_name: string, version: string): string {
+    return path.join(this.cache_dir, sanitize_name(source_name), sanitize_version(version));
+  }
+
+  private get_meta_path(source_name: string, version: string): string {
+    return path.join(this.get_entry_path(source_name, version), META_FILENAME);
+  }
+
+  private get_templates_path(source_name: string, version: string): string {
+    return path.join(this.get_entry_path(source_name, version), TEMPLATES_DIRNAME);
+  }
+
+  async get(
+    source_name: string,
+    version: string,
+  ): Promise<ResultType<CacheEntryType | null, KustodianErrorType>> {
+    const meta_path = this.get_meta_path(source_name, version);
+
+    try {
+      const meta_content = await fs.readFile(meta_path, 'utf-8');
+      const meta_json = JSON.parse(meta_content);
+      const parse_result = cache_meta_schema.safeParse(meta_json);
+
+      if (!parse_result.success) {
+        return failure(Errors.cache_corrupt(meta_path));
+      }
+
+      const meta = parse_result.data;
+      const expires_at = meta.expires_at ? new Date(meta.expires_at) : null;
+
+      // Check if expired (only for mutable refs)
+      if (is_expired(expires_at)) {
+        return success(null);
+      }
+
+      const templates_path = this.get_templates_path(source_name, version);
+
+      // Verify templates directory exists
+      try {
+        await fs.access(templates_path);
+      } catch {
+        return success(null);
+      }
+
+      const entry: CacheEntryType = {
+        source_name: meta.source_name,
+        source_type: meta.source_type,
+        version: meta.version,
+        path: templates_path,
+        fetched_at: new Date(meta.fetched_at),
+        expires_at,
+      };
+      if (meta.checksum) {
+        entry.checksum = meta.checksum;
+      }
+      return success(entry);
+    } catch (error) {
+      // File doesn't exist - not an error, just no cache entry
+      if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+        return success(null);
+      }
+      return failure(Errors.cache_read_error(meta_path, error));
+    }
+  }
+
+  async put(
+    source_name: string,
+    source_type: 'git' | 'http' | 'oci',
+    version: string,
+    content_path: string,
+    mutable: boolean,
+    ttl?: string,
+  ): Promise<ResultType<CacheEntryType, KustodianErrorType>> {
+    const entry_path = this.get_entry_path(source_name, version);
+    const meta_path = this.get_meta_path(source_name, version);
+    const templates_path = this.get_templates_path(source_name, version);
+
+    try {
+      // Create directory structure
+      await fs.mkdir(entry_path, { recursive: true });
+
+      // Copy content to templates directory
+      await fs.cp(content_path, templates_path, { recursive: true });
+
+      const fetched_at = new Date();
+      const expires_at = calculate_expiry(mutable, ttl);
+
+      const meta: CacheMetaType = {
+        source_name,
+        source_type,
+        version,
+        fetched_at: fetched_at.toISOString(),
+        expires_at: expires_at?.toISOString() ?? null,
+      };
+
+      // Write metadata
+      await fs.writeFile(meta_path, JSON.stringify(meta, null, 2));
+
+      return success({
+        source_name,
+        source_type,
+        version,
+        path: templates_path,
+        fetched_at,
+        expires_at,
+      });
+    } catch (error) {
+      return failure(Errors.cache_write_error(entry_path, error));
+    }
+  }
+
+  async invalidate(
+    source_name: string,
+    version?: string,
+  ): Promise<ResultType<void, KustodianErrorType>> {
+    try {
+      if (version) {
+        // Invalidate specific version
+        const entry_path = this.get_entry_path(source_name, version);
+        await fs.rm(entry_path, { recursive: true, force: true });
+      } else {
+        // Invalidate all versions for this source
+        const source_path = path.join(this.cache_dir, sanitize_name(source_name));
+        await fs.rm(source_path, { recursive: true, force: true });
+      }
+      return success(undefined);
+    } catch (error) {
+      return failure(Errors.cache_write_error(this.cache_dir, error));
+    }
+  }
+
+  async prune(): Promise<ResultType<number, KustodianErrorType>> {
+    let pruned = 0;
+
+    try {
+      // Check if cache directory exists
+      try {
+        await fs.access(this.cache_dir);
+      } catch {
+        return success(0);
+      }
+
+      const sources = await fs.readdir(this.cache_dir);
+
+      for (const source of sources) {
+        const source_path = path.join(this.cache_dir, source);
+        const stat = await fs.stat(source_path);
+        if (!stat.isDirectory()) continue;
+
+        const versions = await fs.readdir(source_path);
+
+        for (const version of versions) {
+          const entry_path = path.join(source_path, version);
+          const meta_path = path.join(entry_path, META_FILENAME);
+
+          try {
+            const meta_content = await fs.readFile(meta_path, 'utf-8');
+            const meta_json = JSON.parse(meta_content);
+            const parse_result = cache_meta_schema.safeParse(meta_json);
+
+            if (parse_result.success) {
+              const expires_at = parse_result.data.expires_at
+                ? new Date(parse_result.data.expires_at)
+                : null;
+              if (is_expired(expires_at)) {
+                await fs.rm(entry_path, { recursive: true, force: true });
+                pruned++;
+              }
+            } else {
+              // Corrupt metadata, remove entry
+              await fs.rm(entry_path, { recursive: true, force: true });
+              pruned++;
+            }
+          } catch {
+            // Can't read metadata, skip
+          }
+        }
+      }
+
+      return success(pruned);
+    } catch (error) {
+      return failure(Errors.cache_read_error(this.cache_dir, error));
+    }
+  }
+
+  async list(): Promise<ResultType<CacheEntryType[], KustodianErrorType>> {
+    const entries: CacheEntryType[] = [];
+
+    try {
+      // Check if cache directory exists
+      try {
+        await fs.access(this.cache_dir);
+      } catch {
+        return success([]);
+      }
+
+      const sources = await fs.readdir(this.cache_dir);
+
+      for (const source of sources) {
+        const source_path = path.join(this.cache_dir, source);
+        const stat = await fs.stat(source_path);
+        if (!stat.isDirectory()) continue;
+
+        const versions = await fs.readdir(source_path);
+
+        for (const version of versions) {
+          const entry_path = path.join(source_path, version);
+          const meta_path = path.join(entry_path, META_FILENAME);
+          const templates_path = path.join(entry_path, TEMPLATES_DIRNAME);
+
+          try {
+            const meta_content = await fs.readFile(meta_path, 'utf-8');
+            const meta_json = JSON.parse(meta_content);
+            const parse_result = cache_meta_schema.safeParse(meta_json);
+
+            if (parse_result.success) {
+              const meta = parse_result.data;
+              const entry: CacheEntryType = {
+                source_name: meta.source_name,
+                source_type: meta.source_type,
+                version: meta.version,
+                path: templates_path,
+                fetched_at: new Date(meta.fetched_at),
+                expires_at: meta.expires_at ? new Date(meta.expires_at) : null,
+              };
+              if (meta.checksum) {
+                entry.checksum = meta.checksum;
+              }
+              entries.push(entry);
+            }
+          } catch {
+            // Can't read metadata, skip
+          }
+        }
+      }
+
+      return success(entries);
+    } catch (error) {
+      return failure(Errors.cache_read_error(this.cache_dir, error));
+    }
+  }
+
+  async size(): Promise<ResultType<number, KustodianErrorType>> {
+    const get_dir_size = async (dir_path: string): Promise<number> => {
+      let total = 0;
+      const entries = await fs.readdir(dir_path, { withFileTypes: true });
+
+      for (const entry of entries) {
+        const full_path = path.join(dir_path, entry.name);
+        if (entry.isDirectory()) {
+          total += await get_dir_size(full_path);
+        } else {
+          const stat = await fs.stat(full_path);
+          total += stat.size;
+        }
+      }
+
+      return total;
+    };
+
+    try {
+      // Check if cache directory exists
+      try {
+        await fs.access(this.cache_dir);
+      } catch {
+        return success(0);
+      }
+
+      const total = await get_dir_size(this.cache_dir);
+      return success(total);
+    } catch (error) {
+      return failure(Errors.cache_read_error(this.cache_dir, error));
+    }
+  }
+
+  async clear(): Promise<ResultType<void, KustodianErrorType>> {
+    try {
+      await fs.rm(this.cache_dir, { recursive: true, force: true });
+      return success(undefined);
+    } catch (error) {
+      return failure(Errors.cache_write_error(this.cache_dir, error));
+    }
+  }
+}

package/src/cache/metadata.ts

@@ -0,0 +1,25 @@
+import { z } from 'zod';
+
+/**
+ * Schema for cache entry metadata stored in _meta.json files.
+ */
+export const cache_meta_schema = z.object({
+  source_name: z.string(),
+  source_type: z.enum(['git', 'http', 'oci']),
+  version: z.string(),
+  fetched_at: z.string().datetime(),
+  expires_at: z.string().datetime().nullable(),
+  checksum: z.string().optional(),
+});
+
+export type CacheMetaType = z.infer<typeof cache_meta_schema>;
+
+/**
+ * Filename for cache metadata.
+ */
+export const META_FILENAME = '_meta.json';
+
+/**
+ * Directory name for cached templates within a version directory.
+ */
+export const TEMPLATES_DIRNAME = 'templates';

package/src/cache/ttl.ts

@@ -0,0 +1,71 @@
+import {
+  Errors,
+  type KustodianErrorType,
+  type ResultType,
+  failure,
+  success,
+} from '@kustodian/core';
+
+/**
+ * Default TTL for mutable sources (1 hour).
+ */
+export const DEFAULT_TTL = '1h';
+
+/**
+ * Parses a TTL duration string into milliseconds.
+ * Supported formats: 30m, 1h, 24h, 7d
+ */
+export function parse_ttl(ttl: string): ResultType<number, KustodianErrorType> {
+  const match = ttl.match(/^(\d+)(m|h|d)$/);
+  if (!match) {
+    return failure(
+      Errors.invalid_argument(
+        'ttl',
+        `Invalid TTL format: ${ttl}. Expected format: <number>(m|h|d)`,
+      ),
+    );
+  }
+
+  const value = match[1] ?? '0';
+  const unit = match[2] as 'm' | 'h' | 'd';
+  const multipliers: Record<'m' | 'h' | 'd', number> = {
+    m: 60 * 1000,
+    h: 60 * 60 * 1000,
+    d: 24 * 60 * 60 * 1000,
+  };
+
+  return success(Number.parseInt(value, 10) * multipliers[unit]);
+}
+
+/**
+ * Calculates the expiration date from a TTL string.
+ * Returns null for immutable sources (no expiration).
+ */
+export function calculate_expiry(mutable: boolean, ttl?: string): Date | null {
+  if (!mutable) {
+    return null;
+  }
+
+  const ttl_ms_result = parse_ttl(ttl ?? DEFAULT_TTL);
+  if (!ttl_ms_result.success) {
+    // Fall back to default TTL on parse error
+    const default_ms_result = parse_ttl(DEFAULT_TTL);
+    if (!default_ms_result.success) {
+      // This should never happen with the hardcoded default
+      return new Date(Date.now() + 60 * 60 * 1000);
+    }
+    return new Date(Date.now() + default_ms_result.value);
+  }
+
+  return new Date(Date.now() + ttl_ms_result.value);
+}
+
+/**
+ * Checks if a cache entry has expired.
+ */
+export function is_expired(expires_at: Date | null): boolean {
+  if (expires_at === null) {
+    return false; // Immutable entries never expire
+  }
+  return new Date() > expires_at;
+}

package/src/fetchers/git.ts

@@ -0,0 +1,213 @@
+import { exec } from 'node:child_process';
+import * as fs from 'node:fs/promises';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { promisify } from 'node:util';
+import {
+  Errors,
+  type KustodianErrorType,
+  type ResultType,
+  failure,
+  success,
+} from '@kustodian/core';
+import { type TemplateSourceType, is_git_source } from '@kustodian/schema';
+import type { FetchOptionsType, FetchResultType, RemoteVersionType } from '../types.js';
+import type { SourceFetcherType } from './types.js';
+
+const exec_async = promisify(exec);
+
+const DEFAULT_TIMEOUT = 120_000; // 2 minutes
+
+/**
+ * Creates a Git source fetcher.
+ */
+export function create_git_fetcher(): SourceFetcherType {
+  return new GitFetcher();
+}
+
+class GitFetcher implements SourceFetcherType {
+  readonly type = 'git' as const;
+
+  is_mutable(source: TemplateSourceType): boolean {
+    if (!is_git_source(source)) return true;
+    // Branches are mutable, tags and commits are immutable
+    return source.git.ref.branch !== undefined;
+  }
+
+  async fetch(
+    source: TemplateSourceType,
+    options?: FetchOptionsType,
+  ): Promise<ResultType<FetchResultType, KustodianErrorType>> {
+    if (!is_git_source(source)) {
+      return failure(Errors.invalid_argument('source', 'Expected a git source'));
+    }
+
+    const { url, ref, path: subpath } = source.git;
+    const timeout = options?.timeout ?? DEFAULT_TIMEOUT;
+
+    // Determine the ref to fetch
+    const git_ref = ref.branch ?? ref.tag ?? ref.commit;
+    if (!git_ref) {
+      return failure(Errors.invalid_argument('source', 'No git ref specified'));
+    }
+
+    // Create temp directory for clone
+    const temp_dir = await fs.mkdtemp(path.join(os.tmpdir(), 'kustodian-git-'));
+
+    try {
+      // Clone with depth=1 for efficiency (shallow clone)
+      // For commits, we need a full clone to checkout specific commits
+      const is_commit = ref.commit !== undefined;
+      const depth_flag = is_commit ? '' : '--depth=1';
+      const branch_flag = ref.branch || ref.tag ? `--branch=${git_ref}` : '';
+
+      const clone_cmd =
+        `git clone ${depth_flag} ${branch_flag} --single-branch "${url}" "${temp_dir}"`.trim();
+
+      await this.exec_git(clone_cmd, timeout, source.name);
+
+      // If fetching a specific commit, checkout that commit
+      if (is_commit) {
+        await this.exec_git(`git -C "${temp_dir}" checkout ${git_ref}`, timeout, source.name);
+      }
+
+      // Get the actual commit SHA for versioning
+      const version = await this.get_commit_sha(temp_dir, timeout, source.name);
+      if (!version.success) {
+        return failure(version.error);
+      }
+
+      // Determine final content path
+      let content_path = temp_dir;
+      if (subpath) {
+        content_path = path.join(temp_dir, subpath);
+        // Verify subpath exists
+        try {
+          await fs.access(content_path);
+        } catch {
+          return failure(Errors.not_found('path', `${url}:${subpath}`));
+        }
+      }
+
+      // Create a clean output directory with just the templates
+      const output_dir = await fs.mkdtemp(path.join(os.tmpdir(), 'kustodian-templates-'));
+
+      // Copy content (excluding .git directory)
+      await this.copy_excluding_git(content_path, output_dir);
+
+      // Cleanup original clone
+      await fs.rm(temp_dir, { recursive: true, force: true });
+
+      return success({
+        path: output_dir,
+        version: version.value,
+        from_cache: false,
+        fetched_at: new Date(),
+      });
+    } catch (error) {
+      // Cleanup on error
+      await fs.rm(temp_dir, { recursive: true, force: true }).catch(() => {});
+
+      if (error instanceof Error && error.message.includes('Authentication')) {
+        return failure(Errors.source_auth_error(source.name, error));
+      }
+      return failure(Errors.source_fetch_error(source.name, error));
+    }
+  }
+
+  async list_versions(
+    source: TemplateSourceType,
+  ): Promise<ResultType<RemoteVersionType[], KustodianErrorType>> {
+    if (!is_git_source(source)) {
+      return failure(Errors.invalid_argument('source', 'Expected a git source'));
+    }
+
+    const { url } = source.git;
+
+    try {
+      // Use ls-remote to list refs without cloning
+      const { stdout } = await exec_async(`git ls-remote --tags --heads "${url}"`, {
+        timeout: DEFAULT_TIMEOUT,
+      });
+
+      const versions: RemoteVersionType[] = [];
+
+      for (const line of stdout.split('\n')) {
+        if (!line.trim()) continue;
+
+        const [sha, ref] = line.split('\t');
+        if (!sha || !ref) continue;
+
+        // Parse ref name
+        let version: string;
+        if (ref.startsWith('refs/tags/')) {
+          version = ref.replace('refs/tags/', '').replace(/\^{}$/, '');
+        } else if (ref.startsWith('refs/heads/')) {
+          version = ref.replace('refs/heads/', '');
+        } else {
+          continue;
+        }
+
+        // Deduplicate (annotated tags appear twice)
+        if (!versions.some((v) => v.version === version)) {
+          versions.push({
+            version,
+            digest: sha,
+          });
+        }
+      }
+
+      return success(versions);
+    } catch (error) {
+      return failure(Errors.source_fetch_error(source.name, error));
+    }
+  }
+
+  private async exec_git(
+    command: string,
+    timeout: number,
+    source_name: string,
+  ): Promise<ResultType<string, KustodianErrorType>> {
+    try {
+      const { stdout } = await exec_async(command, { timeout });
+      return success(stdout);
+    } catch (error) {
+      if (error instanceof Error && 'killed' in error && error.killed) {
+        return failure(Errors.source_timeout(source_name, timeout));
+      }
+      throw error;
+    }
+  }
+
+  private async get_commit_sha(
+    repo_path: string,
+    timeout: number,
+    source_name: string,
+  ): Promise<ResultType<string, KustodianErrorType>> {
+    const result = await this.exec_git(
+      `git -C "${repo_path}" rev-parse HEAD`,
+      timeout,
+      source_name,
+    );
+    if (!result.success) return result;
+    return success(result.value.trim());
+  }
+
+  private async copy_excluding_git(src: string, dest: string): Promise<void> {
+    const entries = await fs.readdir(src, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (entry.name === '.git') continue;
+
+      const src_path = path.join(src, entry.name);
+      const dest_path = path.join(dest, entry.name);
+
+      if (entry.isDirectory()) {
+        await fs.mkdir(dest_path, { recursive: true });
+        await this.copy_excluding_git(src_path, dest_path);
+      } else {
+        await fs.copyFile(src_path, dest_path);
+      }
+    }
+  }
+}