@kustodian/sources 1.1.0

package/src/fetchers/http.ts

@@ -0,0 +1,217 @@
+import { exec } from 'node:child_process';
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs/promises';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { promisify } from 'node:util';
+import {
+  Errors,
+  type KustodianErrorType,
+  type ResultType,
+  failure,
+  success,
+} from '@kustodian/core';
+import { type TemplateSourceType, is_http_source } from '@kustodian/schema';
+import type { FetchOptionsType, FetchResultType, RemoteVersionType } from '../types.js';
+import type { SourceFetcherType } from './types.js';
+
+const exec_async = promisify(exec);
+
+const DEFAULT_TIMEOUT = 120_000; // 2 minutes
+
+/**
+ * Creates an HTTP source fetcher.
+ */
+export function create_http_fetcher(): SourceFetcherType {
+  return new HttpFetcher();
+}
+
+class HttpFetcher implements SourceFetcherType {
+  readonly type = 'http' as const;
+
+  is_mutable(source: TemplateSourceType): boolean {
+    if (!is_http_source(source)) return true;
+    // HTTP sources with checksums are immutable (content is verified)
+    // Without checksum, we assume mutable
+    return source.http.checksum === undefined;
+  }
+
+  async fetch(
+    source: TemplateSourceType,
+    options?: FetchOptionsType,
+  ): Promise<ResultType<FetchResultType, KustodianErrorType>> {
+    if (!is_http_source(source)) {
+      return failure(Errors.invalid_argument('source', 'Expected an http source'));
+    }
+
+    const { url, checksum, headers } = source.http;
+    const timeout = options?.timeout ?? DEFAULT_TIMEOUT;
+
+    // Create temp directory for download
+    const temp_dir = await fs.mkdtemp(path.join(os.tmpdir(), 'kustodian-http-'));
+    const archive_path = path.join(temp_dir, 'archive');
+
+    try {
+      // Download the archive
+      const controller = new AbortController();
+      const timeout_id = setTimeout(() => controller.abort(), timeout);
+
+      const fetch_options: RequestInit = {
+        signal: controller.signal,
+      };
+
+      if (headers) {
+        fetch_options.headers = headers;
+      }
+
+      const response = await fetch(url, fetch_options);
+      clearTimeout(timeout_id);
+
+      if (!response.ok) {
+        if (response.status === 401 || response.status === 403) {
+          return failure(Errors.source_auth_error(source.name));
+        }
+        return failure(
+          Errors.source_fetch_error(
+            source.name,
+            new Error(`HTTP ${response.status}: ${response.statusText}`),
+          ),
+        );
+      }
+
+      // Write to file
+      const buffer = await response.arrayBuffer();
+      await fs.writeFile(archive_path, new Uint8Array(buffer));
+
+      // Verify checksum if provided
+      if (checksum) {
+        const verify_result = await this.verify_checksum(archive_path, checksum, source.name);
+        if (!verify_result.success) {
+          return verify_result;
+        }
+      }
+
+      // Compute checksum for version identifier
+      const actual_checksum = await this.compute_checksum(archive_path);
+
+      // Extract archive
+      const extract_dir = path.join(temp_dir, 'extracted');
+      await fs.mkdir(extract_dir, { recursive: true });
+
+      const extract_result = await this.extract_archive(
+        archive_path,
+        extract_dir,
+        url,
+        source.name,
+      );
+      if (!extract_result.success) {
+        return extract_result;
+      }
+
+      // Find the actual content directory (archives often have a root folder)
+      const content_dir = await this.find_content_dir(extract_dir);
+
+      // Create clean output directory
+      const output_dir = await fs.mkdtemp(path.join(os.tmpdir(), 'kustodian-templates-'));
+      await fs.cp(content_dir, output_dir, { recursive: true });
+
+      // Cleanup
+      await fs.rm(temp_dir, { recursive: true, force: true });
+
+      return success({
+        path: output_dir,
+        version: checksum ?? `sha256:${actual_checksum.slice(0, 16)}`,
+        from_cache: false,
+        fetched_at: new Date(),
+      });
+    } catch (error) {
+      // Cleanup on error
+      await fs.rm(temp_dir, { recursive: true, force: true }).catch(() => {});
+
+      if (error instanceof Error && error.name === 'AbortError') {
+        return failure(Errors.source_timeout(source.name, timeout));
+      }
+      return failure(Errors.source_fetch_error(source.name, error));
+    }
+  }
+
+  async list_versions(
+    _source: TemplateSourceType,
+  ): Promise<ResultType<RemoteVersionType[], KustodianErrorType>> {
+    // HTTP sources don't have version listing capability
+    return success([]);
+  }
+
+  private async verify_checksum(
+    file_path: string,
+    expected: string,
+    source_name: string,
+  ): Promise<ResultType<void, KustodianErrorType>> {
+    // Parse expected checksum (format: algorithm:hash or just hash for sha256)
+    const [algorithm, hash] = expected.includes(':')
+      ? (expected.split(':') as [string, string])
+      : ['sha256', expected];
+
+    const actual = await this.compute_checksum(file_path, algorithm);
+
+    if (actual.toLowerCase() !== hash.toLowerCase()) {
+      return failure(
+        Errors.source_checksum_mismatch(source_name, expected, `${algorithm}:${actual}`),
+      );
+    }
+
+    return success(undefined);
+  }
+
+  private async compute_checksum(file_path: string, algorithm = 'sha256'): Promise<string> {
+    const content = await fs.readFile(file_path);
+    return crypto.createHash(algorithm).update(new Uint8Array(content)).digest('hex');
+  }
+
+  private async extract_archive(
+    archive_path: string,
+    dest_dir: string,
+    url: string,
+    source_name: string,
+  ): Promise<ResultType<void, KustodianErrorType>> {
+    const lower_url = url.toLowerCase();
+
+    try {
+      if (lower_url.endsWith('.tar.gz') || lower_url.endsWith('.tgz')) {
+        await exec_async(`tar -xzf "${archive_path}" -C "${dest_dir}"`);
+      } else if (lower_url.endsWith('.tar')) {
+        await exec_async(`tar -xf "${archive_path}" -C "${dest_dir}"`);
+      } else if (lower_url.endsWith('.zip')) {
+        await exec_async(`unzip -q "${archive_path}" -d "${dest_dir}"`);
+      } else {
+        // Try to detect format from content
+        const { stdout } = await exec_async(`file "${archive_path}"`);
+        if (stdout.includes('gzip')) {
+          await exec_async(`tar -xzf "${archive_path}" -C "${dest_dir}"`);
+        } else if (stdout.includes('Zip')) {
+          await exec_async(`unzip -q "${archive_path}" -d "${dest_dir}"`);
+        } else if (stdout.includes('tar')) {
+          await exec_async(`tar -xf "${archive_path}" -C "${dest_dir}"`);
+        } else {
+          return failure(Errors.invalid_argument('source', `Unknown archive format for ${url}`));
+        }
+      }
+      return success(undefined);
+    } catch (error) {
+      return failure(Errors.source_fetch_error(source_name, error));
+    }
+  }
+
+  private async find_content_dir(extract_dir: string): Promise<string> {
+    const entries = await fs.readdir(extract_dir, { withFileTypes: true });
+    const dirs = entries.filter((e) => e.isDirectory());
+
+    // If there's exactly one directory, assume it's the content root
+    if (dirs.length === 1 && entries.length === 1 && dirs[0]) {
+      return path.join(extract_dir, dirs[0].name);
+    }
+
+    // Otherwise, use the extract directory itself
+    return extract_dir;
+  }
+}

package/src/fetchers/index.ts

@@ -0,0 +1,32 @@
+export type { SourceFetcherType } from './types.js';
+export { create_git_fetcher } from './git.js';
+export { create_http_fetcher } from './http.js';
+export { create_oci_fetcher } from './oci.js';
+
+import {
+  type TemplateSourceType,
+  is_git_source,
+  is_http_source,
+  is_oci_source,
+} from '@kustodian/schema';
+import { create_git_fetcher } from './git.js';
+import { create_http_fetcher } from './http.js';
+import { create_oci_fetcher } from './oci.js';
+import type { SourceFetcherType } from './types.js';
+
+/**
+ * Gets the appropriate fetcher for a source type.
+ */
+export function get_fetcher_for_source(source: TemplateSourceType): SourceFetcherType {
+  if (is_git_source(source)) {
+    return create_git_fetcher();
+  }
+  if (is_http_source(source)) {
+    return create_http_fetcher();
+  }
+  if (is_oci_source(source)) {
+    return create_oci_fetcher();
+  }
+  // This should never happen due to schema validation
+  throw new Error(`Unknown source type for source: ${source.name}`);
+}

package/src/fetchers/oci.ts

@@ -0,0 +1,206 @@
+import { exec } from 'node:child_process';
+import * as fs from 'node:fs/promises';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { promisify } from 'node:util';
+import {
+  Errors,
+  type KustodianErrorType,
+  type ResultType,
+  failure,
+  success,
+} from '@kustodian/core';
+import { type TemplateSourceType, is_oci_source } from '@kustodian/schema';
+import type { FetchOptionsType, FetchResultType, RemoteVersionType } from '../types.js';
+import type { SourceFetcherType } from './types.js';
+
+const exec_async = promisify(exec);
+
+const DEFAULT_TIMEOUT = 120_000; // 2 minutes
+
+/**
+ * Creates an OCI source fetcher.
+ */
+export function create_oci_fetcher(): SourceFetcherType {
+  return new OciFetcher();
+}
+
+class OciFetcher implements SourceFetcherType {
+  readonly type = 'oci' as const;
+
+  is_mutable(source: TemplateSourceType): boolean {
+    if (!is_oci_source(source)) return true;
+    // Digests are immutable, 'latest' tag is mutable, other tags are treated as immutable
+    if (source.oci.digest) return false;
+    return source.oci.tag === 'latest';
+  }
+
+  async fetch(
+    source: TemplateSourceType,
+    options?: FetchOptionsType,
+  ): Promise<ResultType<FetchResultType, KustodianErrorType>> {
+    if (!is_oci_source(source)) {
+      return failure(Errors.invalid_argument('source', 'Expected an oci source'));
+    }
+
+    const { registry, repository, tag, digest } = source.oci;
+    const timeout = options?.timeout ?? DEFAULT_TIMEOUT;
+
+    // Build OCI reference
+    const ref = digest ?? tag;
+    if (!ref) {
+      return failure(Errors.invalid_argument('source', 'No OCI tag or digest specified'));
+    }
+
+    const oci_ref = `${registry}/${repository}:${ref}`;
+
+    // Create temp directory for pull
+    const output_dir = await fs.mkdtemp(path.join(os.tmpdir(), 'kustodian-oci-'));
+
+    try {
+      // Try flux first, then oras
+      let pull_result = await this.try_flux_pull(oci_ref, output_dir, timeout, source.name);
+
+      if (!pull_result.success && pull_result.error.code === 'SOURCE_FETCH_ERROR') {
+        // Flux not available, try oras
+        pull_result = await this.try_oras_pull(oci_ref, output_dir, timeout, source.name);
+      }
+
+      if (!pull_result.success) {
+        return pull_result;
+      }
+
+      // Get the actual digest for version identifier
+      const version =
+        digest ?? (await this.get_digest(registry, repository, ref, timeout, source.name));
+
+      return success({
+        path: output_dir,
+        version: version ?? ref,
+        from_cache: false,
+        fetched_at: new Date(),
+      });
+    } catch (error) {
+      // Cleanup on error
+      await fs.rm(output_dir, { recursive: true, force: true }).catch(() => {});
+      return failure(Errors.source_fetch_error(source.name, error));
+    }
+  }
+
+  async list_versions(
+    source: TemplateSourceType,
+  ): Promise<ResultType<RemoteVersionType[], KustodianErrorType>> {
+    if (!is_oci_source(source)) {
+      return failure(Errors.invalid_argument('source', 'Expected an oci source'));
+    }
+
+    const { registry, repository } = source.oci;
+
+    try {
+      // Try using oras to list tags
+      const { stdout } = await exec_async(`oras repo tags ${registry}/${repository}`, {
+        timeout: DEFAULT_TIMEOUT,
+      });
+
+      const versions: RemoteVersionType[] = stdout
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0)
+        .map((version) => ({ version }));
+
+      return success(versions);
+    } catch {
+      // If oras fails, try crane
+      try {
+        const { stdout } = await exec_async(`crane ls ${registry}/${repository}`, {
+          timeout: DEFAULT_TIMEOUT,
+        });
+
+        const versions: RemoteVersionType[] = stdout
+          .split('\n')
+          .map((line) => line.trim())
+          .filter((line) => line.length > 0)
+          .map((version) => ({ version }));
+
+        return success(versions);
+      } catch (error) {
+        return failure(Errors.source_fetch_error(source.name, error));
+      }
+    }
+  }
+
+  private async try_flux_pull(
+    oci_ref: string,
+    output_dir: string,
+    timeout: number,
+    source_name: string,
+  ): Promise<ResultType<void, KustodianErrorType>> {
+    try {
+      await exec_async(`flux pull artifact oci://${oci_ref} --output="${output_dir}"`, { timeout });
+      return success(undefined);
+    } catch (error) {
+      if (error instanceof Error && error.message.includes('not found')) {
+        return failure(Errors.source_fetch_error(source_name, new Error('flux CLI not found')));
+      }
+      if (error instanceof Error && error.message.includes('unauthorized')) {
+        return failure(Errors.source_auth_error(source_name, error));
+      }
+      if (error instanceof Error && 'killed' in error && error.killed) {
+        return failure(Errors.source_timeout(source_name, timeout));
+      }
+      return failure(Errors.source_fetch_error(source_name, error));
+    }
+  }
+
+  private async try_oras_pull(
+    oci_ref: string,
+    output_dir: string,
+    timeout: number,
+    source_name: string,
+  ): Promise<ResultType<void, KustodianErrorType>> {
+    try {
+      await exec_async(`oras pull ${oci_ref} --output="${output_dir}"`, { timeout });
+      return success(undefined);
+    } catch (error) {
+      if (error instanceof Error && error.message.includes('not found')) {
+        return failure(
+          Errors.source_fetch_error(source_name, new Error('Neither flux nor oras CLI found')),
+        );
+      }
+      if (error instanceof Error && error.message.includes('unauthorized')) {
+        return failure(Errors.source_auth_error(source_name, error));
+      }
+      if (error instanceof Error && 'killed' in error && error.killed) {
+        return failure(Errors.source_timeout(source_name, timeout));
+      }
+      return failure(Errors.source_fetch_error(source_name, error));
+    }
+  }
+
+  private async get_digest(
+    registry: string,
+    repository: string,
+    tag: string,
+    timeout: number,
+    _source_name: string,
+  ): Promise<string | null> {
+    try {
+      // Try crane first
+      const { stdout } = await exec_async(`crane digest ${registry}/${repository}:${tag}`, {
+        timeout,
+      });
+      return stdout.trim();
+    } catch {
+      // If crane fails, try oras
+      try {
+        const { stdout } = await exec_async(
+          `oras manifest fetch ${registry}/${repository}:${tag} --descriptor | jq -r .digest`,
+          { timeout },
+        );
+        return stdout.trim() || null;
+      } catch {
+        return null;
+      }
+    }
+  }
+}

package/src/fetchers/types.ts

@@ -0,0 +1,33 @@
+import type { KustodianErrorType, ResultType } from '@kustodian/core';
+import type { TemplateSourceType } from '@kustodian/schema';
+import type { FetchOptionsType, FetchResultType, RemoteVersionType } from '../types.js';
+
+/**
+ * Base interface for source fetchers.
+ * Each source type (git, http, oci) implements this interface.
+ */
+export interface SourceFetcherType {
+  /** Unique identifier for this fetcher type */
+  readonly type: 'git' | 'http' | 'oci';
+
+  /**
+   * Fetches templates from the source to a temporary directory.
+   * The caller is responsible for caching the result.
+   */
+  fetch(
+    source: TemplateSourceType,
+    options?: FetchOptionsType,
+  ): Promise<ResultType<FetchResultType, KustodianErrorType>>;
+
+  /**
+   * Lists available versions from the remote.
+   */
+  list_versions(
+    source: TemplateSourceType,
+  ): Promise<ResultType<RemoteVersionType[], KustodianErrorType>>;
+
+  /**
+   * Determines if this source reference is mutable.
+   */
+  is_mutable(source: TemplateSourceType): boolean;
+}

package/src/index.ts

@@ -0,0 +1,46 @@
+// Types
+export type {
+  CacheEntryType,
+  CacheManagerType,
+  FetchOptionsType,
+  FetchResultType,
+  RemoteVersionType,
+  ResolvedSourceType,
+  ResolverOptionsType,
+  SourceFetcherType,
+  SourceResolverType,
+} from './types.js';
+
+// Cache
+export {
+  create_cache_manager,
+  parse_ttl,
+  calculate_expiry,
+  is_expired,
+  DEFAULT_TTL,
+  META_FILENAME,
+  TEMPLATES_DIRNAME,
+} from './cache/index.js';
+
+// Fetchers
+export {
+  create_git_fetcher,
+  create_http_fetcher,
+  create_oci_fetcher,
+  get_fetcher_for_source,
+} from './fetchers/index.js';
+
+// Resolver
+export {
+  create_source_resolver,
+  DEFAULT_CACHE_DIR,
+  type CreateResolverOptionsType,
+} from './resolver.js';
+
+// Template loader integration
+export {
+  load_templates_from_sources,
+  type LoadedSourcesResultType,
+  type LoadSourcesOptionsType,
+  type SourcedTemplateType,
+} from './loader.js';

package/src/loader.ts

@@ -0,0 +1,139 @@
+import * as path from 'node:path';
+import {
+  type KustodianErrorType,
+  type ResultType,
+  failure,
+  is_success,
+  success,
+} from '@kustodian/core';
+import { type LoadedTemplateType, list_directories, load_template } from '@kustodian/loader';
+import type { TemplateSourceType } from '@kustodian/schema';
+import { type CreateResolverOptionsType, create_source_resolver } from './resolver.js';
+import type { FetchOptionsType, ResolvedSourceType } from './types.js';
+
+/**
+ * A loaded template with source information.
+ */
+export interface SourcedTemplateType extends LoadedTemplateType {
+  /** Name of the source this template came from */
+  source_name: string;
+}
+
+/**
+ * Result of loading templates from sources.
+ */
+export interface LoadedSourcesResultType {
+  /** All loaded templates from all sources */
+  templates: SourcedTemplateType[];
+  /** Successfully resolved sources */
+  resolved: ResolvedSourceType[];
+}
+
+/**
+ * Options for loading templates from sources.
+ */
+export interface LoadSourcesOptionsType extends FetchOptionsType, CreateResolverOptionsType {
+  /** Run fetches in parallel (default: true) */
+  parallel?: boolean;
+}
+
+/**
+ * Loads templates from all configured sources.
+ * Fetches from remote (or cache) and loads all template.yaml files found.
+ */
+export async function load_templates_from_sources(
+  sources: TemplateSourceType[],
+  options?: LoadSourcesOptionsType,
+): Promise<ResultType<LoadedSourcesResultType, KustodianErrorType>> {
+  if (sources.length === 0) {
+    return success({ templates: [], resolved: [] });
+  }
+
+  // Fetch all sources
+  const resolver = create_source_resolver(options);
+  const fetch_result = await resolver.resolve_all(sources, options);
+
+  if (!fetch_result.success) {
+    return fetch_result;
+  }
+
+  // Load templates from each fetched source
+  const all_templates: SourcedTemplateType[] = [];
+  const errors: string[] = [];
+
+  for (const resolved of fetch_result.value) {
+    const templates_result = await load_templates_from_path(resolved.fetch_result.path);
+
+    if (is_success(templates_result)) {
+      // Add source name to each template
+      for (const loaded of templates_result.value) {
+        all_templates.push({
+          ...loaded,
+          source_name: resolved.source.name,
+        });
+      }
+    } else {
+      errors.push(`${resolved.source.name}: ${templates_result.error.message}`);
+    }
+  }
+
+  if (errors.length > 0) {
+    return failure({
+      code: 'VALIDATION_ERROR',
+      message: `Failed to load templates from sources:\n${errors.join('\n')}`,
+    });
+  }
+
+  return success({
+    templates: all_templates,
+    resolved: fetch_result.value,
+  });
+}
+
+/**
+ * Loads all templates from a directory path.
+ * Looks for subdirectories containing template.yaml files.
+ */
+async function load_templates_from_path(
+  source_path: string,
+): Promise<ResultType<LoadedTemplateType[], KustodianErrorType>> {
+  // List all subdirectories that might be templates
+  const dirs_result = await list_directories(source_path);
+
+  if (!is_success(dirs_result)) {
+    // Check if this path itself is a template
+    const direct_result = await load_template(source_path);
+    if (is_success(direct_result)) {
+      return success([direct_result.value]);
+    }
+    return failure({
+      code: 'NOT_FOUND',
+      message: `No templates found in ${source_path}`,
+    });
+  }
+
+  const templates: LoadedTemplateType[] = [];
+  const errors: string[] = [];
+
+  for (const dir of dirs_result.value) {
+    const result = await load_template(dir);
+    if (is_success(result)) {
+      templates.push(result.value);
+    } else {
+      // Only log errors for directories that look like templates
+      // (i.e., they have a template.yaml that failed validation)
+      if (result.error.code === 'SCHEMA_VALIDATION_ERROR') {
+        errors.push(`${path.basename(dir)}: ${result.error.message}`);
+      }
+    }
+  }
+
+  if (templates.length === 0 && errors.length > 0) {
+    return failure({
+      code: 'VALIDATION_ERROR',
+      message: `Failed to load templates:\n${errors.join('\n')}`,
+    });
+  }
+
+  return success(templates);
+}