@thinkingsage/kanon 0.8.0

package/src/backends/github.ts

@@ -0,0 +1,194 @@
+import { access, mkdir } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { CatalogEntry, HarnessName } from "../schemas";
+import type { ArtifactBackend, GitHubBackendConfig } from "./types";
+
+/**
+ * GitHub releases backend.
+ *
+ * Uses the `gh` CLI for authentication and asset downloads — no separate
+ * credential management required beyond `gh auth login`.
+ *
+ * Asset naming convention for releases:
+ *   catalog.json               — standalone catalog for browsing
+ *   dist-<harness>.tar.gz      — per-harness compiled artifact tree
+ *   release-manifest.json      — version metadata
+ */
+export class GitHubBackend implements ArtifactBackend {
+	readonly label: string;
+	private readonly releasePrefix: string;
+
+	constructor(
+		private readonly config: GitHubBackendConfig,
+		private readonly tag?: string,
+	) {
+		this.releasePrefix = config.releasePrefix ?? "v";
+		this.label = `github:${config.repo}${tag ? `@${tag}` : ""}`;
+	}
+
+	async fetchCatalog(): Promise<CatalogEntry[]> {
+		const tag = this.tag ?? (await this.latestTag());
+		const tmp = join(tmpdir(), `forge-catalog-${Date.now()}`);
+		await mkdir(tmp, { recursive: true });
+		const outPath = join(tmp, "catalog.json");
+
+		const proc = Bun.spawnSync(
+			[
+				"gh",
+				"release",
+				"download",
+				tag,
+				"--repo",
+				this.config.repo,
+				"--pattern",
+				"catalog.json",
+				"--output",
+				outPath,
+			],
+			{ stdout: "pipe", stderr: "pipe" },
+		);
+
+		if (proc.exitCode !== 0) {
+			const stderr = new TextDecoder().decode(proc.stderr);
+			throw new Error(
+				`Failed to fetch catalog from GitHub release ${tag}: ${stderr}`,
+			);
+		}
+
+		const raw = await Bun.file(outPath).text();
+		return JSON.parse(raw) as CatalogEntry[];
+	}
+
+	async fetchArtifact(
+		name: string,
+		harness: HarnessName,
+		version?: string,
+	): Promise<string> {
+		const tag = version ?? this.tag ?? (await this.latestTag());
+		const assetName = `dist-${harness}.tar.gz`;
+		const cacheDir = join(
+			process.env.HOME ?? tmpdir(),
+			".forge",
+			"cache",
+			`github-${this.config.repo.replace("/", "-")}`,
+			tag,
+		);
+
+		const extractedDir = join(cacheDir, harness);
+		const artifactDir = join(extractedDir, name);
+
+		// Return cached copy if available
+		try {
+			await access(artifactDir);
+			return artifactDir;
+		} catch {
+			// not cached yet
+		}
+
+		await mkdir(extractedDir, { recursive: true });
+
+		// Download the per-harness tarball
+		const downloadPath = join(cacheDir, assetName);
+		const _assetUrl = await this.resolveAssetUrl(tag, assetName);
+
+		const dlProc = Bun.spawnSync(
+			[
+				"gh",
+				"release",
+				"download",
+				tag,
+				"--repo",
+				this.config.repo,
+				"--pattern",
+				assetName,
+				"--output",
+				downloadPath,
+			],
+			{ stdout: "pipe", stderr: "pipe" },
+		);
+
+		if (dlProc.exitCode !== 0) {
+			const stderr = new TextDecoder().decode(dlProc.stderr);
+			throw new Error(`Failed to download ${assetName} from ${tag}: ${stderr}`);
+		}
+
+		// Extract the tarball
+		const extractProc = Bun.spawnSync(
+			["tar", "-xzf", downloadPath, "-C", cacheDir],
+			{ stdout: "pipe", stderr: "pipe" },
+		);
+
+		if (extractProc.exitCode !== 0) {
+			throw new Error(`Failed to extract ${assetName}`);
+		}
+
+		try {
+			await access(artifactDir);
+		} catch {
+			throw new Error(
+				`Artifact "${name}" not found in ${assetName} for harness "${harness}"`,
+			);
+		}
+
+		return artifactDir;
+	}
+
+	async listVersions(): Promise<string[]> {
+		const proc = Bun.spawnSync(
+			[
+				"gh",
+				"release",
+				"list",
+				"--repo",
+				this.config.repo,
+				"--json",
+				"tagName",
+				"--limit",
+				"20",
+			],
+			{ stdout: "pipe", stderr: "pipe" },
+		);
+
+		if (proc.exitCode !== 0) return [];
+
+		const releases = JSON.parse(
+			new TextDecoder().decode(proc.stdout),
+		) as Array<{ tagName: string }>;
+		return releases.map((r) => r.tagName);
+	}
+
+	private async latestTag(): Promise<string> {
+		const proc = Bun.spawnSync(
+			[
+				"gh",
+				"release",
+				"view",
+				"--repo",
+				this.config.repo,
+				"--json",
+				"tagName",
+			],
+			{ stdout: "pipe", stderr: "pipe" },
+		);
+
+		if (proc.exitCode !== 0) {
+			throw new Error(
+				`Could not determine latest release for ${this.config.repo}`,
+			);
+		}
+
+		const data = JSON.parse(new TextDecoder().decode(proc.stdout)) as {
+			tagName: string;
+		};
+		return data.tagName;
+	}
+
+	private async resolveAssetUrl(
+		_tag: string,
+		_assetName: string,
+	): Promise<string> {
+		return `repos/${this.config.repo}/releases/assets`; // resolved via gh CLI
+		// Note: actual URL resolution handled by gh CLI download command
+	}
+}

package/src/backends/http.ts

@@ -0,0 +1,122 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { CatalogEntry, HarnessName } from "../schemas";
+import type { ArtifactBackend, HttpBackendConfig } from "./types";
+
+/**
+ * Generic HTTPS backend — works with any HTTP server that hosts kanon artifacts
+ * in the standard layout:
+ *
+ *   <baseUrl>/catalog.json
+ *   <baseUrl>/<version>/catalog.json
+ *   <baseUrl>/<version>/dist-<harness>.tar.gz
+ *   <baseUrl>/<version>/release-manifest.json
+ *
+ * Bearer token auth is optional. Token can reference an env var via ${ENV_VAR} syntax.
+ */
+export class HttpBackend implements ArtifactBackend {
+	readonly label: string;
+	private readonly token: string | undefined;
+
+	constructor(
+		private readonly config: HttpBackendConfig,
+		private readonly version?: string,
+	) {
+		this.label = `http:${config.baseUrl}${version ? `@${version}` : ""}`;
+		this.token = this.resolveToken(config.token);
+	}
+
+	async fetchCatalog(): Promise<CatalogEntry[]> {
+		const url = this.version
+			? `${this.config.baseUrl}/${this.version}/catalog.json`
+			: `${this.config.baseUrl}/catalog.json`;
+
+		const res = await this.fetchUrl(url);
+		return res.json() as Promise<CatalogEntry[]>;
+	}
+
+	async fetchArtifact(
+		name: string,
+		harness: HarnessName,
+		version?: string,
+	): Promise<string> {
+		const ver = version ?? this.version ?? "latest";
+		const assetName = `dist-${harness}.tar.gz`;
+		const url = `${this.config.baseUrl}/${ver}/${assetName}`;
+
+		const cacheDir = join(
+			process.env.HOME ?? tmpdir(),
+			".forge",
+			"cache",
+			`http-${this.config.baseUrl.replace(/[^a-z0-9]/gi, "-")}`,
+			ver,
+			harness,
+		);
+
+		const artifactDir = join(cacheDir, name);
+		if (await Bun.file(artifactDir).exists()) {
+			return artifactDir;
+		}
+
+		await mkdir(cacheDir, { recursive: true });
+
+		const tarPath = join(cacheDir, assetName);
+		const res = await this.fetchUrl(url);
+		if (!res.ok) {
+			throw new Error(`HTTP ${res.status} downloading ${url}`);
+		}
+
+		const buffer = await res.arrayBuffer();
+		await writeFile(tarPath, Buffer.from(buffer));
+
+		const extractProc = Bun.spawnSync(
+			["tar", "-xzf", tarPath, "-C", cacheDir],
+			{
+				stdout: "pipe",
+				stderr: "pipe",
+			},
+		);
+
+		if (extractProc.exitCode !== 0) {
+			throw new Error(`Failed to extract ${assetName}`);
+		}
+
+		if (!(await Bun.file(artifactDir).exists())) {
+			throw new Error(`Artifact "${name}" not found in ${assetName}`);
+		}
+
+		return artifactDir;
+	}
+
+	async listVersions(): Promise<string[]> {
+		try {
+			const url = `${this.config.baseUrl}/versions.json`;
+			const res = await this.fetchUrl(url);
+			if (!res.ok) return [];
+			return res.json() as Promise<string[]>;
+		} catch {
+			return [];
+		}
+	}
+
+	private async fetchUrl(url: string): Promise<Response> {
+		const headers: Record<string, string> = {
+			Accept: "application/json, application/octet-stream",
+		};
+		if (this.token) {
+			headers.Authorization = `Bearer ${this.token}`;
+		}
+		return fetch(url, { headers });
+	}
+
+	private resolveToken(raw: string | undefined): string | undefined {
+		if (!raw) return undefined;
+		// Support ${ENV_VAR} syntax for referencing environment variables
+		const match = raw.match(/^\$\{(.+)\}$/);
+		if (match) {
+			return process.env[match[1]];
+		}
+		return raw;
+	}
+}

package/src/backends/index.ts

@@ -0,0 +1,39 @@
+import { GitHubBackend } from "./github";
+import { HttpBackend } from "./http";
+import { LocalBackend } from "./local";
+import { S3Backend } from "./s3";
+import type { ArtifactBackend, BackendConfig } from "./types";
+
+/**
+ * Resolve a BackendConfig into a concrete ArtifactBackend instance.
+ *
+ * @param config - The backend configuration from kanon.config.yaml or CLI flags
+ * @param version - Optional version/tag to pin to (passed through to backends that support it)
+ */
+export function resolveBackend(
+	config: BackendConfig,
+	version?: string,
+): ArtifactBackend {
+	switch (config.type) {
+		case "local":
+			return new LocalBackend(config.path);
+		case "github":
+			return new GitHubBackend(config, version);
+		case "s3":
+			return new S3Backend(config, version);
+		case "http":
+			return new HttpBackend(config, version);
+		default: {
+			const _exhaustive: never = config;
+			throw new Error(
+				`Unknown backend type: ${(_exhaustive as { type: string }).type}`,
+			);
+		}
+	}
+}
+
+export { GitHubBackend } from "./github";
+export { HttpBackend } from "./http";
+export { LocalBackend } from "./local";
+export { S3Backend } from "./s3";
+export type { ArtifactBackend, BackendConfig };

package/src/backends/local.ts

@@ -0,0 +1,47 @@
+import { exists } from "node:fs/promises";
+import { join } from "node:path";
+import { generateCatalog } from "../catalog";
+import type { CatalogEntry, HarnessName } from "../schemas";
+import type { ArtifactBackend } from "./types";
+
+/**
+ * Local filesystem backend — reads from an existing dist/ directory.
+ * This is the default backend for `kanon install` when no remote source is configured.
+ */
+export class LocalBackend implements ArtifactBackend {
+	readonly label: string;
+
+	constructor(private readonly basePath: string) {
+		this.label = `local:${basePath}`;
+	}
+
+	async fetchCatalog(): Promise<CatalogEntry[]> {
+		// Try catalog.json first (faster), fall back to scanning source dirs
+		const catalogPath = join(this.basePath, "catalog.json");
+		if (await exists(catalogPath)) {
+			const raw = await Bun.file(catalogPath).text();
+			return JSON.parse(raw) as CatalogEntry[];
+		}
+		// Fall back to generating from source
+		return generateCatalog([
+			join(this.basePath, "knowledge"),
+			join(this.basePath, "packages"),
+		]);
+	}
+
+	async fetchArtifact(name: string, harness: HarnessName): Promise<string> {
+		const artifactDistDir = join(this.basePath, "dist", harness, name);
+		if (!(await exists(artifactDistDir))) {
+			throw new Error(
+				`Artifact "${name}" for harness "${harness}" not found at ${artifactDistDir}`,
+			);
+		}
+		// For local backend, return the path directly (no copy needed)
+		return artifactDistDir;
+	}
+
+	async listVersions(): Promise<string[]> {
+		// Local backend doesn't have versions; return current as "local"
+		return ["local"];
+	}
+}

package/src/backends/s3.ts

@@ -0,0 +1,157 @@
+import { mkdir } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { CatalogEntry, HarnessName } from "../schemas";
+import type { ArtifactBackend, S3BackendConfig } from "./types";
+
+/**
+ * S3 backend — reads from an S3 bucket (or any S3-compatible store like R2 or MinIO).
+ *
+ * Expected bucket layout:
+ *   <prefix>/catalog.json
+ *   <prefix>/<version>/catalog.json
+ *   <prefix>/<version>/dist-<harness>.tar.gz
+ *   <prefix>/<version>/release-manifest.json
+ *
+ * Credentials: standard AWS credential chain (env vars, ~/.aws/credentials, IAM role).
+ * For read-only public access, no credentials are required.
+ * For presigned URL access, set the FORGE_S3_<BACKEND_NAME>_URL env var.
+ */
+export class S3Backend implements ArtifactBackend {
+	readonly label: string;
+	private readonly prefix: string;
+	private readonly region: string;
+	private readonly endpoint: string | undefined;
+
+	constructor(
+		private readonly config: S3BackendConfig,
+		private readonly version?: string,
+	) {
+		this.prefix = config.prefix?.replace(/\/$/, "") ?? "";
+		this.region =
+			config.region ?? process.env.AWS_DEFAULT_REGION ?? "us-east-1";
+		this.endpoint = config.endpoint;
+		this.label = `s3://${config.bucket}/${this.prefix}${version ? `@${version}` : ""}`;
+	}
+
+	async fetchCatalog(): Promise<CatalogEntry[]> {
+		const key = this.version
+			? `${this.prefix}/${this.version}/catalog.json`
+			: `${this.prefix}/catalog.json`;
+
+		const content = await this.s3Get(key);
+		return JSON.parse(content) as CatalogEntry[];
+	}
+
+	async fetchArtifact(
+		name: string,
+		harness: HarnessName,
+		version?: string,
+	): Promise<string> {
+		const ver = version ?? this.version ?? "latest";
+		const assetName = `dist-${harness}.tar.gz`;
+		const key = `${this.prefix}/${ver}/${assetName}`;
+
+		const cacheDir = join(
+			process.env.HOME ?? tmpdir(),
+			".forge",
+			"cache",
+			`s3-${this.config.bucket.replace(/[^a-z0-9]/gi, "-")}`,
+			ver,
+			harness,
+		);
+
+		const artifactDir = join(cacheDir, name);
+		if (await Bun.file(artifactDir).exists()) {
+			return artifactDir;
+		}
+
+		await mkdir(cacheDir, { recursive: true });
+
+		const tarPath = join(cacheDir, assetName);
+		await this.s3Download(key, tarPath);
+
+		const extractProc = Bun.spawnSync(
+			["tar", "-xzf", tarPath, "-C", cacheDir],
+			{
+				stdout: "pipe",
+				stderr: "pipe",
+			},
+		);
+
+		if (extractProc.exitCode !== 0) {
+			throw new Error(`Failed to extract ${assetName} from S3`);
+		}
+
+		if (!(await Bun.file(artifactDir).exists())) {
+			throw new Error(`Artifact "${name}" not found in ${assetName}`);
+		}
+
+		return artifactDir;
+	}
+
+	async listVersions(): Promise<string[]> {
+		// List common prefixes under the backend prefix to find version dirs
+		const args = this.buildAwsArgs([
+			"s3api",
+			"list-objects-v2",
+			"--bucket",
+			this.config.bucket,
+			"--prefix",
+			`${this.prefix}/`,
+			"--delimiter",
+			"/",
+			"--query",
+			"CommonPrefixes[].Prefix",
+			"--output",
+			"json",
+		]);
+
+		const proc = Bun.spawnSync(args, { stdout: "pipe", stderr: "pipe" });
+		if (proc.exitCode !== 0) return [];
+
+		const prefixes = JSON.parse(new TextDecoder().decode(proc.stdout)) as
+			| string[]
+			| null;
+		if (!prefixes) return [];
+
+		// Extract version from prefix: "<prefix>/<version>/" → "<version>"
+		return prefixes
+			.map((p) => p.replace(`${this.prefix}/`, "").replace(/\/$/, ""))
+			.filter((v) => v.length > 0);
+	}
+
+	private async s3Get(key: string): Promise<string> {
+		const args = this.buildAwsArgs([
+			"s3api",
+			"get-object",
+			"--bucket",
+			this.config.bucket,
+			"--key",
+			key,
+			"/dev/stdout",
+		]);
+		const proc = Bun.spawnSync(args, { stdout: "pipe", stderr: "pipe" });
+		if (proc.exitCode !== 0) {
+			throw new Error(`S3 GET failed for s3://${this.config.bucket}/${key}`);
+		}
+		return new TextDecoder().decode(proc.stdout);
+	}
+
+	private async s3Download(key: string, destPath: string): Promise<void> {
+		const s3Uri = `s3://${this.config.bucket}/${key}`;
+		const args = this.buildAwsArgs(["s3", "cp", s3Uri, destPath]);
+		const proc = Bun.spawnSync(args, { stdout: "pipe", stderr: "pipe" });
+		if (proc.exitCode !== 0) {
+			const stderr = new TextDecoder().decode(proc.stderr);
+			throw new Error(`S3 download failed for ${s3Uri}: ${stderr}`);
+		}
+	}
+
+	private buildAwsArgs(base: string[]): string[] {
+		const args = [...base];
+		if (this.region) args.push("--region", this.region);
+		if (this.endpoint) args.push("--endpoint-url", this.endpoint);
+		return args;
+	}
+}

package/src/backends/types.ts

@@ -0,0 +1,59 @@
+import type { CatalogEntry, HarnessName } from "../schemas";
+
+/**
+ * A backend is a source from which artifact dist files can be fetched and
+ * to which they can be published. All install sources — local filesystem,
+ * GitHub releases, S3 buckets, HTTP endpoints — implement this interface.
+ */
+export interface ArtifactBackend {
+	/** Human-readable label, e.g. "github:my-org/kanon@v1.2.0" */
+	readonly label: string;
+
+	/** Fetch and parse the catalog from this backend. */
+	fetchCatalog(): Promise<CatalogEntry[]>;
+
+	/**
+	 * Download the pre-built dist for a single artifact and harness.
+	 * Returns the path to a local temporary directory containing the artifact files.
+	 */
+	fetchArtifact(
+		name: string,
+		harness: HarnessName,
+		version?: string,
+	): Promise<string>;
+
+	/** List available version identifiers (semver tags, S3 prefixes, etc.). */
+	listVersions(): Promise<string[]>;
+}
+
+/** Configuration shapes for each backend type (as stored in kanon.config.yaml). */
+export interface GitHubBackendConfig {
+	type: "github";
+	repo: string; // e.g. "my-org/kanon-artifacts"
+	releasePrefix?: string; // e.g. "v" — default "v"
+}
+
+export interface S3BackendConfig {
+	type: "s3";
+	bucket: string;
+	prefix?: string; // key prefix, e.g. "skills/"
+	region?: string; // default: AWS_DEFAULT_REGION env
+	endpoint?: string; // for S3-compatible services (MinIO, R2, etc.)
+}
+
+export interface HttpBackendConfig {
+	type: "http";
+	baseUrl: string; // e.g. "https://artifacts.mycompany.com/forge"
+	token?: string; // bearer token (or read from env via ${ENV_VAR})
+}
+
+export interface LocalBackendConfig {
+	type: "local";
+	path: string; // path to a kanon repo or dist directory
+}
+
+export type BackendConfig =
+	| GitHubBackendConfig
+	| S3BackendConfig
+	| HttpBackendConfig
+	| LocalBackendConfig;