@telorun/kernel 0.41.0 → 0.43.0

package/src/transports/registry-transport.ts

@@ -0,0 +1,281 @@
+import {
+  DEFAULT_MANIFEST_FILENAME,
+  HttpSource,
+  IntegrityError,
+  RegistrySource,
+  isRegistryRef,
+  parseModuleRef,
+  splitIntegrity,
+  type ManifestSource,
+} from "@telorun/analyzer";
+import { createHash } from "crypto";
+
+import { computeFilesIntegrity, injectFilesIntegrity } from "../bundle/files-integrity.js";
+import { readOwnerManifest } from "../bundle/module-manifest.js";
+import { makeTarGz, readTarGz, toPayloadFiles } from "../bundle/tar.js";
+import type {
+  FetchedArtifact,
+  PublishBundle,
+  PublishOptions,
+  PublishResult,
+  SiblingIdentity,
+  Transport,
+} from "./transport.js";
+
+const DEFAULT_REGISTRY_URL = "https://registry.telo.run";
+const HTTP_NAMESPACE = "__http";
+const QUERY_HASH_LENGTH = 12;
+const MAX_PUSH_ATTEMPTS = 4;
+const PUSH_BASE_DELAY_MS = 1000;
+
+/** Registry / object-storage backends treat 408/425/429/5xx as transient. */
+function isRetryableStatus(status: number): boolean {
+  return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Mirror `HttpSource.read`'s `fetchUrl` derivation: when the URL does not
+ *  already point at a YAML file, append `/telo.yaml`, so a raw import URL and
+ *  the canonical source it resolves to map to the same cache path. */
+function normalizePathname(rawUrl: string, parsed: URL): string {
+  let pathname = parsed.pathname;
+  if (!rawUrl.includes(".yaml")) {
+    pathname = pathname.endsWith("/")
+      ? `${pathname}${DEFAULT_MANIFEST_FILENAME}`
+      : `${pathname}/${DEFAULT_MANIFEST_FILENAME}`;
+  }
+  return pathname;
+}
+
+/** Short hash of `search + hash` so two URLs that differ only in query /
+ *  fragment do not collide at the same cache path. */
+function disambiguatePath(pathname: string, search: string, hash: string): string {
+  if (!search && !hash) return pathname;
+  const digest = createHash("sha256")
+    .update(search + hash)
+    .digest("hex")
+    .slice(0, QUERY_HASH_LENGTH);
+  const dotIdx = pathname.lastIndexOf(".");
+  const slashIdx = pathname.lastIndexOf("/");
+  const ext = dotIdx > slashIdx ? pathname.slice(dotIdx) : "";
+  const base = pathname.slice(0, pathname.length - ext.length);
+  return `${base}.${digest}${ext}`;
+}
+
+interface VersionsResponse {
+  versions?: string[];
+}
+
+/** The default HTTP transport: bare `namespace/name@version` registry refs and
+ *  direct `https://…` URLs, resolving against `registry.telo.run` (or a
+ *  configured registry). Its resolution `source` composes the browser-safe
+ *  `RegistrySource` / `HttpSource` from `analyzer`; the Node-only management
+ *  methods live here. This is the fallback transport for any ref that carries
+ *  no owning scheme, so `oci://` (or a future `s3://`) never falls through to
+ *  it — those refs are claimed by their own transport's `supports()`. */
+export class RegistryTransport implements Transport {
+  private readonly registrySource: RegistrySource;
+  private readonly httpSource: HttpSource;
+  readonly source: ManifestSource;
+
+  constructor(private readonly registryUrl: string = DEFAULT_REGISTRY_URL) {
+    this.registrySource = new RegistrySource(registryUrl);
+    this.httpSource = new HttpSource();
+    const pick = (ref: string): ManifestSource =>
+      this.httpSource.supports(ref) ? this.httpSource : this.registrySource;
+    this.source = {
+      supports: (url) => this.supports(url),
+      read: (url) => pick(url).read(url),
+      resolveRelative: (base, relative) => pick(base).resolveRelative(base, relative),
+    };
+  }
+
+  supports(ref: string): boolean {
+    const { base } = splitIntegrity(ref);
+    return base.startsWith("http://") || base.startsWith("https://") || isRegistryRef(ref);
+  }
+
+  cacheLocation(ref: string): string[] | null {
+    const url = splitIntegrity(ref).base;
+    const trimmedRegistry = this.registryUrl.replace(/\/+$/, "");
+
+    // 1. Registry ref form: namespace/name@version
+    if (isRegistryRef(url)) {
+      let parsed: ReturnType<typeof parseModuleRef>;
+      try {
+        parsed = parseModuleRef(url);
+      } catch {
+        return null;
+      }
+      return [parsed.modulePath, parsed.version, DEFAULT_MANIFEST_FILENAME];
+    }
+
+    // 2. HTTP(S) URL — a direct registry URL or arbitrary external.
+    if (url.startsWith("http://") || url.startsWith("https://")) {
+      let parsed: URL;
+      try {
+        parsed = new URL(url);
+      } catch {
+        return null;
+      }
+      const pathname = normalizePathname(url, parsed);
+
+      // 2a. On the configured registry, no query/fragment: fold into the
+      //     registry layout so a ref and a direct URL land on the same file.
+      const normalizedUrl = `${parsed.protocol}//${parsed.host}${pathname}`;
+      if (
+        !parsed.search &&
+        !parsed.hash &&
+        (normalizedUrl === trimmedRegistry || normalizedUrl.startsWith(`${trimmedRegistry}/`))
+      ) {
+        const rel = normalizedUrl.slice(trimmedRegistry.length + 1);
+        if (!rel) return null;
+        return rel.split("/");
+      }
+
+      // 2b. Arbitrary HTTP(S) → __http subtree, query-hash suffix on collision.
+      const cleanPath = pathname.startsWith("/") ? pathname.slice(1) : pathname;
+      const disambiguated = disambiguatePath(cleanPath, parsed.search, parsed.hash);
+      return [HTTP_NAMESPACE, parsed.host, ...disambiguated.split("/")];
+    }
+
+    return null;
+  }
+
+  async listVersions(ref: string): Promise<string[] | null> {
+    // Only bare registry refs are version-enumerable — a direct `https://` URL
+    // has no version-list endpoint.
+    if (!isRegistryRef(ref)) return null;
+    const { modulePath } = parseModuleRef(ref);
+    const url = `${this.registryUrl.replace(/\/+$/, "")}/${modulePath}`;
+    const res = await fetch(url, { headers: { accept: "application/json" } });
+    if (res.status === 404) return null;
+    if (!res.ok) {
+      throw new Error(`Registry returned ${res.status} ${res.statusText} for ${modulePath}`);
+    }
+    const body = (await res.json()) as VersionsResponse;
+    return Array.isArray(body.versions) ? body.versions : [];
+  }
+
+  async fetchArtifact(ref: string): Promise<FetchedArtifact> {
+    // `read` verifies the manifest bytes against the inline `#sha256-...` hash.
+    const { text: manifest, source } = await this.source.read(ref);
+    const meta = readOwnerManifest(manifest);
+    if (!meta.declaresFiles) return { manifest, files: [] };
+
+    // The payload rides beside the manifest as `module.tar.gz`.
+    const tarUrl = source.replace(/\/telo\.yaml$/, "/module.tar.gz");
+    const res = await fetch(tarUrl);
+    if (!res.ok) {
+      throw new Error(`could not fetch bundle ${tarUrl}: ${res.status} ${res.statusText}`);
+    }
+    const files = toPayloadFiles(await readTarGz(Buffer.from(await res.arrayBuffer())));
+
+    // Verify the payload against the manifest's `filesIntegrity` before handing
+    // it back — a mismatch is terminal (a tampered bundle must never be used).
+    // The manifest that carries the hash is itself pinned by the inline hash.
+    if (meta.filesIntegrity) {
+      const actual = await computeFilesIntegrity(files);
+      if (actual !== meta.filesIntegrity) {
+        throw new IntegrityError(
+          `Integrity check failed for bundle ${tarUrl}: filesIntegrity expected ` +
+            `${meta.filesIntegrity}, got ${actual}. The payload does not match the recorded ` +
+            `hash — the module may have been tampered with or republished.`,
+        );
+      }
+    }
+
+    return { manifest, files };
+  }
+
+  async publish(
+    destination: string,
+    bundle: PublishBundle,
+    opts: PublishOptions = {},
+  ): Promise<PublishResult> {
+    // Pin the payload in the manifest before it enters the tarball, then choose
+    // the artifact body: a `module.tar.gz` when there are files, else raw YAML.
+    let manifest = bundle.manifest;
+    let body: string | Uint8Array = manifest;
+    let contentType = "text/yaml";
+    let urlSuffix = "";
+    if (bundle.files.length > 0) {
+      manifest = injectFilesIntegrity(manifest, await computeFilesIntegrity(bundle.files));
+      const entries = [
+        { name: DEFAULT_MANIFEST_FILENAME, content: manifest },
+        ...bundle.files.map((f) => ({ name: f.name, content: Buffer.from(f.content) })),
+      ];
+      body = await makeTarGz(entries);
+      contentType = "application/gzip";
+      urlSuffix = "/module.tar.gz";
+    }
+
+    const { namespace, name, version } = readOwnerManifest(manifest);
+    if (!namespace || !name || !version) {
+      throw new Error("metadata must include namespace, name, and version.");
+    }
+    const identity = { namespace, name, version };
+    const base = `${destination.replace(/\/+$/, "")}/${identity.namespace}/${identity.name}/${identity.version}`;
+    const url = `${base}${urlSuffix}`;
+    const label = `${identity.namespace}/${identity.name}@${identity.version}`;
+
+    const headers: Record<string, string> = { "content-type": contentType };
+    if (opts.token) headers.authorization = `Bearer ${opts.token}`;
+
+    let res: Response | null = null;
+    let networkErr: unknown = null;
+    for (let attempt = 1; attempt <= MAX_PUSH_ATTEMPTS; attempt++) {
+      networkErr = null;
+      try {
+        res = await fetch(url, { method: "PUT", headers, body });
+      } catch (err) {
+        networkErr = err;
+        res = null;
+      }
+
+      const transient = networkErr != null || (res != null && isRetryableStatus(res.status));
+      if (!transient) break;
+      if (attempt === MAX_PUSH_ATTEMPTS) break;
+
+      const reason = networkErr
+        ? `network error: ${networkErr instanceof Error ? networkErr.message : String(networkErr)}`
+        : `HTTP ${res!.status}`;
+      // Drain the body so the underlying connection can be reused for the retry.
+      if (res) await res.text().catch(() => {});
+      const delayMs = PUSH_BASE_DELAY_MS * 2 ** (attempt - 1) + Math.floor(Math.random() * 250);
+      opts.onRetry?.({ reason, attempt, maxAttempts: MAX_PUSH_ATTEMPTS, delayMs });
+      await sleep(delayMs);
+    }
+
+    if (networkErr) {
+      throw new Error(
+        `Network error: ${networkErr instanceof Error ? networkErr.message : String(networkErr)} ` +
+          `(after ${MAX_PUSH_ATTEMPTS} attempts)`,
+      );
+    }
+    if (!res!.ok) {
+      const ct = res!.headers.get("content-type") ?? "";
+      const errBody = ct.includes("application/json") ? await res!.json() : await res!.text();
+      throw new Error(`Push failed (${res!.status}): ${JSON.stringify(errBody)}`);
+    }
+
+    return { label, url };
+  }
+
+  canonicalizeSiblingRef(
+    _destination: string,
+    _relativeSource: string,
+    sibling: SiblingIdentity,
+  ): string {
+    // An HTTP registry path defaults to the sibling's own `<namespace>/<name>`.
+    if (!sibling.namespace || !sibling.name) {
+      throw new Error(
+        "a relative import canonicalized to an HTTP registry needs the sibling's metadata.namespace and metadata.name.",
+      );
+    }
+    return `${sibling.namespace}/${sibling.name}@${sibling.version}`;
+  }
+}

package/src/transports/transport-registry.ts

@@ -0,0 +1,91 @@
+import type { ManifestSource } from "@telorun/analyzer";
+
+import { OciTransport } from "./oci/oci-transport.js";
+import { RegistryTransport } from "./registry-transport.js";
+import type {
+  FetchedArtifact,
+  PublishBundle,
+  PublishOptions,
+  PublishResult,
+  Transport,
+} from "./transport.js";
+
+/** Dispatches ref-scheme-specific operations to the transport that owns a ref.
+ *  The loader, cache source, `upgrade`, and `publish` consult this instead of
+ *  branching on ref shape. `RegistryTransport` is always last so it is the
+ *  fallback for bare / `https` refs, and a scheme-owning transport (OCI, later
+ *  S3) claims its refs via `supports()` before the fallback is reached. */
+export class TransportRegistry {
+  constructor(private readonly transports: Transport[]) {}
+
+  /** The transport that owns `ref`, or `undefined` if none does. */
+  forRef(ref: string): Transport | undefined {
+    return this.transports.find((t) => t.supports(ref));
+  }
+
+  /** The resolution `ManifestSource`s of every registered transport, in order —
+   *  the browser-safe subset the loader appends to its source chain. */
+  sources(): ManifestSource[] {
+    return this.transports.map((t) => t.source);
+  }
+
+  /** Cache-path segments for `ref`, from its owning transport; `null` when no
+   *  transport owns it or the ref is not cacheable. */
+  cacheLocation(ref: string): string[] | null {
+    return this.forRef(ref)?.cacheLocation(ref) ?? null;
+  }
+
+  /** Published versions for `ref` via its owning transport; `null` when the
+   *  module is unpublished. Throws when no transport owns the ref. */
+  listVersions(ref: string): Promise<string[] | null> {
+    return this.require(ref).listVersions(ref);
+  }
+
+  /** Full artifact for `ref` via its owning transport. Throws when no transport
+   *  owns the ref. */
+  fetchArtifact(ref: string): Promise<FetchedArtifact> {
+    return this.require(ref).fetchArtifact(ref);
+  }
+
+  /** Publish `bundle` to `destination` via the transport its scheme selects.
+   *  Throws when no transport owns the destination. */
+  publish(
+    destination: string,
+    bundle: PublishBundle,
+    opts?: PublishOptions,
+  ): Promise<PublishResult> {
+    return this.require(destination).publish(destination, bundle, opts);
+  }
+
+  private require(ref: string): Transport {
+    const transport = this.forRef(ref);
+    if (!transport) {
+      throw new Error(`no transport owns ref '${ref}'`);
+    }
+    return transport;
+  }
+}
+
+/** The default transport set. Scheme-owning transports come first; the
+ *  `RegistryTransport` is last, the fallback for bare / `https` refs. OCI (and
+ *  later S3) claim their `oci://` / `s3://` refs before the fallback is reached. */
+export function defaultTransports(registryUrl?: string): Transport[] {
+  return [new OciTransport(), new RegistryTransport(registryUrl)];
+}
+
+const defaultRegistryCache = new Map<string, TransportRegistry>();
+
+/** A `TransportRegistry` seeded with {@link defaultTransports}, memoized per
+ *  `registryUrl`. The default transports are stateless config (a fresh
+ *  `OciClient` with its own token cache is created per OCI operation), so one
+ *  shared instance per registry URL is safe — and avoids re-instantiating the
+ *  whole set on hot paths like `cachePathForCanonical`. */
+export function defaultTransportRegistry(registryUrl?: string): TransportRegistry {
+  const key = registryUrl ?? "";
+  let cached = defaultRegistryCache.get(key);
+  if (!cached) {
+    cached = new TransportRegistry(defaultTransports(registryUrl));
+    defaultRegistryCache.set(key, cached);
+  }
+  return cached;
+}

package/src/transports/transport.ts

@@ -0,0 +1,112 @@
+import type { ManifestSource } from "@telorun/analyzer";
+
+import type { PayloadFile } from "../bundle/files-integrity.js";
+
+/** The full module artifact a transport delivers: the `telo.yaml` bytes plus
+ *  the decompressed `files:` payload (empty for a manifest-only module). The
+ *  manifest is already verified against the import's inline hash; the payload
+ *  against the manifest-embedded `filesIntegrity`. */
+export interface FetchedArtifact {
+  manifest: string;
+  files: PayloadFile[];
+}
+
+/** The module bundle handed to a transport for publishing: the final,
+ *  already-analyzed / pinned / canonicalized `telo.yaml` bytes plus the `files:`
+ *  payload (empty for a manifest-only module). The transport is responsible for
+ *  pinning the payload (`filesIntegrity`) and for the artifact shape it writes
+ *  (HTTP: `telo.yaml` + `module.tar.gz`; OCI: a single blob). */
+export interface PublishBundle {
+  manifest: string;
+  files: PayloadFile[];
+}
+
+export interface PublishResult {
+  /** Human-readable `<ns>/<name>@<version>` label of what was pushed. */
+  label: string;
+  /** The location the artifact was written to. */
+  url: string;
+}
+
+/** Identity of a sibling library, read from its own manifest, that a relative
+ *  import canonicalizes to. `version` is always required; `namespace`/`name` are
+ *  used only by transports whose location is metadata-derived (HTTP registry). */
+export interface SiblingIdentity {
+  namespace?: string;
+  name?: string;
+  version: string;
+}
+
+export interface PublishOptions {
+  /** Bearer token for registries that require auth. */
+  token?: string;
+  /** Notified before each backoff sleep on a transient push failure, so the
+   *  caller can surface retry progress. */
+  onRetry?: (info: {
+    reason: string;
+    attempt: number;
+    maxAttempts: number;
+    delayMs: number;
+  }) => void;
+}
+
+/** A Transport owns everything ref-scheme-specific about a module's lifecycle:
+ *  resolution (through its `ManifestSource`), cache placement, version
+ *  enumeration, full-artifact fetch, and publish. Registering a transport is
+ *  the only thing needed to add a backend — the loader, cache, `upgrade`, and
+ *  `publish` never branch on scheme again; they ask the {@link TransportRegistry}
+ *  which transport owns a ref and delegate.
+ *
+ *  A Transport *composes* a resolution `ManifestSource`, it does not extend it:
+ *  `ManifestSource` is the browser-safe resolution primitive (also implemented
+ *  by the cache / local / memory sources, which have no versions and nothing to
+ *  publish), so it stays in `analyzer`, while the Node-only management methods
+ *  (`cacheLocation` and, in later phases, `listVersions` / `fetchArtifact` /
+ *  `publish`) live on the Transport here in `kernel`. */
+export interface Transport {
+  /** True when this transport owns the given ref (or publish destination). */
+  supports(ref: string): boolean;
+
+  /** The resolution primitive: fetch + verify `telo.yaml`, resolve relatives.
+   *  Browser-safe for a browser-reachable transport (HTTP/registry), so it can
+   *  live in `analyzer`; a Node-only transport has no browser-safe source. */
+  readonly source: ManifestSource;
+
+  /** Deterministic cache-path segments for a ref, joined under the cache root by
+   *  the cache source. Returns `null` when the ref is not cacheable here
+   *  (unsupported scheme, malformed ref, or path-traversal in the ref). */
+  cacheLocation(ref: string): string[] | null;
+
+  /** The versions published for the module `ref` names, newest-first order not
+   *  guaranteed (the caller sorts). Returns `null` when the module is not
+   *  published (e.g. a 404), distinct from `[]` (published, no versions). Used
+   *  by `telo upgrade`. */
+  listVersions(ref: string): Promise<string[] | null>;
+
+  /** Retrieve the full artifact for `ref` — the `telo.yaml` and its `files:`
+   *  payload — verifying the manifest against the inline hash and the payload
+   *  against the manifest's `filesIntegrity`. Used by `telo install`; subsumes
+   *  the out-of-band bundle fetch that used to sit outside the source chain. */
+  fetchArtifact(ref: string): Promise<FetchedArtifact>;
+
+  /** Push `bundle` to `destination` (a base ref / repo whose scheme this
+   *  transport owns), pinning the payload and writing the transport-native
+   *  artifact shape. Throws on failure. Used by `telo publish`. */
+  publish(
+    destination: string,
+    bundle: PublishBundle,
+    opts?: PublishOptions,
+  ): Promise<PublishResult>;
+
+  /** Canonicalize a relative sibling import (`../lib`) declared in a module
+   *  being published to `destination` into the absolute ref it will resolve to
+   *  once published. Owns the scheme-specific "where does a sibling land" rule —
+   *  OCI derives the repo from the destination, HTTP from the sibling's
+   *  `<namespace>/<name>` — so `telo publish` delegates instead of branching on
+   *  transport shape. */
+  canonicalizeSiblingRef(
+    destination: string,
+    relativeSource: string,
+    sibling: SiblingIdentity,
+  ): string;
+}