@telorun/kernel 0.41.0 → 0.43.0

package/src/bundle/tar.ts

@@ -0,0 +1,78 @@
+import { extract as tarExtract, pack as tarPack } from "tar-stream";
+import { gunzipSync, gzipSync } from "node:zlib";
+import { Readable } from "node:stream";
+
+import type { PayloadFile } from "./files-integrity.js";
+
+export interface BundleEntry {
+  /** POSIX-relative path inside the archive (e.g. `telo.yaml`, `public/app.js`). */
+  name: string;
+  content: Buffer | string;
+}
+
+/** Normalize decoded tar entries to `PayloadFile`s (Buffer-backed content) —
+ *  the fixed step both transports run after `readTarGz`. */
+export function toPayloadFiles(entries: BundleEntry[]): PayloadFile[] {
+  return entries.map((e) => ({
+    name: e.name,
+    content: typeof e.content === "string" ? Buffer.from(e.content) : e.content,
+  }));
+}
+
+/**
+ * Pack `entries` into a gzipped tar (`module.tar.gz`) — the module-artifact
+ * writer shared by `telo publish` and the transports. Artifacts are small (a
+ * manifest plus a built frontend), so buffering the whole archive before gzip
+ * is fine. (Distinct from `apps/k8s-runner/src/tar.ts`, which is coupled to
+ * `@telorun/runner-core`'s `RunBundle`.)
+ */
+export async function makeTarGz(entries: BundleEntry[]): Promise<Buffer> {
+  const pack = tarPack();
+  const chunks: Buffer[] = [];
+  pack.on("data", (c: Buffer) => chunks.push(c));
+
+  const done = new Promise<void>((resolve, reject) => {
+    pack.on("end", resolve);
+    pack.on("error", reject);
+  });
+
+  for (const entry of entries) {
+    const buf = typeof entry.content === "string" ? Buffer.from(entry.content, "utf-8") : entry.content;
+    await new Promise<void>((resolve, reject) => {
+      pack.entry({ name: entry.name }, buf, (err) => (err ? reject(err) : resolve()));
+    });
+  }
+  pack.finalize();
+  await done;
+
+  return gzipSync(Buffer.concat(chunks));
+}
+
+/** Decompress + untar a `module.tar.gz` buffer into its file entries. */
+export async function readTarGz(buf: Buffer): Promise<BundleEntry[]> {
+  const tar = gunzipSync(buf);
+  const ex = tarExtract();
+  const entries: BundleEntry[] = [];
+
+  await new Promise<void>((resolve, reject) => {
+    ex.on("entry", (header, stream, next) => {
+      if (header.type !== "file") {
+        stream.on("end", next);
+        stream.resume();
+        return;
+      }
+      const chunks: Buffer[] = [];
+      stream.on("data", (c: Buffer) => chunks.push(c));
+      stream.on("end", () => {
+        entries.push({ name: header.name, content: Buffer.concat(chunks) });
+        next();
+      });
+      stream.on("error", reject);
+    });
+    ex.on("finish", resolve);
+    ex.on("error", reject);
+    Readable.from(tar).pipe(ex);
+  });
+
+  return entries;
+}

package/src/controller-loaders/bundle-builder.ts

@@ -45,6 +45,12 @@ function sanitize(s: string): string {
   return s.replace(/[^a-zA-Z0-9._-]/g, "_");
 }
 
+/** Extensions of a package's own relative modules that are externalized (kept a
+ *  single loose copy) rather than inlined per controller bundle. TS extensions
+ *  are included because published modules are loaded from their `src/*.ts` under
+ *  a TS-aware runtime (Bun), so the shared `.ts` file must keep one identity. */
+const EXTERNALIZABLE_EXTENSIONS = new Set([".js", ".mjs", ".cjs", ".ts", ".mts", ".cts"]);
+
 async function realpathOrNull(p: string): Promise<string | null> {
   try {
     return await fs.realpath(p);
@@ -320,9 +326,15 @@ function nativeExternalsPlugin(bundleDir: string, pkgRoot: string | null): impor
         const relToPkg = path.relative(pkgRoot, resolved.path);
         if (relToPkg.startsWith("..") || path.isAbsolute(relToPkg)) return null;
         if (relToPkg.split(path.sep).includes("node_modules")) return null;
-        // Must be a runtime-loadable JS file; otherwise inline (transpile).
+        // Must be a runtime-loadable module; otherwise inline (transpile). The
+        // loose file is loaded by the SAME runtime as this bundle — and a package
+        // shipped as `.ts` source (loaded via Bun) has `.ts` entries, so a `.ts`
+        // sibling resolves for that runtime too. TS extensions MUST be
+        // externalized, not inlined: inlining gives each controller its own copy
+        // of a shared module (e.g. record-stream's `journal-store`), splitting
+        // `instanceof` / process-local state across the package's controllers.
         const ext = path.extname(resolved.path).toLowerCase();
-        if (ext !== ".js" && ext !== ".mjs" && ext !== ".cjs") return null;
+        if (!EXTERNALIZABLE_EXTENSIONS.has(ext)) return null;
         const rel = path.relative(bundleDir, resolved.path).split(path.sep).join("/");
         return { path: rel.startsWith(".") ? rel : `./${rel}`, external: true };
       });

package/src/controllers/module/import-controller.ts

@@ -1,4 +1,4 @@
-import { AnalysisRegistry, DiagnosticSeverity, parseExportEntry, StaticAnalyzer } from "@telorun/analyzer";
+import { AnalysisRegistry, DiagnosticSeverity, foldIntegrity, parseExportEntry, StaticAnalyzer } from "@telorun/analyzer";
 import type { ResourceInstance } from "@telorun/sdk";
 import { RuntimeError } from "@telorun/sdk";
 import type { BuiltinControllerContext } from "../../internal-context.js";
@@ -11,7 +11,11 @@ export async function create(
 ): Promise<ResourceInstance> {
   const alias = resource.metadata.name as string;
 
-  const moduleSource: string = resource.module ?? resource.source;
+  const rawSource: string = resource.module ?? resource.source;
+  // A directly-authored Telo.Import may carry integrity as a sibling field;
+  // fold it into the ref as a `#sha256-...` fragment (the desugared inline
+  // imports already inline it) so the source chain verifies the fetched bytes.
+  const moduleSource: string = foldIntegrity(rawSource, resource.integrity);
 
   // Resolve relative source paths against the manifest's OWN file URL (stamped onto
   // `metadata.source` by the loader), not the parent module context's source. When a
@@ -334,6 +338,7 @@ export const schema = {
       additionalProperties: true,
     },
     source: { type: "string" },
+    integrity: { type: "string" },
     variables: { type: "object" },
     secrets: { type: "object" },
     runtime: {

package/src/index.ts

@@ -10,6 +10,28 @@ export {
   writeManifestCache,
 } from "./manifest-sources/local-manifest-cache-source.js";
 export { MemorySource } from "./manifest-sources/memory-source.js";
+export type { Transport } from "./transports/transport.js";
+export { RegistryTransport } from "./transports/registry-transport.js";
+export { OciTransport } from "./transports/oci/oci-transport.js";
+export { OciClient } from "./transports/oci/oci-client.js";
+export { isOciRef, parseOciRef, type ParsedOciRef } from "./transports/oci/oci-ref.js";
+export {
+  TransportRegistry,
+  defaultTransports,
+  defaultTransportRegistry,
+} from "./transports/transport-registry.js";
+export { makeTarGz, readTarGz, type BundleEntry } from "./bundle/tar.js";
+export {
+  computeFilesIntegrity,
+  injectFilesIntegrity,
+  type PayloadFile,
+} from "./bundle/files-integrity.js";
+export type {
+  FetchedArtifact,
+  PublishBundle,
+  PublishResult,
+  PublishOptions,
+} from "./transports/transport.js";
 export { ExecutionContext } from "./execution-context.js";
 export { Kernel, type KernelOptions } from "./kernel.js";
 export { nodeCelHandlers } from "./cel-handlers.js";

package/src/kernel.ts

@@ -1,7 +1,6 @@
 import {
   AnalysisRegistry,
   buildEvalPaths,
-  defaultSources,
   flattenForAnalyzer,
   flattenLoadedModule,
   isModuleKind,
@@ -48,6 +47,7 @@ import {
 import {
   resolveCacheRoot,
 } from "./manifest-sources/local-manifest-cache-source.js";
+import { defaultTransportRegistry } from "./transports/transport-registry.js";
 import {
   collectDeclaredEnvKeys,
   precompileApplicationEnvSchemas,
@@ -156,7 +156,12 @@ export class Kernel implements IKernel {
     this.env = options.env ?? hostEnv();
     this.argv = options.argv ?? [];
     this.registryUrl = options.registryUrl;
-    this.loader = new Loader(defaultSources(this.registryUrl), { celHandlers: nodeCelHandlers });
+    // Resolution sources come from the transport registry, so a scheme-owning
+    // transport (OCI, later S3) joins the loader's dispatch chain by being
+    // registered — no source-chain edits here.
+    this.loader = new Loader(defaultTransportRegistry(this.registryUrl).sources(), {
+      celHandlers: nodeCelHandlers,
+    });
     for (const source of options.sources) {
       this.loader.register(source);
     }

package/src/manifest-sources/local-manifest-cache-source.ts

@@ -1,17 +1,14 @@
-import type { LoadedGraph, ManifestSource } from "@telorun/analyzer";
-import { createHash } from "crypto";
+import { splitIntegrity, verifyIntegrity, type LoadedGraph, type ManifestSource } from "@telorun/analyzer";
 import { statSync } from "fs";
 import * as fs from "fs/promises";
 import * as path from "path";
 import { fileURLToPath, pathToFileURL } from "url";
 
 import { hostEnv } from "../host-env.js";
+import { TransportRegistry, defaultTransportRegistry } from "../transports/transport-registry.js";
 
 const CACHE_SUBDIR = ".telo/manifests";
-const HTTP_NAMESPACE = "__http";
-const DEFAULT_MANIFEST_FILENAME = "telo.yaml";
 const DEFAULT_REGISTRY_URL = "https://registry.telo.run";
-const QUERY_HASH_LENGTH = 12;
 
 /** Verify that `candidate` resolves to a path under `root`. Returns the
  *  candidate path on success, `null` when any segment escapes the root.
@@ -27,101 +24,22 @@ function joinUnder(root: string, ...segments: string[]): string | null {
   return candidate;
 }
 
-/** Mirror `HttpSource.read`'s `fetchUrl` derivation: when the URL does not
- *  already point at a YAML file, append `/telo.yaml`. Used by both the
- *  reader and writer so a raw import URL and the canonical source it
- *  resolves to map to the same cache path. */
-function normalizePathname(url: string, parsed: URL): string {
-  let pathname = parsed.pathname;
-  if (!url.includes(".yaml")) {
-    pathname = pathname.endsWith("/")
-      ? `${pathname}${DEFAULT_MANIFEST_FILENAME}`
-      : `${pathname}/${DEFAULT_MANIFEST_FILENAME}`;
-  }
-  return pathname;
-}
-
-/** Compute a short hash of `search + hash` so two URLs that differ only in
- *  query / fragment do not collide at the same cache path. Inserted before
- *  the final extension so the filename stays readable. */
-function disambiguatePath(pathname: string, search: string, hash: string): string {
-  if (!search && !hash) return pathname;
-  const digest = createHash("sha256")
-    .update(search + hash)
-    .digest("hex")
-    .slice(0, QUERY_HASH_LENGTH);
-  const ext = path.extname(pathname);
-  const base = pathname.slice(0, pathname.length - ext.length);
-  return `${base}.${digest}${ext}`;
-}
-
 /** Single source of truth for URL → cache path. Used identically by the
  *  reader (cache lookup) and writer (install-time persistence). For any
- *  given import URL — registry ref, direct registry URL, or arbitrary
- *  HTTP — both sides land on the same file.
+ *  given import ref — registry ref, direct registry URL, arbitrary HTTP, or
+ *  (once registered) `oci://` — both sides land on the same file, because the
+ *  owning transport decides the layout via {@link Transport.cacheLocation}.
  *
- *  Returns `null` for unsupported URLs (file://, memory://, relative paths)
- *  or for path-traversal attempts that would escape `cacheRoot`. */
+ *  Returns `null` for unsupported refs (file://, memory://, relative paths) or
+ *  for path-traversal attempts that would escape `cacheRoot`. */
 function cachePathForUrl(
-  url: string,
+  rawUrl: string,
   cacheRoot: string,
-  registryUrl: string,
+  transports: TransportRegistry,
 ): string | null {
-  const trimmedRegistry = registryUrl.replace(/\/+$/, "");
-
-  // 1. Registry ref form: namespace/name@version
-  if (
-    !url.startsWith("http://") &&
-    !url.startsWith("https://") &&
-    !url.startsWith("/") &&
-    !url.startsWith(".") &&
-    !url.startsWith("file://") &&
-    !url.startsWith("memory://") &&
-    url.includes("@") &&
-    url.includes("/")
-  ) {
-    const atIdx = url.lastIndexOf("@");
-    if (atIdx <= 0 || atIdx === url.length - 1) return null;
-    const modulePath = url.slice(0, atIdx);
-    if (!modulePath.includes("/")) return null;
-    const version = url.slice(atIdx + 1).replace(/^v/, "");
-    if (!version) return null;
-    return joinUnder(cacheRoot, modulePath, version, DEFAULT_MANIFEST_FILENAME);
-  }
-
-  // 2. HTTP(S) URL — could be a direct registry URL or arbitrary external.
-  if (url.startsWith("http://") || url.startsWith("https://")) {
-    let parsed: URL;
-    try {
-      parsed = new URL(url);
-    } catch {
-      return null;
-    }
-    const pathname = normalizePathname(url, parsed);
-
-    // 2a. URL is on the configured registry, with no query/fragment:
-    //     fold into the registry layout so the writer that received a
-    //     registry-ref canonical and the reader that sees a direct URL
-    //     both resolve to the same file.
-    const normalizedUrl = `${parsed.protocol}//${parsed.host}${pathname}`;
-    if (
-      !parsed.search &&
-      !parsed.hash &&
-      (normalizedUrl === trimmedRegistry || normalizedUrl.startsWith(`${trimmedRegistry}/`))
-    ) {
-      const rel = normalizedUrl.slice(trimmedRegistry.length + 1);
-      if (!rel) return null;
-      return joinUnder(cacheRoot, ...rel.split("/"));
-    }
-
-    // 2b. Arbitrary HTTP(S) URL → __http subtree, with a short query-hash
-    //     suffix when query / fragment are present to prevent collisions.
-    const cleanPath = pathname.startsWith("/") ? pathname.slice(1) : pathname;
-    const disambiguated = disambiguatePath(cleanPath, parsed.search, parsed.hash);
-    return joinUnder(cacheRoot, HTTP_NAMESPACE, parsed.host, ...disambiguated.split("/"));
-  }
-
-  return null;
+  const segments = transports.cacheLocation(rawUrl);
+  if (!segments) return null;
+  return joinUnder(cacheRoot, ...segments);
 }
 
 /**
@@ -133,7 +51,7 @@ function cachePathForUrl(
  */
 export class LocalManifestCacheSource implements ManifestSource {
   private readonly cacheRoot: string;
-  private readonly registryUrl: string;
+  private readonly transports: TransportRegistry;
 
   constructor(
     entryDir: string,
@@ -144,7 +62,7 @@ export class LocalManifestCacheSource implements ManifestSource {
     // single `resolveCacheRoot` (honours `TELO_CACHE_DIR`); when absent we fall
     // back to the entry-anchored default so library/test callers are unchanged.
     this.cacheRoot = manifestsDir ?? path.join(entryDir, CACHE_SUBDIR);
-    this.registryUrl = registryUrl;
+    this.transports = defaultTransportRegistry(registryUrl);
   }
 
   supports(url: string): boolean {
@@ -158,6 +76,15 @@ export class LocalManifestCacheSource implements ManifestSource {
         `LocalManifestCacheSource does not support '${url}' (cache miss or unsupported scheme)`,
       );
     }
+    // Verify the cached bytes against the import's inline hash before serving.
+    // A mismatch is a terminal error — a poisoned cache must never be trusted,
+    // and unlike the compiled-validator cache this is not a self-healing miss.
+    const { integrity } = splitIntegrity(url);
+    if (integrity) {
+      const bytes = await fs.readFile(mapped);
+      await verifyIntegrity(new Uint8Array(bytes), integrity, splitIntegrity(url).base);
+      return { text: bytes.toString("utf-8"), source: pathToFileURL(mapped).href };
+    }
     const text = await fs.readFile(mapped, "utf-8");
     return { text, source: pathToFileURL(mapped).href };
   }
@@ -172,7 +99,7 @@ export class LocalManifestCacheSource implements ManifestSource {
   }
 
   private tryMap(url: string): string | null {
-    const candidate = cachePathForUrl(url, this.cacheRoot, this.registryUrl);
+    const candidate = cachePathForUrl(url, this.cacheRoot, this.transports);
     if (!candidate) return null;
     // Require a regular file. A directory, dangling symlink, or stat failure
     // (ENOENT, EACCES, EISDIR-on-component) all fall through as a cache miss
@@ -201,7 +128,7 @@ export function cachePathForCanonical(
   manifestsDir?: string,
 ): string | null {
   const cacheRoot = manifestsDir ?? path.join(entryDir, CACHE_SUBDIR);
-  return cachePathForUrl(canonicalSource, cacheRoot, registryUrl);
+  return cachePathForUrl(canonicalSource, cacheRoot, defaultTransportRegistry(registryUrl));
 }
 
 /**

package/src/transports/oci/docker-credentials.ts

@@ -0,0 +1,85 @@
+import { spawn } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import * as path from "node:path";
+
+import { hostEnv } from "../../host-env.js";
+
+export interface DockerCredential {
+  username: string;
+  password: string;
+}
+
+interface DockerConfig {
+  auths?: Record<string, { auth?: string; username?: string; password?: string }>;
+  credHelpers?: Record<string, string>;
+  credsStore?: string;
+}
+
+/** Path to `config.json`, honouring `DOCKER_CONFIG`. */
+function dockerConfigPath(): string {
+  const override = hostEnv().DOCKER_CONFIG;
+  const dir = override && override.trim() ? override.trim() : path.join(homedir(), ".docker");
+  return path.join(dir, "config.json");
+}
+
+async function readDockerConfig(): Promise<DockerConfig | null> {
+  try {
+    return JSON.parse(await readFile(dockerConfigPath(), "utf-8")) as DockerConfig;
+  } catch {
+    return null; // no config, unreadable, or malformed → anonymous
+  }
+}
+
+/** Run `docker-credential-<helper> get` with `host` on stdin, per the Docker
+ *  credential-helper protocol, returning `{Username, Secret}` or null. */
+function runCredentialHelper(helper: string, host: string): Promise<DockerCredential | null> {
+  return new Promise((resolve) => {
+    const proc = spawn(`docker-credential-${helper}`, ["get"], {
+      stdio: ["pipe", "pipe", "ignore"],
+    });
+    let out = "";
+    proc.stdout.on("data", (c) => (out += c));
+    proc.on("error", () => resolve(null)); // helper binary not installed
+    proc.on("close", (code) => {
+      if (code !== 0) return resolve(null);
+      try {
+        const parsed = JSON.parse(out) as { Username?: string; Secret?: string };
+        if (parsed.Username && parsed.Secret) {
+          resolve({ username: parsed.Username, password: parsed.Secret });
+        } else {
+          resolve(null);
+        }
+      } catch {
+        resolve(null);
+      }
+    });
+    proc.stdin.end(host);
+  });
+}
+
+/** Resolve credentials for `host` from the ambient Docker credential chain:
+ *  a per-registry credential helper, the global credential store, then the
+ *  inline `auths` entry (base64 `user:password` or explicit fields). Returns
+ *  null for anonymous access — the caller falls back to an anonymous token. */
+export async function resolveDockerCredential(host: string): Promise<DockerCredential | null> {
+  const config = await readDockerConfig();
+  if (!config) return null;
+
+  const helper = config.credHelpers?.[host];
+  if (helper) return runCredentialHelper(helper, host);
+  if (config.credsStore) return runCredentialHelper(config.credsStore, host);
+
+  const entry = config.auths?.[host];
+  if (entry?.auth) {
+    const decoded = Buffer.from(entry.auth, "base64").toString("utf-8");
+    const colon = decoded.indexOf(":");
+    if (colon > 0) {
+      return { username: decoded.slice(0, colon), password: decoded.slice(colon + 1) };
+    }
+  }
+  if (entry?.username && entry?.password) {
+    return { username: entry.username, password: entry.password };
+  }
+  return null;
+}