@platforma-sdk/package-builder-lib 1.1.0

package/src/core/util.ts

@@ -0,0 +1,390 @@
+import * as path from "node:path";
+import * as os from "node:os";
+import fs from "node:fs";
+import type { Hash } from "node:crypto";
+import { createHash } from "node:crypto";
+import winston from "winston";
+import type { z } from "zod/v4";
+import * as envs from "./envs";
+
+export const packageJsonName = "package.json";
+export const softwareConfigName = "package.json";
+
+export function assertNever(_a: never) {
+  throw new Error(`code logic error: assertNever() call for ${JSON.stringify(_a)}`);
+}
+
+export function trimPrefix(str: string, prefix: string): string {
+  if (str.startsWith(prefix)) {
+    return str.slice(prefix.length);
+  }
+  return str;
+}
+
+export function trimSuffix(str: string, suffix: string): string {
+  if (str.endsWith(suffix)) {
+    return str.slice(0, -suffix.length);
+  }
+  return str;
+}
+
+export function hashDirMetaSync(folder: string, hasher?: Hash): Hash {
+  const hash = hasher ? hasher : createHash("sha256");
+  const info = fs.readdirSync(folder, { withFileTypes: true });
+
+  for (const item of info) {
+    const fullPath = path.join(folder, item.name);
+
+    if (item.isFile()) {
+      const statInfo = fs.statSync(fullPath);
+      const fileInfo = `${fullPath}:${statInfo.size}:${statInfo.mtimeMs}`;
+      hash.update(fileInfo);
+    } else if (item.isDirectory()) {
+      hashDirMetaSync(fullPath, hash);
+    }
+  }
+
+  return hash;
+}
+
+export function hashDirSync(rootDir: string, hasher?: Hash, subdir?: string): Hash {
+  const hash = hasher ? hasher : createHash("sha256");
+  const folder = path.join(rootDir, subdir ?? ".");
+
+  const info = fs.readdirSync(folder, { withFileTypes: true });
+
+  for (const item of info) {
+    const relPath = path.join(subdir ?? ".", item.name);
+    const fullPath = path.join(folder, item.name);
+
+    hash.update(relPath);
+
+    if (item.isFile()) {
+      const fileDescriptor = fs.openSync(fullPath, "r");
+      const buffer = Buffer.alloc(65536);
+      let bytesRead: number;
+
+      while ((bytesRead = fs.readSync(fileDescriptor, buffer, 0, buffer.length, null)) !== 0) {
+        hash.update(buffer.subarray(0, bytesRead));
+      }
+
+      fs.closeSync(fileDescriptor);
+    } else if (item.isDirectory()) {
+      hashDirSync(fullPath, hash);
+    }
+  }
+
+  return hash;
+}
+
+//
+// Creates all intermediate directories down to given <dirPath>.
+//
+export function ensureDirsExist(dirPath: string) {
+  if (fs.existsSync(dirPath)) {
+    return;
+  }
+  const parentDir = path.dirname(dirPath);
+  ensureDirsExist(parentDir);
+  fs.mkdirSync(dirPath);
+}
+
+export function findPackageRoot(logger: winston.Logger, startPath?: string): string {
+  if (!startPath) {
+    startPath = process.cwd();
+  }
+  logger.debug(`Detecting package root...`);
+  const pkgRoot = searchPathUp(startPath, startPath, packageJsonName);
+  logger.debug(`  package root found at '${pkgRoot}'`);
+
+  return pkgRoot;
+}
+
+export function findNodeModules(logger: winston.Logger, startPath?: string): string {
+  if (!startPath) {
+    startPath = process.cwd();
+  }
+  logger.debug(`Detecting 'node_modules' directory...`);
+  const nodeModules = searchPathUp(startPath, startPath, "node_modules");
+  logger.debug(`  'node_modules' found at '${nodeModules}'`);
+
+  return path.join(nodeModules, "node_modules");
+}
+
+export function findInstalledModule(
+  logger: winston.Logger,
+  packageName: string,
+  startPath?: string,
+): string {
+  const nodeModules = findNodeModules(logger, startPath);
+  const packagePath = path.resolve(nodeModules, packageName);
+
+  if (!fs.existsSync(packagePath)) {
+    throw CLIError(
+      `package '${packageName}' not found in '${nodeModules}'. Did you forget to add it as a dependency?`,
+    );
+  }
+
+  return packagePath;
+}
+
+function searchPathUp(startPath: string, pathToCheck: string, itemToCheck: string): string {
+  const itemPath = path.resolve(pathToCheck, itemToCheck);
+
+  if (fs.existsSync(itemPath)) {
+    return pathToCheck;
+  }
+
+  const parentDir = path.dirname(pathToCheck);
+  if (parentDir === pathToCheck || pathToCheck === "") {
+    throw CLIError(
+      `failed to find '${itemToCheck}' file in any of parent directories starting from '${startPath}'`,
+    );
+  }
+
+  return searchPathUp(startPath, parentDir, itemToCheck);
+}
+
+export function createLogger(level: string = "debug"): winston.Logger {
+  if (envs.isCI() && envs.isRunnerDebug()) {
+    level = "debug";
+  }
+
+  return winston.createLogger({
+    level: level,
+
+    format: winston.format.printf(({ level, message }) => {
+      const indent = " ".repeat(level.length + 2); // For ': ' after the level
+      if (typeof message !== "string") {
+        const messageJson = JSON.stringify(message);
+        throw new Error(`logger message ${messageJson} is not a string`);
+      }
+      const indentedMessage = message
+        .split("\n")
+        .map((line: string, index: number) => (index === 0 ? line : indent + line))
+        .join("\n");
+
+      const colorize = (l: string) => winston.format.colorize().colorize(l, l);
+
+      return `${colorize(level)}: ${indentedMessage}`;
+    }),
+
+    transports: [
+      new winston.transports.Console({
+        stderrLevels: ["error", "warn", "info", "debug"],
+        handleExceptions: true,
+      }),
+    ],
+  });
+}
+
+export function rSplit(input: string, delimiter: string, limit?: number): string[] {
+  const parts = input.split(delimiter);
+  if (!limit || parts.length <= limit) {
+    return parts;
+  }
+
+  return [parts.slice(0, -limit + 1).join(delimiter), ...parts.slice(-limit + 1)];
+}
+
+const replaceDuplicateSlashes = new RegExp("//+", "g");
+export function urlJoin(url: string, appendPath: string): string {
+  const u = new URL(url, "file:/nonexistent-9672674c/");
+  if (u.protocol === "file:" && u.pathname.startsWith("/nonexistent-9672674c/")) {
+    throw new Error(`string '${url}' is not valid URL`);
+  }
+
+  u.pathname = u.pathname + "/" + appendPath;
+  u.pathname = u.pathname.replaceAll(replaceDuplicateSlashes, "/");
+  return u.toString();
+}
+
+export function wrapErr(e: any, msg: string): any {
+  if (!(e instanceof Error)) {
+    return e;
+  }
+
+  const wrappedError = new Error(`${msg}: ${e.message}`);
+  wrappedError.name = e.name;
+  wrappedError.stack = e.stack;
+  // Copy any additional custom properties
+  Object.assign(wrappedError, e);
+  return wrappedError;
+}
+
+/**
+ * Converts any value to integer number 1 if value is truthful and 0 otherwice.
+ * Provides simple way to count values that are defined in an object:
+ *   toInt(options.a) + toInt(options.b) + toInt(options.c) === 1
+ * @param input: any
+ * @returns number - '1' or '0'
+ */
+export function toInt(input: any): number {
+  return input ? 1 : 0;
+}
+
+export const OSes = ["linux", "macosx", "windows"] as const;
+export type OSType = (typeof OSes)[number];
+
+export function currentOS(): OSType {
+  const platform = os.platform();
+  switch (platform) {
+    case "darwin":
+      return "macosx";
+    case "linux":
+      return "linux";
+    case "win32":
+      return "windows";
+    default:
+      throw CLIError(
+        `operating system '${platform}' is not currently supported by Platforma ecosystem. The list of OSes supported: ` +
+          JSON.stringify(OSes),
+      );
+  }
+}
+
+export const Arches = ["x64", "aarch64"] as const;
+export type ArchType = (typeof Arches)[number];
+
+export function currentArch(): ArchType {
+  const arch = os.arch();
+  switch (arch) {
+    case "arm64":
+      return "aarch64";
+    case "x64":
+      return "x64";
+    default:
+      throw CLIError(
+        `processor architecture '${arch}' is not currently supported by Platforma ecosystem. The list of architectures supported: ` +
+          JSON.stringify(Arches),
+      );
+  }
+}
+
+// Combinartions of OS and architecture, Platforma supports.
+export const AllPlatforms: `${OSType}-${ArchType}`[] = [
+  "linux-x64",
+  "linux-aarch64",
+  "macosx-x64",
+  "macosx-aarch64",
+  "windows-x64",
+] as const;
+export type PlatformType = (typeof AllPlatforms)[number];
+
+export function splitPlatform(platform: PlatformType): { os: OSType; arch: ArchType } {
+  const parts = platform.split("-");
+
+  return {
+    os: parts[0] as OSType,
+    arch: parts[1] as ArchType,
+  };
+}
+
+export function joinPlatform(os: OSType, arch: ArchType): PlatformType {
+  return `${os}-${arch}`;
+}
+
+export function isPlatformSupported(os: OSType, arch: ArchType): boolean {
+  return AllPlatforms.includes(`${os}-${arch}`);
+}
+
+export function currentPlatform(): PlatformType {
+  return `${currentOS()}-${currentArch()}`;
+}
+
+export const AllSoftwareSources = ["archive", "docker"] as const; // add 'image', '<whatever>' here when supported
+export type SoftwareSource = (typeof AllSoftwareSources)[number];
+
+export type BuildMode = "dev-local" | "release";
+
+/** True for dev-local build mode. */
+export function isDevLocalMode(mode: BuildMode): mode is "dev-local" {
+  return mode === "dev-local";
+}
+
+export type artifactID = {
+  package: string;
+  name: string;
+};
+
+export function artifactIDFromString(s: string): artifactID {
+  const parts = s.split(":", 2);
+  if (parts.length < 2) {
+    throw CLIError(
+      `string '${s}' is not an artifact ID (artifact ID format is <packageName>:<artifactName>)`,
+    );
+  }
+  return {
+    package: parts[0],
+    name: parts[1],
+  };
+}
+
+export function artifactIDToString(a: artifactID): string {
+  return `${a.package}:${a.name}`;
+}
+
+export function indentText(txt: string, indent: string | number, skipFirstLine: boolean = false) {
+  indent = typeof indent === "number" ? " ".repeat(indent) : indent;
+
+  return txt
+    .split("\n")
+    .map((line, index) => (index === 0 && skipFirstLine ? line : `${indent}${line}`))
+    .join("\n");
+}
+
+export const formatZodIssues = (
+  issues: z.core.$ZodIssue[],
+  i: string = "",
+  p: PropertyKey[] = [],
+): string => {
+  const _errors: string[] = [];
+
+  const formatSubpath = (path: PropertyKey[], prefix: string = "") => {
+    let pp = path.join(".");
+    if (pp != "" && p.length > 0) pp = `.${pp}`;
+    if (pp != "") pp = `${prefix}${pp}`;
+    return pp;
+  };
+
+  const addItem = (item: string) => _errors.push(indentText(item, i));
+
+  for (const issue of issues) {
+    if (issue.code === "invalid_union") {
+      addItem(`✖ Expected one of several possible values`);
+      const path = formatSubpath(issue.path, "");
+      addItem(`  → at '${path}':`);
+      for (const errGrp of issue.errors) {
+        _errors.push(formatZodIssues(errGrp, `${i}    `, [...p, ...issue.path]));
+      }
+    } else if (issue.code === "invalid_element" || issue.code === "invalid_key") {
+      addItem(`✖ ${issue.code.replaceAll("_", " ")}:`);
+      _errors.push(formatZodIssues(issue.issues, `${i}  `, [...p, ...issue.path]));
+    } else if (issue.code === "invalid_type") {
+      const path = formatSubpath(issue.path, " ");
+      const prefix = `✖ (${issue.expected})${path}: `;
+      const msg = indentText(issue.message, Math.min(prefix.length, 20), true);
+      addItem(`${prefix}${msg}`);
+    } else {
+      const path = formatSubpath(issue.path, " ");
+      const prefix = `✖ (${issue.code.replaceAll("_", " ")})${path}: `;
+      const msg = indentText(issue.message, Math.min(prefix.length, 20), true);
+      addItem(`${prefix}${msg}`);
+    }
+  }
+
+  return _errors.join("\n");
+};
+
+// User-facing CLI error. The top-level handler (bin/run.mjs) prints its
+// message without a stack trace and exits non-zero.
+export class CLIErrorClass extends Error {
+  constructor(msg: string | Error) {
+    super(typeof msg === "string" ? msg : msg.message);
+    this.name = "CLIError";
+  }
+}
+
+export function CLIError(msg: string | Error): CLIErrorClass {
+  return new CLIErrorClass(msg);
+}

package/src/defaults.ts

@@ -0,0 +1,38 @@
+// This file contains default values that are part of external contract from the package-builder prospective.
+//
+// DO NOT CHANGE THEIR VALUES UNLESS YOU KNOW WHAT YOU ARE DOING,
+//
+// as these values could be used in different parts of the system (i.e. tengo code, go code of backend and so on).
+
+// Global
+export const BIN_REGISTRY_NAME = "platforma-open";
+export const BIN_REGISTRY_DOWNLOAD_URL = "https://bin.pl-open.science/";
+
+// Python
+export const PYTHON_TOOLSET = "pip";
+export const PYTHON_REQUIREMENTS_FILE = "requirements.txt";
+
+export const PYTHON_DOCKER_BASE_IMAGE = "python:3.12-slim";
+
+// Conda
+export const CONDA_MICROMAMBA_VERSION = "2.3.2-0";
+export const CONDA_SOURCE_ENV_SPEC_FILE = "conda-spec.yaml"; // inside git repo: what env to build in CI
+export const CONDA_FROZEN_ENV_SPEC_FILE = "frozen-spec.yaml"; // inside built package: how to restore env on backend side
+export const CONDA_DATA_LOCATION = "conda-env-data"; // inside built package: where MICROMAMBA_ROOT_PREFIX should point on backend side
+
+export const CONDA_DOCKER_BASE_IMAGE = "ubuntu:22.04";
+
+// Docker
+export const DOCKER_REGISTRY = "containers.pl-open.science/milaboratories/pl-containers";
+export const DOCKER_DOCKERFILE = "Dockerfile"; // inside git repo: path to Dockerfile to build by default
+export const DOCKER_PLACEHOLDER_PKG = "/app"; // {pkg} placeholder value to by used in 'cmd' (for autogenerated docker images)
+
+// Block-software dev pushes (pl-pkg build --dev=local) target this ECR by default.
+// Override per-run with --docker-registry / PL_DOCKER_REGISTRY.
+export const DEV_DOCKER_REGISTRY = "public.ecr.aws/u5p1x5q2/pl-containers";
+
+// All docker images are built for and pushed as linux/amd64, regardless of host arch.
+// Platforma's K8s nodes are linux/amd64; building for any other platform produces
+// a non-runnable artifact. On non-x64 hosts the build cross-compiles via the
+// Docker daemon's qemu emulation.
+export const DOCKER_BUILD_PLATFORM = "linux/amd64";

package/src/index.ts

@@ -0,0 +1,31 @@
+import type winston from "winston";
+import { Core } from "./core/core";
+
+export * as util from "./core/util";
+export * as envs from "./core/envs";
+export * as defaults from "./defaults";
+
+/**
+ * Public surface of the build engine. Derived from `Core` so signatures never
+ * drift, while the `Core` class itself stays internal (per A-0023: the engine is
+ * a library, not a half-binary with an exported orchestrator class). Consumers
+ * (`pl-pkg`, and later `block-tools software`) wrap this, they do not subclass it.
+ */
+export type Builder = Pick<
+  Core,
+  | "buildMode"
+  | "targetPlatform"
+  | "allPlatforms"
+  | "fullDirHash"
+  | "version"
+  | "isPrivate"
+  | "buildDockerImages"
+  | "buildSoftwareArchives"
+  | "buildSwJsonFiles"
+  | "publishPackages"
+  | "publishDockerImages"
+>;
+
+export function createBuilder(logger: winston.Logger, opts?: { packageRoot?: string }): Builder {
+  return new Core(logger, opts);
+}

package/src/requirements.txt

@@ -0,0 +1 @@
+# No dependencies specified